mac80211: Update to version 4.19.120
[openwrt/openwrt.git] / package / kernel / mac80211 / patches / subsys / 322-mac80211-Add-airtime-accounting-and-scheduling-to-TX.patch
1 From: =?UTF-8?q?Toke=20H=C3=B8iland-J=C3=B8rgensen?= <toke@toke.dk>
2 Date: Tue, 18 Dec 2018 17:02:08 -0800
3 Subject: [PATCH] mac80211: Add airtime accounting and scheduling to TXQs
4 MIME-Version: 1.0
5 Content-Type: text/plain; charset=UTF-8
6 Content-Transfer-Encoding: 8bit
7
8 This adds airtime accounting and scheduling to the mac80211 TXQ
9 scheduler. A new callback, ieee80211_sta_register_airtime(), is added
10 that drivers can call to report airtime usage for stations.
11
12 When airtime information is present, mac80211 will schedule TXQs
13 (through ieee80211_next_txq()) in a way that enforces airtime fairness
14 between active stations. This scheduling works the same way as the ath9k
15 in-driver airtime fairness scheduling. If no airtime usage is reported
16 by the driver, the scheduler will default to round-robin scheduling.
17
18 For drivers that don't control TXQ scheduling in software, a new API
19 function, ieee80211_txq_may_transmit(), is added which the driver can use
20 to check if the TXQ is eligible for transmission, or should be throttled to
21 enforce fairness. Calls to this function must also be enclosed in
22 ieee80211_txq_schedule_{start,end}() calls to ensure proper locking.
23
24 The API ieee80211_txq_may_transmit() also ensures that TXQ list will be
25 aligned aginst driver's own round-robin scheduler list. i.e it rotates
26 the TXQ list till it makes the requested node becomes the first entry
27 in TXQ list. Thus both the TXQ list and driver's list are in sync.
28
29 Co-developed-by: Rajkumar Manoharan <rmanohar@codeaurora.org>
30 Signed-off-by: Louie Lu <git@louie.lu>
31 [added debugfs write op to reset airtime counter]
32 Signed-off-by: Toke Høiland-Jørgensen <toke@toke.dk>
33 Signed-off-by: Rajkumar Manoharan <rmanohar@codeaurora.org>
34 Signed-off-by: Johannes Berg <johannes.berg@intel.com>
35 ---
36
37 --- a/include/net/mac80211.h
38 +++ b/include/net/mac80211.h
39 @@ -2304,6 +2304,9 @@ enum ieee80211_hw_flags {
40 * supported by HW.
41 * @max_nan_de_entries: maximum number of NAN DE functions supported by the
42 * device.
43 + *
44 + * @weight_multipler: Driver specific airtime weight multiplier used while
45 + * refilling deficit of each TXQ.
46 */
47 struct ieee80211_hw {
48 struct ieee80211_conf conf;
49 @@ -2339,6 +2342,7 @@ struct ieee80211_hw {
50 u8 n_cipher_schemes;
51 const struct ieee80211_cipher_scheme *cipher_schemes;
52 u8 max_nan_de_entries;
53 + u8 weight_multiplier;
54 };
55
56 static inline bool _ieee80211_hw_check(struct ieee80211_hw *hw,
57 @@ -5299,6 +5303,34 @@ void ieee80211_sta_eosp(struct ieee80211
58 void ieee80211_send_eosp_nullfunc(struct ieee80211_sta *pubsta, int tid);
59
60 /**
61 + * ieee80211_sta_register_airtime - register airtime usage for a sta/tid
62 + *
63 + * Register airtime usage for a given sta on a given tid. The driver can call
64 + * this function to notify mac80211 that a station used a certain amount of
65 + * airtime. This information will be used by the TXQ scheduler to schedule
66 + * stations in a way that ensures airtime fairness.
67 + *
68 + * The reported airtime should as a minimum include all time that is spent
69 + * transmitting to the remote station, including overhead and padding, but not
70 + * including time spent waiting for a TXOP. If the time is not reported by the
71 + * hardware it can in some cases be calculated from the rate and known frame
72 + * composition. When possible, the time should include any failed transmission
73 + * attempts.
74 + *
75 + * The driver can either call this function synchronously for every packet or
76 + * aggregate, or asynchronously as airtime usage information becomes available.
77 + * TX and RX airtime can be reported together, or separately by setting one of
78 + * them to 0.
79 + *
80 + * @pubsta: the station
81 + * @tid: the TID to register airtime for
82 + * @tx_airtime: airtime used during TX (in usec)
83 + * @rx_airtime: airtime used during RX (in usec)
84 + */
85 +void ieee80211_sta_register_airtime(struct ieee80211_sta *pubsta, u8 tid,
86 + u32 tx_airtime, u32 rx_airtime);
87 +
88 +/**
89 * ieee80211_iter_keys - iterate keys programmed into the device
90 * @hw: pointer obtained from ieee80211_alloc_hw()
91 * @vif: virtual interface to iterate, may be %NULL for all
92 @@ -6042,6 +6074,33 @@ void ieee80211_txq_schedule_end(struct i
93 __releases(txq_lock);
94
95 /**
96 + * ieee80211_txq_may_transmit - check whether TXQ is allowed to transmit
97 + *
98 + * This function is used to check whether given txq is allowed to transmit by
99 + * the airtime scheduler, and can be used by drivers to access the airtime
100 + * fairness accounting without going using the scheduling order enfored by
101 + * next_txq().
102 + *
103 + * Returns %true if the airtime scheduler thinks the TXQ should be allowed to
104 + * transmit, and %false if it should be throttled. This function can also have
105 + * the side effect of rotating the TXQ in the scheduler rotation, which will
106 + * eventually bring the deficit to positive and allow the station to transmit
107 + * again.
108 + *
109 + * The API ieee80211_txq_may_transmit() also ensures that TXQ list will be
110 + * aligned aginst driver's own round-robin scheduler list. i.e it rotates
111 + * the TXQ list till it makes the requested node becomes the first entry
112 + * in TXQ list. Thus both the TXQ list and driver's list are in sync. If this
113 + * function returns %true, the driver is expected to schedule packets
114 + * for transmission, and then return the TXQ through ieee80211_return_txq().
115 + *
116 + * @hw: pointer as obtained from ieee80211_alloc_hw()
117 + * @txq: pointer obtained from station or virtual interface
118 + */
119 +bool ieee80211_txq_may_transmit(struct ieee80211_hw *hw,
120 + struct ieee80211_txq *txq);
121 +
122 +/**
123 * ieee80211_txq_get_depth - get pending frame/byte count of given txq
124 *
125 * The values are not guaranteed to be coherent with regard to each other, i.e.
126 --- a/net/mac80211/cfg.c
127 +++ b/net/mac80211/cfg.c
128 @@ -1390,6 +1390,9 @@ static int sta_apply_parameters(struct i
129 if (ieee80211_vif_is_mesh(&sdata->vif))
130 sta_apply_mesh_params(local, sta, params);
131
132 + if (params->airtime_weight)
133 + sta->airtime_weight = params->airtime_weight;
134 +
135 /* set the STA state after all sta info from usermode has been set */
136 if (test_sta_flag(sta, WLAN_STA_TDLS_PEER) ||
137 set & BIT(NL80211_STA_FLAG_ASSOCIATED)) {
138 --- a/net/mac80211/debugfs.c
139 +++ b/net/mac80211/debugfs.c
140 @@ -380,6 +380,9 @@ void debugfs_hw_add(struct ieee80211_loc
141 if (local->ops->wake_tx_queue)
142 DEBUGFS_ADD_MODE(aqm, 0600);
143
144 + debugfs_create_u16("airtime_flags", 0600,
145 + phyd, &local->airtime_flags);
146 +
147 statsd = debugfs_create_dir("statistics", phyd);
148
149 /* if the dir failed, don't put all the other things into the root! */
150 --- a/net/mac80211/debugfs_sta.c
151 +++ b/net/mac80211/debugfs_sta.c
152 @@ -178,9 +178,9 @@ static ssize_t sta_aqm_read(struct file
153 txqi->tin.tx_bytes,
154 txqi->tin.tx_packets,
155 txqi->flags,
156 - txqi->flags & (1<<IEEE80211_TXQ_STOP) ? "STOP" : "RUN",
157 - txqi->flags & (1<<IEEE80211_TXQ_AMPDU) ? " AMPDU" : "",
158 - txqi->flags & (1<<IEEE80211_TXQ_NO_AMSDU) ? " NO-AMSDU" : "");
159 + test_bit(IEEE80211_TXQ_STOP, &txqi->flags) ? "STOP" : "RUN",
160 + test_bit(IEEE80211_TXQ_AMPDU, &txqi->flags) ? " AMPDU" : "",
161 + test_bit(IEEE80211_TXQ_NO_AMSDU, &txqi->flags) ? " NO-AMSDU" : "");
162 }
163
164 rcu_read_unlock();
165 @@ -192,6 +192,64 @@ static ssize_t sta_aqm_read(struct file
166 }
167 STA_OPS(aqm);
168
169 +static ssize_t sta_airtime_read(struct file *file, char __user *userbuf,
170 + size_t count, loff_t *ppos)
171 +{
172 + struct sta_info *sta = file->private_data;
173 + struct ieee80211_local *local = sta->sdata->local;
174 + size_t bufsz = 200;
175 + char *buf = kzalloc(bufsz, GFP_KERNEL), *p = buf;
176 + u64 rx_airtime = 0, tx_airtime = 0;
177 + s64 deficit[IEEE80211_NUM_ACS];
178 + ssize_t rv;
179 + int ac;
180 +
181 + if (!buf)
182 + return -ENOMEM;
183 +
184 + for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) {
185 + spin_lock_bh(&local->active_txq_lock[ac]);
186 + rx_airtime += sta->airtime[ac].rx_airtime;
187 + tx_airtime += sta->airtime[ac].tx_airtime;
188 + deficit[ac] = sta->airtime[ac].deficit;
189 + spin_unlock_bh(&local->active_txq_lock[ac]);
190 + }
191 +
192 + p += scnprintf(p, bufsz + buf - p,
193 + "RX: %llu us\nTX: %llu us\nWeight: %u\n"
194 + "Deficit: VO: %lld us VI: %lld us BE: %lld us BK: %lld us\n",
195 + rx_airtime,
196 + tx_airtime,
197 + sta->airtime_weight,
198 + deficit[0],
199 + deficit[1],
200 + deficit[2],
201 + deficit[3]);
202 +
203 + rv = simple_read_from_buffer(userbuf, count, ppos, buf, p - buf);
204 + kfree(buf);
205 + return rv;
206 +}
207 +
208 +static ssize_t sta_airtime_write(struct file *file, const char __user *userbuf,
209 + size_t count, loff_t *ppos)
210 +{
211 + struct sta_info *sta = file->private_data;
212 + struct ieee80211_local *local = sta->sdata->local;
213 + int ac;
214 +
215 + for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) {
216 + spin_lock_bh(&local->active_txq_lock[ac]);
217 + sta->airtime[ac].rx_airtime = 0;
218 + sta->airtime[ac].tx_airtime = 0;
219 + sta->airtime[ac].deficit = sta->airtime_weight;
220 + spin_unlock_bh(&local->active_txq_lock[ac]);
221 + }
222 +
223 + return count;
224 +}
225 +STA_OPS_RW(airtime);
226 +
227 static ssize_t sta_agg_status_read(struct file *file, char __user *userbuf,
228 size_t count, loff_t *ppos)
229 {
230 @@ -546,6 +604,10 @@ void ieee80211_sta_debugfs_add(struct st
231 if (local->ops->wake_tx_queue)
232 DEBUGFS_ADD(aqm);
233
234 + if (wiphy_ext_feature_isset(local->hw.wiphy,
235 + NL80211_EXT_FEATURE_AIRTIME_FAIRNESS))
236 + DEBUGFS_ADD(airtime);
237 +
238 if (sizeof(sta->driver_buffered_tids) == sizeof(u32))
239 debugfs_create_x32("driver_buffered_tids", 0400,
240 sta->debugfs_dir,
241 --- a/net/mac80211/ieee80211_i.h
242 +++ b/net/mac80211/ieee80211_i.h
243 @@ -1136,6 +1136,8 @@ struct ieee80211_local {
244 struct list_head active_txqs[IEEE80211_NUM_ACS];
245 u16 schedule_round[IEEE80211_NUM_ACS];
246
247 + u16 airtime_flags;
248 +
249 const struct ieee80211_ops *ops;
250
251 /*
252 --- a/net/mac80211/main.c
253 +++ b/net/mac80211/main.c
254 @@ -656,6 +656,7 @@ struct ieee80211_hw *ieee80211_alloc_hw_
255 INIT_LIST_HEAD(&local->active_txqs[i]);
256 spin_lock_init(&local->active_txq_lock[i]);
257 }
258 + local->airtime_flags = AIRTIME_USE_TX | AIRTIME_USE_RX;
259
260 INIT_LIST_HEAD(&local->chanctx_list);
261 mutex_init(&local->chanctx_mtx);
262 @@ -1142,6 +1143,9 @@ int ieee80211_register_hw(struct ieee802
263 if (!local->hw.max_nan_de_entries)
264 local->hw.max_nan_de_entries = IEEE80211_MAX_NAN_INSTANCE_ID;
265
266 + if (!local->hw.weight_multiplier)
267 + local->hw.weight_multiplier = 1;
268 +
269 result = ieee80211_wep_init(local);
270 if (result < 0)
271 wiphy_debug(local->hw.wiphy, "Failed to initialize wep: %d\n",
272 --- a/net/mac80211/sta_info.c
273 +++ b/net/mac80211/sta_info.c
274 @@ -90,7 +90,6 @@ static void __cleanup_single_sta(struct
275 struct tid_ampdu_tx *tid_tx;
276 struct ieee80211_sub_if_data *sdata = sta->sdata;
277 struct ieee80211_local *local = sdata->local;
278 - struct fq *fq = &local->fq;
279 struct ps_data *ps;
280
281 if (test_sta_flag(sta, WLAN_STA_PS_STA) ||
282 @@ -115,9 +114,7 @@ static void __cleanup_single_sta(struct
283 for (i = 0; i < ARRAY_SIZE(sta->sta.txq); i++) {
284 struct txq_info *txqi = to_txq_info(sta->sta.txq[i]);
285
286 - spin_lock_bh(&fq->lock);
287 ieee80211_txq_purge(local, txqi);
288 - spin_unlock_bh(&fq->lock);
289 }
290 }
291
292 @@ -381,9 +378,12 @@ struct sta_info *sta_info_alloc(struct i
293 if (sta_prepare_rate_control(local, sta, gfp))
294 goto free_txq;
295
296 + sta->airtime_weight = IEEE80211_DEFAULT_AIRTIME_WEIGHT;
297 +
298 for (i = 0; i < IEEE80211_NUM_ACS; i++) {
299 skb_queue_head_init(&sta->ps_tx_buf[i]);
300 skb_queue_head_init(&sta->tx_filtered[i]);
301 + sta->airtime[i].deficit = sta->airtime_weight;
302 }
303
304 for (i = 0; i < IEEE80211_NUM_TIDS; i++)
305 @@ -1826,6 +1826,27 @@ void ieee80211_sta_set_buffered(struct i
306 }
307 EXPORT_SYMBOL(ieee80211_sta_set_buffered);
308
309 +void ieee80211_sta_register_airtime(struct ieee80211_sta *pubsta, u8 tid,
310 + u32 tx_airtime, u32 rx_airtime)
311 +{
312 + struct sta_info *sta = container_of(pubsta, struct sta_info, sta);
313 + struct ieee80211_local *local = sta->sdata->local;
314 + u8 ac = ieee80211_ac_from_tid(tid);
315 + u32 airtime = 0;
316 +
317 + if (sta->local->airtime_flags & AIRTIME_USE_TX)
318 + airtime += tx_airtime;
319 + if (sta->local->airtime_flags & AIRTIME_USE_RX)
320 + airtime += rx_airtime;
321 +
322 + spin_lock_bh(&local->active_txq_lock[ac]);
323 + sta->airtime[ac].tx_airtime += tx_airtime;
324 + sta->airtime[ac].rx_airtime += rx_airtime;
325 + sta->airtime[ac].deficit -= airtime;
326 + spin_unlock_bh(&local->active_txq_lock[ac]);
327 +}
328 +EXPORT_SYMBOL(ieee80211_sta_register_airtime);
329 +
330 int sta_info_move_state(struct sta_info *sta,
331 enum ieee80211_sta_state new_state)
332 {
333 @@ -2192,6 +2213,23 @@ void sta_set_sinfo(struct sta_info *sta,
334 sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_FAILED);
335 }
336
337 + if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_RX_DURATION))) {
338 + for (ac = 0; ac < IEEE80211_NUM_ACS; ac++)
339 + sinfo->rx_duration += sta->airtime[ac].rx_airtime;
340 + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_DURATION);
341 + }
342 +
343 + if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_TX_DURATION))) {
344 + for (ac = 0; ac < IEEE80211_NUM_ACS; ac++)
345 + sinfo->tx_duration += sta->airtime[ac].tx_airtime;
346 + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_DURATION);
347 + }
348 +
349 + if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_AIRTIME_WEIGHT))) {
350 + sinfo->airtime_weight = sta->airtime_weight;
351 + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_AIRTIME_WEIGHT);
352 + }
353 +
354 sinfo->rx_dropped_misc = sta->rx_stats.dropped;
355 if (sta->pcpu_rx_stats) {
356 for_each_possible_cpu(cpu) {
357 --- a/net/mac80211/sta_info.h
358 +++ b/net/mac80211/sta_info.h
359 @@ -127,6 +127,16 @@ enum ieee80211_agg_stop_reason {
360 AGG_STOP_DESTROY_STA,
361 };
362
363 +/* Debugfs flags to enable/disable use of RX/TX airtime in scheduler */
364 +#define AIRTIME_USE_TX BIT(0)
365 +#define AIRTIME_USE_RX BIT(1)
366 +
367 +struct airtime_info {
368 + u64 rx_airtime;
369 + u64 tx_airtime;
370 + s64 deficit;
371 +};
372 +
373 struct sta_info;
374
375 /**
376 @@ -563,6 +573,9 @@ struct sta_info {
377 } tx_stats;
378 u16 tid_seq[IEEE80211_QOS_CTL_TID_MASK + 1];
379
380 + struct airtime_info airtime[IEEE80211_NUM_ACS];
381 + u16 airtime_weight;
382 +
383 /*
384 * Aggregation information, locked with lock.
385 */
386 --- a/net/mac80211/status.c
387 +++ b/net/mac80211/status.c
388 @@ -828,6 +828,12 @@ static void __ieee80211_tx_status(struct
389 ieee80211_sta_tx_notify(sta->sdata, (void *) skb->data,
390 acked, info->status.tx_time);
391
392 + if (info->status.tx_time &&
393 + wiphy_ext_feature_isset(local->hw.wiphy,
394 + NL80211_EXT_FEATURE_AIRTIME_FAIRNESS))
395 + ieee80211_sta_register_airtime(&sta->sta, tid,
396 + info->status.tx_time, 0);
397 +
398 if (ieee80211_hw_check(&local->hw, REPORTS_TX_ACK_STATUS)) {
399 if (acked) {
400 if (sta->status_stats.lost_packets)
401 --- a/net/mac80211/tx.c
402 +++ b/net/mac80211/tx.c
403 @@ -1463,8 +1463,11 @@ void ieee80211_txq_purge(struct ieee8021
404 struct fq *fq = &local->fq;
405 struct fq_tin *tin = &txqi->tin;
406
407 + spin_lock_bh(&fq->lock);
408 fq_tin_reset(fq, tin, fq_skb_free_func);
409 ieee80211_purge_tx_queue(&local->hw, &txqi->frags);
410 + spin_unlock_bh(&fq->lock);
411 +
412 spin_lock_bh(&local->active_txq_lock[txqi->txq.ac]);
413 list_del_init(&txqi->schedule_order);
414 spin_unlock_bh(&local->active_txq_lock[txqi->txq.ac]);
415 @@ -3631,11 +3634,28 @@ struct ieee80211_txq *ieee80211_next_txq
416
417 lockdep_assert_held(&local->active_txq_lock[ac]);
418
419 + begin:
420 txqi = list_first_entry_or_null(&local->active_txqs[ac],
421 struct txq_info,
422 schedule_order);
423 + if (!txqi)
424 + return NULL;
425 +
426 + if (txqi->txq.sta) {
427 + struct sta_info *sta = container_of(txqi->txq.sta,
428 + struct sta_info, sta);
429 +
430 + if (sta->airtime[txqi->txq.ac].deficit < 0) {
431 + sta->airtime[txqi->txq.ac].deficit +=
432 + sta->airtime_weight;
433 + list_move_tail(&txqi->schedule_order,
434 + &local->active_txqs[txqi->txq.ac]);
435 + goto begin;
436 + }
437 + }
438 +
439
440 - if (!txqi || txqi->schedule_round == local->schedule_round[ac])
441 + if (txqi->schedule_round == local->schedule_round[ac])
442 return NULL;
443
444 list_del_init(&txqi->schedule_order);
445 @@ -3653,12 +3673,74 @@ void ieee80211_return_txq(struct ieee802
446 lockdep_assert_held(&local->active_txq_lock[txq->ac]);
447
448 if (list_empty(&txqi->schedule_order) &&
449 - (!skb_queue_empty(&txqi->frags) || txqi->tin.backlog_packets))
450 - list_add_tail(&txqi->schedule_order,
451 - &local->active_txqs[txq->ac]);
452 + (!skb_queue_empty(&txqi->frags) || txqi->tin.backlog_packets)) {
453 + /* If airtime accounting is active, always enqueue STAs at the
454 + * head of the list to ensure that they only get moved to the
455 + * back by the airtime DRR scheduler once they have a negative
456 + * deficit. A station that already has a negative deficit will
457 + * get immediately moved to the back of the list on the next
458 + * call to ieee80211_next_txq().
459 + */
460 + if (txqi->txq.sta &&
461 + wiphy_ext_feature_isset(local->hw.wiphy,
462 + NL80211_EXT_FEATURE_AIRTIME_FAIRNESS))
463 + list_add(&txqi->schedule_order,
464 + &local->active_txqs[txq->ac]);
465 + else
466 + list_add_tail(&txqi->schedule_order,
467 + &local->active_txqs[txq->ac]);
468 + }
469 }
470 EXPORT_SYMBOL(ieee80211_return_txq);
471
472 +bool ieee80211_txq_may_transmit(struct ieee80211_hw *hw,
473 + struct ieee80211_txq *txq)
474 +{
475 + struct ieee80211_local *local = hw_to_local(hw);
476 + struct txq_info *iter, *tmp, *txqi = to_txq_info(txq);
477 + struct sta_info *sta;
478 + u8 ac = txq->ac;
479 +
480 + lockdep_assert_held(&local->active_txq_lock[ac]);
481 +
482 + if (!txqi->txq.sta)
483 + goto out;
484 +
485 + if (list_empty(&txqi->schedule_order))
486 + goto out;
487 +
488 + list_for_each_entry_safe(iter, tmp, &local->active_txqs[ac],
489 + schedule_order) {
490 + if (iter == txqi)
491 + break;
492 +
493 + if (!iter->txq.sta) {
494 + list_move_tail(&iter->schedule_order,
495 + &local->active_txqs[ac]);
496 + continue;
497 + }
498 + sta = container_of(iter->txq.sta, struct sta_info, sta);
499 + if (sta->airtime[ac].deficit < 0)
500 + sta->airtime[ac].deficit += sta->airtime_weight;
501 + list_move_tail(&iter->schedule_order, &local->active_txqs[ac]);
502 + }
503 +
504 + sta = container_of(txqi->txq.sta, struct sta_info, sta);
505 + if (sta->airtime[ac].deficit >= 0)
506 + goto out;
507 +
508 + sta->airtime[ac].deficit += sta->airtime_weight;
509 + list_move_tail(&txqi->schedule_order, &local->active_txqs[ac]);
510 +
511 + return false;
512 +out:
513 + if (!list_empty(&txqi->schedule_order))
514 + list_del_init(&txqi->schedule_order);
515 +
516 + return true;
517 +}
518 +EXPORT_SYMBOL(ieee80211_txq_may_transmit);
519 +
520 void ieee80211_txq_schedule_start(struct ieee80211_hw *hw, u8 ac)
521 __acquires(txq_lock)
522 {