1 From: Felix Fietkau <nbd@nbd.name>
2 Date: Sun, 26 Jul 2020 14:03:21 +0200
3 Subject: [PATCH] net: add support for threaded NAPI polling
5 For some drivers (especially 802.11 drivers), doing a lot of work in the NAPI
6 poll function does not perform well. Since NAPI poll is bound to the CPU it
7 was scheduled from, we can easily end up with a few very busy CPUs spending
8 most of their time in softirq/ksoftirqd and some idle ones.
10 Introduce threaded NAPI for such drivers based on a workqueue. The API is the
11 same except for using netif_threaded_napi_add instead of netif_napi_add.
13 In my tests with mt76 on MT7621 using threaded NAPI + a thread for tx scheduling
14 improves LAN->WLAN bridging throughput by 10-50%. Throughput without threaded
15 NAPI is wildly inconsistent, depending on the CPU that runs the tx scheduling
18 With threaded NAPI it seems stable and consistent (and higher than the best
19 results I got without it).
21 Based on a patch by Hillf Danton
23 Cc: Hillf Danton <hdanton@sina.com>
24 Signed-off-by: Felix Fietkau <nbd@nbd.name>
27 --- a/include/linux/netdevice.h
28 +++ b/include/linux/netdevice.h
29 @@ -340,6 +340,7 @@ struct napi_struct {
30 struct list_head dev_list;
31 struct hlist_node napi_hash_node;
33 + struct work_struct work;
37 @@ -350,6 +351,7 @@ enum {
38 NAPI_STATE_HASHED, /* In NAPI hash (busy polling possible) */
39 NAPI_STATE_NO_BUSY_POLL,/* Do not add in napi_hash, no busy polling */
40 NAPI_STATE_IN_BUSY_POLL,/* sk_busy_loop() owns this NAPI */
41 + NAPI_STATE_THREADED, /* Use threaded NAPI */
45 @@ -360,6 +362,7 @@ enum {
46 NAPIF_STATE_HASHED = BIT(NAPI_STATE_HASHED),
47 NAPIF_STATE_NO_BUSY_POLL = BIT(NAPI_STATE_NO_BUSY_POLL),
48 NAPIF_STATE_IN_BUSY_POLL = BIT(NAPI_STATE_IN_BUSY_POLL),
49 + NAPIF_STATE_THREADED = BIT(NAPI_STATE_THREADED),
53 @@ -2101,6 +2104,7 @@ struct net_device {
54 struct lock_class_key addr_list_lock_key;
56 unsigned wol_enabled:1;
57 + unsigned threaded:1;
59 #define to_net_dev(d) container_of(d, struct net_device, dev)
61 @@ -2281,6 +2285,26 @@ void netif_napi_add(struct net_device *d
62 int (*poll)(struct napi_struct *, int), int weight);
65 + * netif_threaded_napi_add - initialize a NAPI context
66 + * @dev: network device
67 + * @napi: NAPI context
68 + * @poll: polling function
69 + * @weight: default weight
71 + * This variant of netif_napi_add() should be used from drivers using NAPI
72 + * with CPU intensive poll functions.
73 + * This will schedule polling from a high priority workqueue
75 +static inline void netif_threaded_napi_add(struct net_device *dev,
76 + struct napi_struct *napi,
77 + int (*poll)(struct napi_struct *, int),
80 + set_bit(NAPI_STATE_THREADED, &napi->state);
81 + netif_napi_add(dev, napi, poll, weight);
85 * netif_tx_napi_add - initialize a NAPI context
86 * @dev: network device
90 @@ -156,6 +156,7 @@ static DEFINE_SPINLOCK(offload_lock);
91 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
92 struct list_head ptype_all __read_mostly; /* Taps */
93 static struct list_head offload_base __read_mostly;
94 +static struct workqueue_struct *napi_workq __read_mostly;
96 static int netif_rx_internal(struct sk_buff *skb);
97 static int call_netdevice_notifiers_info(unsigned long val,
98 @@ -5937,6 +5938,11 @@ void __napi_schedule(struct napi_struct
102 + if (test_bit(NAPI_STATE_THREADED, &n->state)) {
103 + queue_work(napi_workq, &n->work);
107 local_irq_save(flags);
108 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
109 local_irq_restore(flags);
110 @@ -5988,6 +5994,10 @@ EXPORT_SYMBOL(napi_schedule_prep);
112 void __napi_schedule_irqoff(struct napi_struct *n)
114 + if (test_bit(NAPI_STATE_THREADED, &n->state)) {
115 + queue_work(napi_workq, &n->work);
118 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
119 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
121 @@ -6252,9 +6262,89 @@ static void init_gro_hash(struct napi_st
122 napi->gro_bitmask = 0;
125 +static int __napi_poll(struct napi_struct *n, bool *repoll)
129 + weight = n->weight;
131 + /* This NAPI_STATE_SCHED test is for avoiding a race
132 + * with netpoll's poll_napi(). Only the entity which
133 + * obtains the lock and sees NAPI_STATE_SCHED set will
134 + * actually make the ->poll() call. Therefore we avoid
135 + * accidentally calling ->poll() when NAPI is not scheduled.
138 + if (test_bit(NAPI_STATE_SCHED, &n->state)) {
139 + work = n->poll(n, weight);
140 + trace_napi_poll(n, work, weight);
143 + WARN_ON_ONCE(work > weight);
145 + if (likely(work < weight))
148 + /* Drivers must not modify the NAPI state if they
149 + * consume the entire weight. In such cases this code
150 + * still "owns" the NAPI instance and therefore can
151 + * move the instance around on the list at-will.
153 + if (unlikely(napi_disable_pending(n))) {
158 + if (n->gro_bitmask) {
159 + /* flush too old packets
160 + * If HZ < 1000, flush all packets.
162 + napi_gro_flush(n, HZ >= 1000);
165 + gro_normal_list(n);
172 +static void napi_workfn(struct work_struct *work)
174 + struct napi_struct *n = container_of(work, struct napi_struct, work);
178 + bool repoll = false;
180 + local_bh_disable();
182 + have = netpoll_poll_lock(n);
183 + __napi_poll(n, &repoll);
184 + netpoll_poll_unlock(have);
191 + if (!need_resched())
195 + * have to pay for the latency of task switch even if
196 + * napi is scheduled
198 + queue_work(napi_workq, work);
203 void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
204 int (*poll)(struct napi_struct *, int), int weight)
207 + set_bit(NAPI_STATE_THREADED, &napi->state);
208 INIT_LIST_HEAD(&napi->poll_list);
209 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
210 napi->timer.function = napi_watchdog;
211 @@ -6271,6 +6361,7 @@ void netif_napi_add(struct net_device *d
212 #ifdef CONFIG_NETPOLL
213 napi->poll_owner = -1;
215 + INIT_WORK(&napi->work, napi_workfn);
216 set_bit(NAPI_STATE_SCHED, &napi->state);
217 set_bit(NAPI_STATE_NPSVC, &napi->state);
218 list_add_rcu(&napi->dev_list, &dev->napi_list);
219 @@ -6311,6 +6402,7 @@ static void flush_gro_hash(struct napi_s
220 void netif_napi_del(struct napi_struct *napi)
223 + cancel_work_sync(&napi->work);
224 if (napi_hash_del(napi))
226 list_del_init(&napi->dev_list);
227 @@ -6323,50 +6415,18 @@ EXPORT_SYMBOL(netif_napi_del);
229 static int napi_poll(struct napi_struct *n, struct list_head *repoll)
231 + bool do_repoll = false;
236 list_del_init(&n->poll_list);
238 have = netpoll_poll_lock(n);
240 - weight = n->weight;
242 - /* This NAPI_STATE_SCHED test is for avoiding a race
243 - * with netpoll's poll_napi(). Only the entity which
244 - * obtains the lock and sees NAPI_STATE_SCHED set will
245 - * actually make the ->poll() call. Therefore we avoid
246 - * accidentally calling ->poll() when NAPI is not scheduled.
249 - if (test_bit(NAPI_STATE_SCHED, &n->state)) {
250 - work = n->poll(n, weight);
251 - trace_napi_poll(n, work, weight);
254 - WARN_ON_ONCE(work > weight);
255 + work = __napi_poll(n, &do_repoll);
257 - if (likely(work < weight))
260 - /* Drivers must not modify the NAPI state if they
261 - * consume the entire weight. In such cases this code
262 - * still "owns" the NAPI instance and therefore can
263 - * move the instance around on the list at-will.
265 - if (unlikely(napi_disable_pending(n))) {
271 - if (n->gro_bitmask) {
272 - /* flush too old packets
273 - * If HZ < 1000, flush all packets.
275 - napi_gro_flush(n, HZ >= 1000);
278 - gro_normal_list(n);
280 /* Some drivers may have called napi_schedule
281 * prior to exhausting their budget.
282 @@ -10346,6 +10406,10 @@ static int __init net_dev_init(void)
283 sd->backlog.weight = weight_p;
286 + napi_workq = alloc_workqueue("napi_workq", WQ_UNBOUND | WQ_HIGHPRI,
287 + WQ_UNBOUND_MAX_ACTIVE | WQ_SYSFS);
288 + BUG_ON(!napi_workq);
292 /* The loopback device is special if any other network devices
293 --- a/net/core/net-sysfs.c
294 +++ b/net/core/net-sysfs.c
295 @@ -442,6 +442,52 @@ static ssize_t proto_down_store(struct d
297 NETDEVICE_SHOW_RW(proto_down, fmt_dec);
299 +static int change_napi_threaded(struct net_device *dev, unsigned long val)
301 + struct napi_struct *napi;
303 + if (list_empty(&dev->napi_list))
304 + return -EOPNOTSUPP;
306 + list_for_each_entry(napi, &dev->napi_list, dev_list) {
308 + set_bit(NAPI_STATE_THREADED, &napi->state);
310 + clear_bit(NAPI_STATE_THREADED, &napi->state);
316 +static ssize_t napi_threaded_store(struct device *dev,
317 + struct device_attribute *attr,
318 + const char *buf, size_t len)
320 + return netdev_store(dev, attr, buf, len, change_napi_threaded);
323 +static ssize_t napi_threaded_show(struct device *dev,
324 + struct device_attribute *attr,
327 + struct net_device *netdev = to_net_dev(dev);
328 + struct napi_struct *napi;
329 + bool enabled = false;
331 + if (!rtnl_trylock())
332 + return restart_syscall();
334 + list_for_each_entry(napi, &netdev->napi_list, dev_list) {
335 + if (test_bit(NAPI_STATE_THREADED, &napi->state))
341 + return sprintf(buf, fmt_dec, enabled);
343 +static DEVICE_ATTR_RW(napi_threaded);
345 static ssize_t phys_port_id_show(struct device *dev,
346 struct device_attribute *attr, char *buf)
348 @@ -532,6 +578,7 @@ static struct attribute *net_class_attrs
349 &dev_attr_flags.attr,
350 &dev_attr_tx_queue_len.attr,
351 &dev_attr_gro_flush_timeout.attr,
352 + &dev_attr_napi_threaded.attr,
353 &dev_attr_phys_port_id.attr,
354 &dev_attr_phys_port_name.attr,
355 &dev_attr_phys_switch_id.attr,