target/linux/generic/pending-5.4/690-net-add-support-for-threaded-NAPI-polling.patch

   1 From: Felix Fietkau <nbd@nbd.name>
   2 Date: Sun, 26 Jul 2020 14:03:21 +0200
   3 Subject: [PATCH] net: add support for threaded NAPI polling
   4
   5 For some drivers (especially 802.11 drivers), doing a lot of work in the NAPI
   6 poll function does not perform well. Since NAPI poll is bound to the CPU it
   7 was scheduled from, we can easily end up with a few very busy CPUs spending
   8 most of their time in softirq/ksoftirqd and some idle ones.
   9
  10 Introduce threaded NAPI for such drivers based on a workqueue. The API is the
  11 same except for using netif_threaded_napi_add instead of netif_napi_add.
  12
  13 In my tests with mt76 on MT7621 using threaded NAPI + a thread for tx scheduling
  14 improves LAN->WLAN bridging throughput by 10-50%. Throughput without threaded
  15 NAPI is wildly inconsistent, depending on the CPU that runs the tx scheduling
  16 thread.
  17
  18 With threaded NAPI it seems stable and consistent (and higher than the best
  19 results I got without it).
  20
  21 Based on a patch by Hillf Danton
  22
  23 Cc: Hillf Danton <hdanton@sina.com>
  24 Signed-off-by: Felix Fietkau <nbd@nbd.name>
  25 ---
  26
  27 --- a/include/linux/netdevice.h
  28 +++ b/include/linux/netdevice.h
  29 @@ -340,6 +340,7 @@ struct napi_struct {
  30         struct list_head        dev_list;
  31         struct hlist_node       napi_hash_node;
  32         unsigned int            napi_id;
  33 +       struct work_struct      work;
  34  };
  35
  36  enum {
  37 @@ -350,6 +351,7 @@ enum {
  38         NAPI_STATE_HASHED,      /* In NAPI hash (busy polling possible) */
  39         NAPI_STATE_NO_BUSY_POLL,/* Do not add in napi_hash, no busy polling */
  40         NAPI_STATE_IN_BUSY_POLL,/* sk_busy_loop() owns this NAPI */
  41 +       NAPI_STATE_THREADED,    /* Use threaded NAPI */
  42  };
  43
  44  enum {
  45 @@ -360,6 +362,7 @@ enum {
  46         NAPIF_STATE_HASHED       = BIT(NAPI_STATE_HASHED),
  47         NAPIF_STATE_NO_BUSY_POLL = BIT(NAPI_STATE_NO_BUSY_POLL),
  48         NAPIF_STATE_IN_BUSY_POLL = BIT(NAPI_STATE_IN_BUSY_POLL),
  49 +       NAPIF_STATE_THREADED     = BIT(NAPI_STATE_THREADED),
  50  };
  51
  52  enum gro_result {
  53 @@ -2249,6 +2252,26 @@ void netif_napi_add(struct net_device *d
  54                     int (*poll)(struct napi_struct *, int), int weight);
  55
  56  /**
  57 + *     netif_threaded_napi_add - initialize a NAPI context
  58 + *     @dev:  network device
  59 + *     @napi: NAPI context
  60 + *     @poll: polling function
  61 + *     @weight: default weight
  62 + *
  63 + * This variant of netif_napi_add() should be used from drivers using NAPI
  64 + * with CPU intensive poll functions.
  65 + * This will schedule polling from a high priority workqueue
  66 + */
  67 +static inline void netif_threaded_napi_add(struct net_device *dev,
  68 +                                          struct napi_struct *napi,
  69 +                                          int (*poll)(struct napi_struct *, int),
  70 +                                          int weight)
  71 +{
  72 +       set_bit(NAPI_STATE_THREADED, &napi->state);
  73 +       netif_napi_add(dev, napi, poll, weight);
  74 +}
  75 +
  76 +/**
  77   *     netif_tx_napi_add - initialize a NAPI context
  78   *     @dev:  network device
  79   *     @napi: NAPI context
  80 --- a/net/core/dev.c
  81 +++ b/net/core/dev.c
  82 @@ -156,6 +156,7 @@ static DEFINE_SPINLOCK(offload_lock);
  83  struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
  84  struct list_head ptype_all __read_mostly;      /* Taps */
  85  static struct list_head offload_base __read_mostly;
  86 +static struct workqueue_struct *napi_workq __read_mostly;
  87
  88  static int netif_rx_internal(struct sk_buff *skb);
  89  static int call_netdevice_notifiers_info(unsigned long val,
  90 @@ -5910,6 +5911,11 @@ void __napi_schedule(struct napi_struct
  91  {
  92         unsigned long flags;
  93
  94 +       if (test_bit(NAPI_STATE_THREADED, &n->state)) {
  95 +               queue_work(napi_workq, &n->work);
  96 +               return;
  97 +       }
  98 +
  99         local_irq_save(flags);
 100         ____napi_schedule(this_cpu_ptr(&softnet_data), n);
 101         local_irq_restore(flags);
 102 @@ -5957,6 +5963,11 @@ EXPORT_SYMBOL(napi_schedule_prep);
 103   */
 104  void __napi_schedule_irqoff(struct napi_struct *n)
 105  {
 106 +       if (test_bit(NAPI_STATE_THREADED, &n->state)) {
 107 +               queue_work(napi_workq, &n->work);
 108 +               return;
 109 +       }
 110 +
 111         ____napi_schedule(this_cpu_ptr(&softnet_data), n);
 112  }
 113  EXPORT_SYMBOL(__napi_schedule_irqoff);
 114 @@ -6218,6 +6229,84 @@ static void init_gro_hash(struct napi_st
 115         napi->gro_bitmask = 0;
 116  }
 117
 118 +static int __napi_poll(struct napi_struct *n, bool *repoll)
 119 +{
 120 +       int work, weight;
 121 +
 122 +       weight = n->weight;
 123 +
 124 +       /* This NAPI_STATE_SCHED test is for avoiding a race
 125 +        * with netpoll's poll_napi().  Only the entity which
 126 +        * obtains the lock and sees NAPI_STATE_SCHED set will
 127 +        * actually make the ->poll() call.  Therefore we avoid
 128 +        * accidentally calling ->poll() when NAPI is not scheduled.
 129 +        */
 130 +       work = 0;
 131 +       if (test_bit(NAPI_STATE_SCHED, &n->state)) {
 132 +               work = n->poll(n, weight);
 133 +               trace_napi_poll(n, work, weight);
 134 +       }
 135 +
 136 +       WARN_ON_ONCE(work > weight);
 137 +
 138 +       if (likely(work < weight))
 139 +               return work;
 140 +
 141 +       /* Drivers must not modify the NAPI state if they
 142 +        * consume the entire weight.  In such cases this code
 143 +        * still "owns" the NAPI instance and therefore can
 144 +        * move the instance around on the list at-will.
 145 +        */
 146 +       if (unlikely(napi_disable_pending(n))) {
 147 +               napi_complete(n);
 148 +               return work;
 149 +       }
 150 +
 151 +       if (n->gro_bitmask) {
 152 +               /* flush too old packets
 153 +                * If HZ < 1000, flush all packets.
 154 +                */
 155 +               napi_gro_flush(n, HZ >= 1000);
 156 +       }
 157 +
 158 +       gro_normal_list(n);
 159 +
 160 +       *repoll = true;
 161 +
 162 +       return work;
 163 +}
 164 +
 165 +static void napi_workfn(struct work_struct *work)
 166 +{
 167 +       struct napi_struct *n = container_of(work, struct napi_struct, work);
 168 +       void *have;
 169 +
 170 +       for (;;) {
 171 +               bool repoll = false;
 172 +
 173 +               local_bh_disable();
 174 +
 175 +               have = netpoll_poll_lock(n);
 176 +               __napi_poll(n, &repoll);
 177 +               netpoll_poll_unlock(have);
 178 +
 179 +               local_bh_enable();
 180 +
 181 +               if (!repoll)
 182 +                       return;
 183 +
 184 +               if (!need_resched())
 185 +                       continue;
 186 +
 187 +               /*
 188 +                * have to pay for the latency of task switch even if
 189 +                * napi is scheduled
 190 +                */
 191 +               queue_work(napi_workq, work);
 192 +               return;
 193 +       }
 194 +}
 195 +
 196  void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
 197                     int (*poll)(struct napi_struct *, int), int weight)
 198  {
 199 @@ -6237,6 +6326,7 @@ void netif_napi_add(struct net_device *d
 200  #ifdef CONFIG_NETPOLL
 201         napi->poll_owner = -1;
 202  #endif
 203 +       INIT_WORK(&napi->work, napi_workfn);
 204         set_bit(NAPI_STATE_SCHED, &napi->state);
 205         set_bit(NAPI_STATE_NPSVC, &napi->state);
 206         list_add_rcu(&napi->dev_list, &dev->napi_list);
 207 @@ -6277,6 +6367,7 @@ static void flush_gro_hash(struct napi_s
 208  void netif_napi_del(struct napi_struct *napi)
 209  {
 210         might_sleep();
 211 +       cancel_work_sync(&napi->work);
 212         if (napi_hash_del(napi))
 213                 synchronize_net();
 214         list_del_init(&napi->dev_list);
 215 @@ -6289,50 +6380,18 @@ EXPORT_SYMBOL(netif_napi_del);
 216
 217  static int napi_poll(struct napi_struct *n, struct list_head *repoll)
 218  {
 219 +       bool do_repoll = false;
 220         void *have;
 221 -       int work, weight;
 222 +       int work;
 223
 224         list_del_init(&n->poll_list);
 225
 226         have = netpoll_poll_lock(n);
 227
 228 -       weight = n->weight;
 229 +       work = __napi_poll(n, &do_repoll);
 230
 231 -       /* This NAPI_STATE_SCHED test is for avoiding a race
 232 -        * with netpoll's poll_napi().  Only the entity which
 233 -        * obtains the lock and sees NAPI_STATE_SCHED set will
 234 -        * actually make the ->poll() call.  Therefore we avoid
 235 -        * accidentally calling ->poll() when NAPI is not scheduled.
 236 -        */
 237 -       work = 0;
 238 -       if (test_bit(NAPI_STATE_SCHED, &n->state)) {
 239 -               work = n->poll(n, weight);
 240 -               trace_napi_poll(n, work, weight);
 241 -       }
 242 -
 243 -       WARN_ON_ONCE(work > weight);
 244 -
 245 -       if (likely(work < weight))
 246 -               goto out_unlock;
 247 -
 248 -       /* Drivers must not modify the NAPI state if they
 249 -        * consume the entire weight.  In such cases this code
 250 -        * still "owns" the NAPI instance and therefore can
 251 -        * move the instance around on the list at-will.
 252 -        */
 253 -       if (unlikely(napi_disable_pending(n))) {
 254 -               napi_complete(n);
 255 +       if (!do_repoll)
 256                 goto out_unlock;
 257 -       }
 258 -
 259 -       if (n->gro_bitmask) {
 260 -               /* flush too old packets
 261 -                * If HZ < 1000, flush all packets.
 262 -                */
 263 -               napi_gro_flush(n, HZ >= 1000);
 264 -       }
 265 -
 266 -       gro_normal_list(n);
 267
 268         /* Some drivers may have called napi_schedule
 269          * prior to exhausting their budget.
 270 @@ -10270,6 +10329,10 @@ static int __init net_dev_init(void)
 271                 sd->backlog.weight = weight_p;
 272         }
 273
 274 +       napi_workq = alloc_workqueue("napi_workq", WQ_UNBOUND | WQ_HIGHPRI,
 275 +                                    WQ_UNBOUND_MAX_ACTIVE | WQ_SYSFS);
 276 +       BUG_ON(!napi_workq);
 277 +
 278         dev_boot_phase = 0;
 279
 280         /* The loopback device is special if any other network devices
 281 --- a/net/core/net-sysfs.c
 282 +++ b/net/core/net-sysfs.c
 283 @@ -442,6 +442,52 @@ static ssize_t proto_down_store(struct d
 284  }
 285  NETDEVICE_SHOW_RW(proto_down, fmt_dec);
 286
 287 +static int change_napi_threaded(struct net_device *dev, unsigned long val)
 288 +{
 289 +       struct napi_struct *napi;
 290 +
 291 +       if (list_empty(&dev->napi_list))
 292 +               return -EOPNOTSUPP;
 293 +
 294 +       list_for_each_entry(napi, &dev->napi_list, dev_list) {
 295 +               if (val)
 296 +                       set_bit(NAPI_STATE_THREADED, &napi->state);
 297 +               else
 298 +                       clear_bit(NAPI_STATE_THREADED, &napi->state);
 299 +       }
 300 +
 301 +       return 0;
 302 +}
 303 +
 304 +static ssize_t napi_threaded_store(struct device *dev,
 305 +                               struct device_attribute *attr,
 306 +                               const char *buf, size_t len)
 307 +{
 308 +       return netdev_store(dev, attr, buf, len, change_napi_threaded);
 309 +}
 310 +
 311 +static ssize_t napi_threaded_show(struct device *dev,
 312 +                                 struct device_attribute *attr,
 313 +                                 char *buf)
 314 +{
 315 +       struct net_device *netdev = to_net_dev(dev);
 316 +       struct napi_struct *napi;
 317 +       bool enabled = false;
 318 +
 319 +       if (!rtnl_trylock())
 320 +               return restart_syscall();
 321 +
 322 +       list_for_each_entry(napi, &netdev->napi_list, dev_list) {
 323 +               if (test_bit(NAPI_STATE_THREADED, &napi->state))
 324 +                       enabled = true;
 325 +       }
 326 +
 327 +       rtnl_unlock();
 328 +
 329 +       return sprintf(buf, fmt_dec, enabled);
 330 +}
 331 +static DEVICE_ATTR_RW(napi_threaded);
 332 +
 333  static ssize_t phys_port_id_show(struct device *dev,
 334                                  struct device_attribute *attr, char *buf)
 335  {
 336 @@ -532,6 +578,7 @@ static struct attribute *net_class_attrs
 337         &dev_attr_flags.attr,
 338         &dev_attr_tx_queue_len.attr,
 339         &dev_attr_gro_flush_timeout.attr,
 340 +       &dev_attr_napi_threaded.attr,
 341         &dev_attr_phys_port_id.attr,
 342         &dev_attr_phys_port_name.attr,
 343         &dev_attr_phys_switch_id.attr,