target/linux/mvebu/patches-5.4/013-net-mvneta-add-basic-XDP-support.patch

   1 From f21366d964bd41a2b823beed36a7c3e7a728857b Mon Sep 17 00:00:00 2001
   2 From: Lorenzo Bianconi <lorenzo@kernel.org>
   3 Date: Sat, 19 Oct 2019 10:13:24 +0200
   4 Subject: [PATCH 4/7] net: mvneta: add basic XDP support
   5
   6 Add basic XDP support to mvneta driver for devices that rely on software
   7 buffer management. Currently supported verdicts are:
   8 - XDP_DROP
   9 - XDP_PASS
  10 - XDP_REDIRECT
  11 - XDP_ABORTED
  12
  13 - iptables drop:
  14 $iptables -t raw -I PREROUTING -p udp --dport 9 -j DROP
  15 $nstat -n && sleep 1 && nstat
  16 IpInReceives            151169          0.0
  17 IpExtInOctets           6953544         0.0
  18 IpExtInNoECTPkts        151165          0.0
  19
  20 - XDP_DROP via xdp1
  21 $./samples/bpf/xdp1 3
  22 proto 0:        421419 pkt/s
  23 proto 0:        421444 pkt/s
  24 proto 0:        421393 pkt/s
  25 proto 0:        421440 pkt/s
  26 proto 0:        421184 pkt/s
  27
  28 Tested-by: Matteo Croce <mcroce@redhat.com>
  29 Signed-off-by: Lorenzo Bianconi <lorenzo@kernel.org>
  30 Signed-off-by: David S. Miller <davem@davemloft.net>
  31 ---
  32  drivers/net/ethernet/marvell/mvneta.c | 148 ++++++++++++++++++++++++--
  33  1 file changed, 139 insertions(+), 9 deletions(-)
  34
  35 --- a/drivers/net/ethernet/marvell/mvneta.c
  36 +++ b/drivers/net/ethernet/marvell/mvneta.c
  37 @@ -38,6 +38,7 @@
  38  #include <net/ipv6.h>
  39  #include <net/tso.h>
  40  #include <net/page_pool.h>
  41 +#include <linux/bpf_trace.h>
  42
  43  /* Registers */
  44  #define MVNETA_RXQ_CONFIG_REG(q)                (0x1400 + ((q) << 2))
  45 @@ -325,8 +326,10 @@
  46               ETH_HLEN + ETH_FCS_LEN,                        \
  47               cache_line_size())
  48
  49 +#define MVNETA_SKB_HEADROOM    (max(XDP_PACKET_HEADROOM, NET_SKB_PAD) + \
  50 +                                NET_IP_ALIGN)
  51  #define MVNETA_SKB_PAD (SKB_DATA_ALIGN(sizeof(struct skb_shared_info) + \
  52 -                        NET_SKB_PAD))
  53 +                        MVNETA_SKB_HEADROOM))
  54  #define MVNETA_SKB_SIZE(len)   (SKB_DATA_ALIGN(len) + MVNETA_SKB_PAD)
  55  #define MVNETA_MAX_RX_BUF_SIZE (PAGE_SIZE - MVNETA_SKB_PAD)
  56
  57 @@ -354,6 +357,11 @@ struct mvneta_statistic {
  58  #define T_REG_64       64
  59  #define T_SW           1
  60
  61 +#define MVNETA_XDP_PASS                BIT(0)
  62 +#define MVNETA_XDP_DROPPED     BIT(1)
  63 +#define MVNETA_XDP_TX          BIT(2)
  64 +#define MVNETA_XDP_REDIR       BIT(3)
  65 +
  66  static const struct mvneta_statistic mvneta_statistics[] = {
  67         { 0x3000, T_REG_64, "good_octets_received", },
  68         { 0x3010, T_REG_32, "good_frames_received", },
  69 @@ -435,6 +443,8 @@ struct mvneta_port {
  70         u32 cause_rx_tx;
  71         struct napi_struct napi;
  72
  73 +       struct bpf_prog *xdp_prog;
  74 +
  75         /* Core clock */
  76         struct clk *clk;
  77         /* AXI clock */
  78 @@ -1965,10 +1975,50 @@ int mvneta_rx_refill_queue(struct mvneta
  79  }
  80
  81  static int
  82 +mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq,
  83 +              struct bpf_prog *prog, struct xdp_buff *xdp)
  84 +{
  85 +       u32 ret, act = bpf_prog_run_xdp(prog, xdp);
  86 +
  87 +       switch (act) {
  88 +       case XDP_PASS:
  89 +               ret = MVNETA_XDP_PASS;
  90 +               break;
  91 +       case XDP_REDIRECT: {
  92 +               int err;
  93 +
  94 +               err = xdp_do_redirect(pp->dev, xdp, prog);
  95 +               if (err) {
  96 +                       ret = MVNETA_XDP_DROPPED;
  97 +                       xdp_return_buff(xdp);
  98 +               } else {
  99 +                       ret = MVNETA_XDP_REDIR;
 100 +               }
 101 +               break;
 102 +       }
 103 +       default:
 104 +               bpf_warn_invalid_xdp_action(act);
 105 +               /* fall through */
 106 +       case XDP_ABORTED:
 107 +               trace_xdp_exception(pp->dev, prog, act);
 108 +               /* fall through */
 109 +       case XDP_DROP:
 110 +               page_pool_recycle_direct(rxq->page_pool,
 111 +                                        virt_to_head_page(xdp->data));
 112 +               ret = MVNETA_XDP_DROPPED;
 113 +               break;
 114 +       }
 115 +
 116 +       return ret;
 117 +}
 118 +
 119 +static int
 120  mvneta_swbm_rx_frame(struct mvneta_port *pp,
 121                      struct mvneta_rx_desc *rx_desc,
 122                      struct mvneta_rx_queue *rxq,
 123 -                    struct page *page)
 124 +                    struct xdp_buff *xdp,
 125 +                    struct bpf_prog *xdp_prog,
 126 +                    struct page *page, u32 *xdp_ret)
 127  {
 128         unsigned char *data = page_address(page);
 129         int data_len = -MVNETA_MH_SIZE, len;
 130 @@ -1988,7 +2038,26 @@ mvneta_swbm_rx_frame(struct mvneta_port
 131                                 rx_desc->buf_phys_addr,
 132                                 len, dma_dir);
 133
 134 -       rxq->skb = build_skb(data, PAGE_SIZE);
 135 +       xdp->data_hard_start = data;
 136 +       xdp->data = data + MVNETA_SKB_HEADROOM + MVNETA_MH_SIZE;
 137 +       xdp->data_end = xdp->data + data_len;
 138 +       xdp_set_data_meta_invalid(xdp);
 139 +
 140 +       if (xdp_prog) {
 141 +               u32 ret;
 142 +
 143 +               ret = mvneta_run_xdp(pp, rxq, xdp_prog, xdp);
 144 +               if (ret != MVNETA_XDP_PASS) {
 145 +                       mvneta_update_stats(pp, 1,
 146 +                                           xdp->data_end - xdp->data,
 147 +                                           false);
 148 +                       rx_desc->buf_phys_addr = 0;
 149 +                       *xdp_ret |= ret;
 150 +                       return ret;
 151 +               }
 152 +       }
 153 +
 154 +       rxq->skb = build_skb(xdp->data_hard_start, PAGE_SIZE);
 155         if (unlikely(!rxq->skb)) {
 156                 netdev_err(dev,
 157                            "Can't allocate skb on queue %d\n",
 158 @@ -1999,8 +2068,9 @@ mvneta_swbm_rx_frame(struct mvneta_port
 159         }
 160         page_pool_release_page(rxq->page_pool, page);
 161
 162 -       skb_reserve(rxq->skb, MVNETA_MH_SIZE + NET_SKB_PAD);
 163 -       skb_put(rxq->skb, data_len);
 164 +       skb_reserve(rxq->skb,
 165 +                   xdp->data - xdp->data_hard_start);
 166 +       skb_put(rxq->skb, xdp->data_end - xdp->data);
 167         mvneta_rx_csum(pp, rx_desc->status, rxq->skb);
 168
 169         rxq->left_size = rx_desc->data_size - len;
 170 @@ -2034,7 +2104,7 @@ mvneta_swbm_add_rx_fragment(struct mvnet
 171                 /* refill descriptor with new buffer later */
 172                 skb_add_rx_frag(rxq->skb,
 173                                 skb_shinfo(rxq->skb)->nr_frags,
 174 -                               page, NET_SKB_PAD, data_len,
 175 +                               page, MVNETA_SKB_HEADROOM, data_len,
 176                                 PAGE_SIZE);
 177         }
 178         page_pool_release_page(rxq->page_pool, page);
 179 @@ -2049,11 +2119,18 @@ static int mvneta_rx_swbm(struct napi_st
 180  {
 181         int rcvd_pkts = 0, rcvd_bytes = 0, rx_proc = 0;
 182         struct net_device *dev = pp->dev;
 183 +       struct bpf_prog *xdp_prog;
 184 +       struct xdp_buff xdp_buf;
 185         int rx_todo, refill;
 186 +       u32 xdp_ret = 0;
 187
 188         /* Get number of received packets */
 189         rx_todo = mvneta_rxq_busy_desc_num_get(pp, rxq);
 190
 191 +       rcu_read_lock();
 192 +       xdp_prog = READ_ONCE(pp->xdp_prog);
 193 +       xdp_buf.rxq = &rxq->xdp_rxq;
 194 +
 195         /* Fairness NAPI loop */
 196         while (rx_proc < budget && rx_proc < rx_todo) {
 197                 struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq);
 198 @@ -2081,7 +2158,8 @@ static int mvneta_rx_swbm(struct napi_st
 199                                 continue;
 200                         }
 201
 202 -                       err = mvneta_swbm_rx_frame(pp, rx_desc, rxq, page);
 203 +                       err = mvneta_swbm_rx_frame(pp, rx_desc, rxq, &xdp_buf,
 204 +                                                  xdp_prog, page, &xdp_ret);
 205                         if (err)
 206                                 continue;
 207                 } else {
 208 @@ -2116,6 +2194,10 @@ static int mvneta_rx_swbm(struct napi_st
 209                 /* clean uncomplete skb pointer in queue */
 210                 rxq->skb = NULL;
 211         }
 212 +       rcu_read_unlock();
 213 +
 214 +       if (xdp_ret & MVNETA_XDP_REDIR)
 215 +               xdp_do_flush_map();
 216
 217         if (rcvd_pkts)
 218                 mvneta_update_stats(pp, rcvd_pkts, rcvd_bytes, false);
 219 @@ -2857,13 +2939,14 @@ static int mvneta_poll(struct napi_struc
 220  static int mvneta_create_page_pool(struct mvneta_port *pp,
 221                                    struct mvneta_rx_queue *rxq, int size)
 222  {
 223 +       struct bpf_prog *xdp_prog = READ_ONCE(pp->xdp_prog);
 224         struct page_pool_params pp_params = {
 225                 .order = 0,
 226                 .flags = PP_FLAG_DMA_MAP,
 227                 .pool_size = size,
 228                 .nid = cpu_to_node(0),
 229                 .dev = pp->dev->dev.parent,
 230 -               .dma_dir = DMA_FROM_DEVICE,
 231 +               .dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE,
 232         };
 233         int err;
 234
 235 @@ -3364,6 +3447,11 @@ static int mvneta_change_mtu(struct net_
 236                 mtu = ALIGN(MVNETA_RX_PKT_SIZE(mtu), 8);
 237         }
 238
 239 +       if (pp->xdp_prog && mtu > MVNETA_MAX_RX_BUF_SIZE) {
 240 +               netdev_info(dev, "Illegal MTU value %d for XDP mode\n", mtu);
 241 +               return -EINVAL;
 242 +       }
 243 +
 244         dev->mtu = mtu;
 245
 246         if (!netif_running(dev)) {
 247 @@ -4029,6 +4117,47 @@ static int mvneta_ioctl(struct net_devic
 248         return phylink_mii_ioctl(pp->phylink, ifr, cmd);
 249  }
 250
 251 +static int mvneta_xdp_setup(struct net_device *dev, struct bpf_prog *prog,
 252 +                           struct netlink_ext_ack *extack)
 253 +{
 254 +       bool need_update, running = netif_running(dev);
 255 +       struct mvneta_port *pp = netdev_priv(dev);
 256 +       struct bpf_prog *old_prog;
 257 +
 258 +       if (prog && dev->mtu > MVNETA_MAX_RX_BUF_SIZE) {
 259 +               NL_SET_ERR_MSG_MOD(extack, "Jumbo frames not supported on XDP");
 260 +               return -EOPNOTSUPP;
 261 +       }
 262 +
 263 +       need_update = !!pp->xdp_prog != !!prog;
 264 +       if (running && need_update)
 265 +               mvneta_stop(dev);
 266 +
 267 +       old_prog = xchg(&pp->xdp_prog, prog);
 268 +       if (old_prog)
 269 +               bpf_prog_put(old_prog);
 270 +
 271 +       if (running && need_update)
 272 +               return mvneta_open(dev);
 273 +
 274 +       return 0;
 275 +}
 276 +
 277 +static int mvneta_xdp(struct net_device *dev, struct netdev_bpf *xdp)
 278 +{
 279 +       struct mvneta_port *pp = netdev_priv(dev);
 280 +
 281 +       switch (xdp->command) {
 282 +       case XDP_SETUP_PROG:
 283 +               return mvneta_xdp_setup(dev, xdp->prog, xdp->extack);
 284 +       case XDP_QUERY_PROG:
 285 +               xdp->prog_id = pp->xdp_prog ? pp->xdp_prog->aux->id : 0;
 286 +               return 0;
 287 +       default:
 288 +               return -EINVAL;
 289 +       }
 290 +}
 291 +
 292  /* Ethtool methods */
 293
 294  /* Set link ksettings (phy address, speed) for ethtools */
 295 @@ -4425,6 +4554,7 @@ static const struct net_device_ops mvnet
 296         .ndo_fix_features    = mvneta_fix_features,
 297         .ndo_get_stats64     = mvneta_get_stats64,
 298         .ndo_do_ioctl        = mvneta_ioctl,
 299 +       .ndo_bpf             = mvneta_xdp,
 300  };
 301
 302  static const struct ethtool_ops mvneta_eth_tool_ops = {
 303 @@ -4713,7 +4843,7 @@ static int mvneta_probe(struct platform_
 304         SET_NETDEV_DEV(dev, &pdev->dev);
 305
 306         pp->id = global_port_id++;
 307 -       pp->rx_offset_correction = NET_SKB_PAD;
 308 +       pp->rx_offset_correction = MVNETA_SKB_HEADROOM;
 309
 310         /* Obtain access to BM resources if enabled and already initialized */
 311         bm_node = of_parse_phandle(dn, "buffer-manager", 0);