1 From 0db51da7a8e99f0803ec3a8e25c1a66234a219cb Mon Sep 17 00:00:00 2001
2 From: Lorenzo Bianconi <lorenzo@kernel.org>
3 Date: Sat, 19 Oct 2019 10:13:24 +0200
4 Subject: [PATCH 4/7] net: mvneta: add basic XDP support
6 Add basic XDP support to mvneta driver for devices that rely on software
7 buffer management. Currently supported verdicts are:
14 $iptables -t raw -I PREROUTING -p udp --dport 9 -j DROP
15 $nstat -n && sleep 1 && nstat
16 IpInReceives 151169 0.0
17 IpExtInOctets 6953544 0.0
18 IpExtInNoECTPkts 151165 0.0
28 Tested-by: Matteo Croce <mcroce@redhat.com>
29 Signed-off-by: Lorenzo Bianconi <lorenzo@kernel.org>
30 Signed-off-by: David S. Miller <davem@davemloft.net>
32 drivers/net/ethernet/marvell/mvneta.c | 148 ++++++++++++++++++++++++--
33 1 file changed, 139 insertions(+), 9 deletions(-)
35 --- a/drivers/net/ethernet/marvell/mvneta.c
36 +++ b/drivers/net/ethernet/marvell/mvneta.c
40 #include <net/page_pool.h>
41 +#include <linux/bpf_trace.h>
44 #define MVNETA_RXQ_CONFIG_REG(q) (0x1400 + ((q) << 2))
46 ETH_HLEN + ETH_FCS_LEN, \
49 +#define MVNETA_SKB_HEADROOM (max(XDP_PACKET_HEADROOM, NET_SKB_PAD) + \
51 #define MVNETA_SKB_PAD (SKB_DATA_ALIGN(sizeof(struct skb_shared_info) + \
53 + MVNETA_SKB_HEADROOM))
54 #define MVNETA_SKB_SIZE(len) (SKB_DATA_ALIGN(len) + MVNETA_SKB_PAD)
55 #define MVNETA_MAX_RX_BUF_SIZE (PAGE_SIZE - MVNETA_SKB_PAD)
57 @@ -354,6 +357,11 @@ struct mvneta_statistic {
61 +#define MVNETA_XDP_PASS BIT(0)
62 +#define MVNETA_XDP_DROPPED BIT(1)
63 +#define MVNETA_XDP_TX BIT(2)
64 +#define MVNETA_XDP_REDIR BIT(3)
66 static const struct mvneta_statistic mvneta_statistics[] = {
67 { 0x3000, T_REG_64, "good_octets_received", },
68 { 0x3010, T_REG_32, "good_frames_received", },
69 @@ -435,6 +443,8 @@ struct mvneta_port {
71 struct napi_struct napi;
73 + struct bpf_prog *xdp_prog;
78 @@ -1965,10 +1975,50 @@ int mvneta_rx_refill_queue(struct mvneta
82 +mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq,
83 + struct bpf_prog *prog, struct xdp_buff *xdp)
85 + u32 ret, act = bpf_prog_run_xdp(prog, xdp);
89 + ret = MVNETA_XDP_PASS;
91 + case XDP_REDIRECT: {
94 + err = xdp_do_redirect(pp->dev, xdp, prog);
96 + ret = MVNETA_XDP_DROPPED;
97 + xdp_return_buff(xdp);
99 + ret = MVNETA_XDP_REDIR;
104 + bpf_warn_invalid_xdp_action(act);
107 + trace_xdp_exception(pp->dev, prog, act);
110 + page_pool_recycle_direct(rxq->page_pool,
111 + virt_to_head_page(xdp->data));
112 + ret = MVNETA_XDP_DROPPED;
120 mvneta_swbm_rx_frame(struct mvneta_port *pp,
121 struct mvneta_rx_desc *rx_desc,
122 struct mvneta_rx_queue *rxq,
124 + struct xdp_buff *xdp,
125 + struct bpf_prog *xdp_prog,
126 + struct page *page, u32 *xdp_ret)
128 unsigned char *data = page_address(page);
129 int data_len = -MVNETA_MH_SIZE, len;
130 @@ -1988,7 +2038,26 @@ mvneta_swbm_rx_frame(struct mvneta_port
131 rx_desc->buf_phys_addr,
134 - rxq->skb = build_skb(data, PAGE_SIZE);
135 + xdp->data_hard_start = data;
136 + xdp->data = data + MVNETA_SKB_HEADROOM + MVNETA_MH_SIZE;
137 + xdp->data_end = xdp->data + data_len;
138 + xdp_set_data_meta_invalid(xdp);
143 + ret = mvneta_run_xdp(pp, rxq, xdp_prog, xdp);
144 + if (ret != MVNETA_XDP_PASS) {
145 + mvneta_update_stats(pp, 1,
146 + xdp->data_end - xdp->data,
148 + rx_desc->buf_phys_addr = 0;
154 + rxq->skb = build_skb(xdp->data_hard_start, PAGE_SIZE);
155 if (unlikely(!rxq->skb)) {
157 "Can't allocate skb on queue %d\n",
158 @@ -1999,8 +2068,9 @@ mvneta_swbm_rx_frame(struct mvneta_port
160 page_pool_release_page(rxq->page_pool, page);
162 - skb_reserve(rxq->skb, MVNETA_MH_SIZE + NET_SKB_PAD);
163 - skb_put(rxq->skb, data_len);
164 + skb_reserve(rxq->skb,
165 + xdp->data - xdp->data_hard_start);
166 + skb_put(rxq->skb, xdp->data_end - xdp->data);
167 mvneta_rx_csum(pp, rx_desc->status, rxq->skb);
169 rxq->left_size = rx_desc->data_size - len;
170 @@ -2034,7 +2104,7 @@ mvneta_swbm_add_rx_fragment(struct mvnet
171 /* refill descriptor with new buffer later */
172 skb_add_rx_frag(rxq->skb,
173 skb_shinfo(rxq->skb)->nr_frags,
174 - page, NET_SKB_PAD, data_len,
175 + page, MVNETA_SKB_HEADROOM, data_len,
178 page_pool_release_page(rxq->page_pool, page);
179 @@ -2049,11 +2119,18 @@ static int mvneta_rx_swbm(struct napi_st
181 int rcvd_pkts = 0, rcvd_bytes = 0, rx_proc = 0;
182 struct net_device *dev = pp->dev;
183 + struct bpf_prog *xdp_prog;
184 + struct xdp_buff xdp_buf;
188 /* Get number of received packets */
189 rx_todo = mvneta_rxq_busy_desc_num_get(pp, rxq);
192 + xdp_prog = READ_ONCE(pp->xdp_prog);
193 + xdp_buf.rxq = &rxq->xdp_rxq;
195 /* Fairness NAPI loop */
196 while (rx_proc < budget && rx_proc < rx_todo) {
197 struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq);
198 @@ -2081,7 +2158,8 @@ static int mvneta_rx_swbm(struct napi_st
202 - err = mvneta_swbm_rx_frame(pp, rx_desc, rxq, page);
203 + err = mvneta_swbm_rx_frame(pp, rx_desc, rxq, &xdp_buf,
204 + xdp_prog, page, &xdp_ret);
208 @@ -2116,6 +2194,10 @@ static int mvneta_rx_swbm(struct napi_st
209 /* clean uncomplete skb pointer in queue */
214 + if (xdp_ret & MVNETA_XDP_REDIR)
215 + xdp_do_flush_map();
218 mvneta_update_stats(pp, rcvd_pkts, rcvd_bytes, false);
219 @@ -2857,13 +2939,14 @@ static int mvneta_poll(struct napi_struc
220 static int mvneta_create_page_pool(struct mvneta_port *pp,
221 struct mvneta_rx_queue *rxq, int size)
223 + struct bpf_prog *xdp_prog = READ_ONCE(pp->xdp_prog);
224 struct page_pool_params pp_params = {
226 .flags = PP_FLAG_DMA_MAP,
228 .nid = cpu_to_node(0),
229 .dev = pp->dev->dev.parent,
230 - .dma_dir = DMA_FROM_DEVICE,
231 + .dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE,
235 @@ -3364,6 +3447,11 @@ static int mvneta_change_mtu(struct net_
236 mtu = ALIGN(MVNETA_RX_PKT_SIZE(mtu), 8);
239 + if (pp->xdp_prog && mtu > MVNETA_MAX_RX_BUF_SIZE) {
240 + netdev_info(dev, "Illegal MTU value %d for XDP mode\n", mtu);
246 if (!netif_running(dev)) {
247 @@ -4029,6 +4117,47 @@ static int mvneta_ioctl(struct net_devic
248 return phylink_mii_ioctl(pp->phylink, ifr, cmd);
251 +static int mvneta_xdp_setup(struct net_device *dev, struct bpf_prog *prog,
252 + struct netlink_ext_ack *extack)
254 + bool need_update, running = netif_running(dev);
255 + struct mvneta_port *pp = netdev_priv(dev);
256 + struct bpf_prog *old_prog;
258 + if (prog && dev->mtu > MVNETA_MAX_RX_BUF_SIZE) {
259 + NL_SET_ERR_MSG_MOD(extack, "Jumbo frames not supported on XDP");
260 + return -EOPNOTSUPP;
263 + need_update = !!pp->xdp_prog != !!prog;
264 + if (running && need_update)
267 + old_prog = xchg(&pp->xdp_prog, prog);
269 + bpf_prog_put(old_prog);
271 + if (running && need_update)
272 + return mvneta_open(dev);
277 +static int mvneta_xdp(struct net_device *dev, struct netdev_bpf *xdp)
279 + struct mvneta_port *pp = netdev_priv(dev);
281 + switch (xdp->command) {
282 + case XDP_SETUP_PROG:
283 + return mvneta_xdp_setup(dev, xdp->prog, xdp->extack);
284 + case XDP_QUERY_PROG:
285 + xdp->prog_id = pp->xdp_prog ? pp->xdp_prog->aux->id : 0;
292 /* Ethtool methods */
294 /* Set link ksettings (phy address, speed) for ethtools */
295 @@ -4425,6 +4554,7 @@ static const struct net_device_ops mvnet
296 .ndo_fix_features = mvneta_fix_features,
297 .ndo_get_stats64 = mvneta_get_stats64,
298 .ndo_do_ioctl = mvneta_ioctl,
299 + .ndo_bpf = mvneta_xdp,
302 static const struct ethtool_ops mvneta_eth_tool_ops = {
303 @@ -4713,7 +4843,7 @@ static int mvneta_probe(struct platform_
304 SET_NETDEV_DEV(dev, &pdev->dev);
306 pp->id = global_port_id++;
307 - pp->rx_offset_correction = NET_SKB_PAD;
308 + pp->rx_offset_correction = MVNETA_SKB_HEADROOM;
310 /* Obtain access to BM resources if enabled and already initialized */
311 bm_node = of_parse_phandle(dn, "buffer-manager", 0);