1 From b0a43db9087a21d96e1a0b716b8d9963064b2d58 Mon Sep 17 00:00:00 2001
2 From: Lorenzo Bianconi <lorenzo@kernel.org>
3 Date: Sat, 19 Oct 2019 10:13:27 +0200
4 Subject: [PATCH 7/7] net: mvneta: add XDP_TX support
6 Implement XDP_TX verdict and ndo_xdp_xmit net_device_ops function
9 Signed-off-by: Lorenzo Bianconi <lorenzo@kernel.org>
10 Signed-off-by: David S. Miller <davem@davemloft.net>
12 drivers/net/ethernet/marvell/mvneta.c | 128 ++++++++++++++++++++++++--
13 1 file changed, 121 insertions(+), 7 deletions(-)
15 --- a/drivers/net/ethernet/marvell/mvneta.c
16 +++ b/drivers/net/ethernet/marvell/mvneta.c
17 @@ -1813,16 +1813,19 @@ static void mvneta_txq_bufs_free(struct
19 mvneta_txq_inc_get(txq);
21 - if (!IS_TSO_HEADER(txq, tx_desc->buf_phys_addr))
22 + if (!IS_TSO_HEADER(txq, tx_desc->buf_phys_addr) &&
23 + buf->type != MVNETA_TYPE_XDP_TX)
24 dma_unmap_single(pp->dev->dev.parent,
25 tx_desc->buf_phys_addr,
26 tx_desc->data_size, DMA_TO_DEVICE);
30 - bytes_compl += buf->skb->len;
32 - dev_kfree_skb_any(buf->skb);
33 + if (buf->type == MVNETA_TYPE_SKB && buf->skb) {
34 + bytes_compl += buf->skb->len;
36 + dev_kfree_skb_any(buf->skb);
37 + } else if (buf->type == MVNETA_TYPE_XDP_TX ||
38 + buf->type == MVNETA_TYPE_XDP_NDO) {
39 + xdp_return_frame(buf->xdpf);
43 netdev_tx_completed_queue(nq, pkts_compl, bytes_compl);
44 @@ -1987,6 +1990,111 @@ int mvneta_rx_refill_queue(struct mvneta
48 +mvneta_xdp_submit_frame(struct mvneta_port *pp, struct mvneta_tx_queue *txq,
49 + struct xdp_frame *xdpf, bool dma_map)
51 + struct mvneta_tx_desc *tx_desc;
52 + struct mvneta_tx_buf *buf;
53 + dma_addr_t dma_addr;
55 + if (txq->count >= txq->tx_stop_threshold)
56 + return MVNETA_XDP_DROPPED;
58 + tx_desc = mvneta_txq_next_desc_get(txq);
60 + buf = &txq->buf[txq->txq_put_index];
63 + dma_addr = dma_map_single(pp->dev->dev.parent, xdpf->data,
64 + xdpf->len, DMA_TO_DEVICE);
65 + if (dma_mapping_error(pp->dev->dev.parent, dma_addr)) {
66 + mvneta_txq_desc_put(txq);
67 + return MVNETA_XDP_DROPPED;
69 + buf->type = MVNETA_TYPE_XDP_NDO;
71 + struct page *page = virt_to_page(xdpf->data);
73 + dma_addr = page_pool_get_dma_addr(page) +
74 + sizeof(*xdpf) + xdpf->headroom;
75 + dma_sync_single_for_device(pp->dev->dev.parent, dma_addr,
76 + xdpf->len, DMA_BIDIRECTIONAL);
77 + buf->type = MVNETA_TYPE_XDP_TX;
81 + tx_desc->command = MVNETA_TXD_FLZ_DESC;
82 + tx_desc->buf_phys_addr = dma_addr;
83 + tx_desc->data_size = xdpf->len;
85 + mvneta_update_stats(pp, 1, xdpf->len, true);
86 + mvneta_txq_inc_put(txq);
90 + return MVNETA_XDP_TX;
94 +mvneta_xdp_xmit_back(struct mvneta_port *pp, struct xdp_buff *xdp)
96 + struct mvneta_tx_queue *txq;
97 + struct netdev_queue *nq;
98 + struct xdp_frame *xdpf;
102 + xdpf = convert_to_xdp_frame(xdp);
103 + if (unlikely(!xdpf))
104 + return MVNETA_XDP_DROPPED;
106 + cpu = smp_processor_id();
107 + txq = &pp->txqs[cpu % txq_number];
108 + nq = netdev_get_tx_queue(pp->dev, txq->id);
110 + __netif_tx_lock(nq, cpu);
111 + ret = mvneta_xdp_submit_frame(pp, txq, xdpf, false);
112 + if (ret == MVNETA_XDP_TX)
113 + mvneta_txq_pend_desc_add(pp, txq, 0);
114 + __netif_tx_unlock(nq);
120 +mvneta_xdp_xmit(struct net_device *dev, int num_frame,
121 + struct xdp_frame **frames, u32 flags)
123 + struct mvneta_port *pp = netdev_priv(dev);
124 + int cpu = smp_processor_id();
125 + struct mvneta_tx_queue *txq;
126 + struct netdev_queue *nq;
130 + if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
133 + txq = &pp->txqs[cpu % txq_number];
134 + nq = netdev_get_tx_queue(pp->dev, txq->id);
136 + __netif_tx_lock(nq, cpu);
137 + for (i = 0; i < num_frame; i++) {
138 + ret = mvneta_xdp_submit_frame(pp, txq, frames[i], true);
139 + if (ret != MVNETA_XDP_TX) {
140 + xdp_return_frame_rx_napi(frames[i]);
145 + if (unlikely(flags & XDP_XMIT_FLUSH))
146 + mvneta_txq_pend_desc_add(pp, txq, 0);
147 + __netif_tx_unlock(nq);
149 + return num_frame - drops;
153 mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq,
154 struct bpf_prog *prog, struct xdp_buff *xdp)
156 @@ -2008,6 +2116,11 @@ mvneta_run_xdp(struct mvneta_port *pp, s
161 + ret = mvneta_xdp_xmit_back(pp, xdp);
162 + if (ret != MVNETA_XDP_TX)
163 + xdp_return_buff(xdp);
166 bpf_warn_invalid_xdp_action(act);
168 @@ -4581,6 +4694,7 @@ static const struct net_device_ops mvnet
169 .ndo_get_stats64 = mvneta_get_stats64,
170 .ndo_do_ioctl = mvneta_ioctl,
171 .ndo_bpf = mvneta_xdp,
172 + .ndo_xdp_xmit = mvneta_xdp_xmit,
175 static const struct ethtool_ops mvneta_eth_tool_ops = {