--- /dev/null
+From 3f57d8c40fea9b20543cab4da12f4680d2ef182c Mon Sep 17 00:00:00 2001
+From: Felix Fietkau <nbd@nbd.name>
+Date: Thu, 22 Apr 2021 22:20:54 -0700
+Subject: [PATCH] net: ethernet: mtk_eth_soc: fix RX VLAN offload
+
+The VLAN ID in the rx descriptor is only valid if the RX_DMA_VTAG bit is
+set. Fixes frames wrongly marked with VLAN tags.
+
+Signed-off-by: Felix Fietkau <nbd@nbd.name>
+[Ilya: fix commit message]
+Signed-off-by: Ilya Lipnitskiy <ilya.lipnitskiy@gmail.com>
+Signed-off-by: David S. Miller <davem@davemloft.net>
+---
+ drivers/net/ethernet/mediatek/mtk_eth_soc.c | 2 +-
+ drivers/net/ethernet/mediatek/mtk_eth_soc.h | 1 +
+ 2 files changed, 2 insertions(+), 1 deletion(-)
+
+--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
++++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+@@ -1319,7 +1319,7 @@ static int mtk_poll_rx(struct napi_struc
+ skb->protocol = eth_type_trans(skb, netdev);
+
+ if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX &&
+- RX_DMA_VID(trxd.rxd3))
++ (trxd.rxd2 & RX_DMA_VTAG))
+ __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
+ RX_DMA_VID(trxd.rxd3));
+ skb_record_rx_queue(skb, 0);
+--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h
++++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
+@@ -300,6 +300,7 @@
+ #define RX_DMA_LSO BIT(30)
+ #define RX_DMA_PLEN0(_x) (((_x) & 0x3fff) << 16)
+ #define RX_DMA_GET_PLEN0(_x) (((_x) >> 16) & 0x3fff)
++#define RX_DMA_VTAG BIT(15)
+
+ /* QDMA descriptor rxd3 */
+ #define RX_DMA_VID(_x) ((_x) & 0xfff)
--- /dev/null
+From 5196c417854942e218a59ec87bf7d414b3bd581e Mon Sep 17 00:00:00 2001
+From: Felix Fietkau <nbd@nbd.name>
+Date: Thu, 22 Apr 2021 22:20:55 -0700
+Subject: [PATCH] net: ethernet: mtk_eth_soc: unmap RX data before calling
+ build_skb
+
+Since build_skb accesses the data area (for initializing shinfo), dma unmap
+needs to happen before that call
+
+Signed-off-by: Felix Fietkau <nbd@nbd.name>
+[Ilya: split build_skb cleanup fix into a separate commit]
+Signed-off-by: Ilya Lipnitskiy <ilya.lipnitskiy@gmail.com>
+Signed-off-by: David S. Miller <davem@davemloft.net>
+---
+ drivers/net/ethernet/mediatek/mtk_eth_soc.c | 5 +++--
+ 1 file changed, 3 insertions(+), 2 deletions(-)
+
+--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
++++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+@@ -1298,6 +1298,9 @@ static int mtk_poll_rx(struct napi_struc
+ goto release_desc;
+ }
+
++ dma_unmap_single(eth->dev, trxd.rxd1,
++ ring->buf_size, DMA_FROM_DEVICE);
++
+ /* receive data */
+ skb = build_skb(data, ring->frag_size);
+ if (unlikely(!skb)) {
+@@ -1307,8 +1310,6 @@ static int mtk_poll_rx(struct napi_struc
+ }
+ skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
+
+- dma_unmap_single(eth->dev, trxd.rxd1,
+- ring->buf_size, DMA_FROM_DEVICE);
+ pktlen = RX_DMA_GET_PLEN0(trxd.rxd2);
+ skb->dev = netdev;
+ skb_put(skb, pktlen);
--- /dev/null
+From 787082ab9f7be4711e52f67c388535eda74a1269 Mon Sep 17 00:00:00 2001
+From: Ilya Lipnitskiy <ilya.lipnitskiy@gmail.com>
+Date: Thu, 22 Apr 2021 22:20:56 -0700
+Subject: [PATCH] net: ethernet: mtk_eth_soc: fix build_skb cleanup
+
+In case build_skb fails, call skb_free_frag on the correct pointer. Also
+update the DMA structures with the new mapping before exiting, because
+the mapping was successful
+
+Suggested-by: Felix Fietkau <nbd@nbd.name>
+Signed-off-by: Ilya Lipnitskiy <ilya.lipnitskiy@gmail.com>
+Signed-off-by: David S. Miller <davem@davemloft.net>
+---
+ drivers/net/ethernet/mediatek/mtk_eth_soc.c | 5 +++--
+ 1 file changed, 3 insertions(+), 2 deletions(-)
+
+--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
++++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+@@ -1304,9 +1304,9 @@ static int mtk_poll_rx(struct napi_struc
+ /* receive data */
+ skb = build_skb(data, ring->frag_size);
+ if (unlikely(!skb)) {
+- skb_free_frag(new_data);
++ skb_free_frag(data);
+ netdev->stats.rx_dropped++;
+- goto release_desc;
++ goto skip_rx;
+ }
+ skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
+
+@@ -1326,6 +1326,7 @@ static int mtk_poll_rx(struct napi_struc
+ skb_record_rx_queue(skb, 0);
+ napi_gro_receive(napi, skb);
+
++skip_rx:
+ ring->data[idx] = new_data;
+ rxd->rxd1 = (unsigned int)dma_addr;
+
--- /dev/null
+From c30c4a82739090a2de4a4e3f245355ea4fb3ec14 Mon Sep 17 00:00:00 2001
+From: Felix Fietkau <nbd@nbd.name>
+Date: Thu, 22 Apr 2021 22:20:57 -0700
+Subject: [PATCH] net: ethernet: mtk_eth_soc: use napi_consume_skb
+
+Should improve performance, since it can use bulk free
+
+Signed-off-by: Felix Fietkau <nbd@nbd.name>
+Signed-off-by: Ilya Lipnitskiy <ilya.lipnitskiy@gmail.com>
+Signed-off-by: David S. Miller <davem@davemloft.net>
+---
+ drivers/net/ethernet/mediatek/mtk_eth_soc.c | 19 ++++++++++++-------
+ 1 file changed, 12 insertions(+), 7 deletions(-)
+
+--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
++++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+@@ -858,7 +858,8 @@ static int txd_to_idx(struct mtk_tx_ring
+ return ((void *)dma - (void *)ring->dma) / sizeof(*dma);
+ }
+
+-static void mtk_tx_unmap(struct mtk_eth *eth, struct mtk_tx_buf *tx_buf)
++static void mtk_tx_unmap(struct mtk_eth *eth, struct mtk_tx_buf *tx_buf,
++ bool napi)
+ {
+ if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
+ if (tx_buf->flags & MTK_TX_FLAGS_SINGLE0) {
+@@ -890,8 +891,12 @@ static void mtk_tx_unmap(struct mtk_eth
+
+ tx_buf->flags = 0;
+ if (tx_buf->skb &&
+- (tx_buf->skb != (struct sk_buff *)MTK_DMA_DUMMY_DESC))
+- dev_kfree_skb_any(tx_buf->skb);
++ (tx_buf->skb != (struct sk_buff *)MTK_DMA_DUMMY_DESC)) {
++ if (napi)
++ napi_consume_skb(tx_buf->skb, napi);
++ else
++ dev_kfree_skb_any(tx_buf->skb);
++ }
+ tx_buf->skb = NULL;
+ }
+
+@@ -1069,7 +1074,7 @@ err_dma:
+ tx_buf = mtk_desc_to_tx_buf(ring, itxd);
+
+ /* unmap dma */
+- mtk_tx_unmap(eth, tx_buf);
++ mtk_tx_unmap(eth, tx_buf, false);
+
+ itxd->txd3 = TX_DMA_LS0 | TX_DMA_OWNER_CPU;
+ if (!MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
+@@ -1388,7 +1393,7 @@ static int mtk_poll_tx_qdma(struct mtk_e
+ done[mac]++;
+ budget--;
+ }
+- mtk_tx_unmap(eth, tx_buf);
++ mtk_tx_unmap(eth, tx_buf, true);
+
+ ring->last_free = desc;
+ atomic_inc(&ring->free_count);
+@@ -1425,7 +1430,7 @@ static int mtk_poll_tx_pdma(struct mtk_e
+ budget--;
+ }
+
+- mtk_tx_unmap(eth, tx_buf);
++ mtk_tx_unmap(eth, tx_buf, true);
+
+ desc = &ring->dma[cpu];
+ ring->last_free = desc;
+@@ -1627,7 +1632,7 @@ static void mtk_tx_clean(struct mtk_eth
+
+ if (ring->buf) {
+ for (i = 0; i < MTK_DMA_SIZE; i++)
+- mtk_tx_unmap(eth, &ring->buf[i]);
++ mtk_tx_unmap(eth, &ring->buf[i], false);
+ kfree(ring->buf);
+ ring->buf = NULL;
+ }
--- /dev/null
+From 3630d519d7c3eab92567658690e44ffe0517d109 Mon Sep 17 00:00:00 2001
+From: Felix Fietkau <nbd@nbd.name>
+Date: Thu, 22 Apr 2021 22:20:58 -0700
+Subject: [PATCH] net: ethernet: mtk_eth_soc: reduce MDIO bus access latency
+
+usleep_range often ends up sleeping much longer than the 10-20us provided
+as a range here. This causes significant latency in mdio bus acceses,
+which easily adds multiple seconds to the boot time on MT7621 when polling
+DSA slave ports.
+Use cond_resched instead of usleep_range, since the MDIO access does not
+take much time
+
+Signed-off-by: Felix Fietkau <nbd@nbd.name>
+Signed-off-by: Ilya Lipnitskiy <ilya.lipnitskiy@gmail.com>
+Signed-off-by: David S. Miller <davem@davemloft.net>
+---
+ drivers/net/ethernet/mediatek/mtk_eth_soc.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
++++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+@@ -86,7 +86,7 @@ static int mtk_mdio_busy_wait(struct mtk
+ return 0;
+ if (time_after(jiffies, t_start + PHY_IAC_TIMEOUT))
+ break;
+- usleep_range(10, 20);
++ cond_resched();
+ }
+
+ dev_err(eth->dev, "mdio: MDIO timeout\n");
--- /dev/null
+From 16ef670789b252b221700adc413497ed2f941d8a Mon Sep 17 00:00:00 2001
+From: Felix Fietkau <nbd@nbd.name>
+Date: Thu, 22 Apr 2021 22:20:59 -0700
+Subject: [PATCH] net: ethernet: mtk_eth_soc: remove unnecessary TX queue stops
+
+When running short on descriptors, only stop the queue for the netdev that
+tx was attempted for. By the time something tries to send on the other
+netdev, the ring might have some more room already.
+
+Signed-off-by: Felix Fietkau <nbd@nbd.name>
+Signed-off-by: Ilya Lipnitskiy <ilya.lipnitskiy@gmail.com>
+Signed-off-by: David S. Miller <davem@davemloft.net>
+---
+ drivers/net/ethernet/mediatek/mtk_eth_soc.c | 15 ++-------------
+ 1 file changed, 2 insertions(+), 13 deletions(-)
+
+--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
++++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+@@ -1131,17 +1131,6 @@ static void mtk_wake_queue(struct mtk_et
+ }
+ }
+
+-static void mtk_stop_queue(struct mtk_eth *eth)
+-{
+- int i;
+-
+- for (i = 0; i < MTK_MAC_COUNT; i++) {
+- if (!eth->netdev[i])
+- continue;
+- netif_stop_queue(eth->netdev[i]);
+- }
+-}
+-
+ static netdev_tx_t mtk_start_xmit(struct sk_buff *skb, struct net_device *dev)
+ {
+ struct mtk_mac *mac = netdev_priv(dev);
+@@ -1162,7 +1151,7 @@ static netdev_tx_t mtk_start_xmit(struct
+
+ tx_num = mtk_cal_txd_req(skb);
+ if (unlikely(atomic_read(&ring->free_count) <= tx_num)) {
+- mtk_stop_queue(eth);
++ netif_stop_queue(dev);
+ netif_err(eth, tx_queued, dev,
+ "Tx Ring full when queue awake!\n");
+ spin_unlock(ð->page_lock);
+@@ -1188,7 +1177,7 @@ static netdev_tx_t mtk_start_xmit(struct
+ goto drop;
+
+ if (unlikely(atomic_read(&ring->free_count) <= ring->thresh))
+- mtk_stop_queue(eth);
++ netif_stop_queue(dev);
+
+ spin_unlock(ð->page_lock);
+
--- /dev/null
+From 59555a8d0dd39bf60b7ca1ba5e7393d293f7398d Mon Sep 17 00:00:00 2001
+From: Felix Fietkau <nbd@nbd.name>
+Date: Thu, 22 Apr 2021 22:21:00 -0700
+Subject: [PATCH] net: ethernet: mtk_eth_soc: use larger burst size for QDMA TX
+
+Improves tx performance
+
+Signed-off-by: Felix Fietkau <nbd@nbd.name>
+Signed-off-by: Ilya Lipnitskiy <ilya.lipnitskiy@gmail.com>
+Signed-off-by: David S. Miller <davem@davemloft.net>
+---
+ drivers/net/ethernet/mediatek/mtk_eth_soc.c | 2 +-
+ drivers/net/ethernet/mediatek/mtk_eth_soc.h | 2 +-
+ 2 files changed, 2 insertions(+), 2 deletions(-)
+
+--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
++++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+@@ -2193,7 +2193,7 @@ static int mtk_start_dma(struct mtk_eth
+ if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
+ mtk_w32(eth,
+ MTK_TX_WB_DDONE | MTK_TX_DMA_EN |
+- MTK_DMA_SIZE_16DWORDS | MTK_NDP_CO_PRO |
++ MTK_TX_BT_32DWORDS | MTK_NDP_CO_PRO |
+ MTK_RX_DMA_EN | MTK_RX_2B_OFFSET |
+ MTK_RX_BT_32DWORDS,
+ MTK_QDMA_GLO_CFG);
+--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h
++++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
+@@ -202,7 +202,7 @@
+ #define MTK_RX_BT_32DWORDS (3 << 11)
+ #define MTK_NDP_CO_PRO BIT(10)
+ #define MTK_TX_WB_DDONE BIT(6)
+-#define MTK_DMA_SIZE_16DWORDS (2 << 4)
++#define MTK_TX_BT_32DWORDS (3 << 4)
+ #define MTK_RX_DMA_BUSY BIT(3)
+ #define MTK_TX_DMA_BUSY BIT(1)
+ #define MTK_RX_DMA_EN BIT(2)
--- /dev/null
+From 6b4423b258b91032c50a5efca15d3d9bb194ea1d Mon Sep 17 00:00:00 2001
+From: Felix Fietkau <nbd@nbd.name>
+Date: Thu, 22 Apr 2021 22:21:01 -0700
+Subject: [PATCH] net: ethernet: mtk_eth_soc: increase DMA ring sizes
+
+256 descriptors is not enough for multi-gigabit traffic under load on
+MT7622. Bump it to 512 to improve performance.
+
+Signed-off-by: Felix Fietkau <nbd@nbd.name>
+Signed-off-by: Ilya Lipnitskiy <ilya.lipnitskiy@gmail.com>
+Signed-off-by: David S. Miller <davem@davemloft.net>
+---
+ drivers/net/ethernet/mediatek/mtk_eth_soc.h | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h
++++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
+@@ -21,7 +21,7 @@
+ #define MTK_QDMA_PAGE_SIZE 2048
+ #define MTK_MAX_RX_LENGTH 1536
+ #define MTK_TX_DMA_BUF_LEN 0x3fff
+-#define MTK_DMA_SIZE 256
++#define MTK_DMA_SIZE 512
+ #define MTK_NAPI_WEIGHT 64
+ #define MTK_MAC_COUNT 2
+ #define MTK_RX_ETH_HLEN (VLAN_ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN)
--- /dev/null
+From e9229ffd550b2d8c4997c67a501dbc3919fd4e26 Mon Sep 17 00:00:00 2001
+From: Felix Fietkau <nbd@nbd.name>
+Date: Thu, 22 Apr 2021 22:21:02 -0700
+Subject: [PATCH] net: ethernet: mtk_eth_soc: implement dynamic interrupt
+ moderation
+
+Reduces the number of interrupts under load
+
+Signed-off-by: Felix Fietkau <nbd@nbd.name>
+[Ilya: add documentation for new struct fields]
+Signed-off-by: Ilya Lipnitskiy <ilya.lipnitskiy@gmail.com>
+Signed-off-by: David S. Miller <davem@davemloft.net>
+---
+ drivers/net/ethernet/mediatek/Kconfig | 1 +
+ drivers/net/ethernet/mediatek/mtk_eth_soc.c | 96 +++++++++++++++++++--
+ drivers/net/ethernet/mediatek/mtk_eth_soc.h | 41 +++++++--
+ 3 files changed, 124 insertions(+), 14 deletions(-)
+
+--- a/drivers/net/ethernet/mediatek/Kconfig
++++ b/drivers/net/ethernet/mediatek/Kconfig
+@@ -10,6 +10,7 @@ if NET_VENDOR_MEDIATEK
+ config NET_MEDIATEK_SOC
+ tristate "MediaTek SoC Gigabit Ethernet support"
+ select PHYLINK
++ select DIMLIB
+ help
+ This driver supports the gigabit ethernet MACs in the
+ MediaTek SoC family.
+--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
++++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+@@ -1233,12 +1233,13 @@ static void mtk_update_rx_cpu_idx(struct
+ static int mtk_poll_rx(struct napi_struct *napi, int budget,
+ struct mtk_eth *eth)
+ {
++ struct dim_sample dim_sample = {};
+ struct mtk_rx_ring *ring;
+ int idx;
+ struct sk_buff *skb;
+ u8 *data, *new_data;
+ struct mtk_rx_dma *rxd, trxd;
+- int done = 0;
++ int done = 0, bytes = 0;
+
+ while (done < budget) {
+ struct net_device *netdev;
+@@ -1312,6 +1313,7 @@ static int mtk_poll_rx(struct napi_struc
+ else
+ skb_checksum_none_assert(skb);
+ skb->protocol = eth_type_trans(skb, netdev);
++ bytes += pktlen;
+
+ if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX &&
+ (trxd.rxd2 & RX_DMA_VTAG))
+@@ -1344,6 +1346,12 @@ rx_done:
+ mtk_update_rx_cpu_idx(eth);
+ }
+
++ eth->rx_packets += done;
++ eth->rx_bytes += bytes;
++ dim_update_sample(eth->rx_events, eth->rx_packets, eth->rx_bytes,
++ &dim_sample);
++ net_dim(ð->rx_dim, dim_sample);
++
+ return done;
+ }
+
+@@ -1436,6 +1444,7 @@ static int mtk_poll_tx_pdma(struct mtk_e
+ static int mtk_poll_tx(struct mtk_eth *eth, int budget)
+ {
+ struct mtk_tx_ring *ring = ð->tx_ring;
++ struct dim_sample dim_sample = {};
+ unsigned int done[MTK_MAX_DEVS];
+ unsigned int bytes[MTK_MAX_DEVS];
+ int total = 0, i;
+@@ -1453,8 +1462,14 @@ static int mtk_poll_tx(struct mtk_eth *e
+ continue;
+ netdev_completed_queue(eth->netdev[i], done[i], bytes[i]);
+ total += done[i];
++ eth->tx_packets += done[i];
++ eth->tx_bytes += bytes[i];
+ }
+
++ dim_update_sample(eth->tx_events, eth->tx_packets, eth->tx_bytes,
++ &dim_sample);
++ net_dim(ð->tx_dim, dim_sample);
++
+ if (mtk_queue_stopped(eth) &&
+ (atomic_read(&ring->free_count) > ring->thresh))
+ mtk_wake_queue(eth);
+@@ -2129,6 +2144,7 @@ static irqreturn_t mtk_handle_irq_rx(int
+ {
+ struct mtk_eth *eth = _eth;
+
++ eth->rx_events++;
+ if (likely(napi_schedule_prep(ð->rx_napi))) {
+ __napi_schedule(ð->rx_napi);
+ mtk_rx_irq_disable(eth, MTK_RX_DONE_INT);
+@@ -2141,6 +2157,7 @@ static irqreturn_t mtk_handle_irq_tx(int
+ {
+ struct mtk_eth *eth = _eth;
+
++ eth->tx_events++;
+ if (likely(napi_schedule_prep(ð->tx_napi))) {
+ __napi_schedule(ð->tx_napi);
+ mtk_tx_irq_disable(eth, MTK_TX_DONE_INT);
+@@ -2325,6 +2342,9 @@ static int mtk_stop(struct net_device *d
+ napi_disable(ð->tx_napi);
+ napi_disable(ð->rx_napi);
+
++ cancel_work_sync(ð->rx_dim.work);
++ cancel_work_sync(ð->tx_dim.work);
++
+ if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
+ mtk_stop_dma(eth, MTK_QDMA_GLO_CFG);
+ mtk_stop_dma(eth, MTK_PDMA_GLO_CFG);
+@@ -2377,6 +2397,64 @@ err_disable_clks:
+ return ret;
+ }
+
++static void mtk_dim_rx(struct work_struct *work)
++{
++ struct dim *dim = container_of(work, struct dim, work);
++ struct mtk_eth *eth = container_of(dim, struct mtk_eth, rx_dim);
++ struct dim_cq_moder cur_profile;
++ u32 val, cur;
++
++ cur_profile = net_dim_get_rx_moderation(eth->rx_dim.mode,
++ dim->profile_ix);
++ spin_lock_bh(ð->dim_lock);
++
++ val = mtk_r32(eth, MTK_PDMA_DELAY_INT);
++ val &= MTK_PDMA_DELAY_TX_MASK;
++ val |= MTK_PDMA_DELAY_RX_EN;
++
++ cur = min_t(u32, DIV_ROUND_UP(cur_profile.usec, 20), MTK_PDMA_DELAY_PTIME_MASK);
++ val |= cur << MTK_PDMA_DELAY_RX_PTIME_SHIFT;
++
++ cur = min_t(u32, cur_profile.pkts, MTK_PDMA_DELAY_PINT_MASK);
++ val |= cur << MTK_PDMA_DELAY_RX_PINT_SHIFT;
++
++ mtk_w32(eth, val, MTK_PDMA_DELAY_INT);
++ mtk_w32(eth, val, MTK_QDMA_DELAY_INT);
++
++ spin_unlock_bh(ð->dim_lock);
++
++ dim->state = DIM_START_MEASURE;
++}
++
++static void mtk_dim_tx(struct work_struct *work)
++{
++ struct dim *dim = container_of(work, struct dim, work);
++ struct mtk_eth *eth = container_of(dim, struct mtk_eth, tx_dim);
++ struct dim_cq_moder cur_profile;
++ u32 val, cur;
++
++ cur_profile = net_dim_get_tx_moderation(eth->tx_dim.mode,
++ dim->profile_ix);
++ spin_lock_bh(ð->dim_lock);
++
++ val = mtk_r32(eth, MTK_PDMA_DELAY_INT);
++ val &= MTK_PDMA_DELAY_RX_MASK;
++ val |= MTK_PDMA_DELAY_TX_EN;
++
++ cur = min_t(u32, DIV_ROUND_UP(cur_profile.usec, 20), MTK_PDMA_DELAY_PTIME_MASK);
++ val |= cur << MTK_PDMA_DELAY_TX_PTIME_SHIFT;
++
++ cur = min_t(u32, cur_profile.pkts, MTK_PDMA_DELAY_PINT_MASK);
++ val |= cur << MTK_PDMA_DELAY_TX_PINT_SHIFT;
++
++ mtk_w32(eth, val, MTK_PDMA_DELAY_INT);
++ mtk_w32(eth, val, MTK_QDMA_DELAY_INT);
++
++ spin_unlock_bh(ð->dim_lock);
++
++ dim->state = DIM_START_MEASURE;
++}
++
+ static int mtk_hw_init(struct mtk_eth *eth)
+ {
+ int i, val, ret;
+@@ -2398,9 +2476,6 @@ static int mtk_hw_init(struct mtk_eth *e
+ goto err_disable_pm;
+ }
+
+- /* enable interrupt delay for RX */
+- mtk_w32(eth, MTK_PDMA_DELAY_RX_DELAY, MTK_PDMA_DELAY_INT);
+-
+ /* disable delay and normal interrupt */
+ mtk_tx_irq_disable(eth, ~0);
+ mtk_rx_irq_disable(eth, ~0);
+@@ -2439,11 +2514,11 @@ static int mtk_hw_init(struct mtk_eth *e
+ /* Enable RX VLan Offloading */
+ mtk_w32(eth, 1, MTK_CDMP_EG_CTRL);
+
+- /* enable interrupt delay for RX */
+- mtk_w32(eth, MTK_PDMA_DELAY_RX_DELAY, MTK_PDMA_DELAY_INT);
++ /* set interrupt delays based on current Net DIM sample */
++ mtk_dim_rx(ð->rx_dim.work);
++ mtk_dim_tx(ð->tx_dim.work);
+
+ /* disable delay and normal interrupt */
+- mtk_w32(eth, 0, MTK_QDMA_DELAY_INT);
+ mtk_tx_irq_disable(eth, ~0);
+ mtk_rx_irq_disable(eth, ~0);
+
+@@ -2948,6 +3023,13 @@ static int mtk_probe(struct platform_dev
+ spin_lock_init(ð->page_lock);
+ spin_lock_init(ð->tx_irq_lock);
+ spin_lock_init(ð->rx_irq_lock);
++ spin_lock_init(ð->dim_lock);
++
++ eth->rx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
++ INIT_WORK(ð->rx_dim.work, mtk_dim_rx);
++
++ eth->tx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
++ INIT_WORK(ð->tx_dim.work, mtk_dim_tx);
+
+ if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) {
+ eth->ethsys = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
+--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h
++++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
+@@ -16,6 +16,7 @@
+ #include <linux/refcount.h>
+ #include <linux/phylink.h>
+ #include <linux/rhashtable.h>
++#include <linux/dim.h>
+ #include "mtk_ppe.h"
+
+ #define MTK_QDMA_PAGE_SIZE 2048
+@@ -136,13 +137,18 @@
+
+ /* PDMA Delay Interrupt Register */
+ #define MTK_PDMA_DELAY_INT 0xa0c
++#define MTK_PDMA_DELAY_RX_MASK GENMASK(15, 0)
+ #define MTK_PDMA_DELAY_RX_EN BIT(15)
+-#define MTK_PDMA_DELAY_RX_PINT 4
+ #define MTK_PDMA_DELAY_RX_PINT_SHIFT 8
+-#define MTK_PDMA_DELAY_RX_PTIME 4
+-#define MTK_PDMA_DELAY_RX_DELAY \
+- (MTK_PDMA_DELAY_RX_EN | MTK_PDMA_DELAY_RX_PTIME | \
+- (MTK_PDMA_DELAY_RX_PINT << MTK_PDMA_DELAY_RX_PINT_SHIFT))
++#define MTK_PDMA_DELAY_RX_PTIME_SHIFT 0
++
++#define MTK_PDMA_DELAY_TX_MASK GENMASK(31, 16)
++#define MTK_PDMA_DELAY_TX_EN BIT(31)
++#define MTK_PDMA_DELAY_TX_PINT_SHIFT 24
++#define MTK_PDMA_DELAY_TX_PTIME_SHIFT 16
++
++#define MTK_PDMA_DELAY_PINT_MASK 0x7f
++#define MTK_PDMA_DELAY_PTIME_MASK 0xff
+
+ /* PDMA Interrupt Status Register */
+ #define MTK_PDMA_INT_STATUS 0xa20
+@@ -224,6 +230,7 @@
+ /* QDMA Interrupt Status Register */
+ #define MTK_QDMA_INT_STATUS 0x1A18
+ #define MTK_RX_DONE_DLY BIT(30)
++#define MTK_TX_DONE_DLY BIT(28)
+ #define MTK_RX_DONE_INT3 BIT(19)
+ #define MTK_RX_DONE_INT2 BIT(18)
+ #define MTK_RX_DONE_INT1 BIT(17)
+@@ -233,8 +240,7 @@
+ #define MTK_TX_DONE_INT1 BIT(1)
+ #define MTK_TX_DONE_INT0 BIT(0)
+ #define MTK_RX_DONE_INT MTK_RX_DONE_DLY
+-#define MTK_TX_DONE_INT (MTK_TX_DONE_INT0 | MTK_TX_DONE_INT1 | \
+- MTK_TX_DONE_INT2 | MTK_TX_DONE_INT3)
++#define MTK_TX_DONE_INT MTK_TX_DONE_DLY
+
+ /* QDMA Interrupt grouping registers */
+ #define MTK_QDMA_INT_GRP1 0x1a20
+@@ -843,6 +849,7 @@ struct mtk_sgmii {
+ * @page_lock: Make sure that register operations are atomic
+ * @tx_irq__lock: Make sure that IRQ register operations are atomic
+ * @rx_irq__lock: Make sure that IRQ register operations are atomic
++ * @dim_lock: Make sure that Net DIM operations are atomic
+ * @dummy_dev: we run 2 netdevs on 1 physical DMA ring and need a
+ * dummy for NAPI to work
+ * @netdev: The netdev instances
+@@ -861,6 +868,14 @@ struct mtk_sgmii {
+ * @rx_ring_qdma: Pointer to the memory holding info about the QDMA RX ring
+ * @tx_napi: The TX NAPI struct
+ * @rx_napi: The RX NAPI struct
++ * @rx_events: Net DIM RX event counter
++ * @rx_packets: Net DIM RX packet counter
++ * @rx_bytes: Net DIM RX byte counter
++ * @rx_dim: Net DIM RX context
++ * @tx_events: Net DIM TX event counter
++ * @tx_packets: Net DIM TX packet counter
++ * @tx_bytes: Net DIM TX byte counter
++ * @tx_dim: Net DIM TX context
+ * @scratch_ring: Newer SoCs need memory for a second HW managed TX ring
+ * @phy_scratch_ring: physical address of scratch_ring
+ * @scratch_head: The scratch memory that scratch_ring points to.
+@@ -905,6 +920,18 @@ struct mtk_eth {
+
+ const struct mtk_soc_data *soc;
+
++ spinlock_t dim_lock;
++
++ u32 rx_events;
++ u32 rx_packets;
++ u32 rx_bytes;
++ struct dim rx_dim;
++
++ u32 tx_events;
++ u32 tx_packets;
++ u32 tx_bytes;
++ struct dim tx_dim;
++
+ u32 tx_int_mask_reg;
+ u32 tx_int_status_reg;
+ u32 rx_dma_l4_valid;
--- /dev/null
+From 4e6bf609569c59b6bd6acf4a607c096cbd820d79 Mon Sep 17 00:00:00 2001
+From: Felix Fietkau <nbd@nbd.name>
+Date: Thu, 22 Apr 2021 22:21:03 -0700
+Subject: [PATCH] net: ethernet: mtk_eth_soc: cache HW pointer of last freed TX
+ descriptor
+
+The value is only updated by the CPU, so it is cheaper to access from the
+ring data structure than from a hardware register.
+
+Signed-off-by: Felix Fietkau <nbd@nbd.name>
+Signed-off-by: Ilya Lipnitskiy <ilya.lipnitskiy@gmail.com>
+Signed-off-by: David S. Miller <davem@davemloft.net>
+---
+ drivers/net/ethernet/mediatek/mtk_eth_soc.c | 8 ++++----
+ drivers/net/ethernet/mediatek/mtk_eth_soc.h | 2 ++
+ 2 files changed, 6 insertions(+), 4 deletions(-)
+
+--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
++++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+@@ -1364,7 +1364,7 @@ static int mtk_poll_tx_qdma(struct mtk_e
+ struct mtk_tx_buf *tx_buf;
+ u32 cpu, dma;
+
+- cpu = mtk_r32(eth, MTK_QTX_CRX_PTR);
++ cpu = ring->last_free_ptr;
+ dma = mtk_r32(eth, MTK_QTX_DRX_PTR);
+
+ desc = mtk_qdma_phys_to_virt(ring, cpu);
+@@ -1398,6 +1398,7 @@ static int mtk_poll_tx_qdma(struct mtk_e
+ cpu = next_cpu;
+ }
+
++ ring->last_free_ptr = cpu;
+ mtk_w32(eth, cpu, MTK_QTX_CRX_PTR);
+
+ return budget;
+@@ -1598,6 +1599,7 @@ static int mtk_tx_alloc(struct mtk_eth *
+ atomic_set(&ring->free_count, MTK_DMA_SIZE - 2);
+ ring->next_free = &ring->dma[0];
+ ring->last_free = &ring->dma[MTK_DMA_SIZE - 1];
++ ring->last_free_ptr = (u32)(ring->phys + ((MTK_DMA_SIZE - 1) * sz));
+ ring->thresh = MAX_SKB_FRAGS;
+
+ /* make sure that all changes to the dma ring are flushed before we
+@@ -1611,9 +1613,7 @@ static int mtk_tx_alloc(struct mtk_eth *
+ mtk_w32(eth,
+ ring->phys + ((MTK_DMA_SIZE - 1) * sz),
+ MTK_QTX_CRX_PTR);
+- mtk_w32(eth,
+- ring->phys + ((MTK_DMA_SIZE - 1) * sz),
+- MTK_QTX_DRX_PTR);
++ mtk_w32(eth, ring->last_free_ptr, MTK_QTX_DRX_PTR);
+ mtk_w32(eth, (QDMA_RES_THRES << 8) | QDMA_RES_THRES,
+ MTK_QTX_CFG(0));
+ } else {
+--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h
++++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
+@@ -636,6 +636,7 @@ struct mtk_tx_buf {
+ * @phys: The physical addr of tx_buf
+ * @next_free: Pointer to the next free descriptor
+ * @last_free: Pointer to the last free descriptor
++ * @last_free_ptr: Hardware pointer value of the last free descriptor
+ * @thresh: The threshold of minimum amount of free descriptors
+ * @free_count: QDMA uses a linked list. Track how many free descriptors
+ * are present
+@@ -646,6 +647,7 @@ struct mtk_tx_ring {
+ dma_addr_t phys;
+ struct mtk_tx_dma *next_free;
+ struct mtk_tx_dma *last_free;
++ u32 last_free_ptr;
+ u16 thresh;
+ atomic_t free_count;
+ int dma_size;
--- /dev/null
+From 816ac3e6e67bdd78d86226c6eb53619780750e92 Mon Sep 17 00:00:00 2001
+From: Felix Fietkau <nbd@nbd.name>
+Date: Thu, 22 Apr 2021 22:21:04 -0700
+Subject: [PATCH] net: ethernet: mtk_eth_soc: only read the full RX descriptor
+ if DMA is done
+
+Uncached memory access is expensive, and there is no need to access all
+descriptor words if we can't process them anyway
+
+Signed-off-by: Felix Fietkau <nbd@nbd.name>
+Signed-off-by: Ilya Lipnitskiy <ilya.lipnitskiy@gmail.com>
+Signed-off-by: David S. Miller <davem@davemloft.net>
+---
+ drivers/net/ethernet/mediatek/mtk_eth_soc.c | 12 ++++++++----
+ 1 file changed, 8 insertions(+), 4 deletions(-)
+
+--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
++++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+@@ -777,13 +777,18 @@ static inline int mtk_max_buf_size(int f
+ return buf_size;
+ }
+
+-static inline void mtk_rx_get_desc(struct mtk_rx_dma *rxd,
++static inline bool mtk_rx_get_desc(struct mtk_rx_dma *rxd,
+ struct mtk_rx_dma *dma_rxd)
+ {
+- rxd->rxd1 = READ_ONCE(dma_rxd->rxd1);
+ rxd->rxd2 = READ_ONCE(dma_rxd->rxd2);
++ if (!(rxd->rxd2 & RX_DMA_DONE))
++ return false;
++
++ rxd->rxd1 = READ_ONCE(dma_rxd->rxd1);
+ rxd->rxd3 = READ_ONCE(dma_rxd->rxd3);
+ rxd->rxd4 = READ_ONCE(dma_rxd->rxd4);
++
++ return true;
+ }
+
+ /* the qdma core needs scratch memory to be setup */
+@@ -1255,8 +1260,7 @@ static int mtk_poll_rx(struct napi_struc
+ rxd = &ring->dma[idx];
+ data = ring->data[idx];
+
+- mtk_rx_get_desc(&trxd, rxd);
+- if (!(trxd.rxd2 & RX_DMA_DONE))
++ if (!mtk_rx_get_desc(&trxd, rxd))
+ break;
+
+ /* find out which mac the packet come from. values start at 1 */
--- /dev/null
+From 16769a8923fad5a5377253bcd76b0e0d64976c73 Mon Sep 17 00:00:00 2001
+From: Felix Fietkau <nbd@nbd.name>
+Date: Thu, 22 Apr 2021 22:21:05 -0700
+Subject: [PATCH] net: ethernet: mtk_eth_soc: reduce unnecessary interrupts
+
+Avoid rearming interrupt if napi_complete returns false
+
+Signed-off-by: Felix Fietkau <nbd@nbd.name>
+Signed-off-by: Ilya Lipnitskiy <ilya.lipnitskiy@gmail.com>
+Signed-off-by: David S. Miller <davem@davemloft.net>
+---
+ drivers/net/ethernet/mediatek/mtk_eth_soc.c | 9 +++++----
+ 1 file changed, 5 insertions(+), 4 deletions(-)
+
+--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
++++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+@@ -1519,8 +1519,8 @@ static int mtk_napi_tx(struct napi_struc
+ if (status & MTK_TX_DONE_INT)
+ return budget;
+
+- napi_complete(napi);
+- mtk_tx_irq_enable(eth, MTK_TX_DONE_INT);
++ if (napi_complete(napi))
++ mtk_tx_irq_enable(eth, MTK_TX_DONE_INT);
+
+ return tx_done;
+ }
+@@ -1553,8 +1553,9 @@ poll_again:
+ remain_budget -= rx_done;
+ goto poll_again;
+ }
+- napi_complete(napi);
+- mtk_rx_irq_enable(eth, MTK_RX_DONE_INT);
++
++ if (napi_complete(napi))
++ mtk_rx_irq_enable(eth, MTK_RX_DONE_INT);
+
+ return rx_done + budget - remain_budget;
+ }
--- /dev/null
+From db2c7b353db3b3f71b55f9ff4627d8a786446fbe Mon Sep 17 00:00:00 2001
+From: Ilya Lipnitskiy <ilya.lipnitskiy@gmail.com>
+Date: Thu, 22 Apr 2021 22:21:06 -0700
+Subject: [PATCH] net: ethernet: mtk_eth_soc: rework NAPI callbacks
+
+Use napi_complete_done to communicate total TX and RX work done to NAPI.
+Count total RX work up instead of remaining work down for clarity.
+Remove unneeded local variables for clarity. Use do {} while instead of
+goto for clarity.
+
+Suggested-by: Jakub Kicinski <kuba@kernel.org>
+Signed-off-by: Ilya Lipnitskiy <ilya.lipnitskiy@gmail.com>
+Signed-off-by: David S. Miller <davem@davemloft.net>
+---
+ drivers/net/ethernet/mediatek/mtk_eth_soc.c | 54 +++++++++------------
+ 1 file changed, 24 insertions(+), 30 deletions(-)
+
+--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
++++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+@@ -1496,7 +1496,6 @@ static void mtk_handle_status_irq(struct
+ static int mtk_napi_tx(struct napi_struct *napi, int budget)
+ {
+ struct mtk_eth *eth = container_of(napi, struct mtk_eth, tx_napi);
+- u32 status, mask;
+ int tx_done = 0;
+
+ if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
+@@ -1505,21 +1504,19 @@ static int mtk_napi_tx(struct napi_struc
+ tx_done = mtk_poll_tx(eth, budget);
+
+ if (unlikely(netif_msg_intr(eth))) {
+- status = mtk_r32(eth, eth->tx_int_status_reg);
+- mask = mtk_r32(eth, eth->tx_int_mask_reg);
+ dev_info(eth->dev,
+- "done tx %d, intr 0x%08x/0x%x\n",
+- tx_done, status, mask);
++ "done tx %d, intr 0x%08x/0x%x\n", tx_done,
++ mtk_r32(eth, eth->tx_int_status_reg),
++ mtk_r32(eth, eth->tx_int_mask_reg));
+ }
+
+ if (tx_done == budget)
+ return budget;
+
+- status = mtk_r32(eth, eth->tx_int_status_reg);
+- if (status & MTK_TX_DONE_INT)
++ if (mtk_r32(eth, eth->tx_int_status_reg) & MTK_TX_DONE_INT)
+ return budget;
+
+- if (napi_complete(napi))
++ if (napi_complete_done(napi, tx_done))
+ mtk_tx_irq_enable(eth, MTK_TX_DONE_INT);
+
+ return tx_done;
+@@ -1528,36 +1525,33 @@ static int mtk_napi_tx(struct napi_struc
+ static int mtk_napi_rx(struct napi_struct *napi, int budget)
+ {
+ struct mtk_eth *eth = container_of(napi, struct mtk_eth, rx_napi);
+- u32 status, mask;
+- int rx_done = 0;
+- int remain_budget = budget;
++ int rx_done_total = 0;
+
+ mtk_handle_status_irq(eth);
+
+-poll_again:
+- mtk_w32(eth, MTK_RX_DONE_INT, MTK_PDMA_INT_STATUS);
+- rx_done = mtk_poll_rx(napi, remain_budget, eth);
++ do {
++ int rx_done;
+
+- if (unlikely(netif_msg_intr(eth))) {
+- status = mtk_r32(eth, MTK_PDMA_INT_STATUS);
+- mask = mtk_r32(eth, MTK_PDMA_INT_MASK);
+- dev_info(eth->dev,
+- "done rx %d, intr 0x%08x/0x%x\n",
+- rx_done, status, mask);
+- }
+- if (rx_done == remain_budget)
+- return budget;
++ mtk_w32(eth, MTK_RX_DONE_INT, MTK_PDMA_INT_STATUS);
++ rx_done = mtk_poll_rx(napi, budget - rx_done_total, eth);
++ rx_done_total += rx_done;
++
++ if (unlikely(netif_msg_intr(eth))) {
++ dev_info(eth->dev,
++ "done rx %d, intr 0x%08x/0x%x\n", rx_done,
++ mtk_r32(eth, MTK_PDMA_INT_STATUS),
++ mtk_r32(eth, MTK_PDMA_INT_MASK));
++ }
+
+- status = mtk_r32(eth, MTK_PDMA_INT_STATUS);
+- if (status & MTK_RX_DONE_INT) {
+- remain_budget -= rx_done;
+- goto poll_again;
+- }
++ if (rx_done_total == budget)
++ return budget;
++
++ } while (mtk_r32(eth, MTK_PDMA_INT_STATUS) & MTK_RX_DONE_INT);
+
+- if (napi_complete(napi))
++ if (napi_complete_done(napi, rx_done_total))
+ mtk_rx_irq_enable(eth, MTK_RX_DONE_INT);
+
+- return rx_done + budget - remain_budget;
++ return rx_done_total;
+ }
+
+ static int mtk_tx_alloc(struct mtk_eth *eth)
--- /dev/null
+From fa817272c37ef78e25dc14e4760ac78a7043a18a Mon Sep 17 00:00:00 2001
+From: Felix Fietkau <nbd@nbd.name>
+Date: Thu, 22 Apr 2021 22:21:07 -0700
+Subject: [PATCH] net: ethernet: mtk_eth_soc: set PPE flow hash as skb hash if
+ present
+
+This improves GRO performance
+
+Signed-off-by: Felix Fietkau <nbd@nbd.name>
+[Ilya: Use MTK_RXD4_FOE_ENTRY instead of GENMASK(13, 0)]
+Signed-off-by: Ilya Lipnitskiy <ilya.lipnitskiy@gmail.com>
+Signed-off-by: David S. Miller <davem@davemloft.net>
+---
+ drivers/net/ethernet/mediatek/mtk_eth_soc.c | 8 ++++++++
+ 1 file changed, 8 insertions(+)
+
+--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
++++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+@@ -19,6 +19,7 @@
+ #include <linux/interrupt.h>
+ #include <linux/pinctrl/devinfo.h>
+ #include <linux/phylink.h>
++#include <linux/jhash.h>
+ #include <net/dsa.h>
+
+ #include "mtk_eth_soc.h"
+@@ -1250,6 +1251,7 @@ static int mtk_poll_rx(struct napi_struc
+ struct net_device *netdev;
+ unsigned int pktlen;
+ dma_addr_t dma_addr;
++ u32 hash;
+ int mac;
+
+ ring = mtk_get_rx_ring(eth);
+@@ -1319,6 +1321,12 @@ static int mtk_poll_rx(struct napi_struc
+ skb->protocol = eth_type_trans(skb, netdev);
+ bytes += pktlen;
+
++ hash = trxd.rxd4 & MTK_RXD4_FOE_ENTRY;
++ if (hash != MTK_RXD4_FOE_ENTRY) {
++ hash = jhash_1word(hash, 0);
++ skb_set_hash(skb, hash, PKT_HASH_TYPE_L4);
++ }
++
+ if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX &&
+ (trxd.rxd2 & RX_DMA_VTAG))
+ __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
--- /dev/null
+From 3bc8e0aff23be0526af0dbc7973a8866a08d73f1 Mon Sep 17 00:00:00 2001
+From: Ilya Lipnitskiy <ilya.lipnitskiy@gmail.com>
+Date: Thu, 22 Apr 2021 22:21:08 -0700
+Subject: [PATCH] net: ethernet: mtk_eth_soc: use iopoll.h macro for DMA init
+
+Replace a tight busy-wait loop without a pause with a standard
+readx_poll_timeout_atomic routine with a 5 us poll period.
+
+Tested by booting a MT7621 device to ensure the driver initializes
+properly.
+
+Signed-off-by: Ilya Lipnitskiy <ilya.lipnitskiy@gmail.com>
+Reviewed-by: Andrew Lunn <andrew@lunn.ch>
+Signed-off-by: David S. Miller <davem@davemloft.net>
+---
+ drivers/net/ethernet/mediatek/mtk_eth_soc.c | 29 +++++++++------------
+ drivers/net/ethernet/mediatek/mtk_eth_soc.h | 2 +-
+ 2 files changed, 14 insertions(+), 17 deletions(-)
+
+--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
++++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+@@ -2033,25 +2033,22 @@ static int mtk_set_features(struct net_d
+ /* wait for DMA to finish whatever it is doing before we start using it again */
+ static int mtk_dma_busy_wait(struct mtk_eth *eth)
+ {
+- unsigned long t_start = jiffies;
++ unsigned int reg;
++ int ret;
++ u32 val;
+
+- while (1) {
+- if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
+- if (!(mtk_r32(eth, MTK_QDMA_GLO_CFG) &
+- (MTK_RX_DMA_BUSY | MTK_TX_DMA_BUSY)))
+- return 0;
+- } else {
+- if (!(mtk_r32(eth, MTK_PDMA_GLO_CFG) &
+- (MTK_RX_DMA_BUSY | MTK_TX_DMA_BUSY)))
+- return 0;
+- }
++ if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
++ reg = MTK_QDMA_GLO_CFG;
++ else
++ reg = MTK_PDMA_GLO_CFG;
+
+- if (time_after(jiffies, t_start + MTK_DMA_BUSY_TIMEOUT))
+- break;
+- }
++ ret = readx_poll_timeout_atomic(__raw_readl, eth->base + reg, val,
++ !(val & (MTK_RX_DMA_BUSY | MTK_TX_DMA_BUSY)),
++ 5, MTK_DMA_BUSY_TIMEOUT_US);
++ if (ret)
++ dev_err(eth->dev, "DMA init timeout\n");
+
+- dev_err(eth->dev, "DMA init timeout\n");
+- return -1;
++ return ret;
+ }
+
+ static int mtk_dma_init(struct mtk_eth *eth)
+--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h
++++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
+@@ -213,7 +213,7 @@
+ #define MTK_TX_DMA_BUSY BIT(1)
+ #define MTK_RX_DMA_EN BIT(2)
+ #define MTK_TX_DMA_EN BIT(0)
+-#define MTK_DMA_BUSY_TIMEOUT HZ
++#define MTK_DMA_BUSY_TIMEOUT_US 1000000
+
+ /* QDMA Reset Index Register */
+ #define MTK_QDMA_RST_IDX 0x1A08
+++ /dev/null
-From: Felix Fietkau <nbd@nbd.name>
-Date: Mon, 8 Jun 2020 17:01:12 +0200
-Subject: [PATCH] net: ethernet: mtk_eth_soc: use napi_consume_skb
-
-Should improve performance, since it can use bulk free
-
-Signed-off-by: Felix Fietkau <nbd@nbd.name>
----
-
---- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
-+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
-@@ -858,7 +858,8 @@ static int txd_to_idx(struct mtk_tx_ring
- return ((void *)dma - (void *)ring->dma) / sizeof(*dma);
- }
-
--static void mtk_tx_unmap(struct mtk_eth *eth, struct mtk_tx_buf *tx_buf)
-+static void mtk_tx_unmap(struct mtk_eth *eth, struct mtk_tx_buf *tx_buf,
-+ bool napi)
- {
- if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
- if (tx_buf->flags & MTK_TX_FLAGS_SINGLE0) {
-@@ -890,8 +891,12 @@ static void mtk_tx_unmap(struct mtk_eth
-
- tx_buf->flags = 0;
- if (tx_buf->skb &&
-- (tx_buf->skb != (struct sk_buff *)MTK_DMA_DUMMY_DESC))
-- dev_kfree_skb_any(tx_buf->skb);
-+ (tx_buf->skb != (struct sk_buff *)MTK_DMA_DUMMY_DESC)) {
-+ if (napi)
-+ napi_consume_skb(tx_buf->skb, napi);
-+ else
-+ dev_kfree_skb_any(tx_buf->skb);
-+ }
- tx_buf->skb = NULL;
- }
-
-@@ -1069,7 +1074,7 @@ err_dma:
- tx_buf = mtk_desc_to_tx_buf(ring, itxd);
-
- /* unmap dma */
-- mtk_tx_unmap(eth, tx_buf);
-+ mtk_tx_unmap(eth, tx_buf, false);
-
- itxd->txd3 = TX_DMA_LS0 | TX_DMA_OWNER_CPU;
- if (!MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
-@@ -1386,7 +1391,7 @@ static int mtk_poll_tx_qdma(struct mtk_e
- done[mac]++;
- budget--;
- }
-- mtk_tx_unmap(eth, tx_buf);
-+ mtk_tx_unmap(eth, tx_buf, true);
-
- ring->last_free = desc;
- atomic_inc(&ring->free_count);
-@@ -1423,7 +1428,7 @@ static int mtk_poll_tx_pdma(struct mtk_e
- budget--;
- }
-
-- mtk_tx_unmap(eth, tx_buf);
-+ mtk_tx_unmap(eth, tx_buf, true);
-
- desc = &ring->dma[cpu];
- ring->last_free = desc;
-@@ -1625,7 +1630,7 @@ static void mtk_tx_clean(struct mtk_eth
-
- if (ring->buf) {
- for (i = 0; i < MTK_DMA_SIZE; i++)
-- mtk_tx_unmap(eth, &ring->buf[i]);
-+ mtk_tx_unmap(eth, &ring->buf[i], false);
- kfree(ring->buf);
- ring->buf = NULL;
- }
+++ /dev/null
-From: Felix Fietkau <nbd@nbd.name>
-Date: Mon, 8 Jun 2020 17:02:39 +0200
-Subject: [PATCH] net: ethernet: mtk_eth_soc: significantly reduce mdio bus
- access latency
-
-usleep_range often ends up sleeping much longer than the 10-20us provided
-as a range here. This causes significant latency in mdio bus acceses,
-which easily adds multiple seconds to the boot time on MT7621 when polling
-DSA slave ports.
-Use cond_resched instead of usleep_range, since the MDIO access does not
-take much time
-
-Signed-off-by: Felix Fietkau <nbd@nbd.name>
----
-
---- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
-+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
-@@ -86,7 +86,7 @@ static int mtk_mdio_busy_wait(struct mtk
- return 0;
- if (time_after(jiffies, t_start + PHY_IAC_TIMEOUT))
- break;
-- usleep_range(10, 20);
-+ cond_resched();
- }
-
- dev_err(eth->dev, "mdio: MDIO timeout\n");
+++ /dev/null
-From: Felix Fietkau <nbd@nbd.name>
-Date: Wed, 26 Aug 2020 16:52:12 +0200
-Subject: [PATCH] net: ethernet: mtk_eth_soc: fix rx vlan offload
-
-The VLAN ID in the rx descriptor is only valid if the RX_DMA_VID bit is set
-Fixes frames wrongly marked with VLAN tags
-
-Signed-off-by: Felix Fietkau <nbd@nbd.name>
----
-
---- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
-+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
-@@ -1324,7 +1324,7 @@ static int mtk_poll_rx(struct napi_struc
- skb->protocol = eth_type_trans(skb, netdev);
-
- if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX &&
-- RX_DMA_VID(trxd.rxd3))
-+ (trxd.rxd2 & RX_DMA_VTAG))
- __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
- RX_DMA_VID(trxd.rxd3));
- skb_record_rx_queue(skb, 0);
---- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h
-+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
-@@ -300,6 +300,7 @@
- #define RX_DMA_LSO BIT(30)
- #define RX_DMA_PLEN0(_x) (((_x) & 0x3fff) << 16)
- #define RX_DMA_GET_PLEN0(_x) (((_x) >> 16) & 0x3fff)
-+#define RX_DMA_VTAG BIT(15)
-
- /* QDMA descriptor rxd3 */
- #define RX_DMA_VID(_x) ((_x) & 0xfff)
+++ /dev/null
-From: Felix Fietkau <nbd@nbd.name>
-Date: Wed, 26 Aug 2020 16:55:54 +0200
-Subject: [PATCH] net: ethernet: mtk_eth_soc: fix unnecessary tx queue
- stops
-
-When running short on descriptors, only stop the queue for the netdev that tx
-was attempted for. By the time the something tries to send on the other netdev,
-the ring might have some more room already
-
-Signed-off-by: Felix Fietkau <nbd@nbd.name>
----
-
---- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
-+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
-@@ -1131,17 +1131,6 @@ static void mtk_wake_queue(struct mtk_et
- }
- }
-
--static void mtk_stop_queue(struct mtk_eth *eth)
--{
-- int i;
--
-- for (i = 0; i < MTK_MAC_COUNT; i++) {
-- if (!eth->netdev[i])
-- continue;
-- netif_stop_queue(eth->netdev[i]);
-- }
--}
--
- static netdev_tx_t mtk_start_xmit(struct sk_buff *skb, struct net_device *dev)
- {
- struct mtk_mac *mac = netdev_priv(dev);
-@@ -1162,7 +1151,7 @@ static netdev_tx_t mtk_start_xmit(struct
-
- tx_num = mtk_cal_txd_req(skb);
- if (unlikely(atomic_read(&ring->free_count) <= tx_num)) {
-- mtk_stop_queue(eth);
-+ netif_stop_queue(dev);
- netif_err(eth, tx_queued, dev,
- "Tx Ring full when queue awake!\n");
- spin_unlock(ð->page_lock);
-@@ -1188,7 +1177,7 @@ static netdev_tx_t mtk_start_xmit(struct
- goto drop;
-
- if (unlikely(atomic_read(&ring->free_count) <= ring->thresh))
-- mtk_stop_queue(eth);
-+ netif_stop_queue(dev);
-
- spin_unlock(ð->page_lock);
-
+++ /dev/null
-From: Felix Fietkau <nbd@nbd.name>
-Date: Wed, 26 Aug 2020 16:58:55 +0200
-Subject: [PATCH] net: ethernet: mtk_eth_soc: use larger burst size for
- qdma tx
-
-Improves tx performance
-
-Signed-off-by: Felix Fietkau <nbd@nbd.name>
----
-
---- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
-+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
-@@ -2191,7 +2191,7 @@ static int mtk_start_dma(struct mtk_eth
- if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
- mtk_w32(eth,
- MTK_TX_WB_DDONE | MTK_TX_DMA_EN |
-- MTK_DMA_SIZE_16DWORDS | MTK_NDP_CO_PRO |
-+ MTK_TX_BT_32DWORDS | MTK_NDP_CO_PRO |
- MTK_RX_DMA_EN | MTK_RX_2B_OFFSET |
- MTK_RX_BT_32DWORDS,
- MTK_QDMA_GLO_CFG);
---- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h
-+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
-@@ -202,7 +202,7 @@
- #define MTK_RX_BT_32DWORDS (3 << 11)
- #define MTK_NDP_CO_PRO BIT(10)
- #define MTK_TX_WB_DDONE BIT(6)
--#define MTK_DMA_SIZE_16DWORDS (2 << 4)
-+#define MTK_TX_BT_32DWORDS (3 << 4)
- #define MTK_RX_DMA_BUSY BIT(3)
- #define MTK_TX_DMA_BUSY BIT(1)
- #define MTK_RX_DMA_EN BIT(2)
+++ /dev/null
-From: Felix Fietkau <nbd@nbd.name>
-Date: Wed, 26 Aug 2020 16:59:41 +0200
-Subject: [PATCH] net: ethernet: mtk_eth_soc: increase DMA ring sizes
-
-256 descriptors is not enough for multi-gigabit traffic under load on MT7622.
-Bump it to 512 to improve performance
-
-Signed-off-by: Felix Fietkau <nbd@nbd.name>
----
-
---- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h
-+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
-@@ -21,7 +21,7 @@
- #define MTK_QDMA_PAGE_SIZE 2048
- #define MTK_MAX_RX_LENGTH 1536
- #define MTK_TX_DMA_BUF_LEN 0x3fff
--#define MTK_DMA_SIZE 256
-+#define MTK_DMA_SIZE 512
- #define MTK_NAPI_WEIGHT 64
- #define MTK_MAC_COUNT 2
- #define MTK_RX_ETH_HLEN (VLAN_ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN)
+++ /dev/null
-From: Felix Fietkau <nbd@nbd.name>
-Date: Wed, 26 Aug 2020 17:02:30 +0200
-Subject: [PATCH] net: ethernet: mtk_eth_soc: implement dynamic interrupt
- moderation
-
-Reduces the number of interrupts under load
-
-Signed-off-by: Felix Fietkau <nbd@nbd.name>
----
-
---- a/drivers/net/ethernet/mediatek/Kconfig
-+++ b/drivers/net/ethernet/mediatek/Kconfig
-@@ -10,6 +10,7 @@ if NET_VENDOR_MEDIATEK
- config NET_MEDIATEK_SOC
- tristate "MediaTek SoC Gigabit Ethernet support"
- select PHYLINK
-+ select DIMLIB
- help
- This driver supports the gigabit ethernet MACs in the
- MediaTek SoC family.
---- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
-+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
-@@ -1233,12 +1233,13 @@ static void mtk_update_rx_cpu_idx(struct
- static int mtk_poll_rx(struct napi_struct *napi, int budget,
- struct mtk_eth *eth)
- {
-+ struct dim_sample dim_sample = {};
- struct mtk_rx_ring *ring;
- int idx;
- struct sk_buff *skb;
- u8 *data, *new_data;
- struct mtk_rx_dma *rxd, trxd;
-- int done = 0;
-+ int done = 0, bytes = 0;
-
- while (done < budget) {
- struct net_device *netdev;
-@@ -1311,6 +1312,7 @@ static int mtk_poll_rx(struct napi_struc
- else
- skb_checksum_none_assert(skb);
- skb->protocol = eth_type_trans(skb, netdev);
-+ bytes += pktlen;
-
- if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX &&
- (trxd.rxd2 & RX_DMA_VTAG))
-@@ -1342,6 +1344,12 @@ rx_done:
- mtk_update_rx_cpu_idx(eth);
- }
-
-+ eth->rx_packets += done;
-+ eth->rx_bytes += bytes;
-+ dim_update_sample(eth->rx_events, eth->rx_packets, eth->rx_bytes,
-+ &dim_sample);
-+ net_dim(ð->rx_dim, dim_sample);
-+
- return done;
- }
-
-@@ -1434,6 +1442,7 @@ static int mtk_poll_tx_pdma(struct mtk_e
- static int mtk_poll_tx(struct mtk_eth *eth, int budget)
- {
- struct mtk_tx_ring *ring = ð->tx_ring;
-+ struct dim_sample dim_sample = {};
- unsigned int done[MTK_MAX_DEVS];
- unsigned int bytes[MTK_MAX_DEVS];
- int total = 0, i;
-@@ -1451,8 +1460,14 @@ static int mtk_poll_tx(struct mtk_eth *e
- continue;
- netdev_completed_queue(eth->netdev[i], done[i], bytes[i]);
- total += done[i];
-+ eth->tx_packets += done[i];
-+ eth->tx_bytes += bytes[i];
- }
-
-+ dim_update_sample(eth->tx_events, eth->tx_packets, eth->tx_bytes,
-+ &dim_sample);
-+ net_dim(ð->tx_dim, dim_sample);
-+
- if (mtk_queue_stopped(eth) &&
- (atomic_read(&ring->free_count) > ring->thresh))
- mtk_wake_queue(eth);
-@@ -2127,6 +2142,7 @@ static irqreturn_t mtk_handle_irq_rx(int
- {
- struct mtk_eth *eth = _eth;
-
-+ eth->rx_events++;
- if (likely(napi_schedule_prep(ð->rx_napi))) {
- __napi_schedule(ð->rx_napi);
- mtk_rx_irq_disable(eth, MTK_RX_DONE_INT);
-@@ -2139,6 +2155,7 @@ static irqreturn_t mtk_handle_irq_tx(int
- {
- struct mtk_eth *eth = _eth;
-
-+ eth->tx_events++;
- if (likely(napi_schedule_prep(ð->tx_napi))) {
- __napi_schedule(ð->tx_napi);
- mtk_tx_irq_disable(eth, MTK_TX_DONE_INT);
-@@ -2323,6 +2340,9 @@ static int mtk_stop(struct net_device *d
- napi_disable(ð->tx_napi);
- napi_disable(ð->rx_napi);
-
-+ cancel_work_sync(ð->rx_dim.work);
-+ cancel_work_sync(ð->tx_dim.work);
-+
- if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
- mtk_stop_dma(eth, MTK_QDMA_GLO_CFG);
- mtk_stop_dma(eth, MTK_PDMA_GLO_CFG);
-@@ -2375,6 +2395,64 @@ err_disable_clks:
- return ret;
- }
-
-+static void mtk_dim_rx(struct work_struct *work)
-+{
-+ struct dim *dim = container_of(work, struct dim, work);
-+ struct mtk_eth *eth = container_of(dim, struct mtk_eth, rx_dim);
-+ struct dim_cq_moder cur_profile;
-+ u32 val, cur;
-+
-+ cur_profile = net_dim_get_rx_moderation(eth->rx_dim.mode,
-+ dim->profile_ix);
-+ spin_lock_bh(ð->dim_lock);
-+
-+ val = mtk_r32(eth, MTK_PDMA_DELAY_INT);
-+ val &= MTK_PDMA_DELAY_TX_MASK;
-+ val |= MTK_PDMA_DELAY_RX_EN;
-+
-+ cur = min_t(u32, DIV_ROUND_UP(cur_profile.usec, 20), MTK_PDMA_DELAY_PTIME_MASK);
-+ val |= cur << MTK_PDMA_DELAY_RX_PTIME_SHIFT;
-+
-+ cur = min_t(u32, cur_profile.pkts, MTK_PDMA_DELAY_PINT_MASK);
-+ val |= cur << MTK_PDMA_DELAY_RX_PINT_SHIFT;
-+
-+ mtk_w32(eth, val, MTK_PDMA_DELAY_INT);
-+ mtk_w32(eth, val, MTK_QDMA_DELAY_INT);
-+
-+ spin_unlock_bh(ð->dim_lock);
-+
-+ dim->state = DIM_START_MEASURE;
-+}
-+
-+static void mtk_dim_tx(struct work_struct *work)
-+{
-+ struct dim *dim = container_of(work, struct dim, work);
-+ struct mtk_eth *eth = container_of(dim, struct mtk_eth, tx_dim);
-+ struct dim_cq_moder cur_profile;
-+ u32 val, cur;
-+
-+ cur_profile = net_dim_get_tx_moderation(eth->tx_dim.mode,
-+ dim->profile_ix);
-+ spin_lock_bh(ð->dim_lock);
-+
-+ val = mtk_r32(eth, MTK_PDMA_DELAY_INT);
-+ val &= MTK_PDMA_DELAY_RX_MASK;
-+ val |= MTK_PDMA_DELAY_TX_EN;
-+
-+ cur = min_t(u32, DIV_ROUND_UP(cur_profile.usec, 20), MTK_PDMA_DELAY_PTIME_MASK);
-+ val |= cur << MTK_PDMA_DELAY_TX_PTIME_SHIFT;
-+
-+ cur = min_t(u32, cur_profile.pkts, MTK_PDMA_DELAY_PINT_MASK);
-+ val |= cur << MTK_PDMA_DELAY_TX_PINT_SHIFT;
-+
-+ mtk_w32(eth, val, MTK_PDMA_DELAY_INT);
-+ mtk_w32(eth, val, MTK_QDMA_DELAY_INT);
-+
-+ spin_unlock_bh(ð->dim_lock);
-+
-+ dim->state = DIM_START_MEASURE;
-+}
-+
- static int mtk_hw_init(struct mtk_eth *eth)
- {
- int i, val, ret;
-@@ -2396,9 +2474,6 @@ static int mtk_hw_init(struct mtk_eth *e
- goto err_disable_pm;
- }
-
-- /* enable interrupt delay for RX */
-- mtk_w32(eth, MTK_PDMA_DELAY_RX_DELAY, MTK_PDMA_DELAY_INT);
--
- /* disable delay and normal interrupt */
- mtk_tx_irq_disable(eth, ~0);
- mtk_rx_irq_disable(eth, ~0);
-@@ -2437,11 +2512,10 @@ static int mtk_hw_init(struct mtk_eth *e
- /* Enable RX VLan Offloading */
- mtk_w32(eth, 1, MTK_CDMP_EG_CTRL);
-
-- /* enable interrupt delay for RX */
-- mtk_w32(eth, MTK_PDMA_DELAY_RX_DELAY, MTK_PDMA_DELAY_INT);
-+ mtk_dim_rx(ð->rx_dim.work);
-+ mtk_dim_tx(ð->tx_dim.work);
-
- /* disable delay and normal interrupt */
-- mtk_w32(eth, 0, MTK_QDMA_DELAY_INT);
- mtk_tx_irq_disable(eth, ~0);
- mtk_rx_irq_disable(eth, ~0);
-
-@@ -2946,6 +3020,13 @@ static int mtk_probe(struct platform_dev
- spin_lock_init(ð->page_lock);
- spin_lock_init(ð->tx_irq_lock);
- spin_lock_init(ð->rx_irq_lock);
-+ spin_lock_init(ð->dim_lock);
-+
-+ eth->rx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
-+ INIT_WORK(ð->rx_dim.work, mtk_dim_rx);
-+
-+ eth->tx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
-+ INIT_WORK(ð->tx_dim.work, mtk_dim_tx);
-
- if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) {
- eth->ethsys = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
---- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h
-+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
-@@ -16,6 +16,7 @@
- #include <linux/refcount.h>
- #include <linux/phylink.h>
- #include <linux/rhashtable.h>
-+#include <linux/dim.h>
- #include "mtk_ppe.h"
-
- #define MTK_QDMA_PAGE_SIZE 2048
-@@ -136,13 +137,18 @@
-
- /* PDMA Delay Interrupt Register */
- #define MTK_PDMA_DELAY_INT 0xa0c
-+#define MTK_PDMA_DELAY_RX_MASK GENMASK(15, 0)
- #define MTK_PDMA_DELAY_RX_EN BIT(15)
--#define MTK_PDMA_DELAY_RX_PINT 4
- #define MTK_PDMA_DELAY_RX_PINT_SHIFT 8
--#define MTK_PDMA_DELAY_RX_PTIME 4
--#define MTK_PDMA_DELAY_RX_DELAY \
-- (MTK_PDMA_DELAY_RX_EN | MTK_PDMA_DELAY_RX_PTIME | \
-- (MTK_PDMA_DELAY_RX_PINT << MTK_PDMA_DELAY_RX_PINT_SHIFT))
-+#define MTK_PDMA_DELAY_RX_PTIME_SHIFT 0
-+
-+#define MTK_PDMA_DELAY_TX_MASK GENMASK(31, 16)
-+#define MTK_PDMA_DELAY_TX_EN BIT(31)
-+#define MTK_PDMA_DELAY_TX_PINT_SHIFT 24
-+#define MTK_PDMA_DELAY_TX_PTIME_SHIFT 16
-+
-+#define MTK_PDMA_DELAY_PINT_MASK 0x7f
-+#define MTK_PDMA_DELAY_PTIME_MASK 0xff
-
- /* PDMA Interrupt Status Register */
- #define MTK_PDMA_INT_STATUS 0xa20
-@@ -224,6 +230,7 @@
- /* QDMA Interrupt Status Register */
- #define MTK_QDMA_INT_STATUS 0x1A18
- #define MTK_RX_DONE_DLY BIT(30)
-+#define MTK_TX_DONE_DLY BIT(28)
- #define MTK_RX_DONE_INT3 BIT(19)
- #define MTK_RX_DONE_INT2 BIT(18)
- #define MTK_RX_DONE_INT1 BIT(17)
-@@ -233,8 +240,7 @@
- #define MTK_TX_DONE_INT1 BIT(1)
- #define MTK_TX_DONE_INT0 BIT(0)
- #define MTK_RX_DONE_INT MTK_RX_DONE_DLY
--#define MTK_TX_DONE_INT (MTK_TX_DONE_INT0 | MTK_TX_DONE_INT1 | \
-- MTK_TX_DONE_INT2 | MTK_TX_DONE_INT3)
-+#define MTK_TX_DONE_INT MTK_TX_DONE_DLY
-
- /* QDMA Interrupt grouping registers */
- #define MTK_QDMA_INT_GRP1 0x1a20
-@@ -905,6 +911,18 @@ struct mtk_eth {
-
- const struct mtk_soc_data *soc;
-
-+ spinlock_t dim_lock;
-+
-+ u32 rx_events;
-+ u32 rx_packets;
-+ u32 rx_bytes;
-+ struct dim rx_dim;
-+
-+ u32 tx_events;
-+ u32 tx_packets;
-+ u32 tx_bytes;
-+ struct dim tx_dim;
-+
- u32 tx_int_mask_reg;
- u32 tx_int_status_reg;
- u32 rx_dma_l4_valid;
+++ /dev/null
-From: Felix Fietkau <nbd@nbd.name>
-Date: Thu, 27 Aug 2020 06:32:03 +0200
-Subject: [PATCH] net: ethernet: mtk_eth_soc: cache hardware pointer of last
- freed tx descriptor
-
-The value is only updated by the CPU, so it is cheaper to access from the ring
-data structure than from a hardware register
-
-Signed-off-by: Felix Fietkau <nbd@nbd.name>
----
-
---- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
-+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
-@@ -1362,7 +1362,7 @@ static int mtk_poll_tx_qdma(struct mtk_e
- struct mtk_tx_buf *tx_buf;
- u32 cpu, dma;
-
-- cpu = mtk_r32(eth, MTK_QTX_CRX_PTR);
-+ cpu = ring->last_free_ptr;
- dma = mtk_r32(eth, MTK_QTX_DRX_PTR);
-
- desc = mtk_qdma_phys_to_virt(ring, cpu);
-@@ -1396,6 +1396,7 @@ static int mtk_poll_tx_qdma(struct mtk_e
- cpu = next_cpu;
- }
-
-+ ring->last_free_ptr = cpu;
- mtk_w32(eth, cpu, MTK_QTX_CRX_PTR);
-
- return budget;
-@@ -1596,6 +1597,7 @@ static int mtk_tx_alloc(struct mtk_eth *
- atomic_set(&ring->free_count, MTK_DMA_SIZE - 2);
- ring->next_free = &ring->dma[0];
- ring->last_free = &ring->dma[MTK_DMA_SIZE - 1];
-+ ring->last_free_ptr = (u32)(ring->phys + ((MTK_DMA_SIZE - 1) * sz));
- ring->thresh = MAX_SKB_FRAGS;
-
- /* make sure that all changes to the dma ring are flushed before we
-@@ -1609,9 +1611,7 @@ static int mtk_tx_alloc(struct mtk_eth *
- mtk_w32(eth,
- ring->phys + ((MTK_DMA_SIZE - 1) * sz),
- MTK_QTX_CRX_PTR);
-- mtk_w32(eth,
-- ring->phys + ((MTK_DMA_SIZE - 1) * sz),
-- MTK_QTX_DRX_PTR);
-+ mtk_w32(eth, ring->last_free_ptr, MTK_QTX_DRX_PTR);
- mtk_w32(eth, (QDMA_RES_THRES << 8) | QDMA_RES_THRES,
- MTK_QTX_CFG(0));
- } else {
---- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h
-+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
-@@ -636,6 +636,7 @@ struct mtk_tx_buf {
- * @phys: The physical addr of tx_buf
- * @next_free: Pointer to the next free descriptor
- * @last_free: Pointer to the last free descriptor
-+ * @last_free_ptr: Hardware pointer value of the last free descriptor
- * @thresh: The threshold of minimum amount of free descriptors
- * @free_count: QDMA uses a linked list. Track how many free descriptors
- * are present
-@@ -646,6 +647,7 @@ struct mtk_tx_ring {
- dma_addr_t phys;
- struct mtk_tx_dma *next_free;
- struct mtk_tx_dma *last_free;
-+ u32 last_free_ptr;
- u16 thresh;
- atomic_t free_count;
- int dma_size;
+++ /dev/null
-From: Felix Fietkau <nbd@nbd.name>
-Date: Thu, 27 Aug 2020 09:24:25 +0200
-Subject: [PATCH] net: ethernet: mtk_eth_soc: only read the full rx
- descriptor if DMA is done
-
-Uncached memory access is expensive, and there is no need to access all
-descriptor words if we can't process them anyway
-
-Signed-off-by: Felix Fietkau <nbd@nbd.name>
----
-
---- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
-+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
-@@ -777,13 +777,18 @@ static inline int mtk_max_buf_size(int f
- return buf_size;
- }
-
--static inline void mtk_rx_get_desc(struct mtk_rx_dma *rxd,
-+static inline bool mtk_rx_get_desc(struct mtk_rx_dma *rxd,
- struct mtk_rx_dma *dma_rxd)
- {
-- rxd->rxd1 = READ_ONCE(dma_rxd->rxd1);
- rxd->rxd2 = READ_ONCE(dma_rxd->rxd2);
-+ if (!(rxd->rxd2 & RX_DMA_DONE))
-+ return false;
-+
-+ rxd->rxd1 = READ_ONCE(dma_rxd->rxd1);
- rxd->rxd3 = READ_ONCE(dma_rxd->rxd3);
- rxd->rxd4 = READ_ONCE(dma_rxd->rxd4);
-+
-+ return true;
- }
-
- /* the qdma core needs scratch memory to be setup */
-@@ -1255,8 +1260,7 @@ static int mtk_poll_rx(struct napi_struc
- rxd = &ring->dma[idx];
- data = ring->data[idx];
-
-- mtk_rx_get_desc(&trxd, rxd);
-- if (!(trxd.rxd2 & RX_DMA_DONE))
-+ if (!mtk_rx_get_desc(&trxd, rxd))
- break;
-
- /* find out which mac the packet come from. values start at 1 */
+++ /dev/null
-From: Felix Fietkau <nbd@nbd.name>
-Date: Thu, 27 Aug 2020 09:44:43 +0200
-Subject: [PATCH] net: ethernet: mtk_eth_soc: unmap rx data before calling
- build_skb
-
-Since build_skb accesses the data area (for initializing shinfo), dma unmap
-needs to happen before that call
-
-Signed-off-by: Felix Fietkau <nbd@nbd.name>
----
-
---- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
-+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
-@@ -1297,17 +1297,18 @@ static int mtk_poll_rx(struct napi_struc
- goto release_desc;
- }
-
-+ dma_unmap_single(eth->dev, trxd.rxd1,
-+ ring->buf_size, DMA_FROM_DEVICE);
-+
- /* receive data */
- skb = build_skb(data, ring->frag_size);
- if (unlikely(!skb)) {
-- skb_free_frag(new_data);
-+ skb_free_frag(data);
- netdev->stats.rx_dropped++;
-- goto release_desc;
-+ goto skip_rx;
- }
- skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
-
-- dma_unmap_single(eth->dev, trxd.rxd1,
-- ring->buf_size, DMA_FROM_DEVICE);
- pktlen = RX_DMA_GET_PLEN0(trxd.rxd2);
- skb->dev = netdev;
- skb_put(skb, pktlen);
-@@ -1325,6 +1326,7 @@ static int mtk_poll_rx(struct napi_struc
- skb_record_rx_queue(skb, 0);
- napi_gro_receive(napi, skb);
-
-+skip_rx:
- ring->data[idx] = new_data;
- rxd->rxd1 = (unsigned int)dma_addr;
-
+++ /dev/null
-From: Felix Fietkau <nbd@nbd.name>
-Date: Fri, 4 Sep 2020 18:14:05 +0200
-Subject: [PATCH] net: ethernet: mtk_eth_soc: avoid rearming interrupt if
- napi_complete returns false
-
-Reduces unnecessary interrupts
-
-Signed-off-by: Felix Fietkau <nbd@nbd.name>
----
-
---- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
-+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
-@@ -1519,8 +1519,8 @@ static int mtk_napi_tx(struct napi_struc
- if (status & MTK_TX_DONE_INT)
- return budget;
-
-- napi_complete(napi);
-- mtk_tx_irq_enable(eth, MTK_TX_DONE_INT);
-+ if (napi_complete(napi))
-+ mtk_tx_irq_enable(eth, MTK_TX_DONE_INT);
-
- return tx_done;
- }
-@@ -1553,8 +1553,9 @@ poll_again:
- remain_budget -= rx_done;
- goto poll_again;
- }
-- napi_complete(napi);
-- mtk_rx_irq_enable(eth, MTK_RX_DONE_INT);
-+
-+ if (napi_complete(napi))
-+ mtk_rx_irq_enable(eth, MTK_RX_DONE_INT);
-
- return rx_done + budget - remain_budget;
- }
+++ /dev/null
-From: Felix Fietkau <nbd@nbd.name>
-Date: Sun, 13 Sep 2020 08:27:24 +0200
-Subject: [PATCH] net: ethernet: mtk_eth_soc: set PPE flow hash as skb hash
- if present
-
-This improves GRO performance
-
-Signed-off-by: Felix Fietkau <nbd@nbd.name>
----
-
---- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
-+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
-@@ -19,6 +19,7 @@
- #include <linux/interrupt.h>
- #include <linux/pinctrl/devinfo.h>
- #include <linux/phylink.h>
-+#include <linux/jhash.h>
- #include <net/dsa.h>
-
- #include "mtk_eth_soc.h"
-@@ -1250,6 +1251,7 @@ static int mtk_poll_rx(struct napi_struc
- struct net_device *netdev;
- unsigned int pktlen;
- dma_addr_t dma_addr;
-+ u32 hash;
- int mac;
-
- ring = mtk_get_rx_ring(eth);
-@@ -1319,6 +1321,12 @@ static int mtk_poll_rx(struct napi_struc
- skb->protocol = eth_type_trans(skb, netdev);
- bytes += pktlen;
-
-+ hash = trxd.rxd4 & GENMASK(13, 0);
-+ if (hash != GENMASK(13, 0)) {
-+ hash = jhash_1word(hash, 0);
-+ skb_set_hash(skb, hash, PKT_HASH_TYPE_L4);
-+ }
-+
- if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX &&
- (trxd.rxd2 & RX_DMA_VTAG))
- __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
#include <linux/mfd/syscon.h>
#include <linux/regmap.h>
#include <linux/clk.h>
-@@ -2482,6 +2483,13 @@ static int mtk_hw_init(struct mtk_eth *e
+@@ -2473,6 +2474,13 @@ static int mtk_hw_init(struct mtk_eth *e
if (ret)
goto err_disable_pm;
if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) {
ret = device_reset(eth->dev);
if (ret) {
-@@ -3061,6 +3069,16 @@ static int mtk_probe(struct platform_dev
+@@ -3053,6 +3061,16 @@ static int mtk_probe(struct platform_dev
}
}
--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
-@@ -2901,6 +2901,7 @@ static const struct net_device_ops mtk_n
+@@ -2893,6 +2893,7 @@ static const struct net_device_ops mtk_n
static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np)
{
const __be32 *_id = of_get_property(np, "reg", NULL);
phy_interface_t phy_mode;
struct phylink *phylink;
-@@ -2993,6 +2994,9 @@ static int mtk_add_mac(struct mtk_eth *e
+@@ -2985,6 +2986,9 @@ static int mtk_add_mac(struct mtk_eth *e
eth->netdev[id]->max_mtu = MTK_MAX_RX_LENGTH - MTK_RX_ETH_HLEN;
projects / openwrt / openwrt.git / commitdiff