1 From 585ddca487c827178cf697f8bc2e87346061d155 Mon Sep 17 00:00:00 2001
2 From: Tim Harvey <tharvey@gateworks.com>
3 Date: Wed, 19 Feb 2020 15:19:36 -0800
4 Subject: [PATCH 02/12] net: thunderx: workaround BGX TX Underflow issue
6 While it is not yet understood why a TX underflow can easily occur
7 for SGMII interfaces resulting in a TX wedge. It has been found that
8 disabling/re-enabling the LMAC resolves the issue.
10 Signed-off-by: Tim Harvey <tharvey@gateworks.com>
11 Reviewed-by: Robert Jones <rjones@gateworks.com>
12 Signed-off-by: David S. Miller <davem@davemloft.net>
14 drivers/net/ethernet/cavium/thunder/thunder_bgx.c | 62 +++++++++++++++++++++--
15 drivers/net/ethernet/cavium/thunder/thunder_bgx.h | 9 ++++
16 2 files changed, 68 insertions(+), 3 deletions(-)
18 diff --git a/drivers/net/ethernet/cavium/thunder/thunder_bgx.c b/drivers/net/ethernet/cavium/thunder/thunder_bgx.c
19 index 5f2db9c..ade414a 100644
20 --- a/drivers/net/ethernet/cavium/thunder/thunder_bgx.c
21 +++ b/drivers/net/ethernet/cavium/thunder/thunder_bgx.c
22 @@ -413,10 +413,19 @@ void bgx_lmac_rx_tx_enable(int node, int bgx_idx, int lmacid, bool enable)
23 lmac = &bgx->lmac[lmacid];
25 cfg = bgx_reg_read(bgx, lmacid, BGX_CMRX_CFG);
28 cfg |= CMR_PKT_RX_EN | CMR_PKT_TX_EN;
31 + /* enable TX FIFO Underflow interrupt */
32 + bgx_reg_modify(bgx, lmacid, BGX_GMP_GMI_TXX_INT_ENA_W1S,
33 + GMI_TXX_INT_UNDFLW);
35 cfg &= ~(CMR_PKT_RX_EN | CMR_PKT_TX_EN);
37 + /* Disable TX FIFO Underflow interrupt */
38 + bgx_reg_modify(bgx, lmacid, BGX_GMP_GMI_TXX_INT_ENA_W1C,
39 + GMI_TXX_INT_UNDFLW);
41 bgx_reg_write(bgx, lmacid, BGX_CMRX_CFG, cfg);
44 @@ -1544,6 +1553,48 @@ static int bgx_init_phy(struct bgx *bgx)
45 return bgx_init_of_phy(bgx);
48 +static irqreturn_t bgx_intr_handler(int irq, void *data)
50 + struct bgx *bgx = (struct bgx *)data;
54 + for (lmac = 0; lmac < bgx->lmac_count; lmac++) {
55 + status = bgx_reg_read(bgx, lmac, BGX_GMP_GMI_TXX_INT);
56 + if (status & GMI_TXX_INT_UNDFLW) {
57 + pci_err(bgx->pdev, "BGX%d lmac%d UNDFLW\n",
59 + val = bgx_reg_read(bgx, lmac, BGX_CMRX_CFG);
61 + bgx_reg_write(bgx, lmac, BGX_CMRX_CFG, val);
63 + bgx_reg_write(bgx, lmac, BGX_CMRX_CFG, val);
65 + /* clear interrupts */
66 + bgx_reg_write(bgx, lmac, BGX_GMP_GMI_TXX_INT, status);
72 +static void bgx_register_intr(struct pci_dev *pdev)
74 + struct bgx *bgx = pci_get_drvdata(pdev);
77 + ret = pci_alloc_irq_vectors(pdev, BGX_LMAC_VEC_OFFSET,
78 + BGX_LMAC_VEC_OFFSET, PCI_IRQ_ALL_TYPES);
80 + pci_err(pdev, "Req for #%d msix vectors failed\n",
81 + BGX_LMAC_VEC_OFFSET);
84 + ret = pci_request_irq(pdev, GMPX_GMI_TX_INT, bgx_intr_handler, NULL,
85 + bgx, "BGX%d", bgx->bgx_id);
87 + pci_free_irq(pdev, GMPX_GMI_TX_INT, bgx);
90 static int bgx_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
93 @@ -1559,7 +1610,7 @@ static int bgx_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
95 pci_set_drvdata(pdev, bgx);
97 - err = pci_enable_device(pdev);
98 + err = pcim_enable_device(pdev);
100 dev_err(dev, "Failed to enable PCI device\n");
101 pci_set_drvdata(pdev, NULL);
102 @@ -1613,6 +1664,8 @@ static int bgx_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
106 + bgx_register_intr(pdev);
108 /* Enable all LMACs */
109 for (lmac = 0; lmac < bgx->lmac_count; lmac++) {
110 err = bgx_lmac_enable(bgx, lmac);
111 @@ -1629,6 +1682,7 @@ static int bgx_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
114 bgx_vnic[bgx->bgx_id] = NULL;
115 + pci_free_irq(pdev, GMPX_GMI_TX_INT, bgx);
117 pci_release_regions(pdev);
119 @@ -1646,6 +1700,8 @@ static void bgx_remove(struct pci_dev *pdev)
120 for (lmac = 0; lmac < bgx->lmac_count; lmac++)
121 bgx_lmac_disable(bgx, lmac);
123 + pci_free_irq(pdev, GMPX_GMI_TX_INT, bgx);
125 bgx_vnic[bgx->bgx_id] = NULL;
126 pci_release_regions(pdev);
127 pci_disable_device(pdev);
128 diff --git a/drivers/net/ethernet/cavium/thunder/thunder_bgx.h b/drivers/net/ethernet/cavium/thunder/thunder_bgx.h
129 index cbdd20b..ac0c89c 100644
130 --- a/drivers/net/ethernet/cavium/thunder/thunder_bgx.h
131 +++ b/drivers/net/ethernet/cavium/thunder/thunder_bgx.h
133 #define BGX_GMP_GMI_TXX_BURST 0x38228
134 #define BGX_GMP_GMI_TXX_MIN_PKT 0x38240
135 #define BGX_GMP_GMI_TXX_SGMII_CTL 0x38300
136 +#define BGX_GMP_GMI_TXX_INT 0x38500
137 +#define BGX_GMP_GMI_TXX_INT_W1S 0x38508
138 +#define BGX_GMP_GMI_TXX_INT_ENA_W1C 0x38510
139 +#define BGX_GMP_GMI_TXX_INT_ENA_W1S 0x38518
140 +#define GMI_TXX_INT_PTP_LOST BIT_ULL(4)
141 +#define GMI_TXX_INT_LATE_COL BIT_ULL(3)
142 +#define GMI_TXX_INT_XSDEF BIT_ULL(2)
143 +#define GMI_TXX_INT_XSCOL BIT_ULL(1)
144 +#define GMI_TXX_INT_UNDFLW BIT_ULL(0)
146 #define BGX_MSIX_VEC_0_29_ADDR 0x400000 /* +(0..29) << 4 */
147 #define BGX_MSIX_VEC_0_29_CTL 0x400008