1 From 0f62a794b41ad14a962c70445844d61a8097805e Mon Sep 17 00:00:00 2001
2 From: Alex Marginean <alexandru.marginean@nxp.com>
3 Date: Tue, 20 Aug 2019 12:34:22 +0300
4 Subject: [PATCH] enetc: WA for MDIO register access issue
6 Due to a hardware issue access to MDIO registers concurrent with other
7 ENETC register access may lead to the MDIO access being dropped or
8 corrupted. The workaround introduces locking for all register access in
9 ENETC space. To reduce performance impact, code except MDIO uses per-cpu
10 locks, MDIO code having to acquire all per-CPU locks to perform an access.
11 To further reduce the performance impact, datapath functions acquire the
12 per-cpu lock fewer times and use _hot accessors. All the rest of the code
13 uses the _wa accessors which lock every time a register is accessed.
15 Signed-off-by: Alex Marginean <alexandru.marginean@nxp.com>
17 drivers/net/ethernet/freescale/enetc/enetc.c | 85 ++++++++++++++----
18 drivers/net/ethernet/freescale/enetc/enetc_hw.h | 105 +++++++++++++++++++++-
19 drivers/net/ethernet/freescale/enetc/enetc_mdio.c | 4 +-
20 drivers/net/ethernet/freescale/enetc/enetc_pf.c | 3 +
21 4 files changed, 176 insertions(+), 21 deletions(-)
23 --- a/drivers/net/ethernet/freescale/enetc/enetc.c
24 +++ b/drivers/net/ethernet/freescale/enetc/enetc.c
25 @@ -20,8 +20,13 @@ netdev_tx_t enetc_xmit(struct sk_buff *s
27 struct enetc_ndev_priv *priv = netdev_priv(ndev);
28 struct enetc_bdr *tx_ring;
29 + unsigned long flags;
30 + /* pointer to per-cpu ENETC lock for register access issue WA */
34 + lock = this_cpu_ptr(&enetc_gregs);
36 tx_ring = priv->tx_ring[skb->queue_mapping];
38 if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS))
39 @@ -34,7 +39,12 @@ netdev_tx_t enetc_xmit(struct sk_buff *s
40 return NETDEV_TX_BUSY;
43 + spin_lock_irqsave(lock, flags);
45 count = enetc_map_tx_buffs(tx_ring, skb, priv->active_offloads);
47 + spin_unlock_irqrestore(lock, flags);
52 @@ -228,7 +238,7 @@ static int enetc_map_tx_buffs(struct ene
53 tx_ring->next_to_use = i;
55 /* let H/W know BD ring has been updated */
56 - enetc_wr_reg(tx_ring->tpir, i); /* includes wmb() */
57 + enetc_wr_reg_hot(tx_ring->tpir, i); /* includes wmb() */
61 @@ -249,13 +259,21 @@ dma_err:
62 static irqreturn_t enetc_msix(int irq, void *data)
64 struct enetc_int_vector *v = data;
65 + unsigned long flags;
66 + /* pointer to per-cpu ENETC lock for register access issue WA */
70 + lock = this_cpu_ptr(&enetc_gregs);
71 + spin_lock_irqsave(lock, flags);
73 /* disable interrupts */
74 - enetc_wr_reg(v->rbier, 0);
75 + enetc_wr_reg_hot(v->rbier, 0);
77 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
78 - enetc_wr_reg(v->tbier_base + ENETC_BDR_OFF(i), 0);
79 + enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0);
81 + spin_unlock_irqrestore(lock, flags);
83 napi_schedule_irqoff(&v->napi);
85 @@ -271,6 +289,9 @@ static int enetc_poll(struct napi_struct
86 struct enetc_int_vector
87 *v = container_of(napi, struct enetc_int_vector, napi);
89 + unsigned long flags;
90 + /* pointer to per-cpu ENETC lock for register access issue WA */
95 @@ -287,19 +308,24 @@ static int enetc_poll(struct napi_struct
97 napi_complete_done(napi, work_done);
99 + lock = this_cpu_ptr(&enetc_gregs);
100 + spin_lock_irqsave(lock, flags);
102 /* enable interrupts */
103 - enetc_wr_reg(v->rbier, ENETC_RBIER_RXTIE);
104 + enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE);
106 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
107 - enetc_wr_reg(v->tbier_base + ENETC_BDR_OFF(i),
108 - ENETC_TBIER_TXTIE);
109 + enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i),
110 + ENETC_TBIER_TXTIE);
112 + spin_unlock_irqrestore(lock, flags);
117 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)
119 - int pi = enetc_rd_reg(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;
120 + int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;
122 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi;
124 @@ -337,9 +363,18 @@ static bool enetc_clean_tx_ring(struct e
128 + unsigned long flags;
129 + /* pointer to per-cpu ENETC lock for register access issue WA */
132 + lock = this_cpu_ptr(&enetc_gregs);
134 i = tx_ring->next_to_clean;
135 tx_swbd = &tx_ring->tx_swbd[i];
137 + spin_lock_irqsave(lock, flags);
138 bds_to_clean = enetc_bd_ready_count(tx_ring, i);
139 + spin_unlock_irqrestore(lock, flags);
143 @@ -382,16 +417,20 @@ static bool enetc_clean_tx_ring(struct e
144 tx_swbd = tx_ring->tx_swbd;
147 + spin_lock_irqsave(lock, flags);
149 /* BD iteration loop end */
152 /* re-arm interrupt source */
153 - enetc_wr_reg(tx_ring->idr, BIT(tx_ring->index) |
154 - BIT(16 + tx_ring->index));
155 + enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) |
156 + BIT(16 + tx_ring->index));
159 if (unlikely(!bds_to_clean))
160 bds_to_clean = enetc_bd_ready_count(tx_ring, i);
162 + spin_unlock_irqrestore(lock, flags);
165 tx_ring->next_to_clean = i;
166 @@ -470,13 +509,14 @@ static int enetc_refill_rx_ring(struct e
167 rx_ring->next_to_alloc = i; /* keep track from page reuse */
168 rx_ring->next_to_use = i;
169 /* update ENETC's consumer index */
170 - enetc_wr_reg(rx_ring->rcir, i);
171 + enetc_wr_reg_hot(rx_ring->rcir, i);
177 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING
178 +/* Must be called with &enetc_gregs spinlock held */
179 static void enetc_get_rx_tstamp(struct net_device *ndev,
180 union enetc_rx_bd *rxbd,
182 @@ -488,8 +528,8 @@ static void enetc_get_rx_tstamp(struct n
185 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) {
186 - lo = enetc_rd(hw, ENETC_SICTR0);
187 - hi = enetc_rd(hw, ENETC_SICTR1);
188 + lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0);
189 + hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1);
190 tstamp_lo = le32_to_cpu(rxbd->r.tstamp);
193 @@ -627,6 +667,12 @@ static int enetc_clean_rx_ring(struct en
194 int rx_frm_cnt = 0, rx_byte_cnt = 0;
197 + unsigned long flags;
198 + /* pointer to per-cpu ENETC lock for register access issue WA */
201 + lock = this_cpu_ptr(&enetc_gregs);
203 cleaned_cnt = enetc_bd_unused(rx_ring);
204 /* next descriptor to process */
205 i = rx_ring->next_to_clean;
206 @@ -637,6 +683,8 @@ static int enetc_clean_rx_ring(struct en
210 + spin_lock_irqsave(lock, flags);
212 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) {
213 int count = enetc_refill_rx_ring(rx_ring, cleaned_cnt);
215 @@ -645,15 +693,19 @@ static int enetc_clean_rx_ring(struct en
217 rxbd = ENETC_RXBD(*rx_ring, i);
218 bd_status = le32_to_cpu(rxbd->r.lstatus);
221 + spin_unlock_irqrestore(lock, flags);
225 - enetc_wr_reg(rx_ring->idr, BIT(rx_ring->index));
226 + enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
227 dma_rmb(); /* for reading other rxbd fields */
228 size = le16_to_cpu(rxbd->r.buf_len);
229 skb = enetc_map_rx_buff_to_skb(rx_ring, i, size);
232 + spin_unlock_irqrestore(lock, flags);
236 enetc_get_offloads(rx_ring, rxbd, skb);
238 @@ -667,6 +719,7 @@ static int enetc_clean_rx_ring(struct en
240 if (unlikely(bd_status &
241 ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))) {
242 + spin_unlock_irqrestore(lock, flags);
244 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
246 @@ -710,6 +763,8 @@ static int enetc_clean_rx_ring(struct en
248 enetc_process_skb(rx_ring, skb);
250 + spin_unlock_irqrestore(lock, flags);
252 napi_gro_receive(napi, skb);
255 --- a/drivers/net/ethernet/freescale/enetc/enetc_hw.h
256 +++ b/drivers/net/ethernet/freescale/enetc/enetc_hw.h
257 @@ -327,8 +327,15 @@ struct enetc_hw {
260 /* general register accessors */
261 -#define enetc_rd_reg(reg) ioread32((reg))
262 -#define enetc_wr_reg(reg, val) iowrite32((val), (reg))
263 +#define enetc_rd_reg(reg) enetc_rd_reg_wa((reg))
264 +#define enetc_wr_reg(reg, val) enetc_wr_reg_wa((reg), (val))
266 +/* accessors for data-path, due to MDIO issue on LS1028 these should be called
267 + * only under enetc_gregs per-cpu lock
269 +#define enetc_rd_reg_hot(reg) ioread32((reg))
270 +#define enetc_wr_reg_hot(reg, val) iowrite32((val), (reg))
273 #define enetc_rd_reg64(reg) ioread64((reg))
275 @@ -347,12 +354,102 @@ static inline u64 enetc_rd_reg64(void __
279 +extern DEFINE_PER_CPU(spinlock_t, enetc_gregs);
281 +static inline u32 enetc_rd_reg_wa(void *reg)
283 + unsigned long flags;
284 + /* pointer to per-cpu ENETC lock for register access issue WA */
288 + lock = this_cpu_ptr(&enetc_gregs);
289 + spin_lock_irqsave(lock, flags);
290 + val = ioread32(reg);
291 + spin_unlock_irqrestore(lock, flags);
296 +static inline void enetc_wr_reg_wa(void *reg, u32 val)
298 + unsigned long flags;
299 + /* pointer to per-cpu ENETC lock for register access issue WA */
302 + lock = this_cpu_ptr(&enetc_gregs);
303 + spin_lock_irqsave(lock, flags);
304 + iowrite32(val, reg);
305 + spin_unlock_irqrestore(lock, flags);
308 +/* NR_CPUS=256 in ARM64 defconfig and using it as array size triggers stack
309 + * frame warnings for the functions below. Use a custom define of 2 for now,
310 + * LS1028 has just two cores.
312 +#define ENETC_NR_CPU_LOCKS 2
314 +static inline u32 enetc_rd_reg_wa_single(void *reg)
318 + /* per-cpu ENETC lock array for register access issue WA */
319 + spinlock_t *lock[ENETC_NR_CPU_LOCKS];
320 + unsigned long flags;
322 + local_irq_save(flags);
325 + for_each_online_cpu(cpu) {
326 + lock[cpu] = per_cpu_ptr(&enetc_gregs, cpu);
327 + spin_lock(lock[cpu]);
330 + val = ioread32(reg);
332 + for_each_online_cpu(cpu)
333 + spin_unlock(lock[cpu]);
334 + local_irq_restore(flags);
341 +static inline void enetc_wr_reg_wa_single(void *reg, u32 val)
344 + /* per-cpu ENETC lock array for register access issue WA */
345 + spinlock_t *lock[ENETC_NR_CPU_LOCKS];
346 + unsigned long flags;
348 + local_irq_save(flags);
351 + for_each_online_cpu(cpu) {
352 + lock[cpu] = per_cpu_ptr(&enetc_gregs, cpu);
353 + spin_lock(lock[cpu]);
356 + iowrite32(val, reg);
358 + for_each_online_cpu(cpu)
359 + spin_unlock(lock[cpu]);
360 + local_irq_restore(flags);
365 #define enetc_rd(hw, off) enetc_rd_reg((hw)->reg + (off))
366 #define enetc_wr(hw, off, val) enetc_wr_reg((hw)->reg + (off), val)
367 #define enetc_rd64(hw, off) enetc_rd_reg64((hw)->reg + (off))
368 /* port register accessors - PF only */
369 -#define enetc_port_rd(hw, off) enetc_rd_reg((hw)->port + (off))
370 -#define enetc_port_wr(hw, off, val) enetc_wr_reg((hw)->port + (off), val)
371 +#define enetc_port_rd(hw, off) enetc_rd_reg_wa((hw)->port + (off))
372 +#define enetc_port_wr(hw, off, val) enetc_wr_reg_wa((hw)->port + (off), val)
373 +#define enetc_port_rd_single(hw, off) enetc_rd_reg_wa_single(\
374 + (hw)->port + (off))
375 +#define enetc_port_wr_single(hw, off, val) enetc_wr_reg_wa_single(\
376 + (hw)->port + (off), val)
377 /* global register accessors - PF only */
378 #define enetc_global_rd(hw, off) enetc_rd_reg((hw)->global + (off))
379 #define enetc_global_wr(hw, off, val) enetc_wr_reg((hw)->global + (off), val)
380 --- a/drivers/net/ethernet/freescale/enetc/enetc_mdio.c
381 +++ b/drivers/net/ethernet/freescale/enetc/enetc_mdio.c
384 static inline u32 _enetc_mdio_rd(struct enetc_mdio_priv *mdio_priv, int off)
386 - return enetc_port_rd(mdio_priv->hw, mdio_priv->mdio_base + off);
387 + return enetc_port_rd_single(mdio_priv->hw, mdio_priv->mdio_base + off);
390 static inline void _enetc_mdio_wr(struct enetc_mdio_priv *mdio_priv, int off,
393 - enetc_port_wr(mdio_priv->hw, mdio_priv->mdio_base + off, val);
394 + enetc_port_wr_single(mdio_priv->hw, mdio_priv->mdio_base + off, val);
397 #define enetc_mdio_rd(mdio_priv, off) \
398 --- a/drivers/net/ethernet/freescale/enetc/enetc_pf.c
399 +++ b/drivers/net/ethernet/freescale/enetc/enetc_pf.c
400 @@ -1076,6 +1076,9 @@ static void enetc_pf_remove(struct pci_d
401 enetc_pci_remove(pdev);
404 +DEFINE_PER_CPU(spinlock_t, enetc_gregs);
405 +EXPORT_PER_CPU_SYMBOL(enetc_gregs);
407 static const struct pci_device_id enetc_pf_id_table[] = {
408 { PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, ENETC_DEV_ID_PF) },
409 { 0, } /* End of table. */