target/linux/layerscape/patches-5.4/701-net-0234-enetc-WA-for-MDIO-register-access-issue.patch

   1 From 0f62a794b41ad14a962c70445844d61a8097805e Mon Sep 17 00:00:00 2001
   2 From: Alex Marginean <alexandru.marginean@nxp.com>
   3 Date: Tue, 20 Aug 2019 12:34:22 +0300
   4 Subject: [PATCH] enetc: WA for MDIO register access issue
   5
   6 Due to a hardware issue access to MDIO registers concurrent with other
   7 ENETC register access may lead to the MDIO access being dropped or
   8 corrupted.  The workaround introduces locking for all register access in
   9 ENETC space.  To reduce performance impact, code except MDIO uses per-cpu
  10 locks, MDIO code having to acquire all per-CPU locks to perform an access.
  11 To further reduce the performance impact, datapath functions acquire the
  12 per-cpu lock fewer times and use _hot accessors.  All the rest of the code
  13 uses the _wa accessors which lock every time a register is accessed.
  14
  15 Signed-off-by: Alex Marginean <alexandru.marginean@nxp.com>
  16 ---
  17  drivers/net/ethernet/freescale/enetc/enetc.c      |  85 ++++++++++++++----
  18  drivers/net/ethernet/freescale/enetc/enetc_hw.h   | 105 +++++++++++++++++++++-
  19  drivers/net/ethernet/freescale/enetc/enetc_mdio.c |   4 +-
  20  drivers/net/ethernet/freescale/enetc/enetc_pf.c   |   3 +
  21  4 files changed, 176 insertions(+), 21 deletions(-)
  22
  23 --- a/drivers/net/ethernet/freescale/enetc/enetc.c
  24 +++ b/drivers/net/ethernet/freescale/enetc/enetc.c
  25 @@ -20,8 +20,13 @@ netdev_tx_t enetc_xmit(struct sk_buff *s
  26  {
  27         struct enetc_ndev_priv *priv = netdev_priv(ndev);
  28         struct enetc_bdr *tx_ring;
  29 +       unsigned long flags;
  30 +       /* pointer to per-cpu ENETC lock for register access issue WA */
  31 +       spinlock_t *lock;
  32         int count;
  33
  34 +       lock = this_cpu_ptr(&enetc_gregs);
  35 +
  36         tx_ring = priv->tx_ring[skb->queue_mapping];
  37
  38         if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS))
  39 @@ -34,7 +39,12 @@ netdev_tx_t enetc_xmit(struct sk_buff *s
  40                 return NETDEV_TX_BUSY;
  41         }
  42
  43 +       spin_lock_irqsave(lock, flags);
  44 +
  45         count = enetc_map_tx_buffs(tx_ring, skb, priv->active_offloads);
  46 +
  47 +       spin_unlock_irqrestore(lock, flags);
  48 +
  49         if (unlikely(!count))
  50                 goto drop_packet_err;
  51
  52 @@ -228,7 +238,7 @@ static int enetc_map_tx_buffs(struct ene
  53         tx_ring->next_to_use = i;
  54
  55         /* let H/W know BD ring has been updated */
  56 -       enetc_wr_reg(tx_ring->tpir, i); /* includes wmb() */
  57 +       enetc_wr_reg_hot(tx_ring->tpir, i); /* includes wmb() */
  58
  59         return count;
  60
  61 @@ -249,13 +259,21 @@ dma_err:
  62  static irqreturn_t enetc_msix(int irq, void *data)
  63  {
  64         struct enetc_int_vector *v = data;
  65 +       unsigned long flags;
  66 +       /* pointer to per-cpu ENETC lock for register access issue WA */
  67 +       spinlock_t *lock;
  68         int i;
  69
  70 +       lock = this_cpu_ptr(&enetc_gregs);
  71 +       spin_lock_irqsave(lock, flags);
  72 +
  73         /* disable interrupts */
  74 -       enetc_wr_reg(v->rbier, 0);
  75 +       enetc_wr_reg_hot(v->rbier, 0);
  76
  77         for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
  78 -               enetc_wr_reg(v->tbier_base + ENETC_BDR_OFF(i), 0);
  79 +               enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0);
  80 +
  81 +       spin_unlock_irqrestore(lock, flags);
  82
  83         napi_schedule_irqoff(&v->napi);
  84
  85 @@ -271,6 +289,9 @@ static int enetc_poll(struct napi_struct
  86         struct enetc_int_vector
  87                 *v = container_of(napi, struct enetc_int_vector, napi);
  88         bool complete = true;
  89 +       unsigned long flags;
  90 +       /* pointer to per-cpu ENETC lock for register access issue WA */
  91 +       spinlock_t *lock;
  92         int work_done;
  93         int i;
  94
  95 @@ -287,19 +308,24 @@ static int enetc_poll(struct napi_struct
  96
  97         napi_complete_done(napi, work_done);
  98
  99 +       lock = this_cpu_ptr(&enetc_gregs);
 100 +       spin_lock_irqsave(lock, flags);
 101 +
 102         /* enable interrupts */
 103 -       enetc_wr_reg(v->rbier, ENETC_RBIER_RXTIE);
 104 +       enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE);
 105
 106         for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
 107 -               enetc_wr_reg(v->tbier_base + ENETC_BDR_OFF(i),
 108 -                            ENETC_TBIER_TXTIE);
 109 +               enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i),
 110 +                                ENETC_TBIER_TXTIE);
 111 +
 112 +       spin_unlock_irqrestore(lock, flags);
 113
 114         return work_done;
 115  }
 116
 117  static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)
 118  {
 119 -       int pi = enetc_rd_reg(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;
 120 +       int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;
 121
 122         return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi;
 123  }
 124 @@ -337,9 +363,18 @@ static bool enetc_clean_tx_ring(struct e
 125         bool do_tstamp;
 126         u64 tstamp = 0;
 127
 128 +       unsigned long flags;
 129 +       /* pointer to per-cpu ENETC lock for register access issue WA */
 130 +       spinlock_t *lock;
 131 +
 132 +       lock = this_cpu_ptr(&enetc_gregs);
 133 +
 134         i = tx_ring->next_to_clean;
 135         tx_swbd = &tx_ring->tx_swbd[i];
 136 +
 137 +       spin_lock_irqsave(lock, flags);
 138         bds_to_clean = enetc_bd_ready_count(tx_ring, i);
 139 +       spin_unlock_irqrestore(lock, flags);
 140
 141         do_tstamp = false;
 142
 143 @@ -382,16 +417,20 @@ static bool enetc_clean_tx_ring(struct e
 144                         tx_swbd = tx_ring->tx_swbd;
 145                 }
 146
 147 +               spin_lock_irqsave(lock, flags);
 148 +
 149                 /* BD iteration loop end */
 150                 if (is_eof) {
 151                         tx_frm_cnt++;
 152                         /* re-arm interrupt source */
 153 -                       enetc_wr_reg(tx_ring->idr, BIT(tx_ring->index) |
 154 -                                    BIT(16 + tx_ring->index));
 155 +                       enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) |
 156 +                                        BIT(16 + tx_ring->index));
 157                 }
 158
 159                 if (unlikely(!bds_to_clean))
 160                         bds_to_clean = enetc_bd_ready_count(tx_ring, i);
 161 +
 162 +               spin_unlock_irqrestore(lock, flags);
 163         }
 164
 165         tx_ring->next_to_clean = i;
 166 @@ -470,13 +509,14 @@ static int enetc_refill_rx_ring(struct e
 167                 rx_ring->next_to_alloc = i; /* keep track from page reuse */
 168                 rx_ring->next_to_use = i;
 169                 /* update ENETC's consumer index */
 170 -               enetc_wr_reg(rx_ring->rcir, i);
 171 +               enetc_wr_reg_hot(rx_ring->rcir, i);
 172         }
 173
 174         return j;
 175  }
 176
 177  #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING
 178 +/* Must be called with &enetc_gregs spinlock held */
 179  static void enetc_get_rx_tstamp(struct net_device *ndev,
 180                                 union enetc_rx_bd *rxbd,
 181                                 struct sk_buff *skb)
 182 @@ -488,8 +528,8 @@ static void enetc_get_rx_tstamp(struct n
 183         u64 tstamp;
 184
 185         if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) {
 186 -               lo = enetc_rd(hw, ENETC_SICTR0);
 187 -               hi = enetc_rd(hw, ENETC_SICTR1);
 188 +               lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0);
 189 +               hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1);
 190                 tstamp_lo = le32_to_cpu(rxbd->r.tstamp);
 191                 if (lo <= tstamp_lo)
 192                         hi -= 1;
 193 @@ -627,6 +667,12 @@ static int enetc_clean_rx_ring(struct en
 194         int rx_frm_cnt = 0, rx_byte_cnt = 0;
 195         int cleaned_cnt, i;
 196
 197 +       unsigned long flags;
 198 +       /* pointer to per-cpu ENETC lock for register access issue WA */
 199 +       spinlock_t *lock;
 200 +
 201 +       lock = this_cpu_ptr(&enetc_gregs);
 202 +
 203         cleaned_cnt = enetc_bd_unused(rx_ring);
 204         /* next descriptor to process */
 205         i = rx_ring->next_to_clean;
 206 @@ -637,6 +683,8 @@ static int enetc_clean_rx_ring(struct en
 207                 u32 bd_status;
 208                 u16 size;
 209
 210 +               spin_lock_irqsave(lock, flags);
 211 +
 212                 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) {
 213                         int count = enetc_refill_rx_ring(rx_ring, cleaned_cnt);
 214
 215 @@ -645,15 +693,19 @@ static int enetc_clean_rx_ring(struct en
 216
 217                 rxbd = ENETC_RXBD(*rx_ring, i);
 218                 bd_status = le32_to_cpu(rxbd->r.lstatus);
 219 -               if (!bd_status)
 220 +               if (!bd_status) {
 221 +                       spin_unlock_irqrestore(lock, flags);
 222                         break;
 223 +               }
 224
 225 -               enetc_wr_reg(rx_ring->idr, BIT(rx_ring->index));
 226 +               enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
 227                 dma_rmb(); /* for reading other rxbd fields */
 228                 size = le16_to_cpu(rxbd->r.buf_len);
 229                 skb = enetc_map_rx_buff_to_skb(rx_ring, i, size);
 230 -               if (!skb)
 231 +               if (!skb) {
 232 +                       spin_unlock_irqrestore(lock, flags);
 233                         break;
 234 +               }
 235
 236                 enetc_get_offloads(rx_ring, rxbd, skb);
 237
 238 @@ -667,6 +719,7 @@ static int enetc_clean_rx_ring(struct en
 239
 240                 if (unlikely(bd_status &
 241                              ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))) {
 242 +                       spin_unlock_irqrestore(lock, flags);
 243                         dev_kfree_skb(skb);
 244                         while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
 245                                 dma_rmb();
 246 @@ -710,6 +763,8 @@ static int enetc_clean_rx_ring(struct en
 247
 248                 enetc_process_skb(rx_ring, skb);
 249
 250 +               spin_unlock_irqrestore(lock, flags);
 251 +
 252                 napi_gro_receive(napi, skb);
 253
 254                 rx_frm_cnt++;
 255 --- a/drivers/net/ethernet/freescale/enetc/enetc_hw.h
 256 +++ b/drivers/net/ethernet/freescale/enetc/enetc_hw.h
 257 @@ -321,8 +321,15 @@ struct enetc_hw {
 258  };
 259
 260  /* general register accessors */
 261 -#define enetc_rd_reg(reg)      ioread32((reg))
 262 -#define enetc_wr_reg(reg, val) iowrite32((val), (reg))
 263 +#define enetc_rd_reg(reg)      enetc_rd_reg_wa((reg))
 264 +#define enetc_wr_reg(reg, val) enetc_wr_reg_wa((reg), (val))
 265 +
 266 +/* accessors for data-path, due to MDIO issue on LS1028 these should be called
 267 + * only under enetc_gregs per-cpu lock
 268 + */
 269 +#define enetc_rd_reg_hot(reg)  ioread32((reg))
 270 +#define enetc_wr_reg_hot(reg, val)     iowrite32((val), (reg))
 271 +
 272  #ifdef ioread64
 273  #define enetc_rd_reg64(reg)    ioread64((reg))
 274  #else
 275 @@ -341,12 +348,102 @@ static inline u64 enetc_rd_reg64(void __
 276  }
 277  #endif
 278
 279 +extern DEFINE_PER_CPU(spinlock_t, enetc_gregs);
 280 +
 281 +static inline u32 enetc_rd_reg_wa(void *reg)
 282 +{
 283 +       unsigned long flags;
 284 +       /* pointer to per-cpu ENETC lock for register access issue WA */
 285 +       spinlock_t *lock;
 286 +       u32 val;
 287 +
 288 +       lock = this_cpu_ptr(&enetc_gregs);
 289 +       spin_lock_irqsave(lock, flags);
 290 +       val = ioread32(reg);
 291 +       spin_unlock_irqrestore(lock, flags);
 292 +
 293 +       return val;
 294 +}
 295 +
 296 +static inline void enetc_wr_reg_wa(void *reg, u32 val)
 297 +{
 298 +       unsigned long flags;
 299 +       /* pointer to per-cpu ENETC lock for register access issue WA */
 300 +       spinlock_t *lock;
 301 +
 302 +       lock = this_cpu_ptr(&enetc_gregs);
 303 +       spin_lock_irqsave(lock, flags);
 304 +       iowrite32(val, reg);
 305 +       spin_unlock_irqrestore(lock, flags);
 306 +}
 307 +
 308 +/* NR_CPUS=256 in ARM64 defconfig and using it as array size triggers stack
 309 + * frame warnings for the functions below.  Use a custom define of 2 for now,
 310 + * LS1028 has just two cores.
 311 + */
 312 +#define ENETC_NR_CPU_LOCKS     2
 313 +
 314 +static inline u32 enetc_rd_reg_wa_single(void *reg)
 315 +{
 316 +       u32 val;
 317 +       int cpu;
 318 +       /* per-cpu ENETC lock array for register access issue WA */
 319 +       spinlock_t *lock[ENETC_NR_CPU_LOCKS];
 320 +       unsigned long flags;
 321 +
 322 +       local_irq_save(flags);
 323 +       preempt_disable();
 324 +
 325 +       for_each_online_cpu(cpu) {
 326 +               lock[cpu] = per_cpu_ptr(&enetc_gregs, cpu);
 327 +               spin_lock(lock[cpu]);
 328 +       }
 329 +
 330 +       val = ioread32(reg);
 331 +
 332 +       for_each_online_cpu(cpu)
 333 +               spin_unlock(lock[cpu]);
 334 +       local_irq_restore(flags);
 335 +
 336 +       preempt_enable();
 337 +
 338 +       return val;
 339 +}
 340 +
 341 +static inline void enetc_wr_reg_wa_single(void *reg, u32 val)
 342 +{
 343 +       int cpu;
 344 +       /* per-cpu ENETC lock array for register access issue WA */
 345 +       spinlock_t *lock[ENETC_NR_CPU_LOCKS];
 346 +       unsigned long flags;
 347 +
 348 +       local_irq_save(flags);
 349 +       preempt_disable();
 350 +
 351 +       for_each_online_cpu(cpu) {
 352 +               lock[cpu] = per_cpu_ptr(&enetc_gregs, cpu);
 353 +               spin_lock(lock[cpu]);
 354 +       }
 355 +
 356 +       iowrite32(val, reg);
 357 +
 358 +       for_each_online_cpu(cpu)
 359 +               spin_unlock(lock[cpu]);
 360 +       local_irq_restore(flags);
 361 +
 362 +       preempt_enable();
 363 +}
 364 +
 365  #define enetc_rd(hw, off)              enetc_rd_reg((hw)->reg + (off))
 366  #define enetc_wr(hw, off, val)         enetc_wr_reg((hw)->reg + (off), val)
 367  #define enetc_rd64(hw, off)            enetc_rd_reg64((hw)->reg + (off))
 368  /* port register accessors - PF only */
 369 -#define enetc_port_rd(hw, off)         enetc_rd_reg((hw)->port + (off))
 370 -#define enetc_port_wr(hw, off, val)    enetc_wr_reg((hw)->port + (off), val)
 371 +#define enetc_port_rd(hw, off)         enetc_rd_reg_wa((hw)->port + (off))
 372 +#define enetc_port_wr(hw, off, val)    enetc_wr_reg_wa((hw)->port + (off), val)
 373 +#define enetc_port_rd_single(hw, off)          enetc_rd_reg_wa_single(\
 374 +                                                       (hw)->port + (off))
 375 +#define enetc_port_wr_single(hw, off, val)     enetc_wr_reg_wa_single(\
 376 +                                                       (hw)->port + (off), val)
 377  /* global register accessors - PF only */
 378  #define enetc_global_rd(hw, off)       enetc_rd_reg((hw)->global + (off))
 379  #define enetc_global_wr(hw, off, val)  enetc_wr_reg((hw)->global + (off), val)
 380 --- a/drivers/net/ethernet/freescale/enetc/enetc_mdio.c
 381 +++ b/drivers/net/ethernet/freescale/enetc/enetc_mdio.c
 382 @@ -16,13 +16,13 @@
 383
 384  static inline u32 _enetc_mdio_rd(struct enetc_mdio_priv *mdio_priv, int off)
 385  {
 386 -       return enetc_port_rd(mdio_priv->hw, mdio_priv->mdio_base + off);
 387 +       return enetc_port_rd_single(mdio_priv->hw, mdio_priv->mdio_base + off);
 388  }
 389
 390  static inline void _enetc_mdio_wr(struct enetc_mdio_priv *mdio_priv, int off,
 391                                   u32 val)
 392  {
 393 -       enetc_port_wr(mdio_priv->hw, mdio_priv->mdio_base + off, val);
 394 +       enetc_port_wr_single(mdio_priv->hw, mdio_priv->mdio_base + off, val);
 395  }
 396
 397  #define enetc_mdio_rd(mdio_priv, off) \
 398 --- a/drivers/net/ethernet/freescale/enetc/enetc_pf.c
 399 +++ b/drivers/net/ethernet/freescale/enetc/enetc_pf.c
 400 @@ -986,6 +986,9 @@ static void enetc_pf_remove(struct pci_d
 401         enetc_pci_remove(pdev);
 402  }
 403
 404 +DEFINE_PER_CPU(spinlock_t, enetc_gregs);
 405 +EXPORT_PER_CPU_SYMBOL(enetc_gregs);
 406 +
 407  static const struct pci_device_id enetc_pf_id_table[] = {
 408         { PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, ENETC_DEV_ID_PF) },
 409         { 0, } /* End of table. */