target/linux/apm821xx/patches-4.14/020-0021-crypto-crypto4xx-fix-stalls-under-heavy-load.patch

   1 From 4b5b79998af61db8b0506fba6c0f33b57ea457bd Mon Sep 17 00:00:00 2001
   2 From: Christian Lamparter <chunkeey@gmail.com>
   3 Date: Wed, 4 Oct 2017 01:00:13 +0200
   4 Subject: [PATCH 21/25] crypto: crypto4xx - fix stalls under heavy load
   5
   6 If the crypto4xx device is continuously loaded by dm-crypt
   7 and ipsec work, it will start to work intermittent after a
   8 few (between 20-30) seconds, hurting throughput and latency.
   9
  10 This patch contains various stability improvements in order
  11 to fix this issue. So far, the hardware has survived more
  12 than a day without suffering any stalls under the continuous
  13 load.
  14
  15 Signed-off-by: Christian Lamparter <chunkeey@gmail.com>
  16 Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
  17 ---
  18  drivers/crypto/amcc/crypto4xx_core.c    | 33 ++++++++++++++++++---------------
  19  drivers/crypto/amcc/crypto4xx_reg_def.h |  3 +++
  20  2 files changed, 21 insertions(+), 15 deletions(-)
  21
  22 --- a/drivers/crypto/amcc/crypto4xx_core.c
  23 +++ b/drivers/crypto/amcc/crypto4xx_core.c
  24 @@ -280,17 +280,20 @@ static u32 crypto4xx_get_pd_from_pdr_nol
  25  static u32 crypto4xx_put_pd_to_pdr(struct crypto4xx_device *dev, u32 idx)
  26  {
  27         struct pd_uinfo *pd_uinfo = &dev->pdr_uinfo[idx];
  28 +       u32 tail;
  29         unsigned long flags;
  30
  31         spin_lock_irqsave(&dev->core_dev->lock, flags);
  32 +       pd_uinfo->state = PD_ENTRY_FREE;
  33 +
  34         if (dev->pdr_tail != PPC4XX_LAST_PD)
  35                 dev->pdr_tail++;
  36         else
  37                 dev->pdr_tail = 0;
  38 -       pd_uinfo->state = PD_ENTRY_FREE;
  39 +       tail = dev->pdr_tail;
  40         spin_unlock_irqrestore(&dev->core_dev->lock, flags);
  41
  42 -       return 0;
  43 +       return tail;
  44  }
  45
  46  /**
  47 @@ -863,16 +866,16 @@ int crypto4xx_build_pd(struct crypto_asy
  48                 }
  49         }
  50
  51 -       sa->sa_command_1.bf.hash_crypto_offset = 0;
  52 -       pd->pd_ctl.w = 0;
  53 -       pd->pd_ctl.bf.hash_final =
  54 -               (crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AHASH);
  55 -       pd->pd_ctl.bf.host_ready = 1;
  56 +       pd->pd_ctl.w = PD_CTL_HOST_READY |
  57 +               ((crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AHASH) |
  58 +                (crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AEAD) ?
  59 +                       PD_CTL_HASH_FINAL : 0);
  60         pd->pd_ctl_len.w = 0x00400000 | datalen;
  61         pd_uinfo->state = PD_ENTRY_INUSE | (is_busy ? PD_ENTRY_BUSY : 0);
  62
  63         wmb();
  64         /* write any value to push engine to read a pd */
  65 +       writel(0, dev->ce_base + CRYPTO4XX_INT_DESCR_RD);
  66         writel(1, dev->ce_base + CRYPTO4XX_INT_DESCR_RD);
  67         return is_busy ? -EBUSY : -EINPROGRESS;
  68  }
  69 @@ -973,23 +976,23 @@ static void crypto4xx_bh_tasklet_cb(unsi
  70         struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev);
  71         struct pd_uinfo *pd_uinfo;
  72         struct ce_pd *pd;
  73 -       u32 tail;
  74 +       u32 tail = core_dev->dev->pdr_tail;
  75 +       u32 head = core_dev->dev->pdr_head;
  76
  77 -       while (core_dev->dev->pdr_head != core_dev->dev->pdr_tail) {
  78 -               tail = core_dev->dev->pdr_tail;
  79 +       do {
  80                 pd_uinfo = &core_dev->dev->pdr_uinfo[tail];
  81                 pd = &core_dev->dev->pdr[tail];
  82                 if ((pd_uinfo->state & PD_ENTRY_INUSE) &&
  83 -                                  pd->pd_ctl.bf.pe_done &&
  84 -                                  !pd->pd_ctl.bf.host_ready) {
  85 -                       pd->pd_ctl.bf.pe_done = 0;
  86 +                    ((READ_ONCE(pd->pd_ctl.w) &
  87 +                      (PD_CTL_PE_DONE | PD_CTL_HOST_READY)) ==
  88 +                      PD_CTL_PE_DONE)) {
  89                         crypto4xx_pd_done(core_dev->dev, tail);
  90 -                       crypto4xx_put_pd_to_pdr(core_dev->dev, tail);
  91 +                       tail = crypto4xx_put_pd_to_pdr(core_dev->dev, tail);
  92                 } else {
  93                         /* if tail not done, break */
  94                         break;
  95                 }
  96 -       }
  97 +       } while (head != tail);
  98  }
  99
 100  /**
 101 --- a/drivers/crypto/amcc/crypto4xx_reg_def.h
 102 +++ b/drivers/crypto/amcc/crypto4xx_reg_def.h
 103 @@ -261,6 +261,9 @@ union ce_pd_ctl {
 104         } bf;
 105         u32 w;
 106  } __attribute__((packed));
 107 +#define PD_CTL_HASH_FINAL      BIT(4)
 108 +#define PD_CTL_PE_DONE         BIT(1)
 109 +#define PD_CTL_HOST_READY      BIT(0)
 110
 111  union ce_pd_ctl_len {
 112         struct {