target/linux/apm821xx/patches-4.14/020-0017-crypto-crypto4xx-add-backlog-queue-support.patch

   1 From 8ef8d195430ca3542d0434cf25e5115484b9fa32 Mon Sep 17 00:00:00 2001
   2 From: Christian Lamparter <chunkeey@gmail.com>
   3 Date: Wed, 4 Oct 2017 01:00:09 +0200
   4 Subject: [PATCH 17/25] crypto: crypto4xx - add backlog queue support
   5
   6 Previously, If the crypto4xx driver used all available
   7 security contexts, it would simply refuse new requests
   8 with -EAGAIN. CRYPTO_TFM_REQ_MAY_BACKLOG was ignored.
   9
  10 in case of dm-crypt.c's crypt_convert() function this was
  11 causing the following errors to manifest, if the system was
  12 pushed hard enough:
  13
  14 | EXT4-fs warning (dm-1): ext4_end_bio:314: I/O error -5 writing to ino ..
  15 | EXT4-fs warning (dm-1): ext4_end_bio:314: I/O error -5 writing to ino ..
  16 | EXT4-fs warning (dm-1): ext4_end_bio:314: I/O error -5 writing to ino ..
  17 | JBD2: Detected IO errors while flushing file data on dm-1-8
  18 | Aborting journal on device dm-1-8.
  19 | EXT4-fs error : ext4_journal_check_start:56: Detected aborted journal
  20 | EXT4-fs (dm-1): Remounting filesystem read-only
  21 | EXT4-fs : ext4_writepages: jbd2_start: 2048 pages, inode 498...; err -30
  22
  23 (This did cause corruptions due to failed writes)
  24
  25 To fix this mess, the crypto4xx driver needs to notifiy the
  26 user to slow down. This can be achieved by returning -EBUSY
  27 on requests, once the crypto hardware was falling behind.
  28
  29 Note: -EBUSY has two different meanings. Setting the flag
  30 CRYPTO_TFM_REQ_MAY_BACKLOG implies that the request was
  31 successfully queued, by the crypto driver. To achieve this
  32 requirement, the implementation introduces a threshold check and
  33 adds logic to the completion routines in much the same way as
  34 AMD's Cryptographic Coprocessor (CCP) driver do.
  35
  36 Note2: Tests showed that dm-crypt starved ipsec traffic.
  37 Under load, ipsec links dropped to 0 Kbits/s. This is because
  38 dm-crypt's callback would instantly queue the next request.
  39 In order to not starve ipsec, the driver reserves a small
  40 portion of the available crypto contexts for this purpose.
  41
  42 Signed-off-by: Christian Lamparter <chunkeey@gmail.com>
  43 Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
  44 ---
  45  drivers/crypto/amcc/crypto4xx_core.c | 47 ++++++++++++++++++++++++++++++------
  46  drivers/crypto/amcc/crypto4xx_core.h |  3 ++-
  47  2 files changed, 41 insertions(+), 9 deletions(-)
  48
  49 --- a/drivers/crypto/amcc/crypto4xx_core.c
  50 +++ b/drivers/crypto/amcc/crypto4xx_core.c
  51 @@ -39,6 +39,7 @@
  52  #include <crypto/ctr.h>
  53  #include <crypto/sha.h>
  54  #include <crypto/scatterwalk.h>
  55 +#include <crypto/internal/skcipher.h>
  56  #include "crypto4xx_reg_def.h"
  57  #include "crypto4xx_core.h"
  58  #include "crypto4xx_sa.h"
  59 @@ -578,8 +579,10 @@ static u32 crypto4xx_ablkcipher_done(str
  60         }
  61
  62         crypto4xx_ret_sg_desc(dev, pd_uinfo);
  63 -       if (ablk_req->base.complete != NULL)
  64 -               ablk_req->base.complete(&ablk_req->base, 0);
  65 +
  66 +       if (pd_uinfo->state & PD_ENTRY_BUSY)
  67 +               ablkcipher_request_complete(ablk_req, -EINPROGRESS);
  68 +       ablkcipher_request_complete(ablk_req, 0);
  69
  70         return 0;
  71  }
  72 @@ -596,9 +599,10 @@ static u32 crypto4xx_ahash_done(struct c
  73         crypto4xx_copy_digest_to_dst(pd_uinfo,
  74                                      crypto_tfm_ctx(ahash_req->base.tfm));
  75         crypto4xx_ret_sg_desc(dev, pd_uinfo);
  76 -       /* call user provided callback function x */
  77 -       if (ahash_req->base.complete != NULL)
  78 -               ahash_req->base.complete(&ahash_req->base, 0);
  79 +
  80 +       if (pd_uinfo->state & PD_ENTRY_BUSY)
  81 +               ahash_request_complete(ahash_req, -EINPROGRESS);
  82 +       ahash_request_complete(ahash_req, 0);
  83
  84         return 0;
  85  }
  86 @@ -709,6 +713,7 @@ u32 crypto4xx_build_pd(struct crypto_asy
  87         struct pd_uinfo *pd_uinfo = NULL;
  88         unsigned int nbytes = datalen, idx;
  89         u32 gd_idx = 0;
  90 +       bool is_busy;
  91
  92         /* figure how many gd is needed */
  93         num_gd = sg_nents_for_len(src, datalen);
  94 @@ -739,6 +744,31 @@ u32 crypto4xx_build_pd(struct crypto_asy
  95          * already got must be return the original place.
  96          */
  97         spin_lock_irqsave(&dev->core_dev->lock, flags);
  98 +       /*
  99 +        * Let the caller know to slow down, once more than 13/16ths = 81%
 100 +        * of the available data contexts are being used simultaneously.
 101 +        *
 102 +        * With PPC4XX_NUM_PD = 256, this will leave a "backlog queue" for
 103 +        * 31 more contexts. Before new requests have to be rejected.
 104 +        */
 105 +       if (req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG) {
 106 +               is_busy = ((dev->pdr_head - dev->pdr_tail) % PPC4XX_NUM_PD) >=
 107 +                       ((PPC4XX_NUM_PD * 13) / 16);
 108 +       } else {
 109 +               /*
 110 +                * To fix contention issues between ipsec (no blacklog) and
 111 +                * dm-crypto (backlog) reserve 32 entries for "no backlog"
 112 +                * data contexts.
 113 +                */
 114 +               is_busy = ((dev->pdr_head - dev->pdr_tail) % PPC4XX_NUM_PD) >=
 115 +                       ((PPC4XX_NUM_PD * 15) / 16);
 116 +
 117 +               if (is_busy) {
 118 +                       spin_unlock_irqrestore(&dev->core_dev->lock, flags);
 119 +                       return -EBUSY;
 120 +               }
 121 +       }
 122 +
 123         if (num_gd) {
 124                 fst_gd = crypto4xx_get_n_gd(dev, num_gd);
 125                 if (fst_gd == ERING_WAS_FULL) {
 126 @@ -893,11 +923,12 @@ u32 crypto4xx_build_pd(struct crypto_asy
 127         sa->sa_command_1.bf.hash_crypto_offset = 0;
 128         pd->pd_ctl.w = ctx->pd_ctl;
 129         pd->pd_ctl_len.w = 0x00400000 | datalen;
 130 -       pd_uinfo->state = PD_ENTRY_INUSE;
 131 +       pd_uinfo->state = PD_ENTRY_INUSE | (is_busy ? PD_ENTRY_BUSY : 0);
 132 +
 133         wmb();
 134         /* write any value to push engine to read a pd */
 135         writel(1, dev->ce_base + CRYPTO4XX_INT_DESCR_RD);
 136 -       return -EINPROGRESS;
 137 +       return is_busy ? -EBUSY : -EINPROGRESS;
 138  }
 139
 140  /**
 141 @@ -1002,7 +1033,7 @@ static void crypto4xx_bh_tasklet_cb(unsi
 142                 tail = core_dev->dev->pdr_tail;
 143                 pd_uinfo = &core_dev->dev->pdr_uinfo[tail];
 144                 pd = &core_dev->dev->pdr[tail];
 145 -               if ((pd_uinfo->state == PD_ENTRY_INUSE) &&
 146 +               if ((pd_uinfo->state & PD_ENTRY_INUSE) &&
 147                                    pd->pd_ctl.bf.pe_done &&
 148                                    !pd->pd_ctl.bf.host_ready) {
 149                         pd->pd_ctl.bf.pe_done = 0;
 150 --- a/drivers/crypto/amcc/crypto4xx_core.h
 151 +++ b/drivers/crypto/amcc/crypto4xx_core.h
 152 @@ -44,7 +44,8 @@
 153  #define PPC4XX_LAST_SD                         (PPC4XX_NUM_SD - 1)
 154  #define PPC4XX_SD_BUFFER_SIZE                  2048
 155
 156 -#define PD_ENTRY_INUSE                         1
 157 +#define PD_ENTRY_BUSY                          BIT(1)
 158 +#define PD_ENTRY_INUSE                         BIT(0)
 159  #define PD_ENTRY_FREE                          0
 160  #define ERING_WAS_FULL                         0xffffffff
 161