target/linux/generic/backport-5.4/080-wireguard-0003-crypto-x86-chacha-depend-on-generic-chacha-library-i.patch

   1 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
   2 From: Ard Biesheuvel <ardb@kernel.org>
   3 Date: Fri, 8 Nov 2019 13:22:09 +0100
   4 Subject: [PATCH] crypto: x86/chacha - depend on generic chacha library instead
   5  of crypto driver
   6
   7 commit 28e8d89b1ce8d2e7badfb5f69971dd635acb8863 upstream.
   8
   9 In preparation of extending the x86 ChaCha driver to also expose the ChaCha
  10 library interface, drop the dependency on the chacha_generic crypto driver
  11 as a non-SIMD fallback, and depend on the generic ChaCha library directly.
  12 This way, we only pull in the code we actually need, without registering
  13 a set of ChaCha skciphers that we will never use.
  14
  15 Since turning the FPU on and off is cheap these days, simplify the SIMD
  16 routine by dropping the per-page yield, which makes for a cleaner switch
  17 to the library API as well. This also allows use to invoke the skcipher
  18 walk routines in non-atomic mode.
  19
  20 Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
  21 Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
  22 Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
  23 ---
  24  arch/x86/crypto/chacha_glue.c | 90 ++++++++++++++---------------------
  25  crypto/Kconfig                |  2 +-
  26  2 files changed, 36 insertions(+), 56 deletions(-)
  27
  28 --- a/arch/x86/crypto/chacha_glue.c
  29 +++ b/arch/x86/crypto/chacha_glue.c
  30 @@ -123,37 +123,38 @@ static void chacha_dosimd(u32 *state, u8
  31         }
  32  }
  33
  34 -static int chacha_simd_stream_xor(struct skcipher_walk *walk,
  35 +static int chacha_simd_stream_xor(struct skcipher_request *req,
  36                                   const struct chacha_ctx *ctx, const u8 *iv)
  37  {
  38         u32 *state, state_buf[16 + 2] __aligned(8);
  39 -       int next_yield = 4096; /* bytes until next FPU yield */
  40 -       int err = 0;
  41 +       struct skcipher_walk walk;
  42 +       int err;
  43 +
  44 +       err = skcipher_walk_virt(&walk, req, false);
  45
  46         BUILD_BUG_ON(CHACHA_STATE_ALIGN != 16);
  47         state = PTR_ALIGN(state_buf + 0, CHACHA_STATE_ALIGN);
  48
  49 -       crypto_chacha_init(state, ctx, iv);
  50 +       chacha_init_generic(state, ctx->key, iv);
  51
  52 -       while (walk->nbytes > 0) {
  53 -               unsigned int nbytes = walk->nbytes;
  54 +       while (walk.nbytes > 0) {
  55 +               unsigned int nbytes = walk.nbytes;
  56
  57 -               if (nbytes < walk->total) {
  58 -                       nbytes = round_down(nbytes, walk->stride);
  59 -                       next_yield -= nbytes;
  60 -               }
  61 -
  62 -               chacha_dosimd(state, walk->dst.virt.addr, walk->src.virt.addr,
  63 -                             nbytes, ctx->nrounds);
  64 +               if (nbytes < walk.total)
  65 +                       nbytes = round_down(nbytes, walk.stride);
  66
  67 -               if (next_yield <= 0) {
  68 -                       /* temporarily allow preemption */
  69 -                       kernel_fpu_end();
  70 +               if (!crypto_simd_usable()) {
  71 +                       chacha_crypt_generic(state, walk.dst.virt.addr,
  72 +                                            walk.src.virt.addr, nbytes,
  73 +                                            ctx->nrounds);
  74 +               } else {
  75                         kernel_fpu_begin();
  76 -                       next_yield = 4096;
  77 +                       chacha_dosimd(state, walk.dst.virt.addr,
  78 +                                     walk.src.virt.addr, nbytes,
  79 +                                     ctx->nrounds);
  80 +                       kernel_fpu_end();
  81                 }
  82 -
  83 -               err = skcipher_walk_done(walk, walk->nbytes - nbytes);
  84 +               err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
  85         }
  86
  87         return err;
  88 @@ -163,55 +164,34 @@ static int chacha_simd(struct skcipher_r
  89  {
  90         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
  91         struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
  92 -       struct skcipher_walk walk;
  93 -       int err;
  94 -
  95 -       if (req->cryptlen <= CHACHA_BLOCK_SIZE || !crypto_simd_usable())
  96 -               return crypto_chacha_crypt(req);
  97
  98 -       err = skcipher_walk_virt(&walk, req, true);
  99 -       if (err)
 100 -               return err;
 101 -
 102 -       kernel_fpu_begin();
 103 -       err = chacha_simd_stream_xor(&walk, ctx, req->iv);
 104 -       kernel_fpu_end();
 105 -       return err;
 106 +       return chacha_simd_stream_xor(req, ctx, req->iv);
 107  }
 108
 109  static int xchacha_simd(struct skcipher_request *req)
 110  {
 111         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 112         struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
 113 -       struct skcipher_walk walk;
 114 -       struct chacha_ctx subctx;
 115         u32 *state, state_buf[16 + 2] __aligned(8);
 116 +       struct chacha_ctx subctx;
 117         u8 real_iv[16];
 118 -       int err;
 119 -
 120 -       if (req->cryptlen <= CHACHA_BLOCK_SIZE || !crypto_simd_usable())
 121 -               return crypto_xchacha_crypt(req);
 122 -
 123 -       err = skcipher_walk_virt(&walk, req, true);
 124 -       if (err)
 125 -               return err;
 126
 127         BUILD_BUG_ON(CHACHA_STATE_ALIGN != 16);
 128         state = PTR_ALIGN(state_buf + 0, CHACHA_STATE_ALIGN);
 129 -       crypto_chacha_init(state, ctx, req->iv);
 130 +       chacha_init_generic(state, ctx->key, req->iv);
 131
 132 -       kernel_fpu_begin();
 133 -
 134 -       hchacha_block_ssse3(state, subctx.key, ctx->nrounds);
 135 +       if (req->cryptlen > CHACHA_BLOCK_SIZE && crypto_simd_usable()) {
 136 +               kernel_fpu_begin();
 137 +               hchacha_block_ssse3(state, subctx.key, ctx->nrounds);
 138 +               kernel_fpu_end();
 139 +       } else {
 140 +               hchacha_block_generic(state, subctx.key, ctx->nrounds);
 141 +       }
 142         subctx.nrounds = ctx->nrounds;
 143
 144         memcpy(&real_iv[0], req->iv + 24, 8);
 145         memcpy(&real_iv[8], req->iv + 16, 8);
 146 -       err = chacha_simd_stream_xor(&walk, &subctx, real_iv);
 147 -
 148 -       kernel_fpu_end();
 149 -
 150 -       return err;
 151 +       return chacha_simd_stream_xor(req, &subctx, real_iv);
 152  }
 153
 154  static struct skcipher_alg algs[] = {
 155 @@ -227,7 +207,7 @@ static struct skcipher_alg algs[] = {
 156                 .max_keysize            = CHACHA_KEY_SIZE,
 157                 .ivsize                 = CHACHA_IV_SIZE,
 158                 .chunksize              = CHACHA_BLOCK_SIZE,
 159 -               .setkey                 = crypto_chacha20_setkey,
 160 +               .setkey                 = chacha20_setkey,
 161                 .encrypt                = chacha_simd,
 162                 .decrypt                = chacha_simd,
 163         }, {
 164 @@ -242,7 +222,7 @@ static struct skcipher_alg algs[] = {
 165                 .max_keysize            = CHACHA_KEY_SIZE,
 166                 .ivsize                 = XCHACHA_IV_SIZE,
 167                 .chunksize              = CHACHA_BLOCK_SIZE,
 168 -               .setkey                 = crypto_chacha20_setkey,
 169 +               .setkey                 = chacha20_setkey,
 170                 .encrypt                = xchacha_simd,
 171                 .decrypt                = xchacha_simd,
 172         }, {
 173 @@ -257,7 +237,7 @@ static struct skcipher_alg algs[] = {
 174                 .max_keysize            = CHACHA_KEY_SIZE,
 175                 .ivsize                 = XCHACHA_IV_SIZE,
 176                 .chunksize              = CHACHA_BLOCK_SIZE,
 177 -               .setkey                 = crypto_chacha12_setkey,
 178 +               .setkey                 = chacha12_setkey,
 179                 .encrypt                = xchacha_simd,
 180                 .decrypt                = xchacha_simd,
 181         },
 182 --- a/crypto/Kconfig
 183 +++ b/crypto/Kconfig
 184 @@ -1417,7 +1417,7 @@ config CRYPTO_CHACHA20_X86_64
 185         tristate "ChaCha stream cipher algorithms (x86_64/SSSE3/AVX2/AVX-512VL)"
 186         depends on X86 && 64BIT
 187         select CRYPTO_BLKCIPHER
 188 -       select CRYPTO_CHACHA20
 189 +       select CRYPTO_LIB_CHACHA_GENERIC
 190         help
 191           SSSE3, AVX2, and AVX-512VL optimized implementations of the ChaCha20,
 192           XChaCha20, and XChaCha12 stream ciphers.