target/linux/generic/backport-5.4/080-wireguard-0006-crypto-arm64-chacha-expose-arm64-ChaCha-routine-as-l.patch

   1 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
   2 From: Ard Biesheuvel <ardb@kernel.org>
   3 Date: Fri, 8 Nov 2019 13:22:12 +0100
   4 Subject: [PATCH] crypto: arm64/chacha - expose arm64 ChaCha routine as library
   5  function
   6
   7 commit b3aad5bad26a01a4bd8c49a5c5f52aec665f3b7c upstream.
   8
   9 Expose the accelerated NEON ChaCha routine directly as a symbol
  10 export so that users of the ChaCha library API can use it directly.
  11
  12 Given that calls into the library API will always go through the
  13 routines in this module if it is enabled, switch to static keys
  14 to select the optimal implementation available (which may be none
  15 at all, in which case we defer to the generic implementation for
  16 all invocations).
  17
  18 Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
  19 Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
  20 Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
  21 ---
  22  arch/arm64/crypto/Kconfig            |  1 +
  23  arch/arm64/crypto/chacha-neon-glue.c | 53 ++++++++++++++++++++++------
  24  2 files changed, 43 insertions(+), 11 deletions(-)
  25
  26 --- a/arch/arm64/crypto/Kconfig
  27 +++ b/arch/arm64/crypto/Kconfig
  28 @@ -104,6 +104,7 @@ config CRYPTO_CHACHA20_NEON
  29         depends on KERNEL_MODE_NEON
  30         select CRYPTO_BLKCIPHER
  31         select CRYPTO_LIB_CHACHA_GENERIC
  32 +       select CRYPTO_ARCH_HAVE_LIB_CHACHA
  33
  34  config CRYPTO_NHPOLY1305_NEON
  35         tristate "NHPoly1305 hash function using NEON instructions (for Adiantum)"
  36 --- a/arch/arm64/crypto/chacha-neon-glue.c
  37 +++ b/arch/arm64/crypto/chacha-neon-glue.c
  38 @@ -23,6 +23,7 @@
  39  #include <crypto/internal/chacha.h>
  40  #include <crypto/internal/simd.h>
  41  #include <crypto/internal/skcipher.h>
  42 +#include <linux/jump_label.h>
  43  #include <linux/kernel.h>
  44  #include <linux/module.h>
  45
  46 @@ -36,6 +37,8 @@ asmlinkage void chacha_4block_xor_neon(u
  47                                        int nrounds, int bytes);
  48  asmlinkage void hchacha_block_neon(const u32 *state, u32 *out, int nrounds);
  49
  50 +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
  51 +
  52  static void chacha_doneon(u32 *state, u8 *dst, const u8 *src,
  53                           int bytes, int nrounds)
  54  {
  55 @@ -59,6 +62,37 @@ static void chacha_doneon(u32 *state, u8
  56         }
  57  }
  58
  59 +void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds)
  60 +{
  61 +       if (!static_branch_likely(&have_neon) || !crypto_simd_usable()) {
  62 +               hchacha_block_generic(state, stream, nrounds);
  63 +       } else {
  64 +               kernel_neon_begin();
  65 +               hchacha_block_neon(state, stream, nrounds);
  66 +               kernel_neon_end();
  67 +       }
  68 +}
  69 +EXPORT_SYMBOL(hchacha_block_arch);
  70 +
  71 +void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv)
  72 +{
  73 +       chacha_init_generic(state, key, iv);
  74 +}
  75 +EXPORT_SYMBOL(chacha_init_arch);
  76 +
  77 +void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes,
  78 +                      int nrounds)
  79 +{
  80 +       if (!static_branch_likely(&have_neon) || bytes <= CHACHA_BLOCK_SIZE ||
  81 +           !crypto_simd_usable())
  82 +               return chacha_crypt_generic(state, dst, src, bytes, nrounds);
  83 +
  84 +       kernel_neon_begin();
  85 +       chacha_doneon(state, dst, src, bytes, nrounds);
  86 +       kernel_neon_end();
  87 +}
  88 +EXPORT_SYMBOL(chacha_crypt_arch);
  89 +
  90  static int chacha_neon_stream_xor(struct skcipher_request *req,
  91                                   const struct chacha_ctx *ctx, const u8 *iv)
  92  {
  93 @@ -76,7 +110,8 @@ static int chacha_neon_stream_xor(struct
  94                 if (nbytes < walk.total)
  95                         nbytes = rounddown(nbytes, walk.stride);
  96
  97 -               if (!crypto_simd_usable()) {
  98 +               if (!static_branch_likely(&have_neon) ||
  99 +                   !crypto_simd_usable()) {
 100                         chacha_crypt_generic(state, walk.dst.virt.addr,
 101                                              walk.src.virt.addr, nbytes,
 102                                              ctx->nrounds);
 103 @@ -109,14 +144,7 @@ static int xchacha_neon(struct skcipher_
 104         u8 real_iv[16];
 105
 106         chacha_init_generic(state, ctx->key, req->iv);
 107 -
 108 -       if (crypto_simd_usable()) {
 109 -               kernel_neon_begin();
 110 -               hchacha_block_neon(state, subctx.key, ctx->nrounds);
 111 -               kernel_neon_end();
 112 -       } else {
 113 -               hchacha_block_generic(state, subctx.key, ctx->nrounds);
 114 -       }
 115 +       hchacha_block_arch(state, subctx.key, ctx->nrounds);
 116         subctx.nrounds = ctx->nrounds;
 117
 118         memcpy(&real_iv[0], req->iv + 24, 8);
 119 @@ -179,14 +207,17 @@ static struct skcipher_alg algs[] = {
 120  static int __init chacha_simd_mod_init(void)
 121  {
 122         if (!cpu_have_named_feature(ASIMD))
 123 -               return -ENODEV;
 124 +               return 0;
 125 +
 126 +       static_branch_enable(&have_neon);
 127
 128         return crypto_register_skciphers(algs, ARRAY_SIZE(algs));
 129  }
 130
 131  static void __exit chacha_simd_mod_fini(void)
 132  {
 133 -       crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));
 134 +       if (cpu_have_named_feature(ASIMD))
 135 +               crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));
 136  }
 137
 138  module_init(chacha_simd_mod_init);