target/linux/generic/backport-5.4/080-wireguard-0017-crypto-x86-poly1305-expose-existing-driver-as-poly13.patch

   1 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
   2 From: Ard Biesheuvel <ardb@kernel.org>
   3 Date: Fri, 8 Nov 2019 13:22:23 +0100
   4 Subject: [PATCH] crypto: x86/poly1305 - expose existing driver as poly1305
   5  library
   6
   7 commit f0e89bcfbb894e5844cd1bbf6b3cf7c63cb0f5ac upstream.
   8
   9 Implement the arch init/update/final Poly1305 library routines in the
  10 accelerated SIMD driver for x86 so they are accessible to users of
  11 the Poly1305 library interface as well.
  12
  13 Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
  14 Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
  15 Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
  16 ---
  17  arch/x86/crypto/poly1305_glue.c | 57 ++++++++++++++++++++++++---------
  18  crypto/Kconfig                  |  1 +
  19  lib/crypto/Kconfig              |  1 +
  20  3 files changed, 43 insertions(+), 16 deletions(-)
  21
  22 --- a/arch/x86/crypto/poly1305_glue.c
  23 +++ b/arch/x86/crypto/poly1305_glue.c
  24 @@ -10,6 +10,7 @@
  25  #include <crypto/internal/poly1305.h>
  26  #include <crypto/internal/simd.h>
  27  #include <linux/crypto.h>
  28 +#include <linux/jump_label.h>
  29  #include <linux/kernel.h>
  30  #include <linux/module.h>
  31  #include <asm/simd.h>
  32 @@ -21,7 +22,8 @@ asmlinkage void poly1305_2block_sse2(u32
  33  asmlinkage void poly1305_4block_avx2(u32 *h, const u8 *src, const u32 *r,
  34                                      unsigned int blocks, const u32 *u);
  35
  36 -static bool poly1305_use_avx2 __ro_after_init;
  37 +static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_simd);
  38 +static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx2);
  39
  40  static void poly1305_simd_mult(u32 *a, const u32 *b)
  41  {
  42 @@ -64,7 +66,7 @@ static unsigned int poly1305_simd_blocks
  43         }
  44
  45         if (IS_ENABLED(CONFIG_AS_AVX2) &&
  46 -           poly1305_use_avx2 &&
  47 +           static_branch_likely(&poly1305_use_avx2) &&
  48             srclen >= POLY1305_BLOCK_SIZE * 4) {
  49                 if (unlikely(dctx->rset < 4)) {
  50                         if (dctx->rset < 2) {
  51 @@ -103,10 +105,15 @@ static unsigned int poly1305_simd_blocks
  52         return srclen;
  53  }
  54
  55 -static int poly1305_simd_update(struct shash_desc *desc,
  56 -                               const u8 *src, unsigned int srclen)
  57 +void poly1305_init_arch(struct poly1305_desc_ctx *desc, const u8 *key)
  58 +{
  59 +       poly1305_init_generic(desc, key);
  60 +}
  61 +EXPORT_SYMBOL(poly1305_init_arch);
  62 +
  63 +void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,
  64 +                         unsigned int srclen)
  65  {
  66 -       struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
  67         unsigned int bytes;
  68
  69         if (unlikely(dctx->buflen)) {
  70 @@ -117,7 +124,8 @@ static int poly1305_simd_update(struct s
  71                 dctx->buflen += bytes;
  72
  73                 if (dctx->buflen == POLY1305_BLOCK_SIZE) {
  74 -                       if (likely(crypto_simd_usable())) {
  75 +                       if (static_branch_likely(&poly1305_use_simd) &&
  76 +                           likely(crypto_simd_usable())) {
  77                                 kernel_fpu_begin();
  78                                 poly1305_simd_blocks(dctx, dctx->buf,
  79                                                      POLY1305_BLOCK_SIZE);
  80 @@ -131,7 +139,8 @@ static int poly1305_simd_update(struct s
  81         }
  82
  83         if (likely(srclen >= POLY1305_BLOCK_SIZE)) {
  84 -               if (likely(crypto_simd_usable())) {
  85 +               if (static_branch_likely(&poly1305_use_simd) &&
  86 +                   likely(crypto_simd_usable())) {
  87                         kernel_fpu_begin();
  88                         bytes = poly1305_simd_blocks(dctx, src, srclen);
  89                         kernel_fpu_end();
  90 @@ -147,6 +156,13 @@ static int poly1305_simd_update(struct s
  91                 memcpy(dctx->buf, src, srclen);
  92         }
  93  }
  94 +EXPORT_SYMBOL(poly1305_update_arch);
  95 +
  96 +void poly1305_final_arch(struct poly1305_desc_ctx *desc, u8 *digest)
  97 +{
  98 +       poly1305_final_generic(desc, digest);
  99 +}
 100 +EXPORT_SYMBOL(poly1305_final_arch);
 101
 102  static int crypto_poly1305_init(struct shash_desc *desc)
 103  {
 104 @@ -171,6 +187,15 @@ static int crypto_poly1305_final(struct
 105         return 0;
 106  }
 107
 108 +static int poly1305_simd_update(struct shash_desc *desc,
 109 +                               const u8 *src, unsigned int srclen)
 110 +{
 111 +       struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
 112 +
 113 +       poly1305_update_arch(dctx, src, srclen);
 114 +       return 0;
 115 +}
 116 +
 117  static struct shash_alg alg = {
 118         .digestsize     = POLY1305_DIGEST_SIZE,
 119         .init           = crypto_poly1305_init,
 120 @@ -189,15 +214,15 @@ static struct shash_alg alg = {
 121  static int __init poly1305_simd_mod_init(void)
 122  {
 123         if (!boot_cpu_has(X86_FEATURE_XMM2))
 124 -               return -ENODEV;
 125 +               return 0;
 126
 127 -       poly1305_use_avx2 = IS_ENABLED(CONFIG_AS_AVX2) &&
 128 -                           boot_cpu_has(X86_FEATURE_AVX) &&
 129 -                           boot_cpu_has(X86_FEATURE_AVX2) &&
 130 -                           cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL);
 131 -       alg.descsize = sizeof(struct poly1305_desc_ctx) + 5 * sizeof(u32);
 132 -       if (poly1305_use_avx2)
 133 -               alg.descsize += 10 * sizeof(u32);
 134 +       static_branch_enable(&poly1305_use_simd);
 135 +
 136 +       if (IS_ENABLED(CONFIG_AS_AVX2) &&
 137 +           boot_cpu_has(X86_FEATURE_AVX) &&
 138 +           boot_cpu_has(X86_FEATURE_AVX2) &&
 139 +           cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
 140 +               static_branch_enable(&poly1305_use_avx2);
 141
 142         return crypto_register_shash(&alg);
 143  }
 144 --- a/crypto/Kconfig
 145 +++ b/crypto/Kconfig
 146 @@ -698,6 +698,7 @@ config CRYPTO_POLY1305_X86_64
 147         tristate "Poly1305 authenticator algorithm (x86_64/SSE2/AVX2)"
 148         depends on X86 && 64BIT
 149         select CRYPTO_LIB_POLY1305_GENERIC
 150 +       select CRYPTO_ARCH_HAVE_LIB_POLY1305
 151         help
 152           Poly1305 authenticator algorithm, RFC7539.
 153
 154 --- a/lib/crypto/Kconfig
 155 +++ b/lib/crypto/Kconfig
 156 @@ -39,6 +39,7 @@ config CRYPTO_LIB_DES
 157
 158  config CRYPTO_LIB_POLY1305_RSIZE
 159         int
 160 +       default 4 if X86_64
 161         default 1
 162
 163  config CRYPTO_ARCH_HAVE_LIB_POLY1305