target/linux/generic/backport-5.4/071-v5.16-04-mips-bpf-Add-JIT-workarounds-for-CPU-errata.patch

   1 From: Johan Almbladh <johan.almbladh@anyfinetworks.com>
   2 Date: Tue, 5 Oct 2021 18:54:06 +0200
   3 Subject: [PATCH] mips: bpf: Add JIT workarounds for CPU errata
   4
   5 This patch adds workarounds for the following CPU errata to the MIPS
   6 eBPF JIT, if enabled in the kernel configuration.
   7
   8   - R10000 ll/sc weak ordering
   9   - Loongson-3 ll/sc weak ordering
  10   - Loongson-2F jump hang
  11
  12 The Loongson-2F nop errata is implemented in uasm, which the JIT uses,
  13 so no additional mitigations are needed for that.
  14
  15 Signed-off-by: Johan Almbladh <johan.almbladh@anyfinetworks.com>
  16 Reviewed-by: Jiaxun Yang <jiaxun.yang@flygoat.com>
  17 ---
  18
  19 --- a/arch/mips/net/bpf_jit_comp.c
  20 +++ b/arch/mips/net/bpf_jit_comp.c
  21 @@ -404,6 +404,7 @@ void emit_alu_r(struct jit_context *ctx,
  22  /* Atomic read-modify-write (32-bit) */
  23  void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code)
  24  {
  25 +       LLSC_sync(ctx);
  26         emit(ctx, ll, MIPS_R_T9, off, dst);
  27         switch (code) {
  28         case BPF_ADD:
  29 @@ -420,18 +421,19 @@ void emit_atomic_r(struct jit_context *c
  30                 break;
  31         }
  32         emit(ctx, sc, MIPS_R_T8, off, dst);
  33 -       emit(ctx, beqz, MIPS_R_T8, -16);
  34 +       emit(ctx, LLSC_beqz, MIPS_R_T8, -16 - LLSC_offset);
  35         emit(ctx, nop); /* Delay slot */
  36  }
  37
  38  /* Atomic compare-and-exchange (32-bit) */
  39  void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off)
  40  {
  41 +       LLSC_sync(ctx);
  42         emit(ctx, ll, MIPS_R_T9, off, dst);
  43         emit(ctx, bne, MIPS_R_T9, res, 12);
  44         emit(ctx, move, MIPS_R_T8, src);     /* Delay slot */
  45         emit(ctx, sc, MIPS_R_T8, off, dst);
  46 -       emit(ctx, beqz, MIPS_R_T8, -20);
  47 +       emit(ctx, LLSC_beqz, MIPS_R_T8, -20 - LLSC_offset);
  48         emit(ctx, move, res, MIPS_R_T9);     /* Delay slot */
  49         clobber_reg(ctx, res);
  50  }
  51 --- a/arch/mips/net/bpf_jit_comp.h
  52 +++ b/arch/mips/net/bpf_jit_comp.h
  53 @@ -87,7 +87,7 @@ struct jit_context {
  54  };
  55
  56  /* Emit the instruction if the JIT memory space has been allocated */
  57 -#define emit(ctx, func, ...)                                   \
  58 +#define __emit(ctx, func, ...)                                 \
  59  do {                                                           \
  60         if ((ctx)->target != NULL) {                            \
  61                 u32 *p = &(ctx)->target[ctx->jit_index];        \
  62 @@ -95,6 +95,30 @@ do {                                                         \
  63         }                                                       \
  64         (ctx)->jit_index++;                                     \
  65  } while (0)
  66 +#define emit(...) __emit(__VA_ARGS__)
  67 +
  68 +/* Workaround for R10000 ll/sc errata */
  69 +#ifdef CONFIG_WAR_R10000
  70 +#define LLSC_beqz      beqzl
  71 +#else
  72 +#define LLSC_beqz      beqz
  73 +#endif
  74 +
  75 +/* Workaround for Loongson-3 ll/sc errata */
  76 +#ifdef CONFIG_CPU_LOONGSON3_WORKAROUNDS
  77 +#define LLSC_sync(ctx) emit(ctx, sync, 0)
  78 +#define LLSC_offset    4
  79 +#else
  80 +#define LLSC_sync(ctx)
  81 +#define LLSC_offset    0
  82 +#endif
  83 +
  84 +/* Workaround for Loongson-2F jump errata */
  85 +#ifdef CONFIG_CPU_JUMP_WORKAROUNDS
  86 +#define JALR_MASK      0xffffffffcfffffffULL
  87 +#else
  88 +#define JALR_MASK      (~0ULL)
  89 +#endif
  90
  91  /*
  92   * Mark a BPF register as accessed, it needs to be
  93 --- a/arch/mips/net/bpf_jit_comp64.c
  94 +++ b/arch/mips/net/bpf_jit_comp64.c
  95 @@ -375,6 +375,7 @@ static void emit_atomic_r64(struct jit_c
  96         u8 t1 = MIPS_R_T6;
  97         u8 t2 = MIPS_R_T7;
  98
  99 +       LLSC_sync(ctx);
 100         emit(ctx, lld, t1, off, dst);
 101         switch (code) {
 102         case BPF_ADD:
 103 @@ -391,7 +392,7 @@ static void emit_atomic_r64(struct jit_c
 104                 break;
 105         }
 106         emit(ctx, scd, t2, off, dst);
 107 -       emit(ctx, beqz, t2, -16);
 108 +       emit(ctx, LLSC_beqz, t2, -16 - LLSC_offset);
 109         emit(ctx, nop); /* Delay slot */
 110  }
 111
 112 @@ -414,7 +415,7 @@ static int emit_call(struct jit_context
 113         push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0);
 114
 115         /* Emit function call */
 116 -       emit_mov_i64(ctx, tmp, addr);
 117 +       emit_mov_i64(ctx, tmp, addr & JALR_MASK);
 118         emit(ctx, jalr, MIPS_R_RA, tmp);
 119         emit(ctx, nop); /* Delay slot */
 120