1 From: Johan Almbladh <johan.almbladh@anyfinetworks.com>
2 Date: Tue, 5 Oct 2021 18:54:06 +0200
3 Subject: [PATCH] mips: bpf: Add JIT workarounds for CPU errata
5 This patch adds workarounds for the following CPU errata to the MIPS
6 eBPF JIT, if enabled in the kernel configuration.
8 - R10000 ll/sc weak ordering
9 - Loongson-3 ll/sc weak ordering
10 - Loongson-2F jump hang
12 The Loongson-2F nop errata is implemented in uasm, which the JIT uses,
13 so no additional mitigations are needed for that.
15 Signed-off-by: Johan Almbladh <johan.almbladh@anyfinetworks.com>
16 Reviewed-by: Jiaxun Yang <jiaxun.yang@flygoat.com>
19 --- a/arch/mips/net/bpf_jit_comp.c
20 +++ b/arch/mips/net/bpf_jit_comp.c
21 @@ -404,6 +404,7 @@ void emit_alu_r(struct jit_context *ctx,
22 /* Atomic read-modify-write (32-bit) */
23 void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code)
26 emit(ctx, ll, MIPS_R_T9, off, dst);
29 @@ -420,18 +421,19 @@ void emit_atomic_r(struct jit_context *c
32 emit(ctx, sc, MIPS_R_T8, off, dst);
33 - emit(ctx, beqz, MIPS_R_T8, -16);
34 + emit(ctx, LLSC_beqz, MIPS_R_T8, -16 - LLSC_offset);
35 emit(ctx, nop); /* Delay slot */
38 /* Atomic compare-and-exchange (32-bit) */
39 void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off)
42 emit(ctx, ll, MIPS_R_T9, off, dst);
43 emit(ctx, bne, MIPS_R_T9, res, 12);
44 emit(ctx, move, MIPS_R_T8, src); /* Delay slot */
45 emit(ctx, sc, MIPS_R_T8, off, dst);
46 - emit(ctx, beqz, MIPS_R_T8, -20);
47 + emit(ctx, LLSC_beqz, MIPS_R_T8, -20 - LLSC_offset);
48 emit(ctx, move, res, MIPS_R_T9); /* Delay slot */
49 clobber_reg(ctx, res);
51 --- a/arch/mips/net/bpf_jit_comp.h
52 +++ b/arch/mips/net/bpf_jit_comp.h
53 @@ -87,7 +87,7 @@ struct jit_context {
56 /* Emit the instruction if the JIT memory space has been allocated */
57 -#define emit(ctx, func, ...) \
58 +#define __emit(ctx, func, ...) \
60 if ((ctx)->target != NULL) { \
61 u32 *p = &(ctx)->target[ctx->jit_index]; \
62 @@ -95,6 +95,30 @@ do { \
66 +#define emit(...) __emit(__VA_ARGS__)
68 +/* Workaround for R10000 ll/sc errata */
69 +#ifdef CONFIG_WAR_R10000
70 +#define LLSC_beqz beqzl
72 +#define LLSC_beqz beqz
75 +/* Workaround for Loongson-3 ll/sc errata */
76 +#ifdef CONFIG_CPU_LOONGSON3_WORKAROUNDS
77 +#define LLSC_sync(ctx) emit(ctx, sync, 0)
78 +#define LLSC_offset 4
80 +#define LLSC_sync(ctx)
81 +#define LLSC_offset 0
84 +/* Workaround for Loongson-2F jump errata */
85 +#ifdef CONFIG_CPU_JUMP_WORKAROUNDS
86 +#define JALR_MASK 0xffffffffcfffffffULL
88 +#define JALR_MASK (~0ULL)
92 * Mark a BPF register as accessed, it needs to be
93 --- a/arch/mips/net/bpf_jit_comp64.c
94 +++ b/arch/mips/net/bpf_jit_comp64.c
95 @@ -375,6 +375,7 @@ static void emit_atomic_r64(struct jit_c
100 emit(ctx, lld, t1, off, dst);
103 @@ -391,7 +392,7 @@ static void emit_atomic_r64(struct jit_c
106 emit(ctx, scd, t2, off, dst);
107 - emit(ctx, beqz, t2, -16);
108 + emit(ctx, LLSC_beqz, t2, -16 - LLSC_offset);
109 emit(ctx, nop); /* Delay slot */
112 @@ -414,7 +415,7 @@ static int emit_call(struct jit_context
113 push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0);
115 /* Emit function call */
116 - emit_mov_i64(ctx, tmp, addr);
117 + emit_mov_i64(ctx, tmp, addr & JALR_MASK);
118 emit(ctx, jalr, MIPS_R_RA, tmp);
119 emit(ctx, nop); /* Delay slot */