diff options
Diffstat (limited to 'arch/arm64/net/bpf_jit_comp.c')
| -rw-r--r-- | arch/arm64/net/bpf_jit_comp.c | 38 |
1 files changed, 22 insertions, 16 deletions
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index d68abde52740..2f0505b5c240 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -27,6 +27,7 @@ #include <asm/byteorder.h> #include <asm/cacheflush.h> #include <asm/debug-monitors.h> +#include <asm/set_memory.h> #include "bpf_jit.h" @@ -35,6 +36,7 @@ int bpf_jit_enable __read_mostly; #define TMP_REG_1 (MAX_BPF_JIT_REG + 0) #define TMP_REG_2 (MAX_BPF_JIT_REG + 1) #define TCALL_CNT (MAX_BPF_JIT_REG + 2) +#define TMP_REG_3 (MAX_BPF_JIT_REG + 3) /* Map BPF registers to A64 registers */ static const int bpf2a64[] = { @@ -56,6 +58,7 @@ static const int bpf2a64[] = { /* temporary registers for internal BPF JIT */ [TMP_REG_1] = A64_R(10), [TMP_REG_2] = A64_R(11), + [TMP_REG_3] = A64_R(12), /* tail_call_cnt */ [TCALL_CNT] = A64_R(26), /* temporary register for blinding constants */ @@ -68,6 +71,7 @@ struct jit_ctx { int epilogue_offset; int *offset; u32 *image; + u32 stack_size; }; static inline void emit(const u32 insn, struct jit_ctx *ctx) @@ -144,16 +148,11 @@ static inline int epilogue_offset(const struct jit_ctx *ctx) /* Stack must be multiples of 16B */ #define STACK_ALIGN(sz) (((sz) + 15) & ~15) -#define _STACK_SIZE \ - (MAX_BPF_STACK \ - + 4 /* extra for skb_copy_bits buffer */) - -#define STACK_SIZE STACK_ALIGN(_STACK_SIZE) - #define PROLOGUE_OFFSET 8 static int build_prologue(struct jit_ctx *ctx) { + const struct bpf_prog *prog = ctx->prog; const u8 r6 = bpf2a64[BPF_REG_6]; const u8 r7 = bpf2a64[BPF_REG_7]; const u8 r8 = bpf2a64[BPF_REG_8]; @@ -175,9 +174,9 @@ static int build_prologue(struct jit_ctx *ctx) * | | * | ... | BPF prog stack * | | - * +-----+ <= (BPF_FP - MAX_BPF_STACK) + * +-----+ <= (BPF_FP - prog->aux->stack_depth) * |RSVD | JIT scratchpad - * current A64_SP => +-----+ <= (BPF_FP - STACK_SIZE) + * current A64_SP => +-----+ <= (BPF_FP - ctx->stack_size) * | | * | ... | Function call stack * | | @@ -201,8 +200,12 @@ static int build_prologue(struct jit_ctx *ctx) /* Initialize tail_call_cnt */ emit(A64_MOVZ(1, tcc, 0, 0), ctx); + /* 4 byte extra for skb_copy_bits buffer */ + ctx->stack_size = prog->aux->stack_depth + 4; + ctx->stack_size = STACK_ALIGN(ctx->stack_size); + /* Set up function call stack */ - emit(A64_SUB_I(1, A64_SP, A64_SP, STACK_SIZE), ctx); + emit(A64_SUB_I(1, A64_SP, A64_SP, ctx->stack_size), ctx); cur_offset = ctx->idx - idx0; if (cur_offset != PROLOGUE_OFFSET) { @@ -252,8 +255,9 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) */ off = offsetof(struct bpf_array, ptrs); emit_a64_mov_i64(tmp, off, ctx); - emit(A64_LDR64(tmp, r2, tmp), ctx); - emit(A64_LDR64(prg, tmp, r3), ctx); + emit(A64_ADD(1, tmp, r2, tmp), ctx); + emit(A64_LSL(1, prg, r3, 3), ctx); + emit(A64_LDR64(prg, tmp, prg), ctx); emit(A64_CBZ(1, prg, jmp_offset), ctx); /* goto *(prog->bpf_func + prologue_size); */ @@ -286,7 +290,7 @@ static void build_epilogue(struct jit_ctx *ctx) const u8 fp = bpf2a64[BPF_REG_FP]; /* We're done with BPF stack */ - emit(A64_ADD_I(1, A64_SP, A64_SP, STACK_SIZE), ctx); + emit(A64_ADD_I(1, A64_SP, A64_SP, ctx->stack_size), ctx); /* Restore fs (x25) and x26 */ emit(A64_POP(fp, A64_R(26), A64_SP), ctx); @@ -317,6 +321,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) const u8 src = bpf2a64[insn->src_reg]; const u8 tmp = bpf2a64[TMP_REG_1]; const u8 tmp2 = bpf2a64[TMP_REG_2]; + const u8 tmp3 = bpf2a64[TMP_REG_3]; const s16 off = insn->off; const s32 imm = insn->imm; const int i = insn - ctx->prog->insnsi; @@ -584,7 +589,7 @@ emit_cond_jmp: break; } /* tail call */ - case BPF_JMP | BPF_CALL | BPF_X: + case BPF_JMP | BPF_TAIL_CALL: if (emit_bpf_tail_call(ctx)) return -EFAULT; break; @@ -687,10 +692,10 @@ emit_cond_jmp: emit(A64_PRFM(tmp, PST, L1, STRM), ctx); emit(A64_LDXR(isdw, tmp2, tmp), ctx); emit(A64_ADD(isdw, tmp2, tmp2, src), ctx); - emit(A64_STXR(isdw, tmp2, tmp, tmp2), ctx); + emit(A64_STXR(isdw, tmp2, tmp, tmp3), ctx); jmp_offset = -3; check_imm19(jmp_offset); - emit(A64_CBNZ(0, tmp2, jmp_offset), ctx); + emit(A64_CBNZ(0, tmp3, jmp_offset), ctx); break; /* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + imm)) */ @@ -730,7 +735,7 @@ emit_cond_jmp: return -EINVAL; } emit_a64_mov_i64(r3, size, ctx); - emit(A64_SUB_I(1, r4, fp, STACK_SIZE), ctx); + emit(A64_SUB_I(1, r4, fp, ctx->stack_size), ctx); emit_a64_mov_i64(r5, (unsigned long)bpf_load_pointer, ctx); emit(A64_BLR(r5), ctx); emit(A64_MOV(1, r0, A64_R(0)), ctx); @@ -898,6 +903,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) bpf_jit_binary_lock_ro(header); prog->bpf_func = (void *)ctx.image; prog->jited = 1; + prog->jited_len = image_size; out_off: kfree(ctx.offset); |