arch/riscv/net/bpf_jit_core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Common functionality for RV32 and RV64 BPF JIT compilers
   4  *
   5  * Copyright (c) 2019 Björn Töpel <[email protected]>
   6  *
   7  */
   8
   9 #include <linux/bpf.h>
  10 #include <linux/filter.h>
  11 #include "bpf_jit.h"
  12
  13 /* Number of iterations to try until offsets converge. */
  14 #define NR_JIT_ITERATIONS       32
  15
  16 static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset)
  17 {
  18         const struct bpf_prog *prog = ctx->prog;
  19         int i;
  20
  21         for (i = 0; i < prog->len; i++) {
  22                 const struct bpf_insn *insn = &prog->insnsi[i];
  23                 int ret;
  24
  25                 ret = bpf_jit_emit_insn(insn, ctx, extra_pass);
  26                 /* BPF_LD | BPF_IMM | BPF_DW: skip the next instruction. */
  27                 if (ret > 0)
  28                         i++;
  29                 if (offset)
  30                         offset[i] = ctx->ninsns;
  31                 if (ret < 0)
  32                         return ret;
  33         }
  34         return 0;
  35 }
  36
  37 bool bpf_jit_needs_zext(void)
  38 {
  39         return true;
  40 }
  41
  42 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
  43 {
  44         unsigned int prog_size = 0, extable_size = 0;
  45         bool tmp_blinded = false, extra_pass = false;
  46         struct bpf_prog *tmp, *orig_prog = prog;
  47         int pass = 0, prev_ninsns = 0, prologue_len, i;
  48         struct rv_jit_data *jit_data;
  49         struct rv_jit_context *ctx;
  50
  51         if (!prog->jit_requested)
  52                 return orig_prog;
  53
  54         tmp = bpf_jit_blind_constants(prog);
  55         if (IS_ERR(tmp))
  56                 return orig_prog;
  57         if (tmp != prog) {
  58                 tmp_blinded = true;
  59                 prog = tmp;
  60         }
  61
  62         jit_data = prog->aux->jit_data;
  63         if (!jit_data) {
  64                 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
  65                 if (!jit_data) {
  66                         prog = orig_prog;
  67                         goto out;
  68                 }
  69                 prog->aux->jit_data = jit_data;
  70         }
  71
  72         ctx = &jit_data->ctx;
  73
  74         if (ctx->offset) {
  75                 extra_pass = true;
  76                 prog_size = sizeof(*ctx->insns) * ctx->ninsns;
  77                 goto skip_init_ctx;
  78         }
  79
  80         ctx->prog = prog;
  81         ctx->offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL);
  82         if (!ctx->offset) {
  83                 prog = orig_prog;
  84                 goto out_offset;
  85         }
  86         for (i = 0; i < prog->len; i++) {
  87                 prev_ninsns += 32;
  88                 ctx->offset[i] = prev_ninsns;
  89         }
  90
  91         for (i = 0; i < NR_JIT_ITERATIONS; i++) {
  92                 pass++;
  93                 ctx->ninsns = 0;
  94                 if (build_body(ctx, extra_pass, ctx->offset)) {
  95                         prog = orig_prog;
  96                         goto out_offset;
  97                 }
  98                 ctx->body_len = ctx->ninsns;
  99                 bpf_jit_build_prologue(ctx);
 100                 ctx->epilogue_offset = ctx->ninsns;
 101                 bpf_jit_build_epilogue(ctx);
 102
 103                 if (ctx->ninsns == prev_ninsns) {
 104                         if (jit_data->header)
 105                                 break;
 106                         /* obtain the actual image size */
 107                         extable_size = prog->aux->num_exentries *
 108                                 sizeof(struct exception_table_entry);
 109                         prog_size = sizeof(*ctx->insns) * ctx->ninsns;
 110
 111                         jit_data->header =
 112                                 bpf_jit_binary_alloc(prog_size + extable_size,
 113                                                      &jit_data->image,
 114                                                      sizeof(u32),
 115                                                      bpf_fill_ill_insns);
 116                         if (!jit_data->header) {
 117                                 prog = orig_prog;
 118                                 goto out_offset;
 119                         }
 120
 121                         ctx->insns = (u16 *)jit_data->image;
 122                         /*
 123                          * Now, when the image is allocated, the image can
 124                          * potentially shrink more (auipc/jalr -> jal).
 125                          */
 126                 }
 127                 prev_ninsns = ctx->ninsns;
 128         }
 129
 130         if (i == NR_JIT_ITERATIONS) {
 131                 pr_err("bpf-jit: image did not converge in <%d passes!\n", i);
 132                 if (jit_data->header)
 133                         bpf_jit_binary_free(jit_data->header);
 134                 prog = orig_prog;
 135                 goto out_offset;
 136         }
 137
 138         if (extable_size)
 139                 prog->aux->extable = (void *)ctx->insns + prog_size;
 140
 141 skip_init_ctx:
 142         pass++;
 143         ctx->ninsns = 0;
 144         ctx->nexentries = 0;
 145
 146         bpf_jit_build_prologue(ctx);
 147         if (build_body(ctx, extra_pass, NULL)) {
 148                 bpf_jit_binary_free(jit_data->header);
 149                 prog = orig_prog;
 150                 goto out_offset;
 151         }
 152         bpf_jit_build_epilogue(ctx);
 153
 154         if (bpf_jit_enable > 1)
 155                 bpf_jit_dump(prog->len, prog_size, pass, ctx->insns);
 156
 157         prog->bpf_func = (void *)ctx->insns;
 158         prog->jited = 1;
 159         prog->jited_len = prog_size;
 160
 161         bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns);
 162
 163         if (!prog->is_func || extra_pass) {
 164                 bpf_jit_binary_lock_ro(jit_data->header);
 165                 prologue_len = ctx->epilogue_offset - ctx->body_len;
 166                 for (i = 0; i < prog->len; i++)
 167                         ctx->offset[i] = ninsns_rvoff(prologue_len +
 168                                                       ctx->offset[i]);
 169                 bpf_prog_fill_jited_linfo(prog, ctx->offset);
 170 out_offset:
 171                 kfree(ctx->offset);
 172                 kfree(jit_data);
 173                 prog->aux->jit_data = NULL;
 174         }
 175 out:
 176
 177         if (tmp_blinded)
 178                 bpf_jit_prog_release_other(prog, prog == orig_prog ?
 179                                            tmp : orig_prog);
 180         return prog;
 181 }
 182
 183 u64 bpf_jit_alloc_exec_limit(void)
 184 {
 185         return BPF_JIT_REGION_SIZE;
 186 }
 187
 188 void *bpf_jit_alloc_exec(unsigned long size)
 189 {
 190         return __vmalloc_node_range(size, PAGE_SIZE, BPF_JIT_REGION_START,
 191                                     BPF_JIT_REGION_END, GFP_KERNEL,
 192                                     PAGE_KERNEL, 0, NUMA_NO_NODE,
 193                                     __builtin_return_address(0));
 194 }
 195
 196 void bpf_jit_free_exec(void *addr)
 197 {
 198         return vfree(addr);
 199 }