arch/riscv/net/bpf_jit_core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Common functionality for RV32 and RV64 BPF JIT compilers
   4  *
   5  * Copyright (c) 2019 Björn Töpel <[email protected]>
   6  *
   7  */
   8
   9 #include <linux/bpf.h>
  10 #include <linux/filter.h>
  11 #include <linux/memory.h>
  12 #include <asm/patch.h>
  13 #include <asm/cfi.h>
  14 #include "bpf_jit.h"
  15
  16 /* Number of iterations to try until offsets converge. */
  17 #define NR_JIT_ITERATIONS       32
  18
  19 static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset)
  20 {
  21         const struct bpf_prog *prog = ctx->prog;
  22         int i;
  23
  24         for (i = 0; i < prog->len; i++) {
  25                 const struct bpf_insn *insn = &prog->insnsi[i];
  26                 int ret;
  27
  28                 ret = bpf_jit_emit_insn(insn, ctx, extra_pass);
  29                 /* BPF_LD | BPF_IMM | BPF_DW: skip the next instruction. */
  30                 if (ret > 0)
  31                         i++;
  32                 if (offset)
  33                         offset[i] = ctx->ninsns;
  34                 if (ret < 0)
  35                         return ret;
  36         }
  37         return 0;
  38 }
  39
  40 bool bpf_jit_needs_zext(void)
  41 {
  42         return true;
  43 }
  44
  45 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
  46 {
  47         unsigned int prog_size = 0, extable_size = 0;
  48         bool tmp_blinded = false, extra_pass = false;
  49         struct bpf_prog *tmp, *orig_prog = prog;
  50         int pass = 0, prev_ninsns = 0, i;
  51         struct rv_jit_data *jit_data;
  52         struct rv_jit_context *ctx;
  53
  54         if (!prog->jit_requested)
  55                 return orig_prog;
  56
  57         tmp = bpf_jit_blind_constants(prog);
  58         if (IS_ERR(tmp))
  59                 return orig_prog;
  60         if (tmp != prog) {
  61                 tmp_blinded = true;
  62                 prog = tmp;
  63         }
  64
  65         jit_data = prog->aux->jit_data;
  66         if (!jit_data) {
  67                 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
  68                 if (!jit_data) {
  69                         prog = orig_prog;
  70                         goto out;
  71                 }
  72                 prog->aux->jit_data = jit_data;
  73         }
  74
  75         ctx = &jit_data->ctx;
  76
  77         if (ctx->offset) {
  78                 extra_pass = true;
  79                 prog_size = sizeof(*ctx->insns) * ctx->ninsns;
  80                 goto skip_init_ctx;
  81         }
  82
  83         ctx->arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena);
  84         ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena);
  85         ctx->prog = prog;
  86         ctx->offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL);
  87         if (!ctx->offset) {
  88                 prog = orig_prog;
  89                 goto out_offset;
  90         }
  91
  92         if (build_body(ctx, extra_pass, NULL)) {
  93                 prog = orig_prog;
  94                 goto out_offset;
  95         }
  96
  97         for (i = 0; i < prog->len; i++) {
  98                 prev_ninsns += 32;
  99                 ctx->offset[i] = prev_ninsns;
 100         }
 101
 102         for (i = 0; i < NR_JIT_ITERATIONS; i++) {
 103                 pass++;
 104                 ctx->ninsns = 0;
 105
 106                 bpf_jit_build_prologue(ctx, bpf_is_subprog(prog));
 107                 ctx->prologue_len = ctx->ninsns;
 108
 109                 if (build_body(ctx, extra_pass, ctx->offset)) {
 110                         prog = orig_prog;
 111                         goto out_offset;
 112                 }
 113
 114                 ctx->epilogue_offset = ctx->ninsns;
 115                 bpf_jit_build_epilogue(ctx);
 116
 117                 if (ctx->ninsns == prev_ninsns) {
 118                         if (jit_data->header)
 119                                 break;
 120                         /* obtain the actual image size */
 121                         extable_size = prog->aux->num_exentries *
 122                                 sizeof(struct exception_table_entry);
 123                         prog_size = sizeof(*ctx->insns) * ctx->ninsns;
 124
 125                         jit_data->ro_header =
 126                                 bpf_jit_binary_pack_alloc(prog_size + extable_size,
 127                                                           &jit_data->ro_image, sizeof(u32),
 128                                                           &jit_data->header, &jit_data->image,
 129                                                           bpf_fill_ill_insns);
 130                         if (!jit_data->ro_header) {
 131                                 prog = orig_prog;
 132                                 goto out_offset;
 133                         }
 134
 135                         /*
 136                          * Use the image(RW) for writing the JITed instructions. But also save
 137                          * the ro_image(RX) for calculating the offsets in the image. The RW
 138                          * image will be later copied to the RX image from where the program
 139                          * will run. The bpf_jit_binary_pack_finalize() will do this copy in the
 140                          * final step.
 141                          */
 142                         ctx->ro_insns = (u16 *)jit_data->ro_image;
 143                         ctx->insns = (u16 *)jit_data->image;
 144                         /*
 145                          * Now, when the image is allocated, the image can
 146                          * potentially shrink more (auipc/jalr -> jal).
 147                          */
 148                 }
 149                 prev_ninsns = ctx->ninsns;
 150         }
 151
 152         if (i == NR_JIT_ITERATIONS) {
 153                 pr_err("bpf-jit: image did not converge in <%d passes!\n", i);
 154                 prog = orig_prog;
 155                 goto out_free_hdr;
 156         }
 157
 158         if (extable_size)
 159                 prog->aux->extable = (void *)ctx->ro_insns + prog_size;
 160
 161 skip_init_ctx:
 162         pass++;
 163         ctx->ninsns = 0;
 164         ctx->nexentries = 0;
 165
 166         bpf_jit_build_prologue(ctx, bpf_is_subprog(prog));
 167         if (build_body(ctx, extra_pass, NULL)) {
 168                 prog = orig_prog;
 169                 goto out_free_hdr;
 170         }
 171         bpf_jit_build_epilogue(ctx);
 172
 173         if (bpf_jit_enable > 1)
 174                 bpf_jit_dump(prog->len, prog_size, pass, ctx->insns);
 175
 176         prog->bpf_func = (void *)ctx->ro_insns + cfi_get_offset();
 177         prog->jited = 1;
 178         prog->jited_len = prog_size - cfi_get_offset();
 179
 180         if (!prog->is_func || extra_pass) {
 181                 if (WARN_ON(bpf_jit_binary_pack_finalize(prog, jit_data->ro_header,
 182                                                          jit_data->header))) {
 183                         /* ro_header has been freed */
 184                         jit_data->ro_header = NULL;
 185                         prog = orig_prog;
 186                         goto out_offset;
 187                 }
 188                 /*
 189                  * The instructions have now been copied to the ROX region from
 190                  * where they will execute.
 191                  * Write any modified data cache blocks out to memory and
 192                  * invalidate the corresponding blocks in the instruction cache.
 193                  */
 194                 bpf_flush_icache(jit_data->ro_header, ctx->ro_insns + ctx->ninsns);
 195                 for (i = 0; i < prog->len; i++)
 196                         ctx->offset[i] = ninsns_rvoff(ctx->offset[i]);
 197                 bpf_prog_fill_jited_linfo(prog, ctx->offset);
 198 out_offset:
 199                 kfree(ctx->offset);
 200                 kfree(jit_data);
 201                 prog->aux->jit_data = NULL;
 202         }
 203 out:
 204
 205         if (tmp_blinded)
 206                 bpf_jit_prog_release_other(prog, prog == orig_prog ?
 207                                            tmp : orig_prog);
 208         return prog;
 209
 210 out_free_hdr:
 211         if (jit_data->header) {
 212                 bpf_arch_text_copy(&jit_data->ro_header->size, &jit_data->header->size,
 213                                    sizeof(jit_data->header->size));
 214                 bpf_jit_binary_pack_free(jit_data->ro_header, jit_data->header);
 215         }
 216         goto out_offset;
 217 }
 218
 219 u64 bpf_jit_alloc_exec_limit(void)
 220 {
 221         return BPF_JIT_REGION_SIZE;
 222 }
 223
 224 void *bpf_arch_text_copy(void *dst, void *src, size_t len)
 225 {
 226         int ret;
 227
 228         mutex_lock(&text_mutex);
 229         ret = patch_text_nosync(dst, src, len);
 230         mutex_unlock(&text_mutex);
 231
 232         if (ret)
 233                 return ERR_PTR(-EINVAL);
 234
 235         return dst;
 236 }
 237
 238 int bpf_arch_text_invalidate(void *dst, size_t len)
 239 {
 240         int ret;
 241
 242         mutex_lock(&text_mutex);
 243         ret = patch_text_set_nosync(dst, 0, len);
 244         mutex_unlock(&text_mutex);
 245
 246         return ret;
 247 }
 248
 249 void bpf_jit_free(struct bpf_prog *prog)
 250 {
 251         if (prog->jited) {
 252                 struct rv_jit_data *jit_data = prog->aux->jit_data;
 253                 struct bpf_binary_header *hdr;
 254
 255                 /*
 256                  * If we fail the final pass of JIT (from jit_subprogs),
 257                  * the program may not be finalized yet. Call finalize here
 258                  * before freeing it.
 259                  */
 260                 if (jit_data) {
 261                         bpf_jit_binary_pack_finalize(prog, jit_data->ro_header, jit_data->header);
 262                         kfree(jit_data);
 263                 }
 264                 hdr = bpf_jit_binary_pack_hdr(prog);
 265                 bpf_jit_binary_pack_free(hdr, NULL);
 266                 WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog));
 267         }
 268
 269         bpf_prog_unlock_free(prog);
 270 }