arch/riscv/net/bpf_jit_core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Common functionality for RV32 and RV64 BPF JIT compilers
   4  *
   5  * Copyright (c) 2019 Björn Töpel <[email protected]>
   6  *
   7  */
   8
   9 #include <linux/bpf.h>
  10 #include <linux/filter.h>
  11 #include "bpf_jit.h"
  12
  13 /* Number of iterations to try until offsets converge. */
  14 #define NR_JIT_ITERATIONS       16
  15
  16 static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset)
  17 {
  18         const struct bpf_prog *prog = ctx->prog;
  19         int i;
  20
  21         for (i = 0; i < prog->len; i++) {
  22                 const struct bpf_insn *insn = &prog->insnsi[i];
  23                 int ret;
  24
  25                 ret = bpf_jit_emit_insn(insn, ctx, extra_pass);
  26                 /* BPF_LD | BPF_IMM | BPF_DW: skip the next instruction. */
  27                 if (ret > 0)
  28                         i++;
  29                 if (offset)
  30                         offset[i] = ctx->ninsns;
  31                 if (ret < 0)
  32                         return ret;
  33         }
  34         return 0;
  35 }
  36
  37 bool bpf_jit_needs_zext(void)
  38 {
  39         return true;
  40 }
  41
  42 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
  43 {
  44         bool tmp_blinded = false, extra_pass = false;
  45         struct bpf_prog *tmp, *orig_prog = prog;
  46         int pass = 0, prev_ninsns = 0, i;
  47         struct rv_jit_data *jit_data;
  48         struct rv_jit_context *ctx;
  49         unsigned int image_size = 0;
  50
  51         if (!prog->jit_requested)
  52                 return orig_prog;
  53
  54         tmp = bpf_jit_blind_constants(prog);
  55         if (IS_ERR(tmp))
  56                 return orig_prog;
  57         if (tmp != prog) {
  58                 tmp_blinded = true;
  59                 prog = tmp;
  60         }
  61
  62         jit_data = prog->aux->jit_data;
  63         if (!jit_data) {
  64                 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
  65                 if (!jit_data) {
  66                         prog = orig_prog;
  67                         goto out;
  68                 }
  69                 prog->aux->jit_data = jit_data;
  70         }
  71
  72         ctx = &jit_data->ctx;
  73
  74         if (ctx->offset) {
  75                 extra_pass = true;
  76                 image_size = sizeof(*ctx->insns) * ctx->ninsns;
  77                 goto skip_init_ctx;
  78         }
  79
  80         ctx->prog = prog;
  81         ctx->offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL);
  82         if (!ctx->offset) {
  83                 prog = orig_prog;
  84                 goto out_offset;
  85         }
  86         for (i = 0; i < prog->len; i++) {
  87                 prev_ninsns += 32;
  88                 ctx->offset[i] = prev_ninsns;
  89         }
  90
  91         for (i = 0; i < NR_JIT_ITERATIONS; i++) {
  92                 pass++;
  93                 ctx->ninsns = 0;
  94                 if (build_body(ctx, extra_pass, ctx->offset)) {
  95                         prog = orig_prog;
  96                         goto out_offset;
  97                 }
  98                 bpf_jit_build_prologue(ctx);
  99                 ctx->epilogue_offset = ctx->ninsns;
 100                 bpf_jit_build_epilogue(ctx);
 101
 102                 if (ctx->ninsns == prev_ninsns) {
 103                         if (jit_data->header)
 104                                 break;
 105
 106                         image_size = sizeof(*ctx->insns) * ctx->ninsns;
 107                         jit_data->header =
 108                                 bpf_jit_binary_alloc(image_size,
 109                                                      &jit_data->image,
 110                                                      sizeof(u32),
 111                                                      bpf_fill_ill_insns);
 112                         if (!jit_data->header) {
 113                                 prog = orig_prog;
 114                                 goto out_offset;
 115                         }
 116
 117                         ctx->insns = (u16 *)jit_data->image;
 118                         /*
 119                          * Now, when the image is allocated, the image can
 120                          * potentially shrink more (auipc/jalr -> jal).
 121                          */
 122                 }
 123                 prev_ninsns = ctx->ninsns;
 124         }
 125
 126         if (i == NR_JIT_ITERATIONS) {
 127                 pr_err("bpf-jit: image did not converge in <%d passes!\n", i);
 128                 bpf_jit_binary_free(jit_data->header);
 129                 prog = orig_prog;
 130                 goto out_offset;
 131         }
 132
 133 skip_init_ctx:
 134         pass++;
 135         ctx->ninsns = 0;
 136
 137         bpf_jit_build_prologue(ctx);
 138         if (build_body(ctx, extra_pass, NULL)) {
 139                 bpf_jit_binary_free(jit_data->header);
 140                 prog = orig_prog;
 141                 goto out_offset;
 142         }
 143         bpf_jit_build_epilogue(ctx);
 144
 145         if (bpf_jit_enable > 1)
 146                 bpf_jit_dump(prog->len, image_size, pass, ctx->insns);
 147
 148         prog->bpf_func = (void *)ctx->insns;
 149         prog->jited = 1;
 150         prog->jited_len = image_size;
 151
 152         bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns);
 153
 154         if (!prog->is_func || extra_pass) {
 155 out_offset:
 156                 kfree(ctx->offset);
 157                 kfree(jit_data);
 158                 prog->aux->jit_data = NULL;
 159         }
 160 out:
 161
 162         if (tmp_blinded)
 163                 bpf_jit_prog_release_other(prog, prog == orig_prog ?
 164                                            tmp : orig_prog);
 165         return prog;
 166 }