Merge tag 'x86-cleanups-2024-01-08' of git://git.kernel.org/pub/scm/linux/kernel...

[J-linux.git] / arch / x86 / net / bpf_jit_comp.c

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 75ae6e5777e40c91ebdf42d53f5cf8837d84e8f2..144a5839acfe5df645d378f7a70c3c9ad12f1f40 100644 (file)
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -3025,3 +3025,49 @@ void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp
 #endif
        WARN(1, "verification of programs using bpf_throw should have failed\n");
 }
+
+void bpf_arch_poke_desc_update(struct bpf_jit_poke_descriptor *poke,
+                              struct bpf_prog *new, struct bpf_prog *old)
+{
+       u8 *old_addr, *new_addr, *old_bypass_addr;
+       int ret;
+
+       old_bypass_addr = old ? NULL : poke->bypass_addr;
+       old_addr = old ? (u8 *)old->bpf_func + poke->adj_off : NULL;
+       new_addr = new ? (u8 *)new->bpf_func + poke->adj_off : NULL;
+
+       /*
+        * On program loading or teardown, the program's kallsym entry
+        * might not be in place, so we use __bpf_arch_text_poke to skip
+        * the kallsyms check.
+        */
+       if (new) {
+               ret = __bpf_arch_text_poke(poke->tailcall_target,
+                                          BPF_MOD_JUMP,
+                                          old_addr, new_addr);
+               BUG_ON(ret < 0);
+               if (!old) {
+                       ret = __bpf_arch_text_poke(poke->tailcall_bypass,
+                                                  BPF_MOD_JUMP,
+                                                  poke->bypass_addr,
+                                                  NULL);
+                       BUG_ON(ret < 0);
+               }
+       } else {
+               ret = __bpf_arch_text_poke(poke->tailcall_bypass,
+                                          BPF_MOD_JUMP,
+                                          old_bypass_addr,
+                                          poke->bypass_addr);
+               BUG_ON(ret < 0);
+               /* let other CPUs finish the execution of program
+                * so that it will not possible to expose them
+                * to invalid nop, stack unwind, nop state
+                */
+               if (!ret)
+                       synchronize_rcu();
+               ret = __bpf_arch_text_poke(poke->tailcall_target,
+                                          BPF_MOD_JUMP,
+                                          old_addr, NULL);
+               BUG_ON(ret < 0);
+       }
+}