livepatch: Use the default ftrace_ops instead of REGS when ARGS is available

When CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS is available, the ftrace call
will be able to set the ip of the calling function. This will improve the
performance of live kernel patching where it does not need all the regs to
be stored just to change the instruction pointer.

If all archs that support live kernel patching also support
HAVE_DYNAMIC_FTRACE_WITH_ARGS, then the architecture specific function
klp_arch_set_pc() could be made generic.

It is possible that an arch can support HAVE_DYNAMIC_FTRACE_WITH_ARGS but
not HAVE_DYNAMIC_FTRACE_WITH_REGS and then have access to live patching.

Cc: Josh Poimboeuf <[email protected]>
Cc: Jiri Kosina <[email protected]>
Cc: [email protected]
Acked-by: Peter Zijlstra (Intel) <[email protected]>
Acked-by: Miroslav Benes <[email protected]>
Signed-off-by: Steven Rostedt (VMware) <[email protected]>

diff --git a/arch/powerpc/include/asm/livepatch.h b/arch/powerpc/include/asm/livepatch.h
index 4a3d5d25fed5c0073fe084796d3a7bfd65d18937..ae25e6e72997953a1403ed9c5f02e05592cd8a57 100644 (file)
--- a/arch/powerpc/include/asm/livepatch.h
+++ b/arch/powerpc/include/asm/livepatch.h
@@ -12,8 +12,10 @@
 #include <linux/sched/task_stack.h>
 
 #ifdef CONFIG_LIVEPATCH
-static inline void klp_arch_set_pc(struct pt_regs *regs, unsigned long ip)
+static inline void klp_arch_set_pc(struct ftrace_regs *fregs, unsigned long ip)
 {
+       struct pt_regs *regs = ftrace_get_regs(fregs);
+
        regs->nip = ip;
 }
 

diff --git a/arch/s390/include/asm/livepatch.h b/arch/s390/include/asm/livepatch.h
index 818612b784cda2a77960461d1eb0c720c29f925d..d578a8c7667654162eb7dcf6d55bbdb82b64802f 100644 (file)
--- a/arch/s390/include/asm/livepatch.h
+++ b/arch/s390/include/asm/livepatch.h
@@ -11,10 +11,13 @@
 #ifndef ASM_LIVEPATCH_H
 #define ASM_LIVEPATCH_H
 
+#include <linux/ftrace.h>
 #include <asm/ptrace.h>
 
-static inline void klp_arch_set_pc(struct pt_regs *regs, unsigned long ip)
+static inline void klp_arch_set_pc(struct ftrace_regs *fregs, unsigned long ip)
 {
+       struct pt_regs *regs = ftrace_get_regs(fregs);
+
        regs->psw.addr = ip;
 }
 

diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h
index e00fe88146e0b7443396caf2ea0fd0b41c78eff6..9f3130f408077703e7156a3ce127e4c25a990dc0 100644 (file)
--- a/arch/x86/include/asm/ftrace.h
+++ b/arch/x86/include/asm/ftrace.h
@@ -54,6 +54,9 @@ arch_ftrace_get_regs(struct ftrace_regs *fregs)
                return NULL;
        return &fregs->regs;
 }
+
+#define ftrace_instruction_pointer_set(fregs, _ip)     \
+       do { (fregs)->regs.ip = (_ip); } while (0)
 #endif
 
 #ifdef CONFIG_DYNAMIC_FTRACE

diff --git a/arch/x86/include/asm/livepatch.h b/arch/x86/include/asm/livepatch.h
index 1fde1ab6559e91926a87da4dbde93ca1c02b4d20..7c5cc6660e4b65f036fdb4318fe87f6a800eed3f 100644 (file)
--- a/arch/x86/include/asm/livepatch.h
+++ b/arch/x86/include/asm/livepatch.h
@@ -12,9 +12,9 @@
 #include <asm/setup.h>
 #include <linux/ftrace.h>
 
-static inline void klp_arch_set_pc(struct pt_regs *regs, unsigned long ip)
+static inline void klp_arch_set_pc(struct ftrace_regs *fregs, unsigned long ip)
 {
-       regs->ip = ip;
+       ftrace_instruction_pointer_set(fregs, ip);
 }
 
 #endif /* _ASM_X86_LIVEPATCH_H */

diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S
index 60e3b64f5ea678d9bfaa89f944a22ff35d73a212..0d54099c2a3a3156b4a18520091eb59966ef9cfe 100644 (file)
--- a/arch/x86/kernel/ftrace_64.S
+++ b/arch/x86/kernel/ftrace_64.S
@@ -157,6 +157,10 @@ SYM_INNER_LABEL(ftrace_caller_op_ptr, SYM_L_GLOBAL)
 SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
        call ftrace_stub
 
+       /* Handlers can change the RIP */
+       movq RIP(%rsp), %rax
+       movq %rax, MCOUNT_REG_SIZE(%rsp)
+
        restore_mcount_regs
 
        /*

diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index 588ea7023a7a3db5cd509509667c5b4cf702c6fc..9a8ce28e4485008c5049de5312da6eb213952fa3 100644 (file)
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -97,6 +97,13 @@ struct ftrace_regs {
 };
 #define arch_ftrace_get_regs(fregs) (&(fregs)->regs)
 
+/*
+ * ftrace_instruction_pointer_set() is to be defined by the architecture
+ * if to allow setting of the instruction pointer from the ftrace_regs
+ * when HAVE_DYNAMIC_FTRACE_WITH_ARGS is set and it supports
+ * live kernel patching.
+ */
+#define ftrace_instruction_pointer_set(fregs, ip) do { } while (0)
 #endif /* CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS */
 
 static __always_inline struct pt_regs *ftrace_get_regs(struct ftrace_regs *fregs)

diff --git a/kernel/livepatch/Kconfig b/kernel/livepatch/Kconfig
index 54102deb50ba9846e3ef4baee4d62f2f90a94155..53d51ed619a3d53a583c86a0ac10e363877296c8 100644 (file)
--- a/kernel/livepatch/Kconfig
+++ b/kernel/livepatch/Kconfig
@@ -6,7 +6,7 @@ config HAVE_LIVEPATCH
 
 config LIVEPATCH
        bool "Kernel Live Patching"
-       depends on DYNAMIC_FTRACE_WITH_REGS
+       depends on DYNAMIC_FTRACE_WITH_REGS || DYNAMIC_FTRACE_WITH_ARGS
        depends on MODULES
        depends on SYSFS
        depends on KALLSYMS_ALL

diff --git a/kernel/livepatch/patch.c b/kernel/livepatch/patch.c
index f89f9e7e9b07453f8245065346ef7741246942a4..e8029aea67f1d80f7bf488b85e0a1df1e5a75437 100644 (file)
--- a/kernel/livepatch/patch.c
+++ b/kernel/livepatch/patch.c
@@ -42,7 +42,6 @@ static void notrace klp_ftrace_handler(unsigned long ip,
                                       struct ftrace_ops *fops,
                                       struct ftrace_regs *fregs)
 {
-       struct pt_regs *regs = ftrace_get_regs(fregs);
        struct klp_ops *ops;
        struct klp_func *func;
        int patch_state;
@@ -118,7 +117,7 @@ static void notrace klp_ftrace_handler(unsigned long ip,
        if (func->nop)
                goto unlock;
 
-       klp_arch_set_pc(regs, (unsigned long)func->new_func);
+       klp_arch_set_pc(fregs, (unsigned long)func->new_func);
 
 unlock:
        preempt_enable_notrace();
@@ -200,8 +199,10 @@ static int klp_patch_func(struct klp_func *func)
                        return -ENOMEM;
 
                ops->fops.func = klp_ftrace_handler;
-               ops->fops.flags = FTRACE_OPS_FL_SAVE_REGS |
-                                 FTRACE_OPS_FL_DYNAMIC |
+               ops->fops.flags = FTRACE_OPS_FL_DYNAMIC |
+#ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS
+                                 FTRACE_OPS_FL_SAVE_REGS |
+#endif
                                  FTRACE_OPS_FL_IPMODIFY |
                                  FTRACE_OPS_FL_PERMANENT;