jump_label: adjust inline asm to be consistent

To avoid duplication of inline asm between C and Rust, we need to
import the inline asm from the relevant `jump_label.h` header into Rust.
To make that easier, this patch updates the header files to expose the
inline asm via a new ARCH_STATIC_BRANCH_ASM macro.

The header files are all updated to define a ARCH_STATIC_BRANCH_ASM that
takes the same arguments in a consistent order so that Rust can use the
same logic for every architecture.

Cc: Masami Hiramatsu <[email protected]>
Cc: Mathieu Desnoyers <[email protected]>
Cc: Josh Poimboeuf <[email protected]>
Cc: Jason Baron <[email protected]>
Cc: Ard Biesheuvel <[email protected]>
Cc: Alex Gaynor <[email protected]>
Cc: Wedson Almeida Filho <[email protected]>
Cc: Boqun Feng <[email protected]>
Cc: Gary Guo <[email protected]>
Cc: " =?utf-8?q?Bj=C3=B6rn_Roy_Baron?= " <[email protected]>
Cc: Benno Lossin <[email protected]>
Cc: Andreas Hindborg <[email protected]>
Cc: Arnd Bergmann <[email protected]>
Cc: Thomas Gleixner <[email protected]>
Cc: Ingo Molnar <[email protected]>
Cc: Borislav Petkov <[email protected]>
Cc: Dave Hansen <[email protected]>
Cc: "H. Peter Anvin" <[email protected]>
Cc: Sean Christopherson <[email protected]>
Cc: Uros Bizjak <[email protected]>
Cc: Will Deacon <[email protected]>
Cc: Marc Zyngier <[email protected]>
Cc: Oliver Upton <[email protected]>
Cc: Mark Rutland <[email protected]>
Cc: Ryan Roberts <[email protected]>
Cc: Fuad Tabba <[email protected]>
Cc: Paul Walmsley <[email protected]>
Cc: Palmer Dabbelt <[email protected]>
Cc: Albert Ou <[email protected]>
Cc: Anup Patel <[email protected]>
Cc: Andrew Jones <[email protected]>
Cc: Alexandre Ghiti <[email protected]>
Cc: Conor Dooley <[email protected]>
Cc: Samuel Holland <[email protected]>
Cc: Huacai Chen <[email protected]>
Cc: WANG Xuerui <[email protected]>
Cc: Bibo Mao <[email protected]>
Cc: Tiezhu Yang <[email protected]>
Cc: Andrew Morton <[email protected]>
Cc: Tianrui Zhao <[email protected]>
Cc: Palmer Dabbelt <[email protected]>
Link: https://lore.kernel.org/[email protected]
Suggested-by: Peter Zijlstra (Intel) <[email protected]>
Co-developed-by: Miguel Ojeda <[email protected]>
Signed-off-by: Miguel Ojeda <[email protected]>
Acked-by: Peter Zijlstra (Intel) <[email protected]>
Acked-by: Catalin Marinas <[email protected]>
Acked-by: Palmer Dabbelt <[email protected]> # RISC-V
Signed-off-by: Alice Ryhl <[email protected]>
Signed-off-by: Steven Rostedt (Google) <[email protected]>

diff --git a/arch/arm/include/asm/jump_label.h b/arch/arm/include/asm/jump_label.h
index e4eb54f6cd9fef41fecad56e25c4136e75455756..a35aba7f548ccb81d6b8a4bf6543be1dec2cf3a3 100644 (file)
--- a/arch/arm/include/asm/jump_label.h
+++ b/arch/arm/include/asm/jump_label.h
@@ -9,13 +9,17 @@
 
 #define JUMP_LABEL_NOP_SIZE 4
 
+/* This macro is also expanded on the Rust side. */
+#define ARCH_STATIC_BRANCH_ASM(key, label)             \
+       "1:\n\t"                                        \
+       WASM(nop) "\n\t"                                \
+       ".pushsection __jump_table,  \"aw\"\n\t"        \
+       ".word 1b, " label ", " key "\n\t"              \
+       ".popsection\n\t"                               \
+
 static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
 {
-       asm goto("1:\n\t"
-                WASM(nop) "\n\t"
-                ".pushsection __jump_table,  \"aw\"\n\t"
-                ".word 1b, %l[l_yes], %c0\n\t"
-                ".popsection\n\t"
+       asm goto(ARCH_STATIC_BRANCH_ASM("%c0", "%l[l_yes]")
                 : :  "i" (&((char *)key)[branch]) :  : l_yes);
 
        return false;

diff --git a/arch/arm64/include/asm/jump_label.h b/arch/arm64/include/asm/jump_label.h
index a0a5bbae7229e8a791748ebaa193c1ccd118ff1c..424ed421cd97974dca19adc9de71cb6c5f8d64dc 100644 (file)
--- a/arch/arm64/include/asm/jump_label.h
+++ b/arch/arm64/include/asm/jump_label.h
@@ -19,10 +19,14 @@
 #define JUMP_TABLE_ENTRY(key, label)                   \
        ".pushsection   __jump_table, \"aw\"\n\t"       \
        ".align         3\n\t"                          \
-       ".long          1b - ., %l["#label"] - .\n\t"   \
-       ".quad          %c0 - .\n\t"                    \
-       ".popsection\n\t"                               \
-       :  :  "i"(key) :  : label
+       ".long          1b - ., " label " - .\n\t"      \
+       ".quad          " key " - .\n\t"                \
+       ".popsection\n\t"
+
+/* This macro is also expanded on the Rust side. */
+#define ARCH_STATIC_BRANCH_ASM(key, label)             \
+       "1:     nop\n\t"                                \
+       JUMP_TABLE_ENTRY(key, label)
 
 static __always_inline bool arch_static_branch(struct static_key * const key,
                                               const bool branch)
@@ -30,8 +34,8 @@ static __always_inline bool arch_static_branch(struct static_key * const key,
        char *k = &((char *)key)[branch];
 
        asm goto(
-               "1:     nop                                     \n\t"
-               JUMP_TABLE_ENTRY(k, l_yes)
+               ARCH_STATIC_BRANCH_ASM("%c0", "%l[l_yes]")
+               :  :  "i"(k) :  : l_yes
                );
 
        return false;
@@ -43,9 +47,11 @@ static __always_inline bool arch_static_branch_jump(struct static_key * const ke
                                                    const bool branch)
 {
        char *k = &((char *)key)[branch];
+
        asm goto(
                "1:     b               %l[l_yes]               \n\t"
-               JUMP_TABLE_ENTRY(k, l_yes)
+               JUMP_TABLE_ENTRY("%c0", "%l[l_yes]")
+               :  :  "i"(k) :  : l_yes
                );
        return false;
 l_yes:

diff --git a/arch/loongarch/include/asm/jump_label.h b/arch/loongarch/include/asm/jump_label.h
index 29acfe3de3faae797beca198e58f9e5a5e570bbc..8a924bd69d19663d6e769db6bf40a29d55570af4 100644 (file)
--- a/arch/loongarch/include/asm/jump_label.h
+++ b/arch/loongarch/include/asm/jump_label.h
@@ -13,18 +13,22 @@
 
 #define JUMP_LABEL_NOP_SIZE    4
 
-#define JUMP_TABLE_ENTRY                               \
+/* This macro is also expanded on the Rust side. */
+#define JUMP_TABLE_ENTRY(key, label)                   \
         ".pushsection  __jump_table, \"aw\"    \n\t"   \
         ".align        3                       \n\t"   \
-        ".long         1b - ., %l[l_yes] - .   \n\t"   \
-        ".quad         %0 - .                  \n\t"   \
+        ".long         1b - ., " label " - .   \n\t"   \
+        ".quad         " key " - .             \n\t"   \
         ".popsection                           \n\t"
 
+#define ARCH_STATIC_BRANCH_ASM(key, label)             \
+       "1:     nop                             \n\t"   \
+       JUMP_TABLE_ENTRY(key, label)
+
 static __always_inline bool arch_static_branch(struct static_key * const key, const bool branch)
 {
        asm goto(
-               "1:     nop                     \n\t"
-               JUMP_TABLE_ENTRY
+               ARCH_STATIC_BRANCH_ASM("%0", "%l[l_yes]")
                :  :  "i"(&((char *)key)[branch]) :  : l_yes);
 
        return false;
@@ -37,7 +41,7 @@ static __always_inline bool arch_static_branch_jump(struct static_key * const ke
 {
        asm goto(
                "1:     b       %l[l_yes]       \n\t"
-               JUMP_TABLE_ENTRY
+               JUMP_TABLE_ENTRY("%0", "%l[l_yes]")
                :  :  "i"(&((char *)key)[branch]) :  : l_yes);
 
        return false;

diff --git a/arch/riscv/include/asm/jump_label.h b/arch/riscv/include/asm/jump_label.h
index 1c768d02bd0c2a9559c373223c0bc11392b38908..87a71cc6d146cef9c8eabc9e03e2fb270b697851 100644 (file)
--- a/arch/riscv/include/asm/jump_label.h
+++ b/arch/riscv/include/asm/jump_label.h
@@ -16,21 +16,28 @@
 
 #define JUMP_LABEL_NOP_SIZE 4
 
+#define JUMP_TABLE_ENTRY(key, label)                   \
+       ".pushsection   __jump_table, \"aw\"    \n\t"   \
+       ".align         " RISCV_LGPTR "         \n\t"   \
+       ".long          1b - ., " label " - .   \n\t"   \
+       "" RISCV_PTR "  " key " - .             \n\t"   \
+       ".popsection                            \n\t"
+
+/* This macro is also expanded on the Rust side. */
+#define ARCH_STATIC_BRANCH_ASM(key, label)             \
+       "       .align          2               \n\t"   \
+       "       .option push                    \n\t"   \
+       "       .option norelax                 \n\t"   \
+       "       .option norvc                   \n\t"   \
+       "1:     nop                             \n\t"   \
+       "       .option pop                     \n\t"   \
+       JUMP_TABLE_ENTRY(key, label)
+
 static __always_inline bool arch_static_branch(struct static_key * const key,
                                               const bool branch)
 {
        asm goto(
-               "       .align          2                       \n\t"
-               "       .option push                            \n\t"
-               "       .option norelax                         \n\t"
-               "       .option norvc                           \n\t"
-               "1:     nop                                     \n\t"
-               "       .option pop                             \n\t"
-               "       .pushsection    __jump_table, \"aw\"    \n\t"
-               "       .align          " RISCV_LGPTR "         \n\t"
-               "       .long           1b - ., %l[label] - .   \n\t"
-               "       " RISCV_PTR "   %0 - .                  \n\t"
-               "       .popsection                             \n\t"
+               ARCH_STATIC_BRANCH_ASM("%0", "%l[label]")
                :  :  "i"(&((char *)key)[branch]) :  : label);
 
        return false;
@@ -38,21 +45,20 @@ label:
        return true;
 }
 
+#define ARCH_STATIC_BRANCH_JUMP_ASM(key, label)                \
+       "       .align          2               \n\t"   \
+       "       .option push                    \n\t"   \
+       "       .option norelax                 \n\t"   \
+       "       .option norvc                   \n\t"   \
+       "1:     j       " label "               \n\t" \
+       "       .option pop                     \n\t"   \
+       JUMP_TABLE_ENTRY(key, label)
+
 static __always_inline bool arch_static_branch_jump(struct static_key * const key,
                                                    const bool branch)
 {
        asm goto(
-               "       .align          2                       \n\t"
-               "       .option push                            \n\t"
-               "       .option norelax                         \n\t"
-               "       .option norvc                           \n\t"
-               "1:     j               %l[label]               \n\t"
-               "       .option pop                             \n\t"
-               "       .pushsection    __jump_table, \"aw\"    \n\t"
-               "       .align          " RISCV_LGPTR "         \n\t"
-               "       .long           1b - ., %l[label] - .   \n\t"
-               "       " RISCV_PTR "   %0 - .                  \n\t"
-               "       .popsection                             \n\t"
+               ARCH_STATIC_BRANCH_JUMP_ASM("%0", "%l[label]")
                :  :  "i"(&((char *)key)[branch]) :  : label);
 
        return false;

diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h
index cbbef32517f0049a3df51842162032ff1946e901..3f1c1d6c0da12d1ce5d72868d66c5857e9878056 100644 (file)
--- a/arch/x86/include/asm/jump_label.h
+++ b/arch/x86/include/asm/jump_label.h
@@ -12,35 +12,28 @@
 #include <linux/stringify.h>
 #include <linux/types.h>
 
-#define JUMP_TABLE_ENTRY                               \
+#define JUMP_TABLE_ENTRY(key, label)                   \
        ".pushsection __jump_table,  \"aw\" \n\t"       \
        _ASM_ALIGN "\n\t"                               \
        ".long 1b - . \n\t"                             \
-       ".long %l[l_yes] - . \n\t"                      \
-       _ASM_PTR "%c0 + %c1 - .\n\t"                    \
+       ".long " label " - . \n\t"                      \
+       _ASM_PTR " " key " - . \n\t"                    \
        ".popsection \n\t"
 
+/* This macro is also expanded on the Rust side. */
 #ifdef CONFIG_HAVE_JUMP_LABEL_HACK
-
-static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
-{
-       asm goto("1:"
-               "jmp %l[l_yes] # objtool NOPs this \n\t"
-               JUMP_TABLE_ENTRY
-               : :  "i" (key), "i" (2 | branch) : : l_yes);
-
-       return false;
-l_yes:
-       return true;
-}
-
+#define ARCH_STATIC_BRANCH_ASM(key, label)             \
+       "1: jmp " label " # objtool NOPs this \n\t"     \
+       JUMP_TABLE_ENTRY(key " + 2", label)
 #else /* !CONFIG_HAVE_JUMP_LABEL_HACK */
+#define ARCH_STATIC_BRANCH_ASM(key, label)             \
+       "1: .byte " __stringify(BYTES_NOP5) "\n\t"      \
+       JUMP_TABLE_ENTRY(key, label)
+#endif /* CONFIG_HAVE_JUMP_LABEL_HACK */
 
 static __always_inline bool arch_static_branch(struct static_key * const key, const bool branch)
 {
-       asm goto("1:"
-               ".byte " __stringify(BYTES_NOP5) "\n\t"
-               JUMP_TABLE_ENTRY
+       asm goto(ARCH_STATIC_BRANCH_ASM("%c0 + %c1", "%l[l_yes]")
                : :  "i" (key), "i" (branch) : : l_yes);
 
        return false;
@@ -48,13 +41,11 @@ l_yes:
        return true;
 }
 
-#endif /* CONFIG_HAVE_JUMP_LABEL_HACK */
-
 static __always_inline bool arch_static_branch_jump(struct static_key * const key, const bool branch)
 {
        asm goto("1:"
                "jmp %l[l_yes]\n\t"
-               JUMP_TABLE_ENTRY
+               JUMP_TABLE_ENTRY("%c0 + %c1", "%l[l_yes]")
                : :  "i" (key), "i" (branch) : : l_yes);
 
        return false;