x86/percpu: Rewrite x86_this_cpu_test_bit() and friends as macros

Rewrite the whole family of x86_this_cpu_test_bit() functions
as macros, so standard __my_cpu_var() and raw_cpu_read() macros
can be used on percpu variables. This approach considerably
simplifies implementation of functions and also introduces
standard checks on accessed percpu variables.

No functional changes intended.

Signed-off-by: Uros Bizjak <[email protected]>
Signed-off-by: Ingo Molnar <[email protected]>
Cc: Linus Torvalds <[email protected]>
Link: https://lore.kernel.org/r/[email protected]

diff --git a/arch/um/include/asm/cpufeature.h b/arch/um/include/asm/cpufeature.h
index 66fe06db872f05bb775f0089a4f134f77563efe4..1eb8b834fbec31876bacbaa853d4a776b4686774 100644 (file)
--- a/arch/um/include/asm/cpufeature.h
+++ b/arch/um/include/asm/cpufeature.h
@@ -38,8 +38,7 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
 
 #define this_cpu_has(bit)                                              \
        (__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 :  \
 
 #define this_cpu_has(bit)                                              \
        (__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 :  \

-        x86_this_cpu_test_bit(bit,                                     \
-               (unsigned long __percpu *)&cpu_info.x86_capability))
+        x86_this_cpu_test_bit(bit, cpu_info.x86_capability))

 
 /*
  * This macro is for detection of features which need kernel
 
 /*
  * This macro is for detection of features which need kernel

diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index 42157ddcc09d436fef9684a813b3e9b9e666e9da..cd4c02dc9c8b5f49ea3a2421d18ebbabe3933edd 100644 (file)
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -127,8 +127,7 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
 
 #define this_cpu_has(bit)                                              \
        (__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 :  \
 
 #define this_cpu_has(bit)                                              \
        (__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 :  \

-        x86_this_cpu_test_bit(bit,                                     \
-               (unsigned long __percpu *)&cpu_info.x86_capability))
+        x86_this_cpu_test_bit(bit, cpu_info.x86_capability))

 
 /*
  * This macro is for detection of features which need kernel
 
 /*
  * This macro is for detection of features which need kernel

diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index cbfbbe836ee23b14bf0f966300a8e879107c3934..d6ff0db32209d21f40a2d4de7ac95b2e77b3a1a7 100644 (file)
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -96,7 +96,7 @@
 #endif /* CONFIG_SMP */
 
 #define __my_cpu_type(var)     typeof(var) __percpu_seg_override
 #endif /* CONFIG_SMP */
 
 #define __my_cpu_type(var)     typeof(var) __percpu_seg_override

-#define __my_cpu_ptr(ptr)      (__my_cpu_type(*ptr)*)(__force uintptr_t)(ptr)
+#define __my_cpu_ptr(ptr)      (__my_cpu_type(*(ptr))*)(__force uintptr_t)(ptr)

 #define __my_cpu_var(var)      (*__my_cpu_ptr(&(var)))
 #define __percpu_arg(x)                __percpu_prefix "%" #x
 #define __force_percpu_arg(x)  __force_percpu_prefix "%" #x
 #define __my_cpu_var(var)      (*__my_cpu_ptr(&(var)))
 #define __percpu_arg(x)                __percpu_prefix "%" #x
 #define __force_percpu_arg(x)  __force_percpu_prefix "%" #x


@@ -568,37 +568,29 @@ do {                                                                      \
 #define this_cpu_read_stable_8(pcp)    ({ BUILD_BUG(); (typeof(pcp))0; })
 #endif
 
 #define this_cpu_read_stable_8(pcp)    ({ BUILD_BUG(); (typeof(pcp))0; })
 #endif
 

-static __always_inline bool x86_this_cpu_constant_test_bit(unsigned int nr,
-                        const unsigned long __percpu *addr)
-{
-       unsigned long __percpu *a =
-               (unsigned long __percpu *)addr + nr / BITS_PER_LONG;
+#define x86_this_cpu_constant_test_bit(_nr, _var)                      \
+({                                                                     \
+       unsigned long __percpu *addr__ =                                \
+               (unsigned long __percpu *)&(_var) + ((_nr) / BITS_PER_LONG); \
+       !!((1UL << ((_nr) % BITS_PER_LONG)) & raw_cpu_read(*addr__));   \
+})

 
 

-#ifdef CONFIG_X86_64
-       return ((1UL << (nr % BITS_PER_LONG)) & raw_cpu_read_8(*a)) != 0;
-#else
-       return ((1UL << (nr % BITS_PER_LONG)) & raw_cpu_read_4(*a)) != 0;
-#endif
-}
-
-static inline bool x86_this_cpu_variable_test_bit(int nr,
-                        const unsigned long __percpu *addr)
-{
-       bool oldbit;
-
-       asm volatile("btl %[nr], " __percpu_arg([var])
-                       CC_SET(c)
-                       : CC_OUT(c) (oldbit)
-                       : [var] "m" (*__my_cpu_ptr((unsigned long __percpu *)(addr))),
-                         [nr] "Ir" (nr));
-
-       return oldbit;
-}
-
-#define x86_this_cpu_test_bit(nr, addr)                        \
-       (__builtin_constant_p((nr))                     \
-        ? x86_this_cpu_constant_test_bit((nr), (addr)) \
-        : x86_this_cpu_variable_test_bit((nr), (addr)))
+#define x86_this_cpu_variable_test_bit(_nr, _var)              \
+({                                                             \
+       bool oldbit;                                            \
+                                                               \
+       asm volatile("btl %[nr], " __percpu_arg([var])          \
+                    CC_SET(c)                                  \
+                    : CC_OUT(c) (oldbit)                       \
+                    : [var] "m" (__my_cpu_var(_var)),          \
+                      [nr] "rI" (_nr));                        \
+       oldbit;                                                 \
+})
+
+#define x86_this_cpu_test_bit(_nr, _var)                       \
+       (__builtin_constant_p(_nr)                              \
+        ? x86_this_cpu_constant_test_bit(_nr, _var)            \
+        : x86_this_cpu_variable_test_bit(_nr, _var))

 
 
 #include <asm-generic/percpu.h>
 
 
 #include <asm-generic/percpu.h>