x86/percpu: Introduce %rip-relative addressing to PER_CPU_VAR()

Introduce x86_64 %rip-relative addressing to the PER_CPU_VAR() macro.
Instructions using %rip-relative address operand are one byte shorter
than their absolute address counterparts and are also compatible with
position independent executable (-fpie) builds. The patch reduces
code size of a test kernel build by 150 bytes.

The PER_CPU_VAR() macro is intended to be applied to a symbol and should
not be used with register operands. Introduce the new __percpu macro and
use it in cmpxchg{8,16}b_emu.S instead.

Also add a missing function comment to this_cpu_cmpxchg8b_emu().

No functional changes intended.

Signed-off-by: Uros Bizjak <[email protected]>
Signed-off-by: Ingo Molnar <[email protected]>
Cc: [email protected]
Cc: Brian Gerst <[email protected]>
Cc: H. Peter Anvin <[email protected]>
Cc: Linus Torvalds <[email protected]>
Cc: Sean Christopherson <[email protected]>

diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index ac3220aeb779a439faec71b803bf36dc933a421b..bbcc1ca737f05445bc8e672555c6fadb68086818 100644 (file)
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -4,17 +4,21 @@
 
 #ifdef CONFIG_X86_64
 #define __percpu_seg           gs
+#define __percpu_rel           (%rip)
 #else
 #define __percpu_seg           fs
+#define __percpu_rel
 #endif
 
 #ifdef __ASSEMBLY__
 
 #ifdef CONFIG_SMP
-#define PER_CPU_VAR(var)       %__percpu_seg:var
-#else /* ! SMP */
-#define PER_CPU_VAR(var)       var
-#endif /* SMP */
+#define __percpu               %__percpu_seg:
+#else
+#define __percpu
+#endif
+
+#define PER_CPU_VAR(var)       __percpu(var)__percpu_rel
 
 #ifdef CONFIG_X86_64_SMP
 #define INIT_PER_CPU_VAR(var)  init_per_cpu__##var

diff --git a/arch/x86/lib/cmpxchg16b_emu.S b/arch/x86/lib/cmpxchg16b_emu.S
index 6962df3157938d9936895d67d5b18806844fb761..4fb44894ad87593266f9de2b8c71ddbad2166cbe 100644 (file)
--- a/arch/x86/lib/cmpxchg16b_emu.S
+++ b/arch/x86/lib/cmpxchg16b_emu.S
@@ -23,14 +23,14 @@ SYM_FUNC_START(this_cpu_cmpxchg16b_emu)
        cli
 
        /* if (*ptr == old) */
-       cmpq    PER_CPU_VAR(0(%rsi)), %rax
+       cmpq    __percpu (%rsi), %rax
        jne     .Lnot_same
-       cmpq    PER_CPU_VAR(8(%rsi)), %rdx
+       cmpq    __percpu 8(%rsi), %rdx
        jne     .Lnot_same
 
        /* *ptr = new */
-       movq    %rbx, PER_CPU_VAR(0(%rsi))
-       movq    %rcx, PER_CPU_VAR(8(%rsi))
+       movq    %rbx, __percpu (%rsi)
+       movq    %rcx, __percpu 8(%rsi)
 
        /* set ZF in EFLAGS to indicate success */
        orl     $X86_EFLAGS_ZF, (%rsp)
@@ -42,8 +42,8 @@ SYM_FUNC_START(this_cpu_cmpxchg16b_emu)
        /* *ptr != old */
 
        /* old = *ptr */
-       movq    PER_CPU_VAR(0(%rsi)), %rax
-       movq    PER_CPU_VAR(8(%rsi)), %rdx
+       movq    __percpu (%rsi), %rax
+       movq    __percpu 8(%rsi), %rdx
 
        /* clear ZF in EFLAGS to indicate failure */
        andl    $(~X86_EFLAGS_ZF), (%rsp)

diff --git a/arch/x86/lib/cmpxchg8b_emu.S b/arch/x86/lib/cmpxchg8b_emu.S
index 49805257b1255b2b6d3be0172dc6be920fa7168e..8632d7dd1f004768de7ce2ec18657fb843506162 100644 (file)
--- a/arch/x86/lib/cmpxchg8b_emu.S
+++ b/arch/x86/lib/cmpxchg8b_emu.S
@@ -24,12 +24,12 @@ SYM_FUNC_START(cmpxchg8b_emu)
        pushfl
        cli
 
-       cmpl    0(%esi), %eax
+       cmpl    (%esi), %eax
        jne     .Lnot_same
        cmpl    4(%esi), %edx
        jne     .Lnot_same
 
-       movl    %ebx, 0(%esi)
+       movl    %ebx, (%esi)
        movl    %ecx, 4(%esi)
 
        orl     $X86_EFLAGS_ZF, (%esp)
@@ -38,7 +38,7 @@ SYM_FUNC_START(cmpxchg8b_emu)
        RET
 
 .Lnot_same:
-       movl    0(%esi), %eax
+       movl    (%esi), %eax
        movl    4(%esi), %edx
 
        andl    $(~X86_EFLAGS_ZF), (%esp)
@@ -53,18 +53,30 @@ EXPORT_SYMBOL(cmpxchg8b_emu)
 
 #ifndef CONFIG_UML
 
+/*
+ * Emulate 'cmpxchg8b %fs:(%rsi)'
+ *
+ * Inputs:
+ * %esi : memory location to compare
+ * %eax : low 32 bits of old value
+ * %edx : high 32 bits of old value
+ * %ebx : low 32 bits of new value
+ * %ecx : high 32 bits of new value
+ *
+ * Notably this is not LOCK prefixed and is not safe against NMIs
+ */
 SYM_FUNC_START(this_cpu_cmpxchg8b_emu)
 
        pushfl
        cli
 
-       cmpl    PER_CPU_VAR(0(%esi)), %eax
+       cmpl    __percpu (%esi), %eax
        jne     .Lnot_same2
-       cmpl    PER_CPU_VAR(4(%esi)), %edx
+       cmpl    __percpu 4(%esi), %edx
        jne     .Lnot_same2
 
-       movl    %ebx, PER_CPU_VAR(0(%esi))
-       movl    %ecx, PER_CPU_VAR(4(%esi))
+       movl    %ebx, __percpu (%esi)
+       movl    %ecx, __percpu 4(%esi)
 
        orl     $X86_EFLAGS_ZF, (%esp)
 
@@ -72,8 +84,8 @@ SYM_FUNC_START(this_cpu_cmpxchg8b_emu)
        RET
 
 .Lnot_same2:
-       movl    PER_CPU_VAR(0(%esi)), %eax
-       movl    PER_CPU_VAR(4(%esi)), %edx
+       movl    __percpu (%esi), %eax
+       movl    __percpu 4(%esi), %edx
 
        andl    $(~X86_EFLAGS_ZF), (%esp)