powerpc/64e: remove implicit soft-masking and interrupt exit restart logic

The implicit soft-masking to speed up interrupt return was going to be
used by 64e as well, but it has not been extensively tested on that
platform and is not considered ready. It was intended to be disabled
before merge. Disable it for now.

Most of the restart code is common with 64s, so with more correctness
and performance testing this could be re-enabled again by adding the
extra soft-mask checks to interrupt handlers and flipping
exit_must_hard_disable().

Fixes: 9d1988ca87dd ("powerpc/64: treat low kernel text as irqs soft-masked")
Signed-off-by: Nicholas Piggin <[email protected]>
Signed-off-by: Michael Ellerman <[email protected]>
Link: https://lore.kernel.org/r/[email protected]

diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h
index 8b4b1e84e1109198ebdd227eb337eaf866d4c76d..f13c93b033c7d9f8c6a0a177e8e760fcef98bf09 100644 (file)
--- a/arch/powerpc/include/asm/interrupt.h
+++ b/arch/powerpc/include/asm/interrupt.h
@@ -73,20 +73,34 @@
 #include <asm/kprobes.h>
 #include <asm/runlatch.h>
 
 #include <asm/kprobes.h>
 #include <asm/runlatch.h>
 

-#ifdef CONFIG_PPC64
+#ifdef CONFIG_PPC_BOOK3S_64

 extern char __end_soft_masked[];
 unsigned long search_kernel_restart_table(unsigned long addr);
 extern char __end_soft_masked[];
 unsigned long search_kernel_restart_table(unsigned long addr);

-#endif

 
 

-#ifdef CONFIG_PPC_BOOK3S_64

 DECLARE_STATIC_KEY_FALSE(interrupt_exit_not_reentrant);
 
 DECLARE_STATIC_KEY_FALSE(interrupt_exit_not_reentrant);
 

+static inline bool is_implicit_soft_masked(struct pt_regs *regs)
+{
+       if (regs->msr & MSR_PR)
+               return false;
+
+       if (regs->nip >= (unsigned long)__end_soft_masked)
+               return false;
+
+       return true;
+}
+

 static inline void srr_regs_clobbered(void)
 {
        local_paca->srr_valid = 0;
        local_paca->hsrr_valid = 0;
 }
 #else
 static inline void srr_regs_clobbered(void)
 {
        local_paca->srr_valid = 0;
        local_paca->hsrr_valid = 0;
 }
 #else

+static inline bool is_implicit_soft_masked(struct pt_regs *regs)
+{
+       return false;
+}
+

 static inline void srr_regs_clobbered(void)
 {
 }
 static inline void srr_regs_clobbered(void)
 {
 }


@@ -150,11 +164,13 @@ static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrup
                 */
                if (TRAP(regs) != INTERRUPT_PROGRAM) {
                        CT_WARN_ON(ct_state() != CONTEXT_KERNEL);
                 */
                if (TRAP(regs) != INTERRUPT_PROGRAM) {
                        CT_WARN_ON(ct_state() != CONTEXT_KERNEL);

-                       BUG_ON(regs->nip < (unsigned long)__end_soft_masked);
+                       BUG_ON(is_implicit_soft_masked(regs));

                }
                }

+#ifdef CONFIG_PPC_BOOK3S

                /* Move this under a debugging check */
                if (arch_irq_disabled_regs(regs))
                        BUG_ON(search_kernel_restart_table(regs->nip));
                /* Move this under a debugging check */
                if (arch_irq_disabled_regs(regs))
                        BUG_ON(search_kernel_restart_table(regs->nip));

+#endif

        }
 #endif
 
        }
 #endif
 


@@ -244,10 +260,9 @@ static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct inte
        local_paca->irq_soft_mask = IRQS_ALL_DISABLED;
        local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
 
        local_paca->irq_soft_mask = IRQS_ALL_DISABLED;
        local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
 

-       if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && !(regs->msr & MSR_PR) &&
-                               regs->nip < (unsigned long)__end_soft_masked) {
-               // Kernel code running below __end_soft_masked is
-               // implicitly soft-masked.
+       if (is_implicit_soft_masked(regs)) {
+               // Adjust regs->softe soft implicit soft-mask, so
+               // arch_irq_disabled_regs(regs) behaves as expected.

                regs->softe = IRQS_ALL_DISABLED;
        }
 
                regs->softe = IRQS_ALL_DISABLED;
        }
 


@@ -282,6 +297,7 @@ static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct inter
         */
 
 #ifdef CONFIG_PPC64
         */
 
 #ifdef CONFIG_PPC64

+#ifdef CONFIG_PPC_BOOK3S

        if (arch_irq_disabled_regs(regs)) {
                unsigned long rst = search_kernel_restart_table(regs->nip);
                if (rst)
        if (arch_irq_disabled_regs(regs)) {
                unsigned long rst = search_kernel_restart_table(regs->nip);
                if (rst)


@@ -289,7 +305,6 @@ static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct inter
        }
 #endif
 
        }
 #endif
 

-#ifdef CONFIG_PPC64

        if (nmi_disables_ftrace(regs))
                this_cpu_set_ftrace_enabled(state->ftrace_enabled);
 
        if (nmi_disables_ftrace(regs))
                this_cpu_set_ftrace_enabled(state->ftrace_enabled);
 

diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S
index d634bfceed2c51b497c2df3667bdff2b2bfe3462..1401787b0b937f3edd429c5730309708814c8b3d 100644 (file)
--- a/arch/powerpc/kernel/exceptions-64e.S
+++ b/arch/powerpc/kernel/exceptions-64e.S
@@ -342,17 +342,7 @@ ret_from_mc_except:
 #define PROLOG_ADDITION_MASKABLE_GEN(n)                                            \
        lbz     r10,PACAIRQSOFTMASK(r13);       /* are irqs soft-masked? */ \
        andi.   r10,r10,IRQS_DISABLED;  /* yes -> go out of line */ \
 #define PROLOG_ADDITION_MASKABLE_GEN(n)                                            \
        lbz     r10,PACAIRQSOFTMASK(r13);       /* are irqs soft-masked? */ \
        andi.   r10,r10,IRQS_DISABLED;  /* yes -> go out of line */ \

-       bne     masked_interrupt_book3e_##n;                                \
-       /* Kernel code below __end_soft_masked is implicitly masked */      \
-       andi.   r10,r11,MSR_PR;                                             \
-       bne     1f;                     /* user -> not masked */            \
-       std     r14,PACA_EXGEN+EX_R14(r13);                                 \
-       LOAD_REG_IMMEDIATE_SYM(r14, r10, __end_soft_masked);                \
-       mfspr   r10,SPRN_SRR0;                                              \
-       cmpld   r10,r14;                                                    \
-       ld      r14,PACA_EXGEN+EX_R14(r13);                                 \
-       blt     masked_interrupt_book3e_##n;                                \
-1:
+       bne     masked_interrupt_book3e_##n

 
 /*
  * Additional regs must be re-loaded from paca before EXCEPTION_COMMON* is
 
 /*
  * Additional regs must be re-loaded from paca before EXCEPTION_COMMON* is

diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c
index 0052702ee5ac262a8e3afbf9769130c211ef0720..21bbd615ca41033b3051b86621f53f471652e6ec 100644 (file)
--- a/arch/powerpc/kernel/interrupt.c
+++ b/arch/powerpc/kernel/interrupt.c
@@ -36,7 +36,7 @@ static inline bool exit_must_hard_disable(void)
 #else
 static inline bool exit_must_hard_disable(void)
 {
 #else
 static inline bool exit_must_hard_disable(void)
 {

-       return IS_ENABLED(CONFIG_PPC32);
+       return true;

 }
 #endif
 
 }
 #endif
 

diff --git a/arch/powerpc/kernel/interrupt_64.S b/arch/powerpc/kernel/interrupt_64.S
index e7a50613a5701995725a67ef70c75532b3518033..09b8d8846c675faec035ef83c787d015e109551b 100644 (file)
--- a/arch/powerpc/kernel/interrupt_64.S
+++ b/arch/powerpc/kernel/interrupt_64.S
@@ -231,7 +231,7 @@ _ASM_NOKPROBE_SYMBOL(system_call_vectored_emulate)
        li      r10,IRQS_ALL_DISABLED
        stb     r10,PACAIRQSOFTMASK(r13)
        b       system_call_vectored_common
        li      r10,IRQS_ALL_DISABLED
        stb     r10,PACAIRQSOFTMASK(r13)
        b       system_call_vectored_common

-#endif
+#endif /* CONFIG_PPC_BOOK3S */

 
        .balign IFETCH_ALIGN_BYTES
        .globl system_call_common_real
 
        .balign IFETCH_ALIGN_BYTES
        .globl system_call_common_real


@@ -320,10 +320,12 @@ END_BTB_FLUSH_SECTION
        li      r5,0 /* !scv */
        bl      syscall_exit_prepare
        std     r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
        li      r5,0 /* !scv */
        bl      syscall_exit_prepare
        std     r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */

+#ifdef CONFIG_PPC_BOOK3S

 .Lsyscall_rst_start:
        lbz     r11,PACAIRQHAPPENED(r13)
        andi.   r11,r11,(~PACA_IRQ_HARD_DIS)@l
        bne-    syscall_restart
 .Lsyscall_rst_start:
        lbz     r11,PACAIRQHAPPENED(r13)
        andi.   r11,r11,(~PACA_IRQ_HARD_DIS)@l
        bne-    syscall_restart

+#endif

        li      r11,IRQS_ENABLED
        stb     r11,PACAIRQSOFTMASK(r13)
        li      r11,0
        li      r11,IRQS_ENABLED
        stb     r11,PACAIRQSOFTMASK(r13)
        li      r11,0


@@ -396,6 +398,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
        b       .Lsyscall_restore_regs_cont
 .Lsyscall_rst_end:
 
        b       .Lsyscall_restore_regs_cont
 .Lsyscall_rst_end:
 

+#ifdef CONFIG_PPC_BOOK3S

 syscall_restart:
        GET_PACA(r13)
        ld      r1,PACA_EXIT_SAVE_R1(r13)
 syscall_restart:
        GET_PACA(r13)
        ld      r1,PACA_EXIT_SAVE_R1(r13)


@@ -409,6 +412,7 @@ syscall_restart:
        b       .Lsyscall_rst_start
 
 RESTART_TABLE(.Lsyscall_rst_start, .Lsyscall_rst_end, syscall_restart)
        b       .Lsyscall_rst_start
 
 RESTART_TABLE(.Lsyscall_rst_start, .Lsyscall_rst_end, syscall_restart)

+#endif

 
 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
 tabort_syscall:
 
 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
 tabort_syscall:


@@ -504,10 +508,12 @@ _ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\())
        bne-    .Lrestore_nvgprs_\srr
 .Lrestore_nvgprs_\srr\()_cont:
        std     r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
        bne-    .Lrestore_nvgprs_\srr
 .Lrestore_nvgprs_\srr\()_cont:
        std     r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */

+#ifdef CONFIG_PPC_BOOK3S

 .Linterrupt_return_\srr\()_user_rst_start:
        lbz     r11,PACAIRQHAPPENED(r13)
        andi.   r11,r11,(~PACA_IRQ_HARD_DIS)@l
        bne-    interrupt_return_\srr\()_user_restart
 .Linterrupt_return_\srr\()_user_rst_start:
        lbz     r11,PACAIRQHAPPENED(r13)
        andi.   r11,r11,(~PACA_IRQ_HARD_DIS)@l
        bne-    interrupt_return_\srr\()_user_restart

+#endif

        li      r11,IRQS_ENABLED
        stb     r11,PACAIRQSOFTMASK(r13)
        li      r11,0
        li      r11,IRQS_ENABLED
        stb     r11,PACAIRQSOFTMASK(r13)
        li      r11,0


@@ -590,6 +596,7 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
        REST_NVGPRS(r1)
        b       .Lrestore_nvgprs_\srr\()_cont
 
        REST_NVGPRS(r1)
        b       .Lrestore_nvgprs_\srr\()_cont
 

+#ifdef CONFIG_PPC_BOOK3S

 interrupt_return_\srr\()_user_restart:
        GET_PACA(r13)
        ld      r1,PACA_EXIT_SAVE_R1(r13)
 interrupt_return_\srr\()_user_restart:
        GET_PACA(r13)
        ld      r1,PACA_EXIT_SAVE_R1(r13)


@@ -602,6 +609,7 @@ interrupt_return_\srr\()_user_restart:
        b       .Linterrupt_return_\srr\()_user_rst_start
 
 RESTART_TABLE(.Linterrupt_return_\srr\()_user_rst_start, .Linterrupt_return_\srr\()_user_rst_end, interrupt_return_\srr\()_user_restart)
        b       .Linterrupt_return_\srr\()_user_rst_start
 
 RESTART_TABLE(.Linterrupt_return_\srr\()_user_rst_start, .Linterrupt_return_\srr\()_user_rst_end, interrupt_return_\srr\()_user_restart)

+#endif

 
        .balign IFETCH_ALIGN_BYTES
 .Lkernel_interrupt_return_\srr\():
 
        .balign IFETCH_ALIGN_BYTES
 .Lkernel_interrupt_return_\srr\():


@@ -615,9 +623,11 @@ RESTART_TABLE(.Linterrupt_return_\srr\()_user_rst_start, .Linterrupt_return_\srr
        cmpwi   r11,IRQS_ENABLED
        stb     r11,PACAIRQSOFTMASK(r13)
        bne     1f
        cmpwi   r11,IRQS_ENABLED
        stb     r11,PACAIRQSOFTMASK(r13)
        bne     1f

+#ifdef CONFIG_PPC_BOOK3S

        lbz     r11,PACAIRQHAPPENED(r13)
        andi.   r11,r11,(~PACA_IRQ_HARD_DIS)@l
        bne-    interrupt_return_\srr\()_kernel_restart
        lbz     r11,PACAIRQHAPPENED(r13)
        andi.   r11,r11,(~PACA_IRQ_HARD_DIS)@l
        bne-    interrupt_return_\srr\()_kernel_restart

+#endif

        li      r11,0
        stb     r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS
 1:
        li      r11,0
        stb     r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS
 1:


@@ -717,6 +727,7 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
        b       .       /* prevent speculative execution */
 .Linterrupt_return_\srr\()_kernel_rst_end:
 
        b       .       /* prevent speculative execution */
 .Linterrupt_return_\srr\()_kernel_rst_end:
 

+#ifdef CONFIG_PPC_BOOK3S

 interrupt_return_\srr\()_kernel_restart:
        GET_PACA(r13)
        ld      r1,PACA_EXIT_SAVE_R1(r13)
 interrupt_return_\srr\()_kernel_restart:
        GET_PACA(r13)
        ld      r1,PACA_EXIT_SAVE_R1(r13)


@@ -729,14 +740,15 @@ interrupt_return_\srr\()_kernel_restart:
        b       .Linterrupt_return_\srr\()_kernel_rst_start
 
 RESTART_TABLE(.Linterrupt_return_\srr\()_kernel_rst_start, .Linterrupt_return_\srr\()_kernel_rst_end, interrupt_return_\srr\()_kernel_restart)
        b       .Linterrupt_return_\srr\()_kernel_rst_start
 
 RESTART_TABLE(.Linterrupt_return_\srr\()_kernel_rst_start, .Linterrupt_return_\srr\()_kernel_rst_end, interrupt_return_\srr\()_kernel_restart)

+#endif

 
 .endm
 
 interrupt_return_macro srr
 #ifdef CONFIG_PPC_BOOK3S
 interrupt_return_macro hsrr
 
 .endm
 
 interrupt_return_macro srr
 #ifdef CONFIG_PPC_BOOK3S
 interrupt_return_macro hsrr

-#endif /* CONFIG_PPC_BOOK3S */

 
        .globl __end_soft_masked
 __end_soft_masked:
 DEFINE_FIXED_SYMBOL(__end_soft_masked)
 
        .globl __end_soft_masked
 __end_soft_masked:
 DEFINE_FIXED_SYMBOL(__end_soft_masked)

+#endif /* CONFIG_PPC_BOOK3S */