arm64: smp: Add arch support for backtrace using pseudo-NMI

Enable arch_trigger_cpumask_backtrace() support on arm64. This enables
things much like they are enabled on arm32 (including some of the
funky logic around NR_IPI, nr_ipi, and MAX_IPI) but with the
difference that, unlike arm32, we'll try to enable the backtrace to
use pseudo-NMI.

NOTE: this patch is a squash of the little bit of code adding the
ability to mark an IPI to try to use pseudo-NMI plus the little bit of
code to hook things up for kgdb. This approach was decided upon in the
discussion of v9 [1].

This patch depends on commit 8d539b84f1e3 ("nmi_backtrace: allow
excluding an arbitrary CPU") since that commit changed the prototype
of arch_trigger_cpumask_backtrace(), which this patch implements.

[1] https://lore.kernel.org/r/ZORY51mF4alI41G1@FVFF77S0Q05N

Co-developed-by: Sumit Garg <[email protected]>
Signed-off-by: Sumit Garg <[email protected]>
Co-developed-by: Mark Rutland <[email protected]>
Signed-off-by: Mark Rutland <[email protected]>
Reviewed-by: Stephen Boyd <[email protected]>
Reviewed-by: Misono Tomohiro <[email protected]>
Tested-by: Chen-Yu Tsai <[email protected]>
Signed-off-by: Douglas Anderson <[email protected]>
Link: https://lore.kernel.org/r/20230906090246.v13.4.Ie6c132b96ebbbcddbf6954b9469ed40a6960343c@changeid
Signed-off-by: Catalin Marinas <[email protected]>

diff --git a/arch/arm64/include/asm/irq.h b/arch/arm64/include/asm/irq.h
index fac08e18bcd512eec9bbafe7fa79a38ccea1d48c..50ce8b697ff361be7ee7fe571144b6a9e12ad374 100644 (file)
--- a/arch/arm64/include/asm/irq.h
+++ b/arch/arm64/include/asm/irq.h
@@ -6,6 +6,9 @@
 
 #include <asm-generic/irq.h>
 
+void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu);
+#define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace
+
 struct pt_regs;
 
 int set_handle_irq(void (*handle_irq)(struct pt_regs *));

diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index a5848f1ef817b26099439d3cd09031c0e5da0f83..28c904ca499a0d000e11af6be679104f950c7ed6 100644 (file)
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -33,6 +33,7 @@
 #include <linux/kernel_stat.h>
 #include <linux/kexec.h>
 #include <linux/kvm_host.h>
+#include <linux/nmi.h>
 
 #include <asm/alternative.h>
 #include <asm/atomic.h>
@@ -72,12 +73,18 @@ enum ipi_msg_type {
        IPI_CPU_CRASH_STOP,
        IPI_TIMER,
        IPI_IRQ_WORK,
-       NR_IPI
+       NR_IPI,
+       /*
+        * Any enum >= NR_IPI and < MAX_IPI is special and not tracable
+        * with trace_ipi_*
+        */
+       IPI_CPU_BACKTRACE = NR_IPI,
+       MAX_IPI
 };
 
 static int ipi_irq_base __read_mostly;
 static int nr_ipi __read_mostly = NR_IPI;
-static struct irq_desc *ipi_desc[NR_IPI] __read_mostly;
+static struct irq_desc *ipi_desc[MAX_IPI] __read_mostly;
 
 static void ipi_setup(int cpu);
 
@@ -845,6 +852,22 @@ static void __noreturn ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs
 #endif
 }
 
+static void arm64_backtrace_ipi(cpumask_t *mask)
+{
+       __ipi_send_mask(ipi_desc[IPI_CPU_BACKTRACE], mask);
+}
+
+void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu)
+{
+       /*
+        * NOTE: though nmi_trigger_cpumask_backtrace() has "nmi_" in the name,
+        * nothing about it truly needs to be implemented using an NMI, it's
+        * just that it's _allowed_ to work with NMIs. If ipi_should_be_nmi()
+        * returned false our backtrace attempt will just use a regular IPI.
+        */
+       nmi_trigger_cpumask_backtrace(mask, exclude_cpu, arm64_backtrace_ipi);
+}
+
 /*
  * Main handler for inter-processor interrupts
  */
@@ -888,6 +911,14 @@ static void do_handle_IPI(int ipinr)
                break;
 #endif
 
+       case IPI_CPU_BACKTRACE:
+               /*
+                * NOTE: in some cases this _won't_ be NMI context. See the
+                * comment in arch_trigger_cpumask_backtrace().
+                */
+               nmi_cpu_backtrace(get_irq_regs());
+               break;
+
        default:
                pr_crit("CPU%u: Unknown IPI message 0x%x\n", cpu, ipinr);
                break;
@@ -909,6 +940,19 @@ static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
        __ipi_send_mask(ipi_desc[ipinr], target);
 }
 
+static bool ipi_should_be_nmi(enum ipi_msg_type ipi)
+{
+       if (!system_uses_irq_prio_masking())
+               return false;
+
+       switch (ipi) {
+       case IPI_CPU_BACKTRACE:
+               return true;
+       default:
+               return false;
+       }
+}
+
 static void ipi_setup(int cpu)
 {
        int i;
@@ -916,8 +960,14 @@ static void ipi_setup(int cpu)
        if (WARN_ON_ONCE(!ipi_irq_base))
                return;
 
-       for (i = 0; i < nr_ipi; i++)
-               enable_percpu_irq(ipi_irq_base + i, 0);
+       for (i = 0; i < nr_ipi; i++) {
+               if (ipi_should_be_nmi(i)) {
+                       prepare_percpu_nmi(ipi_irq_base + i);
+                       enable_percpu_nmi(ipi_irq_base + i, 0);
+               } else {
+                       enable_percpu_irq(ipi_irq_base + i, 0);
+               }
+       }
 }
 
 #ifdef CONFIG_HOTPLUG_CPU
@@ -928,8 +978,14 @@ static void ipi_teardown(int cpu)
        if (WARN_ON_ONCE(!ipi_irq_base))
                return;
 
-       for (i = 0; i < nr_ipi; i++)
-               disable_percpu_irq(ipi_irq_base + i);
+       for (i = 0; i < nr_ipi; i++) {
+               if (ipi_should_be_nmi(i)) {
+                       disable_percpu_nmi(ipi_irq_base + i);
+                       teardown_percpu_nmi(ipi_irq_base + i);
+               } else {
+                       disable_percpu_irq(ipi_irq_base + i);
+               }
+       }
 }
 #endif
 
@@ -937,15 +993,23 @@ void __init set_smp_ipi_range(int ipi_base, int n)
 {
        int i;
 
-       WARN_ON(n < NR_IPI);
-       nr_ipi = min(n, NR_IPI);
+       WARN_ON(n < MAX_IPI);
+       nr_ipi = min(n, MAX_IPI);
 
        for (i = 0; i < nr_ipi; i++) {
                int err;
 
-               err = request_percpu_irq(ipi_base + i, ipi_handler,
-                                        "IPI", &cpu_number);
-               WARN_ON(err);
+               if (ipi_should_be_nmi(i)) {
+                       err = request_percpu_nmi(ipi_base + i, ipi_handler,
+                                                "IPI", &cpu_number);
+                       WARN(err, "Could not request IPI %d as NMI, err=%d\n",
+                            i, err);
+               } else {
+                       err = request_percpu_irq(ipi_base + i, ipi_handler,
+                                                "IPI", &cpu_number);
+                       WARN(err, "Could not request IPI %d as IRQ, err=%d\n",
+                            i, err);
+               }
 
                ipi_desc[i] = irq_to_desc(ipi_base + i);
                irq_set_status_flags(ipi_base + i, IRQ_HIDDEN);