arch/x86/entry/entry_fred.c

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  * The FRED specific kernel/user entry functions which are invoked from
   4  * assembly code and dispatch to the associated handlers.
   5  */
   6 #include <linux/kernel.h>
   7 #include <linux/kdebug.h>
   8 #include <linux/nospec.h>
   9
  10 #include <asm/desc.h>
  11 #include <asm/fred.h>
  12 #include <asm/idtentry.h>
  13 #include <asm/syscall.h>
  14 #include <asm/trapnr.h>
  15 #include <asm/traps.h>
  16
  17 /* FRED EVENT_TYPE_OTHER vector numbers */
  18 #define FRED_SYSCALL                    1
  19 #define FRED_SYSENTER                   2
  20
  21 static noinstr void fred_bad_type(struct pt_regs *regs, unsigned long error_code)
  22 {
  23         irqentry_state_t irq_state = irqentry_nmi_enter(regs);
  24
  25         instrumentation_begin();
  26
  27         /* Panic on events from a high stack level */
  28         if (regs->fred_cs.sl > 0) {
  29                 pr_emerg("PANIC: invalid or fatal FRED event; event type %u "
  30                          "vector %u error 0x%lx aux 0x%lx at %04x:%016lx\n",
  31                          regs->fred_ss.type, regs->fred_ss.vector, error_code,
  32                          fred_event_data(regs), regs->cs, regs->ip);
  33                 die("invalid or fatal FRED event", regs, error_code);
  34                 panic("invalid or fatal FRED event");
  35         } else {
  36                 unsigned long flags = oops_begin();
  37                 int sig = SIGKILL;
  38
  39                 pr_alert("BUG: invalid or fatal FRED event; event type %u "
  40                          "vector %u error 0x%lx aux 0x%lx at %04x:%016lx\n",
  41                          regs->fred_ss.type, regs->fred_ss.vector, error_code,
  42                          fred_event_data(regs), regs->cs, regs->ip);
  43
  44                 if (__die("Invalid or fatal FRED event", regs, error_code))
  45                         sig = 0;
  46
  47                 oops_end(flags, regs, sig);
  48         }
  49
  50         instrumentation_end();
  51         irqentry_nmi_exit(regs, irq_state);
  52 }
  53
  54 static noinstr void fred_intx(struct pt_regs *regs)
  55 {
  56         switch (regs->fred_ss.vector) {
  57         /* Opcode 0xcd, 0x3, NOT INT3 (opcode 0xcc) */
  58         case X86_TRAP_BP:
  59                 return exc_int3(regs);
  60
  61         /* Opcode 0xcd, 0x4, NOT INTO (opcode 0xce) */
  62         case X86_TRAP_OF:
  63                 return exc_overflow(regs);
  64
  65 #ifdef CONFIG_IA32_EMULATION
  66         /* INT80 */
  67         case IA32_SYSCALL_VECTOR:
  68                 if (ia32_enabled())
  69                         return fred_int80_emulation(regs);
  70                 fallthrough;
  71 #endif
  72
  73         default:
  74                 return exc_general_protection(regs, 0);
  75         }
  76 }
  77
  78 static __always_inline void fred_other(struct pt_regs *regs)
  79 {
  80         /* The compiler can fold these conditions into a single test */
  81         if (likely(regs->fred_ss.vector == FRED_SYSCALL && regs->fred_ss.lm)) {
  82                 regs->orig_ax = regs->ax;
  83                 regs->ax = -ENOSYS;
  84                 do_syscall_64(regs, regs->orig_ax);
  85                 return;
  86         } else if (ia32_enabled() &&
  87                    likely(regs->fred_ss.vector == FRED_SYSENTER && !regs->fred_ss.lm)) {
  88                 regs->orig_ax = regs->ax;
  89                 regs->ax = -ENOSYS;
  90                 do_fast_syscall_32(regs);
  91                 return;
  92         } else {
  93                 exc_invalid_op(regs);
  94                 return;
  95         }
  96 }
  97
  98 #define SYSVEC(_vector, _function) [_vector - FIRST_SYSTEM_VECTOR] = fred_sysvec_##_function
  99
 100 static idtentry_t sysvec_table[NR_SYSTEM_VECTORS] __ro_after_init = {
 101         SYSVEC(ERROR_APIC_VECTOR,               error_interrupt),
 102         SYSVEC(SPURIOUS_APIC_VECTOR,            spurious_apic_interrupt),
 103         SYSVEC(LOCAL_TIMER_VECTOR,              apic_timer_interrupt),
 104         SYSVEC(X86_PLATFORM_IPI_VECTOR,         x86_platform_ipi),
 105
 106         SYSVEC(RESCHEDULE_VECTOR,               reschedule_ipi),
 107         SYSVEC(CALL_FUNCTION_SINGLE_VECTOR,     call_function_single),
 108         SYSVEC(CALL_FUNCTION_VECTOR,            call_function),
 109         SYSVEC(REBOOT_VECTOR,                   reboot),
 110
 111         SYSVEC(THRESHOLD_APIC_VECTOR,           threshold),
 112         SYSVEC(DEFERRED_ERROR_VECTOR,           deferred_error),
 113         SYSVEC(THERMAL_APIC_VECTOR,             thermal),
 114
 115         SYSVEC(IRQ_WORK_VECTOR,                 irq_work),
 116
 117         SYSVEC(POSTED_INTR_VECTOR,              kvm_posted_intr_ipi),
 118         SYSVEC(POSTED_INTR_WAKEUP_VECTOR,       kvm_posted_intr_wakeup_ipi),
 119         SYSVEC(POSTED_INTR_NESTED_VECTOR,       kvm_posted_intr_nested_ipi),
 120
 121         SYSVEC(POSTED_MSI_NOTIFICATION_VECTOR,  posted_msi_notification),
 122 };
 123
 124 static bool fred_setup_done __initdata;
 125
 126 void __init fred_install_sysvec(unsigned int sysvec, idtentry_t handler)
 127 {
 128         if (WARN_ON_ONCE(sysvec < FIRST_SYSTEM_VECTOR))
 129                 return;
 130
 131         if (WARN_ON_ONCE(fred_setup_done))
 132                 return;
 133
 134         if (!WARN_ON_ONCE(sysvec_table[sysvec - FIRST_SYSTEM_VECTOR]))
 135                  sysvec_table[sysvec - FIRST_SYSTEM_VECTOR] = handler;
 136 }
 137
 138 static noinstr void fred_handle_spurious_interrupt(struct pt_regs *regs)
 139 {
 140         spurious_interrupt(regs, regs->fred_ss.vector);
 141 }
 142
 143 void __init fred_complete_exception_setup(void)
 144 {
 145         unsigned int vector;
 146
 147         for (vector = 0; vector < FIRST_EXTERNAL_VECTOR; vector++)
 148                 set_bit(vector, system_vectors);
 149
 150         for (vector = 0; vector < NR_SYSTEM_VECTORS; vector++) {
 151                 if (sysvec_table[vector])
 152                         set_bit(vector + FIRST_SYSTEM_VECTOR, system_vectors);
 153                 else
 154                         sysvec_table[vector] = fred_handle_spurious_interrupt;
 155         }
 156         fred_setup_done = true;
 157 }
 158
 159 static noinstr void fred_extint(struct pt_regs *regs)
 160 {
 161         unsigned int vector = regs->fred_ss.vector;
 162         unsigned int index = array_index_nospec(vector - FIRST_SYSTEM_VECTOR,
 163                                                 NR_SYSTEM_VECTORS);
 164
 165         if (WARN_ON_ONCE(vector < FIRST_EXTERNAL_VECTOR))
 166                 return;
 167
 168         if (likely(vector >= FIRST_SYSTEM_VECTOR)) {
 169                 irqentry_state_t state = irqentry_enter(regs);
 170
 171                 instrumentation_begin();
 172                 sysvec_table[index](regs);
 173                 instrumentation_end();
 174                 irqentry_exit(regs, state);
 175         } else {
 176                 common_interrupt(regs, vector);
 177         }
 178 }
 179
 180 static noinstr void fred_hwexc(struct pt_regs *regs, unsigned long error_code)
 181 {
 182         /* Optimize for #PF. That's the only exception which matters performance wise */
 183         if (likely(regs->fred_ss.vector == X86_TRAP_PF))
 184                 return exc_page_fault(regs, error_code);
 185
 186         switch (regs->fred_ss.vector) {
 187         case X86_TRAP_DE: return exc_divide_error(regs);
 188         case X86_TRAP_DB: return fred_exc_debug(regs);
 189         case X86_TRAP_BR: return exc_bounds(regs);
 190         case X86_TRAP_UD: return exc_invalid_op(regs);
 191         case X86_TRAP_NM: return exc_device_not_available(regs);
 192         case X86_TRAP_DF: return exc_double_fault(regs, error_code);
 193         case X86_TRAP_TS: return exc_invalid_tss(regs, error_code);
 194         case X86_TRAP_NP: return exc_segment_not_present(regs, error_code);
 195         case X86_TRAP_SS: return exc_stack_segment(regs, error_code);
 196         case X86_TRAP_GP: return exc_general_protection(regs, error_code);
 197         case X86_TRAP_MF: return exc_coprocessor_error(regs);
 198         case X86_TRAP_AC: return exc_alignment_check(regs, error_code);
 199         case X86_TRAP_XF: return exc_simd_coprocessor_error(regs);
 200
 201 #ifdef CONFIG_X86_MCE
 202         case X86_TRAP_MC: return fred_exc_machine_check(regs);
 203 #endif
 204 #ifdef CONFIG_INTEL_TDX_GUEST
 205         case X86_TRAP_VE: return exc_virtualization_exception(regs);
 206 #endif
 207 #ifdef CONFIG_X86_CET
 208         case X86_TRAP_CP: return exc_control_protection(regs, error_code);
 209 #endif
 210         default: return fred_bad_type(regs, error_code);
 211         }
 212
 213 }
 214
 215 static noinstr void fred_swexc(struct pt_regs *regs, unsigned long error_code)
 216 {
 217         switch (regs->fred_ss.vector) {
 218         case X86_TRAP_BP: return exc_int3(regs);
 219         case X86_TRAP_OF: return exc_overflow(regs);
 220         default: return fred_bad_type(regs, error_code);
 221         }
 222 }
 223
 224 __visible noinstr void fred_entry_from_user(struct pt_regs *regs)
 225 {
 226         unsigned long error_code = regs->orig_ax;
 227
 228         /* Invalidate orig_ax so that syscall_get_nr() works correctly */
 229         regs->orig_ax = -1;
 230
 231         switch (regs->fred_ss.type) {
 232         case EVENT_TYPE_EXTINT:
 233                 return fred_extint(regs);
 234         case EVENT_TYPE_NMI:
 235                 if (likely(regs->fred_ss.vector == X86_TRAP_NMI))
 236                         return fred_exc_nmi(regs);
 237                 break;
 238         case EVENT_TYPE_HWEXC:
 239                 return fred_hwexc(regs, error_code);
 240         case EVENT_TYPE_SWINT:
 241                 return fred_intx(regs);
 242         case EVENT_TYPE_PRIV_SWEXC:
 243                 if (likely(regs->fred_ss.vector == X86_TRAP_DB))
 244                         return fred_exc_debug(regs);
 245                 break;
 246         case EVENT_TYPE_SWEXC:
 247                 return fred_swexc(regs, error_code);
 248         case EVENT_TYPE_OTHER:
 249                 return fred_other(regs);
 250         default: break;
 251         }
 252
 253         return fred_bad_type(regs, error_code);
 254 }
 255
 256 __visible noinstr void fred_entry_from_kernel(struct pt_regs *regs)
 257 {
 258         unsigned long error_code = regs->orig_ax;
 259
 260         /* Invalidate orig_ax so that syscall_get_nr() works correctly */
 261         regs->orig_ax = -1;
 262
 263         switch (regs->fred_ss.type) {
 264         case EVENT_TYPE_EXTINT:
 265                 return fred_extint(regs);
 266         case EVENT_TYPE_NMI:
 267                 if (likely(regs->fred_ss.vector == X86_TRAP_NMI))
 268                         return fred_exc_nmi(regs);
 269                 break;
 270         case EVENT_TYPE_HWEXC:
 271                 return fred_hwexc(regs, error_code);
 272         case EVENT_TYPE_PRIV_SWEXC:
 273                 if (likely(regs->fred_ss.vector == X86_TRAP_DB))
 274                         return fred_exc_debug(regs);
 275                 break;
 276         case EVENT_TYPE_SWEXC:
 277                 return fred_swexc(regs, error_code);
 278         default: break;
 279         }
 280
 281         return fred_bad_type(regs, error_code);
 282 }
 283
 284 #if IS_ENABLED(CONFIG_KVM_INTEL)
 285 __visible noinstr void __fred_entry_from_kvm(struct pt_regs *regs)
 286 {
 287         switch (regs->fred_ss.type) {
 288         case EVENT_TYPE_EXTINT:
 289                 return fred_extint(regs);
 290         case EVENT_TYPE_NMI:
 291                 return fred_exc_nmi(regs);
 292         default:
 293                 WARN_ON_ONCE(1);
 294         }
 295 }
 296 #endif