arch/arm/vfp/vfpmodule.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  linux/arch/arm/vfp/vfpmodule.c
   4  *
   5  *  Copyright (C) 2004 ARM Limited.
   6  *  Written by Deep Blue Solutions Limited.
   7  */
   8 #include <linux/types.h>
   9 #include <linux/cpu.h>
  10 #include <linux/cpu_pm.h>
  11 #include <linux/hardirq.h>
  12 #include <linux/kernel.h>
  13 #include <linux/notifier.h>
  14 #include <linux/signal.h>
  15 #include <linux/sched/signal.h>
  16 #include <linux/smp.h>
  17 #include <linux/init.h>
  18 #include <linux/uaccess.h>
  19 #include <linux/user.h>
  20 #include <linux/export.h>
  21 #include <linux/perf_event.h>
  22
  23 #include <asm/cp15.h>
  24 #include <asm/cputype.h>
  25 #include <asm/system_info.h>
  26 #include <asm/thread_notify.h>
  27 #include <asm/traps.h>
  28 #include <asm/vfp.h>
  29 #include <asm/neon.h>
  30
  31 #include "vfpinstr.h"
  32 #include "vfp.h"
  33
  34 static bool have_vfp __ro_after_init;
  35
  36 /*
  37  * Dual-use variable.
  38  * Used in startup: set to non-zero if VFP checks fail
  39  * After startup, holds VFP architecture
  40  */
  41 static unsigned int VFP_arch;
  42
  43 #ifdef CONFIG_CPU_FEROCEON
  44 extern unsigned int VFP_arch_feroceon __alias(VFP_arch);
  45 #endif
  46
  47 /*
  48  * The pointer to the vfpstate structure of the thread which currently
  49  * owns the context held in the VFP hardware, or NULL if the hardware
  50  * context is invalid.
  51  *
  52  * For UP, this is sufficient to tell which thread owns the VFP context.
  53  * However, for SMP, we also need to check the CPU number stored in the
  54  * saved state too to catch migrations.
  55  */
  56 union vfp_state *vfp_current_hw_state[NR_CPUS];
  57
  58 /*
  59  * Claim ownership of the VFP unit.
  60  *
  61  * The caller may change VFP registers until vfp_state_release() is called.
  62  *
  63  * local_bh_disable() is used to disable preemption and to disable VFP
  64  * processing in softirq context. On PREEMPT_RT kernels local_bh_disable() is
  65  * not sufficient because it only serializes soft interrupt related sections
  66  * via a local lock, but stays preemptible. Disabling preemption is the right
  67  * choice here as bottom half processing is always in thread context on RT
  68  * kernels so it implicitly prevents bottom half processing as well.
  69  */
  70 static void vfp_state_hold(void)
  71 {
  72         if (!IS_ENABLED(CONFIG_PREEMPT_RT))
  73                 local_bh_disable();
  74         else
  75                 preempt_disable();
  76 }
  77
  78 static void vfp_state_release(void)
  79 {
  80         if (!IS_ENABLED(CONFIG_PREEMPT_RT))
  81                 local_bh_enable();
  82         else
  83                 preempt_enable();
  84 }
  85
  86 /*
  87  * Is 'thread's most up to date state stored in this CPUs hardware?
  88  * Must be called from non-preemptible context.
  89  */
  90 static bool vfp_state_in_hw(unsigned int cpu, struct thread_info *thread)
  91 {
  92 #ifdef CONFIG_SMP
  93         if (thread->vfpstate.hard.cpu != cpu)
  94                 return false;
  95 #endif
  96         return vfp_current_hw_state[cpu] == &thread->vfpstate;
  97 }
  98
  99 /*
 100  * Force a reload of the VFP context from the thread structure.  We do
 101  * this by ensuring that access to the VFP hardware is disabled, and
 102  * clear vfp_current_hw_state.  Must be called from non-preemptible context.
 103  */
 104 static void vfp_force_reload(unsigned int cpu, struct thread_info *thread)
 105 {
 106         if (vfp_state_in_hw(cpu, thread)) {
 107                 fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
 108                 vfp_current_hw_state[cpu] = NULL;
 109         }
 110 #ifdef CONFIG_SMP
 111         thread->vfpstate.hard.cpu = NR_CPUS;
 112 #endif
 113 }
 114
 115 /*
 116  * Per-thread VFP initialization.
 117  */
 118 static void vfp_thread_flush(struct thread_info *thread)
 119 {
 120         union vfp_state *vfp = &thread->vfpstate;
 121         unsigned int cpu;
 122
 123         /*
 124          * Disable VFP to ensure we initialize it first.  We must ensure
 125          * that the modification of vfp_current_hw_state[] and hardware
 126          * disable are done for the same CPU and without preemption.
 127          *
 128          * Do this first to ensure that preemption won't overwrite our
 129          * state saving should access to the VFP be enabled at this point.
 130          */
 131         cpu = get_cpu();
 132         if (vfp_current_hw_state[cpu] == vfp)
 133                 vfp_current_hw_state[cpu] = NULL;
 134         fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
 135         put_cpu();
 136
 137         memset(vfp, 0, sizeof(union vfp_state));
 138
 139         vfp->hard.fpexc = FPEXC_EN;
 140         vfp->hard.fpscr = FPSCR_ROUND_NEAREST;
 141 #ifdef CONFIG_SMP
 142         vfp->hard.cpu = NR_CPUS;
 143 #endif
 144 }
 145
 146 static void vfp_thread_exit(struct thread_info *thread)
 147 {
 148         /* release case: Per-thread VFP cleanup. */
 149         union vfp_state *vfp = &thread->vfpstate;
 150         unsigned int cpu = get_cpu();
 151
 152         if (vfp_current_hw_state[cpu] == vfp)
 153                 vfp_current_hw_state[cpu] = NULL;
 154         put_cpu();
 155 }
 156
 157 static void vfp_thread_copy(struct thread_info *thread)
 158 {
 159         struct thread_info *parent = current_thread_info();
 160
 161         vfp_sync_hwstate(parent);
 162         thread->vfpstate = parent->vfpstate;
 163 #ifdef CONFIG_SMP
 164         thread->vfpstate.hard.cpu = NR_CPUS;
 165 #endif
 166 }
 167
 168 /*
 169  * When this function is called with the following 'cmd's, the following
 170  * is true while this function is being run:
 171  *  THREAD_NOFTIFY_SWTICH:
 172  *   - the previously running thread will not be scheduled onto another CPU.
 173  *   - the next thread to be run (v) will not be running on another CPU.
 174  *   - thread->cpu is the local CPU number
 175  *   - not preemptible as we're called in the middle of a thread switch
 176  *  THREAD_NOTIFY_FLUSH:
 177  *   - the thread (v) will be running on the local CPU, so
 178  *      v === current_thread_info()
 179  *   - thread->cpu is the local CPU number at the time it is accessed,
 180  *      but may change at any time.
 181  *   - we could be preempted if tree preempt rcu is enabled, so
 182  *      it is unsafe to use thread->cpu.
 183  *  THREAD_NOTIFY_EXIT
 184  *   - we could be preempted if tree preempt rcu is enabled, so
 185  *      it is unsafe to use thread->cpu.
 186  */
 187 static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v)
 188 {
 189         struct thread_info *thread = v;
 190         u32 fpexc;
 191 #ifdef CONFIG_SMP
 192         unsigned int cpu;
 193 #endif
 194
 195         switch (cmd) {
 196         case THREAD_NOTIFY_SWITCH:
 197                 fpexc = fmrx(FPEXC);
 198
 199 #ifdef CONFIG_SMP
 200                 cpu = thread->cpu;
 201
 202                 /*
 203                  * On SMP, if VFP is enabled, save the old state in
 204                  * case the thread migrates to a different CPU. The
 205                  * restoring is done lazily.
 206                  */
 207                 if ((fpexc & FPEXC_EN) && vfp_current_hw_state[cpu])
 208                         vfp_save_state(vfp_current_hw_state[cpu], fpexc);
 209 #endif
 210
 211                 /*
 212                  * Always disable VFP so we can lazily save/restore the
 213                  * old state.
 214                  */
 215                 fmxr(FPEXC, fpexc & ~FPEXC_EN);
 216                 break;
 217
 218         case THREAD_NOTIFY_FLUSH:
 219                 vfp_thread_flush(thread);
 220                 break;
 221
 222         case THREAD_NOTIFY_EXIT:
 223                 vfp_thread_exit(thread);
 224                 break;
 225
 226         case THREAD_NOTIFY_COPY:
 227                 vfp_thread_copy(thread);
 228                 break;
 229         }
 230
 231         return NOTIFY_DONE;
 232 }
 233
 234 static struct notifier_block vfp_notifier_block = {
 235         .notifier_call  = vfp_notifier,
 236 };
 237
 238 /*
 239  * Raise a SIGFPE for the current process.
 240  * sicode describes the signal being raised.
 241  */
 242 static void vfp_raise_sigfpe(unsigned int sicode, struct pt_regs *regs)
 243 {
 244         /*
 245          * This is the same as NWFPE, because it's not clear what
 246          * this is used for
 247          */
 248         current->thread.error_code = 0;
 249         current->thread.trap_no = 6;
 250
 251         send_sig_fault(SIGFPE, sicode,
 252                        (void __user *)(instruction_pointer(regs) - 4),
 253                        current);
 254 }
 255
 256 static void vfp_panic(char *reason, u32 inst)
 257 {
 258         int i;
 259
 260         pr_err("VFP: Error: %s\n", reason);
 261         pr_err("VFP: EXC 0x%08x SCR 0x%08x INST 0x%08x\n",
 262                 fmrx(FPEXC), fmrx(FPSCR), inst);
 263         for (i = 0; i < 32; i += 2)
 264                 pr_err("VFP: s%2u: 0x%08x s%2u: 0x%08x\n",
 265                        i, vfp_get_float(i), i+1, vfp_get_float(i+1));
 266 }
 267
 268 /*
 269  * Process bitmask of exception conditions.
 270  */
 271 static int vfp_raise_exceptions(u32 exceptions, u32 inst, u32 fpscr)
 272 {
 273         int si_code = 0;
 274
 275         pr_debug("VFP: raising exceptions %08x\n", exceptions);
 276
 277         if (exceptions == VFP_EXCEPTION_ERROR) {
 278                 vfp_panic("unhandled bounce", inst);
 279                 return FPE_FLTINV;
 280         }
 281
 282         /*
 283          * If any of the status flags are set, update the FPSCR.
 284          * Comparison instructions always return at least one of
 285          * these flags set.
 286          */
 287         if (exceptions & (FPSCR_N|FPSCR_Z|FPSCR_C|FPSCR_V))
 288                 fpscr &= ~(FPSCR_N|FPSCR_Z|FPSCR_C|FPSCR_V);
 289
 290         fpscr |= exceptions;
 291
 292         fmxr(FPSCR, fpscr);
 293
 294 #define RAISE(stat,en,sig)                              \
 295         if (exceptions & stat && fpscr & en)            \
 296                 si_code = sig;
 297
 298         /*
 299          * These are arranged in priority order, least to highest.
 300          */
 301         RAISE(FPSCR_DZC, FPSCR_DZE, FPE_FLTDIV);
 302         RAISE(FPSCR_IXC, FPSCR_IXE, FPE_FLTRES);
 303         RAISE(FPSCR_UFC, FPSCR_UFE, FPE_FLTUND);
 304         RAISE(FPSCR_OFC, FPSCR_OFE, FPE_FLTOVF);
 305         RAISE(FPSCR_IOC, FPSCR_IOE, FPE_FLTINV);
 306
 307         return si_code;
 308 }
 309
 310 /*
 311  * Emulate a VFP instruction.
 312  */
 313 static u32 vfp_emulate_instruction(u32 inst, u32 fpscr, struct pt_regs *regs)
 314 {
 315         u32 exceptions = VFP_EXCEPTION_ERROR;
 316
 317         pr_debug("VFP: emulate: INST=0x%08x SCR=0x%08x\n", inst, fpscr);
 318
 319         if (INST_CPRTDO(inst)) {
 320                 if (!INST_CPRT(inst)) {
 321                         /*
 322                          * CPDO
 323                          */
 324                         if (vfp_single(inst)) {
 325                                 exceptions = vfp_single_cpdo(inst, fpscr);
 326                         } else {
 327                                 exceptions = vfp_double_cpdo(inst, fpscr);
 328                         }
 329                 } else {
 330                         /*
 331                          * A CPRT instruction can not appear in FPINST2, nor
 332                          * can it cause an exception.  Therefore, we do not
 333                          * have to emulate it.
 334                          */
 335                 }
 336         } else {
 337                 /*
 338                  * A CPDT instruction can not appear in FPINST2, nor can
 339                  * it cause an exception.  Therefore, we do not have to
 340                  * emulate it.
 341                  */
 342         }
 343         perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->ARM_pc);
 344         return exceptions & ~VFP_NAN_FLAG;
 345 }
 346
 347 /*
 348  * Package up a bounce condition.
 349  */
 350 static void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs)
 351 {
 352         u32 fpscr, orig_fpscr, fpsid, exceptions;
 353         int si_code2 = 0;
 354         int si_code = 0;
 355
 356         pr_debug("VFP: bounce: trigger %08x fpexc %08x\n", trigger, fpexc);
 357
 358         /*
 359          * At this point, FPEXC can have the following configuration:
 360          *
 361          *  EX DEX IXE
 362          *  0   1   x   - synchronous exception
 363          *  1   x   0   - asynchronous exception
 364          *  1   x   1   - sychronous on VFP subarch 1 and asynchronous on later
 365          *  0   0   1   - synchronous on VFP9 (non-standard subarch 1
 366          *                implementation), undefined otherwise
 367          *
 368          * Clear various bits and enable access to the VFP so we can
 369          * handle the bounce.
 370          */
 371         fmxr(FPEXC, fpexc & ~(FPEXC_EX|FPEXC_DEX|FPEXC_FP2V|FPEXC_VV|FPEXC_TRAP_MASK));
 372
 373         fpsid = fmrx(FPSID);
 374         orig_fpscr = fpscr = fmrx(FPSCR);
 375
 376         /*
 377          * Check for the special VFP subarch 1 and FPSCR.IXE bit case
 378          */
 379         if ((fpsid & FPSID_ARCH_MASK) == (1 << FPSID_ARCH_BIT)
 380             && (fpscr & FPSCR_IXE)) {
 381                 /*
 382                  * Synchronous exception, emulate the trigger instruction
 383                  */
 384                 goto emulate;
 385         }
 386
 387         if (fpexc & FPEXC_EX) {
 388                 /*
 389                  * Asynchronous exception. The instruction is read from FPINST
 390                  * and the interrupted instruction has to be restarted.
 391                  */
 392                 trigger = fmrx(FPINST);
 393                 regs->ARM_pc -= 4;
 394         } else if (!(fpexc & FPEXC_DEX)) {
 395                 /*
 396                  * Illegal combination of bits. It can be caused by an
 397                  * unallocated VFP instruction but with FPSCR.IXE set and not
 398                  * on VFP subarch 1.
 399                  */
 400                 si_code = vfp_raise_exceptions(VFP_EXCEPTION_ERROR, trigger, fpscr);
 401                 goto exit;
 402         }
 403
 404         /*
 405          * Modify fpscr to indicate the number of iterations remaining.
 406          * If FPEXC.EX is 0, FPEXC.DEX is 1 and the FPEXC.VV bit indicates
 407          * whether FPEXC.VECITR or FPSCR.LEN is used.
 408          */
 409         if (fpexc & (FPEXC_EX | FPEXC_VV)) {
 410                 u32 len;
 411
 412                 len = fpexc + (1 << FPEXC_LENGTH_BIT);
 413
 414                 fpscr &= ~FPSCR_LENGTH_MASK;
 415                 fpscr |= (len & FPEXC_LENGTH_MASK) << (FPSCR_LENGTH_BIT - FPEXC_LENGTH_BIT);
 416         }
 417
 418         /*
 419          * Handle the first FP instruction.  We used to take note of the
 420          * FPEXC bounce reason, but this appears to be unreliable.
 421          * Emulate the bounced instruction instead.
 422          */
 423         exceptions = vfp_emulate_instruction(trigger, fpscr, regs);
 424         if (exceptions)
 425                 si_code2 = vfp_raise_exceptions(exceptions, trigger, orig_fpscr);
 426
 427         /*
 428          * If there isn't a second FP instruction, exit now. Note that
 429          * the FPEXC.FP2V bit is valid only if FPEXC.EX is 1.
 430          */
 431         if ((fpexc & (FPEXC_EX | FPEXC_FP2V)) != (FPEXC_EX | FPEXC_FP2V))
 432                 goto exit;
 433
 434         /*
 435          * The barrier() here prevents fpinst2 being read
 436          * before the condition above.
 437          */
 438         barrier();
 439         trigger = fmrx(FPINST2);
 440
 441  emulate:
 442         exceptions = vfp_emulate_instruction(trigger, orig_fpscr, regs);
 443         if (exceptions)
 444                 si_code = vfp_raise_exceptions(exceptions, trigger, orig_fpscr);
 445 exit:
 446         vfp_state_release();
 447         if (si_code2)
 448                 vfp_raise_sigfpe(si_code2, regs);
 449         if (si_code)
 450                 vfp_raise_sigfpe(si_code, regs);
 451 }
 452
 453 static void vfp_enable(void *unused)
 454 {
 455         u32 access;
 456
 457         BUG_ON(preemptible());
 458         access = get_copro_access();
 459
 460         /*
 461          * Enable full access to VFP (cp10 and cp11)
 462          */
 463         set_copro_access(access | CPACC_FULL(10) | CPACC_FULL(11));
 464 }
 465
 466 /* Called by platforms on which we want to disable VFP because it may not be
 467  * present on all CPUs within a SMP complex. Needs to be called prior to
 468  * vfp_init().
 469  */
 470 void __init vfp_disable(void)
 471 {
 472         if (VFP_arch) {
 473                 pr_debug("%s: should be called prior to vfp_init\n", __func__);
 474                 return;
 475         }
 476         VFP_arch = 1;
 477 }
 478
 479 #ifdef CONFIG_CPU_PM
 480 static int vfp_pm_suspend(void)
 481 {
 482         struct thread_info *ti = current_thread_info();
 483         u32 fpexc = fmrx(FPEXC);
 484
 485         /* if vfp is on, then save state for resumption */
 486         if (fpexc & FPEXC_EN) {
 487                 pr_debug("%s: saving vfp state\n", __func__);
 488                 vfp_save_state(&ti->vfpstate, fpexc);
 489
 490                 /* disable, just in case */
 491                 fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
 492         } else if (vfp_current_hw_state[ti->cpu]) {
 493 #ifndef CONFIG_SMP
 494                 fmxr(FPEXC, fpexc | FPEXC_EN);
 495                 vfp_save_state(vfp_current_hw_state[ti->cpu], fpexc);
 496                 fmxr(FPEXC, fpexc);
 497 #endif
 498         }
 499
 500         /* clear any information we had about last context state */
 501         vfp_current_hw_state[ti->cpu] = NULL;
 502
 503         return 0;
 504 }
 505
 506 static void vfp_pm_resume(void)
 507 {
 508         /* ensure we have access to the vfp */
 509         vfp_enable(NULL);
 510
 511         /* and disable it to ensure the next usage restores the state */
 512         fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
 513 }
 514
 515 static int vfp_cpu_pm_notifier(struct notifier_block *self, unsigned long cmd,
 516         void *v)
 517 {
 518         switch (cmd) {
 519         case CPU_PM_ENTER:
 520                 vfp_pm_suspend();
 521                 break;
 522         case CPU_PM_ENTER_FAILED:
 523         case CPU_PM_EXIT:
 524                 vfp_pm_resume();
 525                 break;
 526         }
 527         return NOTIFY_OK;
 528 }
 529
 530 static struct notifier_block vfp_cpu_pm_notifier_block = {
 531         .notifier_call = vfp_cpu_pm_notifier,
 532 };
 533
 534 static void vfp_pm_init(void)
 535 {
 536         cpu_pm_register_notifier(&vfp_cpu_pm_notifier_block);
 537 }
 538
 539 #else
 540 static inline void vfp_pm_init(void) { }
 541 #endif /* CONFIG_CPU_PM */
 542
 543 /*
 544  * Ensure that the VFP state stored in 'thread->vfpstate' is up to date
 545  * with the hardware state.
 546  */
 547 void vfp_sync_hwstate(struct thread_info *thread)
 548 {
 549         vfp_state_hold();
 550
 551         if (vfp_state_in_hw(raw_smp_processor_id(), thread)) {
 552                 u32 fpexc = fmrx(FPEXC);
 553
 554                 /*
 555                  * Save the last VFP state on this CPU.
 556                  */
 557                 fmxr(FPEXC, fpexc | FPEXC_EN);
 558                 vfp_save_state(&thread->vfpstate, fpexc | FPEXC_EN);
 559                 fmxr(FPEXC, fpexc);
 560         }
 561
 562         vfp_state_release();
 563 }
 564
 565 /* Ensure that the thread reloads the hardware VFP state on the next use. */
 566 void vfp_flush_hwstate(struct thread_info *thread)
 567 {
 568         unsigned int cpu = get_cpu();
 569
 570         vfp_force_reload(cpu, thread);
 571
 572         put_cpu();
 573 }
 574
 575 /*
 576  * Save the current VFP state into the provided structures and prepare
 577  * for entry into a new function (signal handler).
 578  */
 579 int vfp_preserve_user_clear_hwstate(struct user_vfp *ufp,
 580                                     struct user_vfp_exc *ufp_exc)
 581 {
 582         struct thread_info *thread = current_thread_info();
 583         struct vfp_hard_struct *hwstate = &thread->vfpstate.hard;
 584
 585         /* Ensure that the saved hwstate is up-to-date. */
 586         vfp_sync_hwstate(thread);
 587
 588         /*
 589          * Copy the floating point registers. There can be unused
 590          * registers see asm/hwcap.h for details.
 591          */
 592         memcpy(&ufp->fpregs, &hwstate->fpregs, sizeof(hwstate->fpregs));
 593
 594         /*
 595          * Copy the status and control register.
 596          */
 597         ufp->fpscr = hwstate->fpscr;
 598
 599         /*
 600          * Copy the exception registers.
 601          */
 602         ufp_exc->fpexc = hwstate->fpexc;
 603         ufp_exc->fpinst = hwstate->fpinst;
 604         ufp_exc->fpinst2 = hwstate->fpinst2;
 605
 606         /* Ensure that VFP is disabled. */
 607         vfp_flush_hwstate(thread);
 608
 609         /*
 610          * As per the PCS, clear the length and stride bits for function
 611          * entry.
 612          */
 613         hwstate->fpscr &= ~(FPSCR_LENGTH_MASK | FPSCR_STRIDE_MASK);
 614         return 0;
 615 }
 616
 617 /* Sanitise and restore the current VFP state from the provided structures. */
 618 int vfp_restore_user_hwstate(struct user_vfp *ufp, struct user_vfp_exc *ufp_exc)
 619 {
 620         struct thread_info *thread = current_thread_info();
 621         struct vfp_hard_struct *hwstate = &thread->vfpstate.hard;
 622         unsigned long fpexc;
 623
 624         /* Disable VFP to avoid corrupting the new thread state. */
 625         vfp_flush_hwstate(thread);
 626
 627         /*
 628          * Copy the floating point registers. There can be unused
 629          * registers see asm/hwcap.h for details.
 630          */
 631         memcpy(&hwstate->fpregs, &ufp->fpregs, sizeof(hwstate->fpregs));
 632         /*
 633          * Copy the status and control register.
 634          */
 635         hwstate->fpscr = ufp->fpscr;
 636
 637         /*
 638          * Sanitise and restore the exception registers.
 639          */
 640         fpexc = ufp_exc->fpexc;
 641
 642         /* Ensure the VFP is enabled. */
 643         fpexc |= FPEXC_EN;
 644
 645         /* Ensure FPINST2 is invalid and the exception flag is cleared. */
 646         fpexc &= ~(FPEXC_EX | FPEXC_FP2V);
 647         hwstate->fpexc = fpexc;
 648
 649         hwstate->fpinst = ufp_exc->fpinst;
 650         hwstate->fpinst2 = ufp_exc->fpinst2;
 651
 652         return 0;
 653 }
 654
 655 /*
 656  * VFP hardware can lose all context when a CPU goes offline.
 657  * As we will be running in SMP mode with CPU hotplug, we will save the
 658  * hardware state at every thread switch.  We clear our held state when
 659  * a CPU has been killed, indicating that the VFP hardware doesn't contain
 660  * a threads VFP state.  When a CPU starts up, we re-enable access to the
 661  * VFP hardware. The callbacks below are called on the CPU which
 662  * is being offlined/onlined.
 663  */
 664 static int vfp_dying_cpu(unsigned int cpu)
 665 {
 666         vfp_current_hw_state[cpu] = NULL;
 667         return 0;
 668 }
 669
 670 static int vfp_starting_cpu(unsigned int unused)
 671 {
 672         vfp_enable(NULL);
 673         return 0;
 674 }
 675
 676 static int vfp_kmode_exception(struct pt_regs *regs, unsigned int instr)
 677 {
 678         /*
 679          * If we reach this point, a floating point exception has been raised
 680          * while running in kernel mode. If the NEON/VFP unit was enabled at the
 681          * time, it means a VFP instruction has been issued that requires
 682          * software assistance to complete, something which is not currently
 683          * supported in kernel mode.
 684          * If the NEON/VFP unit was disabled, and the location pointed to below
 685          * is properly preceded by a call to kernel_neon_begin(), something has
 686          * caused the task to be scheduled out and back in again. In this case,
 687          * rebuilding and running with CONFIG_DEBUG_ATOMIC_SLEEP enabled should
 688          * be helpful in localizing the problem.
 689          */
 690         if (fmrx(FPEXC) & FPEXC_EN)
 691                 pr_crit("BUG: unsupported FP instruction in kernel mode\n");
 692         else
 693                 pr_crit("BUG: FP instruction issued in kernel mode with FP unit disabled\n");
 694         pr_crit("FPEXC == 0x%08x\n", fmrx(FPEXC));
 695         return 1;
 696 }
 697
 698 /*
 699  * vfp_support_entry - Handle VFP exception
 700  *
 701  * @regs:       pt_regs structure holding the register state at exception entry
 702  * @trigger:    The opcode of the instruction that triggered the exception
 703  *
 704  * Returns 0 if the exception was handled, or an error code otherwise.
 705  */
 706 static int vfp_support_entry(struct pt_regs *regs, u32 trigger)
 707 {
 708         struct thread_info *ti = current_thread_info();
 709         u32 fpexc;
 710
 711         if (unlikely(!have_vfp))
 712                 return -ENODEV;
 713
 714         if (!user_mode(regs))
 715                 return vfp_kmode_exception(regs, trigger);
 716
 717         vfp_state_hold();
 718         fpexc = fmrx(FPEXC);
 719
 720         /*
 721          * If the VFP unit was not enabled yet, we have to check whether the
 722          * VFP state in the CPU's registers is the most recent VFP state
 723          * associated with the process. On UP systems, we don't save the VFP
 724          * state eagerly on a context switch, so we may need to save the
 725          * VFP state to memory first, as it may belong to another process.
 726          */
 727         if (!(fpexc & FPEXC_EN)) {
 728                 /*
 729                  * Enable the VFP unit but mask the FP exception flag for the
 730                  * time being, so we can access all the registers.
 731                  */
 732                 fpexc |= FPEXC_EN;
 733                 fmxr(FPEXC, fpexc & ~FPEXC_EX);
 734
 735                 /*
 736                  * Check whether or not the VFP state in the CPU's registers is
 737                  * the most recent VFP state associated with this task. On SMP,
 738                  * migration may result in multiple CPUs holding VFP states
 739                  * that belong to the same task, but only the most recent one
 740                  * is valid.
 741                  */
 742                 if (!vfp_state_in_hw(ti->cpu, ti)) {
 743                         if (!IS_ENABLED(CONFIG_SMP) &&
 744                             vfp_current_hw_state[ti->cpu] != NULL) {
 745                                 /*
 746                                  * This CPU is currently holding the most
 747                                  * recent VFP state associated with another
 748                                  * task, and we must save that to memory first.
 749                                  */
 750                                 vfp_save_state(vfp_current_hw_state[ti->cpu],
 751                                                fpexc);
 752                         }
 753
 754                         /*
 755                          * We can now proceed with loading the task's VFP state
 756                          * from memory into the CPU registers.
 757                          */
 758                         fpexc = vfp_load_state(&ti->vfpstate);
 759                         vfp_current_hw_state[ti->cpu] = &ti->vfpstate;
 760 #ifdef CONFIG_SMP
 761                         /*
 762                          * Record that this CPU is now the one holding the most
 763                          * recent VFP state of the task.
 764                          */
 765                         ti->vfpstate.hard.cpu = ti->cpu;
 766 #endif
 767                 }
 768
 769                 if (fpexc & FPEXC_EX)
 770                         /*
 771                          * Might as well handle the pending exception before
 772                          * retrying branch out before setting an FPEXC that
 773                          * stops us reading stuff.
 774                          */
 775                         goto bounce;
 776
 777                 /*
 778                  * No FP exception is pending: just enable the VFP and
 779                  * replay the instruction that trapped.
 780                  */
 781                 fmxr(FPEXC, fpexc);
 782                 vfp_state_release();
 783         } else {
 784                 /* Check for synchronous or asynchronous exceptions */
 785                 if (!(fpexc & (FPEXC_EX | FPEXC_DEX))) {
 786                         u32 fpscr = fmrx(FPSCR);
 787
 788                         /*
 789                          * On some implementations of the VFP subarch 1,
 790                          * setting FPSCR.IXE causes all the CDP instructions to
 791                          * be bounced synchronously without setting the
 792                          * FPEXC.EX bit
 793                          */
 794                         if (!(fpscr & FPSCR_IXE)) {
 795                                 if (!(fpscr & FPSCR_LENGTH_MASK)) {
 796                                         pr_debug("not VFP\n");
 797                                         vfp_state_release();
 798                                         return -ENOEXEC;
 799                                 }
 800                                 fpexc |= FPEXC_DEX;
 801                         }
 802                 }
 803 bounce:         regs->ARM_pc += 4;
 804                 /* VFP_bounce() will invoke vfp_state_release() */
 805                 VFP_bounce(trigger, fpexc, regs);
 806         }
 807
 808         return 0;
 809 }
 810
 811 static struct undef_hook neon_support_hook[] = {{
 812         .instr_mask     = 0xfe000000,
 813         .instr_val      = 0xf2000000,
 814         .cpsr_mask      = PSR_T_BIT,
 815         .cpsr_val       = 0,
 816         .fn             = vfp_support_entry,
 817 }, {
 818         .instr_mask     = 0xff100000,
 819         .instr_val      = 0xf4000000,
 820         .cpsr_mask      = PSR_T_BIT,
 821         .cpsr_val       = 0,
 822         .fn             = vfp_support_entry,
 823 }, {
 824         .instr_mask     = 0xef000000,
 825         .instr_val      = 0xef000000,
 826         .cpsr_mask      = PSR_T_BIT,
 827         .cpsr_val       = PSR_T_BIT,
 828         .fn             = vfp_support_entry,
 829 }, {
 830         .instr_mask     = 0xff100000,
 831         .instr_val      = 0xf9000000,
 832         .cpsr_mask      = PSR_T_BIT,
 833         .cpsr_val       = PSR_T_BIT,
 834         .fn             = vfp_support_entry,
 835 }, {
 836         .instr_mask     = 0xff000800,
 837         .instr_val      = 0xfc000800,
 838         .cpsr_mask      = 0,
 839         .cpsr_val       = 0,
 840         .fn             = vfp_support_entry,
 841 }, {
 842         .instr_mask     = 0xff000800,
 843         .instr_val      = 0xfd000800,
 844         .cpsr_mask      = 0,
 845         .cpsr_val       = 0,
 846         .fn             = vfp_support_entry,
 847 }, {
 848         .instr_mask     = 0xff000800,
 849         .instr_val      = 0xfe000800,
 850         .cpsr_mask      = 0,
 851         .cpsr_val       = 0,
 852         .fn             = vfp_support_entry,
 853 }};
 854
 855 static struct undef_hook vfp_support_hook = {
 856         .instr_mask     = 0x0c000e00,
 857         .instr_val      = 0x0c000a00,
 858         .fn             = vfp_support_entry,
 859 };
 860
 861 #ifdef CONFIG_KERNEL_MODE_NEON
 862
 863 /*
 864  * Kernel-side NEON support functions
 865  */
 866 void kernel_neon_begin(void)
 867 {
 868         struct thread_info *thread = current_thread_info();
 869         unsigned int cpu;
 870         u32 fpexc;
 871
 872         vfp_state_hold();
 873
 874         /*
 875          * Kernel mode NEON is only allowed outside of hardirq context with
 876          * preemption and softirq processing disabled. This will make sure that
 877          * the kernel mode NEON register contents never need to be preserved.
 878          */
 879         BUG_ON(in_hardirq());
 880         cpu = __smp_processor_id();
 881
 882         fpexc = fmrx(FPEXC) | FPEXC_EN;
 883         fmxr(FPEXC, fpexc);
 884
 885         /*
 886          * Save the userland NEON/VFP state. Under UP,
 887          * the owner could be a task other than 'current'
 888          */
 889         if (vfp_state_in_hw(cpu, thread))
 890                 vfp_save_state(&thread->vfpstate, fpexc);
 891 #ifndef CONFIG_SMP
 892         else if (vfp_current_hw_state[cpu] != NULL)
 893                 vfp_save_state(vfp_current_hw_state[cpu], fpexc);
 894 #endif
 895         vfp_current_hw_state[cpu] = NULL;
 896 }
 897 EXPORT_SYMBOL(kernel_neon_begin);
 898
 899 void kernel_neon_end(void)
 900 {
 901         /* Disable the NEON/VFP unit. */
 902         fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
 903         vfp_state_release();
 904 }
 905 EXPORT_SYMBOL(kernel_neon_end);
 906
 907 #endif /* CONFIG_KERNEL_MODE_NEON */
 908
 909 static int __init vfp_detect(struct pt_regs *regs, unsigned int instr)
 910 {
 911         VFP_arch = UINT_MAX;    /* mark as not present */
 912         regs->ARM_pc += 4;
 913         return 0;
 914 }
 915
 916 static struct undef_hook vfp_detect_hook __initdata = {
 917         .instr_mask     = 0x0c000e00,
 918         .instr_val      = 0x0c000a00,
 919         .cpsr_mask      = MODE_MASK,
 920         .cpsr_val       = SVC_MODE,
 921         .fn             = vfp_detect,
 922 };
 923
 924 /*
 925  * VFP support code initialisation.
 926  */
 927 static int __init vfp_init(void)
 928 {
 929         unsigned int vfpsid;
 930         unsigned int cpu_arch = cpu_architecture();
 931         unsigned int isar6;
 932
 933         /*
 934          * Enable the access to the VFP on all online CPUs so the
 935          * following test on FPSID will succeed.
 936          */
 937         if (cpu_arch >= CPU_ARCH_ARMv6)
 938                 on_each_cpu(vfp_enable, NULL, 1);
 939
 940         /*
 941          * First check that there is a VFP that we can use.
 942          * The handler is already setup to just log calls, so
 943          * we just need to read the VFPSID register.
 944          */
 945         register_undef_hook(&vfp_detect_hook);
 946         barrier();
 947         vfpsid = fmrx(FPSID);
 948         barrier();
 949         unregister_undef_hook(&vfp_detect_hook);
 950
 951         pr_info("VFP support v0.3: ");
 952         if (VFP_arch) {
 953                 pr_cont("not present\n");
 954                 return 0;
 955         /* Extract the architecture on CPUID scheme */
 956         } else if ((read_cpuid_id() & 0x000f0000) == 0x000f0000) {
 957                 VFP_arch = vfpsid & FPSID_CPUID_ARCH_MASK;
 958                 VFP_arch >>= FPSID_ARCH_BIT;
 959                 /*
 960                  * Check for the presence of the Advanced SIMD
 961                  * load/store instructions, integer and single
 962                  * precision floating point operations. Only check
 963                  * for NEON if the hardware has the MVFR registers.
 964                  */
 965                 if (IS_ENABLED(CONFIG_NEON) &&
 966                     (fmrx(MVFR1) & 0x000fff00) == 0x00011100) {
 967                         elf_hwcap |= HWCAP_NEON;
 968                         for (int i = 0; i < ARRAY_SIZE(neon_support_hook); i++)
 969                                 register_undef_hook(&neon_support_hook[i]);
 970                 }
 971
 972                 if (IS_ENABLED(CONFIG_VFPv3)) {
 973                         u32 mvfr0 = fmrx(MVFR0);
 974                         if (((mvfr0 & MVFR0_DP_MASK) >> MVFR0_DP_BIT) == 0x2 ||
 975                             ((mvfr0 & MVFR0_SP_MASK) >> MVFR0_SP_BIT) == 0x2) {
 976                                 elf_hwcap |= HWCAP_VFPv3;
 977                                 /*
 978                                  * Check for VFPv3 D16 and VFPv4 D16.  CPUs in
 979                                  * this configuration only have 16 x 64bit
 980                                  * registers.
 981                                  */
 982                                 if ((mvfr0 & MVFR0_A_SIMD_MASK) == 1)
 983                                         /* also v4-D16 */
 984                                         elf_hwcap |= HWCAP_VFPv3D16;
 985                                 else
 986                                         elf_hwcap |= HWCAP_VFPD32;
 987                         }
 988
 989                         if ((fmrx(MVFR1) & 0xf0000000) == 0x10000000)
 990                                 elf_hwcap |= HWCAP_VFPv4;
 991                         if (((fmrx(MVFR1) & MVFR1_ASIMDHP_MASK) >> MVFR1_ASIMDHP_BIT) == 0x2)
 992                                 elf_hwcap |= HWCAP_ASIMDHP;
 993                         if (((fmrx(MVFR1) & MVFR1_FPHP_MASK) >> MVFR1_FPHP_BIT) == 0x3)
 994                                 elf_hwcap |= HWCAP_FPHP;
 995                 }
 996
 997                 /*
 998                  * Check for the presence of Advanced SIMD Dot Product
 999                  * instructions.
1000                  */
1001                 isar6 = read_cpuid_ext(CPUID_EXT_ISAR6);
1002                 if (cpuid_feature_extract_field(isar6, 4) == 0x1)
1003                         elf_hwcap |= HWCAP_ASIMDDP;
1004                 /*
1005                  * Check for the presence of Advanced SIMD Floating point
1006                  * half-precision multiplication instructions.
1007                  */
1008                 if (cpuid_feature_extract_field(isar6, 8) == 0x1)
1009                         elf_hwcap |= HWCAP_ASIMDFHM;
1010                 /*
1011                  * Check for the presence of Advanced SIMD Bfloat16
1012                  * floating point instructions.
1013                  */
1014                 if (cpuid_feature_extract_field(isar6, 20) == 0x1)
1015                         elf_hwcap |= HWCAP_ASIMDBF16;
1016                 /*
1017                  * Check for the presence of Advanced SIMD and floating point
1018                  * Int8 matrix multiplication instructions instructions.
1019                  */
1020                 if (cpuid_feature_extract_field(isar6, 24) == 0x1)
1021                         elf_hwcap |= HWCAP_I8MM;
1022
1023         /* Extract the architecture version on pre-cpuid scheme */
1024         } else {
1025                 if (vfpsid & FPSID_NODOUBLE) {
1026                         pr_cont("no double precision support\n");
1027                         return 0;
1028                 }
1029
1030                 VFP_arch = (vfpsid & FPSID_ARCH_MASK) >> FPSID_ARCH_BIT;
1031         }
1032
1033         cpuhp_setup_state_nocalls(CPUHP_AP_ARM_VFP_STARTING,
1034                                   "arm/vfp:starting", vfp_starting_cpu,
1035                                   vfp_dying_cpu);
1036
1037         have_vfp = true;
1038
1039         register_undef_hook(&vfp_support_hook);
1040         thread_register_notifier(&vfp_notifier_block);
1041         vfp_pm_init();
1042
1043         /*
1044          * We detected VFP, and the support code is
1045          * in place; report VFP support to userspace.
1046          */
1047         elf_hwcap |= HWCAP_VFP;
1048
1049         pr_cont("implementor %02x architecture %d part %02x variant %x rev %x\n",
1050                 (vfpsid & FPSID_IMPLEMENTER_MASK) >> FPSID_IMPLEMENTER_BIT,
1051                 VFP_arch,
1052                 (vfpsid & FPSID_PART_MASK) >> FPSID_PART_BIT,
1053                 (vfpsid & FPSID_VARIANT_MASK) >> FPSID_VARIANT_BIT,
1054                 (vfpsid & FPSID_REV_MASK) >> FPSID_REV_BIT);
1055
1056         return 0;
1057 }
1058
1059 core_initcall(vfp_init);