arch/x86/kernel/cpu/bugs.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *  Copyright (C) 1994  Linus Torvalds
   4  *
   5  *  Cyrix stuff, June 1998 by:
   6  *      - Rafael R. Reilova (moved everything from head.S),
   7  *        <[email protected]>
   8  *      - Channing Corn (tests & fixes),
   9  *      - Andrew D. Balsa (code cleanup).
  10  */
  11 #include <linux/init.h>
  12 #include <linux/cpu.h>
  13 #include <linux/module.h>
  14 #include <linux/nospec.h>
  15 #include <linux/prctl.h>
  16 #include <linux/sched/smt.h>
  17 #include <linux/pgtable.h>
  18 #include <linux/bpf.h>
  19
  20 #include <asm/spec-ctrl.h>
  21 #include <asm/cmdline.h>
  22 #include <asm/bugs.h>
  23 #include <asm/processor.h>
  24 #include <asm/processor-flags.h>
  25 #include <asm/fpu/api.h>
  26 #include <asm/msr.h>
  27 #include <asm/vmx.h>
  28 #include <asm/paravirt.h>
  29 #include <asm/cpu_device_id.h>
  30 #include <asm/e820/api.h>
  31 #include <asm/hypervisor.h>
  32 #include <asm/tlbflush.h>
  33 #include <asm/cpu.h>
  34
  35 #include "cpu.h"
  36
  37 static void __init spectre_v1_select_mitigation(void);
  38 static void __init spectre_v2_select_mitigation(void);
  39 static void __init retbleed_select_mitigation(void);
  40 static void __init spectre_v2_user_select_mitigation(void);
  41 static void __init ssb_select_mitigation(void);
  42 static void __init l1tf_select_mitigation(void);
  43 static void __init mds_select_mitigation(void);
  44 static void __init md_clear_update_mitigation(void);
  45 static void __init md_clear_select_mitigation(void);
  46 static void __init taa_select_mitigation(void);
  47 static void __init mmio_select_mitigation(void);
  48 static void __init srbds_select_mitigation(void);
  49 static void __init l1d_flush_select_mitigation(void);
  50 static void __init srso_select_mitigation(void);
  51 static void __init gds_select_mitigation(void);
  52
  53 /* The base value of the SPEC_CTRL MSR without task-specific bits set */
  54 u64 x86_spec_ctrl_base;
  55 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base);
  56
  57 /* The current value of the SPEC_CTRL MSR with task-specific bits set */
  58 DEFINE_PER_CPU(u64, x86_spec_ctrl_current);
  59 EXPORT_PER_CPU_SYMBOL_GPL(x86_spec_ctrl_current);
  60
  61 u64 x86_pred_cmd __ro_after_init = PRED_CMD_IBPB;
  62 EXPORT_SYMBOL_GPL(x86_pred_cmd);
  63
  64 static u64 __ro_after_init x86_arch_cap_msr;
  65
  66 static DEFINE_MUTEX(spec_ctrl_mutex);
  67
  68 void (*x86_return_thunk)(void) __ro_after_init = __x86_return_thunk;
  69
  70 /* Update SPEC_CTRL MSR and its cached copy unconditionally */
  71 static void update_spec_ctrl(u64 val)
  72 {
  73         this_cpu_write(x86_spec_ctrl_current, val);
  74         wrmsrl(MSR_IA32_SPEC_CTRL, val);
  75 }
  76
  77 /*
  78  * Keep track of the SPEC_CTRL MSR value for the current task, which may differ
  79  * from x86_spec_ctrl_base due to STIBP/SSB in __speculation_ctrl_update().
  80  */
  81 void update_spec_ctrl_cond(u64 val)
  82 {
  83         if (this_cpu_read(x86_spec_ctrl_current) == val)
  84                 return;
  85
  86         this_cpu_write(x86_spec_ctrl_current, val);
  87
  88         /*
  89          * When KERNEL_IBRS this MSR is written on return-to-user, unless
  90          * forced the update can be delayed until that time.
  91          */
  92         if (!cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS))
  93                 wrmsrl(MSR_IA32_SPEC_CTRL, val);
  94 }
  95
  96 noinstr u64 spec_ctrl_current(void)
  97 {
  98         return this_cpu_read(x86_spec_ctrl_current);
  99 }
 100 EXPORT_SYMBOL_GPL(spec_ctrl_current);
 101
 102 /*
 103  * AMD specific MSR info for Speculative Store Bypass control.
 104  * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu().
 105  */
 106 u64 __ro_after_init x86_amd_ls_cfg_base;
 107 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask;
 108
 109 /* Control conditional STIBP in switch_to() */
 110 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp);
 111 /* Control conditional IBPB in switch_mm() */
 112 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
 113 /* Control unconditional IBPB in switch_mm() */
 114 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
 115
 116 /* Control MDS CPU buffer clear before idling (halt, mwait) */
 117 DEFINE_STATIC_KEY_FALSE(mds_idle_clear);
 118 EXPORT_SYMBOL_GPL(mds_idle_clear);
 119
 120 /*
 121  * Controls whether l1d flush based mitigations are enabled,
 122  * based on hw features and admin setting via boot parameter
 123  * defaults to false
 124  */
 125 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush);
 126
 127 /* Controls CPU Fill buffer clear before KVM guest MMIO accesses */
 128 DEFINE_STATIC_KEY_FALSE(mmio_stale_data_clear);
 129 EXPORT_SYMBOL_GPL(mmio_stale_data_clear);
 130
 131 void __init cpu_select_mitigations(void)
 132 {
 133         /*
 134          * Read the SPEC_CTRL MSR to account for reserved bits which may
 135          * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD
 136          * init code as it is not enumerated and depends on the family.
 137          */
 138         if (cpu_feature_enabled(X86_FEATURE_MSR_SPEC_CTRL)) {
 139                 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 140
 141                 /*
 142                  * Previously running kernel (kexec), may have some controls
 143                  * turned ON. Clear them and let the mitigations setup below
 144                  * rediscover them based on configuration.
 145                  */
 146                 x86_spec_ctrl_base &= ~SPEC_CTRL_MITIGATIONS_MASK;
 147         }
 148
 149         x86_arch_cap_msr = x86_read_arch_cap_msr();
 150
 151         /* Select the proper CPU mitigations before patching alternatives: */
 152         spectre_v1_select_mitigation();
 153         spectre_v2_select_mitigation();
 154         /*
 155          * retbleed_select_mitigation() relies on the state set by
 156          * spectre_v2_select_mitigation(); specifically it wants to know about
 157          * spectre_v2=ibrs.
 158          */
 159         retbleed_select_mitigation();
 160         /*
 161          * spectre_v2_user_select_mitigation() relies on the state set by
 162          * retbleed_select_mitigation(); specifically the STIBP selection is
 163          * forced for UNRET or IBPB.
 164          */
 165         spectre_v2_user_select_mitigation();
 166         ssb_select_mitigation();
 167         l1tf_select_mitigation();
 168         md_clear_select_mitigation();
 169         srbds_select_mitigation();
 170         l1d_flush_select_mitigation();
 171
 172         /*
 173          * srso_select_mitigation() depends and must run after
 174          * retbleed_select_mitigation().
 175          */
 176         srso_select_mitigation();
 177         gds_select_mitigation();
 178 }
 179
 180 /*
 181  * NOTE: This function is *only* called for SVM, since Intel uses
 182  * MSR_IA32_SPEC_CTRL for SSBD.
 183  */
 184 void
 185 x86_virt_spec_ctrl(u64 guest_virt_spec_ctrl, bool setguest)
 186 {
 187         u64 guestval, hostval;
 188         struct thread_info *ti = current_thread_info();
 189
 190         /*
 191          * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
 192          * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.
 193          */
 194         if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
 195             !static_cpu_has(X86_FEATURE_VIRT_SSBD))
 196                 return;
 197
 198         /*
 199          * If the host has SSBD mitigation enabled, force it in the host's
 200          * virtual MSR value. If its not permanently enabled, evaluate
 201          * current's TIF_SSBD thread flag.
 202          */
 203         if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE))
 204                 hostval = SPEC_CTRL_SSBD;
 205         else
 206                 hostval = ssbd_tif_to_spec_ctrl(ti->flags);
 207
 208         /* Sanitize the guest value */
 209         guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD;
 210
 211         if (hostval != guestval) {
 212                 unsigned long tif;
 213
 214                 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) :
 215                                  ssbd_spec_ctrl_to_tif(hostval);
 216
 217                 speculation_ctrl_update(tif);
 218         }
 219 }
 220 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl);
 221
 222 static void x86_amd_ssb_disable(void)
 223 {
 224         u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask;
 225
 226         if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
 227                 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD);
 228         else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD))
 229                 wrmsrl(MSR_AMD64_LS_CFG, msrval);
 230 }
 231
 232 #undef pr_fmt
 233 #define pr_fmt(fmt)     "MDS: " fmt
 234
 235 /* Default mitigation for MDS-affected CPUs */
 236 static enum mds_mitigations mds_mitigation __ro_after_init =
 237         IS_ENABLED(CONFIG_MITIGATION_MDS) ? MDS_MITIGATION_FULL : MDS_MITIGATION_OFF;
 238 static bool mds_nosmt __ro_after_init = false;
 239
 240 static const char * const mds_strings[] = {
 241         [MDS_MITIGATION_OFF]    = "Vulnerable",
 242         [MDS_MITIGATION_FULL]   = "Mitigation: Clear CPU buffers",
 243         [MDS_MITIGATION_VMWERV] = "Vulnerable: Clear CPU buffers attempted, no microcode",
 244 };
 245
 246 static void __init mds_select_mitigation(void)
 247 {
 248         if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) {
 249                 mds_mitigation = MDS_MITIGATION_OFF;
 250                 return;
 251         }
 252
 253         if (mds_mitigation == MDS_MITIGATION_FULL) {
 254                 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR))
 255                         mds_mitigation = MDS_MITIGATION_VMWERV;
 256
 257                 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF);
 258
 259                 if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) &&
 260                     (mds_nosmt || cpu_mitigations_auto_nosmt()))
 261                         cpu_smt_disable(false);
 262         }
 263 }
 264
 265 static int __init mds_cmdline(char *str)
 266 {
 267         if (!boot_cpu_has_bug(X86_BUG_MDS))
 268                 return 0;
 269
 270         if (!str)
 271                 return -EINVAL;
 272
 273         if (!strcmp(str, "off"))
 274                 mds_mitigation = MDS_MITIGATION_OFF;
 275         else if (!strcmp(str, "full"))
 276                 mds_mitigation = MDS_MITIGATION_FULL;
 277         else if (!strcmp(str, "full,nosmt")) {
 278                 mds_mitigation = MDS_MITIGATION_FULL;
 279                 mds_nosmt = true;
 280         }
 281
 282         return 0;
 283 }
 284 early_param("mds", mds_cmdline);
 285
 286 #undef pr_fmt
 287 #define pr_fmt(fmt)     "TAA: " fmt
 288
 289 enum taa_mitigations {
 290         TAA_MITIGATION_OFF,
 291         TAA_MITIGATION_UCODE_NEEDED,
 292         TAA_MITIGATION_VERW,
 293         TAA_MITIGATION_TSX_DISABLED,
 294 };
 295
 296 /* Default mitigation for TAA-affected CPUs */
 297 static enum taa_mitigations taa_mitigation __ro_after_init =
 298         IS_ENABLED(CONFIG_MITIGATION_TAA) ? TAA_MITIGATION_VERW : TAA_MITIGATION_OFF;
 299 static bool taa_nosmt __ro_after_init;
 300
 301 static const char * const taa_strings[] = {
 302         [TAA_MITIGATION_OFF]            = "Vulnerable",
 303         [TAA_MITIGATION_UCODE_NEEDED]   = "Vulnerable: Clear CPU buffers attempted, no microcode",
 304         [TAA_MITIGATION_VERW]           = "Mitigation: Clear CPU buffers",
 305         [TAA_MITIGATION_TSX_DISABLED]   = "Mitigation: TSX disabled",
 306 };
 307
 308 static void __init taa_select_mitigation(void)
 309 {
 310         if (!boot_cpu_has_bug(X86_BUG_TAA)) {
 311                 taa_mitigation = TAA_MITIGATION_OFF;
 312                 return;
 313         }
 314
 315         /* TSX previously disabled by tsx=off */
 316         if (!boot_cpu_has(X86_FEATURE_RTM)) {
 317                 taa_mitigation = TAA_MITIGATION_TSX_DISABLED;
 318                 return;
 319         }
 320
 321         if (cpu_mitigations_off()) {
 322                 taa_mitigation = TAA_MITIGATION_OFF;
 323                 return;
 324         }
 325
 326         /*
 327          * TAA mitigation via VERW is turned off if both
 328          * tsx_async_abort=off and mds=off are specified.
 329          */
 330         if (taa_mitigation == TAA_MITIGATION_OFF &&
 331             mds_mitigation == MDS_MITIGATION_OFF)
 332                 return;
 333
 334         if (boot_cpu_has(X86_FEATURE_MD_CLEAR))
 335                 taa_mitigation = TAA_MITIGATION_VERW;
 336         else
 337                 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED;
 338
 339         /*
 340          * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1.
 341          * A microcode update fixes this behavior to clear CPU buffers. It also
 342          * adds support for MSR_IA32_TSX_CTRL which is enumerated by the
 343          * ARCH_CAP_TSX_CTRL_MSR bit.
 344          *
 345          * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode
 346          * update is required.
 347          */
 348         if ( (x86_arch_cap_msr & ARCH_CAP_MDS_NO) &&
 349             !(x86_arch_cap_msr & ARCH_CAP_TSX_CTRL_MSR))
 350                 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED;
 351
 352         /*
 353          * TSX is enabled, select alternate mitigation for TAA which is
 354          * the same as MDS. Enable MDS static branch to clear CPU buffers.
 355          *
 356          * For guests that can't determine whether the correct microcode is
 357          * present on host, enable the mitigation for UCODE_NEEDED as well.
 358          */
 359         setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF);
 360
 361         if (taa_nosmt || cpu_mitigations_auto_nosmt())
 362                 cpu_smt_disable(false);
 363 }
 364
 365 static int __init tsx_async_abort_parse_cmdline(char *str)
 366 {
 367         if (!boot_cpu_has_bug(X86_BUG_TAA))
 368                 return 0;
 369
 370         if (!str)
 371                 return -EINVAL;
 372
 373         if (!strcmp(str, "off")) {
 374                 taa_mitigation = TAA_MITIGATION_OFF;
 375         } else if (!strcmp(str, "full")) {
 376                 taa_mitigation = TAA_MITIGATION_VERW;
 377         } else if (!strcmp(str, "full,nosmt")) {
 378                 taa_mitigation = TAA_MITIGATION_VERW;
 379                 taa_nosmt = true;
 380         }
 381
 382         return 0;
 383 }
 384 early_param("tsx_async_abort", tsx_async_abort_parse_cmdline);
 385
 386 #undef pr_fmt
 387 #define pr_fmt(fmt)     "MMIO Stale Data: " fmt
 388
 389 enum mmio_mitigations {
 390         MMIO_MITIGATION_OFF,
 391         MMIO_MITIGATION_UCODE_NEEDED,
 392         MMIO_MITIGATION_VERW,
 393 };
 394
 395 /* Default mitigation for Processor MMIO Stale Data vulnerabilities */
 396 static enum mmio_mitigations mmio_mitigation __ro_after_init =
 397         IS_ENABLED(CONFIG_MITIGATION_MMIO_STALE_DATA) ? MMIO_MITIGATION_VERW : MMIO_MITIGATION_OFF;
 398 static bool mmio_nosmt __ro_after_init = false;
 399
 400 static const char * const mmio_strings[] = {
 401         [MMIO_MITIGATION_OFF]           = "Vulnerable",
 402         [MMIO_MITIGATION_UCODE_NEEDED]  = "Vulnerable: Clear CPU buffers attempted, no microcode",
 403         [MMIO_MITIGATION_VERW]          = "Mitigation: Clear CPU buffers",
 404 };
 405
 406 static void __init mmio_select_mitigation(void)
 407 {
 408         if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA) ||
 409              boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN) ||
 410              cpu_mitigations_off()) {
 411                 mmio_mitigation = MMIO_MITIGATION_OFF;
 412                 return;
 413         }
 414
 415         if (mmio_mitigation == MMIO_MITIGATION_OFF)
 416                 return;
 417
 418         /*
 419          * Enable CPU buffer clear mitigation for host and VMM, if also affected
 420          * by MDS or TAA. Otherwise, enable mitigation for VMM only.
 421          */
 422         if (boot_cpu_has_bug(X86_BUG_MDS) || (boot_cpu_has_bug(X86_BUG_TAA) &&
 423                                               boot_cpu_has(X86_FEATURE_RTM)))
 424                 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF);
 425
 426         /*
 427          * X86_FEATURE_CLEAR_CPU_BUF could be enabled by other VERW based
 428          * mitigations, disable KVM-only mitigation in that case.
 429          */
 430         if (boot_cpu_has(X86_FEATURE_CLEAR_CPU_BUF))
 431                 static_branch_disable(&mmio_stale_data_clear);
 432         else
 433                 static_branch_enable(&mmio_stale_data_clear);
 434
 435         /*
 436          * If Processor-MMIO-Stale-Data bug is present and Fill Buffer data can
 437          * be propagated to uncore buffers, clearing the Fill buffers on idle
 438          * is required irrespective of SMT state.
 439          */
 440         if (!(x86_arch_cap_msr & ARCH_CAP_FBSDP_NO))
 441                 static_branch_enable(&mds_idle_clear);
 442
 443         /*
 444          * Check if the system has the right microcode.
 445          *
 446          * CPU Fill buffer clear mitigation is enumerated by either an explicit
 447          * FB_CLEAR or by the presence of both MD_CLEAR and L1D_FLUSH on MDS
 448          * affected systems.
 449          */
 450         if ((x86_arch_cap_msr & ARCH_CAP_FB_CLEAR) ||
 451             (boot_cpu_has(X86_FEATURE_MD_CLEAR) &&
 452              boot_cpu_has(X86_FEATURE_FLUSH_L1D) &&
 453              !(x86_arch_cap_msr & ARCH_CAP_MDS_NO)))
 454                 mmio_mitigation = MMIO_MITIGATION_VERW;
 455         else
 456                 mmio_mitigation = MMIO_MITIGATION_UCODE_NEEDED;
 457
 458         if (mmio_nosmt || cpu_mitigations_auto_nosmt())
 459                 cpu_smt_disable(false);
 460 }
 461
 462 static int __init mmio_stale_data_parse_cmdline(char *str)
 463 {
 464         if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA))
 465                 return 0;
 466
 467         if (!str)
 468                 return -EINVAL;
 469
 470         if (!strcmp(str, "off")) {
 471                 mmio_mitigation = MMIO_MITIGATION_OFF;
 472         } else if (!strcmp(str, "full")) {
 473                 mmio_mitigation = MMIO_MITIGATION_VERW;
 474         } else if (!strcmp(str, "full,nosmt")) {
 475                 mmio_mitigation = MMIO_MITIGATION_VERW;
 476                 mmio_nosmt = true;
 477         }
 478
 479         return 0;
 480 }
 481 early_param("mmio_stale_data", mmio_stale_data_parse_cmdline);
 482
 483 #undef pr_fmt
 484 #define pr_fmt(fmt)     "Register File Data Sampling: " fmt
 485
 486 enum rfds_mitigations {
 487         RFDS_MITIGATION_OFF,
 488         RFDS_MITIGATION_VERW,
 489         RFDS_MITIGATION_UCODE_NEEDED,
 490 };
 491
 492 /* Default mitigation for Register File Data Sampling */
 493 static enum rfds_mitigations rfds_mitigation __ro_after_init =
 494         IS_ENABLED(CONFIG_MITIGATION_RFDS) ? RFDS_MITIGATION_VERW : RFDS_MITIGATION_OFF;
 495
 496 static const char * const rfds_strings[] = {
 497         [RFDS_MITIGATION_OFF]                   = "Vulnerable",
 498         [RFDS_MITIGATION_VERW]                  = "Mitigation: Clear Register File",
 499         [RFDS_MITIGATION_UCODE_NEEDED]          = "Vulnerable: No microcode",
 500 };
 501
 502 static void __init rfds_select_mitigation(void)
 503 {
 504         if (!boot_cpu_has_bug(X86_BUG_RFDS) || cpu_mitigations_off()) {
 505                 rfds_mitigation = RFDS_MITIGATION_OFF;
 506                 return;
 507         }
 508         if (rfds_mitigation == RFDS_MITIGATION_OFF)
 509                 return;
 510
 511         if (x86_arch_cap_msr & ARCH_CAP_RFDS_CLEAR)
 512                 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF);
 513         else
 514                 rfds_mitigation = RFDS_MITIGATION_UCODE_NEEDED;
 515 }
 516
 517 static __init int rfds_parse_cmdline(char *str)
 518 {
 519         if (!str)
 520                 return -EINVAL;
 521
 522         if (!boot_cpu_has_bug(X86_BUG_RFDS))
 523                 return 0;
 524
 525         if (!strcmp(str, "off"))
 526                 rfds_mitigation = RFDS_MITIGATION_OFF;
 527         else if (!strcmp(str, "on"))
 528                 rfds_mitigation = RFDS_MITIGATION_VERW;
 529
 530         return 0;
 531 }
 532 early_param("reg_file_data_sampling", rfds_parse_cmdline);
 533
 534 #undef pr_fmt
 535 #define pr_fmt(fmt)     "" fmt
 536
 537 static void __init md_clear_update_mitigation(void)
 538 {
 539         if (cpu_mitigations_off())
 540                 return;
 541
 542         if (!boot_cpu_has(X86_FEATURE_CLEAR_CPU_BUF))
 543                 goto out;
 544
 545         /*
 546          * X86_FEATURE_CLEAR_CPU_BUF is now enabled. Update MDS, TAA and MMIO
 547          * Stale Data mitigation, if necessary.
 548          */
 549         if (mds_mitigation == MDS_MITIGATION_OFF &&
 550             boot_cpu_has_bug(X86_BUG_MDS)) {
 551                 mds_mitigation = MDS_MITIGATION_FULL;
 552                 mds_select_mitigation();
 553         }
 554         if (taa_mitigation == TAA_MITIGATION_OFF &&
 555             boot_cpu_has_bug(X86_BUG_TAA)) {
 556                 taa_mitigation = TAA_MITIGATION_VERW;
 557                 taa_select_mitigation();
 558         }
 559         /*
 560          * MMIO_MITIGATION_OFF is not checked here so that mmio_stale_data_clear
 561          * gets updated correctly as per X86_FEATURE_CLEAR_CPU_BUF state.
 562          */
 563         if (boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) {
 564                 mmio_mitigation = MMIO_MITIGATION_VERW;
 565                 mmio_select_mitigation();
 566         }
 567         if (rfds_mitigation == RFDS_MITIGATION_OFF &&
 568             boot_cpu_has_bug(X86_BUG_RFDS)) {
 569                 rfds_mitigation = RFDS_MITIGATION_VERW;
 570                 rfds_select_mitigation();
 571         }
 572 out:
 573         if (boot_cpu_has_bug(X86_BUG_MDS))
 574                 pr_info("MDS: %s\n", mds_strings[mds_mitigation]);
 575         if (boot_cpu_has_bug(X86_BUG_TAA))
 576                 pr_info("TAA: %s\n", taa_strings[taa_mitigation]);
 577         if (boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA))
 578                 pr_info("MMIO Stale Data: %s\n", mmio_strings[mmio_mitigation]);
 579         else if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN))
 580                 pr_info("MMIO Stale Data: Unknown: No mitigations\n");
 581         if (boot_cpu_has_bug(X86_BUG_RFDS))
 582                 pr_info("Register File Data Sampling: %s\n", rfds_strings[rfds_mitigation]);
 583 }
 584
 585 static void __init md_clear_select_mitigation(void)
 586 {
 587         mds_select_mitigation();
 588         taa_select_mitigation();
 589         mmio_select_mitigation();
 590         rfds_select_mitigation();
 591
 592         /*
 593          * As these mitigations are inter-related and rely on VERW instruction
 594          * to clear the microarchitural buffers, update and print their status
 595          * after mitigation selection is done for each of these vulnerabilities.
 596          */
 597         md_clear_update_mitigation();
 598 }
 599
 600 #undef pr_fmt
 601 #define pr_fmt(fmt)     "SRBDS: " fmt
 602
 603 enum srbds_mitigations {
 604         SRBDS_MITIGATION_OFF,
 605         SRBDS_MITIGATION_UCODE_NEEDED,
 606         SRBDS_MITIGATION_FULL,
 607         SRBDS_MITIGATION_TSX_OFF,
 608         SRBDS_MITIGATION_HYPERVISOR,
 609 };
 610
 611 static enum srbds_mitigations srbds_mitigation __ro_after_init =
 612         IS_ENABLED(CONFIG_MITIGATION_SRBDS) ? SRBDS_MITIGATION_FULL : SRBDS_MITIGATION_OFF;
 613
 614 static const char * const srbds_strings[] = {
 615         [SRBDS_MITIGATION_OFF]          = "Vulnerable",
 616         [SRBDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode",
 617         [SRBDS_MITIGATION_FULL]         = "Mitigation: Microcode",
 618         [SRBDS_MITIGATION_TSX_OFF]      = "Mitigation: TSX disabled",
 619         [SRBDS_MITIGATION_HYPERVISOR]   = "Unknown: Dependent on hypervisor status",
 620 };
 621
 622 static bool srbds_off;
 623
 624 void update_srbds_msr(void)
 625 {
 626         u64 mcu_ctrl;
 627
 628         if (!boot_cpu_has_bug(X86_BUG_SRBDS))
 629                 return;
 630
 631         if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
 632                 return;
 633
 634         if (srbds_mitigation == SRBDS_MITIGATION_UCODE_NEEDED)
 635                 return;
 636
 637         /*
 638          * A MDS_NO CPU for which SRBDS mitigation is not needed due to TSX
 639          * being disabled and it hasn't received the SRBDS MSR microcode.
 640          */
 641         if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL))
 642                 return;
 643
 644         rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 645
 646         switch (srbds_mitigation) {
 647         case SRBDS_MITIGATION_OFF:
 648         case SRBDS_MITIGATION_TSX_OFF:
 649                 mcu_ctrl |= RNGDS_MITG_DIS;
 650                 break;
 651         case SRBDS_MITIGATION_FULL:
 652                 mcu_ctrl &= ~RNGDS_MITG_DIS;
 653                 break;
 654         default:
 655                 break;
 656         }
 657
 658         wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 659 }
 660
 661 static void __init srbds_select_mitigation(void)
 662 {
 663         if (!boot_cpu_has_bug(X86_BUG_SRBDS))
 664                 return;
 665
 666         /*
 667          * Check to see if this is one of the MDS_NO systems supporting TSX that
 668          * are only exposed to SRBDS when TSX is enabled or when CPU is affected
 669          * by Processor MMIO Stale Data vulnerability.
 670          */
 671         if ((x86_arch_cap_msr & ARCH_CAP_MDS_NO) && !boot_cpu_has(X86_FEATURE_RTM) &&
 672             !boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA))
 673                 srbds_mitigation = SRBDS_MITIGATION_TSX_OFF;
 674         else if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
 675                 srbds_mitigation = SRBDS_MITIGATION_HYPERVISOR;
 676         else if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL))
 677                 srbds_mitigation = SRBDS_MITIGATION_UCODE_NEEDED;
 678         else if (cpu_mitigations_off() || srbds_off)
 679                 srbds_mitigation = SRBDS_MITIGATION_OFF;
 680
 681         update_srbds_msr();
 682         pr_info("%s\n", srbds_strings[srbds_mitigation]);
 683 }
 684
 685 static int __init srbds_parse_cmdline(char *str)
 686 {
 687         if (!str)
 688                 return -EINVAL;
 689
 690         if (!boot_cpu_has_bug(X86_BUG_SRBDS))
 691                 return 0;
 692
 693         srbds_off = !strcmp(str, "off");
 694         return 0;
 695 }
 696 early_param("srbds", srbds_parse_cmdline);
 697
 698 #undef pr_fmt
 699 #define pr_fmt(fmt)     "L1D Flush : " fmt
 700
 701 enum l1d_flush_mitigations {
 702         L1D_FLUSH_OFF = 0,
 703         L1D_FLUSH_ON,
 704 };
 705
 706 static enum l1d_flush_mitigations l1d_flush_mitigation __initdata = L1D_FLUSH_OFF;
 707
 708 static void __init l1d_flush_select_mitigation(void)
 709 {
 710         if (!l1d_flush_mitigation || !boot_cpu_has(X86_FEATURE_FLUSH_L1D))
 711                 return;
 712
 713         static_branch_enable(&switch_mm_cond_l1d_flush);
 714         pr_info("Conditional flush on switch_mm() enabled\n");
 715 }
 716
 717 static int __init l1d_flush_parse_cmdline(char *str)
 718 {
 719         if (!strcmp(str, "on"))
 720                 l1d_flush_mitigation = L1D_FLUSH_ON;
 721
 722         return 0;
 723 }
 724 early_param("l1d_flush", l1d_flush_parse_cmdline);
 725
 726 #undef pr_fmt
 727 #define pr_fmt(fmt)     "GDS: " fmt
 728
 729 enum gds_mitigations {
 730         GDS_MITIGATION_OFF,
 731         GDS_MITIGATION_UCODE_NEEDED,
 732         GDS_MITIGATION_FORCE,
 733         GDS_MITIGATION_FULL,
 734         GDS_MITIGATION_FULL_LOCKED,
 735         GDS_MITIGATION_HYPERVISOR,
 736 };
 737
 738 static enum gds_mitigations gds_mitigation __ro_after_init =
 739         IS_ENABLED(CONFIG_MITIGATION_GDS) ? GDS_MITIGATION_FULL : GDS_MITIGATION_OFF;
 740
 741 static const char * const gds_strings[] = {
 742         [GDS_MITIGATION_OFF]            = "Vulnerable",
 743         [GDS_MITIGATION_UCODE_NEEDED]   = "Vulnerable: No microcode",
 744         [GDS_MITIGATION_FORCE]          = "Mitigation: AVX disabled, no microcode",
 745         [GDS_MITIGATION_FULL]           = "Mitigation: Microcode",
 746         [GDS_MITIGATION_FULL_LOCKED]    = "Mitigation: Microcode (locked)",
 747         [GDS_MITIGATION_HYPERVISOR]     = "Unknown: Dependent on hypervisor status",
 748 };
 749
 750 bool gds_ucode_mitigated(void)
 751 {
 752         return (gds_mitigation == GDS_MITIGATION_FULL ||
 753                 gds_mitigation == GDS_MITIGATION_FULL_LOCKED);
 754 }
 755 EXPORT_SYMBOL_GPL(gds_ucode_mitigated);
 756
 757 void update_gds_msr(void)
 758 {
 759         u64 mcu_ctrl_after;
 760         u64 mcu_ctrl;
 761
 762         switch (gds_mitigation) {
 763         case GDS_MITIGATION_OFF:
 764                 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 765                 mcu_ctrl |= GDS_MITG_DIS;
 766                 break;
 767         case GDS_MITIGATION_FULL_LOCKED:
 768                 /*
 769                  * The LOCKED state comes from the boot CPU. APs might not have
 770                  * the same state. Make sure the mitigation is enabled on all
 771                  * CPUs.
 772                  */
 773         case GDS_MITIGATION_FULL:
 774                 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 775                 mcu_ctrl &= ~GDS_MITG_DIS;
 776                 break;
 777         case GDS_MITIGATION_FORCE:
 778         case GDS_MITIGATION_UCODE_NEEDED:
 779         case GDS_MITIGATION_HYPERVISOR:
 780                 return;
 781         }
 782
 783         wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 784
 785         /*
 786          * Check to make sure that the WRMSR value was not ignored. Writes to
 787          * GDS_MITG_DIS will be ignored if this processor is locked but the boot
 788          * processor was not.
 789          */
 790         rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl_after);
 791         WARN_ON_ONCE(mcu_ctrl != mcu_ctrl_after);
 792 }
 793
 794 static void __init gds_select_mitigation(void)
 795 {
 796         u64 mcu_ctrl;
 797
 798         if (!boot_cpu_has_bug(X86_BUG_GDS))
 799                 return;
 800
 801         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
 802                 gds_mitigation = GDS_MITIGATION_HYPERVISOR;
 803                 goto out;
 804         }
 805
 806         if (cpu_mitigations_off())
 807                 gds_mitigation = GDS_MITIGATION_OFF;
 808         /* Will verify below that mitigation _can_ be disabled */
 809
 810         /* No microcode */
 811         if (!(x86_arch_cap_msr & ARCH_CAP_GDS_CTRL)) {
 812                 if (gds_mitigation == GDS_MITIGATION_FORCE) {
 813                         /*
 814                          * This only needs to be done on the boot CPU so do it
 815                          * here rather than in update_gds_msr()
 816                          */
 817                         setup_clear_cpu_cap(X86_FEATURE_AVX);
 818                         pr_warn("Microcode update needed! Disabling AVX as mitigation.\n");
 819                 } else {
 820                         gds_mitigation = GDS_MITIGATION_UCODE_NEEDED;
 821                 }
 822                 goto out;
 823         }
 824
 825         /* Microcode has mitigation, use it */
 826         if (gds_mitigation == GDS_MITIGATION_FORCE)
 827                 gds_mitigation = GDS_MITIGATION_FULL;
 828
 829         rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 830         if (mcu_ctrl & GDS_MITG_LOCKED) {
 831                 if (gds_mitigation == GDS_MITIGATION_OFF)
 832                         pr_warn("Mitigation locked. Disable failed.\n");
 833
 834                 /*
 835                  * The mitigation is selected from the boot CPU. All other CPUs
 836                  * _should_ have the same state. If the boot CPU isn't locked
 837                  * but others are then update_gds_msr() will WARN() of the state
 838                  * mismatch. If the boot CPU is locked update_gds_msr() will
 839                  * ensure the other CPUs have the mitigation enabled.
 840                  */
 841                 gds_mitigation = GDS_MITIGATION_FULL_LOCKED;
 842         }
 843
 844         update_gds_msr();
 845 out:
 846         pr_info("%s\n", gds_strings[gds_mitigation]);
 847 }
 848
 849 static int __init gds_parse_cmdline(char *str)
 850 {
 851         if (!str)
 852                 return -EINVAL;
 853
 854         if (!boot_cpu_has_bug(X86_BUG_GDS))
 855                 return 0;
 856
 857         if (!strcmp(str, "off"))
 858                 gds_mitigation = GDS_MITIGATION_OFF;
 859         else if (!strcmp(str, "force"))
 860                 gds_mitigation = GDS_MITIGATION_FORCE;
 861
 862         return 0;
 863 }
 864 early_param("gather_data_sampling", gds_parse_cmdline);
 865
 866 #undef pr_fmt
 867 #define pr_fmt(fmt)     "Spectre V1 : " fmt
 868
 869 enum spectre_v1_mitigation {
 870         SPECTRE_V1_MITIGATION_NONE,
 871         SPECTRE_V1_MITIGATION_AUTO,
 872 };
 873
 874 static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init =
 875         IS_ENABLED(CONFIG_MITIGATION_SPECTRE_V1) ?
 876                 SPECTRE_V1_MITIGATION_AUTO : SPECTRE_V1_MITIGATION_NONE;
 877
 878 static const char * const spectre_v1_strings[] = {
 879         [SPECTRE_V1_MITIGATION_NONE] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers",
 880         [SPECTRE_V1_MITIGATION_AUTO] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization",
 881 };
 882
 883 /*
 884  * Does SMAP provide full mitigation against speculative kernel access to
 885  * userspace?
 886  */
 887 static bool smap_works_speculatively(void)
 888 {
 889         if (!boot_cpu_has(X86_FEATURE_SMAP))
 890                 return false;
 891
 892         /*
 893          * On CPUs which are vulnerable to Meltdown, SMAP does not
 894          * prevent speculative access to user data in the L1 cache.
 895          * Consider SMAP to be non-functional as a mitigation on these
 896          * CPUs.
 897          */
 898         if (boot_cpu_has(X86_BUG_CPU_MELTDOWN))
 899                 return false;
 900
 901         return true;
 902 }
 903
 904 static void __init spectre_v1_select_mitigation(void)
 905 {
 906         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1) || cpu_mitigations_off()) {
 907                 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE;
 908                 return;
 909         }
 910
 911         if (spectre_v1_mitigation == SPECTRE_V1_MITIGATION_AUTO) {
 912                 /*
 913                  * With Spectre v1, a user can speculatively control either
 914                  * path of a conditional swapgs with a user-controlled GS
 915                  * value.  The mitigation is to add lfences to both code paths.
 916                  *
 917                  * If FSGSBASE is enabled, the user can put a kernel address in
 918                  * GS, in which case SMAP provides no protection.
 919                  *
 920                  * If FSGSBASE is disabled, the user can only put a user space
 921                  * address in GS.  That makes an attack harder, but still
 922                  * possible if there's no SMAP protection.
 923                  */
 924                 if (boot_cpu_has(X86_FEATURE_FSGSBASE) ||
 925                     !smap_works_speculatively()) {
 926                         /*
 927                          * Mitigation can be provided from SWAPGS itself or
 928                          * PTI as the CR3 write in the Meltdown mitigation
 929                          * is serializing.
 930                          *
 931                          * If neither is there, mitigate with an LFENCE to
 932                          * stop speculation through swapgs.
 933                          */
 934                         if (boot_cpu_has_bug(X86_BUG_SWAPGS) &&
 935                             !boot_cpu_has(X86_FEATURE_PTI))
 936                                 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER);
 937
 938                         /*
 939                          * Enable lfences in the kernel entry (non-swapgs)
 940                          * paths, to prevent user entry from speculatively
 941                          * skipping swapgs.
 942                          */
 943                         setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL);
 944                 }
 945         }
 946
 947         pr_info("%s\n", spectre_v1_strings[spectre_v1_mitigation]);
 948 }
 949
 950 static int __init nospectre_v1_cmdline(char *str)
 951 {
 952         spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE;
 953         return 0;
 954 }
 955 early_param("nospectre_v1", nospectre_v1_cmdline);
 956
 957 enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init = SPECTRE_V2_NONE;
 958
 959 #undef pr_fmt
 960 #define pr_fmt(fmt)     "RETBleed: " fmt
 961
 962 enum retbleed_mitigation {
 963         RETBLEED_MITIGATION_NONE,
 964         RETBLEED_MITIGATION_UNRET,
 965         RETBLEED_MITIGATION_IBPB,
 966         RETBLEED_MITIGATION_IBRS,
 967         RETBLEED_MITIGATION_EIBRS,
 968         RETBLEED_MITIGATION_STUFF,
 969 };
 970
 971 enum retbleed_mitigation_cmd {
 972         RETBLEED_CMD_OFF,
 973         RETBLEED_CMD_AUTO,
 974         RETBLEED_CMD_UNRET,
 975         RETBLEED_CMD_IBPB,
 976         RETBLEED_CMD_STUFF,
 977 };
 978
 979 static const char * const retbleed_strings[] = {
 980         [RETBLEED_MITIGATION_NONE]      = "Vulnerable",
 981         [RETBLEED_MITIGATION_UNRET]     = "Mitigation: untrained return thunk",
 982         [RETBLEED_MITIGATION_IBPB]      = "Mitigation: IBPB",
 983         [RETBLEED_MITIGATION_IBRS]      = "Mitigation: IBRS",
 984         [RETBLEED_MITIGATION_EIBRS]     = "Mitigation: Enhanced IBRS",
 985         [RETBLEED_MITIGATION_STUFF]     = "Mitigation: Stuffing",
 986 };
 987
 988 static enum retbleed_mitigation retbleed_mitigation __ro_after_init =
 989         RETBLEED_MITIGATION_NONE;
 990 static enum retbleed_mitigation_cmd retbleed_cmd __ro_after_init =
 991         IS_ENABLED(CONFIG_MITIGATION_RETBLEED) ? RETBLEED_CMD_AUTO : RETBLEED_CMD_OFF;
 992
 993 static int __ro_after_init retbleed_nosmt = false;
 994
 995 static int __init retbleed_parse_cmdline(char *str)
 996 {
 997         if (!str)
 998                 return -EINVAL;
 999
1000         while (str) {
1001                 char *next = strchr(str, ',');
1002                 if (next) {
1003                         *next = 0;
1004                         next++;
1005                 }
1006
1007                 if (!strcmp(str, "off")) {
1008                         retbleed_cmd = RETBLEED_CMD_OFF;
1009                 } else if (!strcmp(str, "auto")) {
1010                         retbleed_cmd = RETBLEED_CMD_AUTO;
1011                 } else if (!strcmp(str, "unret")) {
1012                         retbleed_cmd = RETBLEED_CMD_UNRET;
1013                 } else if (!strcmp(str, "ibpb")) {
1014                         retbleed_cmd = RETBLEED_CMD_IBPB;
1015                 } else if (!strcmp(str, "stuff")) {
1016                         retbleed_cmd = RETBLEED_CMD_STUFF;
1017                 } else if (!strcmp(str, "nosmt")) {
1018                         retbleed_nosmt = true;
1019                 } else if (!strcmp(str, "force")) {
1020                         setup_force_cpu_bug(X86_BUG_RETBLEED);
1021                 } else {
1022                         pr_err("Ignoring unknown retbleed option (%s).", str);
1023                 }
1024
1025                 str = next;
1026         }
1027
1028         return 0;
1029 }
1030 early_param("retbleed", retbleed_parse_cmdline);
1031
1032 #define RETBLEED_UNTRAIN_MSG "WARNING: BTB untrained return thunk mitigation is only effective on AMD/Hygon!\n"
1033 #define RETBLEED_INTEL_MSG "WARNING: Spectre v2 mitigation leaves CPU vulnerable to RETBleed attacks, data leaks possible!\n"
1034
1035 static void __init retbleed_select_mitigation(void)
1036 {
1037         bool mitigate_smt = false;
1038
1039         if (!boot_cpu_has_bug(X86_BUG_RETBLEED) || cpu_mitigations_off())
1040                 return;
1041
1042         switch (retbleed_cmd) {
1043         case RETBLEED_CMD_OFF:
1044                 return;
1045
1046         case RETBLEED_CMD_UNRET:
1047                 if (IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY)) {
1048                         retbleed_mitigation = RETBLEED_MITIGATION_UNRET;
1049                 } else {
1050                         pr_err("WARNING: kernel not compiled with MITIGATION_UNRET_ENTRY.\n");
1051                         goto do_cmd_auto;
1052                 }
1053                 break;
1054
1055         case RETBLEED_CMD_IBPB:
1056                 if (!boot_cpu_has(X86_FEATURE_IBPB)) {
1057                         pr_err("WARNING: CPU does not support IBPB.\n");
1058                         goto do_cmd_auto;
1059                 } else if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) {
1060                         retbleed_mitigation = RETBLEED_MITIGATION_IBPB;
1061                 } else {
1062                         pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n");
1063                         goto do_cmd_auto;
1064                 }
1065                 break;
1066
1067         case RETBLEED_CMD_STUFF:
1068                 if (IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING) &&
1069                     spectre_v2_enabled == SPECTRE_V2_RETPOLINE) {
1070                         retbleed_mitigation = RETBLEED_MITIGATION_STUFF;
1071
1072                 } else {
1073                         if (IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING))
1074                                 pr_err("WARNING: retbleed=stuff depends on spectre_v2=retpoline\n");
1075                         else
1076                                 pr_err("WARNING: kernel not compiled with MITIGATION_CALL_DEPTH_TRACKING.\n");
1077
1078                         goto do_cmd_auto;
1079                 }
1080                 break;
1081
1082 do_cmd_auto:
1083         case RETBLEED_CMD_AUTO:
1084                 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
1085                     boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) {
1086                         if (IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY))
1087                                 retbleed_mitigation = RETBLEED_MITIGATION_UNRET;
1088                         else if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY) &&
1089                                  boot_cpu_has(X86_FEATURE_IBPB))
1090                                 retbleed_mitigation = RETBLEED_MITIGATION_IBPB;
1091                 }
1092
1093                 /*
1094                  * The Intel mitigation (IBRS or eIBRS) was already selected in
1095                  * spectre_v2_select_mitigation().  'retbleed_mitigation' will
1096                  * be set accordingly below.
1097                  */
1098
1099                 break;
1100         }
1101
1102         switch (retbleed_mitigation) {
1103         case RETBLEED_MITIGATION_UNRET:
1104                 setup_force_cpu_cap(X86_FEATURE_RETHUNK);
1105                 setup_force_cpu_cap(X86_FEATURE_UNRET);
1106
1107                 x86_return_thunk = retbleed_return_thunk;
1108
1109                 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
1110                     boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
1111                         pr_err(RETBLEED_UNTRAIN_MSG);
1112
1113                 mitigate_smt = true;
1114                 break;
1115
1116         case RETBLEED_MITIGATION_IBPB:
1117                 setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB);
1118                 setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT);
1119                 mitigate_smt = true;
1120
1121                 /*
1122                  * IBPB on entry already obviates the need for
1123                  * software-based untraining so clear those in case some
1124                  * other mitigation like SRSO has selected them.
1125                  */
1126                 setup_clear_cpu_cap(X86_FEATURE_UNRET);
1127                 setup_clear_cpu_cap(X86_FEATURE_RETHUNK);
1128
1129                 /*
1130                  * There is no need for RSB filling: entry_ibpb() ensures
1131                  * all predictions, including the RSB, are invalidated,
1132                  * regardless of IBPB implementation.
1133                  */
1134                 setup_clear_cpu_cap(X86_FEATURE_RSB_VMEXIT);
1135
1136                 break;
1137
1138         case RETBLEED_MITIGATION_STUFF:
1139                 setup_force_cpu_cap(X86_FEATURE_RETHUNK);
1140                 setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH);
1141
1142                 x86_return_thunk = call_depth_return_thunk;
1143                 break;
1144
1145         default:
1146                 break;
1147         }
1148
1149         if (mitigate_smt && !boot_cpu_has(X86_FEATURE_STIBP) &&
1150             (retbleed_nosmt || cpu_mitigations_auto_nosmt()))
1151                 cpu_smt_disable(false);
1152
1153         /*
1154          * Let IBRS trump all on Intel without affecting the effects of the
1155          * retbleed= cmdline option except for call depth based stuffing
1156          */
1157         if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
1158                 switch (spectre_v2_enabled) {
1159                 case SPECTRE_V2_IBRS:
1160                         retbleed_mitigation = RETBLEED_MITIGATION_IBRS;
1161                         break;
1162                 case SPECTRE_V2_EIBRS:
1163                 case SPECTRE_V2_EIBRS_RETPOLINE:
1164                 case SPECTRE_V2_EIBRS_LFENCE:
1165                         retbleed_mitigation = RETBLEED_MITIGATION_EIBRS;
1166                         break;
1167                 default:
1168                         if (retbleed_mitigation != RETBLEED_MITIGATION_STUFF)
1169                                 pr_err(RETBLEED_INTEL_MSG);
1170                 }
1171         }
1172
1173         pr_info("%s\n", retbleed_strings[retbleed_mitigation]);
1174 }
1175
1176 #undef pr_fmt
1177 #define pr_fmt(fmt)     "Spectre V2 : " fmt
1178
1179 static enum spectre_v2_user_mitigation spectre_v2_user_stibp __ro_after_init =
1180         SPECTRE_V2_USER_NONE;
1181 static enum spectre_v2_user_mitigation spectre_v2_user_ibpb __ro_after_init =
1182         SPECTRE_V2_USER_NONE;
1183
1184 #ifdef CONFIG_MITIGATION_RETPOLINE
1185 static bool spectre_v2_bad_module;
1186
1187 bool retpoline_module_ok(bool has_retpoline)
1188 {
1189         if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline)
1190                 return true;
1191
1192         pr_err("System may be vulnerable to spectre v2\n");
1193         spectre_v2_bad_module = true;
1194         return false;
1195 }
1196
1197 static inline const char *spectre_v2_module_string(void)
1198 {
1199         return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
1200 }
1201 #else
1202 static inline const char *spectre_v2_module_string(void) { return ""; }
1203 #endif
1204
1205 #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n"
1206 #define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n"
1207 #define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n"
1208 #define SPECTRE_V2_IBRS_PERF_MSG "WARNING: IBRS mitigation selected on Enhanced IBRS CPU, this may cause unnecessary performance loss\n"
1209
1210 #ifdef CONFIG_BPF_SYSCALL
1211 void unpriv_ebpf_notify(int new_state)
1212 {
1213         if (new_state)
1214                 return;
1215
1216         /* Unprivileged eBPF is enabled */
1217
1218         switch (spectre_v2_enabled) {
1219         case SPECTRE_V2_EIBRS:
1220                 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG);
1221                 break;
1222         case SPECTRE_V2_EIBRS_LFENCE:
1223                 if (sched_smt_active())
1224                         pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG);
1225                 break;
1226         default:
1227                 break;
1228         }
1229 }
1230 #endif
1231
1232 static inline bool match_option(const char *arg, int arglen, const char *opt)
1233 {
1234         int len = strlen(opt);
1235
1236         return len == arglen && !strncmp(arg, opt, len);
1237 }
1238
1239 /* The kernel command line selection for spectre v2 */
1240 enum spectre_v2_mitigation_cmd {
1241         SPECTRE_V2_CMD_NONE,
1242         SPECTRE_V2_CMD_AUTO,
1243         SPECTRE_V2_CMD_FORCE,
1244         SPECTRE_V2_CMD_RETPOLINE,
1245         SPECTRE_V2_CMD_RETPOLINE_GENERIC,
1246         SPECTRE_V2_CMD_RETPOLINE_LFENCE,
1247         SPECTRE_V2_CMD_EIBRS,
1248         SPECTRE_V2_CMD_EIBRS_RETPOLINE,
1249         SPECTRE_V2_CMD_EIBRS_LFENCE,
1250         SPECTRE_V2_CMD_IBRS,
1251 };
1252
1253 enum spectre_v2_user_cmd {
1254         SPECTRE_V2_USER_CMD_NONE,
1255         SPECTRE_V2_USER_CMD_AUTO,
1256         SPECTRE_V2_USER_CMD_FORCE,
1257         SPECTRE_V2_USER_CMD_PRCTL,
1258         SPECTRE_V2_USER_CMD_PRCTL_IBPB,
1259         SPECTRE_V2_USER_CMD_SECCOMP,
1260         SPECTRE_V2_USER_CMD_SECCOMP_IBPB,
1261 };
1262
1263 static const char * const spectre_v2_user_strings[] = {
1264         [SPECTRE_V2_USER_NONE]                  = "User space: Vulnerable",
1265         [SPECTRE_V2_USER_STRICT]                = "User space: Mitigation: STIBP protection",
1266         [SPECTRE_V2_USER_STRICT_PREFERRED]      = "User space: Mitigation: STIBP always-on protection",
1267         [SPECTRE_V2_USER_PRCTL]                 = "User space: Mitigation: STIBP via prctl",
1268         [SPECTRE_V2_USER_SECCOMP]               = "User space: Mitigation: STIBP via seccomp and prctl",
1269 };
1270
1271 static const struct {
1272         const char                      *option;
1273         enum spectre_v2_user_cmd        cmd;
1274         bool                            secure;
1275 } v2_user_options[] __initconst = {
1276         { "auto",               SPECTRE_V2_USER_CMD_AUTO,               false },
1277         { "off",                SPECTRE_V2_USER_CMD_NONE,               false },
1278         { "on",                 SPECTRE_V2_USER_CMD_FORCE,              true  },
1279         { "prctl",              SPECTRE_V2_USER_CMD_PRCTL,              false },
1280         { "prctl,ibpb",         SPECTRE_V2_USER_CMD_PRCTL_IBPB,         false },
1281         { "seccomp",            SPECTRE_V2_USER_CMD_SECCOMP,            false },
1282         { "seccomp,ibpb",       SPECTRE_V2_USER_CMD_SECCOMP_IBPB,       false },
1283 };
1284
1285 static void __init spec_v2_user_print_cond(const char *reason, bool secure)
1286 {
1287         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
1288                 pr_info("spectre_v2_user=%s forced on command line.\n", reason);
1289 }
1290
1291 static __ro_after_init enum spectre_v2_mitigation_cmd spectre_v2_cmd;
1292
1293 static enum spectre_v2_user_cmd __init
1294 spectre_v2_parse_user_cmdline(void)
1295 {
1296         char arg[20];
1297         int ret, i;
1298
1299         switch (spectre_v2_cmd) {
1300         case SPECTRE_V2_CMD_NONE:
1301                 return SPECTRE_V2_USER_CMD_NONE;
1302         case SPECTRE_V2_CMD_FORCE:
1303                 return SPECTRE_V2_USER_CMD_FORCE;
1304         default:
1305                 break;
1306         }
1307
1308         ret = cmdline_find_option(boot_command_line, "spectre_v2_user",
1309                                   arg, sizeof(arg));
1310         if (ret < 0)
1311                 return SPECTRE_V2_USER_CMD_AUTO;
1312
1313         for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) {
1314                 if (match_option(arg, ret, v2_user_options[i].option)) {
1315                         spec_v2_user_print_cond(v2_user_options[i].option,
1316                                                 v2_user_options[i].secure);
1317                         return v2_user_options[i].cmd;
1318                 }
1319         }
1320
1321         pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg);
1322         return SPECTRE_V2_USER_CMD_AUTO;
1323 }
1324
1325 static inline bool spectre_v2_in_ibrs_mode(enum spectre_v2_mitigation mode)
1326 {
1327         return spectre_v2_in_eibrs_mode(mode) || mode == SPECTRE_V2_IBRS;
1328 }
1329
1330 static void __init
1331 spectre_v2_user_select_mitigation(void)
1332 {
1333         enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE;
1334         bool smt_possible = IS_ENABLED(CONFIG_SMP);
1335         enum spectre_v2_user_cmd cmd;
1336
1337         if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP))
1338                 return;
1339
1340         if (cpu_smt_control == CPU_SMT_FORCE_DISABLED ||
1341             cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
1342                 smt_possible = false;
1343
1344         cmd = spectre_v2_parse_user_cmdline();
1345         switch (cmd) {
1346         case SPECTRE_V2_USER_CMD_NONE:
1347                 goto set_mode;
1348         case SPECTRE_V2_USER_CMD_FORCE:
1349                 mode = SPECTRE_V2_USER_STRICT;
1350                 break;
1351         case SPECTRE_V2_USER_CMD_AUTO:
1352         case SPECTRE_V2_USER_CMD_PRCTL:
1353         case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
1354                 mode = SPECTRE_V2_USER_PRCTL;
1355                 break;
1356         case SPECTRE_V2_USER_CMD_SECCOMP:
1357         case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
1358                 if (IS_ENABLED(CONFIG_SECCOMP))
1359                         mode = SPECTRE_V2_USER_SECCOMP;
1360                 else
1361                         mode = SPECTRE_V2_USER_PRCTL;
1362                 break;
1363         }
1364
1365         /* Initialize Indirect Branch Prediction Barrier */
1366         if (boot_cpu_has(X86_FEATURE_IBPB)) {
1367                 setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
1368
1369                 spectre_v2_user_ibpb = mode;
1370                 switch (cmd) {
1371                 case SPECTRE_V2_USER_CMD_NONE:
1372                         break;
1373                 case SPECTRE_V2_USER_CMD_FORCE:
1374                 case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
1375                 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
1376                         static_branch_enable(&switch_mm_always_ibpb);
1377                         spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT;
1378                         break;
1379                 case SPECTRE_V2_USER_CMD_PRCTL:
1380                 case SPECTRE_V2_USER_CMD_AUTO:
1381                 case SPECTRE_V2_USER_CMD_SECCOMP:
1382                         static_branch_enable(&switch_mm_cond_ibpb);
1383                         break;
1384                 }
1385
1386                 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n",
1387                         static_key_enabled(&switch_mm_always_ibpb) ?
1388                         "always-on" : "conditional");
1389         }
1390
1391         /*
1392          * If no STIBP, Intel enhanced IBRS is enabled, or SMT impossible, STIBP
1393          * is not required.
1394          *
1395          * Intel's Enhanced IBRS also protects against cross-thread branch target
1396          * injection in user-mode as the IBRS bit remains always set which
1397          * implicitly enables cross-thread protections.  However, in legacy IBRS
1398          * mode, the IBRS bit is set only on kernel entry and cleared on return
1399          * to userspace.  AMD Automatic IBRS also does not protect userspace.
1400          * These modes therefore disable the implicit cross-thread protection,
1401          * so allow for STIBP to be selected in those cases.
1402          */
1403         if (!boot_cpu_has(X86_FEATURE_STIBP) ||
1404             !smt_possible ||
1405             (spectre_v2_in_eibrs_mode(spectre_v2_enabled) &&
1406              !boot_cpu_has(X86_FEATURE_AUTOIBRS)))
1407                 return;
1408
1409         /*
1410          * At this point, an STIBP mode other than "off" has been set.
1411          * If STIBP support is not being forced, check if STIBP always-on
1412          * is preferred.
1413          */
1414         if (mode != SPECTRE_V2_USER_STRICT &&
1415             boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON))
1416                 mode = SPECTRE_V2_USER_STRICT_PREFERRED;
1417
1418         if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET ||
1419             retbleed_mitigation == RETBLEED_MITIGATION_IBPB) {
1420                 if (mode != SPECTRE_V2_USER_STRICT &&
1421                     mode != SPECTRE_V2_USER_STRICT_PREFERRED)
1422                         pr_info("Selecting STIBP always-on mode to complement retbleed mitigation\n");
1423                 mode = SPECTRE_V2_USER_STRICT_PREFERRED;
1424         }
1425
1426         spectre_v2_user_stibp = mode;
1427
1428 set_mode:
1429         pr_info("%s\n", spectre_v2_user_strings[mode]);
1430 }
1431
1432 static const char * const spectre_v2_strings[] = {
1433         [SPECTRE_V2_NONE]                       = "Vulnerable",
1434         [SPECTRE_V2_RETPOLINE]                  = "Mitigation: Retpolines",
1435         [SPECTRE_V2_LFENCE]                     = "Mitigation: LFENCE",
1436         [SPECTRE_V2_EIBRS]                      = "Mitigation: Enhanced / Automatic IBRS",
1437         [SPECTRE_V2_EIBRS_LFENCE]               = "Mitigation: Enhanced / Automatic IBRS + LFENCE",
1438         [SPECTRE_V2_EIBRS_RETPOLINE]            = "Mitigation: Enhanced / Automatic IBRS + Retpolines",
1439         [SPECTRE_V2_IBRS]                       = "Mitigation: IBRS",
1440 };
1441
1442 static const struct {
1443         const char *option;
1444         enum spectre_v2_mitigation_cmd cmd;
1445         bool secure;
1446 } mitigation_options[] __initconst = {
1447         { "off",                SPECTRE_V2_CMD_NONE,              false },
1448         { "on",                 SPECTRE_V2_CMD_FORCE,             true  },
1449         { "retpoline",          SPECTRE_V2_CMD_RETPOLINE,         false },
1450         { "retpoline,amd",      SPECTRE_V2_CMD_RETPOLINE_LFENCE,  false },
1451         { "retpoline,lfence",   SPECTRE_V2_CMD_RETPOLINE_LFENCE,  false },
1452         { "retpoline,generic",  SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
1453         { "eibrs",              SPECTRE_V2_CMD_EIBRS,             false },
1454         { "eibrs,lfence",       SPECTRE_V2_CMD_EIBRS_LFENCE,      false },
1455         { "eibrs,retpoline",    SPECTRE_V2_CMD_EIBRS_RETPOLINE,   false },
1456         { "auto",               SPECTRE_V2_CMD_AUTO,              false },
1457         { "ibrs",               SPECTRE_V2_CMD_IBRS,              false },
1458 };
1459
1460 static void __init spec_v2_print_cond(const char *reason, bool secure)
1461 {
1462         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
1463                 pr_info("%s selected on command line.\n", reason);
1464 }
1465
1466 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
1467 {
1468         enum spectre_v2_mitigation_cmd cmd;
1469         char arg[20];
1470         int ret, i;
1471
1472         cmd = IS_ENABLED(CONFIG_MITIGATION_SPECTRE_V2) ?  SPECTRE_V2_CMD_AUTO : SPECTRE_V2_CMD_NONE;
1473         if (cmdline_find_option_bool(boot_command_line, "nospectre_v2") ||
1474             cpu_mitigations_off())
1475                 return SPECTRE_V2_CMD_NONE;
1476
1477         ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg));
1478         if (ret < 0)
1479                 return cmd;
1480
1481         for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) {
1482                 if (!match_option(arg, ret, mitigation_options[i].option))
1483                         continue;
1484                 cmd = mitigation_options[i].cmd;
1485                 break;
1486         }
1487
1488         if (i >= ARRAY_SIZE(mitigation_options)) {
1489                 pr_err("unknown option (%s). Switching to default mode\n", arg);
1490                 return cmd;
1491         }
1492
1493         if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
1494              cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||
1495              cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC ||
1496              cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||
1497              cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
1498             !IS_ENABLED(CONFIG_MITIGATION_RETPOLINE)) {
1499                 pr_err("%s selected but not compiled in. Switching to AUTO select\n",
1500                        mitigation_options[i].option);
1501                 return SPECTRE_V2_CMD_AUTO;
1502         }
1503
1504         if ((cmd == SPECTRE_V2_CMD_EIBRS ||
1505              cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||
1506              cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
1507             !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
1508                 pr_err("%s selected but CPU doesn't have Enhanced or Automatic IBRS. Switching to AUTO select\n",
1509                        mitigation_options[i].option);
1510                 return SPECTRE_V2_CMD_AUTO;
1511         }
1512
1513         if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||
1514              cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) &&
1515             !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
1516                 pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n",
1517                        mitigation_options[i].option);
1518                 return SPECTRE_V2_CMD_AUTO;
1519         }
1520
1521         if (cmd == SPECTRE_V2_CMD_IBRS && !IS_ENABLED(CONFIG_MITIGATION_IBRS_ENTRY)) {
1522                 pr_err("%s selected but not compiled in. Switching to AUTO select\n",
1523                        mitigation_options[i].option);
1524                 return SPECTRE_V2_CMD_AUTO;
1525         }
1526
1527         if (cmd == SPECTRE_V2_CMD_IBRS && boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {
1528                 pr_err("%s selected but not Intel CPU. Switching to AUTO select\n",
1529                        mitigation_options[i].option);
1530                 return SPECTRE_V2_CMD_AUTO;
1531         }
1532
1533         if (cmd == SPECTRE_V2_CMD_IBRS && !boot_cpu_has(X86_FEATURE_IBRS)) {
1534                 pr_err("%s selected but CPU doesn't have IBRS. Switching to AUTO select\n",
1535                        mitigation_options[i].option);
1536                 return SPECTRE_V2_CMD_AUTO;
1537         }
1538
1539         if (cmd == SPECTRE_V2_CMD_IBRS && cpu_feature_enabled(X86_FEATURE_XENPV)) {
1540                 pr_err("%s selected but running as XenPV guest. Switching to AUTO select\n",
1541                        mitigation_options[i].option);
1542                 return SPECTRE_V2_CMD_AUTO;
1543         }
1544
1545         spec_v2_print_cond(mitigation_options[i].option,
1546                            mitigation_options[i].secure);
1547         return cmd;
1548 }
1549
1550 static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void)
1551 {
1552         if (!IS_ENABLED(CONFIG_MITIGATION_RETPOLINE)) {
1553                 pr_err("Kernel not compiled with retpoline; no mitigation available!");
1554                 return SPECTRE_V2_NONE;
1555         }
1556
1557         return SPECTRE_V2_RETPOLINE;
1558 }
1559
1560 static bool __ro_after_init rrsba_disabled;
1561
1562 /* Disable in-kernel use of non-RSB RET predictors */
1563 static void __init spec_ctrl_disable_kernel_rrsba(void)
1564 {
1565         if (rrsba_disabled)
1566                 return;
1567
1568         if (!(x86_arch_cap_msr & ARCH_CAP_RRSBA)) {
1569                 rrsba_disabled = true;
1570                 return;
1571         }
1572
1573         if (!boot_cpu_has(X86_FEATURE_RRSBA_CTRL))
1574                 return;
1575
1576         x86_spec_ctrl_base |= SPEC_CTRL_RRSBA_DIS_S;
1577         update_spec_ctrl(x86_spec_ctrl_base);
1578         rrsba_disabled = true;
1579 }
1580
1581 static void __init spectre_v2_determine_rsb_fill_type_at_vmexit(enum spectre_v2_mitigation mode)
1582 {
1583         /*
1584          * Similar to context switches, there are two types of RSB attacks
1585          * after VM exit:
1586          *
1587          * 1) RSB underflow
1588          *
1589          * 2) Poisoned RSB entry
1590          *
1591          * When retpoline is enabled, both are mitigated by filling/clearing
1592          * the RSB.
1593          *
1594          * When IBRS is enabled, while #1 would be mitigated by the IBRS branch
1595          * prediction isolation protections, RSB still needs to be cleared
1596          * because of #2.  Note that SMEP provides no protection here, unlike
1597          * user-space-poisoned RSB entries.
1598          *
1599          * eIBRS should protect against RSB poisoning, but if the EIBRS_PBRSB
1600          * bug is present then a LITE version of RSB protection is required,
1601          * just a single call needs to retire before a RET is executed.
1602          */
1603         switch (mode) {
1604         case SPECTRE_V2_NONE:
1605                 return;
1606
1607         case SPECTRE_V2_EIBRS_LFENCE:
1608         case SPECTRE_V2_EIBRS:
1609                 if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) {
1610                         setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT_LITE);
1611                         pr_info("Spectre v2 / PBRSB-eIBRS: Retire a single CALL on VMEXIT\n");
1612                 }
1613                 return;
1614
1615         case SPECTRE_V2_EIBRS_RETPOLINE:
1616         case SPECTRE_V2_RETPOLINE:
1617         case SPECTRE_V2_LFENCE:
1618         case SPECTRE_V2_IBRS:
1619                 setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT);
1620                 pr_info("Spectre v2 / SpectreRSB : Filling RSB on VMEXIT\n");
1621                 return;
1622         }
1623
1624         pr_warn_once("Unknown Spectre v2 mode, disabling RSB mitigation at VM exit");
1625         dump_stack();
1626 }
1627
1628 /*
1629  * Set BHI_DIS_S to prevent indirect branches in kernel to be influenced by
1630  * branch history in userspace. Not needed if BHI_NO is set.
1631  */
1632 static bool __init spec_ctrl_bhi_dis(void)
1633 {
1634         if (!boot_cpu_has(X86_FEATURE_BHI_CTRL))
1635                 return false;
1636
1637         x86_spec_ctrl_base |= SPEC_CTRL_BHI_DIS_S;
1638         update_spec_ctrl(x86_spec_ctrl_base);
1639         setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_HW);
1640
1641         return true;
1642 }
1643
1644 enum bhi_mitigations {
1645         BHI_MITIGATION_OFF,
1646         BHI_MITIGATION_ON,
1647         BHI_MITIGATION_VMEXIT_ONLY,
1648 };
1649
1650 static enum bhi_mitigations bhi_mitigation __ro_after_init =
1651         IS_ENABLED(CONFIG_MITIGATION_SPECTRE_BHI) ? BHI_MITIGATION_ON : BHI_MITIGATION_OFF;
1652
1653 static int __init spectre_bhi_parse_cmdline(char *str)
1654 {
1655         if (!str)
1656                 return -EINVAL;
1657
1658         if (!strcmp(str, "off"))
1659                 bhi_mitigation = BHI_MITIGATION_OFF;
1660         else if (!strcmp(str, "on"))
1661                 bhi_mitigation = BHI_MITIGATION_ON;
1662         else if (!strcmp(str, "vmexit"))
1663                 bhi_mitigation = BHI_MITIGATION_VMEXIT_ONLY;
1664         else
1665                 pr_err("Ignoring unknown spectre_bhi option (%s)", str);
1666
1667         return 0;
1668 }
1669 early_param("spectre_bhi", spectre_bhi_parse_cmdline);
1670
1671 static void __init bhi_select_mitigation(void)
1672 {
1673         if (bhi_mitigation == BHI_MITIGATION_OFF)
1674                 return;
1675
1676         /* Retpoline mitigates against BHI unless the CPU has RRSBA behavior */
1677         if (boot_cpu_has(X86_FEATURE_RETPOLINE) &&
1678             !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE)) {
1679                 spec_ctrl_disable_kernel_rrsba();
1680                 if (rrsba_disabled)
1681                         return;
1682         }
1683
1684         /* Mitigate in hardware if supported */
1685         if (spec_ctrl_bhi_dis())
1686                 return;
1687
1688         if (!IS_ENABLED(CONFIG_X86_64))
1689                 return;
1690
1691         if (bhi_mitigation == BHI_MITIGATION_VMEXIT_ONLY) {
1692                 pr_info("Spectre BHI mitigation: SW BHB clearing on VM exit only\n");
1693                 setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT);
1694                 return;
1695         }
1696
1697         pr_info("Spectre BHI mitigation: SW BHB clearing on syscall and VM exit\n");
1698         setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_LOOP);
1699         setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT);
1700 }
1701
1702 static void __init spectre_v2_select_mitigation(void)
1703 {
1704         enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
1705         enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
1706
1707         /*
1708          * If the CPU is not affected and the command line mode is NONE or AUTO
1709          * then nothing to do.
1710          */
1711         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
1712             (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
1713                 return;
1714
1715         switch (cmd) {
1716         case SPECTRE_V2_CMD_NONE:
1717                 return;
1718
1719         case SPECTRE_V2_CMD_FORCE:
1720         case SPECTRE_V2_CMD_AUTO:
1721                 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
1722                         mode = SPECTRE_V2_EIBRS;
1723                         break;
1724                 }
1725
1726                 if (IS_ENABLED(CONFIG_MITIGATION_IBRS_ENTRY) &&
1727                     boot_cpu_has_bug(X86_BUG_RETBLEED) &&
1728                     retbleed_cmd != RETBLEED_CMD_OFF &&
1729                     retbleed_cmd != RETBLEED_CMD_STUFF &&
1730                     boot_cpu_has(X86_FEATURE_IBRS) &&
1731                     boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
1732                         mode = SPECTRE_V2_IBRS;
1733                         break;
1734                 }
1735
1736                 mode = spectre_v2_select_retpoline();
1737                 break;
1738
1739         case SPECTRE_V2_CMD_RETPOLINE_LFENCE:
1740                 pr_err(SPECTRE_V2_LFENCE_MSG);
1741                 mode = SPECTRE_V2_LFENCE;
1742                 break;
1743
1744         case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
1745                 mode = SPECTRE_V2_RETPOLINE;
1746                 break;
1747
1748         case SPECTRE_V2_CMD_RETPOLINE:
1749                 mode = spectre_v2_select_retpoline();
1750                 break;
1751
1752         case SPECTRE_V2_CMD_IBRS:
1753                 mode = SPECTRE_V2_IBRS;
1754                 break;
1755
1756         case SPECTRE_V2_CMD_EIBRS:
1757                 mode = SPECTRE_V2_EIBRS;
1758                 break;
1759
1760         case SPECTRE_V2_CMD_EIBRS_LFENCE:
1761                 mode = SPECTRE_V2_EIBRS_LFENCE;
1762                 break;
1763
1764         case SPECTRE_V2_CMD_EIBRS_RETPOLINE:
1765                 mode = SPECTRE_V2_EIBRS_RETPOLINE;
1766                 break;
1767         }
1768
1769         if (mode == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
1770                 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG);
1771
1772         if (spectre_v2_in_ibrs_mode(mode)) {
1773                 if (boot_cpu_has(X86_FEATURE_AUTOIBRS)) {
1774                         msr_set_bit(MSR_EFER, _EFER_AUTOIBRS);
1775                 } else {
1776                         x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
1777                         update_spec_ctrl(x86_spec_ctrl_base);
1778                 }
1779         }
1780
1781         switch (mode) {
1782         case SPECTRE_V2_NONE:
1783         case SPECTRE_V2_EIBRS:
1784                 break;
1785
1786         case SPECTRE_V2_IBRS:
1787                 setup_force_cpu_cap(X86_FEATURE_KERNEL_IBRS);
1788                 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED))
1789                         pr_warn(SPECTRE_V2_IBRS_PERF_MSG);
1790                 break;
1791
1792         case SPECTRE_V2_LFENCE:
1793         case SPECTRE_V2_EIBRS_LFENCE:
1794                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE);
1795                 fallthrough;
1796
1797         case SPECTRE_V2_RETPOLINE:
1798         case SPECTRE_V2_EIBRS_RETPOLINE:
1799                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
1800                 break;
1801         }
1802
1803         /*
1804          * Disable alternate RSB predictions in kernel when indirect CALLs and
1805          * JMPs gets protection against BHI and Intramode-BTI, but RET
1806          * prediction from a non-RSB predictor is still a risk.
1807          */
1808         if (mode == SPECTRE_V2_EIBRS_LFENCE ||
1809             mode == SPECTRE_V2_EIBRS_RETPOLINE ||
1810             mode == SPECTRE_V2_RETPOLINE)
1811                 spec_ctrl_disable_kernel_rrsba();
1812
1813         if (boot_cpu_has(X86_BUG_BHI))
1814                 bhi_select_mitigation();
1815
1816         spectre_v2_enabled = mode;
1817         pr_info("%s\n", spectre_v2_strings[mode]);
1818
1819         /*
1820          * If Spectre v2 protection has been enabled, fill the RSB during a
1821          * context switch.  In general there are two types of RSB attacks
1822          * across context switches, for which the CALLs/RETs may be unbalanced.
1823          *
1824          * 1) RSB underflow
1825          *
1826          *    Some Intel parts have "bottomless RSB".  When the RSB is empty,
1827          *    speculated return targets may come from the branch predictor,
1828          *    which could have a user-poisoned BTB or BHB entry.
1829          *
1830          *    AMD has it even worse: *all* returns are speculated from the BTB,
1831          *    regardless of the state of the RSB.
1832          *
1833          *    When IBRS or eIBRS is enabled, the "user -> kernel" attack
1834          *    scenario is mitigated by the IBRS branch prediction isolation
1835          *    properties, so the RSB buffer filling wouldn't be necessary to
1836          *    protect against this type of attack.
1837          *
1838          *    The "user -> user" attack scenario is mitigated by RSB filling.
1839          *
1840          * 2) Poisoned RSB entry
1841          *
1842          *    If the 'next' in-kernel return stack is shorter than 'prev',
1843          *    'next' could be tricked into speculating with a user-poisoned RSB
1844          *    entry.
1845          *
1846          *    The "user -> kernel" attack scenario is mitigated by SMEP and
1847          *    eIBRS.
1848          *
1849          *    The "user -> user" scenario, also known as SpectreBHB, requires
1850          *    RSB clearing.
1851          *
1852          * So to mitigate all cases, unconditionally fill RSB on context
1853          * switches.
1854          *
1855          * FIXME: Is this pointless for retbleed-affected AMD?
1856          */
1857         setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
1858         pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n");
1859
1860         spectre_v2_determine_rsb_fill_type_at_vmexit(mode);
1861
1862         /*
1863          * Retpoline protects the kernel, but doesn't protect firmware.  IBRS
1864          * and Enhanced IBRS protect firmware too, so enable IBRS around
1865          * firmware calls only when IBRS / Enhanced / Automatic IBRS aren't
1866          * otherwise enabled.
1867          *
1868          * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because
1869          * the user might select retpoline on the kernel command line and if
1870          * the CPU supports Enhanced IBRS, kernel might un-intentionally not
1871          * enable IBRS around firmware calls.
1872          */
1873         if (boot_cpu_has_bug(X86_BUG_RETBLEED) &&
1874             boot_cpu_has(X86_FEATURE_IBPB) &&
1875             (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
1876              boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)) {
1877
1878                 if (retbleed_cmd != RETBLEED_CMD_IBPB) {
1879                         setup_force_cpu_cap(X86_FEATURE_USE_IBPB_FW);
1880                         pr_info("Enabling Speculation Barrier for firmware calls\n");
1881                 }
1882
1883         } else if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_ibrs_mode(mode)) {
1884                 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
1885                 pr_info("Enabling Restricted Speculation for firmware calls\n");
1886         }
1887
1888         /* Set up IBPB and STIBP depending on the general spectre V2 command */
1889         spectre_v2_cmd = cmd;
1890 }
1891
1892 static void update_stibp_msr(void * __unused)
1893 {
1894         u64 val = spec_ctrl_current() | (x86_spec_ctrl_base & SPEC_CTRL_STIBP);
1895         update_spec_ctrl(val);
1896 }
1897
1898 /* Update x86_spec_ctrl_base in case SMT state changed. */
1899 static void update_stibp_strict(void)
1900 {
1901         u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP;
1902
1903         if (sched_smt_active())
1904                 mask |= SPEC_CTRL_STIBP;
1905
1906         if (mask == x86_spec_ctrl_base)
1907                 return;
1908
1909         pr_info("Update user space SMT mitigation: STIBP %s\n",
1910                 mask & SPEC_CTRL_STIBP ? "always-on" : "off");
1911         x86_spec_ctrl_base = mask;
1912         on_each_cpu(update_stibp_msr, NULL, 1);
1913 }
1914
1915 /* Update the static key controlling the evaluation of TIF_SPEC_IB */
1916 static void update_indir_branch_cond(void)
1917 {
1918         if (sched_smt_active())
1919                 static_branch_enable(&switch_to_cond_stibp);
1920         else
1921                 static_branch_disable(&switch_to_cond_stibp);
1922 }
1923
1924 #undef pr_fmt
1925 #define pr_fmt(fmt) fmt
1926
1927 /* Update the static key controlling the MDS CPU buffer clear in idle */
1928 static void update_mds_branch_idle(void)
1929 {
1930         /*
1931          * Enable the idle clearing if SMT is active on CPUs which are
1932          * affected only by MSBDS and not any other MDS variant.
1933          *
1934          * The other variants cannot be mitigated when SMT is enabled, so
1935          * clearing the buffers on idle just to prevent the Store Buffer
1936          * repartitioning leak would be a window dressing exercise.
1937          */
1938         if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY))
1939                 return;
1940
1941         if (sched_smt_active()) {
1942                 static_branch_enable(&mds_idle_clear);
1943         } else if (mmio_mitigation == MMIO_MITIGATION_OFF ||
1944                    (x86_arch_cap_msr & ARCH_CAP_FBSDP_NO)) {
1945                 static_branch_disable(&mds_idle_clear);
1946         }
1947 }
1948
1949 #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n"
1950 #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n"
1951 #define MMIO_MSG_SMT "MMIO Stale Data CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/processor_mmio_stale_data.html for more details.\n"
1952
1953 void cpu_bugs_smt_update(void)
1954 {
1955         mutex_lock(&spec_ctrl_mutex);
1956
1957         if (sched_smt_active() && unprivileged_ebpf_enabled() &&
1958             spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE)
1959                 pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG);
1960
1961         switch (spectre_v2_user_stibp) {
1962         case SPECTRE_V2_USER_NONE:
1963                 break;
1964         case SPECTRE_V2_USER_STRICT:
1965         case SPECTRE_V2_USER_STRICT_PREFERRED:
1966                 update_stibp_strict();
1967                 break;
1968         case SPECTRE_V2_USER_PRCTL:
1969         case SPECTRE_V2_USER_SECCOMP:
1970                 update_indir_branch_cond();
1971                 break;
1972         }
1973
1974         switch (mds_mitigation) {
1975         case MDS_MITIGATION_FULL:
1976         case MDS_MITIGATION_VMWERV:
1977                 if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY))
1978                         pr_warn_once(MDS_MSG_SMT);
1979                 update_mds_branch_idle();
1980                 break;
1981         case MDS_MITIGATION_OFF:
1982                 break;
1983         }
1984
1985         switch (taa_mitigation) {
1986         case TAA_MITIGATION_VERW:
1987         case TAA_MITIGATION_UCODE_NEEDED:
1988                 if (sched_smt_active())
1989                         pr_warn_once(TAA_MSG_SMT);
1990                 break;
1991         case TAA_MITIGATION_TSX_DISABLED:
1992         case TAA_MITIGATION_OFF:
1993                 break;
1994         }
1995
1996         switch (mmio_mitigation) {
1997         case MMIO_MITIGATION_VERW:
1998         case MMIO_MITIGATION_UCODE_NEEDED:
1999                 if (sched_smt_active())
2000                         pr_warn_once(MMIO_MSG_SMT);
2001                 break;
2002         case MMIO_MITIGATION_OFF:
2003                 break;
2004         }
2005
2006         mutex_unlock(&spec_ctrl_mutex);
2007 }
2008
2009 #undef pr_fmt
2010 #define pr_fmt(fmt)     "Speculative Store Bypass: " fmt
2011
2012 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE;
2013
2014 /* The kernel command line selection */
2015 enum ssb_mitigation_cmd {
2016         SPEC_STORE_BYPASS_CMD_NONE,
2017         SPEC_STORE_BYPASS_CMD_AUTO,
2018         SPEC_STORE_BYPASS_CMD_ON,
2019         SPEC_STORE_BYPASS_CMD_PRCTL,
2020         SPEC_STORE_BYPASS_CMD_SECCOMP,
2021 };
2022
2023 static const char * const ssb_strings[] = {
2024         [SPEC_STORE_BYPASS_NONE]        = "Vulnerable",
2025         [SPEC_STORE_BYPASS_DISABLE]     = "Mitigation: Speculative Store Bypass disabled",
2026         [SPEC_STORE_BYPASS_PRCTL]       = "Mitigation: Speculative Store Bypass disabled via prctl",
2027         [SPEC_STORE_BYPASS_SECCOMP]     = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp",
2028 };
2029
2030 static const struct {
2031         const char *option;
2032         enum ssb_mitigation_cmd cmd;
2033 } ssb_mitigation_options[]  __initconst = {
2034         { "auto",       SPEC_STORE_BYPASS_CMD_AUTO },    /* Platform decides */
2035         { "on",         SPEC_STORE_BYPASS_CMD_ON },      /* Disable Speculative Store Bypass */
2036         { "off",        SPEC_STORE_BYPASS_CMD_NONE },    /* Don't touch Speculative Store Bypass */
2037         { "prctl",      SPEC_STORE_BYPASS_CMD_PRCTL },   /* Disable Speculative Store Bypass via prctl */
2038         { "seccomp",    SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */
2039 };
2040
2041 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void)
2042 {
2043         enum ssb_mitigation_cmd cmd;
2044         char arg[20];
2045         int ret, i;
2046
2047         cmd = IS_ENABLED(CONFIG_MITIGATION_SSB) ?
2048                 SPEC_STORE_BYPASS_CMD_AUTO : SPEC_STORE_BYPASS_CMD_NONE;
2049         if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable") ||
2050             cpu_mitigations_off()) {
2051                 return SPEC_STORE_BYPASS_CMD_NONE;
2052         } else {
2053                 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable",
2054                                           arg, sizeof(arg));
2055                 if (ret < 0)
2056                         return cmd;
2057
2058                 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) {
2059                         if (!match_option(arg, ret, ssb_mitigation_options[i].option))
2060                                 continue;
2061
2062                         cmd = ssb_mitigation_options[i].cmd;
2063                         break;
2064                 }
2065
2066                 if (i >= ARRAY_SIZE(ssb_mitigation_options)) {
2067                         pr_err("unknown option (%s). Switching to default mode\n", arg);
2068                         return cmd;
2069                 }
2070         }
2071
2072         return cmd;
2073 }
2074
2075 static enum ssb_mitigation __init __ssb_select_mitigation(void)
2076 {
2077         enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE;
2078         enum ssb_mitigation_cmd cmd;
2079
2080         if (!boot_cpu_has(X86_FEATURE_SSBD))
2081                 return mode;
2082
2083         cmd = ssb_parse_cmdline();
2084         if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) &&
2085             (cmd == SPEC_STORE_BYPASS_CMD_NONE ||
2086              cmd == SPEC_STORE_BYPASS_CMD_AUTO))
2087                 return mode;
2088
2089         switch (cmd) {
2090         case SPEC_STORE_BYPASS_CMD_SECCOMP:
2091                 /*
2092                  * Choose prctl+seccomp as the default mode if seccomp is
2093                  * enabled.
2094                  */
2095                 if (IS_ENABLED(CONFIG_SECCOMP))
2096                         mode = SPEC_STORE_BYPASS_SECCOMP;
2097                 else
2098                         mode = SPEC_STORE_BYPASS_PRCTL;
2099                 break;
2100         case SPEC_STORE_BYPASS_CMD_ON:
2101                 mode = SPEC_STORE_BYPASS_DISABLE;
2102                 break;
2103         case SPEC_STORE_BYPASS_CMD_AUTO:
2104         case SPEC_STORE_BYPASS_CMD_PRCTL:
2105                 mode = SPEC_STORE_BYPASS_PRCTL;
2106                 break;
2107         case SPEC_STORE_BYPASS_CMD_NONE:
2108                 break;
2109         }
2110
2111         /*
2112          * We have three CPU feature flags that are in play here:
2113          *  - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
2114          *  - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass
2115          *  - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation
2116          */
2117         if (mode == SPEC_STORE_BYPASS_DISABLE) {
2118                 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE);
2119                 /*
2120                  * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may
2121                  * use a completely different MSR and bit dependent on family.
2122                  */
2123                 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) &&
2124                     !static_cpu_has(X86_FEATURE_AMD_SSBD)) {
2125                         x86_amd_ssb_disable();
2126                 } else {
2127                         x86_spec_ctrl_base |= SPEC_CTRL_SSBD;
2128                         update_spec_ctrl(x86_spec_ctrl_base);
2129                 }
2130         }
2131
2132         return mode;
2133 }
2134
2135 static void ssb_select_mitigation(void)
2136 {
2137         ssb_mode = __ssb_select_mitigation();
2138
2139         if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
2140                 pr_info("%s\n", ssb_strings[ssb_mode]);
2141 }
2142
2143 #undef pr_fmt
2144 #define pr_fmt(fmt)     "Speculation prctl: " fmt
2145
2146 static void task_update_spec_tif(struct task_struct *tsk)
2147 {
2148         /* Force the update of the real TIF bits */
2149         set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE);
2150
2151         /*
2152          * Immediately update the speculation control MSRs for the current
2153          * task, but for a non-current task delay setting the CPU
2154          * mitigation until it is scheduled next.
2155          *
2156          * This can only happen for SECCOMP mitigation. For PRCTL it's
2157          * always the current task.
2158          */
2159         if (tsk == current)
2160                 speculation_ctrl_update_current();
2161 }
2162
2163 static int l1d_flush_prctl_set(struct task_struct *task, unsigned long ctrl)
2164 {
2165
2166         if (!static_branch_unlikely(&switch_mm_cond_l1d_flush))
2167                 return -EPERM;
2168
2169         switch (ctrl) {
2170         case PR_SPEC_ENABLE:
2171                 set_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH);
2172                 return 0;
2173         case PR_SPEC_DISABLE:
2174                 clear_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH);
2175                 return 0;
2176         default:
2177                 return -ERANGE;
2178         }
2179 }
2180
2181 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl)
2182 {
2183         if (ssb_mode != SPEC_STORE_BYPASS_PRCTL &&
2184             ssb_mode != SPEC_STORE_BYPASS_SECCOMP)
2185                 return -ENXIO;
2186
2187         switch (ctrl) {
2188         case PR_SPEC_ENABLE:
2189                 /* If speculation is force disabled, enable is not allowed */
2190                 if (task_spec_ssb_force_disable(task))
2191                         return -EPERM;
2192                 task_clear_spec_ssb_disable(task);
2193                 task_clear_spec_ssb_noexec(task);
2194                 task_update_spec_tif(task);
2195                 break;
2196         case PR_SPEC_DISABLE:
2197                 task_set_spec_ssb_disable(task);
2198                 task_clear_spec_ssb_noexec(task);
2199                 task_update_spec_tif(task);
2200                 break;
2201         case PR_SPEC_FORCE_DISABLE:
2202                 task_set_spec_ssb_disable(task);
2203                 task_set_spec_ssb_force_disable(task);
2204                 task_clear_spec_ssb_noexec(task);
2205                 task_update_spec_tif(task);
2206                 break;
2207         case PR_SPEC_DISABLE_NOEXEC:
2208                 if (task_spec_ssb_force_disable(task))
2209                         return -EPERM;
2210                 task_set_spec_ssb_disable(task);
2211                 task_set_spec_ssb_noexec(task);
2212                 task_update_spec_tif(task);
2213                 break;
2214         default:
2215                 return -ERANGE;
2216         }
2217         return 0;
2218 }
2219
2220 static bool is_spec_ib_user_controlled(void)
2221 {
2222         return spectre_v2_user_ibpb == SPECTRE_V2_USER_PRCTL ||
2223                 spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP ||
2224                 spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL ||
2225                 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP;
2226 }
2227
2228 static int ib_prctl_set(struct task_struct *task, unsigned long ctrl)
2229 {
2230         switch (ctrl) {
2231         case PR_SPEC_ENABLE:
2232                 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
2233                     spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
2234                         return 0;
2235
2236                 /*
2237                  * With strict mode for both IBPB and STIBP, the instruction
2238                  * code paths avoid checking this task flag and instead,
2239                  * unconditionally run the instruction. However, STIBP and IBPB
2240                  * are independent and either can be set to conditionally
2241                  * enabled regardless of the mode of the other.
2242                  *
2243                  * If either is set to conditional, allow the task flag to be
2244                  * updated, unless it was force-disabled by a previous prctl
2245                  * call. Currently, this is possible on an AMD CPU which has the
2246                  * feature X86_FEATURE_AMD_STIBP_ALWAYS_ON. In this case, if the
2247                  * kernel is booted with 'spectre_v2_user=seccomp', then
2248                  * spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP and
2249                  * spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED.
2250                  */
2251                 if (!is_spec_ib_user_controlled() ||
2252                     task_spec_ib_force_disable(task))
2253                         return -EPERM;
2254
2255                 task_clear_spec_ib_disable(task);
2256                 task_update_spec_tif(task);
2257                 break;
2258         case PR_SPEC_DISABLE:
2259         case PR_SPEC_FORCE_DISABLE:
2260                 /*
2261                  * Indirect branch speculation is always allowed when
2262                  * mitigation is force disabled.
2263                  */
2264                 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
2265                     spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
2266                         return -EPERM;
2267
2268                 if (!is_spec_ib_user_controlled())
2269                         return 0;
2270
2271                 task_set_spec_ib_disable(task);
2272                 if (ctrl == PR_SPEC_FORCE_DISABLE)
2273                         task_set_spec_ib_force_disable(task);
2274                 task_update_spec_tif(task);
2275                 if (task == current)
2276                         indirect_branch_prediction_barrier();
2277                 break;
2278         default:
2279                 return -ERANGE;
2280         }
2281         return 0;
2282 }
2283
2284 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
2285                              unsigned long ctrl)
2286 {
2287         switch (which) {
2288         case PR_SPEC_STORE_BYPASS:
2289                 return ssb_prctl_set(task, ctrl);
2290         case PR_SPEC_INDIRECT_BRANCH:
2291                 return ib_prctl_set(task, ctrl);
2292         case PR_SPEC_L1D_FLUSH:
2293                 return l1d_flush_prctl_set(task, ctrl);
2294         default:
2295                 return -ENODEV;
2296         }
2297 }
2298
2299 #ifdef CONFIG_SECCOMP
2300 void arch_seccomp_spec_mitigate(struct task_struct *task)
2301 {
2302         if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP)
2303                 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE);
2304         if (spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP ||
2305             spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP)
2306                 ib_prctl_set(task, PR_SPEC_FORCE_DISABLE);
2307 }
2308 #endif
2309
2310 static int l1d_flush_prctl_get(struct task_struct *task)
2311 {
2312         if (!static_branch_unlikely(&switch_mm_cond_l1d_flush))
2313                 return PR_SPEC_FORCE_DISABLE;
2314
2315         if (test_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH))
2316                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
2317         else
2318                 return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
2319 }
2320
2321 static int ssb_prctl_get(struct task_struct *task)
2322 {
2323         switch (ssb_mode) {
2324         case SPEC_STORE_BYPASS_NONE:
2325                 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
2326                         return PR_SPEC_ENABLE;
2327                 return PR_SPEC_NOT_AFFECTED;
2328         case SPEC_STORE_BYPASS_DISABLE:
2329                 return PR_SPEC_DISABLE;
2330         case SPEC_STORE_BYPASS_SECCOMP:
2331         case SPEC_STORE_BYPASS_PRCTL:
2332                 if (task_spec_ssb_force_disable(task))
2333                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
2334                 if (task_spec_ssb_noexec(task))
2335                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC;
2336                 if (task_spec_ssb_disable(task))
2337                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
2338                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
2339         }
2340         BUG();
2341 }
2342
2343 static int ib_prctl_get(struct task_struct *task)
2344 {
2345         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
2346                 return PR_SPEC_NOT_AFFECTED;
2347
2348         if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
2349             spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
2350                 return PR_SPEC_ENABLE;
2351         else if (is_spec_ib_user_controlled()) {
2352                 if (task_spec_ib_force_disable(task))
2353                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
2354                 if (task_spec_ib_disable(task))
2355                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
2356                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
2357         } else if (spectre_v2_user_ibpb == SPECTRE_V2_USER_STRICT ||
2358             spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT ||
2359             spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED)
2360                 return PR_SPEC_DISABLE;
2361         else
2362                 return PR_SPEC_NOT_AFFECTED;
2363 }
2364
2365 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
2366 {
2367         switch (which) {
2368         case PR_SPEC_STORE_BYPASS:
2369                 return ssb_prctl_get(task);
2370         case PR_SPEC_INDIRECT_BRANCH:
2371                 return ib_prctl_get(task);
2372         case PR_SPEC_L1D_FLUSH:
2373                 return l1d_flush_prctl_get(task);
2374         default:
2375                 return -ENODEV;
2376         }
2377 }
2378
2379 void x86_spec_ctrl_setup_ap(void)
2380 {
2381         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
2382                 update_spec_ctrl(x86_spec_ctrl_base);
2383
2384         if (ssb_mode == SPEC_STORE_BYPASS_DISABLE)
2385                 x86_amd_ssb_disable();
2386 }
2387
2388 bool itlb_multihit_kvm_mitigation;
2389 EXPORT_SYMBOL_GPL(itlb_multihit_kvm_mitigation);
2390
2391 #undef pr_fmt
2392 #define pr_fmt(fmt)     "L1TF: " fmt
2393
2394 /* Default mitigation for L1TF-affected CPUs */
2395 enum l1tf_mitigations l1tf_mitigation __ro_after_init =
2396         IS_ENABLED(CONFIG_MITIGATION_L1TF) ? L1TF_MITIGATION_FLUSH : L1TF_MITIGATION_OFF;
2397 #if IS_ENABLED(CONFIG_KVM_INTEL)
2398 EXPORT_SYMBOL_GPL(l1tf_mitigation);
2399 #endif
2400 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
2401 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation);
2402
2403 /*
2404  * These CPUs all support 44bits physical address space internally in the
2405  * cache but CPUID can report a smaller number of physical address bits.
2406  *
2407  * The L1TF mitigation uses the top most address bit for the inversion of
2408  * non present PTEs. When the installed memory reaches into the top most
2409  * address bit due to memory holes, which has been observed on machines
2410  * which report 36bits physical address bits and have 32G RAM installed,
2411  * then the mitigation range check in l1tf_select_mitigation() triggers.
2412  * This is a false positive because the mitigation is still possible due to
2413  * the fact that the cache uses 44bit internally. Use the cache bits
2414  * instead of the reported physical bits and adjust them on the affected
2415  * machines to 44bit if the reported bits are less than 44.
2416  */
2417 static void override_cache_bits(struct cpuinfo_x86 *c)
2418 {
2419         if (c->x86 != 6)
2420                 return;
2421
2422         switch (c->x86_vfm) {
2423         case INTEL_NEHALEM:
2424         case INTEL_WESTMERE:
2425         case INTEL_SANDYBRIDGE:
2426         case INTEL_IVYBRIDGE:
2427         case INTEL_HASWELL:
2428         case INTEL_HASWELL_L:
2429         case INTEL_HASWELL_G:
2430         case INTEL_BROADWELL:
2431         case INTEL_BROADWELL_G:
2432         case INTEL_SKYLAKE_L:
2433         case INTEL_SKYLAKE:
2434         case INTEL_KABYLAKE_L:
2435         case INTEL_KABYLAKE:
2436                 if (c->x86_cache_bits < 44)
2437                         c->x86_cache_bits = 44;
2438                 break;
2439         }
2440 }
2441
2442 static void __init l1tf_select_mitigation(void)
2443 {
2444         u64 half_pa;
2445
2446         if (!boot_cpu_has_bug(X86_BUG_L1TF))
2447                 return;
2448
2449         if (cpu_mitigations_off())
2450                 l1tf_mitigation = L1TF_MITIGATION_OFF;
2451         else if (cpu_mitigations_auto_nosmt())
2452                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
2453
2454         override_cache_bits(&boot_cpu_data);
2455
2456         switch (l1tf_mitigation) {
2457         case L1TF_MITIGATION_OFF:
2458         case L1TF_MITIGATION_FLUSH_NOWARN:
2459         case L1TF_MITIGATION_FLUSH:
2460                 break;
2461         case L1TF_MITIGATION_FLUSH_NOSMT:
2462         case L1TF_MITIGATION_FULL:
2463                 cpu_smt_disable(false);
2464                 break;
2465         case L1TF_MITIGATION_FULL_FORCE:
2466                 cpu_smt_disable(true);
2467                 break;
2468         }
2469
2470 #if CONFIG_PGTABLE_LEVELS == 2
2471         pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n");
2472         return;
2473 #endif
2474
2475         half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT;
2476         if (l1tf_mitigation != L1TF_MITIGATION_OFF &&
2477                         e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) {
2478                 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n");
2479                 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n",
2480                                 half_pa);
2481                 pr_info("However, doing so will make a part of your RAM unusable.\n");
2482                 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n");
2483                 return;
2484         }
2485
2486         setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV);
2487 }
2488
2489 static int __init l1tf_cmdline(char *str)
2490 {
2491         if (!boot_cpu_has_bug(X86_BUG_L1TF))
2492                 return 0;
2493
2494         if (!str)
2495                 return -EINVAL;
2496
2497         if (!strcmp(str, "off"))
2498                 l1tf_mitigation = L1TF_MITIGATION_OFF;
2499         else if (!strcmp(str, "flush,nowarn"))
2500                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN;
2501         else if (!strcmp(str, "flush"))
2502                 l1tf_mitigation = L1TF_MITIGATION_FLUSH;
2503         else if (!strcmp(str, "flush,nosmt"))
2504                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
2505         else if (!strcmp(str, "full"))
2506                 l1tf_mitigation = L1TF_MITIGATION_FULL;
2507         else if (!strcmp(str, "full,force"))
2508                 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE;
2509
2510         return 0;
2511 }
2512 early_param("l1tf", l1tf_cmdline);
2513
2514 #undef pr_fmt
2515 #define pr_fmt(fmt)     "Speculative Return Stack Overflow: " fmt
2516
2517 enum srso_mitigation {
2518         SRSO_MITIGATION_NONE,
2519         SRSO_MITIGATION_UCODE_NEEDED,
2520         SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED,
2521         SRSO_MITIGATION_MICROCODE,
2522         SRSO_MITIGATION_SAFE_RET,
2523         SRSO_MITIGATION_IBPB,
2524         SRSO_MITIGATION_IBPB_ON_VMEXIT,
2525 };
2526
2527 enum srso_mitigation_cmd {
2528         SRSO_CMD_OFF,
2529         SRSO_CMD_MICROCODE,
2530         SRSO_CMD_SAFE_RET,
2531         SRSO_CMD_IBPB,
2532         SRSO_CMD_IBPB_ON_VMEXIT,
2533 };
2534
2535 static const char * const srso_strings[] = {
2536         [SRSO_MITIGATION_NONE]                  = "Vulnerable",
2537         [SRSO_MITIGATION_UCODE_NEEDED]          = "Vulnerable: No microcode",
2538         [SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED] = "Vulnerable: Safe RET, no microcode",
2539         [SRSO_MITIGATION_MICROCODE]             = "Vulnerable: Microcode, no safe RET",
2540         [SRSO_MITIGATION_SAFE_RET]              = "Mitigation: Safe RET",
2541         [SRSO_MITIGATION_IBPB]                  = "Mitigation: IBPB",
2542         [SRSO_MITIGATION_IBPB_ON_VMEXIT]        = "Mitigation: IBPB on VMEXIT only"
2543 };
2544
2545 static enum srso_mitigation srso_mitigation __ro_after_init = SRSO_MITIGATION_NONE;
2546 static enum srso_mitigation_cmd srso_cmd __ro_after_init = SRSO_CMD_SAFE_RET;
2547
2548 static int __init srso_parse_cmdline(char *str)
2549 {
2550         if (!str)
2551                 return -EINVAL;
2552
2553         if (!strcmp(str, "off"))
2554                 srso_cmd = SRSO_CMD_OFF;
2555         else if (!strcmp(str, "microcode"))
2556                 srso_cmd = SRSO_CMD_MICROCODE;
2557         else if (!strcmp(str, "safe-ret"))
2558                 srso_cmd = SRSO_CMD_SAFE_RET;
2559         else if (!strcmp(str, "ibpb"))
2560                 srso_cmd = SRSO_CMD_IBPB;
2561         else if (!strcmp(str, "ibpb-vmexit"))
2562                 srso_cmd = SRSO_CMD_IBPB_ON_VMEXIT;
2563         else
2564                 pr_err("Ignoring unknown SRSO option (%s).", str);
2565
2566         return 0;
2567 }
2568 early_param("spec_rstack_overflow", srso_parse_cmdline);
2569
2570 #define SRSO_NOTICE "WARNING: See https://kernel.org/doc/html/latest/admin-guide/hw-vuln/srso.html for mitigation options."
2571
2572 static void __init srso_select_mitigation(void)
2573 {
2574         bool has_microcode = boot_cpu_has(X86_FEATURE_IBPB_BRTYPE);
2575
2576         if (!boot_cpu_has_bug(X86_BUG_SRSO) ||
2577             cpu_mitigations_off() ||
2578             srso_cmd == SRSO_CMD_OFF) {
2579                 if (boot_cpu_has(X86_FEATURE_SBPB))
2580                         x86_pred_cmd = PRED_CMD_SBPB;
2581                 return;
2582         }
2583
2584         if (has_microcode) {
2585                 /*
2586                  * Zen1/2 with SMT off aren't vulnerable after the right
2587                  * IBPB microcode has been applied.
2588                  *
2589                  * Zen1/2 don't have SBPB, no need to try to enable it here.
2590                  */
2591                 if (boot_cpu_data.x86 < 0x19 && !cpu_smt_possible()) {
2592                         setup_force_cpu_cap(X86_FEATURE_SRSO_NO);
2593                         return;
2594                 }
2595
2596                 if (retbleed_mitigation == RETBLEED_MITIGATION_IBPB) {
2597                         srso_mitigation = SRSO_MITIGATION_IBPB;
2598                         goto out;
2599                 }
2600         } else {
2601                 pr_warn("IBPB-extending microcode not applied!\n");
2602                 pr_warn(SRSO_NOTICE);
2603
2604                 /* may be overwritten by SRSO_CMD_SAFE_RET below */
2605                 srso_mitigation = SRSO_MITIGATION_UCODE_NEEDED;
2606         }
2607
2608         switch (srso_cmd) {
2609         case SRSO_CMD_MICROCODE:
2610                 if (has_microcode) {
2611                         srso_mitigation = SRSO_MITIGATION_MICROCODE;
2612                         pr_warn(SRSO_NOTICE);
2613                 }
2614                 break;
2615
2616         case SRSO_CMD_SAFE_RET:
2617                 if (boot_cpu_has(X86_FEATURE_SRSO_USER_KERNEL_NO))
2618                         goto ibpb_on_vmexit;
2619
2620                 if (IS_ENABLED(CONFIG_MITIGATION_SRSO)) {
2621                         /*
2622                          * Enable the return thunk for generated code
2623                          * like ftrace, static_call, etc.
2624                          */
2625                         setup_force_cpu_cap(X86_FEATURE_RETHUNK);
2626                         setup_force_cpu_cap(X86_FEATURE_UNRET);
2627
2628                         if (boot_cpu_data.x86 == 0x19) {
2629                                 setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS);
2630                                 x86_return_thunk = srso_alias_return_thunk;
2631                         } else {
2632                                 setup_force_cpu_cap(X86_FEATURE_SRSO);
2633                                 x86_return_thunk = srso_return_thunk;
2634                         }
2635                         if (has_microcode)
2636                                 srso_mitigation = SRSO_MITIGATION_SAFE_RET;
2637                         else
2638                                 srso_mitigation = SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED;
2639                 } else {
2640                         pr_err("WARNING: kernel not compiled with MITIGATION_SRSO.\n");
2641                 }
2642                 break;
2643
2644         case SRSO_CMD_IBPB:
2645                 if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) {
2646                         if (has_microcode) {
2647                                 setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB);
2648                                 setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT);
2649                                 srso_mitigation = SRSO_MITIGATION_IBPB;
2650
2651                                 /*
2652                                  * IBPB on entry already obviates the need for
2653                                  * software-based untraining so clear those in case some
2654                                  * other mitigation like Retbleed has selected them.
2655                                  */
2656                                 setup_clear_cpu_cap(X86_FEATURE_UNRET);
2657                                 setup_clear_cpu_cap(X86_FEATURE_RETHUNK);
2658
2659                                 /*
2660                                  * There is no need for RSB filling: entry_ibpb() ensures
2661                                  * all predictions, including the RSB, are invalidated,
2662                                  * regardless of IBPB implementation.
2663                                  */
2664                                 setup_clear_cpu_cap(X86_FEATURE_RSB_VMEXIT);
2665                         }
2666                 } else {
2667                         pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n");
2668                 }
2669                 break;
2670
2671 ibpb_on_vmexit:
2672         case SRSO_CMD_IBPB_ON_VMEXIT:
2673                 if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) {
2674                         if (has_microcode) {
2675                                 setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT);
2676                                 srso_mitigation = SRSO_MITIGATION_IBPB_ON_VMEXIT;
2677
2678                                 /*
2679                                  * There is no need for RSB filling: entry_ibpb() ensures
2680                                  * all predictions, including the RSB, are invalidated,
2681                                  * regardless of IBPB implementation.
2682                                  */
2683                                 setup_clear_cpu_cap(X86_FEATURE_RSB_VMEXIT);
2684                         }
2685                 } else {
2686                         pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n");
2687                 }
2688                 break;
2689         default:
2690                 break;
2691         }
2692
2693 out:
2694         pr_info("%s\n", srso_strings[srso_mitigation]);
2695 }
2696
2697 #undef pr_fmt
2698 #define pr_fmt(fmt) fmt
2699
2700 #ifdef CONFIG_SYSFS
2701
2702 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion"
2703
2704 #if IS_ENABLED(CONFIG_KVM_INTEL)
2705 static const char * const l1tf_vmx_states[] = {
2706         [VMENTER_L1D_FLUSH_AUTO]                = "auto",
2707         [VMENTER_L1D_FLUSH_NEVER]               = "vulnerable",
2708         [VMENTER_L1D_FLUSH_COND]                = "conditional cache flushes",
2709         [VMENTER_L1D_FLUSH_ALWAYS]              = "cache flushes",
2710         [VMENTER_L1D_FLUSH_EPT_DISABLED]        = "EPT disabled",
2711         [VMENTER_L1D_FLUSH_NOT_REQUIRED]        = "flush not necessary"
2712 };
2713
2714 static ssize_t l1tf_show_state(char *buf)
2715 {
2716         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO)
2717                 return sysfs_emit(buf, "%s\n", L1TF_DEFAULT_MSG);
2718
2719         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED ||
2720             (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER &&
2721              sched_smt_active())) {
2722                 return sysfs_emit(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG,
2723                                   l1tf_vmx_states[l1tf_vmx_mitigation]);
2724         }
2725
2726         return sysfs_emit(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG,
2727                           l1tf_vmx_states[l1tf_vmx_mitigation],
2728                           sched_smt_active() ? "vulnerable" : "disabled");
2729 }
2730
2731 static ssize_t itlb_multihit_show_state(char *buf)
2732 {
2733         if (!boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) ||
2734             !boot_cpu_has(X86_FEATURE_VMX))
2735                 return sysfs_emit(buf, "KVM: Mitigation: VMX unsupported\n");
2736         else if (!(cr4_read_shadow() & X86_CR4_VMXE))
2737                 return sysfs_emit(buf, "KVM: Mitigation: VMX disabled\n");
2738         else if (itlb_multihit_kvm_mitigation)
2739                 return sysfs_emit(buf, "KVM: Mitigation: Split huge pages\n");
2740         else
2741                 return sysfs_emit(buf, "KVM: Vulnerable\n");
2742 }
2743 #else
2744 static ssize_t l1tf_show_state(char *buf)
2745 {
2746         return sysfs_emit(buf, "%s\n", L1TF_DEFAULT_MSG);
2747 }
2748
2749 static ssize_t itlb_multihit_show_state(char *buf)
2750 {
2751         return sysfs_emit(buf, "Processor vulnerable\n");
2752 }
2753 #endif
2754
2755 static ssize_t mds_show_state(char *buf)
2756 {
2757         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
2758                 return sysfs_emit(buf, "%s; SMT Host state unknown\n",
2759                                   mds_strings[mds_mitigation]);
2760         }
2761
2762         if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) {
2763                 return sysfs_emit(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
2764                                   (mds_mitigation == MDS_MITIGATION_OFF ? "vulnerable" :
2765                                    sched_smt_active() ? "mitigated" : "disabled"));
2766         }
2767
2768         return sysfs_emit(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
2769                           sched_smt_active() ? "vulnerable" : "disabled");
2770 }
2771
2772 static ssize_t tsx_async_abort_show_state(char *buf)
2773 {
2774         if ((taa_mitigation == TAA_MITIGATION_TSX_DISABLED) ||
2775             (taa_mitigation == TAA_MITIGATION_OFF))
2776                 return sysfs_emit(buf, "%s\n", taa_strings[taa_mitigation]);
2777
2778         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
2779                 return sysfs_emit(buf, "%s; SMT Host state unknown\n",
2780                                   taa_strings[taa_mitigation]);
2781         }
2782
2783         return sysfs_emit(buf, "%s; SMT %s\n", taa_strings[taa_mitigation],
2784                           sched_smt_active() ? "vulnerable" : "disabled");
2785 }
2786
2787 static ssize_t mmio_stale_data_show_state(char *buf)
2788 {
2789         if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN))
2790                 return sysfs_emit(buf, "Unknown: No mitigations\n");
2791
2792         if (mmio_mitigation == MMIO_MITIGATION_OFF)
2793                 return sysfs_emit(buf, "%s\n", mmio_strings[mmio_mitigation]);
2794
2795         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
2796                 return sysfs_emit(buf, "%s; SMT Host state unknown\n",
2797                                   mmio_strings[mmio_mitigation]);
2798         }
2799
2800         return sysfs_emit(buf, "%s; SMT %s\n", mmio_strings[mmio_mitigation],
2801                           sched_smt_active() ? "vulnerable" : "disabled");
2802 }
2803
2804 static ssize_t rfds_show_state(char *buf)
2805 {
2806         return sysfs_emit(buf, "%s\n", rfds_strings[rfds_mitigation]);
2807 }
2808
2809 static char *stibp_state(void)
2810 {
2811         if (spectre_v2_in_eibrs_mode(spectre_v2_enabled) &&
2812             !boot_cpu_has(X86_FEATURE_AUTOIBRS))
2813                 return "";
2814
2815         switch (spectre_v2_user_stibp) {
2816         case SPECTRE_V2_USER_NONE:
2817                 return "; STIBP: disabled";
2818         case SPECTRE_V2_USER_STRICT:
2819                 return "; STIBP: forced";
2820         case SPECTRE_V2_USER_STRICT_PREFERRED:
2821                 return "; STIBP: always-on";
2822         case SPECTRE_V2_USER_PRCTL:
2823         case SPECTRE_V2_USER_SECCOMP:
2824                 if (static_key_enabled(&switch_to_cond_stibp))
2825                         return "; STIBP: conditional";
2826         }
2827         return "";
2828 }
2829
2830 static char *ibpb_state(void)
2831 {
2832         if (boot_cpu_has(X86_FEATURE_IBPB)) {
2833                 if (static_key_enabled(&switch_mm_always_ibpb))
2834                         return "; IBPB: always-on";
2835                 if (static_key_enabled(&switch_mm_cond_ibpb))
2836                         return "; IBPB: conditional";
2837                 return "; IBPB: disabled";
2838         }
2839         return "";
2840 }
2841
2842 static char *pbrsb_eibrs_state(void)
2843 {
2844         if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) {
2845                 if (boot_cpu_has(X86_FEATURE_RSB_VMEXIT_LITE) ||
2846                     boot_cpu_has(X86_FEATURE_RSB_VMEXIT))
2847                         return "; PBRSB-eIBRS: SW sequence";
2848                 else
2849                         return "; PBRSB-eIBRS: Vulnerable";
2850         } else {
2851                 return "; PBRSB-eIBRS: Not affected";
2852         }
2853 }
2854
2855 static const char *spectre_bhi_state(void)
2856 {
2857         if (!boot_cpu_has_bug(X86_BUG_BHI))
2858                 return "; BHI: Not affected";
2859         else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_HW))
2860                 return "; BHI: BHI_DIS_S";
2861         else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_LOOP))
2862                 return "; BHI: SW loop, KVM: SW loop";
2863         else if (boot_cpu_has(X86_FEATURE_RETPOLINE) &&
2864                  !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE) &&
2865                  rrsba_disabled)
2866                 return "; BHI: Retpoline";
2867         else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT))
2868                 return "; BHI: Vulnerable, KVM: SW loop";
2869
2870         return "; BHI: Vulnerable";
2871 }
2872
2873 static ssize_t spectre_v2_show_state(char *buf)
2874 {
2875         if (spectre_v2_enabled == SPECTRE_V2_LFENCE)
2876                 return sysfs_emit(buf, "Vulnerable: LFENCE\n");
2877
2878         if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
2879                 return sysfs_emit(buf, "Vulnerable: eIBRS with unprivileged eBPF\n");
2880
2881         if (sched_smt_active() && unprivileged_ebpf_enabled() &&
2882             spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE)
2883                 return sysfs_emit(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n");
2884
2885         return sysfs_emit(buf, "%s%s%s%s%s%s%s%s\n",
2886                           spectre_v2_strings[spectre_v2_enabled],
2887                           ibpb_state(),
2888                           boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? "; IBRS_FW" : "",
2889                           stibp_state(),
2890                           boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? "; RSB filling" : "",
2891                           pbrsb_eibrs_state(),
2892                           spectre_bhi_state(),
2893                           /* this should always be at the end */
2894                           spectre_v2_module_string());
2895 }
2896
2897 static ssize_t srbds_show_state(char *buf)
2898 {
2899         return sysfs_emit(buf, "%s\n", srbds_strings[srbds_mitigation]);
2900 }
2901
2902 static ssize_t retbleed_show_state(char *buf)
2903 {
2904         if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET ||
2905             retbleed_mitigation == RETBLEED_MITIGATION_IBPB) {
2906                 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
2907                     boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
2908                         return sysfs_emit(buf, "Vulnerable: untrained return thunk / IBPB on non-AMD based uarch\n");
2909
2910                 return sysfs_emit(buf, "%s; SMT %s\n", retbleed_strings[retbleed_mitigation],
2911                                   !sched_smt_active() ? "disabled" :
2912                                   spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT ||
2913                                   spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED ?
2914                                   "enabled with STIBP protection" : "vulnerable");
2915         }
2916
2917         return sysfs_emit(buf, "%s\n", retbleed_strings[retbleed_mitigation]);
2918 }
2919
2920 static ssize_t srso_show_state(char *buf)
2921 {
2922         if (boot_cpu_has(X86_FEATURE_SRSO_NO))
2923                 return sysfs_emit(buf, "Mitigation: SMT disabled\n");
2924
2925         return sysfs_emit(buf, "%s\n", srso_strings[srso_mitigation]);
2926 }
2927
2928 static ssize_t gds_show_state(char *buf)
2929 {
2930         return sysfs_emit(buf, "%s\n", gds_strings[gds_mitigation]);
2931 }
2932
2933 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
2934                                char *buf, unsigned int bug)
2935 {
2936         if (!boot_cpu_has_bug(bug))
2937                 return sysfs_emit(buf, "Not affected\n");
2938
2939         switch (bug) {
2940         case X86_BUG_CPU_MELTDOWN:
2941                 if (boot_cpu_has(X86_FEATURE_PTI))
2942                         return sysfs_emit(buf, "Mitigation: PTI\n");
2943
2944                 if (hypervisor_is_type(X86_HYPER_XEN_PV))
2945                         return sysfs_emit(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n");
2946
2947                 break;
2948
2949         case X86_BUG_SPECTRE_V1:
2950                 return sysfs_emit(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]);
2951
2952         case X86_BUG_SPECTRE_V2:
2953                 return spectre_v2_show_state(buf);
2954
2955         case X86_BUG_SPEC_STORE_BYPASS:
2956                 return sysfs_emit(buf, "%s\n", ssb_strings[ssb_mode]);
2957
2958         case X86_BUG_L1TF:
2959                 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV))
2960                         return l1tf_show_state(buf);
2961                 break;
2962
2963         case X86_BUG_MDS:
2964                 return mds_show_state(buf);
2965
2966         case X86_BUG_TAA:
2967                 return tsx_async_abort_show_state(buf);
2968
2969         case X86_BUG_ITLB_MULTIHIT:
2970                 return itlb_multihit_show_state(buf);
2971
2972         case X86_BUG_SRBDS:
2973                 return srbds_show_state(buf);
2974
2975         case X86_BUG_MMIO_STALE_DATA:
2976         case X86_BUG_MMIO_UNKNOWN:
2977                 return mmio_stale_data_show_state(buf);
2978
2979         case X86_BUG_RETBLEED:
2980                 return retbleed_show_state(buf);
2981
2982         case X86_BUG_SRSO:
2983                 return srso_show_state(buf);
2984
2985         case X86_BUG_GDS:
2986                 return gds_show_state(buf);
2987
2988         case X86_BUG_RFDS:
2989                 return rfds_show_state(buf);
2990
2991         default:
2992                 break;
2993         }
2994
2995         return sysfs_emit(buf, "Vulnerable\n");
2996 }
2997
2998 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
2999 {
3000         return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN);
3001 }
3002
3003 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
3004 {
3005         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1);
3006 }
3007
3008 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
3009 {
3010         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2);
3011 }
3012
3013 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)
3014 {
3015         return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS);
3016 }
3017
3018 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf)
3019 {
3020         return cpu_show_common(dev, attr, buf, X86_BUG_L1TF);
3021 }
3022
3023 ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf)
3024 {
3025         return cpu_show_common(dev, attr, buf, X86_BUG_MDS);
3026 }
3027
3028 ssize_t cpu_show_tsx_async_abort(struct device *dev, struct device_attribute *attr, char *buf)
3029 {
3030         return cpu_show_common(dev, attr, buf, X86_BUG_TAA);
3031 }
3032
3033 ssize_t cpu_show_itlb_multihit(struct device *dev, struct device_attribute *attr, char *buf)
3034 {
3035         return cpu_show_common(dev, attr, buf, X86_BUG_ITLB_MULTIHIT);
3036 }
3037
3038 ssize_t cpu_show_srbds(struct device *dev, struct device_attribute *attr, char *buf)
3039 {
3040         return cpu_show_common(dev, attr, buf, X86_BUG_SRBDS);
3041 }
3042
3043 ssize_t cpu_show_mmio_stale_data(struct device *dev, struct device_attribute *attr, char *buf)
3044 {
3045         if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN))
3046                 return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_UNKNOWN);
3047         else
3048                 return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_STALE_DATA);
3049 }
3050
3051 ssize_t cpu_show_retbleed(struct device *dev, struct device_attribute *attr, char *buf)
3052 {
3053         return cpu_show_common(dev, attr, buf, X86_BUG_RETBLEED);
3054 }
3055
3056 ssize_t cpu_show_spec_rstack_overflow(struct device *dev, struct device_attribute *attr, char *buf)
3057 {
3058         return cpu_show_common(dev, attr, buf, X86_BUG_SRSO);
3059 }
3060
3061 ssize_t cpu_show_gds(struct device *dev, struct device_attribute *attr, char *buf)
3062 {
3063         return cpu_show_common(dev, attr, buf, X86_BUG_GDS);
3064 }
3065
3066 ssize_t cpu_show_reg_file_data_sampling(struct device *dev, struct device_attribute *attr, char *buf)
3067 {
3068         return cpu_show_common(dev, attr, buf, X86_BUG_RFDS);
3069 }
3070 #endif
3071
3072 void __warn_thunk(void)
3073 {
3074         WARN_ONCE(1, "Unpatched return thunk in use. This should not happen!\n");
3075 }