Merge tag 'x86_bugs_pbrsb' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 eIBRS fixes from Borislav Petkov:
 "More from the CPU vulnerability nightmares front:

  Intel eIBRS machines do not sufficiently mitigate against RET
  mispredictions when doing a VM Exit therefore an additional RSB,
  one-entry stuffing is needed"

* tag 'x86_bugs_pbrsb' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86/speculation: Add LFENCE to RSB fill sequence
  x86/speculation: Add RSB VM Exit protections

diff --combined arch/x86/include/asm/cpufeatures.h
index 14ed039dff551f0077707f4a34593e11f3e6902c,ede8990f3e416940fdb6bde272e5f3cb2c51285a..235dc85c91c3e372980b8b428e9713869115ead4
--- 1/arch/x86/include/asm/cpufeatures.h
--- 2/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@@ -219,7 -219,7 +219,7 @@@
  #define X86_FEATURE_IBRS              ( 7*32+25) /* Indirect Branch Restricted Speculation */
  #define X86_FEATURE_IBPB              ( 7*32+26) /* Indirect Branch Prediction Barrier */
  #define X86_FEATURE_STIBP             ( 7*32+27) /* Single Thread Indirect Branch Predictors */
 -#define X86_FEATURE_ZEN                       ( 7*32+28) /* "" CPU is AMD family 0x17 or above (Zen) */
 +#define X86_FEATURE_ZEN                       (7*32+28) /* "" CPU based on Zen microarchitecture */
  #define X86_FEATURE_L1TF_PTEINV               ( 7*32+29) /* "" L1TF workaround PTE inversion */
  #define X86_FEATURE_IBRS_ENHANCED     ( 7*32+30) /* Enhanced IBRS */
  #define X86_FEATURE_MSR_IA32_FEAT_CTL ( 7*32+31) /* "" MSR IA32_FEAT_CTL configured */
@@@ -303,6 -303,7 +303,7 @@@
  #define X86_FEATURE_RETHUNK           (11*32+14) /* "" Use REturn THUNK */
  #define X86_FEATURE_UNRET             (11*32+15) /* "" AMD BTB untrain return */
  #define X86_FEATURE_USE_IBPB_FW               (11*32+16) /* "" Use IBPB during runtime firmware calls */
+ #define X86_FEATURE_RSB_VMEXIT_LITE   (11*32+17) /* "" Fill RSB on VM exit when EIBRS is enabled */
  
  /* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */
  #define X86_FEATURE_AVX_VNNI          (12*32+ 4) /* AVX VNNI instructions */
@@@ -353,7 -354,6 +354,7 @@@
  #define X86_FEATURE_AVIC              (15*32+13) /* Virtual Interrupt Controller */
  #define X86_FEATURE_V_VMSAVE_VMLOAD   (15*32+15) /* Virtual VMSAVE VMLOAD */
  #define X86_FEATURE_VGIF              (15*32+16) /* Virtual GIF */
 +#define X86_FEATURE_X2AVIC            (15*32+18) /* Virtual x2apic */
  #define X86_FEATURE_V_SPEC_CTRL               (15*32+20) /* Virtual SPEC_CTRL */
  #define X86_FEATURE_SVME_ADDR_CHK     (15*32+28) /* "" SVME addr check */
  
@@@ -457,5 -457,6 +458,6 @@@
  #define X86_BUG_SRBDS                 X86_BUG(24) /* CPU may leak RNG bits if not mitigated */
  #define X86_BUG_MMIO_STALE_DATA               X86_BUG(25) /* CPU is affected by Processor MMIO Stale Data vulnerabilities */
  #define X86_BUG_RETBLEED              X86_BUG(26) /* CPU is affected by RETBleed */
+ #define X86_BUG_EIBRS_PBRSB           X86_BUG(27) /* EIBRS is vulnerable to Post Barrier RSB Predictions */
  
  #endif /* _ASM_X86_CPUFEATURES_H */

diff --combined arch/x86/include/asm/msr-index.h
index 182b2a1f71fea110cb0ad4c5fce4da0727e94035,e057e039173cb627baeb133052b4a4991b44e382..6674bdb096f346d940e353e3c4df7491fd5e0779
--- 1/arch/x86/include/asm/msr-index.h
--- 2/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@@ -150,6 -150,10 +150,10 @@@
                                                 * are restricted to targets in
                                                 * kernel.
                                                 */
+ #define ARCH_CAP_PBRSB_NO             BIT(24) /*
+                                                * Not susceptible to Post-Barrier
+                                                * Return Stack Buffer Predictions.
+                                                */
  
  #define MSR_IA32_FLUSH_CMD            0x0000010b
  #define L1D_FLUSH                     BIT(0)  /*
@@@ -231,12 -235,6 +235,12 @@@
  #define PERF_CAP_PT_IDX                       16
  
  #define MSR_PEBS_LD_LAT_THRESHOLD     0x000003f6
 +#define PERF_CAP_PEBS_TRAP             BIT_ULL(6)
 +#define PERF_CAP_ARCH_REG              BIT_ULL(7)
 +#define PERF_CAP_PEBS_FORMAT           0xf00
 +#define PERF_CAP_PEBS_BASELINE         BIT_ULL(14)
 +#define PERF_CAP_PEBS_MASK    (PERF_CAP_PEBS_TRAP | PERF_CAP_ARCH_REG | \
 +                               PERF_CAP_PEBS_FORMAT | PERF_CAP_PEBS_BASELINE)
  
  #define MSR_IA32_RTIT_CTL             0x00000570
  #define RTIT_CTL_TRACEEN              BIT(0)
@@@ -394,7 -392,6 +398,7 @@@
  #define MSR_TURBO_ACTIVATION_RATIO    0x0000064C
  
  #define MSR_PLATFORM_ENERGY_STATUS    0x0000064D
 +#define MSR_SECONDARY_TURBO_RATIO_LIMIT       0x00000650
  
  #define MSR_PKG_WEIGHTED_CORE_C0_RES  0x00000658
  #define MSR_PKG_ANY_CORE_C0_RES               0x00000659
@@@ -1025,7 -1022,6 +1029,7 @@@
  #define MSR_IA32_VMX_TRUE_EXIT_CTLS      0x0000048f
  #define MSR_IA32_VMX_TRUE_ENTRY_CTLS     0x00000490
  #define MSR_IA32_VMX_VMFUNC             0x00000491
 +#define MSR_IA32_VMX_PROCBASED_CTLS3  0x00000492
  
  /* VMX_BASIC bits and bitmasks */
  #define VMX_BASIC_VMCS_SIZE_SHIFT     32

diff --combined arch/x86/include/asm/nospec-branch.h
index cba942006ffe4385615cdce64797bc1ddba4cb34,d3a3cc6772ee193d0288d4cc6274a1b509a6e468..e64fd20778b61a23b59c8d04eba6cd5efc8cde0f
--- 1/arch/x86/include/asm/nospec-branch.h
--- 2/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@@ -60,7 -60,9 +60,9 @@@
  774:                                          \
        add     $(BITS_PER_LONG/8) * 2, sp;     \
        dec     reg;                            \
-       jnz     771b;
+       jnz     771b;                           \
+       /* barrier for jnz misprediction */     \
+       lfence;
  
  #ifdef __ASSEMBLY__
  
@@@ -93,19 -95,6 +95,19 @@@
  #endif
  .endm
  
 +/*
 + * Equivalent to -mindirect-branch-cs-prefix; emit the 5 byte jmp/call
 + * to the retpoline thunk with a CS prefix when the register requires
 + * a RAX prefix byte to encode. Also see apply_retpolines().
 + */
 +.macro __CS_PREFIX reg:req
 +      .irp rs,r8,r9,r10,r11,r12,r13,r14,r15
 +      .ifc \reg,\rs
 +      .byte 0x2e
 +      .endif
 +      .endr
 +.endm
 +
  /*
   * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
   * indirect jmp/call which may be susceptible to the Spectre variant 2
@@@ -113,30 -102,46 +115,45 @@@
   */
  .macro JMP_NOSPEC reg:req
  #ifdef CONFIG_RETPOLINE
 -      ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \
 -                    __stringify(jmp __x86_indirect_thunk_\reg), X86_FEATURE_RETPOLINE, \
 -                    __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_LFENCE
 +      __CS_PREFIX \reg
 +      jmp     __x86_indirect_thunk_\reg
  #else
        jmp     *%\reg
 +      int3
  #endif
  .endm
  
  .macro CALL_NOSPEC reg:req
  #ifdef CONFIG_RETPOLINE
 -      ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *%\reg), \
 -                    __stringify(call __x86_indirect_thunk_\reg), X86_FEATURE_RETPOLINE, \
 -                    __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *%\reg), X86_FEATURE_RETPOLINE_LFENCE
 +      __CS_PREFIX \reg
 +      call    __x86_indirect_thunk_\reg
  #else
        call    *%\reg
  #endif
  .endm
  
+ .macro ISSUE_UNBALANCED_RET_GUARD
+       ANNOTATE_INTRA_FUNCTION_CALL
+       call .Lunbalanced_ret_guard_\@
+       int3
+ .Lunbalanced_ret_guard_\@:
+       add $(BITS_PER_LONG/8), %_ASM_SP
+       lfence
+ .endm
+ 
   /*
    * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
    * monstrosity above, manually.
    */
- .macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
+ .macro FILL_RETURN_BUFFER reg:req nr:req ftr:req ftr2
+ .ifb \ftr2
        ALTERNATIVE "jmp .Lskip_rsb_\@", "", \ftr
+ .else
+       ALTERNATIVE_2 "jmp .Lskip_rsb_\@", "", \ftr, "jmp .Lunbalanced_\@", \ftr2
+ .endif
        __FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)
+ .Lunbalanced_\@:
+       ISSUE_UNBALANCED_RET_GUARD
  .Lskip_rsb_\@:
  .endm