Merge branch kvm-arm64/mops into kvmarm/next

* kvm-arm64/mops:
  : KVM support for MOPS, courtesy of Kristina Martsenko
  :
  : MOPS adds new instructions for accelerating memcpy(), memset(), and
  : memmove() operations in hardware. This series brings virtualization
  : support for KVM guests, and allows VMs to run on asymmetrict systems
  : that may have different MOPS implementations.
  KVM: arm64: Expose MOPS instructions to guests
  KVM: arm64: Add handler for MOPS exceptions

Signed-off-by: Oliver Upton <[email protected]>

diff --combined arch/arm64/kvm/hyp/vhe/switch.c
index b0cafd7c5f8fdf90386578e02975acba4f649ea1,796202f2e08f0f710d19735c72004b9dae55c1a4..c8e70e8ceeeb40e876a372cbdf27d4abcfbc72c8
--- 1/arch/arm64/kvm/hyp/vhe/switch.c
--- 2/arch/arm64/kvm/hyp/vhe/switch.c
+++ b/arch/arm64/kvm/hyp/vhe/switch.c
@@@ -93,12 -93,12 +93,12 @@@ static void __deactivate_traps(struct k
  NOKPROBE_SYMBOL(__deactivate_traps);
  
  /*
 - * Disable IRQs in {activate,deactivate}_traps_vhe_{load,put}() to
 + * Disable IRQs in __vcpu_{load,put}_{activate,deactivate}_traps() to
   * prevent a race condition between context switching of PMUSERENR_EL0
   * in __{activate,deactivate}_traps_common() and IPIs that attempts to
   * update PMUSERENR_EL0. See also kvm_set_pmuserenr().
   */
 -void activate_traps_vhe_load(struct kvm_vcpu *vcpu)
 +static void __vcpu_load_activate_traps(struct kvm_vcpu *vcpu)
  {
        unsigned long flags;
  
@@@ -107,7 -107,7 +107,7 @@@
        local_irq_restore(flags);
  }
  
 -void deactivate_traps_vhe_put(struct kvm_vcpu *vcpu)
 +static void __vcpu_put_deactivate_traps(struct kvm_vcpu *vcpu)
  {
        unsigned long flags;
  
@@@ -116,19 -116,6 +116,19 @@@
        local_irq_restore(flags);
  }
  
 +void kvm_vcpu_load_vhe(struct kvm_vcpu *vcpu)
 +{
 +      __vcpu_load_switch_sysregs(vcpu);
 +      __vcpu_load_activate_traps(vcpu);
 +      __load_stage2(vcpu->arch.hw_mmu, vcpu->arch.hw_mmu->arch);
 +}
 +
 +void kvm_vcpu_put_vhe(struct kvm_vcpu *vcpu)
 +{
 +      __vcpu_put_deactivate_traps(vcpu);
 +      __vcpu_put_switch_sysregs(vcpu);
 +}
 +
  static const exit_handler_fn hyp_exit_handlers[] = {
        [0 ... ESR_ELx_EC_MAX]          = NULL,
        [ESR_ELx_EC_CP15_32]            = kvm_hyp_handle_cp15_32,
@@@ -139,6 -126,7 +139,7 @@@
        [ESR_ELx_EC_DABT_LOW]           = kvm_hyp_handle_dabt_low,
        [ESR_ELx_EC_WATCHPT_LOW]        = kvm_hyp_handle_watchpt_low,
        [ESR_ELx_EC_PAC]                = kvm_hyp_handle_ptrauth,
+       [ESR_ELx_EC_MOPS]               = kvm_hyp_handle_mops,
  };
  
  static const exit_handler_fn *kvm_get_exit_handler_array(struct kvm_vcpu *vcpu)
@@@ -183,11 -171,17 +184,11 @@@ static int __kvm_vcpu_run_vhe(struct kv
        sysreg_save_host_state_vhe(host_ctxt);
  
        /*
 -       * ARM erratum 1165522 requires us to configure both stage 1 and
 -       * stage 2 translation for the guest context before we clear
 -       * HCR_EL2.TGE.
 -       *
 -       * We have already configured the guest's stage 1 translation in
 -       * kvm_vcpu_load_sysregs_vhe above.  We must now call
 -       * __load_stage2 before __activate_traps, because
 -       * __load_stage2 configures stage 2 translation, and
 -       * __activate_traps clear HCR_EL2.TGE (among other things).
 +       * Note that ARM erratum 1165522 requires us to configure both stage 1
 +       * and stage 2 translation for the guest context before we clear
 +       * HCR_EL2.TGE. The stage 1 and stage 2 guest context has already been
 +       * loaded on the CPU in kvm_vcpu_load_vhe().
         */
 -      __load_stage2(vcpu->arch.hw_mmu, vcpu->arch.hw_mmu->arch);
        __activate_traps(vcpu);
  
        __kvm_adjust_pc(vcpu);

diff --combined arch/arm64/kvm/sys_regs.c
index 2c4923fce573e5974c26170130e844eaf7591cad,9601af2aa0624da0899fb475016de20419d29774..d541ce9f48b9a57c15c00c2741d2204c1f90496f
--- 1/arch/arm64/kvm/sys_regs.c
--- 2/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@@ -746,12 -746,8 +746,12 @@@ static u64 reset_pmevcntr(struct kvm_vc
  
  static u64 reset_pmevtyper(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
  {
 +      /* This thing will UNDEF, who cares about the reset value? */
 +      if (!kvm_vcpu_has_pmu(vcpu))
 +              return 0;
 +
        reset_unknown(vcpu, r);
 -      __vcpu_sys_reg(vcpu, r->reg) &= ARMV8_PMU_EVTYPE_MASK;
 +      __vcpu_sys_reg(vcpu, r->reg) &= kvm_pmu_evtyper_mask(vcpu->kvm);
  
        return __vcpu_sys_reg(vcpu, r->reg);
  }
@@@ -992,7 -988,7 +992,7 @@@ static bool access_pmu_evtyper(struct k
                kvm_pmu_set_counter_event_type(vcpu, p->regval, idx);
                kvm_vcpu_pmu_restore_guest(vcpu);
        } else {
 -              p->regval = __vcpu_sys_reg(vcpu, reg) & ARMV8_PMU_EVTYPE_MASK;
 +              p->regval = __vcpu_sys_reg(vcpu, reg);
        }
  
        return true;
@@@ -1238,7 -1234,7 +1238,7 @@@ static s64 kvm_arm64_ftr_safe_value(u3
        return arm64_ftr_safe_value(&kvm_ftr, new, cur);
  }
  
 -/**
 +/*
   * arm64_check_features() - Check if a feature register value constitutes
   * a subset of features indicated by the idreg's KVM sanitised limit.
   *
@@@ -1348,7 -1344,6 +1348,6 @@@ static u64 __kvm_read_sanitised_id_reg(
                                 ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_GPA3));
                if (!cpus_have_final_cap(ARM64_HAS_WFXT))
                        val &= ~ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_WFxT);
-               val &= ~ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_MOPS);
                break;
        case SYS_ID_AA64MMFR2_EL1:
                val &= ~ID_AA64MMFR2_EL1_CCIDX_MASK;
@@@ -1829,8 -1824,8 +1828,8 @@@ static unsigned int el2_visibility(cons
   * HCR_EL2.E2H==1, and only in the sysreg table for convenience of
   * handling traps. Given that, they are always hidden from userspace.
   */
 -static unsigned int elx2_visibility(const struct kvm_vcpu *vcpu,
 -                                  const struct sys_reg_desc *rd)
 +static unsigned int hidden_user_visibility(const struct kvm_vcpu *vcpu,
 +                                         const struct sys_reg_desc *rd)
  {
        return REG_HIDDEN_USER;
  }
@@@ -1841,7 -1836,7 +1840,7 @@@
        .reset = rst,                           \
        .reg = name##_EL1,                      \
        .val = v,                               \
 -      .visibility = elx2_visibility,          \
 +      .visibility = hidden_user_visibility,   \
  }
  
  /*
@@@ -2007,7 -2002,7 +2006,7 @@@ static const struct sys_reg_desc sys_re
        // DBGDTR[TR]X_EL0 share the same encoding
        { SYS_DESC(SYS_DBGDTRTX_EL0), trap_raz_wi },
  
 -      { SYS_DESC(SYS_DBGVCR32_EL2), NULL, reset_val, DBGVCR32_EL2, 0 },
 +      { SYS_DESC(SYS_DBGVCR32_EL2), trap_undef, reset_val, DBGVCR32_EL2, 0 },
  
        { SYS_DESC(SYS_MPIDR_EL1), NULL, reset_mpidr, MPIDR_EL1 },
  
@@@ -2099,7 -2094,6 +2098,6 @@@
                                        ID_AA64ISAR1_EL1_API |
                                        ID_AA64ISAR1_EL1_APA)),
        ID_WRITABLE(ID_AA64ISAR2_EL1, ~(ID_AA64ISAR2_EL1_RES0 |
-                                       ID_AA64ISAR2_EL1_MOPS |
                                        ID_AA64ISAR2_EL1_APA3 |
                                        ID_AA64ISAR2_EL1_GPA3)),
        ID_UNALLOCATED(6,3),
@@@ -2454,28 -2448,18 +2452,28 @@@
        EL2_REG(VTTBR_EL2, access_rw, reset_val, 0),
        EL2_REG(VTCR_EL2, access_rw, reset_val, 0),
  
 -      { SYS_DESC(SYS_DACR32_EL2), NULL, reset_unknown, DACR32_EL2 },
 +      { SYS_DESC(SYS_DACR32_EL2), trap_undef, reset_unknown, DACR32_EL2 },
        EL2_REG(HDFGRTR_EL2, access_rw, reset_val, 0),
        EL2_REG(HDFGWTR_EL2, access_rw, reset_val, 0),
        EL2_REG(SPSR_EL2, access_rw, reset_val, 0),
        EL2_REG(ELR_EL2, access_rw, reset_val, 0),
        { SYS_DESC(SYS_SP_EL1), access_sp_el1},
  
 -      { SYS_DESC(SYS_IFSR32_EL2), NULL, reset_unknown, IFSR32_EL2 },
 +      /* AArch32 SPSR_* are RES0 if trapped from a NV guest */
 +      { SYS_DESC(SYS_SPSR_irq), .access = trap_raz_wi,
 +        .visibility = hidden_user_visibility },
 +      { SYS_DESC(SYS_SPSR_abt), .access = trap_raz_wi,
 +        .visibility = hidden_user_visibility },
 +      { SYS_DESC(SYS_SPSR_und), .access = trap_raz_wi,
 +        .visibility = hidden_user_visibility },
 +      { SYS_DESC(SYS_SPSR_fiq), .access = trap_raz_wi,
 +        .visibility = hidden_user_visibility },
 +
 +      { SYS_DESC(SYS_IFSR32_EL2), trap_undef, reset_unknown, IFSR32_EL2 },
        EL2_REG(AFSR0_EL2, access_rw, reset_val, 0),
        EL2_REG(AFSR1_EL2, access_rw, reset_val, 0),
        EL2_REG(ESR_EL2, access_rw, reset_val, 0),
 -      { SYS_DESC(SYS_FPEXC32_EL2), NULL, reset_val, FPEXC32_EL2, 0x700 },
 +      { SYS_DESC(SYS_FPEXC32_EL2), trap_undef, reset_val, FPEXC32_EL2, 0x700 },
  
        EL2_REG(FAR_EL2, access_rw, reset_val, 0),
        EL2_REG(HPFAR_EL2, access_rw, reset_val, 0),