LoongArch: KVM: Add LASX (256bit SIMD) support

This patch adds LASX (256bit SIMD) support for LoongArch KVM.

There will be LASX exception in KVM when guest use the LASX instructions.
KVM will enable LASX and restore the vector registers for guest and then
return to guest to continue running.

Reviewed-by: Bibo Mao <[email protected]>
Signed-off-by: Tianrui Zhao <[email protected]>
Signed-off-by: Huacai Chen <[email protected]>

diff --git a/arch/loongarch/include/asm/kvm_host.h b/arch/loongarch/include/asm/kvm_host.h
index b0c5cdd8014c21a408bae0e3852103a4d180d26b..5bdb34b2c5d677099aa02b8800060864aecea72a 100644 (file)
--- a/arch/loongarch/include/asm/kvm_host.h
+++ b/arch/loongarch/include/asm/kvm_host.h
@@ -96,8 +96,9 @@ enum emulation_result {
 
 #define KVM_LARCH_FPU          (0x1 << 0)
 #define KVM_LARCH_LSX          (0x1 << 1)
 
 #define KVM_LARCH_FPU          (0x1 << 0)
 #define KVM_LARCH_LSX          (0x1 << 1)

-#define KVM_LARCH_SWCSR_LATEST (0x1 << 2)
-#define KVM_LARCH_HWCSR_USABLE (0x1 << 3)
+#define KVM_LARCH_LASX         (0x1 << 2)
+#define KVM_LARCH_SWCSR_LATEST (0x1 << 3)
+#define KVM_LARCH_HWCSR_USABLE (0x1 << 4)

 
 struct kvm_vcpu_arch {
        /*
 
 struct kvm_vcpu_arch {
        /*


@@ -189,6 +190,11 @@ static inline bool kvm_guest_has_lsx(struct kvm_vcpu_arch *arch)
        return arch->cpucfg[2] & CPUCFG2_LSX;
 }
 
        return arch->cpucfg[2] & CPUCFG2_LSX;
 }
 

+static inline bool kvm_guest_has_lasx(struct kvm_vcpu_arch *arch)
+{
+       return arch->cpucfg[2] & CPUCFG2_LASX;
+}
+

 /* Debug: dump vcpu state */
 int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu);
 
 /* Debug: dump vcpu state */
 int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu);
 

diff --git a/arch/loongarch/include/asm/kvm_vcpu.h b/arch/loongarch/include/asm/kvm_vcpu.h
index db08dd46b525ab4d71d2142f32a3b9bac2dc6f1a..e71ceb88f29eecdbebe40c407c1a93e4cd433337 100644 (file)
--- a/arch/loongarch/include/asm/kvm_vcpu.h
+++ b/arch/loongarch/include/asm/kvm_vcpu.h
@@ -65,6 +65,16 @@ static inline void kvm_save_lsx(struct loongarch_fpu *fpu) { }
 static inline void kvm_restore_lsx(struct loongarch_fpu *fpu) { }
 #endif
 
 static inline void kvm_restore_lsx(struct loongarch_fpu *fpu) { }
 #endif
 

+#ifdef CONFIG_CPU_HAS_LASX
+int kvm_own_lasx(struct kvm_vcpu *vcpu);
+void kvm_save_lasx(struct loongarch_fpu *fpu);
+void kvm_restore_lasx(struct loongarch_fpu *fpu);
+#else
+static inline int kvm_own_lasx(struct kvm_vcpu *vcpu) { }
+static inline void kvm_save_lasx(struct loongarch_fpu *fpu) { }
+static inline void kvm_restore_lasx(struct loongarch_fpu *fpu) { }
+#endif
+

 void kvm_init_timer(struct kvm_vcpu *vcpu, unsigned long hz);
 void kvm_reset_timer(struct kvm_vcpu *vcpu);
 void kvm_save_timer(struct kvm_vcpu *vcpu);
 void kvm_init_timer(struct kvm_vcpu *vcpu, unsigned long hz);
 void kvm_reset_timer(struct kvm_vcpu *vcpu);
 void kvm_save_timer(struct kvm_vcpu *vcpu);

diff --git a/arch/loongarch/kernel/fpu.S b/arch/loongarch/kernel/fpu.S
index a400924c0348bffeca7ee6d7ad7e2628d41fb5c7..4382e36ae3d44466663aefaa5af1f23717876f35 100644 (file)
--- a/arch/loongarch/kernel/fpu.S
+++ b/arch/loongarch/kernel/fpu.S
@@ -385,6 +385,7 @@ SYM_FUNC_START(_restore_lasx_upper)
        lasx_restore_all_upper a0 t0 t1
        jr      ra
 SYM_FUNC_END(_restore_lasx_upper)
        lasx_restore_all_upper a0 t0 t1
        jr      ra
 SYM_FUNC_END(_restore_lasx_upper)

+EXPORT_SYMBOL(_restore_lasx_upper)

 
 SYM_FUNC_START(_init_lasx_upper)
        lasx_init_all_upper t1
 
 SYM_FUNC_START(_init_lasx_upper)
        lasx_init_all_upper t1

diff --git a/arch/loongarch/kvm/exit.c b/arch/loongarch/kvm/exit.c
index 676f7a3a335c297e90b0d6f9a8a2493bf218ef43..ed1d89d53e2e6da0c8b73ed17d97146721347d16 100644 (file)
--- a/arch/loongarch/kvm/exit.c
+++ b/arch/loongarch/kvm/exit.c
@@ -670,6 +670,21 @@ static int kvm_handle_lsx_disabled(struct kvm_vcpu *vcpu)
        return RESUME_GUEST;
 }
 
        return RESUME_GUEST;
 }
 

+/*
+ * kvm_handle_lasx_disabled() - Guest used LASX while disabled in root.
+ * @vcpu:      Virtual CPU context.
+ *
+ * Handle when the guest attempts to use LASX when it is disabled in the root
+ * context.
+ */
+static int kvm_handle_lasx_disabled(struct kvm_vcpu *vcpu)
+{
+       if (kvm_own_lasx(vcpu))
+               kvm_queue_exception(vcpu, EXCCODE_INE, 0);
+
+       return RESUME_GUEST;
+}
+

 /*
  * LoongArch KVM callback handling for unimplemented guest exiting
  */
 /*
  * LoongArch KVM callback handling for unimplemented guest exiting
  */


@@ -699,6 +714,7 @@ static exit_handle_fn kvm_fault_tables[EXCCODE_INT_START] = {
        [EXCCODE_TLBM]                  = kvm_handle_write_fault,
        [EXCCODE_FPDIS]                 = kvm_handle_fpu_disabled,
        [EXCCODE_LSXDIS]                = kvm_handle_lsx_disabled,
        [EXCCODE_TLBM]                  = kvm_handle_write_fault,
        [EXCCODE_FPDIS]                 = kvm_handle_fpu_disabled,
        [EXCCODE_LSXDIS]                = kvm_handle_lsx_disabled,

+       [EXCCODE_LASXDIS]               = kvm_handle_lasx_disabled,

        [EXCCODE_GSPR]                  = kvm_handle_gspr,
 };
 
        [EXCCODE_GSPR]                  = kvm_handle_gspr,
 };
 

diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S
index 00fbf772d16f9969f16f72c127178de0f2d460f7..ba976509bfe819ec51fdaa08f2a1ba4a334755cd 100644 (file)
--- a/arch/loongarch/kvm/switch.S
+++ b/arch/loongarch/kvm/switch.S
@@ -261,6 +261,21 @@ SYM_FUNC_START(kvm_restore_lsx)
 SYM_FUNC_END(kvm_restore_lsx)
 #endif
 
 SYM_FUNC_END(kvm_restore_lsx)
 #endif
 

+#ifdef CONFIG_CPU_HAS_LASX
+SYM_FUNC_START(kvm_save_lasx)
+       fpu_save_csr    a0 t1
+       fpu_save_cc     a0 t1 t2
+       lasx_save_data  a0 t1
+       jr              ra
+SYM_FUNC_END(kvm_save_lasx)
+
+SYM_FUNC_START(kvm_restore_lasx)
+       lasx_restore_data a0 t1
+       fpu_restore_cc    a0 t1 t2
+       fpu_restore_csr   a0 t1 t2
+       jr                ra
+SYM_FUNC_END(kvm_restore_lasx)
+#endif

        .section ".rodata"
 SYM_DATA(kvm_exception_size, .quad kvm_exc_entry_end - kvm_exc_entry)
 SYM_DATA(kvm_enter_guest_size, .quad kvm_enter_guest_end - kvm_enter_guest)
        .section ".rodata"
 SYM_DATA(kvm_exception_size, .quad kvm_exc_entry_end - kvm_exc_entry)
 SYM_DATA(kvm_enter_guest_size, .quad kvm_enter_guest_end - kvm_enter_guest)

diff --git a/arch/loongarch/kvm/trace.h b/arch/loongarch/kvm/trace.h
index 7da4e230e896f49fca7721140b9cfa6ab761d0e1..c2484ad4cffa2102a61f2bd4d0a6f537a6501906 100644 (file)
--- a/arch/loongarch/kvm/trace.h
+++ b/arch/loongarch/kvm/trace.h
@@ -103,6 +103,7 @@ TRACE_EVENT(kvm_exit_gspr,
 
 #define KVM_TRACE_AUX_FPU              1
 #define KVM_TRACE_AUX_LSX              2
 
 #define KVM_TRACE_AUX_FPU              1
 #define KVM_TRACE_AUX_LSX              2

+#define KVM_TRACE_AUX_LASX             3

 
 #define kvm_trace_symbol_aux_op                                \
        { KVM_TRACE_AUX_SAVE,           "save" },       \
 
 #define kvm_trace_symbol_aux_op                                \
        { KVM_TRACE_AUX_SAVE,           "save" },       \


@@ -113,7 +114,8 @@ TRACE_EVENT(kvm_exit_gspr,
 
 #define kvm_trace_symbol_aux_state                     \
        { KVM_TRACE_AUX_FPU,     "FPU" },               \
 
 #define kvm_trace_symbol_aux_state                     \
        { KVM_TRACE_AUX_FPU,     "FPU" },               \

-       { KVM_TRACE_AUX_LSX,     "LSX" }
+       { KVM_TRACE_AUX_LSX,     "LSX" },               \
+       { KVM_TRACE_AUX_LASX,    "LASX" }

 
 TRACE_EVENT(kvm_aux,
            TP_PROTO(struct kvm_vcpu *vcpu, unsigned int op,
 
 TRACE_EVENT(kvm_aux,
            TP_PROTO(struct kvm_vcpu *vcpu, unsigned int op,

diff --git a/arch/loongarch/kvm/vcpu.c b/arch/loongarch/kvm/vcpu.c
index 80487d177ca4a32c00777c74c8b4bb80f1e06a44..27701991886dda7e3a6f75bd8a7f71a86995735b 100644 (file)
--- a/arch/loongarch/kvm/vcpu.c
+++ b/arch/loongarch/kvm/vcpu.c
@@ -317,6 +317,13 @@ static int _kvm_get_cpucfg(int id, u64 *v)
                 */
                if (cpu_has_lsx)
                        *v |= CPUCFG2_LSX;
                 */
                if (cpu_has_lsx)
                        *v |= CPUCFG2_LSX;

+               /*
+                * if LASX is supported by CPU, it is also supported by KVM,
+                * as we implement it.
+                */
+               if (cpu_has_lasx)
+                       *v |= CPUCFG2_LASX;
+

                break;
        default:
                ret = -EINVAL;
                break;
        default:
                ret = -EINVAL;


@@ -753,12 +760,54 @@ int kvm_own_lsx(struct kvm_vcpu *vcpu)
 }
 #endif
 
 }
 #endif
 

+#ifdef CONFIG_CPU_HAS_LASX
+/* Enable LASX and restore context */
+int kvm_own_lasx(struct kvm_vcpu *vcpu)
+{
+       if (!kvm_guest_has_fpu(&vcpu->arch) || !kvm_guest_has_lsx(&vcpu->arch) || !kvm_guest_has_lasx(&vcpu->arch))
+               return -EINVAL;
+
+       preempt_disable();
+
+       set_csr_euen(CSR_EUEN_FPEN | CSR_EUEN_LSXEN | CSR_EUEN_LASXEN);
+       switch (vcpu->arch.aux_inuse & (KVM_LARCH_FPU | KVM_LARCH_LSX)) {
+       case KVM_LARCH_LSX:
+       case KVM_LARCH_LSX | KVM_LARCH_FPU:
+               /* Guest LSX state already loaded, only restore upper LASX state */
+               _restore_lasx_upper(&vcpu->arch.fpu);
+               break;
+       case KVM_LARCH_FPU:
+               /* Guest FP state already loaded, only restore upper LSX & LASX state */
+               _restore_lsx_upper(&vcpu->arch.fpu);
+               _restore_lasx_upper(&vcpu->arch.fpu);
+               break;
+       default:
+               /* Neither FP or LSX already active, restore full LASX state */
+               kvm_restore_lasx(&vcpu->arch.fpu);
+               break;
+       }
+
+       trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_LASX);
+       vcpu->arch.aux_inuse |= KVM_LARCH_LASX | KVM_LARCH_LSX | KVM_LARCH_FPU;
+       preempt_enable();
+
+       return 0;
+}
+#endif
+

 /* Save context and disable FPU */
 void kvm_lose_fpu(struct kvm_vcpu *vcpu)
 {
        preempt_disable();
 
 /* Save context and disable FPU */
 void kvm_lose_fpu(struct kvm_vcpu *vcpu)
 {
        preempt_disable();
 

-       if (vcpu->arch.aux_inuse & KVM_LARCH_LSX) {
+       if (vcpu->arch.aux_inuse & KVM_LARCH_LASX) {
+               kvm_save_lasx(&vcpu->arch.fpu);
+               vcpu->arch.aux_inuse &= ~(KVM_LARCH_LSX | KVM_LARCH_FPU | KVM_LARCH_LASX);
+               trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_LASX);
+
+               /* Disable LASX & LSX & FPU */
+               clear_csr_euen(CSR_EUEN_FPEN | CSR_EUEN_LSXEN | CSR_EUEN_LASXEN);
+       } else if (vcpu->arch.aux_inuse & KVM_LARCH_LSX) {

                kvm_save_lsx(&vcpu->arch.fpu);
                vcpu->arch.aux_inuse &= ~(KVM_LARCH_LSX | KVM_LARCH_FPU);
                trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_LSX);
                kvm_save_lsx(&vcpu->arch.fpu);
                vcpu->arch.aux_inuse &= ~(KVM_LARCH_LSX | KVM_LARCH_FPU);
                trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_LSX);