]> Git Repo - linux.git/blame - arch/x86/kvm/svm.c
projects / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
KVM: update gfn_to_hva() to use gfn_to_hva_memslot()
[linux.git] / arch / x86 / kvm / svm.c
CommitLineData
6aa8b732
AK		 1/*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * AMD SVM support
5 *
6 * Copyright (C) 2006 Qumranet, Inc.
7 *
8 * Authors:
9 * Yaniv Kamay <[email protected]>
10 * Avi Kivity <[email protected]>
11 *
12 * This work is licensed under the terms of the GNU GPL, version 2. See
13 * the COPYING file in the top-level directory.
14 *
15 */
edf88417
AK		 16#include <linux/kvm_host.h>
17
85f455f7 18#include "irq.h"
1d737c8a 19#include "mmu.h"
5fdbf976 20#include "kvm_cache_regs.h"
fe4c7b19 21#include "x86.h"
e495606d 22
6aa8b732 23#include <linux/module.h>
9d8f549d 24#include <linux/kernel.h>
6aa8b732
AK		 25#include <linux/vmalloc.h>
26#include <linux/highmem.h>
e8edc6e0 27#include <linux/sched.h>
229456fc 28#include <linux/ftrace_event.h>
5a0e3ad6 29#include <linux/slab.h>
6aa8b732 30
e495606d 31#include <asm/desc.h>
6aa8b732 32
63d1142f 33#include <asm/virtext.h>
229456fc 34#include "trace.h"
63d1142f 35
4ecac3fd
AK		 36#define __ex(x) __kvm_handle_fault_on_reboot(x)
37
6aa8b732
AK		 38MODULE_AUTHOR("Qumranet");
39MODULE_LICENSE("GPL");
40
41#define IOPM_ALLOC_ORDER 2
42#define MSRPM_ALLOC_ORDER 1
43
6aa8b732
AK		 44#define SEG_TYPE_LDT 2
45#define SEG_TYPE_BUSY_TSS16 3
46
80b7706e
JR		 47#define SVM_FEATURE_NPT (1 << 0)
48#define SVM_FEATURE_LBRV (1 << 1)
94c935a1 49#define SVM_FEATURE_SVML (1 << 2)
66b7138f 50#define SVM_FEATURE_NRIP (1 << 3)
565d0998 51#define SVM_FEATURE_PAUSE_FILTER (1 << 10)
80b7706e 52
410e4d57
JR		 53#define NESTED_EXIT_HOST 0 /* Exit handled on host level */
54#define NESTED_EXIT_DONE 1 /* Exit caused nested vmexit */
55#define NESTED_EXIT_CONTINUE 2 /* Further checks needed */
56
24e09cbf
JR		 57#define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
58
6c8166a7
AK		 59static const u32 host_save_user_msrs[] = {
60#ifdef CONFIG_X86_64
61 MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE,
62 MSR_FS_BASE,
63#endif
64 MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
65};
66
67#define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)
68
69struct kvm_vcpu;
70
e6aa9abd
JR		 71struct nested_state {
72 struct vmcb *hsave;
73 u64 hsave_msr;
4a810181 74 u64 vm_cr_msr;
e6aa9abd
JR		 75 u64 vmcb;
76
77 /* These are the merged vectors */
78 u32 *msrpm;
79
80 /* gpa pointers to the real vectors */
81 u64 vmcb_msrpm;
aad42c64 82
cd3ff653
JR		 83 /* A VMEXIT is required but not yet emulated */
84 bool exit_required;
85
aad42c64
JR		 86 /* cache for intercepts of the guest */
87 u16 intercept_cr_read;
88 u16 intercept_cr_write;
89 u16 intercept_dr_read;
90 u16 intercept_dr_write;
91 u32 intercept_exceptions;
92 u64 intercept;
93
e6aa9abd
JR		 94};
95
6c8166a7
AK		 96struct vcpu_svm {
97 struct kvm_vcpu vcpu;
98 struct vmcb *vmcb;
99 unsigned long vmcb_pa;
100 struct svm_cpu_data *svm_data;
101 uint64_t asid_generation;
102 uint64_t sysenter_esp;
103 uint64_t sysenter_eip;
104
105 u64 next_rip;
106
107 u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
108 u64 host_gs_base;
6c8166a7
AK		 109
110 u32 *msrpm;
6c8166a7 111
e6aa9abd 112 struct nested_state nested;
6be7d306
JK		 113
114 bool nmi_singlestep;
66b7138f
JK		 115
116 unsigned int3_injected;
117 unsigned long int3_rip;
6c8166a7
AK		 118};
119
709ddebf
JR		 120/* enable NPT for AMD64 and X86 with PAE */
121#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
122static bool npt_enabled = true;
123#else
e0231715 124static bool npt_enabled;
709ddebf 125#endif
6c7dac72
JR		 126static int npt = 1;
127
128module_param(npt, int, S_IRUGO);
e3da3acd 129
4b6e4dca 130static int nested = 1;
236de055
AG		 131module_param(nested, int, S_IRUGO);
132
44874f84 133static void svm_flush_tlb(struct kvm_vcpu *vcpu);
a5c3832d 134static void svm_complete_interrupts(struct vcpu_svm *svm);
04d2cc77 135
410e4d57 136static int nested_svm_exit_handled(struct vcpu_svm *svm);
b8e88bc8 137static int nested_svm_intercept(struct vcpu_svm *svm);
cf74a78b 138static int nested_svm_vmexit(struct vcpu_svm *svm);
cf74a78b
AG		 139static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
140 bool has_error_code, u32 error_code);
141
a2fa3e9f
GH		 142static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
143{
fb3f0f51 144 return container_of(vcpu, struct vcpu_svm, vcpu);
a2fa3e9f
GH		 145}
146
3d6368ef
AG		 147static inline bool is_nested(struct vcpu_svm *svm)
148{
e6aa9abd 149 return svm->nested.vmcb;
3d6368ef
AG		 150}
151
2af9194d
JR		 152static inline void enable_gif(struct vcpu_svm *svm)
153{
154 svm->vcpu.arch.hflags |= HF_GIF_MASK;
155}
156
157static inline void disable_gif(struct vcpu_svm *svm)
158{
159 svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
160}
161
162static inline bool gif_set(struct vcpu_svm *svm)
163{
164 return !!(svm->vcpu.arch.hflags & HF_GIF_MASK);
165}
166
4866d5e3 167static unsigned long iopm_base;
6aa8b732
AK		 168
169struct kvm_ldttss_desc {
170 u16 limit0;
171 u16 base0;
e0231715
JR		 172 unsigned base1:8, type:5, dpl:2, p:1;
173 unsigned limit1:4, zero0:3, g:1, base2:8;
6aa8b732
AK		 174 u32 base3;
175 u32 zero1;
176} __attribute__((packed));
177
178struct svm_cpu_data {
179 int cpu;
180
5008fdf5
AK		 181 u64 asid_generation;
182 u32 max_asid;
183 u32 next_asid;
6aa8b732
AK		 184 struct kvm_ldttss_desc *tss_desc;
185
186 struct page *save_area;
187};
188
189static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
80b7706e 190static uint32_t svm_features;
6aa8b732
AK		 191
192struct svm_init_data {
193 int cpu;
194 int r;
195};
196
197static u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
198
9d8f549d 199#define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
6aa8b732
AK		 200#define MSRS_RANGE_SIZE 2048
201#define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
202
203#define MAX_INST_SIZE 15
204
80b7706e
JR		 205static inline u32 svm_has(u32 feat)
206{
207 return svm_features & feat;
208}
209
6aa8b732
AK		 210static inline void clgi(void)
211{
4ecac3fd 212 asm volatile (__ex(SVM_CLGI));
6aa8b732
AK		 213}
214
215static inline void stgi(void)
216{
4ecac3fd 217 asm volatile (__ex(SVM_STGI));
6aa8b732
AK		 218}
219
220static inline void invlpga(unsigned long addr, u32 asid)
221{
e0231715 222 asm volatile (__ex(SVM_INVLPGA) : : "a"(addr), "c"(asid));
6aa8b732
AK		 223}
224
6aa8b732
AK		 225static inline void force_new_asid(struct kvm_vcpu *vcpu)
226{
a2fa3e9f 227 to_svm(vcpu)->asid_generation--;
6aa8b732
AK		 228}
229
230static inline void flush_guest_tlb(struct kvm_vcpu *vcpu)
231{
232 force_new_asid(vcpu);
233}
234
235static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
236{
709ddebf 237 if (!npt_enabled && !(efer & EFER_LMA))
2b5203ee 238 efer &= ~EFER_LME;
6aa8b732 239
9962d032 240 to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME;
f6801dff 241 vcpu->arch.efer = efer;
6aa8b732
AK		 242}
243
6aa8b732
AK		 244static int is_external_interrupt(u32 info)
245{
246 info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
247 return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
248}
249
2809f5d2
GC		 250static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
251{
252 struct vcpu_svm *svm = to_svm(vcpu);
253 u32 ret = 0;
254
255 if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
48005f64 256 ret |= KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
2809f5d2
GC		 257 return ret & mask;
258}
259
260static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
261{
262 struct vcpu_svm *svm = to_svm(vcpu);
263
264 if (mask == 0)
265 svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
266 else
267 svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
268
269}
270
6aa8b732
AK		 271static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
272{
a2fa3e9f
GH		 273 struct vcpu_svm *svm = to_svm(vcpu);
274
275 if (!svm->next_rip) {
851ba692 276 if (emulate_instruction(vcpu, 0, 0, EMULTYPE_SKIP) !=
f629cf84
GN		 277 EMULATE_DONE)
278 printk(KERN_DEBUG "%s: NOP\n", __func__);
6aa8b732
AK		 279 return;
280 }
5fdbf976
MT		 281 if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)
282 printk(KERN_ERR "%s: ip 0x%lx next 0x%llx\n",
283 __func__, kvm_rip_read(vcpu), svm->next_rip);
6aa8b732 284
5fdbf976 285 kvm_rip_write(vcpu, svm->next_rip);
2809f5d2 286 svm_set_interrupt_shadow(vcpu, 0);
6aa8b732
AK		 287}
288
116a4752
JK		 289static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
290 bool has_error_code, u32 error_code)
291{
292 struct vcpu_svm *svm = to_svm(vcpu);
293
e0231715
JR		 294 /*
295 * If we are within a nested VM we'd better #VMEXIT and let the guest
296 * handle the exception
297 */
116a4752
JK		 298 if (nested_svm_check_exception(svm, nr, has_error_code, error_code))
299 return;
300
66b7138f
JK		 301 if (nr == BP_VECTOR && !svm_has(SVM_FEATURE_NRIP)) {
302 unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu);
303
304 /*
305 * For guest debugging where we have to reinject #BP if some
306 * INT3 is guest-owned:
307 * Emulate nRIP by moving RIP forward. Will fail if injection
308 * raises a fault that is not intercepted. Still better than
309 * failing in all cases.
310 */
311 skip_emulated_instruction(&svm->vcpu);
312 rip = kvm_rip_read(&svm->vcpu);
313 svm->int3_rip = rip + svm->vmcb->save.cs.base;
314 svm->int3_injected = rip - old_rip;
315 }
316
116a4752
JK		 317 svm->vmcb->control.event_inj = nr
318 | SVM_EVTINJ_VALID
319 | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
320 | SVM_EVTINJ_TYPE_EXEPT;
321 svm->vmcb->control.event_inj_err = error_code;
322}
323
6aa8b732
AK		 324static int has_svm(void)
325{
63d1142f 326 const char *msg;
6aa8b732 327
63d1142f 328 if (!cpu_has_svm(&msg)) {
ff81ff10 329 printk(KERN_INFO "has_svm: %s\n", msg);
6aa8b732
AK		 330 return 0;
331 }
332
6aa8b732
AK		 333 return 1;
334}
335
336static void svm_hardware_disable(void *garbage)
337{
2c8dceeb 338 cpu_svm_disable();
6aa8b732
AK		 339}
340
10474ae8 341static int svm_hardware_enable(void *garbage)
6aa8b732
AK		 342{
343
0fe1e009 344 struct svm_cpu_data *sd;
6aa8b732 345 uint64_t efer;
89a27f4d 346 struct desc_ptr gdt_descr;
6aa8b732
AK		 347 struct desc_struct *gdt;
348 int me = raw_smp_processor_id();
349
10474ae8
AG		 350 rdmsrl(MSR_EFER, efer);
351 if (efer & EFER_SVME)
352 return -EBUSY;
353
6aa8b732 354 if (!has_svm()) {
e6732a5a
ZA		 355 printk(KERN_ERR "svm_hardware_enable: err EOPNOTSUPP on %d\n",
356 me);
10474ae8 357 return -EINVAL;
6aa8b732 358 }
0fe1e009 359 sd = per_cpu(svm_data, me);
6aa8b732 360
0fe1e009 361 if (!sd) {
e6732a5a 362 printk(KERN_ERR "svm_hardware_enable: svm_data is NULL on %d\n",
6aa8b732 363 me);
10474ae8 364 return -EINVAL;
6aa8b732
AK		 365 }
366
0fe1e009
TH		 367 sd->asid_generation = 1;
368 sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
369 sd->next_asid = sd->max_asid + 1;
6aa8b732 370
b792c344 371 kvm_get_gdt(&gdt_descr);
89a27f4d 372 gdt = (struct desc_struct *)gdt_descr.address;
0fe1e009 373 sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
6aa8b732 374
9962d032 375 wrmsrl(MSR_EFER, efer | EFER_SVME);
6aa8b732 376
d0316554 377 wrmsrl(MSR_VM_HSAVE_PA, page_to_pfn(sd->save_area) << PAGE_SHIFT);
10474ae8
AG		 378
379 return 0;
6aa8b732
AK		 380}
381
0da1db75
JR		 382static void svm_cpu_uninit(int cpu)
383{
0fe1e009 384 struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id());
0da1db75 385
0fe1e009 386 if (!sd)
0da1db75
JR		 387 return;
388
389 per_cpu(svm_data, raw_smp_processor_id()) = NULL;
0fe1e009
TH		 390 __free_page(sd->save_area);
391 kfree(sd);
0da1db75
JR		 392}
393
6aa8b732
AK		 394static int svm_cpu_init(int cpu)
395{
0fe1e009 396 struct svm_cpu_data *sd;
6aa8b732
AK		 397 int r;
398
0fe1e009
TH		 399 sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
400 if (!sd)
6aa8b732 401 return -ENOMEM;
0fe1e009
TH		 402 sd->cpu = cpu;
403 sd->save_area = alloc_page(GFP_KERNEL);
6aa8b732 404 r = -ENOMEM;
0fe1e009 405 if (!sd->save_area)
6aa8b732
AK		 406 goto err_1;
407
0fe1e009 408 per_cpu(svm_data, cpu) = sd;
6aa8b732
AK		 409
410 return 0;
411
412err_1:
0fe1e009 413 kfree(sd);
6aa8b732
AK		 414 return r;
415
416}
417
bfc733a7
RR		 418static void set_msr_interception(u32 *msrpm, unsigned msr,
419 int read, int write)
6aa8b732
AK		 420{
421 int i;
422
423 for (i = 0; i < NUM_MSR_MAPS; i++) {
424 if (msr >= msrpm_ranges[i] &&
425 msr < msrpm_ranges[i] + MSRS_IN_RANGE) {
426 u32 msr_offset = (i * MSRS_IN_RANGE + msr -
427 msrpm_ranges[i]) * 2;
428
429 u32 *base = msrpm + (msr_offset / 32);
430 u32 msr_shift = msr_offset % 32;
431 u32 mask = ((write) ? 0 : 2) | ((read) ? 0 : 1);
432 *base = (*base & ~(0x3 << msr_shift)) |
433 (mask << msr_shift);
bfc733a7 434 return;
6aa8b732
AK		 435 }
436 }
bfc733a7 437 BUG();
6aa8b732
AK		 438}
439
f65c229c
JR		 440static void svm_vcpu_init_msrpm(u32 *msrpm)
441{
442 memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));
443
444#ifdef CONFIG_X86_64
445 set_msr_interception(msrpm, MSR_GS_BASE, 1, 1);
446 set_msr_interception(msrpm, MSR_FS_BASE, 1, 1);
447 set_msr_interception(msrpm, MSR_KERNEL_GS_BASE, 1, 1);
448 set_msr_interception(msrpm, MSR_LSTAR, 1, 1);
449 set_msr_interception(msrpm, MSR_CSTAR, 1, 1);
450 set_msr_interception(msrpm, MSR_SYSCALL_MASK, 1, 1);
451#endif
452 set_msr_interception(msrpm, MSR_K6_STAR, 1, 1);
453 set_msr_interception(msrpm, MSR_IA32_SYSENTER_CS, 1, 1);
f65c229c
JR		 454}
455
24e09cbf
JR		 456static void svm_enable_lbrv(struct vcpu_svm *svm)
457{
458 u32 *msrpm = svm->msrpm;
459
460 svm->vmcb->control.lbr_ctl = 1;
461 set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
462 set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
463 set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
464 set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
465}
466
467static void svm_disable_lbrv(struct vcpu_svm *svm)
468{
469 u32 *msrpm = svm->msrpm;
470
471 svm->vmcb->control.lbr_ctl = 0;
472 set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
473 set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
474 set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
475 set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
476}
477
6aa8b732
AK		 478static __init int svm_hardware_setup(void)
479{
480 int cpu;
481 struct page *iopm_pages;
f65c229c 482 void *iopm_va;
6aa8b732
AK		 483 int r;
484
6aa8b732
AK		 485 iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);
486
487 if (!iopm_pages)
488 return -ENOMEM;
c8681339
AL		 489
490 iopm_va = page_address(iopm_pages);
491 memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
6aa8b732
AK		 492 iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
493
50a37eb4
JR		 494 if (boot_cpu_has(X86_FEATURE_NX))
495 kvm_enable_efer_bits(EFER_NX);
496
1b2fd70c
AG		 497 if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
498 kvm_enable_efer_bits(EFER_FFXSR);
499
236de055
AG		 500 if (nested) {
501 printk(KERN_INFO "kvm: Nested Virtualization enabled\n");
502 kvm_enable_efer_bits(EFER_SVME);
503 }
504
3230bb47 505 for_each_possible_cpu(cpu) {
6aa8b732
AK		 506 r = svm_cpu_init(cpu);
507 if (r)
f65c229c 508 goto err;
6aa8b732 509 }
33bd6a0b
JR		 510
511 svm_features = cpuid_edx(SVM_CPUID_FUNC);
512
e3da3acd
JR		 513 if (!svm_has(SVM_FEATURE_NPT))
514 npt_enabled = false;
515
6c7dac72
JR		 516 if (npt_enabled && !npt) {
517 printk(KERN_INFO "kvm: Nested Paging disabled\n");
518 npt_enabled = false;
519 }
520
18552672 521 if (npt_enabled) {
e3da3acd 522 printk(KERN_INFO "kvm: Nested Paging enabled\n");
18552672 523 kvm_enable_tdp();
5f4cb662
JR		 524 } else
525 kvm_disable_tdp();
e3da3acd 526
6aa8b732
AK		 527 return 0;
528
f65c229c 529err:
6aa8b732
AK		 530 __free_pages(iopm_pages, IOPM_ALLOC_ORDER);
531 iopm_base = 0;
532 return r;
533}
534
535static __exit void svm_hardware_unsetup(void)
536{
0da1db75
JR		 537 int cpu;
538
3230bb47 539 for_each_possible_cpu(cpu)
0da1db75
JR		 540 svm_cpu_uninit(cpu);
541
6aa8b732 542 __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
f65c229c 543 iopm_base = 0;
6aa8b732
AK		 544}
545
546static void init_seg(struct vmcb_seg *seg)
547{
548 seg->selector = 0;
549 seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
e0231715 550 SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
6aa8b732
AK		 551 seg->limit = 0xffff;
552 seg->base = 0;
553}
554
555static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
556{
557 seg->selector = 0;
558 seg->attrib = SVM_SELECTOR_P_MASK | type;
559 seg->limit = 0xffff;
560 seg->base = 0;
561}
562
e6101a96 563static void init_vmcb(struct vcpu_svm *svm)
6aa8b732 564{
e6101a96
JR		 565 struct vmcb_control_area *control = &svm->vmcb->control;
566 struct vmcb_save_area *save = &svm->vmcb->save;
6aa8b732 567
bff78274
AK		 568 svm->vcpu.fpu_active = 1;
569
e0231715 570 control->intercept_cr_read = INTERCEPT_CR0_MASK |
6aa8b732 571 INTERCEPT_CR3_MASK |
649d6864 572 INTERCEPT_CR4_MASK;
6aa8b732 573
e0231715 574 control->intercept_cr_write = INTERCEPT_CR0_MASK |
6aa8b732 575 INTERCEPT_CR3_MASK |
80a8119c
AK		 576 INTERCEPT_CR4_MASK |
577 INTERCEPT_CR8_MASK;
6aa8b732 578
e0231715 579 control->intercept_dr_read = INTERCEPT_DR0_MASK |
6aa8b732
AK		 580 INTERCEPT_DR1_MASK |
581 INTERCEPT_DR2_MASK |
727f5a23
JK		 582 INTERCEPT_DR3_MASK |
583 INTERCEPT_DR4_MASK |
584 INTERCEPT_DR5_MASK |
585 INTERCEPT_DR6_MASK |
586 INTERCEPT_DR7_MASK;
6aa8b732 587
e0231715 588 control->intercept_dr_write = INTERCEPT_DR0_MASK |
6aa8b732
AK		 589 INTERCEPT_DR1_MASK |
590 INTERCEPT_DR2_MASK |
591 INTERCEPT_DR3_MASK |
727f5a23 592 INTERCEPT_DR4_MASK |
6aa8b732 593 INTERCEPT_DR5_MASK |
727f5a23 594 INTERCEPT_DR6_MASK |
6aa8b732
AK		 595 INTERCEPT_DR7_MASK;
596
7aa81cc0 597 control->intercept_exceptions = (1 << PF_VECTOR) |
53371b50
JR		 598 (1 << UD_VECTOR) |
599 (1 << MC_VECTOR);
6aa8b732
AK		 600
601
e0231715 602 control->intercept = (1ULL << INTERCEPT_INTR) |
6aa8b732 603 (1ULL << INTERCEPT_NMI) |
0152527b 604 (1ULL << INTERCEPT_SMI) |
d225157b 605 (1ULL << INTERCEPT_SELECTIVE_CR0) |
6aa8b732 606 (1ULL << INTERCEPT_CPUID) |
cf5a94d1 607 (1ULL << INTERCEPT_INVD) |
6aa8b732 608 (1ULL << INTERCEPT_HLT) |
a7052897 609 (1ULL << INTERCEPT_INVLPG) |
6aa8b732
AK		 610 (1ULL << INTERCEPT_INVLPGA) |
611 (1ULL << INTERCEPT_IOIO_PROT) |
612 (1ULL << INTERCEPT_MSR_PROT) |
613 (1ULL << INTERCEPT_TASK_SWITCH) |
46fe4ddd 614 (1ULL << INTERCEPT_SHUTDOWN) |
6aa8b732
AK		 615 (1ULL << INTERCEPT_VMRUN) |
616 (1ULL << INTERCEPT_VMMCALL) |
617 (1ULL << INTERCEPT_VMLOAD) |
618 (1ULL << INTERCEPT_VMSAVE) |
619 (1ULL << INTERCEPT_STGI) |
620 (1ULL << INTERCEPT_CLGI) |
916ce236 621 (1ULL << INTERCEPT_SKINIT) |
cf5a94d1 622 (1ULL << INTERCEPT_WBINVD) |
916ce236
JR		 623 (1ULL << INTERCEPT_MONITOR) |
624 (1ULL << INTERCEPT_MWAIT);
6aa8b732
AK		 625
626 control->iopm_base_pa = iopm_base;
f65c229c 627 control->msrpm_base_pa = __pa(svm->msrpm);
0cc5064d 628 control->tsc_offset = 0;
6aa8b732
AK		 629 control->int_ctl = V_INTR_MASKING_MASK;
630
631 init_seg(&save->es);
632 init_seg(&save->ss);
633 init_seg(&save->ds);
634 init_seg(&save->fs);
635 init_seg(&save->gs);
636
637 save->cs.selector = 0xf000;
638 /* Executable/Readable Code Segment */
639 save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
640 SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
641 save->cs.limit = 0xffff;
d92899a0
AK		 642 /*
643 * cs.base should really be 0xffff0000, but vmx can't handle that, so
644 * be consistent with it.
645 *
646 * Replace when we have real mode working for vmx.
647 */
648 save->cs.base = 0xf0000;
6aa8b732
AK		 649
650 save->gdtr.limit = 0xffff;
651 save->idtr.limit = 0xffff;
652
653 init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
654 init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
655
9962d032 656 save->efer = EFER_SVME;
d77c26fc 657 save->dr6 = 0xffff0ff0;
6aa8b732
AK		 658 save->dr7 = 0x400;
659 save->rflags = 2;
660 save->rip = 0x0000fff0;
5fdbf976 661 svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
6aa8b732 662
e0231715
JR		 663 /*
664 * This is the guest-visible cr0 value.
18fa000a 665 * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0.
6aa8b732 666 */
18fa000a
EH		 667 svm->vcpu.arch.cr0 = X86_CR0_NW | X86_CR0_CD | X86_CR0_ET;
668 kvm_set_cr0(&svm->vcpu, svm->vcpu.arch.cr0);
669
66aee91a 670 save->cr4 = X86_CR4_PAE;
6aa8b732 671 /* rdx = ?? */
709ddebf
JR		 672
673 if (npt_enabled) {
674 /* Setup VMCB for Nested Paging */
675 control->nested_ctl = 1;
a7052897
MT		 676 control->intercept &= ~((1ULL << INTERCEPT_TASK_SWITCH) |
677 (1ULL << INTERCEPT_INVLPG));
709ddebf 678 control->intercept_exceptions &= ~(1 << PF_VECTOR);
888f9f3e
AK		 679 control->intercept_cr_read &= ~INTERCEPT_CR3_MASK;
680 control->intercept_cr_write &= ~INTERCEPT_CR3_MASK;
709ddebf 681 save->g_pat = 0x0007040600070406ULL;
709ddebf
JR		 682 save->cr3 = 0;
683 save->cr4 = 0;
684 }
a79d2f18 685 force_new_asid(&svm->vcpu);
1371d904 686
e6aa9abd 687 svm->nested.vmcb = 0;
2af9194d
JR		 688 svm->vcpu.arch.hflags = 0;
689
565d0998
ML		 690 if (svm_has(SVM_FEATURE_PAUSE_FILTER)) {
691 control->pause_filter_count = 3000;
692 control->intercept |= (1ULL << INTERCEPT_PAUSE);
693 }
694
2af9194d 695 enable_gif(svm);
6aa8b732
AK		 696}
697
e00c8cf2 698static int svm_vcpu_reset(struct kvm_vcpu *vcpu)
04d2cc77
AK		 699{
700 struct vcpu_svm *svm = to_svm(vcpu);
701
e6101a96 702 init_vmcb(svm);
70433389 703
c5af89b6 704 if (!kvm_vcpu_is_bsp(vcpu)) {
5fdbf976 705 kvm_rip_write(vcpu, 0);
ad312c7c
ZX		 706 svm->vmcb->save.cs.base = svm->vcpu.arch.sipi_vector << 12;
707 svm->vmcb->save.cs.selector = svm->vcpu.arch.sipi_vector << 8;
70433389 708 }
5fdbf976
MT		 709 vcpu->arch.regs_avail = ~0;
710 vcpu->arch.regs_dirty = ~0;
e00c8cf2
AK		 711
712 return 0;
04d2cc77
AK		 713}
714
fb3f0f51 715static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
6aa8b732 716{
a2fa3e9f 717 struct vcpu_svm *svm;
6aa8b732 718 struct page *page;
f65c229c 719 struct page *msrpm_pages;
b286d5d8 720 struct page *hsave_page;
3d6368ef 721 struct page *nested_msrpm_pages;
fb3f0f51 722 int err;
6aa8b732 723
c16f862d 724 svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
fb3f0f51
RR		 725 if (!svm) {
726 err = -ENOMEM;
727 goto out;
728 }
729
730 err = kvm_vcpu_init(&svm->vcpu, kvm, id);
731 if (err)
732 goto free_svm;
733
b7af4043 734 err = -ENOMEM;
6aa8b732 735 page = alloc_page(GFP_KERNEL);
b7af4043 736 if (!page)
fb3f0f51 737 goto uninit;
6aa8b732 738
f65c229c
JR		 739 msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
740 if (!msrpm_pages)
b7af4043 741 goto free_page1;
3d6368ef
AG		 742
743 nested_msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
744 if (!nested_msrpm_pages)
b7af4043 745 goto free_page2;
f65c229c 746
b286d5d8
AG		 747 hsave_page = alloc_page(GFP_KERNEL);
748 if (!hsave_page)
b7af4043
TY		 749 goto free_page3;
750
e6aa9abd 751 svm->nested.hsave = page_address(hsave_page);
b286d5d8 752
b7af4043
TY		 753 svm->msrpm = page_address(msrpm_pages);
754 svm_vcpu_init_msrpm(svm->msrpm);
755
e6aa9abd 756 svm->nested.msrpm = page_address(nested_msrpm_pages);
3d6368ef 757
a2fa3e9f
GH		 758 svm->vmcb = page_address(page);
759 clear_page(svm->vmcb);
760 svm->vmcb_pa = page_to_pfn(page) << PAGE_SHIFT;
761 svm->asid_generation = 0;
e6101a96 762 init_vmcb(svm);
a2fa3e9f 763
fb3f0f51 764 fx_init(&svm->vcpu);
ad312c7c 765 svm->vcpu.arch.apic_base = 0xfee00000 | MSR_IA32_APICBASE_ENABLE;
c5af89b6 766 if (kvm_vcpu_is_bsp(&svm->vcpu))
ad312c7c 767 svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
6aa8b732 768
fb3f0f51 769 return &svm->vcpu;
36241b8c 770
b7af4043
TY		 771free_page3:
772 __free_pages(nested_msrpm_pages, MSRPM_ALLOC_ORDER);
773free_page2:
774 __free_pages(msrpm_pages, MSRPM_ALLOC_ORDER);
775free_page1:
776 __free_page(page);
fb3f0f51
RR		 777uninit:
778 kvm_vcpu_uninit(&svm->vcpu);
779free_svm:
a4770347 780 kmem_cache_free(kvm_vcpu_cache, svm);
fb3f0f51
RR		 781out:
782 return ERR_PTR(err);
6aa8b732
AK		 783}
784
785static void svm_free_vcpu(struct kvm_vcpu *vcpu)
786{
a2fa3e9f
GH		 787 struct vcpu_svm *svm = to_svm(vcpu);
788
fb3f0f51 789 __free_page(pfn_to_page(svm->vmcb_pa >> PAGE_SHIFT));
f65c229c 790 __free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
e6aa9abd
JR		 791 __free_page(virt_to_page(svm->nested.hsave));
792 __free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER);
fb3f0f51 793 kvm_vcpu_uninit(vcpu);
a4770347 794 kmem_cache_free(kvm_vcpu_cache, svm);
6aa8b732
AK		 795}
796
15ad7146 797static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
6aa8b732 798{
a2fa3e9f 799 struct vcpu_svm *svm = to_svm(vcpu);
15ad7146 800 int i;
0cc5064d 801
0cc5064d 802 if (unlikely(cpu != vcpu->cpu)) {
e935d48e 803 u64 delta;
0cc5064d 804
953899b6
JR		 805 if (check_tsc_unstable()) {
806 /*
807 * Make sure that the guest sees a monotonically
808 * increasing TSC.
809 */
810 delta = vcpu->arch.host_tsc - native_read_tsc();
811 svm->vmcb->control.tsc_offset += delta;
812 if (is_nested(svm))
813 svm->nested.hsave->control.tsc_offset += delta;
814 }
0cc5064d 815 vcpu->cpu = cpu;
2f599714 816 kvm_migrate_timers(vcpu);
4b656b12 817 svm->asid_generation = 0;
0cc5064d 818 }
94dfbdb3
AL		 819
820 for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
a2fa3e9f 821 rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
6aa8b732
AK		 822}
823
824static void svm_vcpu_put(struct kvm_vcpu *vcpu)
825{
a2fa3e9f 826 struct vcpu_svm *svm = to_svm(vcpu);
94dfbdb3
AL		 827 int i;
828
e1beb1d3 829 ++vcpu->stat.host_state_reload;
94dfbdb3 830 for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
a2fa3e9f 831 wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
94dfbdb3 832
e935d48e 833 vcpu->arch.host_tsc = native_read_tsc();
6aa8b732
AK		 834}
835
6aa8b732
AK		 836static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
837{
a2fa3e9f 838 return to_svm(vcpu)->vmcb->save.rflags;
6aa8b732
AK		 839}
840
841static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
842{
a2fa3e9f 843 to_svm(vcpu)->vmcb->save.rflags = rflags;
6aa8b732
AK		 844}
845
6de4f3ad
AK		 846static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
847{
848 switch (reg) {
849 case VCPU_EXREG_PDPTR:
850 BUG_ON(!npt_enabled);
851 load_pdptrs(vcpu, vcpu->arch.cr3);
852 break;
853 default:
854 BUG();
855 }
856}
857
f0b85051
AG		 858static void svm_set_vintr(struct vcpu_svm *svm)
859{
860 svm->vmcb->control.intercept |= 1ULL << INTERCEPT_VINTR;
861}
862
863static void svm_clear_vintr(struct vcpu_svm *svm)
864{
865 svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VINTR);
866}
867
6aa8b732
AK		 868static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
869{
a2fa3e9f 870 struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
6aa8b732
AK		 871
872 switch (seg) {
873 case VCPU_SREG_CS: return &save->cs;
874 case VCPU_SREG_DS: return &save->ds;
875 case VCPU_SREG_ES: return &save->es;
876 case VCPU_SREG_FS: return &save->fs;
877 case VCPU_SREG_GS: return &save->gs;
878 case VCPU_SREG_SS: return &save->ss;
879 case VCPU_SREG_TR: return &save->tr;
880 case VCPU_SREG_LDTR: return &save->ldtr;
881 }
882 BUG();
8b6d44c7 883 return NULL;
6aa8b732
AK		 884}
885
886static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
887{
888 struct vmcb_seg *s = svm_seg(vcpu, seg);
889
890 return s->base;
891}
892
893static void svm_get_segment(struct kvm_vcpu *vcpu,
894 struct kvm_segment *var, int seg)
895{
896 struct vmcb_seg *s = svm_seg(vcpu, seg);
897
898 var->base = s->base;
899 var->limit = s->limit;
900 var->selector = s->selector;
901 var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
902 var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
903 var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
904 var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
905 var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
906 var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
907 var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
908 var->g = (s->attrib >> SVM_SELECTOR_G_SHIFT) & 1;
25022acc 909
e0231715
JR		 910 /*
911 * AMD's VMCB does not have an explicit unusable field, so emulate it
19bca6ab
AP		 912 * for cross vendor migration purposes by "not present"
913 */
914 var->unusable = !var->present || (var->type == 0);
915
1fbdc7a5
AP		 916 switch (seg) {
917 case VCPU_SREG_CS:
918 /*
919 * SVM always stores 0 for the 'G' bit in the CS selector in
920 * the VMCB on a VMEXIT. This hurts cross-vendor migration:
921 * Intel's VMENTRY has a check on the 'G' bit.
922 */
25022acc 923 var->g = s->limit > 0xfffff;
1fbdc7a5
AP		 924 break;
925 case VCPU_SREG_TR:
926 /*
927 * Work around a bug where the busy flag in the tr selector
928 * isn't exposed
929 */
c0d09828 930 var->type |= 0x2;
1fbdc7a5
AP		 931 break;
932 case VCPU_SREG_DS:
933 case VCPU_SREG_ES:
934 case VCPU_SREG_FS:
935 case VCPU_SREG_GS:
936 /*
937 * The accessed bit must always be set in the segment
938 * descriptor cache, although it can be cleared in the
939 * descriptor, the cached bit always remains at 1. Since
940 * Intel has a check on this, set it here to support
941 * cross-vendor migration.
942 */
943 if (!var->unusable)
944 var->type |= 0x1;
945 break;
b586eb02 946 case VCPU_SREG_SS:
e0231715
JR		 947 /*
948 * On AMD CPUs sometimes the DB bit in the segment
b586eb02
AP		 949 * descriptor is left as 1, although the whole segment has
950 * been made unusable. Clear it here to pass an Intel VMX
951 * entry check when cross vendor migrating.
952 */
953 if (var->unusable)
954 var->db = 0;
955 break;
1fbdc7a5 956 }
6aa8b732
AK		 957}
958
2e4d2653
IE		 959static int svm_get_cpl(struct kvm_vcpu *vcpu)
960{
961 struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
962
963 return save->cpl;
964}
965
89a27f4d 966static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
6aa8b732 967{
a2fa3e9f
GH		 968 struct vcpu_svm *svm = to_svm(vcpu);
969
89a27f4d
GN		 970 dt->size = svm->vmcb->save.idtr.limit;
971 dt->address = svm->vmcb->save.idtr.base;
6aa8b732
AK		 972}
973
89a27f4d 974static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
6aa8b732 975{
a2fa3e9f
GH		 976 struct vcpu_svm *svm = to_svm(vcpu);
977
89a27f4d
GN		 978 svm->vmcb->save.idtr.limit = dt->size;
979 svm->vmcb->save.idtr.base = dt->address ;
6aa8b732
AK		 980}
981
89a27f4d 982static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
6aa8b732 983{
a2fa3e9f
GH		 984 struct vcpu_svm *svm = to_svm(vcpu);
985
89a27f4d
GN		 986 dt->size = svm->vmcb->save.gdtr.limit;
987 dt->address = svm->vmcb->save.gdtr.base;
6aa8b732
AK		 988}
989
89a27f4d 990static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
6aa8b732 991{
a2fa3e9f
GH		 992 struct vcpu_svm *svm = to_svm(vcpu);
993
89a27f4d
GN		 994 svm->vmcb->save.gdtr.limit = dt->size;
995 svm->vmcb->save.gdtr.base = dt->address ;
6aa8b732
AK		 996}
997
e8467fda
AK		 998static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
999{
1000}
1001
25c4c276 1002static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
399badf3
AK		 1003{
1004}
1005
d225157b
AK		 1006static void update_cr0_intercept(struct vcpu_svm *svm)
1007{
66a562f7 1008 struct vmcb *vmcb = svm->vmcb;
d225157b
AK		 1009 ulong gcr0 = svm->vcpu.arch.cr0;
1010 u64 *hcr0 = &svm->vmcb->save.cr0;
1011
1012 if (!svm->vcpu.fpu_active)
1013 *hcr0 |= SVM_CR0_SELECTIVE_MASK;
1014 else
1015 *hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK)
1016 | (gcr0 & SVM_CR0_SELECTIVE_MASK);
1017
1018
1019 if (gcr0 == *hcr0 && svm->vcpu.fpu_active) {
66a562f7
JR		 1020 vmcb->control.intercept_cr_read &= ~INTERCEPT_CR0_MASK;
1021 vmcb->control.intercept_cr_write &= ~INTERCEPT_CR0_MASK;
1022 if (is_nested(svm)) {
1023 struct vmcb *hsave = svm->nested.hsave;
1024
1025 hsave->control.intercept_cr_read &= ~INTERCEPT_CR0_MASK;
1026 hsave->control.intercept_cr_write &= ~INTERCEPT_CR0_MASK;
1027 vmcb->control.intercept_cr_read |= svm->nested.intercept_cr_read;
1028 vmcb->control.intercept_cr_write |= svm->nested.intercept_cr_write;
1029 }
d225157b
AK		 1030 } else {
1031 svm->vmcb->control.intercept_cr_read |= INTERCEPT_CR0_MASK;
1032 svm->vmcb->control.intercept_cr_write |= INTERCEPT_CR0_MASK;
66a562f7
JR		 1033 if (is_nested(svm)) {
1034 struct vmcb *hsave = svm->nested.hsave;
1035
1036 hsave->control.intercept_cr_read |= INTERCEPT_CR0_MASK;
1037 hsave->control.intercept_cr_write |= INTERCEPT_CR0_MASK;
1038 }
d225157b
AK		 1039 }
1040}
1041
6aa8b732
AK		 1042static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1043{
a2fa3e9f
GH		 1044 struct vcpu_svm *svm = to_svm(vcpu);
1045
7f5d8b56
JR		 1046 if (is_nested(svm)) {
1047 /*
1048 * We are here because we run in nested mode, the host kvm
1049 * intercepts cr0 writes but the l1 hypervisor does not.
1050 * But the L1 hypervisor may intercept selective cr0 writes.
1051 * This needs to be checked here.
1052 */
1053 unsigned long old, new;
1054
1055 /* Remove bits that would trigger a real cr0 write intercept */
1056 old = vcpu->arch.cr0 & SVM_CR0_SELECTIVE_MASK;
1057 new = cr0 & SVM_CR0_SELECTIVE_MASK;
1058
1059 if (old == new) {
1060 /* cr0 write with ts and mp unchanged */
1061 svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
1062 if (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE)
1063 return;
1064 }
1065 }
1066
05b3e0c2 1067#ifdef CONFIG_X86_64
f6801dff 1068 if (vcpu->arch.efer & EFER_LME) {
707d92fa 1069 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
f6801dff 1070 vcpu->arch.efer |= EFER_LMA;
2b5203ee 1071 svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
6aa8b732
AK		 1072 }
1073
d77c26fc 1074 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
f6801dff 1075 vcpu->arch.efer &= ~EFER_LMA;
2b5203ee 1076 svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
6aa8b732
AK		 1077 }
1078 }
1079#endif
ad312c7c 1080 vcpu->arch.cr0 = cr0;
888f9f3e
AK		 1081
1082 if (!npt_enabled)
1083 cr0 |= X86_CR0_PG | X86_CR0_WP;
02daab21
AK		 1084
1085 if (!vcpu->fpu_active)
334df50a 1086 cr0 |= X86_CR0_TS;
709ddebf
JR		 1087 /*
1088 * re-enable caching here because the QEMU bios
1089 * does not do it - this results in some delay at
1090 * reboot
1091 */
1092 cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
a2fa3e9f 1093 svm->vmcb->save.cr0 = cr0;
d225157b 1094 update_cr0_intercept(svm);
6aa8b732
AK		 1095}
1096
1097static void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1098{
6394b649 1099 unsigned long host_cr4_mce = read_cr4() & X86_CR4_MCE;
e5eab0ce
JR		 1100 unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;
1101
1102 if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
1103 force_new_asid(vcpu);
6394b649 1104
ec077263
JR		 1105 vcpu->arch.cr4 = cr4;
1106 if (!npt_enabled)
1107 cr4 |= X86_CR4_PAE;
6394b649 1108 cr4 |= host_cr4_mce;
ec077263 1109 to_svm(vcpu)->vmcb->save.cr4 = cr4;
6aa8b732
AK		 1110}
1111
1112static void svm_set_segment(struct kvm_vcpu *vcpu,
1113 struct kvm_segment *var, int seg)
1114{
a2fa3e9f 1115 struct vcpu_svm *svm = to_svm(vcpu);
6aa8b732
AK		 1116 struct vmcb_seg *s = svm_seg(vcpu, seg);
1117
1118 s->base = var->base;
1119 s->limit = var->limit;
1120 s->selector = var->selector;
1121 if (var->unusable)
1122 s->attrib = 0;
1123 else {
1124 s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
1125 s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
1126 s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
1127 s->attrib |= (var->present & 1) << SVM_SELECTOR_P_SHIFT;
1128 s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
1129 s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
1130 s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
1131 s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
1132 }
1133 if (seg == VCPU_SREG_CS)
a2fa3e9f
GH		 1134 svm->vmcb->save.cpl
1135 = (svm->vmcb->save.cs.attrib
6aa8b732
AK		 1136 >> SVM_SELECTOR_DPL_SHIFT) & 3;
1137
1138}
1139
44c11430 1140static void update_db_intercept(struct kvm_vcpu *vcpu)
6aa8b732 1141{
d0bfb940
JK		 1142 struct vcpu_svm *svm = to_svm(vcpu);
1143
d0bfb940
JK		 1144 svm->vmcb->control.intercept_exceptions &=
1145 ~((1 << DB_VECTOR) | (1 << BP_VECTOR));
44c11430 1146
6be7d306 1147 if (svm->nmi_singlestep)
44c11430
GN		 1148 svm->vmcb->control.intercept_exceptions |= (1 << DB_VECTOR);
1149
d0bfb940
JK		 1150 if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
1151 if (vcpu->guest_debug &
1152 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
1153 svm->vmcb->control.intercept_exceptions |=
1154 1 << DB_VECTOR;
1155 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
1156 svm->vmcb->control.intercept_exceptions |=
1157 1 << BP_VECTOR;
1158 } else
1159 vcpu->guest_debug = 0;
44c11430
GN		 1160}
1161
355be0b9 1162static void svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg)
44c11430 1163{
44c11430
GN		 1164 struct vcpu_svm *svm = to_svm(vcpu);
1165
ae675ef0
JK		 1166 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
1167 svm->vmcb->save.dr7 = dbg->arch.debugreg[7];
1168 else
1169 svm->vmcb->save.dr7 = vcpu->arch.dr7;
1170
355be0b9 1171 update_db_intercept(vcpu);
6aa8b732
AK		 1172}
1173
1174static void load_host_msrs(struct kvm_vcpu *vcpu)
1175{
94dfbdb3 1176#ifdef CONFIG_X86_64
a2fa3e9f 1177 wrmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base);
94dfbdb3 1178#endif
6aa8b732
AK		 1179}
1180
1181static void save_host_msrs(struct kvm_vcpu *vcpu)
1182{
94dfbdb3 1183#ifdef CONFIG_X86_64
a2fa3e9f 1184 rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base);
94dfbdb3 1185#endif
6aa8b732
AK		 1186}
1187
0fe1e009 1188static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
6aa8b732 1189{
0fe1e009
TH		 1190 if (sd->next_asid > sd->max_asid) {
1191 ++sd->asid_generation;
1192 sd->next_asid = 1;
a2fa3e9f 1193 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
6aa8b732
AK		 1194 }
1195
0fe1e009
TH		 1196 svm->asid_generation = sd->asid_generation;
1197 svm->vmcb->control.asid = sd->next_asid++;
6aa8b732
AK		 1198}
1199
c76de350 1200static int svm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *dest)
6aa8b732 1201{
42dbaa5a 1202 struct vcpu_svm *svm = to_svm(vcpu);
42dbaa5a
JK		 1203
1204 switch (dr) {
1205 case 0 ... 3:
c76de350 1206 *dest = vcpu->arch.db[dr];
42dbaa5a 1207 break;
c76de350
JK		 1208 case 4:
1209 if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
1210 return EMULATE_FAIL; /* will re-inject UD */
1211 /* fall through */
42dbaa5a
JK		 1212 case 6:
1213 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
c76de350 1214 *dest = vcpu->arch.dr6;
42dbaa5a 1215 else
c76de350 1216 *dest = svm->vmcb->save.dr6;
42dbaa5a 1217 break;
c76de350
JK		 1218 case 5:
1219 if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
1220 return EMULATE_FAIL; /* will re-inject UD */
1221 /* fall through */
42dbaa5a
JK		 1222 case 7:
1223 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
c76de350 1224 *dest = vcpu->arch.dr7;
42dbaa5a 1225 else
c76de350 1226 *dest = svm->vmcb->save.dr7;
42dbaa5a 1227 break;
42dbaa5a
JK		 1228 }
1229
c76de350 1230 return EMULATE_DONE;
6aa8b732
AK		 1231}
1232
c76de350 1233static int svm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long value)
6aa8b732 1234{
a2fa3e9f
GH		 1235 struct vcpu_svm *svm = to_svm(vcpu);
1236
6aa8b732
AK		 1237 switch (dr) {
1238 case 0 ... 3:
42dbaa5a
JK		 1239 vcpu->arch.db[dr] = value;
1240 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
1241 vcpu->arch.eff_db[dr] = value;
c76de350
JK		 1242 break;
1243 case 4:
1244 if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
1245 return EMULATE_FAIL; /* will re-inject UD */
1246 /* fall through */
42dbaa5a 1247 case 6:
42dbaa5a 1248 vcpu->arch.dr6 = (value & DR6_VOLATILE) | DR6_FIXED_1;
c76de350
JK		 1249 break;
1250 case 5:
1251 if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
1252 return EMULATE_FAIL; /* will re-inject UD */
1253 /* fall through */
42dbaa5a 1254 case 7:
42dbaa5a
JK		 1255 vcpu->arch.dr7 = (value & DR7_VOLATILE) | DR7_FIXED_1;
1256 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
1257 svm->vmcb->save.dr7 = vcpu->arch.dr7;
1258 vcpu->arch.switch_db_regs = (value & DR7_BP_EN_MASK);
1259 }
c76de350 1260 break;
6aa8b732 1261 }
c76de350
JK		 1262
1263 return EMULATE_DONE;
6aa8b732
AK		 1264}
1265
851ba692 1266static int pf_interception(struct vcpu_svm *svm)
6aa8b732 1267{
6aa8b732
AK		 1268 u64 fault_address;
1269 u32 error_code;
6aa8b732 1270
a2fa3e9f
GH		 1271 fault_address = svm->vmcb->control.exit_info_2;
1272 error_code = svm->vmcb->control.exit_info_1;
af9ca2d7 1273
229456fc 1274 trace_kvm_page_fault(fault_address, error_code);
52c7847d
AK		 1275 if (!npt_enabled && kvm_event_needs_reinjection(&svm->vcpu))
1276 kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address);
3067714c 1277 return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code);
6aa8b732
AK		 1278}
1279
851ba692 1280static int db_interception(struct vcpu_svm *svm)
d0bfb940 1281{
851ba692
AK		 1282 struct kvm_run *kvm_run = svm->vcpu.run;
1283
d0bfb940 1284 if (!(svm->vcpu.guest_debug &
44c11430 1285 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
6be7d306 1286 !svm->nmi_singlestep) {
d0bfb940
JK		 1287 kvm_queue_exception(&svm->vcpu, DB_VECTOR);
1288 return 1;
1289 }
44c11430 1290
6be7d306
JK		 1291 if (svm->nmi_singlestep) {
1292 svm->nmi_singlestep = false;
44c11430
GN		 1293 if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP))
1294 svm->vmcb->save.rflags &=
1295 ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
1296 update_db_intercept(&svm->vcpu);
1297 }
1298
1299 if (svm->vcpu.guest_debug &
e0231715 1300 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
44c11430
GN		 1301 kvm_run->exit_reason = KVM_EXIT_DEBUG;
1302 kvm_run->debug.arch.pc =
1303 svm->vmcb->save.cs.base + svm->vmcb->save.rip;
1304 kvm_run->debug.arch.exception = DB_VECTOR;
1305 return 0;
1306 }
1307
1308 return 1;
d0bfb940
JK		 1309}
1310
851ba692 1311static int bp_interception(struct vcpu_svm *svm)
d0bfb940 1312{
851ba692
AK		 1313 struct kvm_run *kvm_run = svm->vcpu.run;
1314
d0bfb940
JK		 1315 kvm_run->exit_reason = KVM_EXIT_DEBUG;
1316 kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
1317 kvm_run->debug.arch.exception = BP_VECTOR;
1318 return 0;
1319}
1320
851ba692 1321static int ud_interception(struct vcpu_svm *svm)
7aa81cc0
AL		 1322{
1323 int er;
1324
851ba692 1325 er = emulate_instruction(&svm->vcpu, 0, 0, EMULTYPE_TRAP_UD);
7aa81cc0 1326 if (er != EMULATE_DONE)
7ee5d940 1327 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
7aa81cc0
AL		 1328 return 1;
1329}
1330
6b52d186 1331static void svm_fpu_activate(struct kvm_vcpu *vcpu)
7807fa6c 1332{
6b52d186 1333 struct vcpu_svm *svm = to_svm(vcpu);
66a562f7
JR		 1334 u32 excp;
1335
1336 if (is_nested(svm)) {
1337 u32 h_excp, n_excp;
1338
1339 h_excp = svm->nested.hsave->control.intercept_exceptions;
1340 n_excp = svm->nested.intercept_exceptions;
1341 h_excp &= ~(1 << NM_VECTOR);
1342 excp = h_excp | n_excp;
1343 } else {
1344 excp = svm->vmcb->control.intercept_exceptions;
e0231715 1345 excp &= ~(1 << NM_VECTOR);
66a562f7
JR		 1346 }
1347
1348 svm->vmcb->control.intercept_exceptions = excp;
1349
e756fc62 1350 svm->vcpu.fpu_active = 1;
d225157b 1351 update_cr0_intercept(svm);
6b52d186 1352}
a2fa3e9f 1353
6b52d186
AK		 1354static int nm_interception(struct vcpu_svm *svm)
1355{
1356 svm_fpu_activate(&svm->vcpu);
a2fa3e9f 1357 return 1;
7807fa6c
AL		 1358}
1359
851ba692 1360static int mc_interception(struct vcpu_svm *svm)
53371b50
JR		 1361{
1362 /*
1363 * On an #MC intercept the MCE handler is not called automatically in
1364 * the host. So do it by hand here.
1365 */
1366 asm volatile (
1367 "int $0x12\n");
1368 /* not sure if we ever come back to this point */
1369
1370 return 1;
1371}
1372
851ba692 1373static int shutdown_interception(struct vcpu_svm *svm)
46fe4ddd 1374{
851ba692
AK		 1375 struct kvm_run *kvm_run = svm->vcpu.run;
1376
46fe4ddd
JR		 1377 /*
1378 * VMCB is undefined after a SHUTDOWN intercept
1379 * so reinitialize it.
1380 */
a2fa3e9f 1381 clear_page(svm->vmcb);
e6101a96 1382 init_vmcb(svm);
46fe4ddd
JR		 1383
1384 kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
1385 return 0;
1386}
1387
851ba692 1388static int io_interception(struct vcpu_svm *svm)
6aa8b732 1389{
d77c26fc 1390 u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
34c33d16 1391 int size, in, string;
039576c0 1392 unsigned port;
6aa8b732 1393
e756fc62 1394 ++svm->vcpu.stat.io_exits;
6aa8b732 1395
a2fa3e9f 1396 svm->next_rip = svm->vmcb->control.exit_info_2;
6aa8b732 1397
e70669ab
LV		 1398 string = (io_info & SVM_IOIO_STR_MASK) != 0;
1399
1400 if (string) {
3427318f 1401 if (emulate_instruction(&svm->vcpu,
851ba692 1402 0, 0, 0) == EMULATE_DO_MMIO)
e70669ab
LV		 1403 return 0;
1404 return 1;
1405 }
1406
039576c0
AK		 1407 in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
1408 port = io_info >> 16;
1409 size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
6aa8b732 1410
e93f36bc 1411 skip_emulated_instruction(&svm->vcpu);
851ba692 1412 return kvm_emulate_pio(&svm->vcpu, in, size, port);
6aa8b732
AK		 1413}
1414
851ba692 1415static int nmi_interception(struct vcpu_svm *svm)
c47f098d
JR		 1416{
1417 return 1;
1418}
1419
851ba692 1420static int intr_interception(struct vcpu_svm *svm)
a0698055
JR		 1421{
1422 ++svm->vcpu.stat.irq_exits;
1423 return 1;
1424}
1425
851ba692 1426static int nop_on_interception(struct vcpu_svm *svm)
6aa8b732
AK		 1427{
1428 return 1;
1429}
1430
851ba692 1431static int halt_interception(struct vcpu_svm *svm)
6aa8b732 1432{
5fdbf976 1433 svm->next_rip = kvm_rip_read(&svm->vcpu) + 1;
e756fc62
RR		 1434 skip_emulated_instruction(&svm->vcpu);
1435 return kvm_emulate_halt(&svm->vcpu);
6aa8b732
AK		 1436}
1437
851ba692 1438static int vmmcall_interception(struct vcpu_svm *svm)
02e235bc 1439{
5fdbf976 1440 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
e756fc62 1441 skip_emulated_instruction(&svm->vcpu);
7aa81cc0
AL		 1442 kvm_emulate_hypercall(&svm->vcpu);
1443 return 1;
02e235bc
AK		 1444}
1445
c0725420
AG		 1446static int nested_svm_check_permissions(struct vcpu_svm *svm)
1447{
f6801dff 1448 if (!(svm->vcpu.arch.efer & EFER_SVME)
c0725420
AG		 1449 || !is_paging(&svm->vcpu)) {
1450 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
1451 return 1;
1452 }
1453
1454 if (svm->vmcb->save.cpl) {
1455 kvm_inject_gp(&svm->vcpu, 0);
1456 return 1;
1457 }
1458
1459 return 0;
1460}
1461
cf74a78b
AG		 1462static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
1463 bool has_error_code, u32 error_code)
1464{
b8e88bc8
JR		 1465 int vmexit;
1466
0295ad7d
JR		 1467 if (!is_nested(svm))
1468 return 0;
cf74a78b 1469
0295ad7d
JR		 1470 svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
1471 svm->vmcb->control.exit_code_hi = 0;
1472 svm->vmcb->control.exit_info_1 = error_code;
1473 svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;
1474
b8e88bc8
JR		 1475 vmexit = nested_svm_intercept(svm);
1476 if (vmexit == NESTED_EXIT_DONE)
1477 svm->nested.exit_required = true;
1478
1479 return vmexit;
cf74a78b
AG		 1480}
1481
8fe54654
JR		 1482/* This function returns true if it is save to enable the irq window */
1483static inline bool nested_svm_intr(struct vcpu_svm *svm)
cf74a78b 1484{
26666957 1485 if (!is_nested(svm))
8fe54654 1486 return true;
cf74a78b 1487
26666957 1488 if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
8fe54654 1489 return true;
cf74a78b 1490
26666957 1491 if (!(svm->vcpu.arch.hflags & HF_HIF_MASK))
8fe54654 1492 return false;
cf74a78b 1493
197717d5
JR		 1494 svm->vmcb->control.exit_code = SVM_EXIT_INTR;
1495 svm->vmcb->control.exit_info_1 = 0;
1496 svm->vmcb->control.exit_info_2 = 0;
26666957 1497
cd3ff653
JR		 1498 if (svm->nested.intercept & 1ULL) {
1499 /*
1500 * The #vmexit can't be emulated here directly because this
1501 * code path runs with irqs and preemtion disabled. A
1502 * #vmexit emulation might sleep. Only signal request for
1503 * the #vmexit here.
1504 */
1505 svm->nested.exit_required = true;
236649de 1506 trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip);
8fe54654 1507 return false;
cf74a78b
AG		 1508 }
1509
8fe54654 1510 return true;
cf74a78b
AG		 1511}
1512
887f500c
JR		 1513/* This function returns true if it is save to enable the nmi window */
1514static inline bool nested_svm_nmi(struct vcpu_svm *svm)
1515{
1516 if (!is_nested(svm))
1517 return true;
1518
1519 if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI)))
1520 return true;
1521
1522 svm->vmcb->control.exit_code = SVM_EXIT_NMI;
1523 svm->nested.exit_required = true;
1524
1525 return false;
1526}
1527
7597f129 1528static void *nested_svm_map(struct vcpu_svm *svm, u64 gpa, struct page **_page)
34f80cfa
JR		 1529{
1530 struct page *page;
1531
6c3bd3d7
JR		 1532 might_sleep();
1533
34f80cfa 1534 page = gfn_to_page(svm->vcpu.kvm, gpa >> PAGE_SHIFT);
34f80cfa
JR		 1535 if (is_error_page(page))
1536 goto error;
1537
7597f129
JR		 1538 *_page = page;
1539
1540 return kmap(page);
34f80cfa
JR		 1541
1542error:
1543 kvm_release_page_clean(page);
1544 kvm_inject_gp(&svm->vcpu, 0);
1545
1546 return NULL;
1547}
1548
7597f129 1549static void nested_svm_unmap(struct page *page)
34f80cfa 1550{
7597f129 1551 kunmap(page);
34f80cfa
JR		 1552 kvm_release_page_dirty(page);
1553}
1554
3d62d9aa 1555static bool nested_svm_exit_handled_msr(struct vcpu_svm *svm)
4c2161ae 1556{
4c2161ae 1557 u32 param = svm->vmcb->control.exit_info_1 & 1;
3d62d9aa
JR		 1558 u32 msr = svm->vcpu.arch.regs[VCPU_REGS_RCX];
1559 bool ret = false;
1560 u32 t0, t1;
4c7da8cb 1561 u8 val;
4c2161ae 1562
3d62d9aa
JR		 1563 if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
1564 return false;
1565
4c2161ae
JR		 1566 switch (msr) {
1567 case 0 ... 0x1fff:
1568 t0 = (msr * 2) % 8;
1569 t1 = msr / 8;
1570 break;
1571 case 0xc0000000 ... 0xc0001fff:
1572 t0 = (8192 + msr - 0xc0000000) * 2;
1573 t1 = (t0 / 8);
1574 t0 %= 8;
1575 break;
1576 case 0xc0010000 ... 0xc0011fff:
1577 t0 = (16384 + msr - 0xc0010000) * 2;
1578 t1 = (t0 / 8);
1579 t0 %= 8;
1580 break;
1581 default:
3d62d9aa
JR		 1582 ret = true;
1583 goto out;
4c2161ae 1584 }
4c2161ae 1585
4c7da8cb
JR		 1586 if (!kvm_read_guest(svm->vcpu.kvm, svm->nested.vmcb_msrpm + t1, &val, 1))
1587 ret = val & ((1 << param) << t0);
3d62d9aa
JR		 1588
1589out:
3d62d9aa 1590 return ret;
4c2161ae
JR		 1591}
1592
410e4d57 1593static int nested_svm_exit_special(struct vcpu_svm *svm)
cf74a78b 1594{
cf74a78b 1595 u32 exit_code = svm->vmcb->control.exit_code;
4c2161ae 1596
410e4d57
JR		 1597 switch (exit_code) {
1598 case SVM_EXIT_INTR:
1599 case SVM_EXIT_NMI:
1600 return NESTED_EXIT_HOST;
410e4d57 1601 case SVM_EXIT_NPF:
e0231715 1602 /* For now we are always handling NPFs when using them */
410e4d57
JR		 1603 if (npt_enabled)
1604 return NESTED_EXIT_HOST;
1605 break;
410e4d57 1606 case SVM_EXIT_EXCP_BASE + PF_VECTOR:
e0231715 1607 /* When we're shadowing, trap PFs */
410e4d57
JR		 1608 if (!npt_enabled)
1609 return NESTED_EXIT_HOST;
1610 break;
66a562f7
JR		 1611 case SVM_EXIT_EXCP_BASE + NM_VECTOR:
1612 nm_interception(svm);
1613 break;
410e4d57
JR		 1614 default:
1615 break;
cf74a78b
AG		 1616 }
1617
410e4d57
JR		 1618 return NESTED_EXIT_CONTINUE;
1619}
1620
1621/*
1622 * If this function returns true, this #vmexit was already handled
1623 */
b8e88bc8 1624static int nested_svm_intercept(struct vcpu_svm *svm)
410e4d57
JR		 1625{
1626 u32 exit_code = svm->vmcb->control.exit_code;
1627 int vmexit = NESTED_EXIT_HOST;
1628
cf74a78b 1629 switch (exit_code) {
9c4e40b9 1630 case SVM_EXIT_MSR:
3d62d9aa 1631 vmexit = nested_svm_exit_handled_msr(svm);
9c4e40b9 1632 break;
cf74a78b
AG		 1633 case SVM_EXIT_READ_CR0 ... SVM_EXIT_READ_CR8: {
1634 u32 cr_bits = 1 << (exit_code - SVM_EXIT_READ_CR0);
aad42c64 1635 if (svm->nested.intercept_cr_read & cr_bits)
410e4d57 1636 vmexit = NESTED_EXIT_DONE;
cf74a78b
AG		 1637 break;
1638 }
1639 case SVM_EXIT_WRITE_CR0 ... SVM_EXIT_WRITE_CR8: {
1640 u32 cr_bits = 1 << (exit_code - SVM_EXIT_WRITE_CR0);
aad42c64 1641 if (svm->nested.intercept_cr_write & cr_bits)
410e4d57 1642 vmexit = NESTED_EXIT_DONE;
cf74a78b
AG		 1643 break;
1644 }
1645 case SVM_EXIT_READ_DR0 ... SVM_EXIT_READ_DR7: {
1646 u32 dr_bits = 1 << (exit_code - SVM_EXIT_READ_DR0);
aad42c64 1647 if (svm->nested.intercept_dr_read & dr_bits)
410e4d57 1648 vmexit = NESTED_EXIT_DONE;
cf74a78b
AG		 1649 break;
1650 }
1651 case SVM_EXIT_WRITE_DR0 ... SVM_EXIT_WRITE_DR7: {
1652 u32 dr_bits = 1 << (exit_code - SVM_EXIT_WRITE_DR0);
aad42c64 1653 if (svm->nested.intercept_dr_write & dr_bits)
410e4d57 1654 vmexit = NESTED_EXIT_DONE;
cf74a78b
AG		 1655 break;
1656 }
1657 case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
1658 u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
aad42c64 1659 if (svm->nested.intercept_exceptions & excp_bits)
410e4d57 1660 vmexit = NESTED_EXIT_DONE;
cf74a78b
AG		 1661 break;
1662 }
1663 default: {
1664 u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR);
aad42c64 1665 if (svm->nested.intercept & exit_bits)
410e4d57 1666 vmexit = NESTED_EXIT_DONE;
cf74a78b
AG		 1667 }
1668 }
1669
b8e88bc8
JR		 1670 return vmexit;
1671}
1672
1673static int nested_svm_exit_handled(struct vcpu_svm *svm)
1674{
1675 int vmexit;
1676
1677 vmexit = nested_svm_intercept(svm);
1678
1679 if (vmexit == NESTED_EXIT_DONE)
9c4e40b9 1680 nested_svm_vmexit(svm);
9c4e40b9
JR		 1681
1682 return vmexit;
cf74a78b
AG		 1683}
1684
0460a979
JR		 1685static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *from_vmcb)
1686{
1687 struct vmcb_control_area *dst = &dst_vmcb->control;
1688 struct vmcb_control_area *from = &from_vmcb->control;
1689
1690 dst->intercept_cr_read = from->intercept_cr_read;
1691 dst->intercept_cr_write = from->intercept_cr_write;
1692 dst->intercept_dr_read = from->intercept_dr_read;
1693 dst->intercept_dr_write = from->intercept_dr_write;
1694 dst->intercept_exceptions = from->intercept_exceptions;
1695 dst->intercept = from->intercept;
1696 dst->iopm_base_pa = from->iopm_base_pa;
1697 dst->msrpm_base_pa = from->msrpm_base_pa;
1698 dst->tsc_offset = from->tsc_offset;
1699 dst->asid = from->asid;
1700 dst->tlb_ctl = from->tlb_ctl;
1701 dst->int_ctl = from->int_ctl;
1702 dst->int_vector = from->int_vector;
1703 dst->int_state = from->int_state;
1704 dst->exit_code = from->exit_code;
1705 dst->exit_code_hi = from->exit_code_hi;
1706 dst->exit_info_1 = from->exit_info_1;
1707 dst->exit_info_2 = from->exit_info_2;
1708 dst->exit_int_info = from->exit_int_info;
1709 dst->exit_int_info_err = from->exit_int_info_err;
1710 dst->nested_ctl = from->nested_ctl;
1711 dst->event_inj = from->event_inj;
1712 dst->event_inj_err = from->event_inj_err;
1713 dst->nested_cr3 = from->nested_cr3;
1714 dst->lbr_ctl = from->lbr_ctl;
1715}
1716
34f80cfa 1717static int nested_svm_vmexit(struct vcpu_svm *svm)
cf74a78b 1718{
34f80cfa 1719 struct vmcb *nested_vmcb;
e6aa9abd 1720 struct vmcb *hsave = svm->nested.hsave;
33740e40 1721 struct vmcb *vmcb = svm->vmcb;
7597f129 1722 struct page *page;
cf74a78b 1723
17897f36
JR		 1724 trace_kvm_nested_vmexit_inject(vmcb->control.exit_code,
1725 vmcb->control.exit_info_1,
1726 vmcb->control.exit_info_2,
1727 vmcb->control.exit_int_info,
1728 vmcb->control.exit_int_info_err);
1729
7597f129 1730 nested_vmcb = nested_svm_map(svm, svm->nested.vmcb, &page);
34f80cfa
JR		 1731 if (!nested_vmcb)
1732 return 1;
1733
06fc7772
JR		 1734 /* Exit nested SVM mode */
1735 svm->nested.vmcb = 0;
1736
cf74a78b 1737 /* Give the current vmcb to the guest */
33740e40
JR		 1738 disable_gif(svm);
1739
1740 nested_vmcb->save.es = vmcb->save.es;
1741 nested_vmcb->save.cs = vmcb->save.cs;
1742 nested_vmcb->save.ss = vmcb->save.ss;
1743 nested_vmcb->save.ds = vmcb->save.ds;
1744 nested_vmcb->save.gdtr = vmcb->save.gdtr;
1745 nested_vmcb->save.idtr = vmcb->save.idtr;
cdbbdc12 1746 nested_vmcb->save.cr0 = kvm_read_cr0(&svm->vcpu);
33740e40
JR		 1747 if (npt_enabled)
1748 nested_vmcb->save.cr3 = vmcb->save.cr3;
cdbbdc12
JR		 1749 else
1750 nested_vmcb->save.cr3 = svm->vcpu.arch.cr3;
33740e40 1751 nested_vmcb->save.cr2 = vmcb->save.cr2;
cdbbdc12 1752 nested_vmcb->save.cr4 = svm->vcpu.arch.cr4;
33740e40
JR		 1753 nested_vmcb->save.rflags = vmcb->save.rflags;
1754 nested_vmcb->save.rip = vmcb->save.rip;
1755 nested_vmcb->save.rsp = vmcb->save.rsp;
1756 nested_vmcb->save.rax = vmcb->save.rax;
1757 nested_vmcb->save.dr7 = vmcb->save.dr7;
1758 nested_vmcb->save.dr6 = vmcb->save.dr6;
1759 nested_vmcb->save.cpl = vmcb->save.cpl;
1760
1761 nested_vmcb->control.int_ctl = vmcb->control.int_ctl;
1762 nested_vmcb->control.int_vector = vmcb->control.int_vector;
1763 nested_vmcb->control.int_state = vmcb->control.int_state;
1764 nested_vmcb->control.exit_code = vmcb->control.exit_code;
1765 nested_vmcb->control.exit_code_hi = vmcb->control.exit_code_hi;
1766 nested_vmcb->control.exit_info_1 = vmcb->control.exit_info_1;
1767 nested_vmcb->control.exit_info_2 = vmcb->control.exit_info_2;
1768 nested_vmcb->control.exit_int_info = vmcb->control.exit_int_info;
1769 nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err;
8d23c466
AG		 1770
1771 /*
1772 * If we emulate a VMRUN/#VMEXIT in the same host #vmexit cycle we have
1773 * to make sure that we do not lose injected events. So check event_inj
1774 * here and copy it to exit_int_info if it is valid.
1775 * Exit_int_info and event_inj can't be both valid because the case
1776 * below only happens on a VMRUN instruction intercept which has
1777 * no valid exit_int_info set.
1778 */
1779 if (vmcb->control.event_inj & SVM_EVTINJ_VALID) {
1780 struct vmcb_control_area *nc = &nested_vmcb->control;
1781
1782 nc->exit_int_info = vmcb->control.event_inj;
1783 nc->exit_int_info_err = vmcb->control.event_inj_err;
1784 }
1785
33740e40
JR		 1786 nested_vmcb->control.tlb_ctl = 0;
1787 nested_vmcb->control.event_inj = 0;
1788 nested_vmcb->control.event_inj_err = 0;
cf74a78b
AG		 1789
1790 /* We always set V_INTR_MASKING and remember the old value in hflags */
1791 if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
1792 nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;
1793
cf74a78b 1794 /* Restore the original control entries */
0460a979 1795 copy_vmcb_control_area(vmcb, hsave);
cf74a78b 1796
219b65dc
AG		 1797 kvm_clear_exception_queue(&svm->vcpu);
1798 kvm_clear_interrupt_queue(&svm->vcpu);
cf74a78b
AG		 1799
1800 /* Restore selected save entries */
1801 svm->vmcb->save.es = hsave->save.es;
1802 svm->vmcb->save.cs = hsave->save.cs;
1803 svm->vmcb->save.ss = hsave->save.ss;
1804 svm->vmcb->save.ds = hsave->save.ds;
1805 svm->vmcb->save.gdtr = hsave->save.gdtr;
1806 svm->vmcb->save.idtr = hsave->save.idtr;
1807 svm->vmcb->save.rflags = hsave->save.rflags;
1808 svm_set_efer(&svm->vcpu, hsave->save.efer);
1809 svm_set_cr0(&svm->vcpu, hsave->save.cr0 | X86_CR0_PE);
1810 svm_set_cr4(&svm->vcpu, hsave->save.cr4);
1811 if (npt_enabled) {
1812 svm->vmcb->save.cr3 = hsave->save.cr3;
1813 svm->vcpu.arch.cr3 = hsave->save.cr3;
1814 } else {
1815 kvm_set_cr3(&svm->vcpu, hsave->save.cr3);
1816 }
1817 kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, hsave->save.rax);
1818 kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, hsave->save.rsp);
1819 kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, hsave->save.rip);
1820 svm->vmcb->save.dr7 = 0;
1821 svm->vmcb->save.cpl = 0;
1822 svm->vmcb->control.exit_int_info = 0;
1823
7597f129 1824 nested_svm_unmap(page);
cf74a78b
AG		 1825
1826 kvm_mmu_reset_context(&svm->vcpu);
1827 kvm_mmu_load(&svm->vcpu);
1828
1829 return 0;
1830}
3d6368ef 1831
9738b2c9 1832static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)
3d6368ef 1833{
9738b2c9 1834 u32 *nested_msrpm;
7597f129 1835 struct page *page;
3d6368ef 1836 int i;
9738b2c9 1837
7597f129 1838 nested_msrpm = nested_svm_map(svm, svm->nested.vmcb_msrpm, &page);
9738b2c9
JR		 1839 if (!nested_msrpm)
1840 return false;
1841
e0231715 1842 for (i = 0; i < PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER) / 4; i++)
e6aa9abd 1843 svm->nested.msrpm[i] = svm->msrpm[i] | nested_msrpm[i];
9738b2c9 1844
e6aa9abd 1845 svm->vmcb->control.msrpm_base_pa = __pa(svm->nested.msrpm);
3d6368ef 1846
7597f129 1847 nested_svm_unmap(page);
9738b2c9
JR		 1848
1849 return true;
3d6368ef
AG		 1850}
1851
9738b2c9 1852static bool nested_svm_vmrun(struct vcpu_svm *svm)
3d6368ef 1853{
9738b2c9 1854 struct vmcb *nested_vmcb;
e6aa9abd 1855 struct vmcb *hsave = svm->nested.hsave;
defbba56 1856 struct vmcb *vmcb = svm->vmcb;
7597f129 1857 struct page *page;
06fc7772
JR		 1858 u64 vmcb_gpa;
1859
1860 vmcb_gpa = svm->vmcb->save.rax;
3d6368ef 1861
7597f129 1862 nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
9738b2c9
JR		 1863 if (!nested_vmcb)
1864 return false;
1865
ecf1405d 1866 trace_kvm_nested_vmrun(svm->vmcb->save.rip - 3, vmcb_gpa,
0ac406de
JR		 1867 nested_vmcb->save.rip,
1868 nested_vmcb->control.int_ctl,
1869 nested_vmcb->control.event_inj,
1870 nested_vmcb->control.nested_ctl);
1871
2e554e8d
JR		 1872 trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr_read,
1873 nested_vmcb->control.intercept_cr_write,
1874 nested_vmcb->control.intercept_exceptions,
1875 nested_vmcb->control.intercept);
1876
3d6368ef 1877 /* Clear internal status */
219b65dc
AG		 1878 kvm_clear_exception_queue(&svm->vcpu);
1879 kvm_clear_interrupt_queue(&svm->vcpu);
3d6368ef 1880
e0231715
JR		 1881 /*
1882 * Save the old vmcb, so we don't need to pick what we save, but can
1883 * restore everything when a VMEXIT occurs
1884 */
defbba56
JR		 1885 hsave->save.es = vmcb->save.es;
1886 hsave->save.cs = vmcb->save.cs;
1887 hsave->save.ss = vmcb->save.ss;
1888 hsave->save.ds = vmcb->save.ds;
1889 hsave->save.gdtr = vmcb->save.gdtr;
1890 hsave->save.idtr = vmcb->save.idtr;
f6801dff 1891 hsave->save.efer = svm->vcpu.arch.efer;
4d4ec087 1892 hsave->save.cr0 = kvm_read_cr0(&svm->vcpu);
defbba56
JR		 1893 hsave->save.cr4 = svm->vcpu.arch.cr4;
1894 hsave->save.rflags = vmcb->save.rflags;
1895 hsave->save.rip = svm->next_rip;
1896 hsave->save.rsp = vmcb->save.rsp;
1897 hsave->save.rax = vmcb->save.rax;
1898 if (npt_enabled)
1899 hsave->save.cr3 = vmcb->save.cr3;
1900 else
1901 hsave->save.cr3 = svm->vcpu.arch.cr3;
1902
0460a979 1903 copy_vmcb_control_area(hsave, vmcb);
3d6368ef
AG		 1904
1905 if (svm->vmcb->save.rflags & X86_EFLAGS_IF)
1906 svm->vcpu.arch.hflags |= HF_HIF_MASK;
1907 else
1908 svm->vcpu.arch.hflags &= ~HF_HIF_MASK;
1909
1910 /* Load the nested guest state */
1911 svm->vmcb->save.es = nested_vmcb->save.es;
1912 svm->vmcb->save.cs = nested_vmcb->save.cs;
1913 svm->vmcb->save.ss = nested_vmcb->save.ss;
1914 svm->vmcb->save.ds = nested_vmcb->save.ds;
1915 svm->vmcb->save.gdtr = nested_vmcb->save.gdtr;
1916 svm->vmcb->save.idtr = nested_vmcb->save.idtr;
1917 svm->vmcb->save.rflags = nested_vmcb->save.rflags;
1918 svm_set_efer(&svm->vcpu, nested_vmcb->save.efer);
1919 svm_set_cr0(&svm->vcpu, nested_vmcb->save.cr0);
1920 svm_set_cr4(&svm->vcpu, nested_vmcb->save.cr4);
1921 if (npt_enabled) {
1922 svm->vmcb->save.cr3 = nested_vmcb->save.cr3;
1923 svm->vcpu.arch.cr3 = nested_vmcb->save.cr3;
0e5cbe36 1924 } else
3d6368ef 1925 kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3);
0e5cbe36
JR		 1926
1927 /* Guest paging mode is active - reset mmu */
1928 kvm_mmu_reset_context(&svm->vcpu);
1929
defbba56 1930 svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = nested_vmcb->save.cr2;
3d6368ef
AG		 1931 kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, nested_vmcb->save.rax);
1932 kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, nested_vmcb->save.rsp);
1933 kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, nested_vmcb->save.rip);
e0231715 1934
3d6368ef
AG		 1935 /* In case we don't even reach vcpu_run, the fields are not updated */
1936 svm->vmcb->save.rax = nested_vmcb->save.rax;
1937 svm->vmcb->save.rsp = nested_vmcb->save.rsp;
1938 svm->vmcb->save.rip = nested_vmcb->save.rip;
1939 svm->vmcb->save.dr7 = nested_vmcb->save.dr7;
1940 svm->vmcb->save.dr6 = nested_vmcb->save.dr6;
1941 svm->vmcb->save.cpl = nested_vmcb->save.cpl;
1942
e6aa9abd 1943 svm->nested.vmcb_msrpm = nested_vmcb->control.msrpm_base_pa;
3d6368ef 1944
aad42c64
JR		 1945 /* cache intercepts */
1946 svm->nested.intercept_cr_read = nested_vmcb->control.intercept_cr_read;
1947 svm->nested.intercept_cr_write = nested_vmcb->control.intercept_cr_write;
1948 svm->nested.intercept_dr_read = nested_vmcb->control.intercept_dr_read;
1949 svm->nested.intercept_dr_write = nested_vmcb->control.intercept_dr_write;
1950 svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions;
1951 svm->nested.intercept = nested_vmcb->control.intercept;
1952
3d6368ef 1953 force_new_asid(&svm->vcpu);
3d6368ef 1954 svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK;
3d6368ef
AG		 1955 if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK)
1956 svm->vcpu.arch.hflags |= HF_VINTR_MASK;
1957 else
1958 svm->vcpu.arch.hflags &= ~HF_VINTR_MASK;
1959
88ab24ad
JR		 1960 if (svm->vcpu.arch.hflags & HF_VINTR_MASK) {
1961 /* We only want the cr8 intercept bits of the guest */
1962 svm->vmcb->control.intercept_cr_read &= ~INTERCEPT_CR8_MASK;
1963 svm->vmcb->control.intercept_cr_write &= ~INTERCEPT_CR8_MASK;
1964 }
1965
e0231715
JR		 1966 /*
1967 * We don't want a nested guest to be more powerful than the guest, so
1968 * all intercepts are ORed
1969 */
88ab24ad
JR		 1970 svm->vmcb->control.intercept_cr_read |=
1971 nested_vmcb->control.intercept_cr_read;
1972 svm->vmcb->control.intercept_cr_write |=
1973 nested_vmcb->control.intercept_cr_write;
1974 svm->vmcb->control.intercept_dr_read |=
1975 nested_vmcb->control.intercept_dr_read;
1976 svm->vmcb->control.intercept_dr_write |=
1977 nested_vmcb->control.intercept_dr_write;
1978 svm->vmcb->control.intercept_exceptions |=
1979 nested_vmcb->control.intercept_exceptions;
1980
1981 svm->vmcb->control.intercept |= nested_vmcb->control.intercept;
1982
1983 svm->vmcb->control.lbr_ctl = nested_vmcb->control.lbr_ctl;
3d6368ef
AG		 1984 svm->vmcb->control.int_vector = nested_vmcb->control.int_vector;
1985 svm->vmcb->control.int_state = nested_vmcb->control.int_state;
1986 svm->vmcb->control.tsc_offset += nested_vmcb->control.tsc_offset;
3d6368ef
AG		 1987 svm->vmcb->control.event_inj = nested_vmcb->control.event_inj;
1988 svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err;
1989
7597f129 1990 nested_svm_unmap(page);
9738b2c9 1991
06fc7772
JR		 1992 /* nested_vmcb is our indicator if nested SVM is activated */
1993 svm->nested.vmcb = vmcb_gpa;
1994
2af9194d 1995 enable_gif(svm);
3d6368ef 1996
9738b2c9 1997 return true;
3d6368ef
AG		 1998}
1999
9966bf68 2000static void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
5542675b
AG		 2001{
2002 to_vmcb->save.fs = from_vmcb->save.fs;
2003 to_vmcb->save.gs = from_vmcb->save.gs;
2004 to_vmcb->save.tr = from_vmcb->save.tr;
2005 to_vmcb->save.ldtr = from_vmcb->save.ldtr;
2006 to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base;
2007 to_vmcb->save.star = from_vmcb->save.star;
2008 to_vmcb->save.lstar = from_vmcb->save.lstar;
2009 to_vmcb->save.cstar = from_vmcb->save.cstar;
2010 to_vmcb->save.sfmask = from_vmcb->save.sfmask;
2011 to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
2012 to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
2013 to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
5542675b
AG		 2014}
2015
851ba692 2016static int vmload_interception(struct vcpu_svm *svm)
5542675b 2017{
9966bf68 2018 struct vmcb *nested_vmcb;
7597f129 2019 struct page *page;
9966bf68 2020
5542675b
AG		 2021 if (nested_svm_check_permissions(svm))
2022 return 1;
2023
2024 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2025 skip_emulated_instruction(&svm->vcpu);
2026
7597f129 2027 nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
9966bf68
JR		 2028 if (!nested_vmcb)
2029 return 1;
2030
2031 nested_svm_vmloadsave(nested_vmcb, svm->vmcb);
7597f129 2032 nested_svm_unmap(page);
5542675b
AG		 2033
2034 return 1;
2035}
2036
851ba692 2037static int vmsave_interception(struct vcpu_svm *svm)
5542675b 2038{
9966bf68 2039 struct vmcb *nested_vmcb;
7597f129 2040 struct page *page;
9966bf68 2041
5542675b
AG		 2042 if (nested_svm_check_permissions(svm))
2043 return 1;
2044
2045 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2046 skip_emulated_instruction(&svm->vcpu);
2047
7597f129 2048 nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
9966bf68
JR		 2049 if (!nested_vmcb)
2050 return 1;
2051
2052 nested_svm_vmloadsave(svm->vmcb, nested_vmcb);
7597f129 2053 nested_svm_unmap(page);
5542675b
AG		 2054
2055 return 1;
2056}
2057
851ba692 2058static int vmrun_interception(struct vcpu_svm *svm)
3d6368ef 2059{
3d6368ef
AG		 2060 if (nested_svm_check_permissions(svm))
2061 return 1;
2062
2063 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2064 skip_emulated_instruction(&svm->vcpu);
2065
9738b2c9 2066 if (!nested_svm_vmrun(svm))
3d6368ef
AG		 2067 return 1;
2068
9738b2c9 2069 if (!nested_svm_vmrun_msrpm(svm))
1f8da478
JR		 2070 goto failed;
2071
2072 return 1;
2073
2074failed:
2075
2076 svm->vmcb->control.exit_code = SVM_EXIT_ERR;
2077 svm->vmcb->control.exit_code_hi = 0;
2078 svm->vmcb->control.exit_info_1 = 0;
2079 svm->vmcb->control.exit_info_2 = 0;
2080
2081 nested_svm_vmexit(svm);
3d6368ef
AG		 2082
2083 return 1;
2084}
2085
851ba692 2086static int stgi_interception(struct vcpu_svm *svm)
1371d904
AG		 2087{
2088 if (nested_svm_check_permissions(svm))
2089 return 1;
2090
2091 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2092 skip_emulated_instruction(&svm->vcpu);
2093
2af9194d 2094 enable_gif(svm);
1371d904
AG		 2095
2096 return 1;
2097}
2098
851ba692 2099static int clgi_interception(struct vcpu_svm *svm)
1371d904
AG		 2100{
2101 if (nested_svm_check_permissions(svm))
2102 return 1;
2103
2104 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2105 skip_emulated_instruction(&svm->vcpu);
2106
2af9194d 2107 disable_gif(svm);
1371d904
AG		 2108
2109 /* After a CLGI no interrupts should come */
2110 svm_clear_vintr(svm);
2111 svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
2112
2113 return 1;
2114}
2115
851ba692 2116static int invlpga_interception(struct vcpu_svm *svm)
ff092385
AG		 2117{
2118 struct kvm_vcpu *vcpu = &svm->vcpu;
ff092385 2119
ec1ff790
JR		 2120 trace_kvm_invlpga(svm->vmcb->save.rip, vcpu->arch.regs[VCPU_REGS_RCX],
2121 vcpu->arch.regs[VCPU_REGS_RAX]);
2122
ff092385
AG		 2123 /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
2124 kvm_mmu_invlpg(vcpu, vcpu->arch.regs[VCPU_REGS_RAX]);
2125
2126 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2127 skip_emulated_instruction(&svm->vcpu);
2128 return 1;
2129}
2130
532a46b9
JR		 2131static int skinit_interception(struct vcpu_svm *svm)
2132{
2133 trace_kvm_skinit(svm->vmcb->save.rip, svm->vcpu.arch.regs[VCPU_REGS_RAX]);
2134
2135 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2136 return 1;
2137}
2138
851ba692 2139static int invalid_op_interception(struct vcpu_svm *svm)
6aa8b732 2140{
7ee5d940 2141 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
6aa8b732
AK		 2142 return 1;
2143}
2144
851ba692 2145static int task_switch_interception(struct vcpu_svm *svm)
6aa8b732 2146{
37817f29 2147 u16 tss_selector;
64a7ec06
GN		 2148 int reason;
2149 int int_type = svm->vmcb->control.exit_int_info &
2150 SVM_EXITINTINFO_TYPE_MASK;
8317c298 2151 int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
fe8e7f83
GN		 2152 uint32_t type =
2153 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
2154 uint32_t idt_v =
2155 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
37817f29
IE		 2156
2157 tss_selector = (u16)svm->vmcb->control.exit_info_1;
64a7ec06 2158
37817f29
IE		 2159 if (svm->vmcb->control.exit_info_2 &
2160 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
64a7ec06
GN		 2161 reason = TASK_SWITCH_IRET;
2162 else if (svm->vmcb->control.exit_info_2 &
2163 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
2164 reason = TASK_SWITCH_JMP;
fe8e7f83 2165 else if (idt_v)
64a7ec06
GN		 2166 reason = TASK_SWITCH_GATE;
2167 else
2168 reason = TASK_SWITCH_CALL;
2169
fe8e7f83
GN		 2170 if (reason == TASK_SWITCH_GATE) {
2171 switch (type) {
2172 case SVM_EXITINTINFO_TYPE_NMI:
2173 svm->vcpu.arch.nmi_injected = false;
2174 break;
2175 case SVM_EXITINTINFO_TYPE_EXEPT:
2176 kvm_clear_exception_queue(&svm->vcpu);
2177 break;
2178 case SVM_EXITINTINFO_TYPE_INTR:
2179 kvm_clear_interrupt_queue(&svm->vcpu);
2180 break;
2181 default:
2182 break;
2183 }
2184 }
64a7ec06 2185
8317c298
GN		 2186 if (reason != TASK_SWITCH_GATE ||
2187 int_type == SVM_EXITINTINFO_TYPE_SOFT ||
2188 (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
f629cf84
GN		 2189 (int_vec == OF_VECTOR || int_vec == BP_VECTOR)))
2190 skip_emulated_instruction(&svm->vcpu);
64a7ec06
GN		 2191
2192 return kvm_task_switch(&svm->vcpu, tss_selector, reason);
6aa8b732
AK		 2193}
2194
851ba692 2195static int cpuid_interception(struct vcpu_svm *svm)
6aa8b732 2196{
5fdbf976 2197 svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
e756fc62 2198 kvm_emulate_cpuid(&svm->vcpu);
06465c5a 2199 return 1;
6aa8b732
AK		 2200}
2201
851ba692 2202static int iret_interception(struct vcpu_svm *svm)
95ba8273
GN		 2203{
2204 ++svm->vcpu.stat.nmi_window_exits;
2205 svm->vmcb->control.intercept &= ~(1UL << INTERCEPT_IRET);
44c11430 2206 svm->vcpu.arch.hflags |= HF_IRET_MASK;
95ba8273
GN		 2207 return 1;
2208}
2209
851ba692 2210static int invlpg_interception(struct vcpu_svm *svm)
a7052897 2211{
851ba692 2212 if (emulate_instruction(&svm->vcpu, 0, 0, 0) != EMULATE_DONE)
a7052897
MT		 2213 pr_unimpl(&svm->vcpu, "%s: failed\n", __func__);
2214 return 1;
2215}
2216
851ba692 2217static int emulate_on_interception(struct vcpu_svm *svm)
6aa8b732 2218{
851ba692 2219 if (emulate_instruction(&svm->vcpu, 0, 0, 0) != EMULATE_DONE)
b8688d51 2220 pr_unimpl(&svm->vcpu, "%s: failed\n", __func__);
6aa8b732
AK		 2221 return 1;
2222}
2223
851ba692 2224static int cr8_write_interception(struct vcpu_svm *svm)
1d075434 2225{
851ba692
AK		 2226 struct kvm_run *kvm_run = svm->vcpu.run;
2227
0a5fff19
GN		 2228 u8 cr8_prev = kvm_get_cr8(&svm->vcpu);
2229 /* instruction emulation calls kvm_set_cr8() */
851ba692 2230 emulate_instruction(&svm->vcpu, 0, 0, 0);
95ba8273
GN		 2231 if (irqchip_in_kernel(svm->vcpu.kvm)) {
2232 svm->vmcb->control.intercept_cr_write &= ~INTERCEPT_CR8_MASK;
1d075434 2233 return 1;
95ba8273 2234 }
0a5fff19
GN		 2235 if (cr8_prev <= kvm_get_cr8(&svm->vcpu))
2236 return 1;
1d075434
JR		 2237 kvm_run->exit_reason = KVM_EXIT_SET_TPR;
2238 return 0;
2239}
2240
6aa8b732
AK		 2241static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data)
2242{
a2fa3e9f
GH		 2243 struct vcpu_svm *svm = to_svm(vcpu);
2244
6aa8b732 2245 switch (ecx) {
af24a4e4 2246 case MSR_IA32_TSC: {
20824f30 2247 u64 tsc_offset;
6aa8b732 2248
20824f30
JR		 2249 if (is_nested(svm))
2250 tsc_offset = svm->nested.hsave->control.tsc_offset;
2251 else
2252 tsc_offset = svm->vmcb->control.tsc_offset;
2253
2254 *data = tsc_offset + native_read_tsc();
6aa8b732
AK		 2255 break;
2256 }
0e859cac 2257 case MSR_K6_STAR:
a2fa3e9f 2258 *data = svm->vmcb->save.star;
6aa8b732 2259 break;
0e859cac 2260#ifdef CONFIG_X86_64
6aa8b732 2261 case MSR_LSTAR:
a2fa3e9f 2262 *data = svm->vmcb->save.lstar;
6aa8b732
AK		 2263 break;
2264 case MSR_CSTAR:
a2fa3e9f 2265 *data = svm->vmcb->save.cstar;
6aa8b732
AK		 2266 break;
2267 case MSR_KERNEL_GS_BASE:
a2fa3e9f 2268 *data = svm->vmcb->save.kernel_gs_base;
6aa8b732
AK		 2269 break;
2270 case MSR_SYSCALL_MASK:
a2fa3e9f 2271 *data = svm->vmcb->save.sfmask;
6aa8b732
AK		 2272 break;
2273#endif
2274 case MSR_IA32_SYSENTER_CS:
a2fa3e9f 2275 *data = svm->vmcb->save.sysenter_cs;
6aa8b732
AK		 2276 break;
2277 case MSR_IA32_SYSENTER_EIP:
017cb99e 2278 *data = svm->sysenter_eip;
6aa8b732
AK		 2279 break;
2280 case MSR_IA32_SYSENTER_ESP:
017cb99e 2281 *data = svm->sysenter_esp;
6aa8b732 2282 break;
e0231715
JR		 2283 /*
2284 * Nobody will change the following 5 values in the VMCB so we can
2285 * safely return them on rdmsr. They will always be 0 until LBRV is
2286 * implemented.
2287 */
a2938c80
JR		 2288 case MSR_IA32_DEBUGCTLMSR:
2289 *data = svm->vmcb->save.dbgctl;
2290 break;
2291 case MSR_IA32_LASTBRANCHFROMIP:
2292 *data = svm->vmcb->save.br_from;
2293 break;
2294 case MSR_IA32_LASTBRANCHTOIP:
2295 *data = svm->vmcb->save.br_to;
2296 break;
2297 case MSR_IA32_LASTINTFROMIP:
2298 *data = svm->vmcb->save.last_excp_from;
2299 break;
2300 case MSR_IA32_LASTINTTOIP:
2301 *data = svm->vmcb->save.last_excp_to;
2302 break;
b286d5d8 2303 case MSR_VM_HSAVE_PA:
e6aa9abd 2304 *data = svm->nested.hsave_msr;
b286d5d8 2305 break;
eb6f302e 2306 case MSR_VM_CR:
4a810181 2307 *data = svm->nested.vm_cr_msr;
eb6f302e 2308 break;
c8a73f18
AG		 2309 case MSR_IA32_UCODE_REV:
2310 *data = 0x01000065;
2311 break;
6aa8b732 2312 default:
3bab1f5d 2313 return kvm_get_msr_common(vcpu, ecx, data);
6aa8b732
AK		 2314 }
2315 return 0;
2316}
2317
851ba692 2318static int rdmsr_interception(struct vcpu_svm *svm)
6aa8b732 2319{
ad312c7c 2320 u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX];
6aa8b732
AK		 2321 u64 data;
2322
59200273
AK		 2323 if (svm_get_msr(&svm->vcpu, ecx, &data)) {
2324 trace_kvm_msr_read_ex(ecx);
c1a5d4f9 2325 kvm_inject_gp(&svm->vcpu, 0);
59200273 2326 } else {
229456fc 2327 trace_kvm_msr_read(ecx, data);
af9ca2d7 2328
5fdbf976 2329 svm->vcpu.arch.regs[VCPU_REGS_RAX] = data & 0xffffffff;
ad312c7c 2330 svm->vcpu.arch.regs[VCPU_REGS_RDX] = data >> 32;
5fdbf976 2331 svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
e756fc62 2332 skip_emulated_instruction(&svm->vcpu);
6aa8b732
AK		 2333 }
2334 return 1;
2335}
2336
4a810181
JR		 2337static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
2338{
2339 struct vcpu_svm *svm = to_svm(vcpu);
2340 int svm_dis, chg_mask;
2341
2342 if (data & ~SVM_VM_CR_VALID_MASK)
2343 return 1;
2344
2345 chg_mask = SVM_VM_CR_VALID_MASK;
2346
2347 if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
2348 chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);
2349
2350 svm->nested.vm_cr_msr &= ~chg_mask;
2351 svm->nested.vm_cr_msr |= (data & chg_mask);
2352
2353 svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;
2354
2355 /* check for svm_disable while efer.svme is set */
2356 if (svm_dis && (vcpu->arch.efer & EFER_SVME))
2357 return 1;
2358
2359 return 0;
2360}
2361
6aa8b732
AK		 2362static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data)
2363{
a2fa3e9f
GH		 2364 struct vcpu_svm *svm = to_svm(vcpu);
2365
6aa8b732 2366 switch (ecx) {
af24a4e4 2367 case MSR_IA32_TSC: {
20824f30
JR		 2368 u64 tsc_offset = data - native_read_tsc();
2369 u64 g_tsc_offset = 0;
2370
2371 if (is_nested(svm)) {
2372 g_tsc_offset = svm->vmcb->control.tsc_offset -
2373 svm->nested.hsave->control.tsc_offset;
2374 svm->nested.hsave->control.tsc_offset = tsc_offset;
2375 }
2376
2377 svm->vmcb->control.tsc_offset = tsc_offset + g_tsc_offset;
6aa8b732 2378
6aa8b732
AK		 2379 break;
2380 }
0e859cac 2381 case MSR_K6_STAR:
a2fa3e9f 2382 svm->vmcb->save.star = data;
6aa8b732 2383 break;
49b14f24 2384#ifdef CONFIG_X86_64
6aa8b732 2385 case MSR_LSTAR:
a2fa3e9f 2386 svm->vmcb->save.lstar = data;
6aa8b732
AK		 2387 break;
2388 case MSR_CSTAR:
a2fa3e9f 2389 svm->vmcb->save.cstar = data;
6aa8b732
AK		 2390 break;
2391 case MSR_KERNEL_GS_BASE:
a2fa3e9f 2392 svm->vmcb->save.kernel_gs_base = data;
6aa8b732
AK		 2393 break;
2394 case MSR_SYSCALL_MASK:
a2fa3e9f 2395 svm->vmcb->save.sfmask = data;
6aa8b732
AK		 2396 break;
2397#endif
2398 case MSR_IA32_SYSENTER_CS:
a2fa3e9f 2399 svm->vmcb->save.sysenter_cs = data;
6aa8b732
AK		 2400 break;
2401 case MSR_IA32_SYSENTER_EIP:
017cb99e 2402 svm->sysenter_eip = data;
a2fa3e9f 2403 svm->vmcb->save.sysenter_eip = data;
6aa8b732
AK		 2404 break;
2405 case MSR_IA32_SYSENTER_ESP:
017cb99e 2406 svm->sysenter_esp = data;
a2fa3e9f 2407 svm->vmcb->save.sysenter_esp = data;
6aa8b732 2408 break;
a2938c80 2409 case MSR_IA32_DEBUGCTLMSR:
24e09cbf
JR		 2410 if (!svm_has(SVM_FEATURE_LBRV)) {
2411 pr_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
b8688d51 2412 __func__, data);
24e09cbf
JR		 2413 break;
2414 }
2415 if (data & DEBUGCTL_RESERVED_BITS)
2416 return 1;
2417
2418 svm->vmcb->save.dbgctl = data;
2419 if (data & (1ULL<<0))
2420 svm_enable_lbrv(svm);
2421 else
2422 svm_disable_lbrv(svm);
a2938c80 2423 break;
b286d5d8 2424 case MSR_VM_HSAVE_PA:
e6aa9abd 2425 svm->nested.hsave_msr = data;
62b9abaa 2426 break;
3c5d0a44 2427 case MSR_VM_CR:
4a810181 2428 return svm_set_vm_cr(vcpu, data);
3c5d0a44 2429 case MSR_VM_IGNNE:
3c5d0a44
AG		 2430 pr_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
2431 break;
6aa8b732 2432 default:
3bab1f5d 2433 return kvm_set_msr_common(vcpu, ecx, data);
6aa8b732
AK		 2434 }
2435 return 0;
2436}
2437
851ba692 2438static int wrmsr_interception(struct vcpu_svm *svm)
6aa8b732 2439{
ad312c7c 2440 u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX];
5fdbf976 2441 u64 data = (svm->vcpu.arch.regs[VCPU_REGS_RAX] & -1u)
ad312c7c 2442 | ((u64)(svm->vcpu.arch.regs[VCPU_REGS_RDX] & -1u) << 32);
af9ca2d7 2443
af9ca2d7 2444
5fdbf976 2445 svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
59200273
AK		 2446 if (svm_set_msr(&svm->vcpu, ecx, data)) {
2447 trace_kvm_msr_write_ex(ecx, data);
c1a5d4f9 2448 kvm_inject_gp(&svm->vcpu, 0);
59200273
AK		 2449 } else {
2450 trace_kvm_msr_write(ecx, data);
e756fc62 2451 skip_emulated_instruction(&svm->vcpu);
59200273 2452 }
6aa8b732
AK		 2453 return 1;
2454}
2455
851ba692 2456static int msr_interception(struct vcpu_svm *svm)
6aa8b732 2457{
e756fc62 2458 if (svm->vmcb->control.exit_info_1)
851ba692 2459 return wrmsr_interception(svm);
6aa8b732 2460 else
851ba692 2461 return rdmsr_interception(svm);
6aa8b732
AK		 2462}
2463
851ba692 2464static int interrupt_window_interception(struct vcpu_svm *svm)
c1150d8c 2465{
851ba692
AK		 2466 struct kvm_run *kvm_run = svm->vcpu.run;
2467
f0b85051 2468 svm_clear_vintr(svm);
85f455f7 2469 svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
c1150d8c
DL		 2470 /*
2471 * If the user space waits to inject interrupts, exit as soon as
2472 * possible
2473 */
8061823a
GN		 2474 if (!irqchip_in_kernel(svm->vcpu.kvm) &&
2475 kvm_run->request_interrupt_window &&
2476 !kvm_cpu_has_interrupt(&svm->vcpu)) {
e756fc62 2477 ++svm->vcpu.stat.irq_window_exits;
c1150d8c
DL		 2478 kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
2479 return 0;
2480 }
2481
2482 return 1;
2483}
2484
565d0998
ML		 2485static int pause_interception(struct vcpu_svm *svm)
2486{
2487 kvm_vcpu_on_spin(&(svm->vcpu));
2488 return 1;
2489}
2490
851ba692 2491static int (*svm_exit_handlers[])(struct vcpu_svm *svm) = {
e0231715
JR		 2492 [SVM_EXIT_READ_CR0] = emulate_on_interception,
2493 [SVM_EXIT_READ_CR3] = emulate_on_interception,
2494 [SVM_EXIT_READ_CR4] = emulate_on_interception,
2495 [SVM_EXIT_READ_CR8] = emulate_on_interception,
d225157b 2496 [SVM_EXIT_CR0_SEL_WRITE] = emulate_on_interception,
e0231715
JR		 2497 [SVM_EXIT_WRITE_CR0] = emulate_on_interception,
2498 [SVM_EXIT_WRITE_CR3] = emulate_on_interception,
2499 [SVM_EXIT_WRITE_CR4] = emulate_on_interception,
2500 [SVM_EXIT_WRITE_CR8] = cr8_write_interception,
2501 [SVM_EXIT_READ_DR0] = emulate_on_interception,
6aa8b732
AK		 2502 [SVM_EXIT_READ_DR1] = emulate_on_interception,
2503 [SVM_EXIT_READ_DR2] = emulate_on_interception,
2504 [SVM_EXIT_READ_DR3] = emulate_on_interception,
727f5a23
JK		 2505 [SVM_EXIT_READ_DR4] = emulate_on_interception,
2506 [SVM_EXIT_READ_DR5] = emulate_on_interception,
2507 [SVM_EXIT_READ_DR6] = emulate_on_interception,
2508 [SVM_EXIT_READ_DR7] = emulate_on_interception,
6aa8b732
AK		 2509 [SVM_EXIT_WRITE_DR0] = emulate_on_interception,
2510 [SVM_EXIT_WRITE_DR1] = emulate_on_interception,
2511 [SVM_EXIT_WRITE_DR2] = emulate_on_interception,
2512 [SVM_EXIT_WRITE_DR3] = emulate_on_interception,
727f5a23 2513 [SVM_EXIT_WRITE_DR4] = emulate_on_interception,
6aa8b732 2514 [SVM_EXIT_WRITE_DR5] = emulate_on_interception,
727f5a23 2515 [SVM_EXIT_WRITE_DR6] = emulate_on_interception,
6aa8b732 2516 [SVM_EXIT_WRITE_DR7] = emulate_on_interception,
d0bfb940
JK		 2517 [SVM_EXIT_EXCP_BASE + DB_VECTOR] = db_interception,
2518 [SVM_EXIT_EXCP_BASE + BP_VECTOR] = bp_interception,
7aa81cc0 2519 [SVM_EXIT_EXCP_BASE + UD_VECTOR] = ud_interception,
e0231715
JR		 2520 [SVM_EXIT_EXCP_BASE + PF_VECTOR] = pf_interception,
2521 [SVM_EXIT_EXCP_BASE + NM_VECTOR] = nm_interception,
2522 [SVM_EXIT_EXCP_BASE + MC_VECTOR] = mc_interception,
2523 [SVM_EXIT_INTR] = intr_interception,
c47f098d 2524 [SVM_EXIT_NMI] = nmi_interception,
6aa8b732
AK		 2525 [SVM_EXIT_SMI] = nop_on_interception,
2526 [SVM_EXIT_INIT] = nop_on_interception,
c1150d8c 2527 [SVM_EXIT_VINTR] = interrupt_window_interception,
6aa8b732 2528 [SVM_EXIT_CPUID] = cpuid_interception,
95ba8273 2529 [SVM_EXIT_IRET] = iret_interception,
cf5a94d1 2530 [SVM_EXIT_INVD] = emulate_on_interception,
565d0998 2531 [SVM_EXIT_PAUSE] = pause_interception,
6aa8b732 2532 [SVM_EXIT_HLT] = halt_interception,
a7052897 2533 [SVM_EXIT_INVLPG] = invlpg_interception,
ff092385 2534 [SVM_EXIT_INVLPGA] = invlpga_interception,
e0231715 2535 [SVM_EXIT_IOIO] = io_interception,
6aa8b732
AK		 2536 [SVM_EXIT_MSR] = msr_interception,
2537 [SVM_EXIT_TASK_SWITCH] = task_switch_interception,
46fe4ddd 2538 [SVM_EXIT_SHUTDOWN] = shutdown_interception,
3d6368ef 2539 [SVM_EXIT_VMRUN] = vmrun_interception,
02e235bc 2540 [SVM_EXIT_VMMCALL] = vmmcall_interception,
5542675b
AG		 2541 [SVM_EXIT_VMLOAD] = vmload_interception,
2542 [SVM_EXIT_VMSAVE] = vmsave_interception,
1371d904
AG		 2543 [SVM_EXIT_STGI] = stgi_interception,
2544 [SVM_EXIT_CLGI] = clgi_interception,
532a46b9 2545 [SVM_EXIT_SKINIT] = skinit_interception,
cf5a94d1 2546 [SVM_EXIT_WBINVD] = emulate_on_interception,
916ce236
JR		 2547 [SVM_EXIT_MONITOR] = invalid_op_interception,
2548 [SVM_EXIT_MWAIT] = invalid_op_interception,
709ddebf 2549 [SVM_EXIT_NPF] = pf_interception,
6aa8b732
AK		 2550};
2551
851ba692 2552static int handle_exit(struct kvm_vcpu *vcpu)
6aa8b732 2553{
04d2cc77 2554 struct vcpu_svm *svm = to_svm(vcpu);
851ba692 2555 struct kvm_run *kvm_run = vcpu->run;
a2fa3e9f 2556 u32 exit_code = svm->vmcb->control.exit_code;
6aa8b732 2557
229456fc 2558 trace_kvm_exit(exit_code, svm->vmcb->save.rip);
af9ca2d7 2559
cd3ff653
JR		 2560 if (unlikely(svm->nested.exit_required)) {
2561 nested_svm_vmexit(svm);
2562 svm->nested.exit_required = false;
2563
2564 return 1;
2565 }
2566
cf74a78b 2567 if (is_nested(svm)) {
410e4d57
JR		 2568 int vmexit;
2569
d8cabddf
JR		 2570 trace_kvm_nested_vmexit(svm->vmcb->save.rip, exit_code,
2571 svm->vmcb->control.exit_info_1,
2572 svm->vmcb->control.exit_info_2,
2573 svm->vmcb->control.exit_int_info,
2574 svm->vmcb->control.exit_int_info_err);
2575
410e4d57
JR		 2576 vmexit = nested_svm_exit_special(svm);
2577
2578 if (vmexit == NESTED_EXIT_CONTINUE)
2579 vmexit = nested_svm_exit_handled(svm);
2580
2581 if (vmexit == NESTED_EXIT_DONE)
cf74a78b 2582 return 1;
cf74a78b
AG		 2583 }
2584
a5c3832d
JR		 2585 svm_complete_interrupts(svm);
2586
888f9f3e 2587 if (!(svm->vmcb->control.intercept_cr_write & INTERCEPT_CR0_MASK))
709ddebf 2588 vcpu->arch.cr0 = svm->vmcb->save.cr0;
888f9f3e 2589 if (npt_enabled)
709ddebf 2590 vcpu->arch.cr3 = svm->vmcb->save.cr3;
04d2cc77
AK		 2591
2592 if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
2593 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
2594 kvm_run->fail_entry.hardware_entry_failure_reason
2595 = svm->vmcb->control.exit_code;
2596 return 0;
2597 }
2598
a2fa3e9f 2599 if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
709ddebf 2600 exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
fe8e7f83 2601 exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH)
6aa8b732
AK		 2602 printk(KERN_ERR "%s: unexpected exit_ini_info 0x%x "
2603 "exit_code 0x%x\n",
b8688d51 2604 __func__, svm->vmcb->control.exit_int_info,
6aa8b732
AK		 2605 exit_code);
2606
9d8f549d 2607 if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
56919c5c 2608 || !svm_exit_handlers[exit_code]) {
6aa8b732 2609 kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
364b625b 2610 kvm_run->hw.hardware_exit_reason = exit_code;
6aa8b732
AK		 2611 return 0;
2612 }
2613
851ba692 2614 return svm_exit_handlers[exit_code](svm);
6aa8b732
AK		 2615}
2616
2617static void reload_tss(struct kvm_vcpu *vcpu)
2618{
2619 int cpu = raw_smp_processor_id();
2620
0fe1e009
TH		 2621 struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
2622 sd->tss_desc->type = 9; /* available 32/64-bit TSS */
6aa8b732
AK		 2623 load_TR_desc();
2624}
2625
e756fc62 2626static void pre_svm_run(struct vcpu_svm *svm)
6aa8b732
AK		 2627{
2628 int cpu = raw_smp_processor_id();
2629
0fe1e009 2630 struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
6aa8b732 2631
a2fa3e9f 2632 svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
4b656b12 2633 /* FIXME: handle wraparound of asid_generation */
0fe1e009
TH		 2634 if (svm->asid_generation != sd->asid_generation)
2635 new_asid(svm, sd);
6aa8b732
AK		 2636}
2637
95ba8273
GN		 2638static void svm_inject_nmi(struct kvm_vcpu *vcpu)
2639{
2640 struct vcpu_svm *svm = to_svm(vcpu);
2641
2642 svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
2643 vcpu->arch.hflags |= HF_NMI_MASK;
2644 svm->vmcb->control.intercept |= (1UL << INTERCEPT_IRET);
2645 ++vcpu->stat.nmi_injections;
2646}
6aa8b732 2647
85f455f7 2648static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
6aa8b732
AK		 2649{
2650 struct vmcb_control_area *control;
2651
229456fc 2652 trace_kvm_inj_virq(irq);
af9ca2d7 2653
fa89a817 2654 ++svm->vcpu.stat.irq_injections;
e756fc62 2655 control = &svm->vmcb->control;
85f455f7 2656 control->int_vector = irq;
6aa8b732
AK		 2657 control->int_ctl &= ~V_INTR_PRIO_MASK;
2658 control->int_ctl |= V_IRQ_MASK |
2659 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
2660}
2661
66fd3f7f 2662static void svm_set_irq(struct kvm_vcpu *vcpu)
2a8067f1
ED		 2663{
2664 struct vcpu_svm *svm = to_svm(vcpu);
2665
2af9194d 2666 BUG_ON(!(gif_set(svm)));
cf74a78b 2667
219b65dc
AG		 2668 svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
2669 SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
2a8067f1
ED		 2670}
2671
95ba8273 2672static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
aaacfc9a
JR		 2673{
2674 struct vcpu_svm *svm = to_svm(vcpu);
aaacfc9a 2675
88ab24ad
JR		 2676 if (is_nested(svm) && (vcpu->arch.hflags & HF_VINTR_MASK))
2677 return;
2678
95ba8273 2679 if (irr == -1)
aaacfc9a
JR		 2680 return;
2681
95ba8273
GN		 2682 if (tpr >= irr)
2683 svm->vmcb->control.intercept_cr_write |= INTERCEPT_CR8_MASK;
2684}
aaacfc9a 2685
95ba8273
GN		 2686static int svm_nmi_allowed(struct kvm_vcpu *vcpu)
2687{
2688 struct vcpu_svm *svm = to_svm(vcpu);
2689 struct vmcb *vmcb = svm->vmcb;
2690 return !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
2691 !(svm->vcpu.arch.hflags & HF_NMI_MASK);
aaacfc9a
JR		 2692}
2693
3cfc3092
JK		 2694static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
2695{
2696 struct vcpu_svm *svm = to_svm(vcpu);
2697
2698 return !!(svm->vcpu.arch.hflags & HF_NMI_MASK);
2699}
2700
2701static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
2702{
2703 struct vcpu_svm *svm = to_svm(vcpu);
2704
2705 if (masked) {
2706 svm->vcpu.arch.hflags |= HF_NMI_MASK;
2707 svm->vmcb->control.intercept |= (1UL << INTERCEPT_IRET);
2708 } else {
2709 svm->vcpu.arch.hflags &= ~HF_NMI_MASK;
2710 svm->vmcb->control.intercept &= ~(1UL << INTERCEPT_IRET);
2711 }
2712}
2713
78646121
GN		 2714static int svm_interrupt_allowed(struct kvm_vcpu *vcpu)
2715{
2716 struct vcpu_svm *svm = to_svm(vcpu);
2717 struct vmcb *vmcb = svm->vmcb;
7fcdb510
JR		 2718 int ret;
2719
2720 if (!gif_set(svm) ||
2721 (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK))
2722 return 0;
2723
2724 ret = !!(vmcb->save.rflags & X86_EFLAGS_IF);
2725
2726 if (is_nested(svm))
2727 return ret && !(svm->vcpu.arch.hflags & HF_VINTR_MASK);
2728
2729 return ret;
78646121
GN		 2730}
2731
9222be18 2732static void enable_irq_window(struct kvm_vcpu *vcpu)
6aa8b732 2733{
219b65dc 2734 struct vcpu_svm *svm = to_svm(vcpu);
219b65dc 2735
e0231715
JR		 2736 /*
2737 * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
2738 * 1, because that's a separate STGI/VMRUN intercept. The next time we
2739 * get that intercept, this function will be called again though and
2740 * we'll get the vintr intercept.
2741 */
8fe54654 2742 if (gif_set(svm) && nested_svm_intr(svm)) {
219b65dc
AG		 2743 svm_set_vintr(svm);
2744 svm_inject_irq(svm, 0x0);
2745 }
85f455f7
ED		 2746}
2747
95ba8273 2748static void enable_nmi_window(struct kvm_vcpu *vcpu)
c1150d8c 2749{
04d2cc77 2750 struct vcpu_svm *svm = to_svm(vcpu);
c1150d8c 2751
44c11430
GN		 2752 if ((svm->vcpu.arch.hflags & (HF_NMI_MASK | HF_IRET_MASK))
2753 == HF_NMI_MASK)
2754 return; /* IRET will cause a vm exit */
2755
e0231715
JR		 2756 /*
2757 * Something prevents NMI from been injected. Single step over possible
2758 * problem (IRET or exception injection or interrupt shadow)
2759 */
887f500c
JR		 2760 if (gif_set(svm) && nested_svm_nmi(svm)) {
2761 svm->nmi_singlestep = true;
2762 svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
2763 update_db_intercept(vcpu);
2764 }
c1150d8c
DL		 2765}
2766
cbc94022
IE		 2767static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
2768{
2769 return 0;
2770}
2771
d9e368d6
AK		 2772static void svm_flush_tlb(struct kvm_vcpu *vcpu)
2773{
2774 force_new_asid(vcpu);
2775}
2776
04d2cc77
AK		 2777static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
2778{
2779}
2780
d7bf8221
JR		 2781static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
2782{
2783 struct vcpu_svm *svm = to_svm(vcpu);
2784
88ab24ad
JR		 2785 if (is_nested(svm) && (vcpu->arch.hflags & HF_VINTR_MASK))
2786 return;
2787
d7bf8221
JR		 2788 if (!(svm->vmcb->control.intercept_cr_write & INTERCEPT_CR8_MASK)) {
2789 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
615d5193 2790 kvm_set_cr8(vcpu, cr8);
d7bf8221
JR		 2791 }
2792}
2793
649d6864
JR		 2794static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
2795{
2796 struct vcpu_svm *svm = to_svm(vcpu);
2797 u64 cr8;
2798
88ab24ad
JR		 2799 if (is_nested(svm) && (vcpu->arch.hflags & HF_VINTR_MASK))
2800 return;
2801
649d6864
JR		 2802 cr8 = kvm_get_cr8(vcpu);
2803 svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
2804 svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
2805}
2806
9222be18
GN		 2807static void svm_complete_interrupts(struct vcpu_svm *svm)
2808{
2809 u8 vector;
2810 int type;
2811 u32 exitintinfo = svm->vmcb->control.exit_int_info;
66b7138f
JK		 2812 unsigned int3_injected = svm->int3_injected;
2813
2814 svm->int3_injected = 0;
9222be18 2815
44c11430
GN		 2816 if (svm->vcpu.arch.hflags & HF_IRET_MASK)
2817 svm->vcpu.arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK);
2818
9222be18
GN		 2819 svm->vcpu.arch.nmi_injected = false;
2820 kvm_clear_exception_queue(&svm->vcpu);
2821 kvm_clear_interrupt_queue(&svm->vcpu);
2822
2823 if (!(exitintinfo & SVM_EXITINTINFO_VALID))
2824 return;
2825
2826 vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
2827 type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
2828
2829 switch (type) {
2830 case SVM_EXITINTINFO_TYPE_NMI:
2831 svm->vcpu.arch.nmi_injected = true;
2832 break;
2833 case SVM_EXITINTINFO_TYPE_EXEPT:
219b65dc
AG		 2834 if (is_nested(svm))
2835 break;
66b7138f
JK		 2836 /*
2837 * In case of software exceptions, do not reinject the vector,
2838 * but re-execute the instruction instead. Rewind RIP first
2839 * if we emulated INT3 before.
2840 */
2841 if (kvm_exception_is_soft(vector)) {
2842 if (vector == BP_VECTOR && int3_injected &&
2843 kvm_is_linear_rip(&svm->vcpu, svm->int3_rip))
2844 kvm_rip_write(&svm->vcpu,
2845 kvm_rip_read(&svm->vcpu) -
2846 int3_injected);
9222be18 2847 break;
66b7138f 2848 }
9222be18
GN		 2849 if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
2850 u32 err = svm->vmcb->control.exit_int_info_err;
2851 kvm_queue_exception_e(&svm->vcpu, vector, err);
2852
2853 } else
2854 kvm_queue_exception(&svm->vcpu, vector);
2855 break;
2856 case SVM_EXITINTINFO_TYPE_INTR:
66fd3f7f 2857 kvm_queue_interrupt(&svm->vcpu, vector, false);
9222be18
GN		 2858 break;
2859 default:
2860 break;
2861 }
2862}
2863
80e31d4f
AK		 2864#ifdef CONFIG_X86_64
2865#define R "r"
2866#else
2867#define R "e"
2868#endif
2869
851ba692 2870static void svm_vcpu_run(struct kvm_vcpu *vcpu)
6aa8b732 2871{
a2fa3e9f 2872 struct vcpu_svm *svm = to_svm(vcpu);
6aa8b732
AK		 2873 u16 fs_selector;
2874 u16 gs_selector;
2875 u16 ldt_selector;
d9e368d6 2876
cd3ff653
JR		 2877 /*
2878 * A vmexit emulation is required before the vcpu can be executed
2879 * again.
2880 */
2881 if (unlikely(svm->nested.exit_required))
2882 return;
2883
5fdbf976
MT		 2884 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
2885 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
2886 svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
2887
e756fc62 2888 pre_svm_run(svm);
6aa8b732 2889
649d6864
JR		 2890 sync_lapic_to_cr8(vcpu);
2891
6aa8b732 2892 save_host_msrs(vcpu);
d6e88aec
AK		 2893 fs_selector = kvm_read_fs();
2894 gs_selector = kvm_read_gs();
2895 ldt_selector = kvm_read_ldt();
cda0ffdd 2896 svm->vmcb->save.cr2 = vcpu->arch.cr2;
709ddebf
JR		 2897 /* required for live migration with NPT */
2898 if (npt_enabled)
2899 svm->vmcb->save.cr3 = vcpu->arch.cr3;
6aa8b732 2900
04d2cc77
AK		 2901 clgi();
2902
2903 local_irq_enable();
36241b8c 2904
6aa8b732 2905 asm volatile (
80e31d4f
AK		 2906 "push %%"R"bp; \n\t"
2907 "mov %c[rbx](%[svm]), %%"R"bx \n\t"
2908 "mov %c[rcx](%[svm]), %%"R"cx \n\t"
2909 "mov %c[rdx](%[svm]), %%"R"dx \n\t"
2910 "mov %c[rsi](%[svm]), %%"R"si \n\t"
2911 "mov %c[rdi](%[svm]), %%"R"di \n\t"
2912 "mov %c[rbp](%[svm]), %%"R"bp \n\t"
05b3e0c2 2913#ifdef CONFIG_X86_64
fb3f0f51
RR		 2914 "mov %c[r8](%[svm]), %%r8 \n\t"
2915 "mov %c[r9](%[svm]), %%r9 \n\t"
2916 "mov %c[r10](%[svm]), %%r10 \n\t"
2917 "mov %c[r11](%[svm]), %%r11 \n\t"
2918 "mov %c[r12](%[svm]), %%r12 \n\t"
2919 "mov %c[r13](%[svm]), %%r13 \n\t"
2920 "mov %c[r14](%[svm]), %%r14 \n\t"
2921 "mov %c[r15](%[svm]), %%r15 \n\t"
6aa8b732
AK		 2922#endif
2923
6aa8b732 2924 /* Enter guest mode */
80e31d4f
AK		 2925 "push %%"R"ax \n\t"
2926 "mov %c[vmcb](%[svm]), %%"R"ax \n\t"
4ecac3fd
AK		 2927 __ex(SVM_VMLOAD) "\n\t"
2928 __ex(SVM_VMRUN) "\n\t"
2929 __ex(SVM_VMSAVE) "\n\t"
80e31d4f 2930 "pop %%"R"ax \n\t"
6aa8b732
AK		 2931
2932 /* Save guest registers, load host registers */
80e31d4f
AK		 2933 "mov %%"R"bx, %c[rbx](%[svm]) \n\t"
2934 "mov %%"R"cx, %c[rcx](%[svm]) \n\t"
2935 "mov %%"R"dx, %c[rdx](%[svm]) \n\t"
2936 "mov %%"R"si, %c[rsi](%[svm]) \n\t"
2937 "mov %%"R"di, %c[rdi](%[svm]) \n\t"
2938 "mov %%"R"bp, %c[rbp](%[svm]) \n\t"
05b3e0c2 2939#ifdef CONFIG_X86_64
fb3f0f51
RR		 2940 "mov %%r8, %c[r8](%[svm]) \n\t"
2941 "mov %%r9, %c[r9](%[svm]) \n\t"
2942 "mov %%r10, %c[r10](%[svm]) \n\t"
2943 "mov %%r11, %c[r11](%[svm]) \n\t"
2944 "mov %%r12, %c[r12](%[svm]) \n\t"
2945 "mov %%r13, %c[r13](%[svm]) \n\t"
2946 "mov %%r14, %c[r14](%[svm]) \n\t"
2947 "mov %%r15, %c[r15](%[svm]) \n\t"
6aa8b732 2948#endif
80e31d4f 2949 "pop %%"R"bp"
6aa8b732 2950 :
fb3f0f51 2951 : [svm]"a"(svm),
6aa8b732 2952 [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)),
ad312c7c
ZX		 2953 [rbx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX])),
2954 [rcx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX])),
2955 [rdx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX])),
2956 [rsi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI])),
2957 [rdi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI])),
2958 [rbp]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP]))
05b3e0c2 2959#ifdef CONFIG_X86_64
ad312c7c
ZX		 2960 , [r8]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8])),
2961 [r9]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9])),
2962 [r10]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10])),
2963 [r11]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11])),
2964 [r12]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12])),
2965 [r13]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13])),
2966 [r14]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14])),
2967 [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15]))
6aa8b732 2968#endif
54a08c04 2969 : "cc", "memory"
80e31d4f 2970 , R"bx", R"cx", R"dx", R"si", R"di"
54a08c04 2971#ifdef CONFIG_X86_64
54a08c04
LV		 2972 , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15"
2973#endif
2974 );
6aa8b732 2975
ad312c7c 2976 vcpu->arch.cr2 = svm->vmcb->save.cr2;
5fdbf976
MT		 2977 vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
2978 vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
2979 vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
6aa8b732 2980
d6e88aec
AK		 2981 kvm_load_fs(fs_selector);
2982 kvm_load_gs(gs_selector);
2983 kvm_load_ldt(ldt_selector);
6aa8b732
AK		 2984 load_host_msrs(vcpu);
2985
2986 reload_tss(vcpu);
2987
56ba47dd
AK		 2988 local_irq_disable();
2989
2990 stgi();
2991
d7bf8221
JR		 2992 sync_cr8_to_lapic(vcpu);
2993
a2fa3e9f 2994 svm->next_rip = 0;
9222be18 2995
6de4f3ad
AK		 2996 if (npt_enabled) {
2997 vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR);
2998 vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR);
2999 }
6aa8b732
AK		 3000}
3001
80e31d4f
AK		 3002#undef R
3003
6aa8b732
AK		 3004static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
3005{
a2fa3e9f
GH		 3006 struct vcpu_svm *svm = to_svm(vcpu);
3007
709ddebf
JR		 3008 if (npt_enabled) {
3009 svm->vmcb->control.nested_cr3 = root;
3010 force_new_asid(vcpu);
3011 return;
3012 }
3013
a2fa3e9f 3014 svm->vmcb->save.cr3 = root;
6aa8b732
AK		 3015 force_new_asid(vcpu);
3016}
3017
6aa8b732
AK		 3018static int is_disabled(void)
3019{
6031a61c
JR		 3020 u64 vm_cr;
3021
3022 rdmsrl(MSR_VM_CR, vm_cr);
3023 if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
3024 return 1;
3025
6aa8b732
AK		 3026 return 0;
3027}
3028
102d8325
IM		 3029static void
3030svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
3031{
3032 /*
3033 * Patch in the VMMCALL instruction:
3034 */
3035 hypercall[0] = 0x0f;
3036 hypercall[1] = 0x01;
3037 hypercall[2] = 0xd9;
102d8325
IM		 3038}
3039
002c7f7c
YS		 3040static void svm_check_processor_compat(void *rtn)
3041{
3042 *(int *)rtn = 0;
3043}
3044
774ead3a
AK		 3045static bool svm_cpu_has_accelerated_tpr(void)
3046{
3047 return false;
3048}
3049
67253af5
SY		 3050static int get_npt_level(void)
3051{
3052#ifdef CONFIG_X86_64
3053 return PT64_ROOT_LEVEL;
3054#else
3055 return PT32E_ROOT_LEVEL;
3056#endif
3057}
3058
4b12f0de 3059static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
64d4d521
SY		 3060{
3061 return 0;
3062}
3063
0e851880
SY		 3064static void svm_cpuid_update(struct kvm_vcpu *vcpu)
3065{
3066}
3067
229456fc 3068static const struct trace_print_flags svm_exit_reasons_str[] = {
e0231715
JR		 3069 { SVM_EXIT_READ_CR0, "read_cr0" },
3070 { SVM_EXIT_READ_CR3, "read_cr3" },
3071 { SVM_EXIT_READ_CR4, "read_cr4" },
3072 { SVM_EXIT_READ_CR8, "read_cr8" },
3073 { SVM_EXIT_WRITE_CR0, "write_cr0" },
3074 { SVM_EXIT_WRITE_CR3, "write_cr3" },
3075 { SVM_EXIT_WRITE_CR4, "write_cr4" },
3076 { SVM_EXIT_WRITE_CR8, "write_cr8" },
3077 { SVM_EXIT_READ_DR0, "read_dr0" },
3078 { SVM_EXIT_READ_DR1, "read_dr1" },
3079 { SVM_EXIT_READ_DR2, "read_dr2" },
3080 { SVM_EXIT_READ_DR3, "read_dr3" },
3081 { SVM_EXIT_WRITE_DR0, "write_dr0" },
3082 { SVM_EXIT_WRITE_DR1, "write_dr1" },
3083 { SVM_EXIT_WRITE_DR2, "write_dr2" },
3084 { SVM_EXIT_WRITE_DR3, "write_dr3" },
3085 { SVM_EXIT_WRITE_DR5, "write_dr5" },
3086 { SVM_EXIT_WRITE_DR7, "write_dr7" },
229456fc
MT		 3087 { SVM_EXIT_EXCP_BASE + DB_VECTOR, "DB excp" },
3088 { SVM_EXIT_EXCP_BASE + BP_VECTOR, "BP excp" },
3089 { SVM_EXIT_EXCP_BASE + UD_VECTOR, "UD excp" },
3090 { SVM_EXIT_EXCP_BASE + PF_VECTOR, "PF excp" },
3091 { SVM_EXIT_EXCP_BASE + NM_VECTOR, "NM excp" },
3092 { SVM_EXIT_EXCP_BASE + MC_VECTOR, "MC excp" },
3093 { SVM_EXIT_INTR, "interrupt" },
3094 { SVM_EXIT_NMI, "nmi" },
3095 { SVM_EXIT_SMI, "smi" },
3096 { SVM_EXIT_INIT, "init" },
3097 { SVM_EXIT_VINTR, "vintr" },
3098 { SVM_EXIT_CPUID, "cpuid" },
3099 { SVM_EXIT_INVD, "invd" },
3100 { SVM_EXIT_HLT, "hlt" },
3101 { SVM_EXIT_INVLPG, "invlpg" },
3102 { SVM_EXIT_INVLPGA, "invlpga" },
3103 { SVM_EXIT_IOIO, "io" },
3104 { SVM_EXIT_MSR, "msr" },
3105 { SVM_EXIT_TASK_SWITCH, "task_switch" },
3106 { SVM_EXIT_SHUTDOWN, "shutdown" },
3107 { SVM_EXIT_VMRUN, "vmrun" },
3108 { SVM_EXIT_VMMCALL, "hypercall" },
3109 { SVM_EXIT_VMLOAD, "vmload" },
3110 { SVM_EXIT_VMSAVE, "vmsave" },
3111 { SVM_EXIT_STGI, "stgi" },
3112 { SVM_EXIT_CLGI, "clgi" },
3113 { SVM_EXIT_SKINIT, "skinit" },
3114 { SVM_EXIT_WBINVD, "wbinvd" },
3115 { SVM_EXIT_MONITOR, "monitor" },
3116 { SVM_EXIT_MWAIT, "mwait" },
3117 { SVM_EXIT_NPF, "npf" },
3118 { -1, NULL }
3119};
3120
17cc3935 3121static int svm_get_lpage_level(void)
344f414f 3122{
17cc3935 3123 return PT_PDPE_LEVEL;
344f414f
JR		 3124}
3125
4e47c7a6
SY		 3126static bool svm_rdtscp_supported(void)
3127{
3128 return false;
3129}
3130
02daab21
AK		 3131static void svm_fpu_deactivate(struct kvm_vcpu *vcpu)
3132{
3133 struct vcpu_svm *svm = to_svm(vcpu);
3134
02daab21 3135 svm->vmcb->control.intercept_exceptions |= 1 << NM_VECTOR;
66a562f7
JR		 3136 if (is_nested(svm))
3137 svm->nested.hsave->control.intercept_exceptions |= 1 << NM_VECTOR;
3138 update_cr0_intercept(svm);
02daab21
AK		 3139}
3140
cbdd1bea 3141static struct kvm_x86_ops svm_x86_ops = {
6aa8b732
AK		 3142 .cpu_has_kvm_support = has_svm,
3143 .disabled_by_bios = is_disabled,
3144 .hardware_setup = svm_hardware_setup,
3145 .hardware_unsetup = svm_hardware_unsetup,
002c7f7c 3146 .check_processor_compatibility = svm_check_processor_compat,
6aa8b732
AK		 3147 .hardware_enable = svm_hardware_enable,
3148 .hardware_disable = svm_hardware_disable,
774ead3a 3149 .cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
6aa8b732
AK		 3150
3151 .vcpu_create = svm_create_vcpu,
3152 .vcpu_free = svm_free_vcpu,
04d2cc77 3153 .vcpu_reset = svm_vcpu_reset,
6aa8b732 3154
04d2cc77 3155 .prepare_guest_switch = svm_prepare_guest_switch,
6aa8b732
AK		 3156 .vcpu_load = svm_vcpu_load,
3157 .vcpu_put = svm_vcpu_put,
3158
3159 .set_guest_debug = svm_guest_debug,
3160 .get_msr = svm_get_msr,
3161 .set_msr = svm_set_msr,
3162 .get_segment_base = svm_get_segment_base,
3163 .get_segment = svm_get_segment,
3164 .set_segment = svm_set_segment,
2e4d2653 3165 .get_cpl = svm_get_cpl,
1747fb71 3166 .get_cs_db_l_bits = kvm_get_cs_db_l_bits,
e8467fda 3167 .decache_cr0_guest_bits = svm_decache_cr0_guest_bits,
25c4c276 3168 .decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
6aa8b732 3169 .set_cr0 = svm_set_cr0,
6aa8b732
AK		 3170 .set_cr3 = svm_set_cr3,
3171 .set_cr4 = svm_set_cr4,
3172 .set_efer = svm_set_efer,
3173 .get_idt = svm_get_idt,
3174 .set_idt = svm_set_idt,
3175 .get_gdt = svm_get_gdt,
3176 .set_gdt = svm_set_gdt,
3177 .get_dr = svm_get_dr,
3178 .set_dr = svm_set_dr,
6de4f3ad 3179 .cache_reg = svm_cache_reg,
6aa8b732
AK		 3180 .get_rflags = svm_get_rflags,
3181 .set_rflags = svm_set_rflags,
6b52d186 3182 .fpu_activate = svm_fpu_activate,
02daab21 3183 .fpu_deactivate = svm_fpu_deactivate,
6aa8b732 3184
6aa8b732 3185 .tlb_flush = svm_flush_tlb,
6aa8b732 3186
6aa8b732 3187 .run = svm_vcpu_run,
04d2cc77 3188 .handle_exit = handle_exit,
6aa8b732 3189 .skip_emulated_instruction = skip_emulated_instruction,
2809f5d2
GC		 3190 .set_interrupt_shadow = svm_set_interrupt_shadow,
3191 .get_interrupt_shadow = svm_get_interrupt_shadow,
102d8325 3192 .patch_hypercall = svm_patch_hypercall,
2a8067f1 3193 .set_irq = svm_set_irq,
95ba8273 3194 .set_nmi = svm_inject_nmi,
298101da 3195 .queue_exception = svm_queue_exception,
78646121 3196 .interrupt_allowed = svm_interrupt_allowed,
95ba8273 3197 .nmi_allowed = svm_nmi_allowed,
3cfc3092
JK		 3198 .get_nmi_mask = svm_get_nmi_mask,
3199 .set_nmi_mask = svm_set_nmi_mask,
95ba8273
GN		 3200 .enable_nmi_window = enable_nmi_window,
3201 .enable_irq_window = enable_irq_window,
3202 .update_cr8_intercept = update_cr8_intercept,
cbc94022
IE		 3203
3204 .set_tss_addr = svm_set_tss_addr,
67253af5 3205 .get_tdp_level = get_npt_level,
4b12f0de 3206 .get_mt_mask = svm_get_mt_mask,
229456fc
MT		 3207
3208 .exit_reasons_str = svm_exit_reasons_str,
17cc3935 3209 .get_lpage_level = svm_get_lpage_level,
0e851880
SY		 3210
3211 .cpuid_update = svm_cpuid_update,
4e47c7a6
SY		 3212
3213 .rdtscp_supported = svm_rdtscp_supported,
6aa8b732
AK		 3214};
3215
3216static int __init svm_init(void)
3217{
cb498ea2 3218 return kvm_init(&svm_x86_ops, sizeof(struct vcpu_svm),
c16f862d 3219 THIS_MODULE);
6aa8b732
AK		 3220}
3221
3222static void __exit svm_exit(void)
3223{
cb498ea2 3224 kvm_exit();
6aa8b732
AK		 3225}
3226
3227module_init(svm_init)
3228module_exit(svm_exit)
Empty description
This page took 1.014967 seconds and 4 git commands to generate.