target/i386/whpx-all.c

   1 /*
   2  * QEMU Windows Hypervisor Platform accelerator (WHPX)
   3  *
   4  * Copyright Microsoft Corp. 2017
   5  *
   6  * This work is licensed under the terms of the GNU GPL, version 2 or later.
   7  * See the COPYING file in the top-level directory.
   8  *
   9  */
  10
  11 #include "qemu/osdep.h"
  12 #include "cpu.h"
  13 #include "exec/address-spaces.h"
  14 #include "exec/ioport.h"
  15 #include "qemu-common.h"
  16 #include "sysemu/accel.h"
  17 #include "sysemu/whpx.h"
  18 #include "sysemu/sysemu.h"
  19 #include "sysemu/cpus.h"
  20 #include "qemu/main-loop.h"
  21 #include "hw/boards.h"
  22 #include "qemu/error-report.h"
  23 #include "qemu/queue.h"
  24 #include "qapi/error.h"
  25 #include "migration/blocker.h"
  26 #include "whp-dispatch.h"
  27
  28 #include <WinHvPlatform.h>
  29 #include <WinHvEmulation.h>
  30
  31 struct whpx_state {
  32     uint64_t mem_quota;
  33     WHV_PARTITION_HANDLE partition;
  34 };
  35
  36 static const WHV_REGISTER_NAME whpx_register_names[] = {
  37
  38     /* X64 General purpose registers */
  39     WHvX64RegisterRax,
  40     WHvX64RegisterRcx,
  41     WHvX64RegisterRdx,
  42     WHvX64RegisterRbx,
  43     WHvX64RegisterRsp,
  44     WHvX64RegisterRbp,
  45     WHvX64RegisterRsi,
  46     WHvX64RegisterRdi,
  47     WHvX64RegisterR8,
  48     WHvX64RegisterR9,
  49     WHvX64RegisterR10,
  50     WHvX64RegisterR11,
  51     WHvX64RegisterR12,
  52     WHvX64RegisterR13,
  53     WHvX64RegisterR14,
  54     WHvX64RegisterR15,
  55     WHvX64RegisterRip,
  56     WHvX64RegisterRflags,
  57
  58     /* X64 Segment registers */
  59     WHvX64RegisterEs,
  60     WHvX64RegisterCs,
  61     WHvX64RegisterSs,
  62     WHvX64RegisterDs,
  63     WHvX64RegisterFs,
  64     WHvX64RegisterGs,
  65     WHvX64RegisterLdtr,
  66     WHvX64RegisterTr,
  67
  68     /* X64 Table registers */
  69     WHvX64RegisterIdtr,
  70     WHvX64RegisterGdtr,
  71
  72     /* X64 Control Registers */
  73     WHvX64RegisterCr0,
  74     WHvX64RegisterCr2,
  75     WHvX64RegisterCr3,
  76     WHvX64RegisterCr4,
  77     WHvX64RegisterCr8,
  78
  79     /* X64 Debug Registers */
  80     /*
  81      * WHvX64RegisterDr0,
  82      * WHvX64RegisterDr1,
  83      * WHvX64RegisterDr2,
  84      * WHvX64RegisterDr3,
  85      * WHvX64RegisterDr6,
  86      * WHvX64RegisterDr7,
  87      */
  88
  89     /* X64 Floating Point and Vector Registers */
  90     WHvX64RegisterXmm0,
  91     WHvX64RegisterXmm1,
  92     WHvX64RegisterXmm2,
  93     WHvX64RegisterXmm3,
  94     WHvX64RegisterXmm4,
  95     WHvX64RegisterXmm5,
  96     WHvX64RegisterXmm6,
  97     WHvX64RegisterXmm7,
  98     WHvX64RegisterXmm8,
  99     WHvX64RegisterXmm9,
 100     WHvX64RegisterXmm10,
 101     WHvX64RegisterXmm11,
 102     WHvX64RegisterXmm12,
 103     WHvX64RegisterXmm13,
 104     WHvX64RegisterXmm14,
 105     WHvX64RegisterXmm15,
 106     WHvX64RegisterFpMmx0,
 107     WHvX64RegisterFpMmx1,
 108     WHvX64RegisterFpMmx2,
 109     WHvX64RegisterFpMmx3,
 110     WHvX64RegisterFpMmx4,
 111     WHvX64RegisterFpMmx5,
 112     WHvX64RegisterFpMmx6,
 113     WHvX64RegisterFpMmx7,
 114     WHvX64RegisterFpControlStatus,
 115     WHvX64RegisterXmmControlStatus,
 116
 117     /* X64 MSRs */
 118     WHvX64RegisterTsc,
 119     WHvX64RegisterEfer,
 120 #ifdef TARGET_X86_64
 121     WHvX64RegisterKernelGsBase,
 122 #endif
 123     WHvX64RegisterApicBase,
 124     /* WHvX64RegisterPat, */
 125     WHvX64RegisterSysenterCs,
 126     WHvX64RegisterSysenterEip,
 127     WHvX64RegisterSysenterEsp,
 128     WHvX64RegisterStar,
 129 #ifdef TARGET_X86_64
 130     WHvX64RegisterLstar,
 131     WHvX64RegisterCstar,
 132     WHvX64RegisterSfmask,
 133 #endif
 134
 135     /* Interrupt / Event Registers */
 136     /*
 137      * WHvRegisterPendingInterruption,
 138      * WHvRegisterInterruptState,
 139      * WHvRegisterPendingEvent0,
 140      * WHvRegisterPendingEvent1
 141      * WHvX64RegisterDeliverabilityNotifications,
 142      */
 143 };
 144
 145 struct whpx_register_set {
 146     WHV_REGISTER_VALUE values[RTL_NUMBER_OF(whpx_register_names)];
 147 };
 148
 149 struct whpx_vcpu {
 150     WHV_EMULATOR_HANDLE emulator;
 151     bool window_registered;
 152     bool interruptable;
 153     uint64_t tpr;
 154     uint64_t apic_base;
 155     bool interruption_pending;
 156
 157     /* Must be the last field as it may have a tail */
 158     WHV_RUN_VP_EXIT_CONTEXT exit_ctx;
 159 };
 160
 161 static bool whpx_allowed;
 162 static bool whp_dispatch_initialized;
 163 static HMODULE hWinHvPlatform, hWinHvEmulation;
 164
 165 struct whpx_state whpx_global;
 166 struct WHPDispatch whp_dispatch;
 167
 168
 169 /*
 170  * VP support
 171  */
 172
 173 static struct whpx_vcpu *get_whpx_vcpu(CPUState *cpu)
 174 {
 175     return (struct whpx_vcpu *)cpu->hax_vcpu;
 176 }
 177
 178 static WHV_X64_SEGMENT_REGISTER whpx_seg_q2h(const SegmentCache *qs, int v86,
 179                                              int r86)
 180 {
 181     WHV_X64_SEGMENT_REGISTER hs;
 182     unsigned flags = qs->flags;
 183
 184     hs.Base = qs->base;
 185     hs.Limit = qs->limit;
 186     hs.Selector = qs->selector;
 187
 188     if (v86) {
 189         hs.Attributes = 0;
 190         hs.SegmentType = 3;
 191         hs.Present = 1;
 192         hs.DescriptorPrivilegeLevel = 3;
 193         hs.NonSystemSegment = 1;
 194
 195     } else {
 196         hs.Attributes = (flags >> DESC_TYPE_SHIFT);
 197
 198         if (r86) {
 199             /* hs.Base &= 0xfffff; */
 200         }
 201     }
 202
 203     return hs;
 204 }
 205
 206 static SegmentCache whpx_seg_h2q(const WHV_X64_SEGMENT_REGISTER *hs)
 207 {
 208     SegmentCache qs;
 209
 210     qs.base = hs->Base;
 211     qs.limit = hs->Limit;
 212     qs.selector = hs->Selector;
 213
 214     qs.flags = ((uint32_t)hs->Attributes) << DESC_TYPE_SHIFT;
 215
 216     return qs;
 217 }
 218
 219 static void whpx_set_registers(CPUState *cpu)
 220 {
 221     struct whpx_state *whpx = &whpx_global;
 222     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
 223     struct CPUX86State *env = (CPUArchState *)(cpu->env_ptr);
 224     X86CPU *x86_cpu = X86_CPU(cpu);
 225     struct whpx_register_set vcxt;
 226     HRESULT hr;
 227     int idx;
 228     int idx_next;
 229     int i;
 230     int v86, r86;
 231
 232     assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
 233
 234     memset(&vcxt, 0, sizeof(struct whpx_register_set));
 235
 236     v86 = (env->eflags & VM_MASK);
 237     r86 = !(env->cr[0] & CR0_PE_MASK);
 238
 239     vcpu->tpr = cpu_get_apic_tpr(x86_cpu->apic_state);
 240     vcpu->apic_base = cpu_get_apic_base(x86_cpu->apic_state);
 241
 242     idx = 0;
 243
 244     /* Indexes for first 16 registers match between HV and QEMU definitions */
 245     idx_next = 16;
 246     for (idx = 0; idx < CPU_NB_REGS; idx += 1) {
 247         vcxt.values[idx].Reg64 = (uint64_t)env->regs[idx];
 248     }
 249     idx = idx_next;
 250
 251     /* Same goes for RIP and RFLAGS */
 252     assert(whpx_register_names[idx] == WHvX64RegisterRip);
 253     vcxt.values[idx++].Reg64 = env->eip;
 254
 255     assert(whpx_register_names[idx] == WHvX64RegisterRflags);
 256     vcxt.values[idx++].Reg64 = env->eflags;
 257
 258     /* Translate 6+4 segment registers. HV and QEMU order matches  */
 259     assert(idx == WHvX64RegisterEs);
 260     for (i = 0; i < 6; i += 1, idx += 1) {
 261         vcxt.values[idx].Segment = whpx_seg_q2h(&env->segs[i], v86, r86);
 262     }
 263
 264     assert(idx == WHvX64RegisterLdtr);
 265     vcxt.values[idx++].Segment = whpx_seg_q2h(&env->ldt, 0, 0);
 266
 267     assert(idx == WHvX64RegisterTr);
 268     vcxt.values[idx++].Segment = whpx_seg_q2h(&env->tr, 0, 0);
 269
 270     assert(idx == WHvX64RegisterIdtr);
 271     vcxt.values[idx].Table.Base = env->idt.base;
 272     vcxt.values[idx].Table.Limit = env->idt.limit;
 273     idx += 1;
 274
 275     assert(idx == WHvX64RegisterGdtr);
 276     vcxt.values[idx].Table.Base = env->gdt.base;
 277     vcxt.values[idx].Table.Limit = env->gdt.limit;
 278     idx += 1;
 279
 280     /* CR0, 2, 3, 4, 8 */
 281     assert(whpx_register_names[idx] == WHvX64RegisterCr0);
 282     vcxt.values[idx++].Reg64 = env->cr[0];
 283     assert(whpx_register_names[idx] == WHvX64RegisterCr2);
 284     vcxt.values[idx++].Reg64 = env->cr[2];
 285     assert(whpx_register_names[idx] == WHvX64RegisterCr3);
 286     vcxt.values[idx++].Reg64 = env->cr[3];
 287     assert(whpx_register_names[idx] == WHvX64RegisterCr4);
 288     vcxt.values[idx++].Reg64 = env->cr[4];
 289     assert(whpx_register_names[idx] == WHvX64RegisterCr8);
 290     vcxt.values[idx++].Reg64 = vcpu->tpr;
 291
 292     /* 8 Debug Registers - Skipped */
 293
 294     /* 16 XMM registers */
 295     assert(whpx_register_names[idx] == WHvX64RegisterXmm0);
 296     idx_next = idx + 16;
 297     for (i = 0; i < sizeof(env->xmm_regs) / sizeof(ZMMReg); i += 1, idx += 1) {
 298         vcxt.values[idx].Reg128.Low64 = env->xmm_regs[i].ZMM_Q(0);
 299         vcxt.values[idx].Reg128.High64 = env->xmm_regs[i].ZMM_Q(1);
 300     }
 301     idx = idx_next;
 302
 303     /* 8 FP registers */
 304     assert(whpx_register_names[idx] == WHvX64RegisterFpMmx0);
 305     for (i = 0; i < 8; i += 1, idx += 1) {
 306         vcxt.values[idx].Fp.AsUINT128.Low64 = env->fpregs[i].mmx.MMX_Q(0);
 307         /* vcxt.values[idx].Fp.AsUINT128.High64 =
 308                env->fpregs[i].mmx.MMX_Q(1);
 309         */
 310     }
 311
 312     /* FP control status register */
 313     assert(whpx_register_names[idx] == WHvX64RegisterFpControlStatus);
 314     vcxt.values[idx].FpControlStatus.FpControl = env->fpuc;
 315     vcxt.values[idx].FpControlStatus.FpStatus =
 316         (env->fpus & ~0x3800) | (env->fpstt & 0x7) << 11;
 317     vcxt.values[idx].FpControlStatus.FpTag = 0;
 318     for (i = 0; i < 8; ++i) {
 319         vcxt.values[idx].FpControlStatus.FpTag |= (!env->fptags[i]) << i;
 320     }
 321     vcxt.values[idx].FpControlStatus.Reserved = 0;
 322     vcxt.values[idx].FpControlStatus.LastFpOp = env->fpop;
 323     vcxt.values[idx].FpControlStatus.LastFpRip = env->fpip;
 324     idx += 1;
 325
 326     /* XMM control status register */
 327     assert(whpx_register_names[idx] == WHvX64RegisterXmmControlStatus);
 328     vcxt.values[idx].XmmControlStatus.LastFpRdp = 0;
 329     vcxt.values[idx].XmmControlStatus.XmmStatusControl = env->mxcsr;
 330     vcxt.values[idx].XmmControlStatus.XmmStatusControlMask = 0x0000ffff;
 331     idx += 1;
 332
 333     /* MSRs */
 334     assert(whpx_register_names[idx] == WHvX64RegisterTsc);
 335     vcxt.values[idx++].Reg64 = env->tsc;
 336     assert(whpx_register_names[idx] == WHvX64RegisterEfer);
 337     vcxt.values[idx++].Reg64 = env->efer;
 338 #ifdef TARGET_X86_64
 339     assert(whpx_register_names[idx] == WHvX64RegisterKernelGsBase);
 340     vcxt.values[idx++].Reg64 = env->kernelgsbase;
 341 #endif
 342
 343     assert(whpx_register_names[idx] == WHvX64RegisterApicBase);
 344     vcxt.values[idx++].Reg64 = vcpu->apic_base;
 345
 346     /* WHvX64RegisterPat - Skipped */
 347
 348     assert(whpx_register_names[idx] == WHvX64RegisterSysenterCs);
 349     vcxt.values[idx++].Reg64 = env->sysenter_cs;
 350     assert(whpx_register_names[idx] == WHvX64RegisterSysenterEip);
 351     vcxt.values[idx++].Reg64 = env->sysenter_eip;
 352     assert(whpx_register_names[idx] == WHvX64RegisterSysenterEsp);
 353     vcxt.values[idx++].Reg64 = env->sysenter_esp;
 354     assert(whpx_register_names[idx] == WHvX64RegisterStar);
 355     vcxt.values[idx++].Reg64 = env->star;
 356 #ifdef TARGET_X86_64
 357     assert(whpx_register_names[idx] == WHvX64RegisterLstar);
 358     vcxt.values[idx++].Reg64 = env->lstar;
 359     assert(whpx_register_names[idx] == WHvX64RegisterCstar);
 360     vcxt.values[idx++].Reg64 = env->cstar;
 361     assert(whpx_register_names[idx] == WHvX64RegisterSfmask);
 362     vcxt.values[idx++].Reg64 = env->fmask;
 363 #endif
 364
 365     /* Interrupt / Event Registers - Skipped */
 366
 367     assert(idx == RTL_NUMBER_OF(whpx_register_names));
 368
 369     hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
 370         whpx->partition, cpu->cpu_index,
 371         whpx_register_names,
 372         RTL_NUMBER_OF(whpx_register_names),
 373         &vcxt.values[0]);
 374
 375     if (FAILED(hr)) {
 376         error_report("WHPX: Failed to set virtual processor context, hr=%08lx",
 377                      hr);
 378     }
 379
 380     return;
 381 }
 382
 383 static void whpx_get_registers(CPUState *cpu)
 384 {
 385     struct whpx_state *whpx = &whpx_global;
 386     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
 387     struct CPUX86State *env = (CPUArchState *)(cpu->env_ptr);
 388     X86CPU *x86_cpu = X86_CPU(cpu);
 389     struct whpx_register_set vcxt;
 390     uint64_t tpr, apic_base;
 391     HRESULT hr;
 392     int idx;
 393     int idx_next;
 394     int i;
 395
 396     assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
 397
 398     hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
 399         whpx->partition, cpu->cpu_index,
 400         whpx_register_names,
 401         RTL_NUMBER_OF(whpx_register_names),
 402         &vcxt.values[0]);
 403     if (FAILED(hr)) {
 404         error_report("WHPX: Failed to get virtual processor context, hr=%08lx",
 405                      hr);
 406     }
 407
 408     idx = 0;
 409
 410     /* Indexes for first 16 registers match between HV and QEMU definitions */
 411     idx_next = 16;
 412     for (idx = 0; idx < CPU_NB_REGS; idx += 1) {
 413         env->regs[idx] = vcxt.values[idx].Reg64;
 414     }
 415     idx = idx_next;
 416
 417     /* Same goes for RIP and RFLAGS */
 418     assert(whpx_register_names[idx] == WHvX64RegisterRip);
 419     env->eip = vcxt.values[idx++].Reg64;
 420     assert(whpx_register_names[idx] == WHvX64RegisterRflags);
 421     env->eflags = vcxt.values[idx++].Reg64;
 422
 423     /* Translate 6+4 segment registers. HV and QEMU order matches  */
 424     assert(idx == WHvX64RegisterEs);
 425     for (i = 0; i < 6; i += 1, idx += 1) {
 426         env->segs[i] = whpx_seg_h2q(&vcxt.values[idx].Segment);
 427     }
 428
 429     assert(idx == WHvX64RegisterLdtr);
 430     env->ldt = whpx_seg_h2q(&vcxt.values[idx++].Segment);
 431     assert(idx == WHvX64RegisterTr);
 432     env->tr = whpx_seg_h2q(&vcxt.values[idx++].Segment);
 433     assert(idx == WHvX64RegisterIdtr);
 434     env->idt.base = vcxt.values[idx].Table.Base;
 435     env->idt.limit = vcxt.values[idx].Table.Limit;
 436     idx += 1;
 437     assert(idx == WHvX64RegisterGdtr);
 438     env->gdt.base = vcxt.values[idx].Table.Base;
 439     env->gdt.limit = vcxt.values[idx].Table.Limit;
 440     idx += 1;
 441
 442     /* CR0, 2, 3, 4, 8 */
 443     assert(whpx_register_names[idx] == WHvX64RegisterCr0);
 444     env->cr[0] = vcxt.values[idx++].Reg64;
 445     assert(whpx_register_names[idx] == WHvX64RegisterCr2);
 446     env->cr[2] = vcxt.values[idx++].Reg64;
 447     assert(whpx_register_names[idx] == WHvX64RegisterCr3);
 448     env->cr[3] = vcxt.values[idx++].Reg64;
 449     assert(whpx_register_names[idx] == WHvX64RegisterCr4);
 450     env->cr[4] = vcxt.values[idx++].Reg64;
 451     assert(whpx_register_names[idx] == WHvX64RegisterCr8);
 452     tpr = vcxt.values[idx++].Reg64;
 453     if (tpr != vcpu->tpr) {
 454         vcpu->tpr = tpr;
 455         cpu_set_apic_tpr(x86_cpu->apic_state, tpr);
 456     }
 457
 458     /* 8 Debug Registers - Skipped */
 459
 460     /* 16 XMM registers */
 461     assert(whpx_register_names[idx] == WHvX64RegisterXmm0);
 462     idx_next = idx + 16;
 463     for (i = 0; i < sizeof(env->xmm_regs) / sizeof(ZMMReg); i += 1, idx += 1) {
 464         env->xmm_regs[i].ZMM_Q(0) = vcxt.values[idx].Reg128.Low64;
 465         env->xmm_regs[i].ZMM_Q(1) = vcxt.values[idx].Reg128.High64;
 466     }
 467     idx = idx_next;
 468
 469     /* 8 FP registers */
 470     assert(whpx_register_names[idx] == WHvX64RegisterFpMmx0);
 471     for (i = 0; i < 8; i += 1, idx += 1) {
 472         env->fpregs[i].mmx.MMX_Q(0) = vcxt.values[idx].Fp.AsUINT128.Low64;
 473         /* env->fpregs[i].mmx.MMX_Q(1) =
 474                vcxt.values[idx].Fp.AsUINT128.High64;
 475         */
 476     }
 477
 478     /* FP control status register */
 479     assert(whpx_register_names[idx] == WHvX64RegisterFpControlStatus);
 480     env->fpuc = vcxt.values[idx].FpControlStatus.FpControl;
 481     env->fpstt = (vcxt.values[idx].FpControlStatus.FpStatus >> 11) & 0x7;
 482     env->fpus = vcxt.values[idx].FpControlStatus.FpStatus & ~0x3800;
 483     for (i = 0; i < 8; ++i) {
 484         env->fptags[i] = !((vcxt.values[idx].FpControlStatus.FpTag >> i) & 1);
 485     }
 486     env->fpop = vcxt.values[idx].FpControlStatus.LastFpOp;
 487     env->fpip = vcxt.values[idx].FpControlStatus.LastFpRip;
 488     idx += 1;
 489
 490     /* XMM control status register */
 491     assert(whpx_register_names[idx] == WHvX64RegisterXmmControlStatus);
 492     env->mxcsr = vcxt.values[idx].XmmControlStatus.XmmStatusControl;
 493     idx += 1;
 494
 495     /* MSRs */
 496     assert(whpx_register_names[idx] == WHvX64RegisterTsc);
 497     env->tsc = vcxt.values[idx++].Reg64;
 498     assert(whpx_register_names[idx] == WHvX64RegisterEfer);
 499     env->efer = vcxt.values[idx++].Reg64;
 500 #ifdef TARGET_X86_64
 501     assert(whpx_register_names[idx] == WHvX64RegisterKernelGsBase);
 502     env->kernelgsbase = vcxt.values[idx++].Reg64;
 503 #endif
 504
 505     assert(whpx_register_names[idx] == WHvX64RegisterApicBase);
 506     apic_base = vcxt.values[idx++].Reg64;
 507     if (apic_base != vcpu->apic_base) {
 508         vcpu->apic_base = apic_base;
 509         cpu_set_apic_base(x86_cpu->apic_state, vcpu->apic_base);
 510     }
 511
 512     /* WHvX64RegisterPat - Skipped */
 513
 514     assert(whpx_register_names[idx] == WHvX64RegisterSysenterCs);
 515     env->sysenter_cs = vcxt.values[idx++].Reg64;;
 516     assert(whpx_register_names[idx] == WHvX64RegisterSysenterEip);
 517     env->sysenter_eip = vcxt.values[idx++].Reg64;
 518     assert(whpx_register_names[idx] == WHvX64RegisterSysenterEsp);
 519     env->sysenter_esp = vcxt.values[idx++].Reg64;
 520     assert(whpx_register_names[idx] == WHvX64RegisterStar);
 521     env->star = vcxt.values[idx++].Reg64;
 522 #ifdef TARGET_X86_64
 523     assert(whpx_register_names[idx] == WHvX64RegisterLstar);
 524     env->lstar = vcxt.values[idx++].Reg64;
 525     assert(whpx_register_names[idx] == WHvX64RegisterCstar);
 526     env->cstar = vcxt.values[idx++].Reg64;
 527     assert(whpx_register_names[idx] == WHvX64RegisterSfmask);
 528     env->fmask = vcxt.values[idx++].Reg64;
 529 #endif
 530
 531     /* Interrupt / Event Registers - Skipped */
 532
 533     assert(idx == RTL_NUMBER_OF(whpx_register_names));
 534
 535     return;
 536 }
 537
 538 static HRESULT CALLBACK whpx_emu_ioport_callback(
 539     void *ctx,
 540     WHV_EMULATOR_IO_ACCESS_INFO *IoAccess)
 541 {
 542     MemTxAttrs attrs = { 0 };
 543     address_space_rw(&address_space_io, IoAccess->Port, attrs,
 544                      (uint8_t *)&IoAccess->Data, IoAccess->AccessSize,
 545                      IoAccess->Direction);
 546     return S_OK;
 547 }
 548
 549 static HRESULT CALLBACK whpx_emu_mmio_callback(
 550     void *ctx,
 551     WHV_EMULATOR_MEMORY_ACCESS_INFO *ma)
 552 {
 553     cpu_physical_memory_rw(ma->GpaAddress, ma->Data, ma->AccessSize,
 554                            ma->Direction);
 555     return S_OK;
 556 }
 557
 558 static HRESULT CALLBACK whpx_emu_getreg_callback(
 559     void *ctx,
 560     const WHV_REGISTER_NAME *RegisterNames,
 561     UINT32 RegisterCount,
 562     WHV_REGISTER_VALUE *RegisterValues)
 563 {
 564     HRESULT hr;
 565     struct whpx_state *whpx = &whpx_global;
 566     CPUState *cpu = (CPUState *)ctx;
 567
 568     hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
 569         whpx->partition, cpu->cpu_index,
 570         RegisterNames, RegisterCount,
 571         RegisterValues);
 572     if (FAILED(hr)) {
 573         error_report("WHPX: Failed to get virtual processor registers,"
 574                      " hr=%08lx", hr);
 575     }
 576
 577     return hr;
 578 }
 579
 580 static HRESULT CALLBACK whpx_emu_setreg_callback(
 581     void *ctx,
 582     const WHV_REGISTER_NAME *RegisterNames,
 583     UINT32 RegisterCount,
 584     const WHV_REGISTER_VALUE *RegisterValues)
 585 {
 586     HRESULT hr;
 587     struct whpx_state *whpx = &whpx_global;
 588     CPUState *cpu = (CPUState *)ctx;
 589
 590     hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
 591         whpx->partition, cpu->cpu_index,
 592         RegisterNames, RegisterCount,
 593         RegisterValues);
 594     if (FAILED(hr)) {
 595         error_report("WHPX: Failed to set virtual processor registers,"
 596                      " hr=%08lx", hr);
 597     }
 598
 599     /*
 600      * The emulator just successfully wrote the register state. We clear the
 601      * dirty state so we avoid the double write on resume of the VP.
 602      */
 603     cpu->vcpu_dirty = false;
 604
 605     return hr;
 606 }
 607
 608 static HRESULT CALLBACK whpx_emu_translate_callback(
 609     void *ctx,
 610     WHV_GUEST_VIRTUAL_ADDRESS Gva,
 611     WHV_TRANSLATE_GVA_FLAGS TranslateFlags,
 612     WHV_TRANSLATE_GVA_RESULT_CODE *TranslationResult,
 613     WHV_GUEST_PHYSICAL_ADDRESS *Gpa)
 614 {
 615     HRESULT hr;
 616     struct whpx_state *whpx = &whpx_global;
 617     CPUState *cpu = (CPUState *)ctx;
 618     WHV_TRANSLATE_GVA_RESULT res;
 619
 620     hr = whp_dispatch.WHvTranslateGva(whpx->partition, cpu->cpu_index,
 621                                       Gva, TranslateFlags, &res, Gpa);
 622     if (FAILED(hr)) {
 623         error_report("WHPX: Failed to translate GVA, hr=%08lx", hr);
 624     } else {
 625         *TranslationResult = res.ResultCode;
 626     }
 627
 628     return hr;
 629 }
 630
 631 static const WHV_EMULATOR_CALLBACKS whpx_emu_callbacks = {
 632     .Size = sizeof(WHV_EMULATOR_CALLBACKS),
 633     .WHvEmulatorIoPortCallback = whpx_emu_ioport_callback,
 634     .WHvEmulatorMemoryCallback = whpx_emu_mmio_callback,
 635     .WHvEmulatorGetVirtualProcessorRegisters = whpx_emu_getreg_callback,
 636     .WHvEmulatorSetVirtualProcessorRegisters = whpx_emu_setreg_callback,
 637     .WHvEmulatorTranslateGvaPage = whpx_emu_translate_callback,
 638 };
 639
 640 static int whpx_handle_mmio(CPUState *cpu, WHV_MEMORY_ACCESS_CONTEXT *ctx)
 641 {
 642     HRESULT hr;
 643     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
 644     WHV_EMULATOR_STATUS emu_status;
 645
 646     hr = whp_dispatch.WHvEmulatorTryMmioEmulation(
 647         vcpu->emulator, cpu,
 648         &vcpu->exit_ctx.VpContext, ctx,
 649         &emu_status);
 650     if (FAILED(hr)) {
 651         error_report("WHPX: Failed to parse MMIO access, hr=%08lx", hr);
 652         return -1;
 653     }
 654
 655     if (!emu_status.EmulationSuccessful) {
 656         error_report("WHPX: Failed to emulate MMIO access with"
 657                      " EmulatorReturnStatus: %u", emu_status.AsUINT32);
 658         return -1;
 659     }
 660
 661     return 0;
 662 }
 663
 664 static int whpx_handle_portio(CPUState *cpu,
 665                               WHV_X64_IO_PORT_ACCESS_CONTEXT *ctx)
 666 {
 667     HRESULT hr;
 668     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
 669     WHV_EMULATOR_STATUS emu_status;
 670
 671     hr = whp_dispatch.WHvEmulatorTryIoEmulation(
 672         vcpu->emulator, cpu,
 673         &vcpu->exit_ctx.VpContext, ctx,
 674         &emu_status);
 675     if (FAILED(hr)) {
 676         error_report("WHPX: Failed to parse PortIO access, hr=%08lx", hr);
 677         return -1;
 678     }
 679
 680     if (!emu_status.EmulationSuccessful) {
 681         error_report("WHPX: Failed to emulate PortIO access with"
 682                      " EmulatorReturnStatus: %u", emu_status.AsUINT32);
 683         return -1;
 684     }
 685
 686     return 0;
 687 }
 688
 689 static int whpx_handle_halt(CPUState *cpu)
 690 {
 691     struct CPUX86State *env = (CPUArchState *)(cpu->env_ptr);
 692     int ret = 0;
 693
 694     qemu_mutex_lock_iothread();
 695     if (!((cpu->interrupt_request & CPU_INTERRUPT_HARD) &&
 696           (env->eflags & IF_MASK)) &&
 697         !(cpu->interrupt_request & CPU_INTERRUPT_NMI)) {
 698         cpu->exception_index = EXCP_HLT;
 699         cpu->halted = true;
 700         ret = 1;
 701     }
 702     qemu_mutex_unlock_iothread();
 703
 704     return ret;
 705 }
 706
 707 static void whpx_vcpu_pre_run(CPUState *cpu)
 708 {
 709     HRESULT hr;
 710     struct whpx_state *whpx = &whpx_global;
 711     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
 712     struct CPUX86State *env = (CPUArchState *)(cpu->env_ptr);
 713     X86CPU *x86_cpu = X86_CPU(cpu);
 714     int irq;
 715     uint8_t tpr;
 716     WHV_X64_PENDING_INTERRUPTION_REGISTER new_int;
 717     UINT32 reg_count = 0;
 718     WHV_REGISTER_VALUE reg_values[3];
 719     WHV_REGISTER_NAME reg_names[3];
 720
 721     memset(&new_int, 0, sizeof(new_int));
 722     memset(reg_values, 0, sizeof(reg_values));
 723
 724     qemu_mutex_lock_iothread();
 725
 726     /* Inject NMI */
 727     if (!vcpu->interruption_pending &&
 728         cpu->interrupt_request & (CPU_INTERRUPT_NMI | CPU_INTERRUPT_SMI)) {
 729         if (cpu->interrupt_request & CPU_INTERRUPT_NMI) {
 730             cpu->interrupt_request &= ~CPU_INTERRUPT_NMI;
 731             vcpu->interruptable = false;
 732             new_int.InterruptionType = WHvX64PendingNmi;
 733             new_int.InterruptionPending = 1;
 734             new_int.InterruptionVector = 2;
 735         }
 736         if (cpu->interrupt_request & CPU_INTERRUPT_SMI) {
 737             cpu->interrupt_request &= ~CPU_INTERRUPT_SMI;
 738         }
 739     }
 740
 741     /*
 742      * Force the VCPU out of its inner loop to process any INIT requests or
 743      * commit pending TPR access.
 744      */
 745     if (cpu->interrupt_request & (CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)) {
 746         if ((cpu->interrupt_request & CPU_INTERRUPT_INIT) &&
 747             !(env->hflags & HF_SMM_MASK)) {
 748             cpu->exit_request = 1;
 749         }
 750         if (cpu->interrupt_request & CPU_INTERRUPT_TPR) {
 751             cpu->exit_request = 1;
 752         }
 753     }
 754
 755     /* Get pending hard interruption or replay one that was overwritten */
 756     if (!vcpu->interruption_pending &&
 757         vcpu->interruptable && (env->eflags & IF_MASK)) {
 758         assert(!new_int.InterruptionPending);
 759         if (cpu->interrupt_request & CPU_INTERRUPT_HARD) {
 760             cpu->interrupt_request &= ~CPU_INTERRUPT_HARD;
 761             irq = cpu_get_pic_interrupt(env);
 762             if (irq >= 0) {
 763                 new_int.InterruptionType = WHvX64PendingInterrupt;
 764                 new_int.InterruptionPending = 1;
 765                 new_int.InterruptionVector = irq;
 766             }
 767         }
 768     }
 769
 770     /* Setup interrupt state if new one was prepared */
 771     if (new_int.InterruptionPending) {
 772         reg_values[reg_count].PendingInterruption = new_int;
 773         reg_names[reg_count] = WHvRegisterPendingInterruption;
 774         reg_count += 1;
 775     }
 776
 777     /* Sync the TPR to the CR8 if was modified during the intercept */
 778     tpr = cpu_get_apic_tpr(x86_cpu->apic_state);
 779     if (tpr != vcpu->tpr) {
 780         vcpu->tpr = tpr;
 781         reg_values[reg_count].Reg64 = tpr;
 782         cpu->exit_request = 1;
 783         reg_names[reg_count] = WHvX64RegisterCr8;
 784         reg_count += 1;
 785     }
 786
 787     /* Update the state of the interrupt delivery notification */
 788     if (!vcpu->window_registered &&
 789         cpu->interrupt_request & CPU_INTERRUPT_HARD) {
 790         reg_values[reg_count].DeliverabilityNotifications.InterruptNotification
 791             = 1;
 792         vcpu->window_registered = 1;
 793         reg_names[reg_count] = WHvX64RegisterDeliverabilityNotifications;
 794         reg_count += 1;
 795     }
 796
 797     qemu_mutex_unlock_iothread();
 798
 799     if (reg_count) {
 800         hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
 801             whpx->partition, cpu->cpu_index,
 802             reg_names, reg_count, reg_values);
 803         if (FAILED(hr)) {
 804             error_report("WHPX: Failed to set interrupt state registers,"
 805                          " hr=%08lx", hr);
 806         }
 807     }
 808
 809     return;
 810 }
 811
 812 static void whpx_vcpu_post_run(CPUState *cpu)
 813 {
 814     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
 815     struct CPUX86State *env = (CPUArchState *)(cpu->env_ptr);
 816     X86CPU *x86_cpu = X86_CPU(cpu);
 817
 818     env->eflags = vcpu->exit_ctx.VpContext.Rflags;
 819
 820     uint64_t tpr = vcpu->exit_ctx.VpContext.Cr8;
 821     if (vcpu->tpr != tpr) {
 822         vcpu->tpr = tpr;
 823         qemu_mutex_lock_iothread();
 824         cpu_set_apic_tpr(x86_cpu->apic_state, vcpu->tpr);
 825         qemu_mutex_unlock_iothread();
 826     }
 827
 828     vcpu->interruption_pending =
 829         vcpu->exit_ctx.VpContext.ExecutionState.InterruptionPending;
 830
 831     vcpu->interruptable =
 832         !vcpu->exit_ctx.VpContext.ExecutionState.InterruptShadow;
 833
 834     return;
 835 }
 836
 837 static void whpx_vcpu_process_async_events(CPUState *cpu)
 838 {
 839     struct CPUX86State *env = (CPUArchState *)(cpu->env_ptr);
 840     X86CPU *x86_cpu = X86_CPU(cpu);
 841     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
 842
 843     if ((cpu->interrupt_request & CPU_INTERRUPT_INIT) &&
 844         !(env->hflags & HF_SMM_MASK)) {
 845
 846         do_cpu_init(x86_cpu);
 847         cpu->vcpu_dirty = true;
 848         vcpu->interruptable = true;
 849     }
 850
 851     if (cpu->interrupt_request & CPU_INTERRUPT_POLL) {
 852         cpu->interrupt_request &= ~CPU_INTERRUPT_POLL;
 853         apic_poll_irq(x86_cpu->apic_state);
 854     }
 855
 856     if (((cpu->interrupt_request & CPU_INTERRUPT_HARD) &&
 857          (env->eflags & IF_MASK)) ||
 858         (cpu->interrupt_request & CPU_INTERRUPT_NMI)) {
 859         cpu->halted = false;
 860     }
 861
 862     if (cpu->interrupt_request & CPU_INTERRUPT_SIPI) {
 863         if (!cpu->vcpu_dirty) {
 864             whpx_get_registers(cpu);
 865         }
 866         do_cpu_sipi(x86_cpu);
 867     }
 868
 869     if (cpu->interrupt_request & CPU_INTERRUPT_TPR) {
 870         cpu->interrupt_request &= ~CPU_INTERRUPT_TPR;
 871         if (!cpu->vcpu_dirty) {
 872             whpx_get_registers(cpu);
 873         }
 874         apic_handle_tpr_access_report(x86_cpu->apic_state, env->eip,
 875                                       env->tpr_access_type);
 876     }
 877
 878     return;
 879 }
 880
 881 static int whpx_vcpu_run(CPUState *cpu)
 882 {
 883     HRESULT hr;
 884     struct whpx_state *whpx = &whpx_global;
 885     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
 886     int ret;
 887
 888     whpx_vcpu_process_async_events(cpu);
 889     if (cpu->halted) {
 890         cpu->exception_index = EXCP_HLT;
 891         atomic_set(&cpu->exit_request, false);
 892         return 0;
 893     }
 894
 895     qemu_mutex_unlock_iothread();
 896     cpu_exec_start(cpu);
 897
 898     do {
 899         if (cpu->vcpu_dirty) {
 900             whpx_set_registers(cpu);
 901             cpu->vcpu_dirty = false;
 902         }
 903
 904         whpx_vcpu_pre_run(cpu);
 905
 906         if (atomic_read(&cpu->exit_request)) {
 907             whpx_vcpu_kick(cpu);
 908         }
 909
 910         hr = whp_dispatch.WHvRunVirtualProcessor(
 911             whpx->partition, cpu->cpu_index,
 912             &vcpu->exit_ctx, sizeof(vcpu->exit_ctx));
 913
 914         if (FAILED(hr)) {
 915             error_report("WHPX: Failed to exec a virtual processor,"
 916                          " hr=%08lx", hr);
 917             ret = -1;
 918             break;
 919         }
 920
 921         whpx_vcpu_post_run(cpu);
 922
 923         switch (vcpu->exit_ctx.ExitReason) {
 924         case WHvRunVpExitReasonMemoryAccess:
 925             ret = whpx_handle_mmio(cpu, &vcpu->exit_ctx.MemoryAccess);
 926             break;
 927
 928         case WHvRunVpExitReasonX64IoPortAccess:
 929             ret = whpx_handle_portio(cpu, &vcpu->exit_ctx.IoPortAccess);
 930             break;
 931
 932         case WHvRunVpExitReasonX64InterruptWindow:
 933             vcpu->window_registered = 0;
 934             ret = 0;
 935             break;
 936
 937         case WHvRunVpExitReasonX64Halt:
 938             ret = whpx_handle_halt(cpu);
 939             break;
 940
 941         case WHvRunVpExitReasonCanceled:
 942             cpu->exception_index = EXCP_INTERRUPT;
 943             ret = 1;
 944             break;
 945
 946         case WHvRunVpExitReasonX64MsrAccess: {
 947             WHV_REGISTER_VALUE reg_values[3] = {0};
 948             WHV_REGISTER_NAME reg_names[3];
 949             UINT32 reg_count;
 950
 951             reg_names[0] = WHvX64RegisterRip;
 952             reg_names[1] = WHvX64RegisterRax;
 953             reg_names[2] = WHvX64RegisterRdx;
 954
 955             reg_values[0].Reg64 =
 956                 vcpu->exit_ctx.VpContext.Rip +
 957                 vcpu->exit_ctx.VpContext.InstructionLength;
 958
 959             /*
 960              * For all unsupported MSR access we:
 961              *     ignore writes
 962              *     return 0 on read.
 963              */
 964             reg_count = vcpu->exit_ctx.MsrAccess.AccessInfo.IsWrite ?
 965                         1 : 3;
 966
 967             hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
 968                 whpx->partition,
 969                 cpu->cpu_index,
 970                 reg_names, reg_count,
 971                 reg_values);
 972
 973             if (FAILED(hr)) {
 974                 error_report("WHPX: Failed to set MsrAccess state "
 975                              " registers, hr=%08lx", hr);
 976             }
 977             ret = 0;
 978             break;
 979         }
 980         case WHvRunVpExitReasonX64Cpuid: {
 981             WHV_REGISTER_VALUE reg_values[5];
 982             WHV_REGISTER_NAME reg_names[5];
 983             UINT32 reg_count = 5;
 984             UINT64 rip, rax, rcx, rdx, rbx;
 985
 986             memset(reg_values, 0, sizeof(reg_values));
 987
 988             rip = vcpu->exit_ctx.VpContext.Rip +
 989                   vcpu->exit_ctx.VpContext.InstructionLength;
 990             switch (vcpu->exit_ctx.CpuidAccess.Rax) {
 991             case 1:
 992                 rax = vcpu->exit_ctx.CpuidAccess.DefaultResultRax;
 993                 /* Advertise that we are running on a hypervisor */
 994                 rcx =
 995                     vcpu->exit_ctx.CpuidAccess.DefaultResultRcx |
 996                     CPUID_EXT_HYPERVISOR;
 997
 998                 rdx = vcpu->exit_ctx.CpuidAccess.DefaultResultRdx;
 999                 rbx = vcpu->exit_ctx.CpuidAccess.DefaultResultRbx;
1000                 break;
1001             case 0x80000001:
1002                 rax = vcpu->exit_ctx.CpuidAccess.DefaultResultRax;
1003                 /* Remove any support of OSVW */
1004                 rcx =
1005                     vcpu->exit_ctx.CpuidAccess.DefaultResultRcx &
1006                     ~CPUID_EXT3_OSVW;
1007
1008                 rdx = vcpu->exit_ctx.CpuidAccess.DefaultResultRdx;
1009                 rbx = vcpu->exit_ctx.CpuidAccess.DefaultResultRbx;
1010                 break;
1011             default:
1012                 rax = vcpu->exit_ctx.CpuidAccess.DefaultResultRax;
1013                 rcx = vcpu->exit_ctx.CpuidAccess.DefaultResultRcx;
1014                 rdx = vcpu->exit_ctx.CpuidAccess.DefaultResultRdx;
1015                 rbx = vcpu->exit_ctx.CpuidAccess.DefaultResultRbx;
1016             }
1017
1018             reg_names[0] = WHvX64RegisterRip;
1019             reg_names[1] = WHvX64RegisterRax;
1020             reg_names[2] = WHvX64RegisterRcx;
1021             reg_names[3] = WHvX64RegisterRdx;
1022             reg_names[4] = WHvX64RegisterRbx;
1023
1024             reg_values[0].Reg64 = rip;
1025             reg_values[1].Reg64 = rax;
1026             reg_values[2].Reg64 = rcx;
1027             reg_values[3].Reg64 = rdx;
1028             reg_values[4].Reg64 = rbx;
1029
1030             hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
1031                 whpx->partition, cpu->cpu_index,
1032                 reg_names,
1033                 reg_count,
1034                 reg_values);
1035
1036             if (FAILED(hr)) {
1037                 error_report("WHPX: Failed to set CpuidAccess state registers,"
1038                              " hr=%08lx", hr);
1039             }
1040             ret = 0;
1041             break;
1042         }
1043         case WHvRunVpExitReasonNone:
1044         case WHvRunVpExitReasonUnrecoverableException:
1045         case WHvRunVpExitReasonInvalidVpRegisterValue:
1046         case WHvRunVpExitReasonUnsupportedFeature:
1047         case WHvRunVpExitReasonException:
1048         default:
1049             error_report("WHPX: Unexpected VP exit code %d",
1050                          vcpu->exit_ctx.ExitReason);
1051             whpx_get_registers(cpu);
1052             qemu_mutex_lock_iothread();
1053             qemu_system_guest_panicked(cpu_get_crash_info(cpu));
1054             qemu_mutex_unlock_iothread();
1055             break;
1056         }
1057
1058     } while (!ret);
1059
1060     cpu_exec_end(cpu);
1061     qemu_mutex_lock_iothread();
1062     current_cpu = cpu;
1063
1064     atomic_set(&cpu->exit_request, false);
1065
1066     return ret < 0;
1067 }
1068
1069 static void do_whpx_cpu_synchronize_state(CPUState *cpu, run_on_cpu_data arg)
1070 {
1071     whpx_get_registers(cpu);
1072     cpu->vcpu_dirty = true;
1073 }
1074
1075 static void do_whpx_cpu_synchronize_post_reset(CPUState *cpu,
1076                                                run_on_cpu_data arg)
1077 {
1078     whpx_set_registers(cpu);
1079     cpu->vcpu_dirty = false;
1080 }
1081
1082 static void do_whpx_cpu_synchronize_post_init(CPUState *cpu,
1083                                               run_on_cpu_data arg)
1084 {
1085     whpx_set_registers(cpu);
1086     cpu->vcpu_dirty = false;
1087 }
1088
1089 static void do_whpx_cpu_synchronize_pre_loadvm(CPUState *cpu,
1090                                                run_on_cpu_data arg)
1091 {
1092     cpu->vcpu_dirty = true;
1093 }
1094
1095 /*
1096  * CPU support.
1097  */
1098
1099 void whpx_cpu_synchronize_state(CPUState *cpu)
1100 {
1101     if (!cpu->vcpu_dirty) {
1102         run_on_cpu(cpu, do_whpx_cpu_synchronize_state, RUN_ON_CPU_NULL);
1103     }
1104 }
1105
1106 void whpx_cpu_synchronize_post_reset(CPUState *cpu)
1107 {
1108     run_on_cpu(cpu, do_whpx_cpu_synchronize_post_reset, RUN_ON_CPU_NULL);
1109 }
1110
1111 void whpx_cpu_synchronize_post_init(CPUState *cpu)
1112 {
1113     run_on_cpu(cpu, do_whpx_cpu_synchronize_post_init, RUN_ON_CPU_NULL);
1114 }
1115
1116 void whpx_cpu_synchronize_pre_loadvm(CPUState *cpu)
1117 {
1118     run_on_cpu(cpu, do_whpx_cpu_synchronize_pre_loadvm, RUN_ON_CPU_NULL);
1119 }
1120
1121 /*
1122  * Vcpu support.
1123  */
1124
1125 static Error *whpx_migration_blocker;
1126
1127 int whpx_init_vcpu(CPUState *cpu)
1128 {
1129     HRESULT hr;
1130     struct whpx_state *whpx = &whpx_global;
1131     struct whpx_vcpu *vcpu;
1132     Error *local_error = NULL;
1133
1134     /* Add migration blockers for all unsupported features of the
1135      * Windows Hypervisor Platform
1136      */
1137     if (whpx_migration_blocker == NULL) {
1138         error_setg(&whpx_migration_blocker,
1139                "State blocked due to non-migratable CPUID feature support,"
1140                "dirty memory tracking support, and XSAVE/XRSTOR support");
1141
1142         (void)migrate_add_blocker(whpx_migration_blocker, &local_error);
1143         if (local_error) {
1144             error_report_err(local_error);
1145             migrate_del_blocker(whpx_migration_blocker);
1146             error_free(whpx_migration_blocker);
1147             return -EINVAL;
1148         }
1149     }
1150
1151     vcpu = g_malloc0(sizeof(struct whpx_vcpu));
1152
1153     if (!vcpu) {
1154         error_report("WHPX: Failed to allocte VCPU context.");
1155         return -ENOMEM;
1156     }
1157
1158     hr = whp_dispatch.WHvEmulatorCreateEmulator(
1159         &whpx_emu_callbacks,
1160         &vcpu->emulator);
1161     if (FAILED(hr)) {
1162         error_report("WHPX: Failed to setup instruction completion support,"
1163                      " hr=%08lx", hr);
1164         g_free(vcpu);
1165         return -EINVAL;
1166     }
1167
1168     hr = whp_dispatch.WHvCreateVirtualProcessor(
1169         whpx->partition, cpu->cpu_index, 0);
1170     if (FAILED(hr)) {
1171         error_report("WHPX: Failed to create a virtual processor,"
1172                      " hr=%08lx", hr);
1173         whp_dispatch.WHvEmulatorDestroyEmulator(vcpu->emulator);
1174         g_free(vcpu);
1175         return -EINVAL;
1176     }
1177
1178     vcpu->interruptable = true;
1179
1180     cpu->vcpu_dirty = true;
1181     cpu->hax_vcpu = (struct hax_vcpu_state *)vcpu;
1182
1183     return 0;
1184 }
1185
1186 int whpx_vcpu_exec(CPUState *cpu)
1187 {
1188     int ret;
1189     int fatal;
1190
1191     for (;;) {
1192         if (cpu->exception_index >= EXCP_INTERRUPT) {
1193             ret = cpu->exception_index;
1194             cpu->exception_index = -1;
1195             break;
1196         }
1197
1198         fatal = whpx_vcpu_run(cpu);
1199
1200         if (fatal) {
1201             error_report("WHPX: Failed to exec a virtual processor");
1202             abort();
1203         }
1204     }
1205
1206     return ret;
1207 }
1208
1209 void whpx_destroy_vcpu(CPUState *cpu)
1210 {
1211     struct whpx_state *whpx = &whpx_global;
1212     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
1213
1214     whp_dispatch.WHvDeleteVirtualProcessor(whpx->partition, cpu->cpu_index);
1215     whp_dispatch.WHvEmulatorDestroyEmulator(vcpu->emulator);
1216     g_free(cpu->hax_vcpu);
1217     return;
1218 }
1219
1220 void whpx_vcpu_kick(CPUState *cpu)
1221 {
1222     struct whpx_state *whpx = &whpx_global;
1223     whp_dispatch.WHvCancelRunVirtualProcessor(
1224         whpx->partition, cpu->cpu_index, 0);
1225 }
1226
1227 /*
1228  * Memory support.
1229  */
1230
1231 static void whpx_update_mapping(hwaddr start_pa, ram_addr_t size,
1232                                 void *host_va, int add, int rom,
1233                                 const char *name)
1234 {
1235     struct whpx_state *whpx = &whpx_global;
1236     HRESULT hr;
1237
1238     /*
1239     if (add) {
1240         printf("WHPX: ADD PA:%p Size:%p, Host:%p, %s, '%s'\n",
1241                (void*)start_pa, (void*)size, host_va,
1242                (rom ? "ROM" : "RAM"), name);
1243     } else {
1244         printf("WHPX: DEL PA:%p Size:%p, Host:%p,      '%s'\n",
1245                (void*)start_pa, (void*)size, host_va, name);
1246     }
1247     */
1248
1249     if (add) {
1250         hr = whp_dispatch.WHvMapGpaRange(whpx->partition,
1251                                          host_va,
1252                                          start_pa,
1253                                          size,
1254                                          (WHvMapGpaRangeFlagRead |
1255                                           WHvMapGpaRangeFlagExecute |
1256                                           (rom ? 0 : WHvMapGpaRangeFlagWrite)));
1257     } else {
1258         hr = whp_dispatch.WHvUnmapGpaRange(whpx->partition,
1259                                            start_pa,
1260                                            size);
1261     }
1262
1263     if (FAILED(hr)) {
1264         error_report("WHPX: Failed to %s GPA range '%s' PA:%p, Size:%p bytes,"
1265                      " Host:%p, hr=%08lx",
1266                      (add ? "MAP" : "UNMAP"), name,
1267                      (void *)(uintptr_t)start_pa, (void *)size, host_va, hr);
1268     }
1269 }
1270
1271 static void whpx_process_section(MemoryRegionSection *section, int add)
1272 {
1273     MemoryRegion *mr = section->mr;
1274     hwaddr start_pa = section->offset_within_address_space;
1275     ram_addr_t size = int128_get64(section->size);
1276     unsigned int delta;
1277     uint64_t host_va;
1278
1279     if (!memory_region_is_ram(mr)) {
1280         return;
1281     }
1282
1283     delta = qemu_real_host_page_size - (start_pa & ~qemu_real_host_page_mask);
1284     delta &= ~qemu_real_host_page_mask;
1285     if (delta > size) {
1286         return;
1287     }
1288     start_pa += delta;
1289     size -= delta;
1290     size &= qemu_real_host_page_mask;
1291     if (!size || (start_pa & ~qemu_real_host_page_mask)) {
1292         return;
1293     }
1294
1295     host_va = (uintptr_t)memory_region_get_ram_ptr(mr)
1296             + section->offset_within_region + delta;
1297
1298     whpx_update_mapping(start_pa, size, (void *)(uintptr_t)host_va, add,
1299                         memory_region_is_rom(mr), mr->name);
1300 }
1301
1302 static void whpx_region_add(MemoryListener *listener,
1303                            MemoryRegionSection *section)
1304 {
1305     memory_region_ref(section->mr);
1306     whpx_process_section(section, 1);
1307 }
1308
1309 static void whpx_region_del(MemoryListener *listener,
1310                            MemoryRegionSection *section)
1311 {
1312     whpx_process_section(section, 0);
1313     memory_region_unref(section->mr);
1314 }
1315
1316 static void whpx_transaction_begin(MemoryListener *listener)
1317 {
1318 }
1319
1320 static void whpx_transaction_commit(MemoryListener *listener)
1321 {
1322 }
1323
1324 static void whpx_log_sync(MemoryListener *listener,
1325                          MemoryRegionSection *section)
1326 {
1327     MemoryRegion *mr = section->mr;
1328
1329     if (!memory_region_is_ram(mr)) {
1330         return;
1331     }
1332
1333     memory_region_set_dirty(mr, 0, int128_get64(section->size));
1334 }
1335
1336 static MemoryListener whpx_memory_listener = {
1337     .begin = whpx_transaction_begin,
1338     .commit = whpx_transaction_commit,
1339     .region_add = whpx_region_add,
1340     .region_del = whpx_region_del,
1341     .log_sync = whpx_log_sync,
1342     .priority = 10,
1343 };
1344
1345 static void whpx_memory_init(void)
1346 {
1347     memory_listener_register(&whpx_memory_listener, &address_space_memory);
1348 }
1349
1350 static void whpx_handle_interrupt(CPUState *cpu, int mask)
1351 {
1352     cpu->interrupt_request |= mask;
1353
1354     if (!qemu_cpu_is_self(cpu)) {
1355         qemu_cpu_kick(cpu);
1356     }
1357 }
1358
1359 /*
1360  * Partition support
1361  */
1362
1363 static int whpx_accel_init(MachineState *ms)
1364 {
1365     struct whpx_state *whpx;
1366     int ret;
1367     HRESULT hr;
1368     WHV_CAPABILITY whpx_cap;
1369     UINT32 whpx_cap_size;
1370     WHV_PARTITION_PROPERTY prop;
1371
1372     whpx = &whpx_global;
1373
1374     if (!init_whp_dispatch()) {
1375         ret = -ENOSYS;
1376         goto error;
1377     }
1378
1379     memset(whpx, 0, sizeof(struct whpx_state));
1380     whpx->mem_quota = ms->ram_size;
1381
1382     hr = whp_dispatch.WHvGetCapability(
1383         WHvCapabilityCodeHypervisorPresent, &whpx_cap,
1384         sizeof(whpx_cap), &whpx_cap_size);
1385     if (FAILED(hr) || !whpx_cap.HypervisorPresent) {
1386         error_report("WHPX: No accelerator found, hr=%08lx", hr);
1387         ret = -ENOSPC;
1388         goto error;
1389     }
1390
1391     hr = whp_dispatch.WHvCreatePartition(&whpx->partition);
1392     if (FAILED(hr)) {
1393         error_report("WHPX: Failed to create partition, hr=%08lx", hr);
1394         ret = -EINVAL;
1395         goto error;
1396     }
1397
1398     memset(&prop, 0, sizeof(WHV_PARTITION_PROPERTY));
1399     prop.ProcessorCount = smp_cpus;
1400     hr = whp_dispatch.WHvSetPartitionProperty(
1401         whpx->partition,
1402         WHvPartitionPropertyCodeProcessorCount,
1403         &prop,
1404         sizeof(WHV_PARTITION_PROPERTY));
1405
1406     if (FAILED(hr)) {
1407         error_report("WHPX: Failed to set partition core count to %d,"
1408                      " hr=%08lx", smp_cores, hr);
1409         ret = -EINVAL;
1410         goto error;
1411     }
1412
1413     memset(&prop, 0, sizeof(WHV_PARTITION_PROPERTY));
1414     prop.ExtendedVmExits.X64MsrExit = 1;
1415     prop.ExtendedVmExits.X64CpuidExit = 1;
1416     hr = whp_dispatch.WHvSetPartitionProperty(
1417         whpx->partition,
1418         WHvPartitionPropertyCodeExtendedVmExits,
1419         &prop,
1420         sizeof(WHV_PARTITION_PROPERTY));
1421
1422     if (FAILED(hr)) {
1423         error_report("WHPX: Failed to enable partition extended X64MsrExit and"
1424                      " X64CpuidExit hr=%08lx", hr);
1425         ret = -EINVAL;
1426         goto error;
1427     }
1428
1429     UINT32 cpuidExitList[] = {1, 0x80000001};
1430     hr = whp_dispatch.WHvSetPartitionProperty(
1431         whpx->partition,
1432         WHvPartitionPropertyCodeCpuidExitList,
1433         cpuidExitList,
1434         RTL_NUMBER_OF(cpuidExitList) * sizeof(UINT32));
1435
1436     if (FAILED(hr)) {
1437         error_report("WHPX: Failed to set partition CpuidExitList hr=%08lx",
1438                      hr);
1439         ret = -EINVAL;
1440         goto error;
1441     }
1442
1443     hr = whp_dispatch.WHvSetupPartition(whpx->partition);
1444     if (FAILED(hr)) {
1445         error_report("WHPX: Failed to setup partition, hr=%08lx", hr);
1446         ret = -EINVAL;
1447         goto error;
1448     }
1449
1450     whpx_memory_init();
1451
1452     cpu_interrupt_handler = whpx_handle_interrupt;
1453
1454     printf("Windows Hypervisor Platform accelerator is operational\n");
1455     return 0;
1456
1457   error:
1458
1459     if (NULL != whpx->partition) {
1460         whp_dispatch.WHvDeletePartition(whpx->partition);
1461         whpx->partition = NULL;
1462     }
1463
1464
1465     return ret;
1466 }
1467
1468 int whpx_enabled(void)
1469 {
1470     return whpx_allowed;
1471 }
1472
1473 static void whpx_accel_class_init(ObjectClass *oc, void *data)
1474 {
1475     AccelClass *ac = ACCEL_CLASS(oc);
1476     ac->name = "WHPX";
1477     ac->init_machine = whpx_accel_init;
1478     ac->allowed = &whpx_allowed;
1479 }
1480
1481 static const TypeInfo whpx_accel_type = {
1482     .name = ACCEL_CLASS_NAME("whpx"),
1483     .parent = TYPE_ACCEL,
1484     .class_init = whpx_accel_class_init,
1485 };
1486
1487 static void whpx_type_init(void)
1488 {
1489     type_register_static(&whpx_accel_type);
1490 }
1491
1492 bool init_whp_dispatch(void)
1493 {
1494     const char *lib_name;
1495     HMODULE hLib;
1496
1497     if (whp_dispatch_initialized) {
1498         return true;
1499     }
1500
1501     #define WHP_LOAD_FIELD(return_type, function_name, signature) \
1502         whp_dispatch.function_name = \
1503             (function_name ## _t)GetProcAddress(hLib, #function_name); \
1504         if (!whp_dispatch.function_name) { \
1505             error_report("Could not load function %s from library %s.", \
1506                          #function_name, lib_name); \
1507             goto error; \
1508         } \
1509
1510     lib_name = "WinHvPlatform.dll";
1511     hWinHvPlatform = LoadLibrary(lib_name);
1512     if (!hWinHvPlatform) {
1513         error_report("Could not load library %s.", lib_name);
1514         goto error;
1515     }
1516     hLib = hWinHvPlatform;
1517     LIST_WINHVPLATFORM_FUNCTIONS(WHP_LOAD_FIELD)
1518
1519     lib_name = "WinHvEmulation.dll";
1520     hWinHvEmulation = LoadLibrary(lib_name);
1521     if (!hWinHvEmulation) {
1522         error_report("Could not load library %s.", lib_name);
1523         goto error;
1524     }
1525     hLib = hWinHvEmulation;
1526     LIST_WINHVEMULATION_FUNCTIONS(WHP_LOAD_FIELD)
1527
1528     whp_dispatch_initialized = true;
1529     return true;
1530
1531     error:
1532
1533     if (hWinHvPlatform) {
1534         FreeLibrary(hWinHvPlatform);
1535     }
1536     if (hWinHvEmulation) {
1537         FreeLibrary(hWinHvEmulation);
1538     }
1539     return false;
1540 }
1541
1542 type_init(whpx_type_init);