cpus.c

   1 /*
   2  * QEMU System Emulator
   3  *
   4  * Copyright (c) 2003-2008 Fabrice Bellard
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to deal
   8  * in the Software without restriction, including without limitation the rights
   9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10  * copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22  * THE SOFTWARE.
  23  */
  24
  25 #include "qemu/osdep.h"
  26 #include "qemu/config-file.h"
  27 #include "cpu.h"
  28 #include "monitor/monitor.h"
  29 #include "qapi/error.h"
  30 #include "qapi/qapi-commands-misc.h"
  31 #include "qapi/qapi-events-run-state.h"
  32 #include "qapi/qmp/qerror.h"
  33 #include "qemu/error-report.h"
  34 #include "sysemu/sysemu.h"
  35 #include "sysemu/block-backend.h"
  36 #include "exec/gdbstub.h"
  37 #include "sysemu/dma.h"
  38 #include "sysemu/hw_accel.h"
  39 #include "sysemu/kvm.h"
  40 #include "sysemu/hax.h"
  41 #include "sysemu/hvf.h"
  42 #include "sysemu/whpx.h"
  43 #include "exec/exec-all.h"
  44
  45 #include "qemu/thread.h"
  46 #include "sysemu/cpus.h"
  47 #include "sysemu/qtest.h"
  48 #include "qemu/main-loop.h"
  49 #include "qemu/option.h"
  50 #include "qemu/bitmap.h"
  51 #include "qemu/seqlock.h"
  52 #include "tcg.h"
  53 #include "hw/nmi.h"
  54 #include "sysemu/replay.h"
  55 #include "hw/boards.h"
  56
  57 #ifdef CONFIG_LINUX
  58
  59 #include <sys/prctl.h>
  60
  61 #ifndef PR_MCE_KILL
  62 #define PR_MCE_KILL 33
  63 #endif
  64
  65 #ifndef PR_MCE_KILL_SET
  66 #define PR_MCE_KILL_SET 1
  67 #endif
  68
  69 #ifndef PR_MCE_KILL_EARLY
  70 #define PR_MCE_KILL_EARLY 1
  71 #endif
  72
  73 #endif /* CONFIG_LINUX */
  74
  75 int64_t max_delay;
  76 int64_t max_advance;
  77
  78 /* vcpu throttling controls */
  79 static QEMUTimer *throttle_timer;
  80 static unsigned int throttle_percentage;
  81
  82 #define CPU_THROTTLE_PCT_MIN 1
  83 #define CPU_THROTTLE_PCT_MAX 99
  84 #define CPU_THROTTLE_TIMESLICE_NS 10000000
  85
  86 bool cpu_is_stopped(CPUState *cpu)
  87 {
  88     return cpu->stopped || !runstate_is_running();
  89 }
  90
  91 static bool cpu_thread_is_idle(CPUState *cpu)
  92 {
  93     if (cpu->stop || cpu->queued_work_first) {
  94         return false;
  95     }
  96     if (cpu_is_stopped(cpu)) {
  97         return true;
  98     }
  99     if (!cpu->halted || cpu_has_work(cpu) ||
 100         kvm_halt_in_kernel()) {
 101         return false;
 102     }
 103     return true;
 104 }
 105
 106 static bool all_cpu_threads_idle(void)
 107 {
 108     CPUState *cpu;
 109
 110     CPU_FOREACH(cpu) {
 111         if (!cpu_thread_is_idle(cpu)) {
 112             return false;
 113         }
 114     }
 115     return true;
 116 }
 117
 118 /***********************************************************/
 119 /* guest cycle counter */
 120
 121 /* Protected by TimersState seqlock */
 122
 123 static bool icount_sleep = true;
 124 /* Arbitrarily pick 1MIPS as the minimum allowable speed.  */
 125 #define MAX_ICOUNT_SHIFT 10
 126
 127 typedef struct TimersState {
 128     /* Protected by BQL.  */
 129     int64_t cpu_ticks_prev;
 130     int64_t cpu_ticks_offset;
 131
 132     /* Protect fields that can be respectively read outside the
 133      * BQL, and written from multiple threads.
 134      */
 135     QemuSeqLock vm_clock_seqlock;
 136     QemuSpin vm_clock_lock;
 137
 138     int16_t cpu_ticks_enabled;
 139
 140     /* Conversion factor from emulated instructions to virtual clock ticks.  */
 141     int16_t icount_time_shift;
 142
 143     /* Compensate for varying guest execution speed.  */
 144     int64_t qemu_icount_bias;
 145
 146     int64_t vm_clock_warp_start;
 147     int64_t cpu_clock_offset;
 148
 149     /* Only written by TCG thread */
 150     int64_t qemu_icount;
 151
 152     /* for adjusting icount */
 153     QEMUTimer *icount_rt_timer;
 154     QEMUTimer *icount_vm_timer;
 155     QEMUTimer *icount_warp_timer;
 156 } TimersState;
 157
 158 static TimersState timers_state;
 159 bool mttcg_enabled;
 160
 161 /*
 162  * We default to false if we know other options have been enabled
 163  * which are currently incompatible with MTTCG. Otherwise when each
 164  * guest (target) has been updated to support:
 165  *   - atomic instructions
 166  *   - memory ordering primitives (barriers)
 167  * they can set the appropriate CONFIG flags in ${target}-softmmu.mak
 168  *
 169  * Once a guest architecture has been converted to the new primitives
 170  * there are two remaining limitations to check.
 171  *
 172  * - The guest can't be oversized (e.g. 64 bit guest on 32 bit host)
 173  * - The host must have a stronger memory order than the guest
 174  *
 175  * It may be possible in future to support strong guests on weak hosts
 176  * but that will require tagging all load/stores in a guest with their
 177  * implicit memory order requirements which would likely slow things
 178  * down a lot.
 179  */
 180
 181 static bool check_tcg_memory_orders_compatible(void)
 182 {
 183 #if defined(TCG_GUEST_DEFAULT_MO) && defined(TCG_TARGET_DEFAULT_MO)
 184     return (TCG_GUEST_DEFAULT_MO & ~TCG_TARGET_DEFAULT_MO) == 0;
 185 #else
 186     return false;
 187 #endif
 188 }
 189
 190 static bool default_mttcg_enabled(void)
 191 {
 192     if (use_icount || TCG_OVERSIZED_GUEST) {
 193         return false;
 194     } else {
 195 #ifdef TARGET_SUPPORTS_MTTCG
 196         return check_tcg_memory_orders_compatible();
 197 #else
 198         return false;
 199 #endif
 200     }
 201 }
 202
 203 void qemu_tcg_configure(QemuOpts *opts, Error **errp)
 204 {
 205     const char *t = qemu_opt_get(opts, "thread");
 206     if (t) {
 207         if (strcmp(t, "multi") == 0) {
 208             if (TCG_OVERSIZED_GUEST) {
 209                 error_setg(errp, "No MTTCG when guest word size > hosts");
 210             } else if (use_icount) {
 211                 error_setg(errp, "No MTTCG when icount is enabled");
 212             } else {
 213 #ifndef TARGET_SUPPORTS_MTTCG
 214                 error_report("Guest not yet converted to MTTCG - "
 215                              "you may get unexpected results");
 216 #endif
 217                 if (!check_tcg_memory_orders_compatible()) {
 218                     error_report("Guest expects a stronger memory ordering "
 219                                  "than the host provides");
 220                     error_printf("This may cause strange/hard to debug errors\n");
 221                 }
 222                 mttcg_enabled = true;
 223             }
 224         } else if (strcmp(t, "single") == 0) {
 225             mttcg_enabled = false;
 226         } else {
 227             error_setg(errp, "Invalid 'thread' setting %s", t);
 228         }
 229     } else {
 230         mttcg_enabled = default_mttcg_enabled();
 231     }
 232 }
 233
 234 /* The current number of executed instructions is based on what we
 235  * originally budgeted minus the current state of the decrementing
 236  * icount counters in extra/u16.low.
 237  */
 238 static int64_t cpu_get_icount_executed(CPUState *cpu)
 239 {
 240     return cpu->icount_budget - (cpu->icount_decr.u16.low + cpu->icount_extra);
 241 }
 242
 243 /*
 244  * Update the global shared timer_state.qemu_icount to take into
 245  * account executed instructions. This is done by the TCG vCPU
 246  * thread so the main-loop can see time has moved forward.
 247  */
 248 static void cpu_update_icount_locked(CPUState *cpu)
 249 {
 250     int64_t executed = cpu_get_icount_executed(cpu);
 251     cpu->icount_budget -= executed;
 252
 253     atomic_set_i64(&timers_state.qemu_icount,
 254                    timers_state.qemu_icount + executed);
 255 }
 256
 257 /*
 258  * Update the global shared timer_state.qemu_icount to take into
 259  * account executed instructions. This is done by the TCG vCPU
 260  * thread so the main-loop can see time has moved forward.
 261  */
 262 void cpu_update_icount(CPUState *cpu)
 263 {
 264     seqlock_write_lock(&timers_state.vm_clock_seqlock,
 265                        &timers_state.vm_clock_lock);
 266     cpu_update_icount_locked(cpu);
 267     seqlock_write_unlock(&timers_state.vm_clock_seqlock,
 268                          &timers_state.vm_clock_lock);
 269 }
 270
 271 static int64_t cpu_get_icount_raw_locked(void)
 272 {
 273     CPUState *cpu = current_cpu;
 274
 275     if (cpu && cpu->running) {
 276         if (!cpu->can_do_io) {
 277             error_report("Bad icount read");
 278             exit(1);
 279         }
 280         /* Take into account what has run */
 281         cpu_update_icount_locked(cpu);
 282     }
 283     /* The read is protected by the seqlock, but needs atomic64 to avoid UB */
 284     return atomic_read_i64(&timers_state.qemu_icount);
 285 }
 286
 287 static int64_t cpu_get_icount_locked(void)
 288 {
 289     int64_t icount = cpu_get_icount_raw_locked();
 290     return atomic_read_i64(&timers_state.qemu_icount_bias) +
 291         cpu_icount_to_ns(icount);
 292 }
 293
 294 int64_t cpu_get_icount_raw(void)
 295 {
 296     int64_t icount;
 297     unsigned start;
 298
 299     do {
 300         start = seqlock_read_begin(&timers_state.vm_clock_seqlock);
 301         icount = cpu_get_icount_raw_locked();
 302     } while (seqlock_read_retry(&timers_state.vm_clock_seqlock, start));
 303
 304     return icount;
 305 }
 306
 307 /* Return the virtual CPU time, based on the instruction counter.  */
 308 int64_t cpu_get_icount(void)
 309 {
 310     int64_t icount;
 311     unsigned start;
 312
 313     do {
 314         start = seqlock_read_begin(&timers_state.vm_clock_seqlock);
 315         icount = cpu_get_icount_locked();
 316     } while (seqlock_read_retry(&timers_state.vm_clock_seqlock, start));
 317
 318     return icount;
 319 }
 320
 321 int64_t cpu_icount_to_ns(int64_t icount)
 322 {
 323     return icount << atomic_read(&timers_state.icount_time_shift);
 324 }
 325
 326 static int64_t cpu_get_ticks_locked(void)
 327 {
 328     int64_t ticks = timers_state.cpu_ticks_offset;
 329     if (timers_state.cpu_ticks_enabled) {
 330         ticks += cpu_get_host_ticks();
 331     }
 332
 333     if (timers_state.cpu_ticks_prev > ticks) {
 334         /* Non increasing ticks may happen if the host uses software suspend.  */
 335         timers_state.cpu_ticks_offset += timers_state.cpu_ticks_prev - ticks;
 336         ticks = timers_state.cpu_ticks_prev;
 337     }
 338
 339     timers_state.cpu_ticks_prev = ticks;
 340     return ticks;
 341 }
 342
 343 /* return the time elapsed in VM between vm_start and vm_stop.  Unless
 344  * icount is active, cpu_get_ticks() uses units of the host CPU cycle
 345  * counter.
 346  */
 347 int64_t cpu_get_ticks(void)
 348 {
 349     int64_t ticks;
 350
 351     if (use_icount) {
 352         return cpu_get_icount();
 353     }
 354
 355     qemu_spin_lock(&timers_state.vm_clock_lock);
 356     ticks = cpu_get_ticks_locked();
 357     qemu_spin_unlock(&timers_state.vm_clock_lock);
 358     return ticks;
 359 }
 360
 361 static int64_t cpu_get_clock_locked(void)
 362 {
 363     int64_t time;
 364
 365     time = timers_state.cpu_clock_offset;
 366     if (timers_state.cpu_ticks_enabled) {
 367         time += get_clock();
 368     }
 369
 370     return time;
 371 }
 372
 373 /* Return the monotonic time elapsed in VM, i.e.,
 374  * the time between vm_start and vm_stop
 375  */
 376 int64_t cpu_get_clock(void)
 377 {
 378     int64_t ti;
 379     unsigned start;
 380
 381     do {
 382         start = seqlock_read_begin(&timers_state.vm_clock_seqlock);
 383         ti = cpu_get_clock_locked();
 384     } while (seqlock_read_retry(&timers_state.vm_clock_seqlock, start));
 385
 386     return ti;
 387 }
 388
 389 /* enable cpu_get_ticks()
 390  * Caller must hold BQL which serves as mutex for vm_clock_seqlock.
 391  */
 392 void cpu_enable_ticks(void)
 393 {
 394     seqlock_write_lock(&timers_state.vm_clock_seqlock,
 395                        &timers_state.vm_clock_lock);
 396     if (!timers_state.cpu_ticks_enabled) {
 397         timers_state.cpu_ticks_offset -= cpu_get_host_ticks();
 398         timers_state.cpu_clock_offset -= get_clock();
 399         timers_state.cpu_ticks_enabled = 1;
 400     }
 401     seqlock_write_unlock(&timers_state.vm_clock_seqlock,
 402                        &timers_state.vm_clock_lock);
 403 }
 404
 405 /* disable cpu_get_ticks() : the clock is stopped. You must not call
 406  * cpu_get_ticks() after that.
 407  * Caller must hold BQL which serves as mutex for vm_clock_seqlock.
 408  */
 409 void cpu_disable_ticks(void)
 410 {
 411     seqlock_write_lock(&timers_state.vm_clock_seqlock,
 412                        &timers_state.vm_clock_lock);
 413     if (timers_state.cpu_ticks_enabled) {
 414         timers_state.cpu_ticks_offset += cpu_get_host_ticks();
 415         timers_state.cpu_clock_offset = cpu_get_clock_locked();
 416         timers_state.cpu_ticks_enabled = 0;
 417     }
 418     seqlock_write_unlock(&timers_state.vm_clock_seqlock,
 419                          &timers_state.vm_clock_lock);
 420 }
 421
 422 /* Correlation between real and virtual time is always going to be
 423    fairly approximate, so ignore small variation.
 424    When the guest is idle real and virtual time will be aligned in
 425    the IO wait loop.  */
 426 #define ICOUNT_WOBBLE (NANOSECONDS_PER_SECOND / 10)
 427
 428 static void icount_adjust(void)
 429 {
 430     int64_t cur_time;
 431     int64_t cur_icount;
 432     int64_t delta;
 433
 434     /* Protected by TimersState mutex.  */
 435     static int64_t last_delta;
 436
 437     /* If the VM is not running, then do nothing.  */
 438     if (!runstate_is_running()) {
 439         return;
 440     }
 441
 442     seqlock_write_lock(&timers_state.vm_clock_seqlock,
 443                        &timers_state.vm_clock_lock);
 444     cur_time = cpu_get_clock_locked();
 445     cur_icount = cpu_get_icount_locked();
 446
 447     delta = cur_icount - cur_time;
 448     /* FIXME: This is a very crude algorithm, somewhat prone to oscillation.  */
 449     if (delta > 0
 450         && last_delta + ICOUNT_WOBBLE < delta * 2
 451         && timers_state.icount_time_shift > 0) {
 452         /* The guest is getting too far ahead.  Slow time down.  */
 453         atomic_set(&timers_state.icount_time_shift,
 454                    timers_state.icount_time_shift - 1);
 455     }
 456     if (delta < 0
 457         && last_delta - ICOUNT_WOBBLE > delta * 2
 458         && timers_state.icount_time_shift < MAX_ICOUNT_SHIFT) {
 459         /* The guest is getting too far behind.  Speed time up.  */
 460         atomic_set(&timers_state.icount_time_shift,
 461                    timers_state.icount_time_shift + 1);
 462     }
 463     last_delta = delta;
 464     atomic_set_i64(&timers_state.qemu_icount_bias,
 465                    cur_icount - (timers_state.qemu_icount
 466                                  << timers_state.icount_time_shift));
 467     seqlock_write_unlock(&timers_state.vm_clock_seqlock,
 468                          &timers_state.vm_clock_lock);
 469 }
 470
 471 static void icount_adjust_rt(void *opaque)
 472 {
 473     timer_mod(timers_state.icount_rt_timer,
 474               qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL_RT) + 1000);
 475     icount_adjust();
 476 }
 477
 478 static void icount_adjust_vm(void *opaque)
 479 {
 480     timer_mod(timers_state.icount_vm_timer,
 481                    qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
 482                    NANOSECONDS_PER_SECOND / 10);
 483     icount_adjust();
 484 }
 485
 486 static int64_t qemu_icount_round(int64_t count)
 487 {
 488     int shift = atomic_read(&timers_state.icount_time_shift);
 489     return (count + (1 << shift) - 1) >> shift;
 490 }
 491
 492 static void icount_warp_rt(void)
 493 {
 494     unsigned seq;
 495     int64_t warp_start;
 496
 497     /* The icount_warp_timer is rescheduled soon after vm_clock_warp_start
 498      * changes from -1 to another value, so the race here is okay.
 499      */
 500     do {
 501         seq = seqlock_read_begin(&timers_state.vm_clock_seqlock);
 502         warp_start = timers_state.vm_clock_warp_start;
 503     } while (seqlock_read_retry(&timers_state.vm_clock_seqlock, seq));
 504
 505     if (warp_start == -1) {
 506         return;
 507     }
 508
 509     seqlock_write_lock(&timers_state.vm_clock_seqlock,
 510                        &timers_state.vm_clock_lock);
 511     if (runstate_is_running()) {
 512         int64_t clock = REPLAY_CLOCK(REPLAY_CLOCK_VIRTUAL_RT,
 513                                      cpu_get_clock_locked());
 514         int64_t warp_delta;
 515
 516         warp_delta = clock - timers_state.vm_clock_warp_start;
 517         if (use_icount == 2) {
 518             /*
 519              * In adaptive mode, do not let QEMU_CLOCK_VIRTUAL run too
 520              * far ahead of real time.
 521              */
 522             int64_t cur_icount = cpu_get_icount_locked();
 523             int64_t delta = clock - cur_icount;
 524             warp_delta = MIN(warp_delta, delta);
 525         }
 526         atomic_set_i64(&timers_state.qemu_icount_bias,
 527                        timers_state.qemu_icount_bias + warp_delta);
 528     }
 529     timers_state.vm_clock_warp_start = -1;
 530     seqlock_write_unlock(&timers_state.vm_clock_seqlock,
 531                        &timers_state.vm_clock_lock);
 532
 533     if (qemu_clock_expired(QEMU_CLOCK_VIRTUAL)) {
 534         qemu_clock_notify(QEMU_CLOCK_VIRTUAL);
 535     }
 536 }
 537
 538 static void icount_timer_cb(void *opaque)
 539 {
 540     /* No need for a checkpoint because the timer already synchronizes
 541      * with CHECKPOINT_CLOCK_VIRTUAL_RT.
 542      */
 543     icount_warp_rt();
 544 }
 545
 546 void qtest_clock_warp(int64_t dest)
 547 {
 548     int64_t clock = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
 549     AioContext *aio_context;
 550     assert(qtest_enabled());
 551     aio_context = qemu_get_aio_context();
 552     while (clock < dest) {
 553         int64_t deadline = qemu_clock_deadline_ns_all(QEMU_CLOCK_VIRTUAL);
 554         int64_t warp = qemu_soonest_timeout(dest - clock, deadline);
 555
 556         seqlock_write_lock(&timers_state.vm_clock_seqlock,
 557                            &timers_state.vm_clock_lock);
 558         atomic_set_i64(&timers_state.qemu_icount_bias,
 559                        timers_state.qemu_icount_bias + warp);
 560         seqlock_write_unlock(&timers_state.vm_clock_seqlock,
 561                              &timers_state.vm_clock_lock);
 562
 563         qemu_clock_run_timers(QEMU_CLOCK_VIRTUAL);
 564         timerlist_run_timers(aio_context->tlg.tl[QEMU_CLOCK_VIRTUAL]);
 565         clock = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
 566     }
 567     qemu_clock_notify(QEMU_CLOCK_VIRTUAL);
 568 }
 569
 570 void qemu_start_warp_timer(void)
 571 {
 572     int64_t clock;
 573     int64_t deadline;
 574
 575     if (!use_icount) {
 576         return;
 577     }
 578
 579     /* Nothing to do if the VM is stopped: QEMU_CLOCK_VIRTUAL timers
 580      * do not fire, so computing the deadline does not make sense.
 581      */
 582     if (!runstate_is_running()) {
 583         return;
 584     }
 585
 586     if (replay_mode != REPLAY_MODE_PLAY) {
 587         if (!all_cpu_threads_idle()) {
 588             return;
 589         }
 590
 591         if (qtest_enabled()) {
 592             /* When testing, qtest commands advance icount.  */
 593             return;
 594         }
 595
 596         replay_checkpoint(CHECKPOINT_CLOCK_WARP_START);
 597     } else {
 598         /* warp clock deterministically in record/replay mode */
 599         if (!replay_checkpoint(CHECKPOINT_CLOCK_WARP_START)) {
 600             /* vCPU is sleeping and warp can't be started.
 601                It is probably a race condition: notification sent
 602                to vCPU was processed in advance and vCPU went to sleep.
 603                Therefore we have to wake it up for doing someting. */
 604             if (replay_has_checkpoint()) {
 605                 qemu_clock_notify(QEMU_CLOCK_VIRTUAL);
 606             }
 607             return;
 608         }
 609     }
 610
 611     /* We want to use the earliest deadline from ALL vm_clocks */
 612     clock = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL_RT);
 613     deadline = qemu_clock_deadline_ns_all(QEMU_CLOCK_VIRTUAL);
 614     if (deadline < 0) {
 615         static bool notified;
 616         if (!icount_sleep && !notified) {
 617             warn_report("icount sleep disabled and no active timers");
 618             notified = true;
 619         }
 620         return;
 621     }
 622
 623     if (deadline > 0) {
 624         /*
 625          * Ensure QEMU_CLOCK_VIRTUAL proceeds even when the virtual CPU goes to
 626          * sleep.  Otherwise, the CPU might be waiting for a future timer
 627          * interrupt to wake it up, but the interrupt never comes because
 628          * the vCPU isn't running any insns and thus doesn't advance the
 629          * QEMU_CLOCK_VIRTUAL.
 630          */
 631         if (!icount_sleep) {
 632             /*
 633              * We never let VCPUs sleep in no sleep icount mode.
 634              * If there is a pending QEMU_CLOCK_VIRTUAL timer we just advance
 635              * to the next QEMU_CLOCK_VIRTUAL event and notify it.
 636              * It is useful when we want a deterministic execution time,
 637              * isolated from host latencies.
 638              */
 639             seqlock_write_lock(&timers_state.vm_clock_seqlock,
 640                                &timers_state.vm_clock_lock);
 641             atomic_set_i64(&timers_state.qemu_icount_bias,
 642                            timers_state.qemu_icount_bias + deadline);
 643             seqlock_write_unlock(&timers_state.vm_clock_seqlock,
 644                                  &timers_state.vm_clock_lock);
 645             qemu_clock_notify(QEMU_CLOCK_VIRTUAL);
 646         } else {
 647             /*
 648              * We do stop VCPUs and only advance QEMU_CLOCK_VIRTUAL after some
 649              * "real" time, (related to the time left until the next event) has
 650              * passed. The QEMU_CLOCK_VIRTUAL_RT clock will do this.
 651              * This avoids that the warps are visible externally; for example,
 652              * you will not be sending network packets continuously instead of
 653              * every 100ms.
 654              */
 655             seqlock_write_lock(&timers_state.vm_clock_seqlock,
 656                                &timers_state.vm_clock_lock);
 657             if (timers_state.vm_clock_warp_start == -1
 658                 || timers_state.vm_clock_warp_start > clock) {
 659                 timers_state.vm_clock_warp_start = clock;
 660             }
 661             seqlock_write_unlock(&timers_state.vm_clock_seqlock,
 662                                  &timers_state.vm_clock_lock);
 663             timer_mod_anticipate(timers_state.icount_warp_timer,
 664                                  clock + deadline);
 665         }
 666     } else if (deadline == 0) {
 667         qemu_clock_notify(QEMU_CLOCK_VIRTUAL);
 668     }
 669 }
 670
 671 static void qemu_account_warp_timer(void)
 672 {
 673     if (!use_icount || !icount_sleep) {
 674         return;
 675     }
 676
 677     /* Nothing to do if the VM is stopped: QEMU_CLOCK_VIRTUAL timers
 678      * do not fire, so computing the deadline does not make sense.
 679      */
 680     if (!runstate_is_running()) {
 681         return;
 682     }
 683
 684     /* warp clock deterministically in record/replay mode */
 685     if (!replay_checkpoint(CHECKPOINT_CLOCK_WARP_ACCOUNT)) {
 686         return;
 687     }
 688
 689     timer_del(timers_state.icount_warp_timer);
 690     icount_warp_rt();
 691 }
 692
 693 static bool icount_state_needed(void *opaque)
 694 {
 695     return use_icount;
 696 }
 697
 698 static bool warp_timer_state_needed(void *opaque)
 699 {
 700     TimersState *s = opaque;
 701     return s->icount_warp_timer != NULL;
 702 }
 703
 704 static bool adjust_timers_state_needed(void *opaque)
 705 {
 706     TimersState *s = opaque;
 707     return s->icount_rt_timer != NULL;
 708 }
 709
 710 /*
 711  * Subsection for warp timer migration is optional, because may not be created
 712  */
 713 static const VMStateDescription icount_vmstate_warp_timer = {
 714     .name = "timer/icount/warp_timer",
 715     .version_id = 1,
 716     .minimum_version_id = 1,
 717     .needed = warp_timer_state_needed,
 718     .fields = (VMStateField[]) {
 719         VMSTATE_INT64(vm_clock_warp_start, TimersState),
 720         VMSTATE_TIMER_PTR(icount_warp_timer, TimersState),
 721         VMSTATE_END_OF_LIST()
 722     }
 723 };
 724
 725 static const VMStateDescription icount_vmstate_adjust_timers = {
 726     .name = "timer/icount/timers",
 727     .version_id = 1,
 728     .minimum_version_id = 1,
 729     .needed = adjust_timers_state_needed,
 730     .fields = (VMStateField[]) {
 731         VMSTATE_TIMER_PTR(icount_rt_timer, TimersState),
 732         VMSTATE_TIMER_PTR(icount_vm_timer, TimersState),
 733         VMSTATE_END_OF_LIST()
 734     }
 735 };
 736
 737 /*
 738  * This is a subsection for icount migration.
 739  */
 740 static const VMStateDescription icount_vmstate_timers = {
 741     .name = "timer/icount",
 742     .version_id = 1,
 743     .minimum_version_id = 1,
 744     .needed = icount_state_needed,
 745     .fields = (VMStateField[]) {
 746         VMSTATE_INT64(qemu_icount_bias, TimersState),
 747         VMSTATE_INT64(qemu_icount, TimersState),
 748         VMSTATE_END_OF_LIST()
 749     },
 750     .subsections = (const VMStateDescription*[]) {
 751         &icount_vmstate_warp_timer,
 752         &icount_vmstate_adjust_timers,
 753         NULL
 754     }
 755 };
 756
 757 static const VMStateDescription vmstate_timers = {
 758     .name = "timer",
 759     .version_id = 2,
 760     .minimum_version_id = 1,
 761     .fields = (VMStateField[]) {
 762         VMSTATE_INT64(cpu_ticks_offset, TimersState),
 763         VMSTATE_UNUSED(8),
 764         VMSTATE_INT64_V(cpu_clock_offset, TimersState, 2),
 765         VMSTATE_END_OF_LIST()
 766     },
 767     .subsections = (const VMStateDescription*[]) {
 768         &icount_vmstate_timers,
 769         NULL
 770     }
 771 };
 772
 773 static void cpu_throttle_thread(CPUState *cpu, run_on_cpu_data opaque)
 774 {
 775     double pct;
 776     double throttle_ratio;
 777     long sleeptime_ns;
 778
 779     if (!cpu_throttle_get_percentage()) {
 780         return;
 781     }
 782
 783     pct = (double)cpu_throttle_get_percentage()/100;
 784     throttle_ratio = pct / (1 - pct);
 785     sleeptime_ns = (long)(throttle_ratio * CPU_THROTTLE_TIMESLICE_NS);
 786
 787     qemu_mutex_unlock_iothread();
 788     g_usleep(sleeptime_ns / 1000); /* Convert ns to us for usleep call */
 789     qemu_mutex_lock_iothread();
 790     atomic_set(&cpu->throttle_thread_scheduled, 0);
 791 }
 792
 793 static void cpu_throttle_timer_tick(void *opaque)
 794 {
 795     CPUState *cpu;
 796     double pct;
 797
 798     /* Stop the timer if needed */
 799     if (!cpu_throttle_get_percentage()) {
 800         return;
 801     }
 802     CPU_FOREACH(cpu) {
 803         if (!atomic_xchg(&cpu->throttle_thread_scheduled, 1)) {
 804             async_run_on_cpu(cpu, cpu_throttle_thread,
 805                              RUN_ON_CPU_NULL);
 806         }
 807     }
 808
 809     pct = (double)cpu_throttle_get_percentage()/100;
 810     timer_mod(throttle_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL_RT) +
 811                                    CPU_THROTTLE_TIMESLICE_NS / (1-pct));
 812 }
 813
 814 void cpu_throttle_set(int new_throttle_pct)
 815 {
 816     /* Ensure throttle percentage is within valid range */
 817     new_throttle_pct = MIN(new_throttle_pct, CPU_THROTTLE_PCT_MAX);
 818     new_throttle_pct = MAX(new_throttle_pct, CPU_THROTTLE_PCT_MIN);
 819
 820     atomic_set(&throttle_percentage, new_throttle_pct);
 821
 822     timer_mod(throttle_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL_RT) +
 823                                        CPU_THROTTLE_TIMESLICE_NS);
 824 }
 825
 826 void cpu_throttle_stop(void)
 827 {
 828     atomic_set(&throttle_percentage, 0);
 829 }
 830
 831 bool cpu_throttle_active(void)
 832 {
 833     return (cpu_throttle_get_percentage() != 0);
 834 }
 835
 836 int cpu_throttle_get_percentage(void)
 837 {
 838     return atomic_read(&throttle_percentage);
 839 }
 840
 841 void cpu_ticks_init(void)
 842 {
 843     seqlock_init(&timers_state.vm_clock_seqlock);
 844     qemu_spin_init(&timers_state.vm_clock_lock);
 845     vmstate_register(NULL, 0, &vmstate_timers, &timers_state);
 846     throttle_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL_RT,
 847                                            cpu_throttle_timer_tick, NULL);
 848 }
 849
 850 void configure_icount(QemuOpts *opts, Error **errp)
 851 {
 852     const char *option;
 853     char *rem_str = NULL;
 854
 855     option = qemu_opt_get(opts, "shift");
 856     if (!option) {
 857         if (qemu_opt_get(opts, "align") != NULL) {
 858             error_setg(errp, "Please specify shift option when using align");
 859         }
 860         return;
 861     }
 862
 863     icount_sleep = qemu_opt_get_bool(opts, "sleep", true);
 864     if (icount_sleep) {
 865         timers_state.icount_warp_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL_RT,
 866                                          icount_timer_cb, NULL);
 867     }
 868
 869     icount_align_option = qemu_opt_get_bool(opts, "align", false);
 870
 871     if (icount_align_option && !icount_sleep) {
 872         error_setg(errp, "align=on and sleep=off are incompatible");
 873     }
 874     if (strcmp(option, "auto") != 0) {
 875         errno = 0;
 876         timers_state.icount_time_shift = strtol(option, &rem_str, 0);
 877         if (errno != 0 || *rem_str != '\0' || !strlen(option)) {
 878             error_setg(errp, "icount: Invalid shift value");
 879         }
 880         use_icount = 1;
 881         return;
 882     } else if (icount_align_option) {
 883         error_setg(errp, "shift=auto and align=on are incompatible");
 884     } else if (!icount_sleep) {
 885         error_setg(errp, "shift=auto and sleep=off are incompatible");
 886     }
 887
 888     use_icount = 2;
 889
 890     /* 125MIPS seems a reasonable initial guess at the guest speed.
 891        It will be corrected fairly quickly anyway.  */
 892     timers_state.icount_time_shift = 3;
 893
 894     /* Have both realtime and virtual time triggers for speed adjustment.
 895        The realtime trigger catches emulated time passing too slowly,
 896        the virtual time trigger catches emulated time passing too fast.
 897        Realtime triggers occur even when idle, so use them less frequently
 898        than VM triggers.  */
 899     timers_state.vm_clock_warp_start = -1;
 900     timers_state.icount_rt_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL_RT,
 901                                    icount_adjust_rt, NULL);
 902     timer_mod(timers_state.icount_rt_timer,
 903                    qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL_RT) + 1000);
 904     timers_state.icount_vm_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL,
 905                                         icount_adjust_vm, NULL);
 906     timer_mod(timers_state.icount_vm_timer,
 907                    qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
 908                    NANOSECONDS_PER_SECOND / 10);
 909 }
 910
 911 /***********************************************************/
 912 /* TCG vCPU kick timer
 913  *
 914  * The kick timer is responsible for moving single threaded vCPU
 915  * emulation on to the next vCPU. If more than one vCPU is running a
 916  * timer event with force a cpu->exit so the next vCPU can get
 917  * scheduled.
 918  *
 919  * The timer is removed if all vCPUs are idle and restarted again once
 920  * idleness is complete.
 921  */
 922
 923 static QEMUTimer *tcg_kick_vcpu_timer;
 924 static CPUState *tcg_current_rr_cpu;
 925
 926 #define TCG_KICK_PERIOD (NANOSECONDS_PER_SECOND / 10)
 927
 928 static inline int64_t qemu_tcg_next_kick(void)
 929 {
 930     return qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + TCG_KICK_PERIOD;
 931 }
 932
 933 /* Kick the currently round-robin scheduled vCPU */
 934 static void qemu_cpu_kick_rr_cpu(void)
 935 {
 936     CPUState *cpu;
 937     do {
 938         cpu = atomic_mb_read(&tcg_current_rr_cpu);
 939         if (cpu) {
 940             cpu_exit(cpu);
 941         }
 942     } while (cpu != atomic_mb_read(&tcg_current_rr_cpu));
 943 }
 944
 945 static void do_nothing(CPUState *cpu, run_on_cpu_data unused)
 946 {
 947 }
 948
 949 void qemu_timer_notify_cb(void *opaque, QEMUClockType type)
 950 {
 951     if (!use_icount || type != QEMU_CLOCK_VIRTUAL) {
 952         qemu_notify_event();
 953         return;
 954     }
 955
 956     if (qemu_in_vcpu_thread()) {
 957         /* A CPU is currently running; kick it back out to the
 958          * tcg_cpu_exec() loop so it will recalculate its
 959          * icount deadline immediately.
 960          */
 961         qemu_cpu_kick(current_cpu);
 962     } else if (first_cpu) {
 963         /* qemu_cpu_kick is not enough to kick a halted CPU out of
 964          * qemu_tcg_wait_io_event.  async_run_on_cpu, instead,
 965          * causes cpu_thread_is_idle to return false.  This way,
 966          * handle_icount_deadline can run.
 967          * If we have no CPUs at all for some reason, we don't
 968          * need to do anything.
 969          */
 970         async_run_on_cpu(first_cpu, do_nothing, RUN_ON_CPU_NULL);
 971     }
 972 }
 973
 974 static void kick_tcg_thread(void *opaque)
 975 {
 976     timer_mod(tcg_kick_vcpu_timer, qemu_tcg_next_kick());
 977     qemu_cpu_kick_rr_cpu();
 978 }
 979
 980 static void start_tcg_kick_timer(void)
 981 {
 982     assert(!mttcg_enabled);
 983     if (!tcg_kick_vcpu_timer && CPU_NEXT(first_cpu)) {
 984         tcg_kick_vcpu_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL,
 985                                            kick_tcg_thread, NULL);
 986     }
 987     if (tcg_kick_vcpu_timer && !timer_pending(tcg_kick_vcpu_timer)) {
 988         timer_mod(tcg_kick_vcpu_timer, qemu_tcg_next_kick());
 989     }
 990 }
 991
 992 static void stop_tcg_kick_timer(void)
 993 {
 994     assert(!mttcg_enabled);
 995     if (tcg_kick_vcpu_timer && timer_pending(tcg_kick_vcpu_timer)) {
 996         timer_del(tcg_kick_vcpu_timer);
 997     }
 998 }
 999
1000 /***********************************************************/
1001 void hw_error(const char *fmt, ...)
1002 {
1003     va_list ap;
1004     CPUState *cpu;
1005
1006     va_start(ap, fmt);
1007     fprintf(stderr, "qemu: hardware error: ");
1008     vfprintf(stderr, fmt, ap);
1009     fprintf(stderr, "\n");
1010     CPU_FOREACH(cpu) {
1011         fprintf(stderr, "CPU #%d:\n", cpu->cpu_index);
1012         cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU);
1013     }
1014     va_end(ap);
1015     abort();
1016 }
1017
1018 void cpu_synchronize_all_states(void)
1019 {
1020     CPUState *cpu;
1021
1022     CPU_FOREACH(cpu) {
1023         cpu_synchronize_state(cpu);
1024         /* TODO: move to cpu_synchronize_state() */
1025         if (hvf_enabled()) {
1026             hvf_cpu_synchronize_state(cpu);
1027         }
1028     }
1029 }
1030
1031 void cpu_synchronize_all_post_reset(void)
1032 {
1033     CPUState *cpu;
1034
1035     CPU_FOREACH(cpu) {
1036         cpu_synchronize_post_reset(cpu);
1037         /* TODO: move to cpu_synchronize_post_reset() */
1038         if (hvf_enabled()) {
1039             hvf_cpu_synchronize_post_reset(cpu);
1040         }
1041     }
1042 }
1043
1044 void cpu_synchronize_all_post_init(void)
1045 {
1046     CPUState *cpu;
1047
1048     CPU_FOREACH(cpu) {
1049         cpu_synchronize_post_init(cpu);
1050         /* TODO: move to cpu_synchronize_post_init() */
1051         if (hvf_enabled()) {
1052             hvf_cpu_synchronize_post_init(cpu);
1053         }
1054     }
1055 }
1056
1057 void cpu_synchronize_all_pre_loadvm(void)
1058 {
1059     CPUState *cpu;
1060
1061     CPU_FOREACH(cpu) {
1062         cpu_synchronize_pre_loadvm(cpu);
1063     }
1064 }
1065
1066 static int do_vm_stop(RunState state, bool send_stop)
1067 {
1068     int ret = 0;
1069
1070     if (runstate_is_running()) {
1071         cpu_disable_ticks();
1072         pause_all_vcpus();
1073         runstate_set(state);
1074         vm_state_notify(0, state);
1075         if (send_stop) {
1076             qapi_event_send_stop();
1077         }
1078     }
1079
1080     bdrv_drain_all();
1081     replay_disable_events();
1082     ret = bdrv_flush_all();
1083
1084     return ret;
1085 }
1086
1087 /* Special vm_stop() variant for terminating the process.  Historically clients
1088  * did not expect a QMP STOP event and so we need to retain compatibility.
1089  */
1090 int vm_shutdown(void)
1091 {
1092     return do_vm_stop(RUN_STATE_SHUTDOWN, false);
1093 }
1094
1095 static bool cpu_can_run(CPUState *cpu)
1096 {
1097     if (cpu->stop) {
1098         return false;
1099     }
1100     if (cpu_is_stopped(cpu)) {
1101         return false;
1102     }
1103     return true;
1104 }
1105
1106 static void cpu_handle_guest_debug(CPUState *cpu)
1107 {
1108     gdb_set_stop_cpu(cpu);
1109     qemu_system_debug_request();
1110     cpu->stopped = true;
1111 }
1112
1113 #ifdef CONFIG_LINUX
1114 static void sigbus_reraise(void)
1115 {
1116     sigset_t set;
1117     struct sigaction action;
1118
1119     memset(&action, 0, sizeof(action));
1120     action.sa_handler = SIG_DFL;
1121     if (!sigaction(SIGBUS, &action, NULL)) {
1122         raise(SIGBUS);
1123         sigemptyset(&set);
1124         sigaddset(&set, SIGBUS);
1125         pthread_sigmask(SIG_UNBLOCK, &set, NULL);
1126     }
1127     perror("Failed to re-raise SIGBUS!\n");
1128     abort();
1129 }
1130
1131 static void sigbus_handler(int n, siginfo_t *siginfo, void *ctx)
1132 {
1133     if (siginfo->si_code != BUS_MCEERR_AO && siginfo->si_code != BUS_MCEERR_AR) {
1134         sigbus_reraise();
1135     }
1136
1137     if (current_cpu) {
1138         /* Called asynchronously in VCPU thread.  */
1139         if (kvm_on_sigbus_vcpu(current_cpu, siginfo->si_code, siginfo->si_addr)) {
1140             sigbus_reraise();
1141         }
1142     } else {
1143         /* Called synchronously (via signalfd) in main thread.  */
1144         if (kvm_on_sigbus(siginfo->si_code, siginfo->si_addr)) {
1145             sigbus_reraise();
1146         }
1147     }
1148 }
1149
1150 static void qemu_init_sigbus(void)
1151 {
1152     struct sigaction action;
1153
1154     memset(&action, 0, sizeof(action));
1155     action.sa_flags = SA_SIGINFO;
1156     action.sa_sigaction = sigbus_handler;
1157     sigaction(SIGBUS, &action, NULL);
1158
1159     prctl(PR_MCE_KILL, PR_MCE_KILL_SET, PR_MCE_KILL_EARLY, 0, 0);
1160 }
1161 #else /* !CONFIG_LINUX */
1162 static void qemu_init_sigbus(void)
1163 {
1164 }
1165 #endif /* !CONFIG_LINUX */
1166
1167 static QemuMutex qemu_global_mutex;
1168
1169 static QemuThread io_thread;
1170
1171 /* cpu creation */
1172 static QemuCond qemu_cpu_cond;
1173 /* system init */
1174 static QemuCond qemu_pause_cond;
1175
1176 void qemu_init_cpu_loop(void)
1177 {
1178     qemu_init_sigbus();
1179     qemu_cond_init(&qemu_cpu_cond);
1180     qemu_cond_init(&qemu_pause_cond);
1181     qemu_mutex_init(&qemu_global_mutex);
1182
1183     qemu_thread_get_self(&io_thread);
1184 }
1185
1186 void run_on_cpu(CPUState *cpu, run_on_cpu_func func, run_on_cpu_data data)
1187 {
1188     do_run_on_cpu(cpu, func, data, &qemu_global_mutex);
1189 }
1190
1191 static void qemu_kvm_destroy_vcpu(CPUState *cpu)
1192 {
1193     if (kvm_destroy_vcpu(cpu) < 0) {
1194         error_report("kvm_destroy_vcpu failed");
1195         exit(EXIT_FAILURE);
1196     }
1197 }
1198
1199 static void qemu_tcg_destroy_vcpu(CPUState *cpu)
1200 {
1201 }
1202
1203 static void qemu_cpu_stop(CPUState *cpu, bool exit)
1204 {
1205     g_assert(qemu_cpu_is_self(cpu));
1206     cpu->stop = false;
1207     cpu->stopped = true;
1208     if (exit) {
1209         cpu_exit(cpu);
1210     }
1211     qemu_cond_broadcast(&qemu_pause_cond);
1212 }
1213
1214 static void qemu_wait_io_event_common(CPUState *cpu)
1215 {
1216     atomic_mb_set(&cpu->thread_kicked, false);
1217     if (cpu->stop) {
1218         qemu_cpu_stop(cpu, false);
1219     }
1220     process_queued_cpu_work(cpu);
1221 }
1222
1223 static void qemu_tcg_rr_wait_io_event(CPUState *cpu)
1224 {
1225     while (all_cpu_threads_idle()) {
1226         stop_tcg_kick_timer();
1227         qemu_cond_wait(cpu->halt_cond, &qemu_global_mutex);
1228     }
1229
1230     start_tcg_kick_timer();
1231
1232     qemu_wait_io_event_common(cpu);
1233 }
1234
1235 static void qemu_wait_io_event(CPUState *cpu)
1236 {
1237     while (cpu_thread_is_idle(cpu)) {
1238         qemu_cond_wait(cpu->halt_cond, &qemu_global_mutex);
1239     }
1240
1241 #ifdef _WIN32
1242     /* Eat dummy APC queued by qemu_cpu_kick_thread.  */
1243     if (!tcg_enabled()) {
1244         SleepEx(0, TRUE);
1245     }
1246 #endif
1247     qemu_wait_io_event_common(cpu);
1248 }
1249
1250 static void *qemu_kvm_cpu_thread_fn(void *arg)
1251 {
1252     CPUState *cpu = arg;
1253     int r;
1254
1255     rcu_register_thread();
1256
1257     qemu_mutex_lock_iothread();
1258     qemu_thread_get_self(cpu->thread);
1259     cpu->thread_id = qemu_get_thread_id();
1260     cpu->can_do_io = 1;
1261     current_cpu = cpu;
1262
1263     r = kvm_init_vcpu(cpu);
1264     if (r < 0) {
1265         error_report("kvm_init_vcpu failed: %s", strerror(-r));
1266         exit(1);
1267     }
1268
1269     kvm_init_cpu_signals(cpu);
1270
1271     /* signal CPU creation */
1272     cpu->created = true;
1273     qemu_cond_signal(&qemu_cpu_cond);
1274
1275     do {
1276         if (cpu_can_run(cpu)) {
1277             r = kvm_cpu_exec(cpu);
1278             if (r == EXCP_DEBUG) {
1279                 cpu_handle_guest_debug(cpu);
1280             }
1281         }
1282         qemu_wait_io_event(cpu);
1283     } while (!cpu->unplug || cpu_can_run(cpu));
1284
1285     qemu_kvm_destroy_vcpu(cpu);
1286     cpu->created = false;
1287     qemu_cond_signal(&qemu_cpu_cond);
1288     qemu_mutex_unlock_iothread();
1289     rcu_unregister_thread();
1290     return NULL;
1291 }
1292
1293 static void *qemu_dummy_cpu_thread_fn(void *arg)
1294 {
1295 #ifdef _WIN32
1296     error_report("qtest is not supported under Windows");
1297     exit(1);
1298 #else
1299     CPUState *cpu = arg;
1300     sigset_t waitset;
1301     int r;
1302
1303     rcu_register_thread();
1304
1305     qemu_mutex_lock_iothread();
1306     qemu_thread_get_self(cpu->thread);
1307     cpu->thread_id = qemu_get_thread_id();
1308     cpu->can_do_io = 1;
1309     current_cpu = cpu;
1310
1311     sigemptyset(&waitset);
1312     sigaddset(&waitset, SIG_IPI);
1313
1314     /* signal CPU creation */
1315     cpu->created = true;
1316     qemu_cond_signal(&qemu_cpu_cond);
1317
1318     do {
1319         qemu_mutex_unlock_iothread();
1320         do {
1321             int sig;
1322             r = sigwait(&waitset, &sig);
1323         } while (r == -1 && (errno == EAGAIN || errno == EINTR));
1324         if (r == -1) {
1325             perror("sigwait");
1326             exit(1);
1327         }
1328         qemu_mutex_lock_iothread();
1329         qemu_wait_io_event(cpu);
1330     } while (!cpu->unplug);
1331
1332     rcu_unregister_thread();
1333     return NULL;
1334 #endif
1335 }
1336
1337 static int64_t tcg_get_icount_limit(void)
1338 {
1339     int64_t deadline;
1340
1341     if (replay_mode != REPLAY_MODE_PLAY) {
1342         deadline = qemu_clock_deadline_ns_all(QEMU_CLOCK_VIRTUAL);
1343
1344         /* Maintain prior (possibly buggy) behaviour where if no deadline
1345          * was set (as there is no QEMU_CLOCK_VIRTUAL timer) or it is more than
1346          * INT32_MAX nanoseconds ahead, we still use INT32_MAX
1347          * nanoseconds.
1348          */
1349         if ((deadline < 0) || (deadline > INT32_MAX)) {
1350             deadline = INT32_MAX;
1351         }
1352
1353         return qemu_icount_round(deadline);
1354     } else {
1355         return replay_get_instructions();
1356     }
1357 }
1358
1359 static void handle_icount_deadline(void)
1360 {
1361     assert(qemu_in_vcpu_thread());
1362     if (use_icount) {
1363         int64_t deadline =
1364             qemu_clock_deadline_ns_all(QEMU_CLOCK_VIRTUAL);
1365
1366         if (deadline == 0) {
1367             /* Wake up other AioContexts.  */
1368             qemu_clock_notify(QEMU_CLOCK_VIRTUAL);
1369             qemu_clock_run_timers(QEMU_CLOCK_VIRTUAL);
1370         }
1371     }
1372 }
1373
1374 static void prepare_icount_for_run(CPUState *cpu)
1375 {
1376     if (use_icount) {
1377         int insns_left;
1378
1379         /* These should always be cleared by process_icount_data after
1380          * each vCPU execution. However u16.high can be raised
1381          * asynchronously by cpu_exit/cpu_interrupt/tcg_handle_interrupt
1382          */
1383         g_assert(cpu->icount_decr.u16.low == 0);
1384         g_assert(cpu->icount_extra == 0);
1385
1386         cpu->icount_budget = tcg_get_icount_limit();
1387         insns_left = MIN(0xffff, cpu->icount_budget);
1388         cpu->icount_decr.u16.low = insns_left;
1389         cpu->icount_extra = cpu->icount_budget - insns_left;
1390
1391         replay_mutex_lock();
1392     }
1393 }
1394
1395 static void process_icount_data(CPUState *cpu)
1396 {
1397     if (use_icount) {
1398         /* Account for executed instructions */
1399         cpu_update_icount(cpu);
1400
1401         /* Reset the counters */
1402         cpu->icount_decr.u16.low = 0;
1403         cpu->icount_extra = 0;
1404         cpu->icount_budget = 0;
1405
1406         replay_account_executed_instructions();
1407
1408         replay_mutex_unlock();
1409     }
1410 }
1411
1412
1413 static int tcg_cpu_exec(CPUState *cpu)
1414 {
1415     int ret;
1416 #ifdef CONFIG_PROFILER
1417     int64_t ti;
1418 #endif
1419
1420     assert(tcg_enabled());
1421 #ifdef CONFIG_PROFILER
1422     ti = profile_getclock();
1423 #endif
1424     cpu_exec_start(cpu);
1425     ret = cpu_exec(cpu);
1426     cpu_exec_end(cpu);
1427 #ifdef CONFIG_PROFILER
1428     tcg_time += profile_getclock() - ti;
1429 #endif
1430     return ret;
1431 }
1432
1433 /* Destroy any remaining vCPUs which have been unplugged and have
1434  * finished running
1435  */
1436 static void deal_with_unplugged_cpus(void)
1437 {
1438     CPUState *cpu;
1439
1440     CPU_FOREACH(cpu) {
1441         if (cpu->unplug && !cpu_can_run(cpu)) {
1442             qemu_tcg_destroy_vcpu(cpu);
1443             cpu->created = false;
1444             qemu_cond_signal(&qemu_cpu_cond);
1445             break;
1446         }
1447     }
1448 }
1449
1450 /* Single-threaded TCG
1451  *
1452  * In the single-threaded case each vCPU is simulated in turn. If
1453  * there is more than a single vCPU we create a simple timer to kick
1454  * the vCPU and ensure we don't get stuck in a tight loop in one vCPU.
1455  * This is done explicitly rather than relying on side-effects
1456  * elsewhere.
1457  */
1458
1459 static void *qemu_tcg_rr_cpu_thread_fn(void *arg)
1460 {
1461     CPUState *cpu = arg;
1462
1463     assert(tcg_enabled());
1464     rcu_register_thread();
1465     tcg_register_thread();
1466
1467     qemu_mutex_lock_iothread();
1468     qemu_thread_get_self(cpu->thread);
1469
1470     cpu->thread_id = qemu_get_thread_id();
1471     cpu->created = true;
1472     cpu->can_do_io = 1;
1473     qemu_cond_signal(&qemu_cpu_cond);
1474
1475     /* wait for initial kick-off after machine start */
1476     while (first_cpu->stopped) {
1477         qemu_cond_wait(first_cpu->halt_cond, &qemu_global_mutex);
1478
1479         /* process any pending work */
1480         CPU_FOREACH(cpu) {
1481             current_cpu = cpu;
1482             qemu_wait_io_event_common(cpu);
1483         }
1484     }
1485
1486     start_tcg_kick_timer();
1487
1488     cpu = first_cpu;
1489
1490     /* process any pending work */
1491     cpu->exit_request = 1;
1492
1493     while (1) {
1494         qemu_mutex_unlock_iothread();
1495         replay_mutex_lock();
1496         qemu_mutex_lock_iothread();
1497         /* Account partial waits to QEMU_CLOCK_VIRTUAL.  */
1498         qemu_account_warp_timer();
1499
1500         /* Run the timers here.  This is much more efficient than
1501          * waking up the I/O thread and waiting for completion.
1502          */
1503         handle_icount_deadline();
1504
1505         replay_mutex_unlock();
1506
1507         if (!cpu) {
1508             cpu = first_cpu;
1509         }
1510
1511         while (cpu && !cpu->queued_work_first && !cpu->exit_request) {
1512
1513             atomic_mb_set(&tcg_current_rr_cpu, cpu);
1514             current_cpu = cpu;
1515
1516             qemu_clock_enable(QEMU_CLOCK_VIRTUAL,
1517                               (cpu->singlestep_enabled & SSTEP_NOTIMER) == 0);
1518
1519             if (cpu_can_run(cpu)) {
1520                 int r;
1521
1522                 qemu_mutex_unlock_iothread();
1523                 prepare_icount_for_run(cpu);
1524
1525                 r = tcg_cpu_exec(cpu);
1526
1527                 process_icount_data(cpu);
1528                 qemu_mutex_lock_iothread();
1529
1530                 if (r == EXCP_DEBUG) {
1531                     cpu_handle_guest_debug(cpu);
1532                     break;
1533                 } else if (r == EXCP_ATOMIC) {
1534                     qemu_mutex_unlock_iothread();
1535                     cpu_exec_step_atomic(cpu);
1536                     qemu_mutex_lock_iothread();
1537                     break;
1538                 }
1539             } else if (cpu->stop) {
1540                 if (cpu->unplug) {
1541                     cpu = CPU_NEXT(cpu);
1542                 }
1543                 break;
1544             }
1545
1546             cpu = CPU_NEXT(cpu);
1547         } /* while (cpu && !cpu->exit_request).. */
1548
1549         /* Does not need atomic_mb_set because a spurious wakeup is okay.  */
1550         atomic_set(&tcg_current_rr_cpu, NULL);
1551
1552         if (cpu && cpu->exit_request) {
1553             atomic_mb_set(&cpu->exit_request, 0);
1554         }
1555
1556         qemu_tcg_rr_wait_io_event(cpu ? cpu : first_cpu);
1557         deal_with_unplugged_cpus();
1558     }
1559
1560     rcu_unregister_thread();
1561     return NULL;
1562 }
1563
1564 static void *qemu_hax_cpu_thread_fn(void *arg)
1565 {
1566     CPUState *cpu = arg;
1567     int r;
1568
1569     rcu_register_thread();
1570     qemu_mutex_lock_iothread();
1571     qemu_thread_get_self(cpu->thread);
1572
1573     cpu->thread_id = qemu_get_thread_id();
1574     cpu->created = true;
1575     cpu->halted = 0;
1576     current_cpu = cpu;
1577
1578     hax_init_vcpu(cpu);
1579     qemu_cond_signal(&qemu_cpu_cond);
1580
1581     do {
1582         if (cpu_can_run(cpu)) {
1583             r = hax_smp_cpu_exec(cpu);
1584             if (r == EXCP_DEBUG) {
1585                 cpu_handle_guest_debug(cpu);
1586             }
1587         }
1588
1589         qemu_wait_io_event(cpu);
1590     } while (!cpu->unplug || cpu_can_run(cpu));
1591     rcu_unregister_thread();
1592     return NULL;
1593 }
1594
1595 /* The HVF-specific vCPU thread function. This one should only run when the host
1596  * CPU supports the VMX "unrestricted guest" feature. */
1597 static void *qemu_hvf_cpu_thread_fn(void *arg)
1598 {
1599     CPUState *cpu = arg;
1600
1601     int r;
1602
1603     assert(hvf_enabled());
1604
1605     rcu_register_thread();
1606
1607     qemu_mutex_lock_iothread();
1608     qemu_thread_get_self(cpu->thread);
1609
1610     cpu->thread_id = qemu_get_thread_id();
1611     cpu->can_do_io = 1;
1612     current_cpu = cpu;
1613
1614     hvf_init_vcpu(cpu);
1615
1616     /* signal CPU creation */
1617     cpu->created = true;
1618     qemu_cond_signal(&qemu_cpu_cond);
1619
1620     do {
1621         if (cpu_can_run(cpu)) {
1622             r = hvf_vcpu_exec(cpu);
1623             if (r == EXCP_DEBUG) {
1624                 cpu_handle_guest_debug(cpu);
1625             }
1626         }
1627         qemu_wait_io_event(cpu);
1628     } while (!cpu->unplug || cpu_can_run(cpu));
1629
1630     hvf_vcpu_destroy(cpu);
1631     cpu->created = false;
1632     qemu_cond_signal(&qemu_cpu_cond);
1633     qemu_mutex_unlock_iothread();
1634     rcu_unregister_thread();
1635     return NULL;
1636 }
1637
1638 static void *qemu_whpx_cpu_thread_fn(void *arg)
1639 {
1640     CPUState *cpu = arg;
1641     int r;
1642
1643     rcu_register_thread();
1644
1645     qemu_mutex_lock_iothread();
1646     qemu_thread_get_self(cpu->thread);
1647     cpu->thread_id = qemu_get_thread_id();
1648     current_cpu = cpu;
1649
1650     r = whpx_init_vcpu(cpu);
1651     if (r < 0) {
1652         fprintf(stderr, "whpx_init_vcpu failed: %s\n", strerror(-r));
1653         exit(1);
1654     }
1655
1656     /* signal CPU creation */
1657     cpu->created = true;
1658     qemu_cond_signal(&qemu_cpu_cond);
1659
1660     do {
1661         if (cpu_can_run(cpu)) {
1662             r = whpx_vcpu_exec(cpu);
1663             if (r == EXCP_DEBUG) {
1664                 cpu_handle_guest_debug(cpu);
1665             }
1666         }
1667         while (cpu_thread_is_idle(cpu)) {
1668             qemu_cond_wait(cpu->halt_cond, &qemu_global_mutex);
1669         }
1670         qemu_wait_io_event_common(cpu);
1671     } while (!cpu->unplug || cpu_can_run(cpu));
1672
1673     whpx_destroy_vcpu(cpu);
1674     cpu->created = false;
1675     qemu_cond_signal(&qemu_cpu_cond);
1676     qemu_mutex_unlock_iothread();
1677     rcu_unregister_thread();
1678     return NULL;
1679 }
1680
1681 #ifdef _WIN32
1682 static void CALLBACK dummy_apc_func(ULONG_PTR unused)
1683 {
1684 }
1685 #endif
1686
1687 /* Multi-threaded TCG
1688  *
1689  * In the multi-threaded case each vCPU has its own thread. The TLS
1690  * variable current_cpu can be used deep in the code to find the
1691  * current CPUState for a given thread.
1692  */
1693
1694 static void *qemu_tcg_cpu_thread_fn(void *arg)
1695 {
1696     CPUState *cpu = arg;
1697
1698     assert(tcg_enabled());
1699     g_assert(!use_icount);
1700
1701     rcu_register_thread();
1702     tcg_register_thread();
1703
1704     qemu_mutex_lock_iothread();
1705     qemu_thread_get_self(cpu->thread);
1706
1707     cpu->thread_id = qemu_get_thread_id();
1708     cpu->created = true;
1709     cpu->can_do_io = 1;
1710     current_cpu = cpu;
1711     qemu_cond_signal(&qemu_cpu_cond);
1712
1713     /* process any pending work */
1714     cpu->exit_request = 1;
1715
1716     do {
1717         if (cpu_can_run(cpu)) {
1718             int r;
1719             qemu_mutex_unlock_iothread();
1720             r = tcg_cpu_exec(cpu);
1721             qemu_mutex_lock_iothread();
1722             switch (r) {
1723             case EXCP_DEBUG:
1724                 cpu_handle_guest_debug(cpu);
1725                 break;
1726             case EXCP_HALTED:
1727                 /* during start-up the vCPU is reset and the thread is
1728                  * kicked several times. If we don't ensure we go back
1729                  * to sleep in the halted state we won't cleanly
1730                  * start-up when the vCPU is enabled.
1731                  *
1732                  * cpu->halted should ensure we sleep in wait_io_event
1733                  */
1734                 g_assert(cpu->halted);
1735                 break;
1736             case EXCP_ATOMIC:
1737                 qemu_mutex_unlock_iothread();
1738                 cpu_exec_step_atomic(cpu);
1739                 qemu_mutex_lock_iothread();
1740             default:
1741                 /* Ignore everything else? */
1742                 break;
1743             }
1744         }
1745
1746         atomic_mb_set(&cpu->exit_request, 0);
1747         qemu_wait_io_event(cpu);
1748     } while (!cpu->unplug || cpu_can_run(cpu));
1749
1750     qemu_tcg_destroy_vcpu(cpu);
1751     cpu->created = false;
1752     qemu_cond_signal(&qemu_cpu_cond);
1753     qemu_mutex_unlock_iothread();
1754     rcu_unregister_thread();
1755     return NULL;
1756 }
1757
1758 static void qemu_cpu_kick_thread(CPUState *cpu)
1759 {
1760 #ifndef _WIN32
1761     int err;
1762
1763     if (cpu->thread_kicked) {
1764         return;
1765     }
1766     cpu->thread_kicked = true;
1767     err = pthread_kill(cpu->thread->thread, SIG_IPI);
1768     if (err) {
1769         fprintf(stderr, "qemu:%s: %s", __func__, strerror(err));
1770         exit(1);
1771     }
1772 #else /* _WIN32 */
1773     if (!qemu_cpu_is_self(cpu)) {
1774         if (whpx_enabled()) {
1775             whpx_vcpu_kick(cpu);
1776         } else if (!QueueUserAPC(dummy_apc_func, cpu->hThread, 0)) {
1777             fprintf(stderr, "%s: QueueUserAPC failed with error %lu\n",
1778                     __func__, GetLastError());
1779             exit(1);
1780         }
1781     }
1782 #endif
1783 }
1784
1785 void qemu_cpu_kick(CPUState *cpu)
1786 {
1787     qemu_cond_broadcast(cpu->halt_cond);
1788     if (tcg_enabled()) {
1789         cpu_exit(cpu);
1790         /* NOP unless doing single-thread RR */
1791         qemu_cpu_kick_rr_cpu();
1792     } else {
1793         if (hax_enabled()) {
1794             /*
1795              * FIXME: race condition with the exit_request check in
1796              * hax_vcpu_hax_exec
1797              */
1798             cpu->exit_request = 1;
1799         }
1800         qemu_cpu_kick_thread(cpu);
1801     }
1802 }
1803
1804 void qemu_cpu_kick_self(void)
1805 {
1806     assert(current_cpu);
1807     qemu_cpu_kick_thread(current_cpu);
1808 }
1809
1810 bool qemu_cpu_is_self(CPUState *cpu)
1811 {
1812     return qemu_thread_is_self(cpu->thread);
1813 }
1814
1815 bool qemu_in_vcpu_thread(void)
1816 {
1817     return current_cpu && qemu_cpu_is_self(current_cpu);
1818 }
1819
1820 static __thread bool iothread_locked = false;
1821
1822 bool qemu_mutex_iothread_locked(void)
1823 {
1824     return iothread_locked;
1825 }
1826
1827 /*
1828  * The BQL is taken from so many places that it is worth profiling the
1829  * callers directly, instead of funneling them all through a single function.
1830  */
1831 void qemu_mutex_lock_iothread_impl(const char *file, int line)
1832 {
1833     QemuMutexLockFunc bql_lock = atomic_read(&qemu_bql_mutex_lock_func);
1834
1835     g_assert(!qemu_mutex_iothread_locked());
1836     bql_lock(&qemu_global_mutex, file, line);
1837     iothread_locked = true;
1838 }
1839
1840 void qemu_mutex_unlock_iothread(void)
1841 {
1842     g_assert(qemu_mutex_iothread_locked());
1843     iothread_locked = false;
1844     qemu_mutex_unlock(&qemu_global_mutex);
1845 }
1846
1847 static bool all_vcpus_paused(void)
1848 {
1849     CPUState *cpu;
1850
1851     CPU_FOREACH(cpu) {
1852         if (!cpu->stopped) {
1853             return false;
1854         }
1855     }
1856
1857     return true;
1858 }
1859
1860 void pause_all_vcpus(void)
1861 {
1862     CPUState *cpu;
1863
1864     qemu_clock_enable(QEMU_CLOCK_VIRTUAL, false);
1865     CPU_FOREACH(cpu) {
1866         if (qemu_cpu_is_self(cpu)) {
1867             qemu_cpu_stop(cpu, true);
1868         } else {
1869             cpu->stop = true;
1870             qemu_cpu_kick(cpu);
1871         }
1872     }
1873
1874     /* We need to drop the replay_lock so any vCPU threads woken up
1875      * can finish their replay tasks
1876      */
1877     replay_mutex_unlock();
1878
1879     while (!all_vcpus_paused()) {
1880         qemu_cond_wait(&qemu_pause_cond, &qemu_global_mutex);
1881         CPU_FOREACH(cpu) {
1882             qemu_cpu_kick(cpu);
1883         }
1884     }
1885
1886     qemu_mutex_unlock_iothread();
1887     replay_mutex_lock();
1888     qemu_mutex_lock_iothread();
1889 }
1890
1891 void cpu_resume(CPUState *cpu)
1892 {
1893     cpu->stop = false;
1894     cpu->stopped = false;
1895     qemu_cpu_kick(cpu);
1896 }
1897
1898 void resume_all_vcpus(void)
1899 {
1900     CPUState *cpu;
1901
1902     qemu_clock_enable(QEMU_CLOCK_VIRTUAL, true);
1903     CPU_FOREACH(cpu) {
1904         cpu_resume(cpu);
1905     }
1906 }
1907
1908 void cpu_remove_sync(CPUState *cpu)
1909 {
1910     cpu->stop = true;
1911     cpu->unplug = true;
1912     qemu_cpu_kick(cpu);
1913     qemu_mutex_unlock_iothread();
1914     qemu_thread_join(cpu->thread);
1915     qemu_mutex_lock_iothread();
1916 }
1917
1918 /* For temporary buffers for forming a name */
1919 #define VCPU_THREAD_NAME_SIZE 16
1920
1921 static void qemu_tcg_init_vcpu(CPUState *cpu)
1922 {
1923     char thread_name[VCPU_THREAD_NAME_SIZE];
1924     static QemuCond *single_tcg_halt_cond;
1925     static QemuThread *single_tcg_cpu_thread;
1926     static int tcg_region_inited;
1927
1928     assert(tcg_enabled());
1929     /*
1930      * Initialize TCG regions--once. Now is a good time, because:
1931      * (1) TCG's init context, prologue and target globals have been set up.
1932      * (2) qemu_tcg_mttcg_enabled() works now (TCG init code runs before the
1933      *     -accel flag is processed, so the check doesn't work then).
1934      */
1935     if (!tcg_region_inited) {
1936         tcg_region_inited = 1;
1937         tcg_region_init();
1938     }
1939
1940     if (qemu_tcg_mttcg_enabled() || !single_tcg_cpu_thread) {
1941         cpu->thread = g_malloc0(sizeof(QemuThread));
1942         cpu->halt_cond = g_malloc0(sizeof(QemuCond));
1943         qemu_cond_init(cpu->halt_cond);
1944
1945         if (qemu_tcg_mttcg_enabled()) {
1946             /* create a thread per vCPU with TCG (MTTCG) */
1947             parallel_cpus = true;
1948             snprintf(thread_name, VCPU_THREAD_NAME_SIZE, "CPU %d/TCG",
1949                  cpu->cpu_index);
1950
1951             qemu_thread_create(cpu->thread, thread_name, qemu_tcg_cpu_thread_fn,
1952                                cpu, QEMU_THREAD_JOINABLE);
1953
1954         } else {
1955             /* share a single thread for all cpus with TCG */
1956             snprintf(thread_name, VCPU_THREAD_NAME_SIZE, "ALL CPUs/TCG");
1957             qemu_thread_create(cpu->thread, thread_name,
1958                                qemu_tcg_rr_cpu_thread_fn,
1959                                cpu, QEMU_THREAD_JOINABLE);
1960
1961             single_tcg_halt_cond = cpu->halt_cond;
1962             single_tcg_cpu_thread = cpu->thread;
1963         }
1964 #ifdef _WIN32
1965         cpu->hThread = qemu_thread_get_handle(cpu->thread);
1966 #endif
1967     } else {
1968         /* For non-MTTCG cases we share the thread */
1969         cpu->thread = single_tcg_cpu_thread;
1970         cpu->halt_cond = single_tcg_halt_cond;
1971         cpu->thread_id = first_cpu->thread_id;
1972         cpu->can_do_io = 1;
1973         cpu->created = true;
1974     }
1975 }
1976
1977 static void qemu_hax_start_vcpu(CPUState *cpu)
1978 {
1979     char thread_name[VCPU_THREAD_NAME_SIZE];
1980
1981     cpu->thread = g_malloc0(sizeof(QemuThread));
1982     cpu->halt_cond = g_malloc0(sizeof(QemuCond));
1983     qemu_cond_init(cpu->halt_cond);
1984
1985     snprintf(thread_name, VCPU_THREAD_NAME_SIZE, "CPU %d/HAX",
1986              cpu->cpu_index);
1987     qemu_thread_create(cpu->thread, thread_name, qemu_hax_cpu_thread_fn,
1988                        cpu, QEMU_THREAD_JOINABLE);
1989 #ifdef _WIN32
1990     cpu->hThread = qemu_thread_get_handle(cpu->thread);
1991 #endif
1992 }
1993
1994 static void qemu_kvm_start_vcpu(CPUState *cpu)
1995 {
1996     char thread_name[VCPU_THREAD_NAME_SIZE];
1997
1998     cpu->thread = g_malloc0(sizeof(QemuThread));
1999     cpu->halt_cond = g_malloc0(sizeof(QemuCond));
2000     qemu_cond_init(cpu->halt_cond);
2001     snprintf(thread_name, VCPU_THREAD_NAME_SIZE, "CPU %d/KVM",
2002              cpu->cpu_index);
2003     qemu_thread_create(cpu->thread, thread_name, qemu_kvm_cpu_thread_fn,
2004                        cpu, QEMU_THREAD_JOINABLE);
2005 }
2006
2007 static void qemu_hvf_start_vcpu(CPUState *cpu)
2008 {
2009     char thread_name[VCPU_THREAD_NAME_SIZE];
2010
2011     /* HVF currently does not support TCG, and only runs in
2012      * unrestricted-guest mode. */
2013     assert(hvf_enabled());
2014
2015     cpu->thread = g_malloc0(sizeof(QemuThread));
2016     cpu->halt_cond = g_malloc0(sizeof(QemuCond));
2017     qemu_cond_init(cpu->halt_cond);
2018
2019     snprintf(thread_name, VCPU_THREAD_NAME_SIZE, "CPU %d/HVF",
2020              cpu->cpu_index);
2021     qemu_thread_create(cpu->thread, thread_name, qemu_hvf_cpu_thread_fn,
2022                        cpu, QEMU_THREAD_JOINABLE);
2023 }
2024
2025 static void qemu_whpx_start_vcpu(CPUState *cpu)
2026 {
2027     char thread_name[VCPU_THREAD_NAME_SIZE];
2028
2029     cpu->thread = g_malloc0(sizeof(QemuThread));
2030     cpu->halt_cond = g_malloc0(sizeof(QemuCond));
2031     qemu_cond_init(cpu->halt_cond);
2032     snprintf(thread_name, VCPU_THREAD_NAME_SIZE, "CPU %d/WHPX",
2033              cpu->cpu_index);
2034     qemu_thread_create(cpu->thread, thread_name, qemu_whpx_cpu_thread_fn,
2035                        cpu, QEMU_THREAD_JOINABLE);
2036 #ifdef _WIN32
2037     cpu->hThread = qemu_thread_get_handle(cpu->thread);
2038 #endif
2039 }
2040
2041 static void qemu_dummy_start_vcpu(CPUState *cpu)
2042 {
2043     char thread_name[VCPU_THREAD_NAME_SIZE];
2044
2045     cpu->thread = g_malloc0(sizeof(QemuThread));
2046     cpu->halt_cond = g_malloc0(sizeof(QemuCond));
2047     qemu_cond_init(cpu->halt_cond);
2048     snprintf(thread_name, VCPU_THREAD_NAME_SIZE, "CPU %d/DUMMY",
2049              cpu->cpu_index);
2050     qemu_thread_create(cpu->thread, thread_name, qemu_dummy_cpu_thread_fn, cpu,
2051                        QEMU_THREAD_JOINABLE);
2052 }
2053
2054 void qemu_init_vcpu(CPUState *cpu)
2055 {
2056     cpu->nr_cores = smp_cores;
2057     cpu->nr_threads = smp_threads;
2058     cpu->stopped = true;
2059
2060     if (!cpu->as) {
2061         /* If the target cpu hasn't set up any address spaces itself,
2062          * give it the default one.
2063          */
2064         cpu->num_ases = 1;
2065         cpu_address_space_init(cpu, 0, "cpu-memory", cpu->memory);
2066     }
2067
2068     if (kvm_enabled()) {
2069         qemu_kvm_start_vcpu(cpu);
2070     } else if (hax_enabled()) {
2071         qemu_hax_start_vcpu(cpu);
2072     } else if (hvf_enabled()) {
2073         qemu_hvf_start_vcpu(cpu);
2074     } else if (tcg_enabled()) {
2075         qemu_tcg_init_vcpu(cpu);
2076     } else if (whpx_enabled()) {
2077         qemu_whpx_start_vcpu(cpu);
2078     } else {
2079         qemu_dummy_start_vcpu(cpu);
2080     }
2081
2082     while (!cpu->created) {
2083         qemu_cond_wait(&qemu_cpu_cond, &qemu_global_mutex);
2084     }
2085 }
2086
2087 void cpu_stop_current(void)
2088 {
2089     if (current_cpu) {
2090         qemu_cpu_stop(current_cpu, true);
2091     }
2092 }
2093
2094 int vm_stop(RunState state)
2095 {
2096     if (qemu_in_vcpu_thread()) {
2097         qemu_system_vmstop_request_prepare();
2098         qemu_system_vmstop_request(state);
2099         /*
2100          * FIXME: should not return to device code in case
2101          * vm_stop() has been requested.
2102          */
2103         cpu_stop_current();
2104         return 0;
2105     }
2106
2107     return do_vm_stop(state, true);
2108 }
2109
2110 /**
2111  * Prepare for (re)starting the VM.
2112  * Returns -1 if the vCPUs are not to be restarted (e.g. if they are already
2113  * running or in case of an error condition), 0 otherwise.
2114  */
2115 int vm_prepare_start(void)
2116 {
2117     RunState requested;
2118
2119     qemu_vmstop_requested(&requested);
2120     if (runstate_is_running() && requested == RUN_STATE__MAX) {
2121         return -1;
2122     }
2123
2124     /* Ensure that a STOP/RESUME pair of events is emitted if a
2125      * vmstop request was pending.  The BLOCK_IO_ERROR event, for
2126      * example, according to documentation is always followed by
2127      * the STOP event.
2128      */
2129     if (runstate_is_running()) {
2130         qapi_event_send_stop();
2131         qapi_event_send_resume();
2132         return -1;
2133     }
2134
2135     /* We are sending this now, but the CPUs will be resumed shortly later */
2136     qapi_event_send_resume();
2137
2138     replay_enable_events();
2139     cpu_enable_ticks();
2140     runstate_set(RUN_STATE_RUNNING);
2141     vm_state_notify(1, RUN_STATE_RUNNING);
2142     return 0;
2143 }
2144
2145 void vm_start(void)
2146 {
2147     if (!vm_prepare_start()) {
2148         resume_all_vcpus();
2149     }
2150 }
2151
2152 /* does a state transition even if the VM is already stopped,
2153    current state is forgotten forever */
2154 int vm_stop_force_state(RunState state)
2155 {
2156     if (runstate_is_running()) {
2157         return vm_stop(state);
2158     } else {
2159         runstate_set(state);
2160
2161         bdrv_drain_all();
2162         /* Make sure to return an error if the flush in a previous vm_stop()
2163          * failed. */
2164         return bdrv_flush_all();
2165     }
2166 }
2167
2168 void list_cpus(FILE *f, fprintf_function cpu_fprintf, const char *optarg)
2169 {
2170     /* XXX: implement xxx_cpu_list for targets that still miss it */
2171 #if defined(cpu_list)
2172     cpu_list(f, cpu_fprintf);
2173 #endif
2174 }
2175
2176 CpuInfoList *qmp_query_cpus(Error **errp)
2177 {
2178     MachineState *ms = MACHINE(qdev_get_machine());
2179     MachineClass *mc = MACHINE_GET_CLASS(ms);
2180     CpuInfoList *head = NULL, *cur_item = NULL;
2181     CPUState *cpu;
2182
2183     CPU_FOREACH(cpu) {
2184         CpuInfoList *info;
2185 #if defined(TARGET_I386)
2186         X86CPU *x86_cpu = X86_CPU(cpu);
2187         CPUX86State *env = &x86_cpu->env;
2188 #elif defined(TARGET_PPC)
2189         PowerPCCPU *ppc_cpu = POWERPC_CPU(cpu);
2190         CPUPPCState *env = &ppc_cpu->env;
2191 #elif defined(TARGET_SPARC)
2192         SPARCCPU *sparc_cpu = SPARC_CPU(cpu);
2193         CPUSPARCState *env = &sparc_cpu->env;
2194 #elif defined(TARGET_RISCV)
2195         RISCVCPU *riscv_cpu = RISCV_CPU(cpu);
2196         CPURISCVState *env = &riscv_cpu->env;
2197 #elif defined(TARGET_MIPS)
2198         MIPSCPU *mips_cpu = MIPS_CPU(cpu);
2199         CPUMIPSState *env = &mips_cpu->env;
2200 #elif defined(TARGET_TRICORE)
2201         TriCoreCPU *tricore_cpu = TRICORE_CPU(cpu);
2202         CPUTriCoreState *env = &tricore_cpu->env;
2203 #elif defined(TARGET_S390X)
2204         S390CPU *s390_cpu = S390_CPU(cpu);
2205         CPUS390XState *env = &s390_cpu->env;
2206 #endif
2207
2208         cpu_synchronize_state(cpu);
2209
2210         info = g_malloc0(sizeof(*info));
2211         info->value = g_malloc0(sizeof(*info->value));
2212         info->value->CPU = cpu->cpu_index;
2213         info->value->current = (cpu == first_cpu);
2214         info->value->halted = cpu->halted;
2215         info->value->qom_path = object_get_canonical_path(OBJECT(cpu));
2216         info->value->thread_id = cpu->thread_id;
2217 #if defined(TARGET_I386)
2218         info->value->arch = CPU_INFO_ARCH_X86;
2219         info->value->u.x86.pc = env->eip + env->segs[R_CS].base;
2220 #elif defined(TARGET_PPC)
2221         info->value->arch = CPU_INFO_ARCH_PPC;
2222         info->value->u.ppc.nip = env->nip;
2223 #elif defined(TARGET_SPARC)
2224         info->value->arch = CPU_INFO_ARCH_SPARC;
2225         info->value->u.q_sparc.pc = env->pc;
2226         info->value->u.q_sparc.npc = env->npc;
2227 #elif defined(TARGET_MIPS)
2228         info->value->arch = CPU_INFO_ARCH_MIPS;
2229         info->value->u.q_mips.PC = env->active_tc.PC;
2230 #elif defined(TARGET_TRICORE)
2231         info->value->arch = CPU_INFO_ARCH_TRICORE;
2232         info->value->u.tricore.PC = env->PC;
2233 #elif defined(TARGET_S390X)
2234         info->value->arch = CPU_INFO_ARCH_S390;
2235         info->value->u.s390.cpu_state = env->cpu_state;
2236 #elif defined(TARGET_RISCV)
2237         info->value->arch = CPU_INFO_ARCH_RISCV;
2238         info->value->u.riscv.pc = env->pc;
2239 #else
2240         info->value->arch = CPU_INFO_ARCH_OTHER;
2241 #endif
2242         info->value->has_props = !!mc->cpu_index_to_instance_props;
2243         if (info->value->has_props) {
2244             CpuInstanceProperties *props;
2245             props = g_malloc0(sizeof(*props));
2246             *props = mc->cpu_index_to_instance_props(ms, cpu->cpu_index);
2247             info->value->props = props;
2248         }
2249
2250         /* XXX: waiting for the qapi to support GSList */
2251         if (!cur_item) {
2252             head = cur_item = info;
2253         } else {
2254             cur_item->next = info;
2255             cur_item = info;
2256         }
2257     }
2258
2259     return head;
2260 }
2261
2262 static CpuInfoArch sysemu_target_to_cpuinfo_arch(SysEmuTarget target)
2263 {
2264     /*
2265      * The @SysEmuTarget -> @CpuInfoArch mapping below is based on the
2266      * TARGET_ARCH -> TARGET_BASE_ARCH mapping in the "configure" script.
2267      */
2268     switch (target) {
2269     case SYS_EMU_TARGET_I386:
2270     case SYS_EMU_TARGET_X86_64:
2271         return CPU_INFO_ARCH_X86;
2272
2273     case SYS_EMU_TARGET_PPC:
2274     case SYS_EMU_TARGET_PPC64:
2275         return CPU_INFO_ARCH_PPC;
2276
2277     case SYS_EMU_TARGET_SPARC:
2278     case SYS_EMU_TARGET_SPARC64:
2279         return CPU_INFO_ARCH_SPARC;
2280
2281     case SYS_EMU_TARGET_MIPS:
2282     case SYS_EMU_TARGET_MIPSEL:
2283     case SYS_EMU_TARGET_MIPS64:
2284     case SYS_EMU_TARGET_MIPS64EL:
2285         return CPU_INFO_ARCH_MIPS;
2286
2287     case SYS_EMU_TARGET_TRICORE:
2288         return CPU_INFO_ARCH_TRICORE;
2289
2290     case SYS_EMU_TARGET_S390X:
2291         return CPU_INFO_ARCH_S390;
2292
2293     case SYS_EMU_TARGET_RISCV32:
2294     case SYS_EMU_TARGET_RISCV64:
2295         return CPU_INFO_ARCH_RISCV;
2296
2297     default:
2298         return CPU_INFO_ARCH_OTHER;
2299     }
2300 }
2301
2302 static void cpustate_to_cpuinfo_s390(CpuInfoS390 *info, const CPUState *cpu)
2303 {
2304 #ifdef TARGET_S390X
2305     S390CPU *s390_cpu = S390_CPU(cpu);
2306     CPUS390XState *env = &s390_cpu->env;
2307
2308     info->cpu_state = env->cpu_state;
2309 #else
2310     abort();
2311 #endif
2312 }
2313
2314 /*
2315  * fast means: we NEVER interrupt vCPU threads to retrieve
2316  * information from KVM.
2317  */
2318 CpuInfoFastList *qmp_query_cpus_fast(Error **errp)
2319 {
2320     MachineState *ms = MACHINE(qdev_get_machine());
2321     MachineClass *mc = MACHINE_GET_CLASS(ms);
2322     CpuInfoFastList *head = NULL, *cur_item = NULL;
2323     SysEmuTarget target = qapi_enum_parse(&SysEmuTarget_lookup, TARGET_NAME,
2324                                           -1, &error_abort);
2325     CPUState *cpu;
2326
2327     CPU_FOREACH(cpu) {
2328         CpuInfoFastList *info = g_malloc0(sizeof(*info));
2329         info->value = g_malloc0(sizeof(*info->value));
2330
2331         info->value->cpu_index = cpu->cpu_index;
2332         info->value->qom_path = object_get_canonical_path(OBJECT(cpu));
2333         info->value->thread_id = cpu->thread_id;
2334
2335         info->value->has_props = !!mc->cpu_index_to_instance_props;
2336         if (info->value->has_props) {
2337             CpuInstanceProperties *props;
2338             props = g_malloc0(sizeof(*props));
2339             *props = mc->cpu_index_to_instance_props(ms, cpu->cpu_index);
2340             info->value->props = props;
2341         }
2342
2343         info->value->arch = sysemu_target_to_cpuinfo_arch(target);
2344         info->value->target = target;
2345         if (target == SYS_EMU_TARGET_S390X) {
2346             cpustate_to_cpuinfo_s390(&info->value->u.s390x, cpu);
2347         }
2348
2349         if (!cur_item) {
2350             head = cur_item = info;
2351         } else {
2352             cur_item->next = info;
2353             cur_item = info;
2354         }
2355     }
2356
2357     return head;
2358 }
2359
2360 void qmp_memsave(int64_t addr, int64_t size, const char *filename,
2361                  bool has_cpu, int64_t cpu_index, Error **errp)
2362 {
2363     FILE *f;
2364     uint32_t l;
2365     CPUState *cpu;
2366     uint8_t buf[1024];
2367     int64_t orig_addr = addr, orig_size = size;
2368
2369     if (!has_cpu) {
2370         cpu_index = 0;
2371     }
2372
2373     cpu = qemu_get_cpu(cpu_index);
2374     if (cpu == NULL) {
2375         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cpu-index",
2376                    "a CPU number");
2377         return;
2378     }
2379
2380     f = fopen(filename, "wb");
2381     if (!f) {
2382         error_setg_file_open(errp, errno, filename);
2383         return;
2384     }
2385
2386     while (size != 0) {
2387         l = sizeof(buf);
2388         if (l > size)
2389             l = size;
2390         if (cpu_memory_rw_debug(cpu, addr, buf, l, 0) != 0) {
2391             error_setg(errp, "Invalid addr 0x%016" PRIx64 "/size %" PRId64
2392                              " specified", orig_addr, orig_size);
2393             goto exit;
2394         }
2395         if (fwrite(buf, 1, l, f) != l) {
2396             error_setg(errp, QERR_IO_ERROR);
2397             goto exit;
2398         }
2399         addr += l;
2400         size -= l;
2401     }
2402
2403 exit:
2404     fclose(f);
2405 }
2406
2407 void qmp_pmemsave(int64_t addr, int64_t size, const char *filename,
2408                   Error **errp)
2409 {
2410     FILE *f;
2411     uint32_t l;
2412     uint8_t buf[1024];
2413
2414     f = fopen(filename, "wb");
2415     if (!f) {
2416         error_setg_file_open(errp, errno, filename);
2417         return;
2418     }
2419
2420     while (size != 0) {
2421         l = sizeof(buf);
2422         if (l > size)
2423             l = size;
2424         cpu_physical_memory_read(addr, buf, l);
2425         if (fwrite(buf, 1, l, f) != l) {
2426             error_setg(errp, QERR_IO_ERROR);
2427             goto exit;
2428         }
2429         addr += l;
2430         size -= l;
2431     }
2432
2433 exit:
2434     fclose(f);
2435 }
2436
2437 void qmp_inject_nmi(Error **errp)
2438 {
2439     nmi_monitor_handle(monitor_get_cpu_index(), errp);
2440 }
2441
2442 void dump_drift_info(FILE *f, fprintf_function cpu_fprintf)
2443 {
2444     if (!use_icount) {
2445         return;
2446     }
2447
2448     cpu_fprintf(f, "Host - Guest clock  %"PRIi64" ms\n",
2449                 (cpu_get_clock() - cpu_get_icount())/SCALE_MS);
2450     if (icount_align_option) {
2451         cpu_fprintf(f, "Max guest delay     %"PRIi64" ms\n", -max_delay/SCALE_MS);
2452         cpu_fprintf(f, "Max guest advance   %"PRIi64" ms\n", max_advance/SCALE_MS);
2453     } else {
2454         cpu_fprintf(f, "Max guest delay     NA\n");
2455         cpu_fprintf(f, "Max guest advance   NA\n");
2456     }
2457 }