migration/migration.c

   1 /*
   2  * QEMU live migration
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *
   6  * Authors:
   7  *  Anthony Liguori   <[email protected]>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  * Contributions after 2012-01-13 are licensed under the terms of the
  13  * GNU GPL, version 2 or (at your option) any later version.
  14  */
  15
  16 #include "qemu/osdep.h"
  17 #include "qemu/cutils.h"
  18 #include "qemu/error-report.h"
  19 #include "qemu/main-loop.h"
  20 #include "migration/blocker.h"
  21 #include "exec.h"
  22 #include "fd.h"
  23 #include "socket.h"
  24 #include "sysemu/runstate.h"
  25 #include "sysemu/sysemu.h"
  26 #include "sysemu/cpu-throttle.h"
  27 #include "rdma.h"
  28 #include "ram.h"
  29 #include "migration/global_state.h"
  30 #include "migration/misc.h"
  31 #include "migration.h"
  32 #include "savevm.h"
  33 #include "qemu-file-channel.h"
  34 #include "qemu-file.h"
  35 #include "migration/vmstate.h"
  36 #include "block/block.h"
  37 #include "qapi/error.h"
  38 #include "qapi/clone-visitor.h"
  39 #include "qapi/qapi-visit-migration.h"
  40 #include "qapi/qapi-visit-sockets.h"
  41 #include "qapi/qapi-commands-migration.h"
  42 #include "qapi/qapi-events-migration.h"
  43 #include "qapi/qmp/qerror.h"
  44 #include "qapi/qmp/qnull.h"
  45 #include "qemu/rcu.h"
  46 #include "block.h"
  47 #include "postcopy-ram.h"
  48 #include "qemu/thread.h"
  49 #include "trace.h"
  50 #include "exec/target_page.h"
  51 #include "io/channel-buffer.h"
  52 #include "migration/colo.h"
  53 #include "hw/boards.h"
  54 #include "hw/qdev-properties.h"
  55 #include "hw/qdev-properties-system.h"
  56 #include "monitor/monitor.h"
  57 #include "net/announce.h"
  58 #include "qemu/queue.h"
  59 #include "multifd.h"
  60 #include "qemu/yank.h"
  61 #include "sysemu/cpus.h"
  62 #include "yank_functions.h"
  63 #include "sysemu/qtest.h"
  64
  65 #define MAX_THROTTLE  (128 << 20)      /* Migration transfer speed throttling */
  66
  67 /* Amount of time to allocate to each "chunk" of bandwidth-throttled
  68  * data. */
  69 #define BUFFER_DELAY     100
  70 #define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY)
  71
  72 /* Time in milliseconds we are allowed to stop the source,
  73  * for sending the last part */
  74 #define DEFAULT_MIGRATE_SET_DOWNTIME 300
  75
  76 /* Maximum migrate downtime set to 2000 seconds */
  77 #define MAX_MIGRATE_DOWNTIME_SECONDS 2000
  78 #define MAX_MIGRATE_DOWNTIME (MAX_MIGRATE_DOWNTIME_SECONDS * 1000)
  79
  80 /* Default compression thread count */
  81 #define DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT 8
  82 /* Default decompression thread count, usually decompression is at
  83  * least 4 times as fast as compression.*/
  84 #define DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT 2
  85 /*0: means nocompress, 1: best speed, ... 9: best compress ratio */
  86 #define DEFAULT_MIGRATE_COMPRESS_LEVEL 1
  87 /* Define default autoconverge cpu throttle migration parameters */
  88 #define DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD 50
  89 #define DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL 20
  90 #define DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT 10
  91 #define DEFAULT_MIGRATE_MAX_CPU_THROTTLE 99
  92
  93 /* Migration XBZRLE default cache size */
  94 #define DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE (64 * 1024 * 1024)
  95
  96 /* The delay time (in ms) between two COLO checkpoints */
  97 #define DEFAULT_MIGRATE_X_CHECKPOINT_DELAY (200 * 100)
  98 #define DEFAULT_MIGRATE_MULTIFD_CHANNELS 2
  99 #define DEFAULT_MIGRATE_MULTIFD_COMPRESSION MULTIFD_COMPRESSION_NONE
 100 /* 0: means nocompress, 1: best speed, ... 9: best compress ratio */
 101 #define DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL 1
 102 /* 0: means nocompress, 1: best speed, ... 20: best compress ratio */
 103 #define DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL 1
 104
 105 /* Background transfer rate for postcopy, 0 means unlimited, note
 106  * that page requests can still exceed this limit.
 107  */
 108 #define DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH 0
 109
 110 /*
 111  * Parameters for self_announce_delay giving a stream of RARP/ARP
 112  * packets after migration.
 113  */
 114 #define DEFAULT_MIGRATE_ANNOUNCE_INITIAL  50
 115 #define DEFAULT_MIGRATE_ANNOUNCE_MAX     550
 116 #define DEFAULT_MIGRATE_ANNOUNCE_ROUNDS    5
 117 #define DEFAULT_MIGRATE_ANNOUNCE_STEP    100
 118
 119 static NotifierList migration_state_notifiers =
 120     NOTIFIER_LIST_INITIALIZER(migration_state_notifiers);
 121
 122 /* Messages sent on the return path from destination to source */
 123 enum mig_rp_message_type {
 124     MIG_RP_MSG_INVALID = 0,  /* Must be 0 */
 125     MIG_RP_MSG_SHUT,         /* sibling will not send any more RP messages */
 126     MIG_RP_MSG_PONG,         /* Response to a PING; data (seq: be32 ) */
 127
 128     MIG_RP_MSG_REQ_PAGES_ID, /* data (start: be64, len: be32, id: string) */
 129     MIG_RP_MSG_REQ_PAGES,    /* data (start: be64, len: be32) */
 130     MIG_RP_MSG_RECV_BITMAP,  /* send recved_bitmap back to source */
 131     MIG_RP_MSG_RESUME_ACK,   /* tell source that we are ready to resume */
 132
 133     MIG_RP_MSG_MAX
 134 };
 135
 136 /* Migration capabilities set */
 137 struct MigrateCapsSet {
 138     int size;                       /* Capability set size */
 139     MigrationCapability caps[];     /* Variadic array of capabilities */
 140 };
 141 typedef struct MigrateCapsSet MigrateCapsSet;
 142
 143 /* Define and initialize MigrateCapsSet */
 144 #define INITIALIZE_MIGRATE_CAPS_SET(_name, ...)   \
 145     MigrateCapsSet _name = {    \
 146         .size = sizeof((int []) { __VA_ARGS__ }) / sizeof(int), \
 147         .caps = { __VA_ARGS__ } \
 148     }
 149
 150 /* Background-snapshot compatibility check list */
 151 static const
 152 INITIALIZE_MIGRATE_CAPS_SET(check_caps_background_snapshot,
 153     MIGRATION_CAPABILITY_POSTCOPY_RAM,
 154     MIGRATION_CAPABILITY_DIRTY_BITMAPS,
 155     MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME,
 156     MIGRATION_CAPABILITY_LATE_BLOCK_ACTIVATE,
 157     MIGRATION_CAPABILITY_RETURN_PATH,
 158     MIGRATION_CAPABILITY_MULTIFD,
 159     MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER,
 160     MIGRATION_CAPABILITY_AUTO_CONVERGE,
 161     MIGRATION_CAPABILITY_RELEASE_RAM,
 162     MIGRATION_CAPABILITY_RDMA_PIN_ALL,
 163     MIGRATION_CAPABILITY_COMPRESS,
 164     MIGRATION_CAPABILITY_XBZRLE,
 165     MIGRATION_CAPABILITY_X_COLO,
 166     MIGRATION_CAPABILITY_VALIDATE_UUID);
 167
 168 /* When we add fault tolerance, we could have several
 169    migrations at once.  For now we don't need to add
 170    dynamic creation of migration */
 171
 172 static MigrationState *current_migration;
 173 static MigrationIncomingState *current_incoming;
 174
 175 static GSList *migration_blockers;
 176
 177 static bool migration_object_check(MigrationState *ms, Error **errp);
 178 static int migration_maybe_pause(MigrationState *s,
 179                                  int *current_active_state,
 180                                  int new_state);
 181 static void migrate_fd_cancel(MigrationState *s);
 182
 183 static gint page_request_addr_cmp(gconstpointer ap, gconstpointer bp)
 184 {
 185     uintptr_t a = (uintptr_t) ap, b = (uintptr_t) bp;
 186
 187     return (a > b) - (a < b);
 188 }
 189
 190 void migration_object_init(void)
 191 {
 192     /* This can only be called once. */
 193     assert(!current_migration);
 194     current_migration = MIGRATION_OBJ(object_new(TYPE_MIGRATION));
 195
 196     /*
 197      * Init the migrate incoming object as well no matter whether
 198      * we'll use it or not.
 199      */
 200     assert(!current_incoming);
 201     current_incoming = g_new0(MigrationIncomingState, 1);
 202     current_incoming->state = MIGRATION_STATUS_NONE;
 203     current_incoming->postcopy_remote_fds =
 204         g_array_new(FALSE, TRUE, sizeof(struct PostCopyFD));
 205     qemu_mutex_init(&current_incoming->rp_mutex);
 206     qemu_event_init(&current_incoming->main_thread_load_event, false);
 207     qemu_sem_init(&current_incoming->postcopy_pause_sem_dst, 0);
 208     qemu_sem_init(&current_incoming->postcopy_pause_sem_fault, 0);
 209     qemu_mutex_init(&current_incoming->page_request_mutex);
 210     current_incoming->page_requested = g_tree_new(page_request_addr_cmp);
 211
 212     migration_object_check(current_migration, &error_fatal);
 213
 214     blk_mig_init();
 215     ram_mig_init();
 216     dirty_bitmap_mig_init();
 217 }
 218
 219 void migration_cancel(const Error *error)
 220 {
 221     if (error) {
 222         migrate_set_error(current_migration, error);
 223     }
 224     migrate_fd_cancel(current_migration);
 225 }
 226
 227 void migration_shutdown(void)
 228 {
 229     /*
 230      * When the QEMU main thread exit, the COLO thread
 231      * may wait a semaphore. So, we should wakeup the
 232      * COLO thread before migration shutdown.
 233      */
 234     colo_shutdown();
 235     /*
 236      * Cancel the current migration - that will (eventually)
 237      * stop the migration using this structure
 238      */
 239     migration_cancel(NULL);
 240     object_unref(OBJECT(current_migration));
 241
 242     /*
 243      * Cancel outgoing migration of dirty bitmaps. It should
 244      * at least unref used block nodes.
 245      */
 246     dirty_bitmap_mig_cancel_outgoing();
 247
 248     /*
 249      * Cancel incoming migration of dirty bitmaps. Dirty bitmaps
 250      * are non-critical data, and their loss never considered as
 251      * something serious.
 252      */
 253     dirty_bitmap_mig_cancel_incoming();
 254 }
 255
 256 /* For outgoing */
 257 MigrationState *migrate_get_current(void)
 258 {
 259     /* This can only be called after the object created. */
 260     assert(current_migration);
 261     return current_migration;
 262 }
 263
 264 MigrationIncomingState *migration_incoming_get_current(void)
 265 {
 266     assert(current_incoming);
 267     return current_incoming;
 268 }
 269
 270 void migration_incoming_transport_cleanup(MigrationIncomingState *mis)
 271 {
 272     if (mis->socket_address_list) {
 273         qapi_free_SocketAddressList(mis->socket_address_list);
 274         mis->socket_address_list = NULL;
 275     }
 276
 277     if (mis->transport_cleanup) {
 278         mis->transport_cleanup(mis->transport_data);
 279         mis->transport_data = mis->transport_cleanup = NULL;
 280     }
 281 }
 282
 283 void migration_incoming_state_destroy(void)
 284 {
 285     struct MigrationIncomingState *mis = migration_incoming_get_current();
 286
 287     if (mis->to_src_file) {
 288         /* Tell source that we are done */
 289         migrate_send_rp_shut(mis, qemu_file_get_error(mis->from_src_file) != 0);
 290         qemu_fclose(mis->to_src_file);
 291         mis->to_src_file = NULL;
 292     }
 293
 294     if (mis->from_src_file) {
 295         migration_ioc_unregister_yank_from_file(mis->from_src_file);
 296         qemu_fclose(mis->from_src_file);
 297         mis->from_src_file = NULL;
 298     }
 299     if (mis->postcopy_remote_fds) {
 300         g_array_free(mis->postcopy_remote_fds, TRUE);
 301         mis->postcopy_remote_fds = NULL;
 302     }
 303
 304     migration_incoming_transport_cleanup(mis);
 305     qemu_event_reset(&mis->main_thread_load_event);
 306
 307     if (mis->page_requested) {
 308         g_tree_destroy(mis->page_requested);
 309         mis->page_requested = NULL;
 310     }
 311
 312     yank_unregister_instance(MIGRATION_YANK_INSTANCE);
 313 }
 314
 315 static void migrate_generate_event(int new_state)
 316 {
 317     if (migrate_use_events()) {
 318         qapi_event_send_migration(new_state);
 319     }
 320 }
 321
 322 static bool migrate_late_block_activate(void)
 323 {
 324     MigrationState *s;
 325
 326     s = migrate_get_current();
 327
 328     return s->enabled_capabilities[
 329         MIGRATION_CAPABILITY_LATE_BLOCK_ACTIVATE];
 330 }
 331
 332 /*
 333  * Send a message on the return channel back to the source
 334  * of the migration.
 335  */
 336 static int migrate_send_rp_message(MigrationIncomingState *mis,
 337                                    enum mig_rp_message_type message_type,
 338                                    uint16_t len, void *data)
 339 {
 340     int ret = 0;
 341
 342     trace_migrate_send_rp_message((int)message_type, len);
 343     QEMU_LOCK_GUARD(&mis->rp_mutex);
 344
 345     /*
 346      * It's possible that the file handle got lost due to network
 347      * failures.
 348      */
 349     if (!mis->to_src_file) {
 350         ret = -EIO;
 351         return ret;
 352     }
 353
 354     qemu_put_be16(mis->to_src_file, (unsigned int)message_type);
 355     qemu_put_be16(mis->to_src_file, len);
 356     qemu_put_buffer(mis->to_src_file, data, len);
 357     qemu_fflush(mis->to_src_file);
 358
 359     /* It's possible that qemu file got error during sending */
 360     ret = qemu_file_get_error(mis->to_src_file);
 361
 362     return ret;
 363 }
 364
 365 /* Request one page from the source VM at the given start address.
 366  *   rb: the RAMBlock to request the page in
 367  *   Start: Address offset within the RB
 368  *   Len: Length in bytes required - must be a multiple of pagesize
 369  */
 370 int migrate_send_rp_message_req_pages(MigrationIncomingState *mis,
 371                                       RAMBlock *rb, ram_addr_t start)
 372 {
 373     uint8_t bufc[12 + 1 + 255]; /* start (8), len (4), rbname up to 256 */
 374     size_t msglen = 12; /* start + len */
 375     size_t len = qemu_ram_pagesize(rb);
 376     enum mig_rp_message_type msg_type;
 377     const char *rbname;
 378     int rbname_len;
 379
 380     *(uint64_t *)bufc = cpu_to_be64((uint64_t)start);
 381     *(uint32_t *)(bufc + 8) = cpu_to_be32((uint32_t)len);
 382
 383     /*
 384      * We maintain the last ramblock that we requested for page.  Note that we
 385      * don't need locking because this function will only be called within the
 386      * postcopy ram fault thread.
 387      */
 388     if (rb != mis->last_rb) {
 389         mis->last_rb = rb;
 390
 391         rbname = qemu_ram_get_idstr(rb);
 392         rbname_len = strlen(rbname);
 393
 394         assert(rbname_len < 256);
 395
 396         bufc[msglen++] = rbname_len;
 397         memcpy(bufc + msglen, rbname, rbname_len);
 398         msglen += rbname_len;
 399         msg_type = MIG_RP_MSG_REQ_PAGES_ID;
 400     } else {
 401         msg_type = MIG_RP_MSG_REQ_PAGES;
 402     }
 403
 404     return migrate_send_rp_message(mis, msg_type, msglen, bufc);
 405 }
 406
 407 int migrate_send_rp_req_pages(MigrationIncomingState *mis,
 408                               RAMBlock *rb, ram_addr_t start, uint64_t haddr)
 409 {
 410     void *aligned = (void *)(uintptr_t)ROUND_DOWN(haddr, qemu_ram_pagesize(rb));
 411     bool received = false;
 412
 413     WITH_QEMU_LOCK_GUARD(&mis->page_request_mutex) {
 414         received = ramblock_recv_bitmap_test_byte_offset(rb, start);
 415         if (!received && !g_tree_lookup(mis->page_requested, aligned)) {
 416             /*
 417              * The page has not been received, and it's not yet in the page
 418              * request list.  Queue it.  Set the value of element to 1, so that
 419              * things like g_tree_lookup() will return TRUE (1) when found.
 420              */
 421             g_tree_insert(mis->page_requested, aligned, (gpointer)1);
 422             mis->page_requested_count++;
 423             trace_postcopy_page_req_add(aligned, mis->page_requested_count);
 424         }
 425     }
 426
 427     /*
 428      * If the page is there, skip sending the message.  We don't even need the
 429      * lock because as long as the page arrived, it'll be there forever.
 430      */
 431     if (received) {
 432         return 0;
 433     }
 434
 435     return migrate_send_rp_message_req_pages(mis, rb, start);
 436 }
 437
 438 static bool migration_colo_enabled;
 439 bool migration_incoming_colo_enabled(void)
 440 {
 441     return migration_colo_enabled;
 442 }
 443
 444 void migration_incoming_disable_colo(void)
 445 {
 446     ram_block_discard_disable(false);
 447     migration_colo_enabled = false;
 448 }
 449
 450 int migration_incoming_enable_colo(void)
 451 {
 452     if (ram_block_discard_disable(true)) {
 453         error_report("COLO: cannot disable RAM discard");
 454         return -EBUSY;
 455     }
 456     migration_colo_enabled = true;
 457     return 0;
 458 }
 459
 460 void migrate_add_address(SocketAddress *address)
 461 {
 462     MigrationIncomingState *mis = migration_incoming_get_current();
 463
 464     QAPI_LIST_PREPEND(mis->socket_address_list,
 465                       QAPI_CLONE(SocketAddress, address));
 466 }
 467
 468 static void qemu_start_incoming_migration(const char *uri, Error **errp)
 469 {
 470     const char *p = NULL;
 471
 472     migrate_protocol_allow_multifd(false); /* reset it anyway */
 473     qapi_event_send_migration(MIGRATION_STATUS_SETUP);
 474     if (strstart(uri, "tcp:", &p) ||
 475         strstart(uri, "unix:", NULL) ||
 476         strstart(uri, "vsock:", NULL)) {
 477         migrate_protocol_allow_multifd(true);
 478         socket_start_incoming_migration(p ? p : uri, errp);
 479 #ifdef CONFIG_RDMA
 480     } else if (strstart(uri, "rdma:", &p)) {
 481         rdma_start_incoming_migration(p, errp);
 482 #endif
 483     } else if (strstart(uri, "exec:", &p)) {
 484         exec_start_incoming_migration(p, errp);
 485     } else if (strstart(uri, "fd:", &p)) {
 486         fd_start_incoming_migration(p, errp);
 487     } else {
 488         error_setg(errp, "unknown migration protocol: %s", uri);
 489     }
 490 }
 491
 492 static void process_incoming_migration_bh(void *opaque)
 493 {
 494     Error *local_err = NULL;
 495     MigrationIncomingState *mis = opaque;
 496
 497     /* If capability late_block_activate is set:
 498      * Only fire up the block code now if we're going to restart the
 499      * VM, else 'cont' will do it.
 500      * This causes file locking to happen; so we don't want it to happen
 501      * unless we really are starting the VM.
 502      */
 503     if (!migrate_late_block_activate() ||
 504          (autostart && (!global_state_received() ||
 505             global_state_get_runstate() == RUN_STATE_RUNNING))) {
 506         /* Make sure all file formats throw away their mutable metadata.
 507          * If we get an error here, just don't restart the VM yet. */
 508         bdrv_activate_all(&local_err);
 509         if (local_err) {
 510             error_report_err(local_err);
 511             local_err = NULL;
 512             autostart = false;
 513         }
 514     }
 515
 516     /*
 517      * This must happen after all error conditions are dealt with and
 518      * we're sure the VM is going to be running on this host.
 519      */
 520     qemu_announce_self(&mis->announce_timer, migrate_announce_params());
 521
 522     if (multifd_load_cleanup(&local_err) != 0) {
 523         error_report_err(local_err);
 524         autostart = false;
 525     }
 526     /* If global state section was not received or we are in running
 527        state, we need to obey autostart. Any other state is set with
 528        runstate_set. */
 529
 530     dirty_bitmap_mig_before_vm_start();
 531
 532     if (!global_state_received() ||
 533         global_state_get_runstate() == RUN_STATE_RUNNING) {
 534         if (autostart) {
 535             vm_start();
 536         } else {
 537             runstate_set(RUN_STATE_PAUSED);
 538         }
 539     } else if (migration_incoming_colo_enabled()) {
 540         migration_incoming_disable_colo();
 541         vm_start();
 542     } else {
 543         runstate_set(global_state_get_runstate());
 544     }
 545     /*
 546      * This must happen after any state changes since as soon as an external
 547      * observer sees this event they might start to prod at the VM assuming
 548      * it's ready to use.
 549      */
 550     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
 551                       MIGRATION_STATUS_COMPLETED);
 552     qemu_bh_delete(mis->bh);
 553     migration_incoming_state_destroy();
 554 }
 555
 556 static void process_incoming_migration_co(void *opaque)
 557 {
 558     MigrationIncomingState *mis = migration_incoming_get_current();
 559     PostcopyState ps;
 560     int ret;
 561     Error *local_err = NULL;
 562
 563     assert(mis->from_src_file);
 564     mis->migration_incoming_co = qemu_coroutine_self();
 565     mis->largest_page_size = qemu_ram_pagesize_largest();
 566     postcopy_state_set(POSTCOPY_INCOMING_NONE);
 567     migrate_set_state(&mis->state, MIGRATION_STATUS_NONE,
 568                       MIGRATION_STATUS_ACTIVE);
 569     ret = qemu_loadvm_state(mis->from_src_file);
 570
 571     ps = postcopy_state_get();
 572     trace_process_incoming_migration_co_end(ret, ps);
 573     if (ps != POSTCOPY_INCOMING_NONE) {
 574         if (ps == POSTCOPY_INCOMING_ADVISE) {
 575             /*
 576              * Where a migration had postcopy enabled (and thus went to advise)
 577              * but managed to complete within the precopy period, we can use
 578              * the normal exit.
 579              */
 580             postcopy_ram_incoming_cleanup(mis);
 581         } else if (ret >= 0) {
 582             /*
 583              * Postcopy was started, cleanup should happen at the end of the
 584              * postcopy thread.
 585              */
 586             trace_process_incoming_migration_co_postcopy_end_main();
 587             return;
 588         }
 589         /* Else if something went wrong then just fall out of the normal exit */
 590     }
 591
 592     /* we get COLO info, and know if we are in COLO mode */
 593     if (!ret && migration_incoming_colo_enabled()) {
 594         /* Make sure all file formats throw away their mutable metadata */
 595         bdrv_activate_all(&local_err);
 596         if (local_err) {
 597             error_report_err(local_err);
 598             goto fail;
 599         }
 600
 601         qemu_thread_create(&mis->colo_incoming_thread, "COLO incoming",
 602              colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE);
 603         mis->have_colo_incoming_thread = true;
 604         qemu_coroutine_yield();
 605
 606         qemu_mutex_unlock_iothread();
 607         /* Wait checkpoint incoming thread exit before free resource */
 608         qemu_thread_join(&mis->colo_incoming_thread);
 609         qemu_mutex_lock_iothread();
 610         /* We hold the global iothread lock, so it is safe here */
 611         colo_release_ram_cache();
 612     }
 613
 614     if (ret < 0) {
 615         error_report("load of migration failed: %s", strerror(-ret));
 616         goto fail;
 617     }
 618     mis->bh = qemu_bh_new(process_incoming_migration_bh, mis);
 619     qemu_bh_schedule(mis->bh);
 620     mis->migration_incoming_co = NULL;
 621     return;
 622 fail:
 623     local_err = NULL;
 624     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
 625                       MIGRATION_STATUS_FAILED);
 626     qemu_fclose(mis->from_src_file);
 627     if (multifd_load_cleanup(&local_err) != 0) {
 628         error_report_err(local_err);
 629     }
 630     exit(EXIT_FAILURE);
 631 }
 632
 633 /**
 634  * migration_incoming_setup: Setup incoming migration
 635  * @f: file for main migration channel
 636  * @errp: where to put errors
 637  *
 638  * Returns: %true on success, %false on error.
 639  */
 640 static bool migration_incoming_setup(QEMUFile *f, Error **errp)
 641 {
 642     MigrationIncomingState *mis = migration_incoming_get_current();
 643
 644     if (multifd_load_setup(errp) != 0) {
 645         return false;
 646     }
 647
 648     if (!mis->from_src_file) {
 649         mis->from_src_file = f;
 650     }
 651     qemu_file_set_blocking(f, false);
 652     return true;
 653 }
 654
 655 void migration_incoming_process(void)
 656 {
 657     Coroutine *co = qemu_coroutine_create(process_incoming_migration_co, NULL);
 658     qemu_coroutine_enter(co);
 659 }
 660
 661 /* Returns true if recovered from a paused migration, otherwise false */
 662 static bool postcopy_try_recover(QEMUFile *f)
 663 {
 664     MigrationIncomingState *mis = migration_incoming_get_current();
 665
 666     if (mis->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
 667         /* Resumed from a paused postcopy migration */
 668
 669         mis->from_src_file = f;
 670         /* Postcopy has standalone thread to do vm load */
 671         qemu_file_set_blocking(f, true);
 672
 673         /* Re-configure the return path */
 674         mis->to_src_file = qemu_file_get_return_path(f);
 675
 676         migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
 677                           MIGRATION_STATUS_POSTCOPY_RECOVER);
 678
 679         /*
 680          * Here, we only wake up the main loading thread (while the
 681          * fault thread will still be waiting), so that we can receive
 682          * commands from source now, and answer it if needed. The
 683          * fault thread will be woken up afterwards until we are sure
 684          * that source is ready to reply to page requests.
 685          */
 686         qemu_sem_post(&mis->postcopy_pause_sem_dst);
 687         return true;
 688     }
 689
 690     return false;
 691 }
 692
 693 void migration_fd_process_incoming(QEMUFile *f, Error **errp)
 694 {
 695     if (postcopy_try_recover(f)) {
 696         return;
 697     }
 698
 699     if (!migration_incoming_setup(f, errp)) {
 700         return;
 701     }
 702     migration_incoming_process();
 703 }
 704
 705 void migration_ioc_process_incoming(QIOChannel *ioc, Error **errp)
 706 {
 707     MigrationIncomingState *mis = migration_incoming_get_current();
 708     Error *local_err = NULL;
 709     bool start_migration;
 710
 711     if (!mis->from_src_file) {
 712         /* The first connection (multifd may have multiple) */
 713         QEMUFile *f = qemu_fopen_channel_input(ioc);
 714
 715         /* If it's a recovery, we're done */
 716         if (postcopy_try_recover(f)) {
 717             return;
 718         }
 719
 720         if (!migration_incoming_setup(f, errp)) {
 721             return;
 722         }
 723
 724         /*
 725          * Common migration only needs one channel, so we can start
 726          * right now.  Multifd needs more than one channel, we wait.
 727          */
 728         start_migration = !migrate_use_multifd();
 729     } else {
 730         /* Multiple connections */
 731         assert(migrate_use_multifd());
 732         start_migration = multifd_recv_new_channel(ioc, &local_err);
 733         if (local_err) {
 734             error_propagate(errp, local_err);
 735             return;
 736         }
 737     }
 738
 739     if (start_migration) {
 740         migration_incoming_process();
 741     }
 742 }
 743
 744 /**
 745  * @migration_has_all_channels: We have received all channels that we need
 746  *
 747  * Returns true when we have got connections to all the channels that
 748  * we need for migration.
 749  */
 750 bool migration_has_all_channels(void)
 751 {
 752     MigrationIncomingState *mis = migration_incoming_get_current();
 753     bool all_channels;
 754
 755     all_channels = multifd_recv_all_channels_created();
 756
 757     return all_channels && mis->from_src_file != NULL;
 758 }
 759
 760 /*
 761  * Send a 'SHUT' message on the return channel with the given value
 762  * to indicate that we've finished with the RP.  Non-0 value indicates
 763  * error.
 764  */
 765 void migrate_send_rp_shut(MigrationIncomingState *mis,
 766                           uint32_t value)
 767 {
 768     uint32_t buf;
 769
 770     buf = cpu_to_be32(value);
 771     migrate_send_rp_message(mis, MIG_RP_MSG_SHUT, sizeof(buf), &buf);
 772 }
 773
 774 /*
 775  * Send a 'PONG' message on the return channel with the given value
 776  * (normally in response to a 'PING')
 777  */
 778 void migrate_send_rp_pong(MigrationIncomingState *mis,
 779                           uint32_t value)
 780 {
 781     uint32_t buf;
 782
 783     buf = cpu_to_be32(value);
 784     migrate_send_rp_message(mis, MIG_RP_MSG_PONG, sizeof(buf), &buf);
 785 }
 786
 787 void migrate_send_rp_recv_bitmap(MigrationIncomingState *mis,
 788                                  char *block_name)
 789 {
 790     char buf[512];
 791     int len;
 792     int64_t res;
 793
 794     /*
 795      * First, we send the header part. It contains only the len of
 796      * idstr, and the idstr itself.
 797      */
 798     len = strlen(block_name);
 799     buf[0] = len;
 800     memcpy(buf + 1, block_name, len);
 801
 802     if (mis->state != MIGRATION_STATUS_POSTCOPY_RECOVER) {
 803         error_report("%s: MSG_RP_RECV_BITMAP only used for recovery",
 804                      __func__);
 805         return;
 806     }
 807
 808     migrate_send_rp_message(mis, MIG_RP_MSG_RECV_BITMAP, len + 1, buf);
 809
 810     /*
 811      * Next, we dump the received bitmap to the stream.
 812      *
 813      * TODO: currently we are safe since we are the only one that is
 814      * using the to_src_file handle (fault thread is still paused),
 815      * and it's ok even not taking the mutex. However the best way is
 816      * to take the lock before sending the message header, and release
 817      * the lock after sending the bitmap.
 818      */
 819     qemu_mutex_lock(&mis->rp_mutex);
 820     res = ramblock_recv_bitmap_send(mis->to_src_file, block_name);
 821     qemu_mutex_unlock(&mis->rp_mutex);
 822
 823     trace_migrate_send_rp_recv_bitmap(block_name, res);
 824 }
 825
 826 void migrate_send_rp_resume_ack(MigrationIncomingState *mis, uint32_t value)
 827 {
 828     uint32_t buf;
 829
 830     buf = cpu_to_be32(value);
 831     migrate_send_rp_message(mis, MIG_RP_MSG_RESUME_ACK, sizeof(buf), &buf);
 832 }
 833
 834 MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp)
 835 {
 836     MigrationCapabilityStatusList *head = NULL, **tail = &head;
 837     MigrationCapabilityStatus *caps;
 838     MigrationState *s = migrate_get_current();
 839     int i;
 840
 841     for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
 842 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
 843         if (i == MIGRATION_CAPABILITY_BLOCK) {
 844             continue;
 845         }
 846 #endif
 847         caps = g_malloc0(sizeof(*caps));
 848         caps->capability = i;
 849         caps->state = s->enabled_capabilities[i];
 850         QAPI_LIST_APPEND(tail, caps);
 851     }
 852
 853     return head;
 854 }
 855
 856 MigrationParameters *qmp_query_migrate_parameters(Error **errp)
 857 {
 858     MigrationParameters *params;
 859     MigrationState *s = migrate_get_current();
 860
 861     /* TODO use QAPI_CLONE() instead of duplicating it inline */
 862     params = g_malloc0(sizeof(*params));
 863     params->has_compress_level = true;
 864     params->compress_level = s->parameters.compress_level;
 865     params->has_compress_threads = true;
 866     params->compress_threads = s->parameters.compress_threads;
 867     params->has_compress_wait_thread = true;
 868     params->compress_wait_thread = s->parameters.compress_wait_thread;
 869     params->has_decompress_threads = true;
 870     params->decompress_threads = s->parameters.decompress_threads;
 871     params->has_throttle_trigger_threshold = true;
 872     params->throttle_trigger_threshold = s->parameters.throttle_trigger_threshold;
 873     params->has_cpu_throttle_initial = true;
 874     params->cpu_throttle_initial = s->parameters.cpu_throttle_initial;
 875     params->has_cpu_throttle_increment = true;
 876     params->cpu_throttle_increment = s->parameters.cpu_throttle_increment;
 877     params->has_cpu_throttle_tailslow = true;
 878     params->cpu_throttle_tailslow = s->parameters.cpu_throttle_tailslow;
 879     params->has_tls_creds = true;
 880     params->tls_creds = g_strdup(s->parameters.tls_creds);
 881     params->has_tls_hostname = true;
 882     params->tls_hostname = g_strdup(s->parameters.tls_hostname);
 883     params->has_tls_authz = true;
 884     params->tls_authz = g_strdup(s->parameters.tls_authz ?
 885                                  s->parameters.tls_authz : "");
 886     params->has_max_bandwidth = true;
 887     params->max_bandwidth = s->parameters.max_bandwidth;
 888     params->has_downtime_limit = true;
 889     params->downtime_limit = s->parameters.downtime_limit;
 890     params->has_x_checkpoint_delay = true;
 891     params->x_checkpoint_delay = s->parameters.x_checkpoint_delay;
 892     params->has_block_incremental = true;
 893     params->block_incremental = s->parameters.block_incremental;
 894     params->has_multifd_channels = true;
 895     params->multifd_channels = s->parameters.multifd_channels;
 896     params->has_multifd_compression = true;
 897     params->multifd_compression = s->parameters.multifd_compression;
 898     params->has_multifd_zlib_level = true;
 899     params->multifd_zlib_level = s->parameters.multifd_zlib_level;
 900     params->has_multifd_zstd_level = true;
 901     params->multifd_zstd_level = s->parameters.multifd_zstd_level;
 902     params->has_xbzrle_cache_size = true;
 903     params->xbzrle_cache_size = s->parameters.xbzrle_cache_size;
 904     params->has_max_postcopy_bandwidth = true;
 905     params->max_postcopy_bandwidth = s->parameters.max_postcopy_bandwidth;
 906     params->has_max_cpu_throttle = true;
 907     params->max_cpu_throttle = s->parameters.max_cpu_throttle;
 908     params->has_announce_initial = true;
 909     params->announce_initial = s->parameters.announce_initial;
 910     params->has_announce_max = true;
 911     params->announce_max = s->parameters.announce_max;
 912     params->has_announce_rounds = true;
 913     params->announce_rounds = s->parameters.announce_rounds;
 914     params->has_announce_step = true;
 915     params->announce_step = s->parameters.announce_step;
 916
 917     if (s->parameters.has_block_bitmap_mapping) {
 918         params->has_block_bitmap_mapping = true;
 919         params->block_bitmap_mapping =
 920             QAPI_CLONE(BitmapMigrationNodeAliasList,
 921                        s->parameters.block_bitmap_mapping);
 922     }
 923
 924     return params;
 925 }
 926
 927 AnnounceParameters *migrate_announce_params(void)
 928 {
 929     static AnnounceParameters ap;
 930
 931     MigrationState *s = migrate_get_current();
 932
 933     ap.initial = s->parameters.announce_initial;
 934     ap.max = s->parameters.announce_max;
 935     ap.rounds = s->parameters.announce_rounds;
 936     ap.step = s->parameters.announce_step;
 937
 938     return &ap;
 939 }
 940
 941 /*
 942  * Return true if we're already in the middle of a migration
 943  * (i.e. any of the active or setup states)
 944  */
 945 bool migration_is_setup_or_active(int state)
 946 {
 947     switch (state) {
 948     case MIGRATION_STATUS_ACTIVE:
 949     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
 950     case MIGRATION_STATUS_POSTCOPY_PAUSED:
 951     case MIGRATION_STATUS_POSTCOPY_RECOVER:
 952     case MIGRATION_STATUS_SETUP:
 953     case MIGRATION_STATUS_PRE_SWITCHOVER:
 954     case MIGRATION_STATUS_DEVICE:
 955     case MIGRATION_STATUS_WAIT_UNPLUG:
 956     case MIGRATION_STATUS_COLO:
 957         return true;
 958
 959     default:
 960         return false;
 961
 962     }
 963 }
 964
 965 bool migration_is_running(int state)
 966 {
 967     switch (state) {
 968     case MIGRATION_STATUS_ACTIVE:
 969     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
 970     case MIGRATION_STATUS_POSTCOPY_PAUSED:
 971     case MIGRATION_STATUS_POSTCOPY_RECOVER:
 972     case MIGRATION_STATUS_SETUP:
 973     case MIGRATION_STATUS_PRE_SWITCHOVER:
 974     case MIGRATION_STATUS_DEVICE:
 975     case MIGRATION_STATUS_WAIT_UNPLUG:
 976     case MIGRATION_STATUS_CANCELLING:
 977         return true;
 978
 979     default:
 980         return false;
 981
 982     }
 983 }
 984
 985 static void populate_time_info(MigrationInfo *info, MigrationState *s)
 986 {
 987     info->has_status = true;
 988     info->has_setup_time = true;
 989     info->setup_time = s->setup_time;
 990     if (s->state == MIGRATION_STATUS_COMPLETED) {
 991         info->has_total_time = true;
 992         info->total_time = s->total_time;
 993         info->has_downtime = true;
 994         info->downtime = s->downtime;
 995     } else {
 996         info->has_total_time = true;
 997         info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) -
 998                            s->start_time;
 999         info->has_expected_downtime = true;
1000         info->expected_downtime = s->expected_downtime;
1001     }
1002 }
1003
1004 static void populate_ram_info(MigrationInfo *info, MigrationState *s)
1005 {
1006     size_t page_size = qemu_target_page_size();
1007
1008     info->has_ram = true;
1009     info->ram = g_malloc0(sizeof(*info->ram));
1010     info->ram->transferred = ram_counters.transferred;
1011     info->ram->total = ram_bytes_total();
1012     info->ram->duplicate = ram_counters.duplicate;
1013     /* legacy value.  It is not used anymore */
1014     info->ram->skipped = 0;
1015     info->ram->normal = ram_counters.normal;
1016     info->ram->normal_bytes = ram_counters.normal * page_size;
1017     info->ram->mbps = s->mbps;
1018     info->ram->dirty_sync_count = ram_counters.dirty_sync_count;
1019     info->ram->postcopy_requests = ram_counters.postcopy_requests;
1020     info->ram->page_size = page_size;
1021     info->ram->multifd_bytes = ram_counters.multifd_bytes;
1022     info->ram->pages_per_second = s->pages_per_second;
1023     info->ram->precopy_bytes = ram_counters.precopy_bytes;
1024     info->ram->downtime_bytes = ram_counters.downtime_bytes;
1025     info->ram->postcopy_bytes = ram_counters.postcopy_bytes;
1026
1027     if (migrate_use_xbzrle()) {
1028         info->has_xbzrle_cache = true;
1029         info->xbzrle_cache = g_malloc0(sizeof(*info->xbzrle_cache));
1030         info->xbzrle_cache->cache_size = migrate_xbzrle_cache_size();
1031         info->xbzrle_cache->bytes = xbzrle_counters.bytes;
1032         info->xbzrle_cache->pages = xbzrle_counters.pages;
1033         info->xbzrle_cache->cache_miss = xbzrle_counters.cache_miss;
1034         info->xbzrle_cache->cache_miss_rate = xbzrle_counters.cache_miss_rate;
1035         info->xbzrle_cache->encoding_rate = xbzrle_counters.encoding_rate;
1036         info->xbzrle_cache->overflow = xbzrle_counters.overflow;
1037     }
1038
1039     if (migrate_use_compression()) {
1040         info->has_compression = true;
1041         info->compression = g_malloc0(sizeof(*info->compression));
1042         info->compression->pages = compression_counters.pages;
1043         info->compression->busy = compression_counters.busy;
1044         info->compression->busy_rate = compression_counters.busy_rate;
1045         info->compression->compressed_size =
1046                                     compression_counters.compressed_size;
1047         info->compression->compression_rate =
1048                                     compression_counters.compression_rate;
1049     }
1050
1051     if (cpu_throttle_active()) {
1052         info->has_cpu_throttle_percentage = true;
1053         info->cpu_throttle_percentage = cpu_throttle_get_percentage();
1054     }
1055
1056     if (s->state != MIGRATION_STATUS_COMPLETED) {
1057         info->ram->remaining = ram_bytes_remaining();
1058         info->ram->dirty_pages_rate = ram_counters.dirty_pages_rate;
1059     }
1060 }
1061
1062 static void populate_disk_info(MigrationInfo *info)
1063 {
1064     if (blk_mig_active()) {
1065         info->has_disk = true;
1066         info->disk = g_malloc0(sizeof(*info->disk));
1067         info->disk->transferred = blk_mig_bytes_transferred();
1068         info->disk->remaining = blk_mig_bytes_remaining();
1069         info->disk->total = blk_mig_bytes_total();
1070     }
1071 }
1072
1073 static void fill_source_migration_info(MigrationInfo *info)
1074 {
1075     MigrationState *s = migrate_get_current();
1076     GSList *cur_blocker = migration_blockers;
1077
1078     info->blocked_reasons = NULL;
1079
1080     /*
1081      * There are two types of reasons a migration might be blocked;
1082      * a) devices marked in VMState as non-migratable, and
1083      * b) Explicit migration blockers
1084      * We need to add both of them here.
1085      */
1086     qemu_savevm_non_migratable_list(&info->blocked_reasons);
1087
1088     while (cur_blocker) {
1089         QAPI_LIST_PREPEND(info->blocked_reasons,
1090                           g_strdup(error_get_pretty(cur_blocker->data)));
1091         cur_blocker = g_slist_next(cur_blocker);
1092     }
1093     info->has_blocked_reasons = info->blocked_reasons != NULL;
1094
1095     switch (s->state) {
1096     case MIGRATION_STATUS_NONE:
1097         /* no migration has happened ever */
1098         /* do not overwrite destination migration status */
1099         return;
1100     case MIGRATION_STATUS_SETUP:
1101         info->has_status = true;
1102         info->has_total_time = false;
1103         break;
1104     case MIGRATION_STATUS_ACTIVE:
1105     case MIGRATION_STATUS_CANCELLING:
1106     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1107     case MIGRATION_STATUS_PRE_SWITCHOVER:
1108     case MIGRATION_STATUS_DEVICE:
1109     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1110     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1111         /* TODO add some postcopy stats */
1112         populate_time_info(info, s);
1113         populate_ram_info(info, s);
1114         populate_disk_info(info);
1115         populate_vfio_info(info);
1116         break;
1117     case MIGRATION_STATUS_COLO:
1118         info->has_status = true;
1119         /* TODO: display COLO specific information (checkpoint info etc.) */
1120         break;
1121     case MIGRATION_STATUS_COMPLETED:
1122         populate_time_info(info, s);
1123         populate_ram_info(info, s);
1124         populate_vfio_info(info);
1125         break;
1126     case MIGRATION_STATUS_FAILED:
1127         info->has_status = true;
1128         if (s->error) {
1129             info->has_error_desc = true;
1130             info->error_desc = g_strdup(error_get_pretty(s->error));
1131         }
1132         break;
1133     case MIGRATION_STATUS_CANCELLED:
1134         info->has_status = true;
1135         break;
1136     case MIGRATION_STATUS_WAIT_UNPLUG:
1137         info->has_status = true;
1138         break;
1139     }
1140     info->status = s->state;
1141 }
1142
1143 typedef enum WriteTrackingSupport {
1144     WT_SUPPORT_UNKNOWN = 0,
1145     WT_SUPPORT_ABSENT,
1146     WT_SUPPORT_AVAILABLE,
1147     WT_SUPPORT_COMPATIBLE
1148 } WriteTrackingSupport;
1149
1150 static
1151 WriteTrackingSupport migrate_query_write_tracking(void)
1152 {
1153     /* Check if kernel supports required UFFD features */
1154     if (!ram_write_tracking_available()) {
1155         return WT_SUPPORT_ABSENT;
1156     }
1157     /*
1158      * Check if current memory configuration is
1159      * compatible with required UFFD features.
1160      */
1161     if (!ram_write_tracking_compatible()) {
1162         return WT_SUPPORT_AVAILABLE;
1163     }
1164
1165     return WT_SUPPORT_COMPATIBLE;
1166 }
1167
1168 /**
1169  * @migration_caps_check - check capability validity
1170  *
1171  * @cap_list: old capability list, array of bool
1172  * @params: new capabilities to be applied soon
1173  * @errp: set *errp if the check failed, with reason
1174  *
1175  * Returns true if check passed, otherwise false.
1176  */
1177 static bool migrate_caps_check(bool *cap_list,
1178                                MigrationCapabilityStatusList *params,
1179                                Error **errp)
1180 {
1181     MigrationCapabilityStatusList *cap;
1182     bool old_postcopy_cap;
1183     MigrationIncomingState *mis = migration_incoming_get_current();
1184
1185     old_postcopy_cap = cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM];
1186
1187     for (cap = params; cap; cap = cap->next) {
1188         cap_list[cap->value->capability] = cap->value->state;
1189     }
1190
1191 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
1192     if (cap_list[MIGRATION_CAPABILITY_BLOCK]) {
1193         error_setg(errp, "QEMU compiled without old-style (blk/-b, inc/-i) "
1194                    "block migration");
1195         error_append_hint(errp, "Use drive_mirror+NBD instead.\n");
1196         return false;
1197     }
1198 #endif
1199
1200 #ifndef CONFIG_REPLICATION
1201     if (cap_list[MIGRATION_CAPABILITY_X_COLO]) {
1202         error_setg(errp, "QEMU compiled without replication module"
1203                    " can't enable COLO");
1204         error_append_hint(errp, "Please enable replication before COLO.\n");
1205         return false;
1206     }
1207 #endif
1208
1209     if (cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM]) {
1210         /* This check is reasonably expensive, so only when it's being
1211          * set the first time, also it's only the destination that needs
1212          * special support.
1213          */
1214         if (!old_postcopy_cap && runstate_check(RUN_STATE_INMIGRATE) &&
1215             !postcopy_ram_supported_by_host(mis)) {
1216             /* postcopy_ram_supported_by_host will have emitted a more
1217              * detailed message
1218              */
1219             error_setg(errp, "Postcopy is not supported");
1220             return false;
1221         }
1222
1223         if (cap_list[MIGRATION_CAPABILITY_X_IGNORE_SHARED]) {
1224             error_setg(errp, "Postcopy is not compatible with ignore-shared");
1225             return false;
1226         }
1227     }
1228
1229     if (cap_list[MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT]) {
1230         WriteTrackingSupport wt_support;
1231         int idx;
1232         /*
1233          * Check if 'background-snapshot' capability is supported by
1234          * host kernel and compatible with guest memory configuration.
1235          */
1236         wt_support = migrate_query_write_tracking();
1237         if (wt_support < WT_SUPPORT_AVAILABLE) {
1238             error_setg(errp, "Background-snapshot is not supported by host kernel");
1239             return false;
1240         }
1241         if (wt_support < WT_SUPPORT_COMPATIBLE) {
1242             error_setg(errp, "Background-snapshot is not compatible "
1243                     "with guest memory configuration");
1244             return false;
1245         }
1246
1247         /*
1248          * Check if there are any migration capabilities
1249          * incompatible with 'background-snapshot'.
1250          */
1251         for (idx = 0; idx < check_caps_background_snapshot.size; idx++) {
1252             int incomp_cap = check_caps_background_snapshot.caps[idx];
1253             if (cap_list[incomp_cap]) {
1254                 error_setg(errp,
1255                         "Background-snapshot is not compatible with %s",
1256                         MigrationCapability_str(incomp_cap));
1257                 return false;
1258             }
1259         }
1260     }
1261
1262     /* incoming side only */
1263     if (runstate_check(RUN_STATE_INMIGRATE) &&
1264         !migrate_multifd_is_allowed() &&
1265         cap_list[MIGRATION_CAPABILITY_MULTIFD]) {
1266         error_setg(errp, "multifd is not supported by current protocol");
1267         return false;
1268     }
1269
1270     return true;
1271 }
1272
1273 static void fill_destination_migration_info(MigrationInfo *info)
1274 {
1275     MigrationIncomingState *mis = migration_incoming_get_current();
1276
1277     if (mis->socket_address_list) {
1278         info->has_socket_address = true;
1279         info->socket_address =
1280             QAPI_CLONE(SocketAddressList, mis->socket_address_list);
1281     }
1282
1283     switch (mis->state) {
1284     case MIGRATION_STATUS_NONE:
1285         return;
1286     case MIGRATION_STATUS_SETUP:
1287     case MIGRATION_STATUS_CANCELLING:
1288     case MIGRATION_STATUS_CANCELLED:
1289     case MIGRATION_STATUS_ACTIVE:
1290     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1291     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1292     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1293     case MIGRATION_STATUS_FAILED:
1294     case MIGRATION_STATUS_COLO:
1295         info->has_status = true;
1296         break;
1297     case MIGRATION_STATUS_COMPLETED:
1298         info->has_status = true;
1299         fill_destination_postcopy_migration_info(info);
1300         break;
1301     }
1302     info->status = mis->state;
1303 }
1304
1305 MigrationInfo *qmp_query_migrate(Error **errp)
1306 {
1307     MigrationInfo *info = g_malloc0(sizeof(*info));
1308
1309     fill_destination_migration_info(info);
1310     fill_source_migration_info(info);
1311
1312     return info;
1313 }
1314
1315 void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params,
1316                                   Error **errp)
1317 {
1318     MigrationState *s = migrate_get_current();
1319     MigrationCapabilityStatusList *cap;
1320     bool cap_list[MIGRATION_CAPABILITY__MAX];
1321
1322     if (migration_is_running(s->state)) {
1323         error_setg(errp, QERR_MIGRATION_ACTIVE);
1324         return;
1325     }
1326
1327     memcpy(cap_list, s->enabled_capabilities, sizeof(cap_list));
1328     if (!migrate_caps_check(cap_list, params, errp)) {
1329         return;
1330     }
1331
1332     for (cap = params; cap; cap = cap->next) {
1333         s->enabled_capabilities[cap->value->capability] = cap->value->state;
1334     }
1335 }
1336
1337 /*
1338  * Check whether the parameters are valid. Error will be put into errp
1339  * (if provided). Return true if valid, otherwise false.
1340  */
1341 static bool migrate_params_check(MigrationParameters *params, Error **errp)
1342 {
1343     if (params->has_compress_level &&
1344         (params->compress_level > 9)) {
1345         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "compress_level",
1346                    "a value between 0 and 9");
1347         return false;
1348     }
1349
1350     if (params->has_compress_threads && (params->compress_threads < 1)) {
1351         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1352                    "compress_threads",
1353                    "a value between 1 and 255");
1354         return false;
1355     }
1356
1357     if (params->has_decompress_threads && (params->decompress_threads < 1)) {
1358         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1359                    "decompress_threads",
1360                    "a value between 1 and 255");
1361         return false;
1362     }
1363
1364     if (params->has_throttle_trigger_threshold &&
1365         (params->throttle_trigger_threshold < 1 ||
1366          params->throttle_trigger_threshold > 100)) {
1367         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1368                    "throttle_trigger_threshold",
1369                    "an integer in the range of 1 to 100");
1370         return false;
1371     }
1372
1373     if (params->has_cpu_throttle_initial &&
1374         (params->cpu_throttle_initial < 1 ||
1375          params->cpu_throttle_initial > 99)) {
1376         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1377                    "cpu_throttle_initial",
1378                    "an integer in the range of 1 to 99");
1379         return false;
1380     }
1381
1382     if (params->has_cpu_throttle_increment &&
1383         (params->cpu_throttle_increment < 1 ||
1384          params->cpu_throttle_increment > 99)) {
1385         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1386                    "cpu_throttle_increment",
1387                    "an integer in the range of 1 to 99");
1388         return false;
1389     }
1390
1391     if (params->has_max_bandwidth && (params->max_bandwidth > SIZE_MAX)) {
1392         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1393                    "max_bandwidth",
1394                    "an integer in the range of 0 to "stringify(SIZE_MAX)
1395                    " bytes/second");
1396         return false;
1397     }
1398
1399     if (params->has_downtime_limit &&
1400         (params->downtime_limit > MAX_MIGRATE_DOWNTIME)) {
1401         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1402                    "downtime_limit",
1403                    "an integer in the range of 0 to "
1404                     stringify(MAX_MIGRATE_DOWNTIME)" ms");
1405         return false;
1406     }
1407
1408     /* x_checkpoint_delay is now always positive */
1409
1410     if (params->has_multifd_channels && (params->multifd_channels < 1)) {
1411         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1412                    "multifd_channels",
1413                    "a value between 1 and 255");
1414         return false;
1415     }
1416
1417     if (params->has_multifd_zlib_level &&
1418         (params->multifd_zlib_level > 9)) {
1419         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zlib_level",
1420                    "a value between 0 and 9");
1421         return false;
1422     }
1423
1424     if (params->has_multifd_zstd_level &&
1425         (params->multifd_zstd_level > 20)) {
1426         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zstd_level",
1427                    "a value between 0 and 20");
1428         return false;
1429     }
1430
1431     if (params->has_xbzrle_cache_size &&
1432         (params->xbzrle_cache_size < qemu_target_page_size() ||
1433          !is_power_of_2(params->xbzrle_cache_size))) {
1434         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1435                    "xbzrle_cache_size",
1436                    "a power of two no less than the target page size");
1437         return false;
1438     }
1439
1440     if (params->has_max_cpu_throttle &&
1441         (params->max_cpu_throttle < params->cpu_throttle_initial ||
1442          params->max_cpu_throttle > 99)) {
1443         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1444                    "max_cpu_throttle",
1445                    "an integer in the range of cpu_throttle_initial to 99");
1446         return false;
1447     }
1448
1449     if (params->has_announce_initial &&
1450         params->announce_initial > 100000) {
1451         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1452                    "announce_initial",
1453                    "a value between 0 and 100000");
1454         return false;
1455     }
1456     if (params->has_announce_max &&
1457         params->announce_max > 100000) {
1458         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1459                    "announce_max",
1460                    "a value between 0 and 100000");
1461        return false;
1462     }
1463     if (params->has_announce_rounds &&
1464         params->announce_rounds > 1000) {
1465         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1466                    "announce_rounds",
1467                    "a value between 0 and 1000");
1468        return false;
1469     }
1470     if (params->has_announce_step &&
1471         (params->announce_step < 1 ||
1472         params->announce_step > 10000)) {
1473         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1474                    "announce_step",
1475                    "a value between 0 and 10000");
1476        return false;
1477     }
1478
1479     if (params->has_block_bitmap_mapping &&
1480         !check_dirty_bitmap_mig_alias_map(params->block_bitmap_mapping, errp)) {
1481         error_prepend(errp, "Invalid mapping given for block-bitmap-mapping: ");
1482         return false;
1483     }
1484
1485     return true;
1486 }
1487
1488 static void migrate_params_test_apply(MigrateSetParameters *params,
1489                                       MigrationParameters *dest)
1490 {
1491     *dest = migrate_get_current()->parameters;
1492
1493     /* TODO use QAPI_CLONE() instead of duplicating it inline */
1494
1495     if (params->has_compress_level) {
1496         dest->compress_level = params->compress_level;
1497     }
1498
1499     if (params->has_compress_threads) {
1500         dest->compress_threads = params->compress_threads;
1501     }
1502
1503     if (params->has_compress_wait_thread) {
1504         dest->compress_wait_thread = params->compress_wait_thread;
1505     }
1506
1507     if (params->has_decompress_threads) {
1508         dest->decompress_threads = params->decompress_threads;
1509     }
1510
1511     if (params->has_throttle_trigger_threshold) {
1512         dest->throttle_trigger_threshold = params->throttle_trigger_threshold;
1513     }
1514
1515     if (params->has_cpu_throttle_initial) {
1516         dest->cpu_throttle_initial = params->cpu_throttle_initial;
1517     }
1518
1519     if (params->has_cpu_throttle_increment) {
1520         dest->cpu_throttle_increment = params->cpu_throttle_increment;
1521     }
1522
1523     if (params->has_cpu_throttle_tailslow) {
1524         dest->cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1525     }
1526
1527     if (params->has_tls_creds) {
1528         assert(params->tls_creds->type == QTYPE_QSTRING);
1529         dest->tls_creds = params->tls_creds->u.s;
1530     }
1531
1532     if (params->has_tls_hostname) {
1533         assert(params->tls_hostname->type == QTYPE_QSTRING);
1534         dest->tls_hostname = params->tls_hostname->u.s;
1535     }
1536
1537     if (params->has_max_bandwidth) {
1538         dest->max_bandwidth = params->max_bandwidth;
1539     }
1540
1541     if (params->has_downtime_limit) {
1542         dest->downtime_limit = params->downtime_limit;
1543     }
1544
1545     if (params->has_x_checkpoint_delay) {
1546         dest->x_checkpoint_delay = params->x_checkpoint_delay;
1547     }
1548
1549     if (params->has_block_incremental) {
1550         dest->block_incremental = params->block_incremental;
1551     }
1552     if (params->has_multifd_channels) {
1553         dest->multifd_channels = params->multifd_channels;
1554     }
1555     if (params->has_multifd_compression) {
1556         dest->multifd_compression = params->multifd_compression;
1557     }
1558     if (params->has_xbzrle_cache_size) {
1559         dest->xbzrle_cache_size = params->xbzrle_cache_size;
1560     }
1561     if (params->has_max_postcopy_bandwidth) {
1562         dest->max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1563     }
1564     if (params->has_max_cpu_throttle) {
1565         dest->max_cpu_throttle = params->max_cpu_throttle;
1566     }
1567     if (params->has_announce_initial) {
1568         dest->announce_initial = params->announce_initial;
1569     }
1570     if (params->has_announce_max) {
1571         dest->announce_max = params->announce_max;
1572     }
1573     if (params->has_announce_rounds) {
1574         dest->announce_rounds = params->announce_rounds;
1575     }
1576     if (params->has_announce_step) {
1577         dest->announce_step = params->announce_step;
1578     }
1579
1580     if (params->has_block_bitmap_mapping) {
1581         dest->has_block_bitmap_mapping = true;
1582         dest->block_bitmap_mapping = params->block_bitmap_mapping;
1583     }
1584 }
1585
1586 static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
1587 {
1588     MigrationState *s = migrate_get_current();
1589
1590     /* TODO use QAPI_CLONE() instead of duplicating it inline */
1591
1592     if (params->has_compress_level) {
1593         s->parameters.compress_level = params->compress_level;
1594     }
1595
1596     if (params->has_compress_threads) {
1597         s->parameters.compress_threads = params->compress_threads;
1598     }
1599
1600     if (params->has_compress_wait_thread) {
1601         s->parameters.compress_wait_thread = params->compress_wait_thread;
1602     }
1603
1604     if (params->has_decompress_threads) {
1605         s->parameters.decompress_threads = params->decompress_threads;
1606     }
1607
1608     if (params->has_throttle_trigger_threshold) {
1609         s->parameters.throttle_trigger_threshold = params->throttle_trigger_threshold;
1610     }
1611
1612     if (params->has_cpu_throttle_initial) {
1613         s->parameters.cpu_throttle_initial = params->cpu_throttle_initial;
1614     }
1615
1616     if (params->has_cpu_throttle_increment) {
1617         s->parameters.cpu_throttle_increment = params->cpu_throttle_increment;
1618     }
1619
1620     if (params->has_cpu_throttle_tailslow) {
1621         s->parameters.cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1622     }
1623
1624     if (params->has_tls_creds) {
1625         g_free(s->parameters.tls_creds);
1626         assert(params->tls_creds->type == QTYPE_QSTRING);
1627         s->parameters.tls_creds = g_strdup(params->tls_creds->u.s);
1628     }
1629
1630     if (params->has_tls_hostname) {
1631         g_free(s->parameters.tls_hostname);
1632         assert(params->tls_hostname->type == QTYPE_QSTRING);
1633         s->parameters.tls_hostname = g_strdup(params->tls_hostname->u.s);
1634     }
1635
1636     if (params->has_tls_authz) {
1637         g_free(s->parameters.tls_authz);
1638         assert(params->tls_authz->type == QTYPE_QSTRING);
1639         s->parameters.tls_authz = g_strdup(params->tls_authz->u.s);
1640     }
1641
1642     if (params->has_max_bandwidth) {
1643         s->parameters.max_bandwidth = params->max_bandwidth;
1644         if (s->to_dst_file && !migration_in_postcopy()) {
1645             qemu_file_set_rate_limit(s->to_dst_file,
1646                                 s->parameters.max_bandwidth / XFER_LIMIT_RATIO);
1647         }
1648     }
1649
1650     if (params->has_downtime_limit) {
1651         s->parameters.downtime_limit = params->downtime_limit;
1652     }
1653
1654     if (params->has_x_checkpoint_delay) {
1655         s->parameters.x_checkpoint_delay = params->x_checkpoint_delay;
1656         if (migration_in_colo_state()) {
1657             colo_checkpoint_notify(s);
1658         }
1659     }
1660
1661     if (params->has_block_incremental) {
1662         s->parameters.block_incremental = params->block_incremental;
1663     }
1664     if (params->has_multifd_channels) {
1665         s->parameters.multifd_channels = params->multifd_channels;
1666     }
1667     if (params->has_multifd_compression) {
1668         s->parameters.multifd_compression = params->multifd_compression;
1669     }
1670     if (params->has_xbzrle_cache_size) {
1671         s->parameters.xbzrle_cache_size = params->xbzrle_cache_size;
1672         xbzrle_cache_resize(params->xbzrle_cache_size, errp);
1673     }
1674     if (params->has_max_postcopy_bandwidth) {
1675         s->parameters.max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1676         if (s->to_dst_file && migration_in_postcopy()) {
1677             qemu_file_set_rate_limit(s->to_dst_file,
1678                     s->parameters.max_postcopy_bandwidth / XFER_LIMIT_RATIO);
1679         }
1680     }
1681     if (params->has_max_cpu_throttle) {
1682         s->parameters.max_cpu_throttle = params->max_cpu_throttle;
1683     }
1684     if (params->has_announce_initial) {
1685         s->parameters.announce_initial = params->announce_initial;
1686     }
1687     if (params->has_announce_max) {
1688         s->parameters.announce_max = params->announce_max;
1689     }
1690     if (params->has_announce_rounds) {
1691         s->parameters.announce_rounds = params->announce_rounds;
1692     }
1693     if (params->has_announce_step) {
1694         s->parameters.announce_step = params->announce_step;
1695     }
1696
1697     if (params->has_block_bitmap_mapping) {
1698         qapi_free_BitmapMigrationNodeAliasList(
1699             s->parameters.block_bitmap_mapping);
1700
1701         s->parameters.has_block_bitmap_mapping = true;
1702         s->parameters.block_bitmap_mapping =
1703             QAPI_CLONE(BitmapMigrationNodeAliasList,
1704                        params->block_bitmap_mapping);
1705     }
1706 }
1707
1708 void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp)
1709 {
1710     MigrationParameters tmp;
1711
1712     /* TODO Rewrite "" to null instead */
1713     if (params->has_tls_creds
1714         && params->tls_creds->type == QTYPE_QNULL) {
1715         qobject_unref(params->tls_creds->u.n);
1716         params->tls_creds->type = QTYPE_QSTRING;
1717         params->tls_creds->u.s = strdup("");
1718     }
1719     /* TODO Rewrite "" to null instead */
1720     if (params->has_tls_hostname
1721         && params->tls_hostname->type == QTYPE_QNULL) {
1722         qobject_unref(params->tls_hostname->u.n);
1723         params->tls_hostname->type = QTYPE_QSTRING;
1724         params->tls_hostname->u.s = strdup("");
1725     }
1726
1727     migrate_params_test_apply(params, &tmp);
1728
1729     if (!migrate_params_check(&tmp, errp)) {
1730         /* Invalid parameter */
1731         return;
1732     }
1733
1734     migrate_params_apply(params, errp);
1735 }
1736
1737
1738 void qmp_migrate_start_postcopy(Error **errp)
1739 {
1740     MigrationState *s = migrate_get_current();
1741
1742     if (!migrate_postcopy()) {
1743         error_setg(errp, "Enable postcopy with migrate_set_capability before"
1744                          " the start of migration");
1745         return;
1746     }
1747
1748     if (s->state == MIGRATION_STATUS_NONE) {
1749         error_setg(errp, "Postcopy must be started after migration has been"
1750                          " started");
1751         return;
1752     }
1753     /*
1754      * we don't error if migration has finished since that would be racy
1755      * with issuing this command.
1756      */
1757     qatomic_set(&s->start_postcopy, true);
1758 }
1759
1760 /* shared migration helpers */
1761
1762 void migrate_set_state(int *state, int old_state, int new_state)
1763 {
1764     assert(new_state < MIGRATION_STATUS__MAX);
1765     if (qatomic_cmpxchg(state, old_state, new_state) == old_state) {
1766         trace_migrate_set_state(MigrationStatus_str(new_state));
1767         migrate_generate_event(new_state);
1768     }
1769 }
1770
1771 static MigrationCapabilityStatus *migrate_cap_add(MigrationCapability index,
1772                                                   bool state)
1773 {
1774     MigrationCapabilityStatus *cap;
1775
1776     cap = g_new0(MigrationCapabilityStatus, 1);
1777     cap->capability = index;
1778     cap->state = state;
1779
1780     return cap;
1781 }
1782
1783 void migrate_set_block_enabled(bool value, Error **errp)
1784 {
1785     MigrationCapabilityStatusList *cap = NULL;
1786
1787     QAPI_LIST_PREPEND(cap, migrate_cap_add(MIGRATION_CAPABILITY_BLOCK, value));
1788     qmp_migrate_set_capabilities(cap, errp);
1789     qapi_free_MigrationCapabilityStatusList(cap);
1790 }
1791
1792 static void migrate_set_block_incremental(MigrationState *s, bool value)
1793 {
1794     s->parameters.block_incremental = value;
1795 }
1796
1797 static void block_cleanup_parameters(MigrationState *s)
1798 {
1799     if (s->must_remove_block_options) {
1800         /* setting to false can never fail */
1801         migrate_set_block_enabled(false, &error_abort);
1802         migrate_set_block_incremental(s, false);
1803         s->must_remove_block_options = false;
1804     }
1805 }
1806
1807 static void migrate_fd_cleanup(MigrationState *s)
1808 {
1809     qemu_bh_delete(s->cleanup_bh);
1810     s->cleanup_bh = NULL;
1811
1812     g_free(s->hostname);
1813     s->hostname = NULL;
1814
1815     qemu_savevm_state_cleanup();
1816
1817     if (s->to_dst_file) {
1818         QEMUFile *tmp;
1819
1820         trace_migrate_fd_cleanup();
1821         qemu_mutex_unlock_iothread();
1822         if (s->migration_thread_running) {
1823             qemu_thread_join(&s->thread);
1824             s->migration_thread_running = false;
1825         }
1826         qemu_mutex_lock_iothread();
1827
1828         multifd_save_cleanup();
1829         qemu_mutex_lock(&s->qemu_file_lock);
1830         tmp = s->to_dst_file;
1831         s->to_dst_file = NULL;
1832         qemu_mutex_unlock(&s->qemu_file_lock);
1833         /*
1834          * Close the file handle without the lock to make sure the
1835          * critical section won't block for long.
1836          */
1837         migration_ioc_unregister_yank_from_file(tmp);
1838         qemu_fclose(tmp);
1839     }
1840
1841     assert(!migration_is_active(s));
1842
1843     if (s->state == MIGRATION_STATUS_CANCELLING) {
1844         migrate_set_state(&s->state, MIGRATION_STATUS_CANCELLING,
1845                           MIGRATION_STATUS_CANCELLED);
1846     }
1847
1848     if (s->error) {
1849         /* It is used on info migrate.  We can't free it */
1850         error_report_err(error_copy(s->error));
1851     }
1852     notifier_list_notify(&migration_state_notifiers, s);
1853     block_cleanup_parameters(s);
1854     yank_unregister_instance(MIGRATION_YANK_INSTANCE);
1855 }
1856
1857 static void migrate_fd_cleanup_schedule(MigrationState *s)
1858 {
1859     /*
1860      * Ref the state for bh, because it may be called when
1861      * there're already no other refs
1862      */
1863     object_ref(OBJECT(s));
1864     qemu_bh_schedule(s->cleanup_bh);
1865 }
1866
1867 static void migrate_fd_cleanup_bh(void *opaque)
1868 {
1869     MigrationState *s = opaque;
1870     migrate_fd_cleanup(s);
1871     object_unref(OBJECT(s));
1872 }
1873
1874 void migrate_set_error(MigrationState *s, const Error *error)
1875 {
1876     QEMU_LOCK_GUARD(&s->error_mutex);
1877     if (!s->error) {
1878         s->error = error_copy(error);
1879     }
1880 }
1881
1882 static void migrate_error_free(MigrationState *s)
1883 {
1884     QEMU_LOCK_GUARD(&s->error_mutex);
1885     if (s->error) {
1886         error_free(s->error);
1887         s->error = NULL;
1888     }
1889 }
1890
1891 void migrate_fd_error(MigrationState *s, const Error *error)
1892 {
1893     trace_migrate_fd_error(error_get_pretty(error));
1894     assert(s->to_dst_file == NULL);
1895     migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
1896                       MIGRATION_STATUS_FAILED);
1897     migrate_set_error(s, error);
1898 }
1899
1900 static void migrate_fd_cancel(MigrationState *s)
1901 {
1902     int old_state ;
1903     QEMUFile *f = migrate_get_current()->to_dst_file;
1904     trace_migrate_fd_cancel();
1905
1906     WITH_QEMU_LOCK_GUARD(&s->qemu_file_lock) {
1907         if (s->rp_state.from_dst_file) {
1908             /* shutdown the rp socket, so causing the rp thread to shutdown */
1909             qemu_file_shutdown(s->rp_state.from_dst_file);
1910         }
1911     }
1912
1913     do {
1914         old_state = s->state;
1915         if (!migration_is_running(old_state)) {
1916             break;
1917         }
1918         /* If the migration is paused, kick it out of the pause */
1919         if (old_state == MIGRATION_STATUS_PRE_SWITCHOVER) {
1920             qemu_sem_post(&s->pause_sem);
1921         }
1922         migrate_set_state(&s->state, old_state, MIGRATION_STATUS_CANCELLING);
1923     } while (s->state != MIGRATION_STATUS_CANCELLING);
1924
1925     /*
1926      * If we're unlucky the migration code might be stuck somewhere in a
1927      * send/write while the network has failed and is waiting to timeout;
1928      * if we've got shutdown(2) available then we can force it to quit.
1929      * The outgoing qemu file gets closed in migrate_fd_cleanup that is
1930      * called in a bh, so there is no race against this cancel.
1931      */
1932     if (s->state == MIGRATION_STATUS_CANCELLING && f) {
1933         qemu_file_shutdown(f);
1934     }
1935     if (s->state == MIGRATION_STATUS_CANCELLING && s->block_inactive) {
1936         Error *local_err = NULL;
1937
1938         bdrv_activate_all(&local_err);
1939         if (local_err) {
1940             error_report_err(local_err);
1941         } else {
1942             s->block_inactive = false;
1943         }
1944     }
1945 }
1946
1947 void add_migration_state_change_notifier(Notifier *notify)
1948 {
1949     notifier_list_add(&migration_state_notifiers, notify);
1950 }
1951
1952 void remove_migration_state_change_notifier(Notifier *notify)
1953 {
1954     notifier_remove(notify);
1955 }
1956
1957 bool migration_in_setup(MigrationState *s)
1958 {
1959     return s->state == MIGRATION_STATUS_SETUP;
1960 }
1961
1962 bool migration_has_finished(MigrationState *s)
1963 {
1964     return s->state == MIGRATION_STATUS_COMPLETED;
1965 }
1966
1967 bool migration_has_failed(MigrationState *s)
1968 {
1969     return (s->state == MIGRATION_STATUS_CANCELLED ||
1970             s->state == MIGRATION_STATUS_FAILED);
1971 }
1972
1973 bool migration_in_postcopy(void)
1974 {
1975     MigrationState *s = migrate_get_current();
1976
1977     switch (s->state) {
1978     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1979     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1980     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1981         return true;
1982     default:
1983         return false;
1984     }
1985 }
1986
1987 bool migration_in_postcopy_after_devices(MigrationState *s)
1988 {
1989     return migration_in_postcopy() && s->postcopy_after_devices;
1990 }
1991
1992 bool migration_in_incoming_postcopy(void)
1993 {
1994     PostcopyState ps = postcopy_state_get();
1995
1996     return ps >= POSTCOPY_INCOMING_DISCARD && ps < POSTCOPY_INCOMING_END;
1997 }
1998
1999 bool migration_in_bg_snapshot(void)
2000 {
2001     MigrationState *s = migrate_get_current();
2002
2003     return migrate_background_snapshot() &&
2004             migration_is_setup_or_active(s->state);
2005 }
2006
2007 bool migration_is_idle(void)
2008 {
2009     MigrationState *s = current_migration;
2010
2011     if (!s) {
2012         return true;
2013     }
2014
2015     switch (s->state) {
2016     case MIGRATION_STATUS_NONE:
2017     case MIGRATION_STATUS_CANCELLED:
2018     case MIGRATION_STATUS_COMPLETED:
2019     case MIGRATION_STATUS_FAILED:
2020         return true;
2021     case MIGRATION_STATUS_SETUP:
2022     case MIGRATION_STATUS_CANCELLING:
2023     case MIGRATION_STATUS_ACTIVE:
2024     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
2025     case MIGRATION_STATUS_COLO:
2026     case MIGRATION_STATUS_PRE_SWITCHOVER:
2027     case MIGRATION_STATUS_DEVICE:
2028     case MIGRATION_STATUS_WAIT_UNPLUG:
2029         return false;
2030     case MIGRATION_STATUS__MAX:
2031         g_assert_not_reached();
2032     }
2033
2034     return false;
2035 }
2036
2037 bool migration_is_active(MigrationState *s)
2038 {
2039     return (s->state == MIGRATION_STATUS_ACTIVE ||
2040             s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
2041 }
2042
2043 void migrate_init(MigrationState *s)
2044 {
2045     /*
2046      * Reinitialise all migration state, except
2047      * parameters/capabilities that the user set, and
2048      * locks.
2049      */
2050     s->cleanup_bh = 0;
2051     s->vm_start_bh = 0;
2052     s->to_dst_file = NULL;
2053     s->state = MIGRATION_STATUS_NONE;
2054     s->rp_state.from_dst_file = NULL;
2055     s->rp_state.error = false;
2056     s->mbps = 0.0;
2057     s->pages_per_second = 0.0;
2058     s->downtime = 0;
2059     s->expected_downtime = 0;
2060     s->setup_time = 0;
2061     s->start_postcopy = false;
2062     s->postcopy_after_devices = false;
2063     s->migration_thread_running = false;
2064     error_free(s->error);
2065     s->error = NULL;
2066     s->hostname = NULL;
2067
2068     migrate_set_state(&s->state, MIGRATION_STATUS_NONE, MIGRATION_STATUS_SETUP);
2069
2070     s->start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
2071     s->total_time = 0;
2072     s->vm_was_running = false;
2073     s->iteration_initial_bytes = 0;
2074     s->threshold_size = 0;
2075 }
2076
2077 int migrate_add_blocker_internal(Error *reason, Error **errp)
2078 {
2079     /* Snapshots are similar to migrations, so check RUN_STATE_SAVE_VM too. */
2080     if (runstate_check(RUN_STATE_SAVE_VM) || !migration_is_idle()) {
2081         error_propagate_prepend(errp, error_copy(reason),
2082                                 "disallowing migration blocker "
2083                                 "(migration/snapshot in progress) for: ");
2084         return -EBUSY;
2085     }
2086
2087     migration_blockers = g_slist_prepend(migration_blockers, reason);
2088     return 0;
2089 }
2090
2091 int migrate_add_blocker(Error *reason, Error **errp)
2092 {
2093     if (only_migratable) {
2094         error_propagate_prepend(errp, error_copy(reason),
2095                                 "disallowing migration blocker "
2096                                 "(--only-migratable) for: ");
2097         return -EACCES;
2098     }
2099
2100     return migrate_add_blocker_internal(reason, errp);
2101 }
2102
2103 void migrate_del_blocker(Error *reason)
2104 {
2105     migration_blockers = g_slist_remove(migration_blockers, reason);
2106 }
2107
2108 void qmp_migrate_incoming(const char *uri, Error **errp)
2109 {
2110     Error *local_err = NULL;
2111     static bool once = true;
2112
2113     if (!once) {
2114         error_setg(errp, "The incoming migration has already been started");
2115         return;
2116     }
2117     if (!runstate_check(RUN_STATE_INMIGRATE)) {
2118         error_setg(errp, "'-incoming' was not specified on the command line");
2119         return;
2120     }
2121
2122     if (!yank_register_instance(MIGRATION_YANK_INSTANCE, errp)) {
2123         return;
2124     }
2125
2126     qemu_start_incoming_migration(uri, &local_err);
2127
2128     if (local_err) {
2129         yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2130         error_propagate(errp, local_err);
2131         return;
2132     }
2133
2134     once = false;
2135 }
2136
2137 void qmp_migrate_recover(const char *uri, Error **errp)
2138 {
2139     MigrationIncomingState *mis = migration_incoming_get_current();
2140
2141     /*
2142      * Don't even bother to use ERRP_GUARD() as it _must_ always be set by
2143      * callers (no one should ignore a recover failure); if there is, it's a
2144      * programming error.
2145      */
2146     assert(errp);
2147
2148     if (mis->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
2149         error_setg(errp, "Migrate recover can only be run "
2150                    "when postcopy is paused.");
2151         return;
2152     }
2153
2154     if (qatomic_cmpxchg(&mis->postcopy_recover_triggered,
2155                        false, true) == true) {
2156         error_setg(errp, "Migrate recovery is triggered already");
2157         return;
2158     }
2159
2160     /*
2161      * Note that this call will never start a real migration; it will
2162      * only re-setup the migration stream and poke existing migration
2163      * to continue using that newly established channel.
2164      */
2165     qemu_start_incoming_migration(uri, errp);
2166
2167     /* Safe to dereference with the assert above */
2168     if (*errp) {
2169         /* Reset the flag so user could still retry */
2170         qatomic_set(&mis->postcopy_recover_triggered, false);
2171     }
2172 }
2173
2174 void qmp_migrate_pause(Error **errp)
2175 {
2176     MigrationState *ms = migrate_get_current();
2177     MigrationIncomingState *mis = migration_incoming_get_current();
2178     int ret;
2179
2180     if (ms->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
2181         /* Source side, during postcopy */
2182         qemu_mutex_lock(&ms->qemu_file_lock);
2183         ret = qemu_file_shutdown(ms->to_dst_file);
2184         qemu_mutex_unlock(&ms->qemu_file_lock);
2185         if (ret) {
2186             error_setg(errp, "Failed to pause source migration");
2187         }
2188         return;
2189     }
2190
2191     if (mis->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
2192         ret = qemu_file_shutdown(mis->from_src_file);
2193         if (ret) {
2194             error_setg(errp, "Failed to pause destination migration");
2195         }
2196         return;
2197     }
2198
2199     error_setg(errp, "migrate-pause is currently only supported "
2200                "during postcopy-active state");
2201 }
2202
2203 bool migration_is_blocked(Error **errp)
2204 {
2205     if (qemu_savevm_state_blocked(errp)) {
2206         return true;
2207     }
2208
2209     if (migration_blockers) {
2210         error_propagate(errp, error_copy(migration_blockers->data));
2211         return true;
2212     }
2213
2214     return false;
2215 }
2216
2217 /* Returns true if continue to migrate, or false if error detected */
2218 static bool migrate_prepare(MigrationState *s, bool blk, bool blk_inc,
2219                             bool resume, Error **errp)
2220 {
2221     Error *local_err = NULL;
2222
2223     if (resume) {
2224         if (s->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
2225             error_setg(errp, "Cannot resume if there is no "
2226                        "paused migration");
2227             return false;
2228         }
2229
2230         /*
2231          * Postcopy recovery won't work well with release-ram
2232          * capability since release-ram will drop the page buffer as
2233          * long as the page is put into the send buffer.  So if there
2234          * is a network failure happened, any page buffers that have
2235          * not yet reached the destination VM but have already been
2236          * sent from the source VM will be lost forever.  Let's refuse
2237          * the client from resuming such a postcopy migration.
2238          * Luckily release-ram was designed to only be used when src
2239          * and destination VMs are on the same host, so it should be
2240          * fine.
2241          */
2242         if (migrate_release_ram()) {
2243             error_setg(errp, "Postcopy recovery cannot work "
2244                        "when release-ram capability is set");
2245             return false;
2246         }
2247
2248         /* This is a resume, skip init status */
2249         return true;
2250     }
2251
2252     if (migration_is_running(s->state)) {
2253         error_setg(errp, QERR_MIGRATION_ACTIVE);
2254         return false;
2255     }
2256
2257     if (runstate_check(RUN_STATE_INMIGRATE)) {
2258         error_setg(errp, "Guest is waiting for an incoming migration");
2259         return false;
2260     }
2261
2262     if (runstate_check(RUN_STATE_POSTMIGRATE)) {
2263         error_setg(errp, "Can't migrate the vm that was paused due to "
2264                    "previous migration");
2265         return false;
2266     }
2267
2268     if (migration_is_blocked(errp)) {
2269         return false;
2270     }
2271
2272     if (blk || blk_inc) {
2273         if (migrate_colo_enabled()) {
2274             error_setg(errp, "No disk migration is required in COLO mode");
2275             return false;
2276         }
2277         if (migrate_use_block() || migrate_use_block_incremental()) {
2278             error_setg(errp, "Command options are incompatible with "
2279                        "current migration capabilities");
2280             return false;
2281         }
2282         migrate_set_block_enabled(true, &local_err);
2283         if (local_err) {
2284             error_propagate(errp, local_err);
2285             return false;
2286         }
2287         s->must_remove_block_options = true;
2288     }
2289
2290     if (blk_inc) {
2291         migrate_set_block_incremental(s, true);
2292     }
2293
2294     migrate_init(s);
2295     /*
2296      * set ram_counters compression_counters memory to zero for a
2297      * new migration
2298      */
2299     memset(&ram_counters, 0, sizeof(ram_counters));
2300     memset(&compression_counters, 0, sizeof(compression_counters));
2301
2302     return true;
2303 }
2304
2305 void qmp_migrate(const char *uri, bool has_blk, bool blk,
2306                  bool has_inc, bool inc, bool has_detach, bool detach,
2307                  bool has_resume, bool resume, Error **errp)
2308 {
2309     Error *local_err = NULL;
2310     MigrationState *s = migrate_get_current();
2311     const char *p = NULL;
2312
2313     if (!migrate_prepare(s, has_blk && blk, has_inc && inc,
2314                          has_resume && resume, errp)) {
2315         /* Error detected, put into errp */
2316         return;
2317     }
2318
2319     if (!(has_resume && resume)) {
2320         if (!yank_register_instance(MIGRATION_YANK_INSTANCE, errp)) {
2321             return;
2322         }
2323     }
2324
2325     migrate_protocol_allow_multifd(false);
2326     if (strstart(uri, "tcp:", &p) ||
2327         strstart(uri, "unix:", NULL) ||
2328         strstart(uri, "vsock:", NULL)) {
2329         migrate_protocol_allow_multifd(true);
2330         socket_start_outgoing_migration(s, p ? p : uri, &local_err);
2331 #ifdef CONFIG_RDMA
2332     } else if (strstart(uri, "rdma:", &p)) {
2333         rdma_start_outgoing_migration(s, p, &local_err);
2334 #endif
2335     } else if (strstart(uri, "exec:", &p)) {
2336         exec_start_outgoing_migration(s, p, &local_err);
2337     } else if (strstart(uri, "fd:", &p)) {
2338         fd_start_outgoing_migration(s, p, &local_err);
2339     } else {
2340         if (!(has_resume && resume)) {
2341             yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2342         }
2343         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "uri",
2344                    "a valid migration protocol");
2345         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
2346                           MIGRATION_STATUS_FAILED);
2347         block_cleanup_parameters(s);
2348         return;
2349     }
2350
2351     if (local_err) {
2352         if (!(has_resume && resume)) {
2353             yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2354         }
2355         migrate_fd_error(s, local_err);
2356         error_propagate(errp, local_err);
2357         return;
2358     }
2359 }
2360
2361 void qmp_migrate_cancel(Error **errp)
2362 {
2363     migration_cancel(NULL);
2364 }
2365
2366 void qmp_migrate_continue(MigrationStatus state, Error **errp)
2367 {
2368     MigrationState *s = migrate_get_current();
2369     if (s->state != state) {
2370         error_setg(errp,  "Migration not in expected state: %s",
2371                    MigrationStatus_str(s->state));
2372         return;
2373     }
2374     qemu_sem_post(&s->pause_sem);
2375 }
2376
2377 bool migrate_release_ram(void)
2378 {
2379     MigrationState *s;
2380
2381     s = migrate_get_current();
2382
2383     return s->enabled_capabilities[MIGRATION_CAPABILITY_RELEASE_RAM];
2384 }
2385
2386 bool migrate_postcopy_ram(void)
2387 {
2388     MigrationState *s;
2389
2390     s = migrate_get_current();
2391
2392     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_RAM];
2393 }
2394
2395 bool migrate_postcopy(void)
2396 {
2397     return migrate_postcopy_ram() || migrate_dirty_bitmaps();
2398 }
2399
2400 bool migrate_auto_converge(void)
2401 {
2402     MigrationState *s;
2403
2404     s = migrate_get_current();
2405
2406     return s->enabled_capabilities[MIGRATION_CAPABILITY_AUTO_CONVERGE];
2407 }
2408
2409 bool migrate_zero_blocks(void)
2410 {
2411     MigrationState *s;
2412
2413     s = migrate_get_current();
2414
2415     return s->enabled_capabilities[MIGRATION_CAPABILITY_ZERO_BLOCKS];
2416 }
2417
2418 bool migrate_postcopy_blocktime(void)
2419 {
2420     MigrationState *s;
2421
2422     s = migrate_get_current();
2423
2424     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME];
2425 }
2426
2427 bool migrate_use_compression(void)
2428 {
2429     MigrationState *s;
2430
2431     s = migrate_get_current();
2432
2433     return s->enabled_capabilities[MIGRATION_CAPABILITY_COMPRESS];
2434 }
2435
2436 int migrate_compress_level(void)
2437 {
2438     MigrationState *s;
2439
2440     s = migrate_get_current();
2441
2442     return s->parameters.compress_level;
2443 }
2444
2445 int migrate_compress_threads(void)
2446 {
2447     MigrationState *s;
2448
2449     s = migrate_get_current();
2450
2451     return s->parameters.compress_threads;
2452 }
2453
2454 int migrate_compress_wait_thread(void)
2455 {
2456     MigrationState *s;
2457
2458     s = migrate_get_current();
2459
2460     return s->parameters.compress_wait_thread;
2461 }
2462
2463 int migrate_decompress_threads(void)
2464 {
2465     MigrationState *s;
2466
2467     s = migrate_get_current();
2468
2469     return s->parameters.decompress_threads;
2470 }
2471
2472 bool migrate_dirty_bitmaps(void)
2473 {
2474     MigrationState *s;
2475
2476     s = migrate_get_current();
2477
2478     return s->enabled_capabilities[MIGRATION_CAPABILITY_DIRTY_BITMAPS];
2479 }
2480
2481 bool migrate_ignore_shared(void)
2482 {
2483     MigrationState *s;
2484
2485     s = migrate_get_current();
2486
2487     return s->enabled_capabilities[MIGRATION_CAPABILITY_X_IGNORE_SHARED];
2488 }
2489
2490 bool migrate_validate_uuid(void)
2491 {
2492     MigrationState *s;
2493
2494     s = migrate_get_current();
2495
2496     return s->enabled_capabilities[MIGRATION_CAPABILITY_VALIDATE_UUID];
2497 }
2498
2499 bool migrate_use_events(void)
2500 {
2501     MigrationState *s;
2502
2503     s = migrate_get_current();
2504
2505     return s->enabled_capabilities[MIGRATION_CAPABILITY_EVENTS];
2506 }
2507
2508 bool migrate_use_multifd(void)
2509 {
2510     MigrationState *s;
2511
2512     s = migrate_get_current();
2513
2514     return s->enabled_capabilities[MIGRATION_CAPABILITY_MULTIFD];
2515 }
2516
2517 bool migrate_pause_before_switchover(void)
2518 {
2519     MigrationState *s;
2520
2521     s = migrate_get_current();
2522
2523     return s->enabled_capabilities[
2524         MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER];
2525 }
2526
2527 int migrate_multifd_channels(void)
2528 {
2529     MigrationState *s;
2530
2531     s = migrate_get_current();
2532
2533     return s->parameters.multifd_channels;
2534 }
2535
2536 MultiFDCompression migrate_multifd_compression(void)
2537 {
2538     MigrationState *s;
2539
2540     s = migrate_get_current();
2541
2542     return s->parameters.multifd_compression;
2543 }
2544
2545 int migrate_multifd_zlib_level(void)
2546 {
2547     MigrationState *s;
2548
2549     s = migrate_get_current();
2550
2551     return s->parameters.multifd_zlib_level;
2552 }
2553
2554 int migrate_multifd_zstd_level(void)
2555 {
2556     MigrationState *s;
2557
2558     s = migrate_get_current();
2559
2560     return s->parameters.multifd_zstd_level;
2561 }
2562
2563 int migrate_use_xbzrle(void)
2564 {
2565     MigrationState *s;
2566
2567     s = migrate_get_current();
2568
2569     return s->enabled_capabilities[MIGRATION_CAPABILITY_XBZRLE];
2570 }
2571
2572 uint64_t migrate_xbzrle_cache_size(void)
2573 {
2574     MigrationState *s;
2575
2576     s = migrate_get_current();
2577
2578     return s->parameters.xbzrle_cache_size;
2579 }
2580
2581 static int64_t migrate_max_postcopy_bandwidth(void)
2582 {
2583     MigrationState *s;
2584
2585     s = migrate_get_current();
2586
2587     return s->parameters.max_postcopy_bandwidth;
2588 }
2589
2590 bool migrate_use_block(void)
2591 {
2592     MigrationState *s;
2593
2594     s = migrate_get_current();
2595
2596     return s->enabled_capabilities[MIGRATION_CAPABILITY_BLOCK];
2597 }
2598
2599 bool migrate_use_return_path(void)
2600 {
2601     MigrationState *s;
2602
2603     s = migrate_get_current();
2604
2605     return s->enabled_capabilities[MIGRATION_CAPABILITY_RETURN_PATH];
2606 }
2607
2608 bool migrate_use_block_incremental(void)
2609 {
2610     MigrationState *s;
2611
2612     s = migrate_get_current();
2613
2614     return s->parameters.block_incremental;
2615 }
2616
2617 bool migrate_background_snapshot(void)
2618 {
2619     MigrationState *s;
2620
2621     s = migrate_get_current();
2622
2623     return s->enabled_capabilities[MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT];
2624 }
2625
2626 /* migration thread support */
2627 /*
2628  * Something bad happened to the RP stream, mark an error
2629  * The caller shall print or trace something to indicate why
2630  */
2631 static void mark_source_rp_bad(MigrationState *s)
2632 {
2633     s->rp_state.error = true;
2634 }
2635
2636 static struct rp_cmd_args {
2637     ssize_t     len; /* -1 = variable */
2638     const char *name;
2639 } rp_cmd_args[] = {
2640     [MIG_RP_MSG_INVALID]        = { .len = -1, .name = "INVALID" },
2641     [MIG_RP_MSG_SHUT]           = { .len =  4, .name = "SHUT" },
2642     [MIG_RP_MSG_PONG]           = { .len =  4, .name = "PONG" },
2643     [MIG_RP_MSG_REQ_PAGES]      = { .len = 12, .name = "REQ_PAGES" },
2644     [MIG_RP_MSG_REQ_PAGES_ID]   = { .len = -1, .name = "REQ_PAGES_ID" },
2645     [MIG_RP_MSG_RECV_BITMAP]    = { .len = -1, .name = "RECV_BITMAP" },
2646     [MIG_RP_MSG_RESUME_ACK]     = { .len =  4, .name = "RESUME_ACK" },
2647     [MIG_RP_MSG_MAX]            = { .len = -1, .name = "MAX" },
2648 };
2649
2650 /*
2651  * Process a request for pages received on the return path,
2652  * We're allowed to send more than requested (e.g. to round to our page size)
2653  * and we don't need to send pages that have already been sent.
2654  */
2655 static void migrate_handle_rp_req_pages(MigrationState *ms, const char* rbname,
2656                                        ram_addr_t start, size_t len)
2657 {
2658     long our_host_ps = qemu_real_host_page_size();
2659
2660     trace_migrate_handle_rp_req_pages(rbname, start, len);
2661
2662     /*
2663      * Since we currently insist on matching page sizes, just sanity check
2664      * we're being asked for whole host pages.
2665      */
2666     if (!QEMU_IS_ALIGNED(start, our_host_ps) ||
2667         !QEMU_IS_ALIGNED(len, our_host_ps)) {
2668         error_report("%s: Misaligned page request, start: " RAM_ADDR_FMT
2669                      " len: %zd", __func__, start, len);
2670         mark_source_rp_bad(ms);
2671         return;
2672     }
2673
2674     if (ram_save_queue_pages(rbname, start, len)) {
2675         mark_source_rp_bad(ms);
2676     }
2677 }
2678
2679 /* Return true to retry, false to quit */
2680 static bool postcopy_pause_return_path_thread(MigrationState *s)
2681 {
2682     trace_postcopy_pause_return_path();
2683
2684     qemu_sem_wait(&s->postcopy_pause_rp_sem);
2685
2686     trace_postcopy_pause_return_path_continued();
2687
2688     return true;
2689 }
2690
2691 static int migrate_handle_rp_recv_bitmap(MigrationState *s, char *block_name)
2692 {
2693     RAMBlock *block = qemu_ram_block_by_name(block_name);
2694
2695     if (!block) {
2696         error_report("%s: invalid block name '%s'", __func__, block_name);
2697         return -EINVAL;
2698     }
2699
2700     /* Fetch the received bitmap and refresh the dirty bitmap */
2701     return ram_dirty_bitmap_reload(s, block);
2702 }
2703
2704 static int migrate_handle_rp_resume_ack(MigrationState *s, uint32_t value)
2705 {
2706     trace_source_return_path_thread_resume_ack(value);
2707
2708     if (value != MIGRATION_RESUME_ACK_VALUE) {
2709         error_report("%s: illegal resume_ack value %"PRIu32,
2710                      __func__, value);
2711         return -1;
2712     }
2713
2714     /* Now both sides are active. */
2715     migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_RECOVER,
2716                       MIGRATION_STATUS_POSTCOPY_ACTIVE);
2717
2718     /* Notify send thread that time to continue send pages */
2719     qemu_sem_post(&s->rp_state.rp_sem);
2720
2721     return 0;
2722 }
2723
2724 /* Release ms->rp_state.from_dst_file in a safe way */
2725 static void migration_release_from_dst_file(MigrationState *ms)
2726 {
2727     QEMUFile *file;
2728
2729     WITH_QEMU_LOCK_GUARD(&ms->qemu_file_lock) {
2730         /*
2731          * Reset the from_dst_file pointer first before releasing it, as we
2732          * can't block within lock section
2733          */
2734         file = ms->rp_state.from_dst_file;
2735         ms->rp_state.from_dst_file = NULL;
2736     }
2737
2738     qemu_fclose(file);
2739 }
2740
2741 /*
2742  * Handles messages sent on the return path towards the source VM
2743  *
2744  */
2745 static void *source_return_path_thread(void *opaque)
2746 {
2747     MigrationState *ms = opaque;
2748     QEMUFile *rp = ms->rp_state.from_dst_file;
2749     uint16_t header_len, header_type;
2750     uint8_t buf[512];
2751     uint32_t tmp32, sibling_error;
2752     ram_addr_t start = 0; /* =0 to silence warning */
2753     size_t  len = 0, expected_len;
2754     int res;
2755
2756     trace_source_return_path_thread_entry();
2757     rcu_register_thread();
2758
2759 retry:
2760     while (!ms->rp_state.error && !qemu_file_get_error(rp) &&
2761            migration_is_setup_or_active(ms->state)) {
2762         trace_source_return_path_thread_loop_top();
2763         header_type = qemu_get_be16(rp);
2764         header_len = qemu_get_be16(rp);
2765
2766         if (qemu_file_get_error(rp)) {
2767             mark_source_rp_bad(ms);
2768             goto out;
2769         }
2770
2771         if (header_type >= MIG_RP_MSG_MAX ||
2772             header_type == MIG_RP_MSG_INVALID) {
2773             error_report("RP: Received invalid message 0x%04x length 0x%04x",
2774                          header_type, header_len);
2775             mark_source_rp_bad(ms);
2776             goto out;
2777         }
2778
2779         if ((rp_cmd_args[header_type].len != -1 &&
2780             header_len != rp_cmd_args[header_type].len) ||
2781             header_len > sizeof(buf)) {
2782             error_report("RP: Received '%s' message (0x%04x) with"
2783                          "incorrect length %d expecting %zu",
2784                          rp_cmd_args[header_type].name, header_type, header_len,
2785                          (size_t)rp_cmd_args[header_type].len);
2786             mark_source_rp_bad(ms);
2787             goto out;
2788         }
2789
2790         /* We know we've got a valid header by this point */
2791         res = qemu_get_buffer(rp, buf, header_len);
2792         if (res != header_len) {
2793             error_report("RP: Failed reading data for message 0x%04x"
2794                          " read %d expected %d",
2795                          header_type, res, header_len);
2796             mark_source_rp_bad(ms);
2797             goto out;
2798         }
2799
2800         /* OK, we have the message and the data */
2801         switch (header_type) {
2802         case MIG_RP_MSG_SHUT:
2803             sibling_error = ldl_be_p(buf);
2804             trace_source_return_path_thread_shut(sibling_error);
2805             if (sibling_error) {
2806                 error_report("RP: Sibling indicated error %d", sibling_error);
2807                 mark_source_rp_bad(ms);
2808             }
2809             /*
2810              * We'll let the main thread deal with closing the RP
2811              * we could do a shutdown(2) on it, but we're the only user
2812              * anyway, so there's nothing gained.
2813              */
2814             goto out;
2815
2816         case MIG_RP_MSG_PONG:
2817             tmp32 = ldl_be_p(buf);
2818             trace_source_return_path_thread_pong(tmp32);
2819             break;
2820
2821         case MIG_RP_MSG_REQ_PAGES:
2822             start = ldq_be_p(buf);
2823             len = ldl_be_p(buf + 8);
2824             migrate_handle_rp_req_pages(ms, NULL, start, len);
2825             break;
2826
2827         case MIG_RP_MSG_REQ_PAGES_ID:
2828             expected_len = 12 + 1; /* header + termination */
2829
2830             if (header_len >= expected_len) {
2831                 start = ldq_be_p(buf);
2832                 len = ldl_be_p(buf + 8);
2833                 /* Now we expect an idstr */
2834                 tmp32 = buf[12]; /* Length of the following idstr */
2835                 buf[13 + tmp32] = '\0';
2836                 expected_len += tmp32;
2837             }
2838             if (header_len != expected_len) {
2839                 error_report("RP: Req_Page_id with length %d expecting %zd",
2840                              header_len, expected_len);
2841                 mark_source_rp_bad(ms);
2842                 goto out;
2843             }
2844             migrate_handle_rp_req_pages(ms, (char *)&buf[13], start, len);
2845             break;
2846
2847         case MIG_RP_MSG_RECV_BITMAP:
2848             if (header_len < 1) {
2849                 error_report("%s: missing block name", __func__);
2850                 mark_source_rp_bad(ms);
2851                 goto out;
2852             }
2853             /* Format: len (1B) + idstr (<255B). This ends the idstr. */
2854             buf[buf[0] + 1] = '\0';
2855             if (migrate_handle_rp_recv_bitmap(ms, (char *)(buf + 1))) {
2856                 mark_source_rp_bad(ms);
2857                 goto out;
2858             }
2859             break;
2860
2861         case MIG_RP_MSG_RESUME_ACK:
2862             tmp32 = ldl_be_p(buf);
2863             if (migrate_handle_rp_resume_ack(ms, tmp32)) {
2864                 mark_source_rp_bad(ms);
2865                 goto out;
2866             }
2867             break;
2868
2869         default:
2870             break;
2871         }
2872     }
2873
2874 out:
2875     res = qemu_file_get_error(rp);
2876     if (res) {
2877         if (res && migration_in_postcopy()) {
2878             /*
2879              * Maybe there is something we can do: it looks like a
2880              * network down issue, and we pause for a recovery.
2881              */
2882             migration_release_from_dst_file(ms);
2883             rp = NULL;
2884             if (postcopy_pause_return_path_thread(ms)) {
2885                 /*
2886                  * Reload rp, reset the rest.  Referencing it is safe since
2887                  * it's reset only by us above, or when migration completes
2888                  */
2889                 rp = ms->rp_state.from_dst_file;
2890                 ms->rp_state.error = false;
2891                 goto retry;
2892             }
2893         }
2894
2895         trace_source_return_path_thread_bad_end();
2896         mark_source_rp_bad(ms);
2897     }
2898
2899     trace_source_return_path_thread_end();
2900     migration_release_from_dst_file(ms);
2901     rcu_unregister_thread();
2902     return NULL;
2903 }
2904
2905 static int open_return_path_on_source(MigrationState *ms,
2906                                       bool create_thread)
2907 {
2908     ms->rp_state.from_dst_file = qemu_file_get_return_path(ms->to_dst_file);
2909     if (!ms->rp_state.from_dst_file) {
2910         return -1;
2911     }
2912
2913     trace_open_return_path_on_source();
2914
2915     if (!create_thread) {
2916         /* We're done */
2917         return 0;
2918     }
2919
2920     qemu_thread_create(&ms->rp_state.rp_thread, "return path",
2921                        source_return_path_thread, ms, QEMU_THREAD_JOINABLE);
2922     ms->rp_state.rp_thread_created = true;
2923
2924     trace_open_return_path_on_source_continue();
2925
2926     return 0;
2927 }
2928
2929 /* Returns 0 if the RP was ok, otherwise there was an error on the RP */
2930 static int await_return_path_close_on_source(MigrationState *ms)
2931 {
2932     /*
2933      * If this is a normal exit then the destination will send a SHUT and the
2934      * rp_thread will exit, however if there's an error we need to cause
2935      * it to exit.
2936      */
2937     if (qemu_file_get_error(ms->to_dst_file) && ms->rp_state.from_dst_file) {
2938         /*
2939          * shutdown(2), if we have it, will cause it to unblock if it's stuck
2940          * waiting for the destination.
2941          */
2942         qemu_file_shutdown(ms->rp_state.from_dst_file);
2943         mark_source_rp_bad(ms);
2944     }
2945     trace_await_return_path_close_on_source_joining();
2946     qemu_thread_join(&ms->rp_state.rp_thread);
2947     ms->rp_state.rp_thread_created = false;
2948     trace_await_return_path_close_on_source_close();
2949     return ms->rp_state.error;
2950 }
2951
2952 /*
2953  * Switch from normal iteration to postcopy
2954  * Returns non-0 on error
2955  */
2956 static int postcopy_start(MigrationState *ms)
2957 {
2958     int ret;
2959     QIOChannelBuffer *bioc;
2960     QEMUFile *fb;
2961     int64_t time_at_stop = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
2962     int64_t bandwidth = migrate_max_postcopy_bandwidth();
2963     bool restart_block = false;
2964     int cur_state = MIGRATION_STATUS_ACTIVE;
2965     if (!migrate_pause_before_switchover()) {
2966         migrate_set_state(&ms->state, MIGRATION_STATUS_ACTIVE,
2967                           MIGRATION_STATUS_POSTCOPY_ACTIVE);
2968     }
2969
2970     trace_postcopy_start();
2971     qemu_mutex_lock_iothread();
2972     trace_postcopy_start_set_run();
2973
2974     qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
2975     global_state_store();
2976     ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
2977     if (ret < 0) {
2978         goto fail;
2979     }
2980
2981     ret = migration_maybe_pause(ms, &cur_state,
2982                                 MIGRATION_STATUS_POSTCOPY_ACTIVE);
2983     if (ret < 0) {
2984         goto fail;
2985     }
2986
2987     ret = bdrv_inactivate_all();
2988     if (ret < 0) {
2989         goto fail;
2990     }
2991     restart_block = true;
2992
2993     /*
2994      * Cause any non-postcopiable, but iterative devices to
2995      * send out their final data.
2996      */
2997     qemu_savevm_state_complete_precopy(ms->to_dst_file, true, false);
2998
2999     /*
3000      * in Finish migrate and with the io-lock held everything should
3001      * be quiet, but we've potentially still got dirty pages and we
3002      * need to tell the destination to throw any pages it's already received
3003      * that are dirty
3004      */
3005     if (migrate_postcopy_ram()) {
3006         ram_postcopy_send_discard_bitmap(ms);
3007     }
3008
3009     /*
3010      * send rest of state - note things that are doing postcopy
3011      * will notice we're in POSTCOPY_ACTIVE and not actually
3012      * wrap their state up here
3013      */
3014     /* 0 max-postcopy-bandwidth means unlimited */
3015     if (!bandwidth) {
3016         qemu_file_set_rate_limit(ms->to_dst_file, INT64_MAX);
3017     } else {
3018         qemu_file_set_rate_limit(ms->to_dst_file, bandwidth / XFER_LIMIT_RATIO);
3019     }
3020     if (migrate_postcopy_ram()) {
3021         /* Ping just for debugging, helps line traces up */
3022         qemu_savevm_send_ping(ms->to_dst_file, 2);
3023     }
3024
3025     /*
3026      * While loading the device state we may trigger page transfer
3027      * requests and the fd must be free to process those, and thus
3028      * the destination must read the whole device state off the fd before
3029      * it starts processing it.  Unfortunately the ad-hoc migration format
3030      * doesn't allow the destination to know the size to read without fully
3031      * parsing it through each devices load-state code (especially the open
3032      * coded devices that use get/put).
3033      * So we wrap the device state up in a package with a length at the start;
3034      * to do this we use a qemu_buf to hold the whole of the device state.
3035      */
3036     bioc = qio_channel_buffer_new(4096);
3037     qio_channel_set_name(QIO_CHANNEL(bioc), "migration-postcopy-buffer");
3038     fb = qemu_fopen_channel_output(QIO_CHANNEL(bioc));
3039     object_unref(OBJECT(bioc));
3040
3041     /*
3042      * Make sure the receiver can get incoming pages before we send the rest
3043      * of the state
3044      */
3045     qemu_savevm_send_postcopy_listen(fb);
3046
3047     qemu_savevm_state_complete_precopy(fb, false, false);
3048     if (migrate_postcopy_ram()) {
3049         qemu_savevm_send_ping(fb, 3);
3050     }
3051
3052     qemu_savevm_send_postcopy_run(fb);
3053
3054     /* <><> end of stuff going into the package */
3055
3056     /* Last point of recovery; as soon as we send the package the destination
3057      * can open devices and potentially start running.
3058      * Lets just check again we've not got any errors.
3059      */
3060     ret = qemu_file_get_error(ms->to_dst_file);
3061     if (ret) {
3062         error_report("postcopy_start: Migration stream errored (pre package)");
3063         goto fail_closefb;
3064     }
3065
3066     restart_block = false;
3067
3068     /* Now send that blob */
3069     if (qemu_savevm_send_packaged(ms->to_dst_file, bioc->data, bioc->usage)) {
3070         goto fail_closefb;
3071     }
3072     qemu_fclose(fb);
3073
3074     /* Send a notify to give a chance for anything that needs to happen
3075      * at the transition to postcopy and after the device state; in particular
3076      * spice needs to trigger a transition now
3077      */
3078     ms->postcopy_after_devices = true;
3079     notifier_list_notify(&migration_state_notifiers, ms);
3080
3081     ms->downtime =  qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - time_at_stop;
3082
3083     qemu_mutex_unlock_iothread();
3084
3085     if (migrate_postcopy_ram()) {
3086         /*
3087          * Although this ping is just for debug, it could potentially be
3088          * used for getting a better measurement of downtime at the source.
3089          */
3090         qemu_savevm_send_ping(ms->to_dst_file, 4);
3091     }
3092
3093     if (migrate_release_ram()) {
3094         ram_postcopy_migrated_memory_release(ms);
3095     }
3096
3097     ret = qemu_file_get_error(ms->to_dst_file);
3098     if (ret) {
3099         error_report("postcopy_start: Migration stream errored");
3100         migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
3101                               MIGRATION_STATUS_FAILED);
3102     }
3103
3104     return ret;
3105
3106 fail_closefb:
3107     qemu_fclose(fb);
3108 fail:
3109     migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
3110                           MIGRATION_STATUS_FAILED);
3111     if (restart_block) {
3112         /* A failure happened early enough that we know the destination hasn't
3113          * accessed block devices, so we're safe to recover.
3114          */
3115         Error *local_err = NULL;
3116
3117         bdrv_activate_all(&local_err);
3118         if (local_err) {
3119             error_report_err(local_err);
3120         }
3121     }
3122     qemu_mutex_unlock_iothread();
3123     return -1;
3124 }
3125
3126 /**
3127  * migration_maybe_pause: Pause if required to by
3128  * migrate_pause_before_switchover called with the iothread locked
3129  * Returns: 0 on success
3130  */
3131 static int migration_maybe_pause(MigrationState *s,
3132                                  int *current_active_state,
3133                                  int new_state)
3134 {
3135     if (!migrate_pause_before_switchover()) {
3136         return 0;
3137     }
3138
3139     /* Since leaving this state is not atomic with posting the semaphore
3140      * it's possible that someone could have issued multiple migrate_continue
3141      * and the semaphore is incorrectly positive at this point;
3142      * the docs say it's undefined to reinit a semaphore that's already
3143      * init'd, so use timedwait to eat up any existing posts.
3144      */
3145     while (qemu_sem_timedwait(&s->pause_sem, 1) == 0) {
3146         /* This block intentionally left blank */
3147     }
3148
3149     /*
3150      * If the migration is cancelled when it is in the completion phase,
3151      * the migration state is set to MIGRATION_STATUS_CANCELLING.
3152      * So we don't need to wait a semaphore, otherwise we would always
3153      * wait for the 'pause_sem' semaphore.
3154      */
3155     if (s->state != MIGRATION_STATUS_CANCELLING) {
3156         qemu_mutex_unlock_iothread();
3157         migrate_set_state(&s->state, *current_active_state,
3158                           MIGRATION_STATUS_PRE_SWITCHOVER);
3159         qemu_sem_wait(&s->pause_sem);
3160         migrate_set_state(&s->state, MIGRATION_STATUS_PRE_SWITCHOVER,
3161                           new_state);
3162         *current_active_state = new_state;
3163         qemu_mutex_lock_iothread();
3164     }
3165
3166     return s->state == new_state ? 0 : -EINVAL;
3167 }
3168
3169 /**
3170  * migration_completion: Used by migration_thread when there's not much left.
3171  *   The caller 'breaks' the loop when this returns.
3172  *
3173  * @s: Current migration state
3174  */
3175 static void migration_completion(MigrationState *s)
3176 {
3177     int ret;
3178     int current_active_state = s->state;
3179
3180     if (s->state == MIGRATION_STATUS_ACTIVE) {
3181         qemu_mutex_lock_iothread();
3182         s->downtime_start = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3183         qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
3184         s->vm_was_running = runstate_is_running();
3185         ret = global_state_store();
3186
3187         if (!ret) {
3188             bool inactivate = !migrate_colo_enabled();
3189             ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
3190             trace_migration_completion_vm_stop(ret);
3191             if (ret >= 0) {
3192                 ret = migration_maybe_pause(s, &current_active_state,
3193                                             MIGRATION_STATUS_DEVICE);
3194             }
3195             if (ret >= 0) {
3196                 qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX);
3197                 ret = qemu_savevm_state_complete_precopy(s->to_dst_file, false,
3198                                                          inactivate);
3199             }
3200             if (inactivate && ret >= 0) {
3201                 s->block_inactive = true;
3202             }
3203         }
3204         qemu_mutex_unlock_iothread();
3205
3206         if (ret < 0) {
3207             goto fail;
3208         }
3209     } else if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
3210         trace_migration_completion_postcopy_end();
3211
3212         qemu_mutex_lock_iothread();
3213         qemu_savevm_state_complete_postcopy(s->to_dst_file);
3214         qemu_mutex_unlock_iothread();
3215
3216         trace_migration_completion_postcopy_end_after_complete();
3217     } else {
3218         goto fail;
3219     }
3220
3221     /*
3222      * If rp was opened we must clean up the thread before
3223      * cleaning everything else up (since if there are no failures
3224      * it will wait for the destination to send it's status in
3225      * a SHUT command).
3226      */
3227     if (s->rp_state.rp_thread_created) {
3228         int rp_error;
3229         trace_migration_return_path_end_before();
3230         rp_error = await_return_path_close_on_source(s);
3231         trace_migration_return_path_end_after(rp_error);
3232         if (rp_error) {
3233             goto fail_invalidate;
3234         }
3235     }
3236
3237     if (qemu_file_get_error(s->to_dst_file)) {
3238         trace_migration_completion_file_err();
3239         goto fail_invalidate;
3240     }
3241
3242     if (migrate_colo_enabled() && s->state == MIGRATION_STATUS_ACTIVE) {
3243         /* COLO does not support postcopy */
3244         migrate_set_state(&s->state, MIGRATION_STATUS_ACTIVE,
3245                           MIGRATION_STATUS_COLO);
3246     } else {
3247         migrate_set_state(&s->state, current_active_state,
3248                           MIGRATION_STATUS_COMPLETED);
3249     }
3250
3251     return;
3252
3253 fail_invalidate:
3254     /* If not doing postcopy, vm_start() will be called: let's regain
3255      * control on images.
3256      */
3257     if (s->state == MIGRATION_STATUS_ACTIVE ||
3258         s->state == MIGRATION_STATUS_DEVICE) {
3259         Error *local_err = NULL;
3260
3261         qemu_mutex_lock_iothread();
3262         bdrv_activate_all(&local_err);
3263         if (local_err) {
3264             error_report_err(local_err);
3265         } else {
3266             s->block_inactive = false;
3267         }
3268         qemu_mutex_unlock_iothread();
3269     }
3270
3271 fail:
3272     migrate_set_state(&s->state, current_active_state,
3273                       MIGRATION_STATUS_FAILED);
3274 }
3275
3276 /**
3277  * bg_migration_completion: Used by bg_migration_thread when after all the
3278  *   RAM has been saved. The caller 'breaks' the loop when this returns.
3279  *
3280  * @s: Current migration state
3281  */
3282 static void bg_migration_completion(MigrationState *s)
3283 {
3284     int current_active_state = s->state;
3285
3286     /*
3287      * Stop tracking RAM writes - un-protect memory, un-register UFFD
3288      * memory ranges, flush kernel wait queues and wake up threads
3289      * waiting for write fault to be resolved.
3290      */
3291     ram_write_tracking_stop();
3292
3293     if (s->state == MIGRATION_STATUS_ACTIVE) {
3294         /*
3295          * By this moment we have RAM content saved into the migration stream.
3296          * The next step is to flush the non-RAM content (device state)
3297          * right after the ram content. The device state has been stored into
3298          * the temporary buffer before RAM saving started.
3299          */
3300         qemu_put_buffer(s->to_dst_file, s->bioc->data, s->bioc->usage);
3301         qemu_fflush(s->to_dst_file);
3302     } else if (s->state == MIGRATION_STATUS_CANCELLING) {
3303         goto fail;
3304     }
3305
3306     if (qemu_file_get_error(s->to_dst_file)) {
3307         trace_migration_completion_file_err();
3308         goto fail;
3309     }
3310
3311     migrate_set_state(&s->state, current_active_state,
3312                       MIGRATION_STATUS_COMPLETED);
3313     return;
3314
3315 fail:
3316     migrate_set_state(&s->state, current_active_state,
3317                       MIGRATION_STATUS_FAILED);
3318 }
3319
3320 bool migrate_colo_enabled(void)
3321 {
3322     MigrationState *s = migrate_get_current();
3323     return s->enabled_capabilities[MIGRATION_CAPABILITY_X_COLO];
3324 }
3325
3326 typedef enum MigThrError {
3327     /* No error detected */
3328     MIG_THR_ERR_NONE = 0,
3329     /* Detected error, but resumed successfully */
3330     MIG_THR_ERR_RECOVERED = 1,
3331     /* Detected fatal error, need to exit */
3332     MIG_THR_ERR_FATAL = 2,
3333 } MigThrError;
3334
3335 static int postcopy_resume_handshake(MigrationState *s)
3336 {
3337     qemu_savevm_send_postcopy_resume(s->to_dst_file);
3338
3339     while (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3340         qemu_sem_wait(&s->rp_state.rp_sem);
3341     }
3342
3343     if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
3344         return 0;
3345     }
3346
3347     return -1;
3348 }
3349
3350 /* Return zero if success, or <0 for error */
3351 static int postcopy_do_resume(MigrationState *s)
3352 {
3353     int ret;
3354
3355     /*
3356      * Call all the resume_prepare() hooks, so that modules can be
3357      * ready for the migration resume.
3358      */
3359     ret = qemu_savevm_state_resume_prepare(s);
3360     if (ret) {
3361         error_report("%s: resume_prepare() failure detected: %d",
3362                      __func__, ret);
3363         return ret;
3364     }
3365
3366     /*
3367      * Last handshake with destination on the resume (destination will
3368      * switch to postcopy-active afterwards)
3369      */
3370     ret = postcopy_resume_handshake(s);
3371     if (ret) {
3372         error_report("%s: handshake failed: %d", __func__, ret);
3373         return ret;
3374     }
3375
3376     return 0;
3377 }
3378
3379 /*
3380  * We don't return until we are in a safe state to continue current
3381  * postcopy migration.  Returns MIG_THR_ERR_RECOVERED if recovered, or
3382  * MIG_THR_ERR_FATAL if unrecovery failure happened.
3383  */
3384 static MigThrError postcopy_pause(MigrationState *s)
3385 {
3386     assert(s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
3387
3388     while (true) {
3389         QEMUFile *file;
3390
3391         /*
3392          * Current channel is possibly broken. Release it.  Note that this is
3393          * guaranteed even without lock because to_dst_file should only be
3394          * modified by the migration thread.  That also guarantees that the
3395          * unregister of yank is safe too without the lock.  It should be safe
3396          * even to be within the qemu_file_lock, but we didn't do that to avoid
3397          * taking more mutex (yank_lock) within qemu_file_lock.  TL;DR: we make
3398          * the qemu_file_lock critical section as small as possible.
3399          */
3400         assert(s->to_dst_file);
3401         migration_ioc_unregister_yank_from_file(s->to_dst_file);
3402         qemu_mutex_lock(&s->qemu_file_lock);
3403         file = s->to_dst_file;
3404         s->to_dst_file = NULL;
3405         qemu_mutex_unlock(&s->qemu_file_lock);
3406
3407         qemu_file_shutdown(file);
3408         qemu_fclose(file);
3409
3410         migrate_set_state(&s->state, s->state,
3411                           MIGRATION_STATUS_POSTCOPY_PAUSED);
3412
3413         error_report("Detected IO failure for postcopy. "
3414                      "Migration paused.");
3415
3416         /*
3417          * We wait until things fixed up. Then someone will setup the
3418          * status back for us.
3419          */
3420         while (s->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
3421             qemu_sem_wait(&s->postcopy_pause_sem);
3422         }
3423
3424         if (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3425             /* Woken up by a recover procedure. Give it a shot */
3426
3427             /*
3428              * Firstly, let's wake up the return path now, with a new
3429              * return path channel.
3430              */
3431             qemu_sem_post(&s->postcopy_pause_rp_sem);
3432
3433             /* Do the resume logic */
3434             if (postcopy_do_resume(s) == 0) {
3435                 /* Let's continue! */
3436                 trace_postcopy_pause_continued();
3437                 return MIG_THR_ERR_RECOVERED;
3438             } else {
3439                 /*
3440                  * Something wrong happened during the recovery, let's
3441                  * pause again. Pause is always better than throwing
3442                  * data away.
3443                  */
3444                 continue;
3445             }
3446         } else {
3447             /* This is not right... Time to quit. */
3448             return MIG_THR_ERR_FATAL;
3449         }
3450     }
3451 }
3452
3453 static MigThrError migration_detect_error(MigrationState *s)
3454 {
3455     int ret;
3456     int state = s->state;
3457     Error *local_error = NULL;
3458
3459     if (state == MIGRATION_STATUS_CANCELLING ||
3460         state == MIGRATION_STATUS_CANCELLED) {
3461         /* End the migration, but don't set the state to failed */
3462         return MIG_THR_ERR_FATAL;
3463     }
3464
3465     /* Try to detect any file errors */
3466     ret = qemu_file_get_error_obj(s->to_dst_file, &local_error);
3467     if (!ret) {
3468         /* Everything is fine */
3469         assert(!local_error);
3470         return MIG_THR_ERR_NONE;
3471     }
3472
3473     if (local_error) {
3474         migrate_set_error(s, local_error);
3475         error_free(local_error);
3476     }
3477
3478     if (state == MIGRATION_STATUS_POSTCOPY_ACTIVE && ret) {
3479         /*
3480          * For postcopy, we allow the network to be down for a
3481          * while. After that, it can be continued by a
3482          * recovery phase.
3483          */
3484         return postcopy_pause(s);
3485     } else {
3486         /*
3487          * For precopy (or postcopy with error outside IO), we fail
3488          * with no time.
3489          */
3490         migrate_set_state(&s->state, state, MIGRATION_STATUS_FAILED);
3491         trace_migration_thread_file_err();
3492
3493         /* Time to stop the migration, now. */
3494         return MIG_THR_ERR_FATAL;
3495     }
3496 }
3497
3498 /* How many bytes have we transferred since the beginning of the migration */
3499 static uint64_t migration_total_bytes(MigrationState *s)
3500 {
3501     return qemu_ftell(s->to_dst_file) + ram_counters.multifd_bytes;
3502 }
3503
3504 static void migration_calculate_complete(MigrationState *s)
3505 {
3506     uint64_t bytes = migration_total_bytes(s);
3507     int64_t end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3508     int64_t transfer_time;
3509
3510     s->total_time = end_time - s->start_time;
3511     if (!s->downtime) {
3512         /*
3513          * It's still not set, so we are precopy migration.  For
3514          * postcopy, downtime is calculated during postcopy_start().
3515          */
3516         s->downtime = end_time - s->downtime_start;
3517     }
3518
3519     transfer_time = s->total_time - s->setup_time;
3520     if (transfer_time) {
3521         s->mbps = ((double) bytes * 8.0) / transfer_time / 1000;
3522     }
3523 }
3524
3525 static void update_iteration_initial_status(MigrationState *s)
3526 {
3527     /*
3528      * Update these three fields at the same time to avoid mismatch info lead
3529      * wrong speed calculation.
3530      */
3531     s->iteration_start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3532     s->iteration_initial_bytes = migration_total_bytes(s);
3533     s->iteration_initial_pages = ram_get_total_transferred_pages();
3534 }
3535
3536 static void migration_update_counters(MigrationState *s,
3537                                       int64_t current_time)
3538 {
3539     uint64_t transferred, transferred_pages, time_spent;
3540     uint64_t current_bytes; /* bytes transferred since the beginning */
3541     double bandwidth;
3542
3543     if (current_time < s->iteration_start_time + BUFFER_DELAY) {
3544         return;
3545     }
3546
3547     current_bytes = migration_total_bytes(s);
3548     transferred = current_bytes - s->iteration_initial_bytes;
3549     time_spent = current_time - s->iteration_start_time;
3550     bandwidth = (double)transferred / time_spent;
3551     s->threshold_size = bandwidth * s->parameters.downtime_limit;
3552
3553     s->mbps = (((double) transferred * 8.0) /
3554                ((double) time_spent / 1000.0)) / 1000.0 / 1000.0;
3555
3556     transferred_pages = ram_get_total_transferred_pages() -
3557                             s->iteration_initial_pages;
3558     s->pages_per_second = (double) transferred_pages /
3559                              (((double) time_spent / 1000.0));
3560
3561     /*
3562      * if we haven't sent anything, we don't want to
3563      * recalculate. 10000 is a small enough number for our purposes
3564      */
3565     if (ram_counters.dirty_pages_rate && transferred > 10000) {
3566         s->expected_downtime = ram_counters.remaining / bandwidth;
3567     }
3568
3569     qemu_file_reset_rate_limit(s->to_dst_file);
3570
3571     update_iteration_initial_status(s);
3572
3573     trace_migrate_transferred(transferred, time_spent,
3574                               bandwidth, s->threshold_size);
3575 }
3576
3577 /* Migration thread iteration status */
3578 typedef enum {
3579     MIG_ITERATE_RESUME,         /* Resume current iteration */
3580     MIG_ITERATE_SKIP,           /* Skip current iteration */
3581     MIG_ITERATE_BREAK,          /* Break the loop */
3582 } MigIterateState;
3583
3584 /*
3585  * Return true if continue to the next iteration directly, false
3586  * otherwise.
3587  */
3588 static MigIterateState migration_iteration_run(MigrationState *s)
3589 {
3590     uint64_t pending_size, pend_pre, pend_compat, pend_post;
3591     bool in_postcopy = s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE;
3592
3593     qemu_savevm_state_pending(s->to_dst_file, s->threshold_size, &pend_pre,
3594                               &pend_compat, &pend_post);
3595     pending_size = pend_pre + pend_compat + pend_post;
3596
3597     trace_migrate_pending(pending_size, s->threshold_size,
3598                           pend_pre, pend_compat, pend_post);
3599
3600     if (pending_size && pending_size >= s->threshold_size) {
3601         /* Still a significant amount to transfer */
3602         if (!in_postcopy && pend_pre <= s->threshold_size &&
3603             qatomic_read(&s->start_postcopy)) {
3604             if (postcopy_start(s)) {
3605                 error_report("%s: postcopy failed to start", __func__);
3606             }
3607             return MIG_ITERATE_SKIP;
3608         }
3609         /* Just another iteration step */
3610         qemu_savevm_state_iterate(s->to_dst_file, in_postcopy);
3611     } else {
3612         trace_migration_thread_low_pending(pending_size);
3613         migration_completion(s);
3614         return MIG_ITERATE_BREAK;
3615     }
3616
3617     return MIG_ITERATE_RESUME;
3618 }
3619
3620 static void migration_iteration_finish(MigrationState *s)
3621 {
3622     /* If we enabled cpu throttling for auto-converge, turn it off. */
3623     cpu_throttle_stop();
3624
3625     qemu_mutex_lock_iothread();
3626     switch (s->state) {
3627     case MIGRATION_STATUS_COMPLETED:
3628         migration_calculate_complete(s);
3629         runstate_set(RUN_STATE_POSTMIGRATE);
3630         break;
3631     case MIGRATION_STATUS_COLO:
3632         if (!migrate_colo_enabled()) {
3633             error_report("%s: critical error: calling COLO code without "
3634                          "COLO enabled", __func__);
3635         }
3636         migrate_start_colo_process(s);
3637         s->vm_was_running = true;
3638         /* Fallthrough */
3639     case MIGRATION_STATUS_FAILED:
3640     case MIGRATION_STATUS_CANCELLED:
3641     case MIGRATION_STATUS_CANCELLING:
3642         if (s->vm_was_running) {
3643             if (!runstate_check(RUN_STATE_SHUTDOWN)) {
3644                 vm_start();
3645             }
3646         } else {
3647             if (runstate_check(RUN_STATE_FINISH_MIGRATE)) {
3648                 runstate_set(RUN_STATE_POSTMIGRATE);
3649             }
3650         }
3651         break;
3652
3653     default:
3654         /* Should not reach here, but if so, forgive the VM. */
3655         error_report("%s: Unknown ending state %d", __func__, s->state);
3656         break;
3657     }
3658     migrate_fd_cleanup_schedule(s);
3659     qemu_mutex_unlock_iothread();
3660 }
3661
3662 static void bg_migration_iteration_finish(MigrationState *s)
3663 {
3664     qemu_mutex_lock_iothread();
3665     switch (s->state) {
3666     case MIGRATION_STATUS_COMPLETED:
3667         migration_calculate_complete(s);
3668         break;
3669
3670     case MIGRATION_STATUS_ACTIVE:
3671     case MIGRATION_STATUS_FAILED:
3672     case MIGRATION_STATUS_CANCELLED:
3673     case MIGRATION_STATUS_CANCELLING:
3674         break;
3675
3676     default:
3677         /* Should not reach here, but if so, forgive the VM. */
3678         error_report("%s: Unknown ending state %d", __func__, s->state);
3679         break;
3680     }
3681
3682     migrate_fd_cleanup_schedule(s);
3683     qemu_mutex_unlock_iothread();
3684 }
3685
3686 /*
3687  * Return true if continue to the next iteration directly, false
3688  * otherwise.
3689  */
3690 static MigIterateState bg_migration_iteration_run(MigrationState *s)
3691 {
3692     int res;
3693
3694     res = qemu_savevm_state_iterate(s->to_dst_file, false);
3695     if (res > 0) {
3696         bg_migration_completion(s);
3697         return MIG_ITERATE_BREAK;
3698     }
3699
3700     return MIG_ITERATE_RESUME;
3701 }
3702
3703 void migration_make_urgent_request(void)
3704 {
3705     qemu_sem_post(&migrate_get_current()->rate_limit_sem);
3706 }
3707
3708 void migration_consume_urgent_request(void)
3709 {
3710     qemu_sem_wait(&migrate_get_current()->rate_limit_sem);
3711 }
3712
3713 /* Returns true if the rate limiting was broken by an urgent request */
3714 bool migration_rate_limit(void)
3715 {
3716     int64_t now = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3717     MigrationState *s = migrate_get_current();
3718
3719     bool urgent = false;
3720     migration_update_counters(s, now);
3721     if (qemu_file_rate_limit(s->to_dst_file)) {
3722
3723         if (qemu_file_get_error(s->to_dst_file)) {
3724             return false;
3725         }
3726         /*
3727          * Wait for a delay to do rate limiting OR
3728          * something urgent to post the semaphore.
3729          */
3730         int ms = s->iteration_start_time + BUFFER_DELAY - now;
3731         trace_migration_rate_limit_pre(ms);
3732         if (qemu_sem_timedwait(&s->rate_limit_sem, ms) == 0) {
3733             /*
3734              * We were woken by one or more urgent things but
3735              * the timedwait will have consumed one of them.
3736              * The service routine for the urgent wake will dec
3737              * the semaphore itself for each item it consumes,
3738              * so add this one we just eat back.
3739              */
3740             qemu_sem_post(&s->rate_limit_sem);
3741             urgent = true;
3742         }
3743         trace_migration_rate_limit_post(urgent);
3744     }
3745     return urgent;
3746 }
3747
3748 /*
3749  * if failover devices are present, wait they are completely
3750  * unplugged
3751  */
3752
3753 static void qemu_savevm_wait_unplug(MigrationState *s, int old_state,
3754                                     int new_state)
3755 {
3756     if (qemu_savevm_state_guest_unplug_pending()) {
3757         migrate_set_state(&s->state, old_state, MIGRATION_STATUS_WAIT_UNPLUG);
3758
3759         while (s->state == MIGRATION_STATUS_WAIT_UNPLUG &&
3760                qemu_savevm_state_guest_unplug_pending()) {
3761             qemu_sem_timedwait(&s->wait_unplug_sem, 250);
3762         }
3763         if (s->state != MIGRATION_STATUS_WAIT_UNPLUG) {
3764             int timeout = 120; /* 30 seconds */
3765             /*
3766              * migration has been canceled
3767              * but as we have started an unplug we must wait the end
3768              * to be able to plug back the card
3769              */
3770             while (timeout-- && qemu_savevm_state_guest_unplug_pending()) {
3771                 qemu_sem_timedwait(&s->wait_unplug_sem, 250);
3772             }
3773             if (qemu_savevm_state_guest_unplug_pending() &&
3774                 !qtest_enabled()) {
3775                 warn_report("migration: partially unplugged device on "
3776                             "failure");
3777             }
3778         }
3779
3780         migrate_set_state(&s->state, MIGRATION_STATUS_WAIT_UNPLUG, new_state);
3781     } else {
3782         migrate_set_state(&s->state, old_state, new_state);
3783     }
3784 }
3785
3786 /*
3787  * Master migration thread on the source VM.
3788  * It drives the migration and pumps the data down the outgoing channel.
3789  */
3790 static void *migration_thread(void *opaque)
3791 {
3792     MigrationState *s = opaque;
3793     int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
3794     MigThrError thr_error;
3795     bool urgent = false;
3796
3797     rcu_register_thread();
3798
3799     object_ref(OBJECT(s));
3800     update_iteration_initial_status(s);
3801
3802     qemu_savevm_state_header(s->to_dst_file);
3803
3804     /*
3805      * If we opened the return path, we need to make sure dst has it
3806      * opened as well.
3807      */
3808     if (s->rp_state.rp_thread_created) {
3809         /* Now tell the dest that it should open its end so it can reply */
3810         qemu_savevm_send_open_return_path(s->to_dst_file);
3811
3812         /* And do a ping that will make stuff easier to debug */
3813         qemu_savevm_send_ping(s->to_dst_file, 1);
3814     }
3815
3816     if (migrate_postcopy()) {
3817         /*
3818          * Tell the destination that we *might* want to do postcopy later;
3819          * if the other end can't do postcopy it should fail now, nice and
3820          * early.
3821          */
3822         qemu_savevm_send_postcopy_advise(s->to_dst_file);
3823     }
3824
3825     if (migrate_colo_enabled()) {
3826         /* Notify migration destination that we enable COLO */
3827         qemu_savevm_send_colo_enable(s->to_dst_file);
3828     }
3829
3830     qemu_savevm_state_setup(s->to_dst_file);
3831
3832     qemu_savevm_wait_unplug(s, MIGRATION_STATUS_SETUP,
3833                                MIGRATION_STATUS_ACTIVE);
3834
3835     s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
3836
3837     trace_migration_thread_setup_complete();
3838
3839     while (migration_is_active(s)) {
3840         if (urgent || !qemu_file_rate_limit(s->to_dst_file)) {
3841             MigIterateState iter_state = migration_iteration_run(s);
3842             if (iter_state == MIG_ITERATE_SKIP) {
3843                 continue;
3844             } else if (iter_state == MIG_ITERATE_BREAK) {
3845                 break;
3846             }
3847         }
3848
3849         /*
3850          * Try to detect any kind of failures, and see whether we
3851          * should stop the migration now.
3852          */
3853         thr_error = migration_detect_error(s);
3854         if (thr_error == MIG_THR_ERR_FATAL) {
3855             /* Stop migration */
3856             break;
3857         } else if (thr_error == MIG_THR_ERR_RECOVERED) {
3858             /*
3859              * Just recovered from a e.g. network failure, reset all
3860              * the local variables. This is important to avoid
3861              * breaking transferred_bytes and bandwidth calculation
3862              */
3863             update_iteration_initial_status(s);
3864         }
3865
3866         urgent = migration_rate_limit();
3867     }
3868
3869     trace_migration_thread_after_loop();
3870     migration_iteration_finish(s);
3871     object_unref(OBJECT(s));
3872     rcu_unregister_thread();
3873     return NULL;
3874 }
3875
3876 static void bg_migration_vm_start_bh(void *opaque)
3877 {
3878     MigrationState *s = opaque;
3879
3880     qemu_bh_delete(s->vm_start_bh);
3881     s->vm_start_bh = NULL;
3882
3883     vm_start();
3884     s->downtime = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - s->downtime_start;
3885 }
3886
3887 /**
3888  * Background snapshot thread, based on live migration code.
3889  * This is an alternative implementation of live migration mechanism
3890  * introduced specifically to support background snapshots.
3891  *
3892  * It takes advantage of userfault_fd write protection mechanism introduced
3893  * in v5.7 kernel. Compared to existing dirty page logging migration much
3894  * lesser stream traffic is produced resulting in smaller snapshot images,
3895  * simply cause of no page duplicates can get into the stream.
3896  *
3897  * Another key point is that generated vmstate stream reflects machine state
3898  * 'frozen' at the beginning of snapshot creation compared to dirty page logging
3899  * mechanism, which effectively results in that saved snapshot is the state of VM
3900  * at the end of the process.
3901  */
3902 static void *bg_migration_thread(void *opaque)
3903 {
3904     MigrationState *s = opaque;
3905     int64_t setup_start;
3906     MigThrError thr_error;
3907     QEMUFile *fb;
3908     bool early_fail = true;
3909
3910     rcu_register_thread();
3911     object_ref(OBJECT(s));
3912
3913     qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX);
3914
3915     setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
3916     /*
3917      * We want to save vmstate for the moment when migration has been
3918      * initiated but also we want to save RAM content while VM is running.
3919      * The RAM content should appear first in the vmstate. So, we first
3920      * stash the non-RAM part of the vmstate to the temporary buffer,
3921      * then write RAM part of the vmstate to the migration stream
3922      * with vCPUs running and, finally, write stashed non-RAM part of
3923      * the vmstate from the buffer to the migration stream.
3924      */
3925     s->bioc = qio_channel_buffer_new(512 * 1024);
3926     qio_channel_set_name(QIO_CHANNEL(s->bioc), "vmstate-buffer");
3927     fb = qemu_fopen_channel_output(QIO_CHANNEL(s->bioc));
3928     object_unref(OBJECT(s->bioc));
3929
3930     update_iteration_initial_status(s);
3931
3932     /*
3933      * Prepare for tracking memory writes with UFFD-WP - populate
3934      * RAM pages before protecting.
3935      */
3936 #ifdef __linux__
3937     ram_write_tracking_prepare();
3938 #endif
3939
3940     qemu_savevm_state_header(s->to_dst_file);
3941     qemu_savevm_state_setup(s->to_dst_file);
3942
3943     qemu_savevm_wait_unplug(s, MIGRATION_STATUS_SETUP,
3944                                MIGRATION_STATUS_ACTIVE);
3945
3946     s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
3947
3948     trace_migration_thread_setup_complete();
3949     s->downtime_start = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3950
3951     qemu_mutex_lock_iothread();
3952
3953     /*
3954      * If VM is currently in suspended state, then, to make a valid runstate
3955      * transition in vm_stop_force_state() we need to wakeup it up.
3956      */
3957     qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
3958     s->vm_was_running = runstate_is_running();
3959
3960     if (global_state_store()) {
3961         goto fail;
3962     }
3963     /* Forcibly stop VM before saving state of vCPUs and devices */
3964     if (vm_stop_force_state(RUN_STATE_PAUSED)) {
3965         goto fail;
3966     }
3967     /*
3968      * Put vCPUs in sync with shadow context structures, then
3969      * save their state to channel-buffer along with devices.
3970      */
3971     cpu_synchronize_all_states();
3972     if (qemu_savevm_state_complete_precopy_non_iterable(fb, false, false)) {
3973         goto fail;
3974     }
3975     /*
3976      * Since we are going to get non-iterable state data directly
3977      * from s->bioc->data, explicit flush is needed here.
3978      */
3979     qemu_fflush(fb);
3980
3981     /* Now initialize UFFD context and start tracking RAM writes */
3982     if (ram_write_tracking_start()) {
3983         goto fail;
3984     }
3985     early_fail = false;
3986
3987     /*
3988      * Start VM from BH handler to avoid write-fault lock here.
3989      * UFFD-WP protection for the whole RAM is already enabled so
3990      * calling VM state change notifiers from vm_start() would initiate
3991      * writes to virtio VQs memory which is in write-protected region.
3992      */
3993     s->vm_start_bh = qemu_bh_new(bg_migration_vm_start_bh, s);
3994     qemu_bh_schedule(s->vm_start_bh);
3995
3996     qemu_mutex_unlock_iothread();
3997
3998     while (migration_is_active(s)) {
3999         MigIterateState iter_state = bg_migration_iteration_run(s);
4000         if (iter_state == MIG_ITERATE_SKIP) {
4001             continue;
4002         } else if (iter_state == MIG_ITERATE_BREAK) {
4003             break;
4004         }
4005
4006         /*
4007          * Try to detect any kind of failures, and see whether we
4008          * should stop the migration now.
4009          */
4010         thr_error = migration_detect_error(s);
4011         if (thr_error == MIG_THR_ERR_FATAL) {
4012             /* Stop migration */
4013             break;
4014         }
4015
4016         migration_update_counters(s, qemu_clock_get_ms(QEMU_CLOCK_REALTIME));
4017     }
4018
4019     trace_migration_thread_after_loop();
4020
4021 fail:
4022     if (early_fail) {
4023         migrate_set_state(&s->state, MIGRATION_STATUS_ACTIVE,
4024                 MIGRATION_STATUS_FAILED);
4025         qemu_mutex_unlock_iothread();
4026     }
4027
4028     bg_migration_iteration_finish(s);
4029
4030     qemu_fclose(fb);
4031     object_unref(OBJECT(s));
4032     rcu_unregister_thread();
4033
4034     return NULL;
4035 }
4036
4037 void migrate_fd_connect(MigrationState *s, Error *error_in)
4038 {
4039     Error *local_err = NULL;
4040     int64_t rate_limit;
4041     bool resume = s->state == MIGRATION_STATUS_POSTCOPY_PAUSED;
4042
4043     /*
4044      * If there's a previous error, free it and prepare for another one.
4045      * Meanwhile if migration completes successfully, there won't have an error
4046      * dumped when calling migrate_fd_cleanup().
4047      */
4048     migrate_error_free(s);
4049
4050     s->expected_downtime = s->parameters.downtime_limit;
4051     if (resume) {
4052         assert(s->cleanup_bh);
4053     } else {
4054         assert(!s->cleanup_bh);
4055         s->cleanup_bh = qemu_bh_new(migrate_fd_cleanup_bh, s);
4056     }
4057     if (error_in) {
4058         migrate_fd_error(s, error_in);
4059         if (resume) {
4060             /*
4061              * Don't do cleanup for resume if channel is invalid, but only dump
4062              * the error.  We wait for another channel connect from the user.
4063              * The error_report still gives HMP user a hint on what failed.
4064              * It's normally done in migrate_fd_cleanup(), but call it here
4065              * explicitly.
4066              */
4067             error_report_err(error_copy(s->error));
4068         } else {
4069             migrate_fd_cleanup(s);
4070         }
4071         return;
4072     }
4073
4074     if (resume) {
4075         /* This is a resumed migration */
4076         rate_limit = s->parameters.max_postcopy_bandwidth /
4077             XFER_LIMIT_RATIO;
4078     } else {
4079         /* This is a fresh new migration */
4080         rate_limit = s->parameters.max_bandwidth / XFER_LIMIT_RATIO;
4081
4082         /* Notify before starting migration thread */
4083         notifier_list_notify(&migration_state_notifiers, s);
4084     }
4085
4086     qemu_file_set_rate_limit(s->to_dst_file, rate_limit);
4087     qemu_file_set_blocking(s->to_dst_file, true);
4088
4089     /*
4090      * Open the return path. For postcopy, it is used exclusively. For
4091      * precopy, only if user specified "return-path" capability would
4092      * QEMU uses the return path.
4093      */
4094     if (migrate_postcopy_ram() || migrate_use_return_path()) {
4095         if (open_return_path_on_source(s, !resume)) {
4096             error_report("Unable to open return-path for postcopy");
4097             migrate_set_state(&s->state, s->state, MIGRATION_STATUS_FAILED);
4098             migrate_fd_cleanup(s);
4099             return;
4100         }
4101     }
4102
4103     if (resume) {
4104         /* Wakeup the main migration thread to do the recovery */
4105         migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
4106                           MIGRATION_STATUS_POSTCOPY_RECOVER);
4107         qemu_sem_post(&s->postcopy_pause_sem);
4108         return;
4109     }
4110
4111     if (multifd_save_setup(&local_err) != 0) {
4112         error_report_err(local_err);
4113         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
4114                           MIGRATION_STATUS_FAILED);
4115         migrate_fd_cleanup(s);
4116         return;
4117     }
4118
4119     if (migrate_background_snapshot()) {
4120         qemu_thread_create(&s->thread, "bg_snapshot",
4121                 bg_migration_thread, s, QEMU_THREAD_JOINABLE);
4122     } else {
4123         qemu_thread_create(&s->thread, "live_migration",
4124                 migration_thread, s, QEMU_THREAD_JOINABLE);
4125     }
4126     s->migration_thread_running = true;
4127 }
4128
4129 void migration_global_dump(Monitor *mon)
4130 {
4131     MigrationState *ms = migrate_get_current();
4132
4133     monitor_printf(mon, "globals:\n");
4134     monitor_printf(mon, "store-global-state: %s\n",
4135                    ms->store_global_state ? "on" : "off");
4136     monitor_printf(mon, "only-migratable: %s\n",
4137                    only_migratable ? "on" : "off");
4138     monitor_printf(mon, "send-configuration: %s\n",
4139                    ms->send_configuration ? "on" : "off");
4140     monitor_printf(mon, "send-section-footer: %s\n",
4141                    ms->send_section_footer ? "on" : "off");
4142     monitor_printf(mon, "decompress-error-check: %s\n",
4143                    ms->decompress_error_check ? "on" : "off");
4144     monitor_printf(mon, "clear-bitmap-shift: %u\n",
4145                    ms->clear_bitmap_shift);
4146 }
4147
4148 #define DEFINE_PROP_MIG_CAP(name, x)             \
4149     DEFINE_PROP_BOOL(name, MigrationState, enabled_capabilities[x], false)
4150
4151 static Property migration_properties[] = {
4152     DEFINE_PROP_BOOL("store-global-state", MigrationState,
4153                      store_global_state, true),
4154     DEFINE_PROP_BOOL("send-configuration", MigrationState,
4155                      send_configuration, true),
4156     DEFINE_PROP_BOOL("send-section-footer", MigrationState,
4157                      send_section_footer, true),
4158     DEFINE_PROP_BOOL("decompress-error-check", MigrationState,
4159                       decompress_error_check, true),
4160     DEFINE_PROP_UINT8("x-clear-bitmap-shift", MigrationState,
4161                       clear_bitmap_shift, CLEAR_BITMAP_SHIFT_DEFAULT),
4162
4163     /* Migration parameters */
4164     DEFINE_PROP_UINT8("x-compress-level", MigrationState,
4165                       parameters.compress_level,
4166                       DEFAULT_MIGRATE_COMPRESS_LEVEL),
4167     DEFINE_PROP_UINT8("x-compress-threads", MigrationState,
4168                       parameters.compress_threads,
4169                       DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT),
4170     DEFINE_PROP_BOOL("x-compress-wait-thread", MigrationState,
4171                       parameters.compress_wait_thread, true),
4172     DEFINE_PROP_UINT8("x-decompress-threads", MigrationState,
4173                       parameters.decompress_threads,
4174                       DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT),
4175     DEFINE_PROP_UINT8("x-throttle-trigger-threshold", MigrationState,
4176                       parameters.throttle_trigger_threshold,
4177                       DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD),
4178     DEFINE_PROP_UINT8("x-cpu-throttle-initial", MigrationState,
4179                       parameters.cpu_throttle_initial,
4180                       DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL),
4181     DEFINE_PROP_UINT8("x-cpu-throttle-increment", MigrationState,
4182                       parameters.cpu_throttle_increment,
4183                       DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT),
4184     DEFINE_PROP_BOOL("x-cpu-throttle-tailslow", MigrationState,
4185                       parameters.cpu_throttle_tailslow, false),
4186     DEFINE_PROP_SIZE("x-max-bandwidth", MigrationState,
4187                       parameters.max_bandwidth, MAX_THROTTLE),
4188     DEFINE_PROP_UINT64("x-downtime-limit", MigrationState,
4189                       parameters.downtime_limit,
4190                       DEFAULT_MIGRATE_SET_DOWNTIME),
4191     DEFINE_PROP_UINT32("x-checkpoint-delay", MigrationState,
4192                       parameters.x_checkpoint_delay,
4193                       DEFAULT_MIGRATE_X_CHECKPOINT_DELAY),
4194     DEFINE_PROP_UINT8("multifd-channels", MigrationState,
4195                       parameters.multifd_channels,
4196                       DEFAULT_MIGRATE_MULTIFD_CHANNELS),
4197     DEFINE_PROP_MULTIFD_COMPRESSION("multifd-compression", MigrationState,
4198                       parameters.multifd_compression,
4199                       DEFAULT_MIGRATE_MULTIFD_COMPRESSION),
4200     DEFINE_PROP_UINT8("multifd-zlib-level", MigrationState,
4201                       parameters.multifd_zlib_level,
4202                       DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL),
4203     DEFINE_PROP_UINT8("multifd-zstd-level", MigrationState,
4204                       parameters.multifd_zstd_level,
4205                       DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL),
4206     DEFINE_PROP_SIZE("xbzrle-cache-size", MigrationState,
4207                       parameters.xbzrle_cache_size,
4208                       DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE),
4209     DEFINE_PROP_SIZE("max-postcopy-bandwidth", MigrationState,
4210                       parameters.max_postcopy_bandwidth,
4211                       DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH),
4212     DEFINE_PROP_UINT8("max-cpu-throttle", MigrationState,
4213                       parameters.max_cpu_throttle,
4214                       DEFAULT_MIGRATE_MAX_CPU_THROTTLE),
4215     DEFINE_PROP_SIZE("announce-initial", MigrationState,
4216                       parameters.announce_initial,
4217                       DEFAULT_MIGRATE_ANNOUNCE_INITIAL),
4218     DEFINE_PROP_SIZE("announce-max", MigrationState,
4219                       parameters.announce_max,
4220                       DEFAULT_MIGRATE_ANNOUNCE_MAX),
4221     DEFINE_PROP_SIZE("announce-rounds", MigrationState,
4222                       parameters.announce_rounds,
4223                       DEFAULT_MIGRATE_ANNOUNCE_ROUNDS),
4224     DEFINE_PROP_SIZE("announce-step", MigrationState,
4225                       parameters.announce_step,
4226                       DEFAULT_MIGRATE_ANNOUNCE_STEP),
4227
4228     /* Migration capabilities */
4229     DEFINE_PROP_MIG_CAP("x-xbzrle", MIGRATION_CAPABILITY_XBZRLE),
4230     DEFINE_PROP_MIG_CAP("x-rdma-pin-all", MIGRATION_CAPABILITY_RDMA_PIN_ALL),
4231     DEFINE_PROP_MIG_CAP("x-auto-converge", MIGRATION_CAPABILITY_AUTO_CONVERGE),
4232     DEFINE_PROP_MIG_CAP("x-zero-blocks", MIGRATION_CAPABILITY_ZERO_BLOCKS),
4233     DEFINE_PROP_MIG_CAP("x-compress", MIGRATION_CAPABILITY_COMPRESS),
4234     DEFINE_PROP_MIG_CAP("x-events", MIGRATION_CAPABILITY_EVENTS),
4235     DEFINE_PROP_MIG_CAP("x-postcopy-ram", MIGRATION_CAPABILITY_POSTCOPY_RAM),
4236     DEFINE_PROP_MIG_CAP("x-colo", MIGRATION_CAPABILITY_X_COLO),
4237     DEFINE_PROP_MIG_CAP("x-release-ram", MIGRATION_CAPABILITY_RELEASE_RAM),
4238     DEFINE_PROP_MIG_CAP("x-block", MIGRATION_CAPABILITY_BLOCK),
4239     DEFINE_PROP_MIG_CAP("x-return-path", MIGRATION_CAPABILITY_RETURN_PATH),
4240     DEFINE_PROP_MIG_CAP("x-multifd", MIGRATION_CAPABILITY_MULTIFD),
4241     DEFINE_PROP_MIG_CAP("x-background-snapshot",
4242             MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT),
4243
4244     DEFINE_PROP_END_OF_LIST(),
4245 };
4246
4247 static void migration_class_init(ObjectClass *klass, void *data)
4248 {
4249     DeviceClass *dc = DEVICE_CLASS(klass);
4250
4251     dc->user_creatable = false;
4252     device_class_set_props(dc, migration_properties);
4253 }
4254
4255 static void migration_instance_finalize(Object *obj)
4256 {
4257     MigrationState *ms = MIGRATION_OBJ(obj);
4258     MigrationParameters *params = &ms->parameters;
4259
4260     qemu_mutex_destroy(&ms->error_mutex);
4261     qemu_mutex_destroy(&ms->qemu_file_lock);
4262     g_free(params->tls_hostname);
4263     g_free(params->tls_creds);
4264     qemu_sem_destroy(&ms->wait_unplug_sem);
4265     qemu_sem_destroy(&ms->rate_limit_sem);
4266     qemu_sem_destroy(&ms->pause_sem);
4267     qemu_sem_destroy(&ms->postcopy_pause_sem);
4268     qemu_sem_destroy(&ms->postcopy_pause_rp_sem);
4269     qemu_sem_destroy(&ms->rp_state.rp_sem);
4270     error_free(ms->error);
4271 }
4272
4273 static void migration_instance_init(Object *obj)
4274 {
4275     MigrationState *ms = MIGRATION_OBJ(obj);
4276     MigrationParameters *params = &ms->parameters;
4277
4278     ms->state = MIGRATION_STATUS_NONE;
4279     ms->mbps = -1;
4280     ms->pages_per_second = -1;
4281     qemu_sem_init(&ms->pause_sem, 0);
4282     qemu_mutex_init(&ms->error_mutex);
4283
4284     params->tls_hostname = g_strdup("");
4285     params->tls_creds = g_strdup("");
4286
4287     /* Set has_* up only for parameter checks */
4288     params->has_compress_level = true;
4289     params->has_compress_threads = true;
4290     params->has_decompress_threads = true;
4291     params->has_throttle_trigger_threshold = true;
4292     params->has_cpu_throttle_initial = true;
4293     params->has_cpu_throttle_increment = true;
4294     params->has_cpu_throttle_tailslow = true;
4295     params->has_max_bandwidth = true;
4296     params->has_downtime_limit = true;
4297     params->has_x_checkpoint_delay = true;
4298     params->has_block_incremental = true;
4299     params->has_multifd_channels = true;
4300     params->has_multifd_compression = true;
4301     params->has_multifd_zlib_level = true;
4302     params->has_multifd_zstd_level = true;
4303     params->has_xbzrle_cache_size = true;
4304     params->has_max_postcopy_bandwidth = true;
4305     params->has_max_cpu_throttle = true;
4306     params->has_announce_initial = true;
4307     params->has_announce_max = true;
4308     params->has_announce_rounds = true;
4309     params->has_announce_step = true;
4310
4311     qemu_sem_init(&ms->postcopy_pause_sem, 0);
4312     qemu_sem_init(&ms->postcopy_pause_rp_sem, 0);
4313     qemu_sem_init(&ms->rp_state.rp_sem, 0);
4314     qemu_sem_init(&ms->rate_limit_sem, 0);
4315     qemu_sem_init(&ms->wait_unplug_sem, 0);
4316     qemu_mutex_init(&ms->qemu_file_lock);
4317 }
4318
4319 /*
4320  * Return true if check pass, false otherwise. Error will be put
4321  * inside errp if provided.
4322  */
4323 static bool migration_object_check(MigrationState *ms, Error **errp)
4324 {
4325     MigrationCapabilityStatusList *head = NULL;
4326     /* Assuming all off */
4327     bool cap_list[MIGRATION_CAPABILITY__MAX] = { 0 }, ret;
4328     int i;
4329
4330     if (!migrate_params_check(&ms->parameters, errp)) {
4331         return false;
4332     }
4333
4334     for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
4335         if (ms->enabled_capabilities[i]) {
4336             QAPI_LIST_PREPEND(head, migrate_cap_add(i, true));
4337         }
4338     }
4339
4340     ret = migrate_caps_check(cap_list, head, errp);
4341
4342     /* It works with head == NULL */
4343     qapi_free_MigrationCapabilityStatusList(head);
4344
4345     return ret;
4346 }
4347
4348 static const TypeInfo migration_type = {
4349     .name = TYPE_MIGRATION,
4350     /*
4351      * NOTE: TYPE_MIGRATION is not really a device, as the object is
4352      * not created using qdev_new(), it is not attached to the qdev
4353      * device tree, and it is never realized.
4354      *
4355      * TODO: Make this TYPE_OBJECT once QOM provides something like
4356      * TYPE_DEVICE's "-global" properties.
4357      */
4358     .parent = TYPE_DEVICE,
4359     .class_init = migration_class_init,
4360     .class_size = sizeof(MigrationClass),
4361     .instance_size = sizeof(MigrationState),
4362     .instance_init = migration_instance_init,
4363     .instance_finalize = migration_instance_finalize,
4364 };
4365
4366 static void register_migration_types(void)
4367 {
4368     type_register_static(&migration_type);
4369 }
4370
4371 type_init(register_migration_types);