migration/migration.c

   1 /*
   2  * QEMU live migration
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *
   6  * Authors:
   7  *  Anthony Liguori   <[email protected]>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  * Contributions after 2012-01-13 are licensed under the terms of the
  13  * GNU GPL, version 2 or (at your option) any later version.
  14  */
  15
  16 #include "qemu/osdep.h"
  17 #include "qemu/cutils.h"
  18 #include "qemu/error-report.h"
  19 #include "qemu/main-loop.h"
  20 #include "migration/blocker.h"
  21 #include "exec.h"
  22 #include "fd.h"
  23 #include "socket.h"
  24 #include "sysemu/runstate.h"
  25 #include "sysemu/sysemu.h"
  26 #include "sysemu/cpu-throttle.h"
  27 #include "rdma.h"
  28 #include "ram.h"
  29 #include "migration/global_state.h"
  30 #include "migration/misc.h"
  31 #include "migration.h"
  32 #include "savevm.h"
  33 #include "qemu-file-channel.h"
  34 #include "qemu-file.h"
  35 #include "migration/vmstate.h"
  36 #include "block/block.h"
  37 #include "qapi/error.h"
  38 #include "qapi/clone-visitor.h"
  39 #include "qapi/qapi-visit-migration.h"
  40 #include "qapi/qapi-visit-sockets.h"
  41 #include "qapi/qapi-commands-migration.h"
  42 #include "qapi/qapi-events-migration.h"
  43 #include "qapi/qmp/qerror.h"
  44 #include "qapi/qmp/qnull.h"
  45 #include "qemu/rcu.h"
  46 #include "block.h"
  47 #include "postcopy-ram.h"
  48 #include "qemu/thread.h"
  49 #include "trace.h"
  50 #include "exec/target_page.h"
  51 #include "io/channel-buffer.h"
  52 #include "migration/colo.h"
  53 #include "hw/boards.h"
  54 #include "hw/qdev-properties.h"
  55 #include "hw/qdev-properties-system.h"
  56 #include "monitor/monitor.h"
  57 #include "net/announce.h"
  58 #include "qemu/queue.h"
  59 #include "multifd.h"
  60 #include "qemu/yank.h"
  61 #include "sysemu/cpus.h"
  62 #include "yank_functions.h"
  63
  64 #define MAX_THROTTLE  (128 << 20)      /* Migration transfer speed throttling */
  65
  66 /* Amount of time to allocate to each "chunk" of bandwidth-throttled
  67  * data. */
  68 #define BUFFER_DELAY     100
  69 #define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY)
  70
  71 /* Time in milliseconds we are allowed to stop the source,
  72  * for sending the last part */
  73 #define DEFAULT_MIGRATE_SET_DOWNTIME 300
  74
  75 /* Maximum migrate downtime set to 2000 seconds */
  76 #define MAX_MIGRATE_DOWNTIME_SECONDS 2000
  77 #define MAX_MIGRATE_DOWNTIME (MAX_MIGRATE_DOWNTIME_SECONDS * 1000)
  78
  79 /* Default compression thread count */
  80 #define DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT 8
  81 /* Default decompression thread count, usually decompression is at
  82  * least 4 times as fast as compression.*/
  83 #define DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT 2
  84 /*0: means nocompress, 1: best speed, ... 9: best compress ratio */
  85 #define DEFAULT_MIGRATE_COMPRESS_LEVEL 1
  86 /* Define default autoconverge cpu throttle migration parameters */
  87 #define DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD 50
  88 #define DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL 20
  89 #define DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT 10
  90 #define DEFAULT_MIGRATE_MAX_CPU_THROTTLE 99
  91
  92 /* Migration XBZRLE default cache size */
  93 #define DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE (64 * 1024 * 1024)
  94
  95 /* The delay time (in ms) between two COLO checkpoints */
  96 #define DEFAULT_MIGRATE_X_CHECKPOINT_DELAY (200 * 100)
  97 #define DEFAULT_MIGRATE_MULTIFD_CHANNELS 2
  98 #define DEFAULT_MIGRATE_MULTIFD_COMPRESSION MULTIFD_COMPRESSION_NONE
  99 /* 0: means nocompress, 1: best speed, ... 9: best compress ratio */
 100 #define DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL 1
 101 /* 0: means nocompress, 1: best speed, ... 20: best compress ratio */
 102 #define DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL 1
 103
 104 /* Background transfer rate for postcopy, 0 means unlimited, note
 105  * that page requests can still exceed this limit.
 106  */
 107 #define DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH 0
 108
 109 /*
 110  * Parameters for self_announce_delay giving a stream of RARP/ARP
 111  * packets after migration.
 112  */
 113 #define DEFAULT_MIGRATE_ANNOUNCE_INITIAL  50
 114 #define DEFAULT_MIGRATE_ANNOUNCE_MAX     550
 115 #define DEFAULT_MIGRATE_ANNOUNCE_ROUNDS    5
 116 #define DEFAULT_MIGRATE_ANNOUNCE_STEP    100
 117
 118 static NotifierList migration_state_notifiers =
 119     NOTIFIER_LIST_INITIALIZER(migration_state_notifiers);
 120
 121 /* Messages sent on the return path from destination to source */
 122 enum mig_rp_message_type {
 123     MIG_RP_MSG_INVALID = 0,  /* Must be 0 */
 124     MIG_RP_MSG_SHUT,         /* sibling will not send any more RP messages */
 125     MIG_RP_MSG_PONG,         /* Response to a PING; data (seq: be32 ) */
 126
 127     MIG_RP_MSG_REQ_PAGES_ID, /* data (start: be64, len: be32, id: string) */
 128     MIG_RP_MSG_REQ_PAGES,    /* data (start: be64, len: be32) */
 129     MIG_RP_MSG_RECV_BITMAP,  /* send recved_bitmap back to source */
 130     MIG_RP_MSG_RESUME_ACK,   /* tell source that we are ready to resume */
 131
 132     MIG_RP_MSG_MAX
 133 };
 134
 135 /* Migration capabilities set */
 136 struct MigrateCapsSet {
 137     int size;                       /* Capability set size */
 138     MigrationCapability caps[];     /* Variadic array of capabilities */
 139 };
 140 typedef struct MigrateCapsSet MigrateCapsSet;
 141
 142 /* Define and initialize MigrateCapsSet */
 143 #define INITIALIZE_MIGRATE_CAPS_SET(_name, ...)   \
 144     MigrateCapsSet _name = {    \
 145         .size = sizeof((int []) { __VA_ARGS__ }) / sizeof(int), \
 146         .caps = { __VA_ARGS__ } \
 147     }
 148
 149 /* Background-snapshot compatibility check list */
 150 static const
 151 INITIALIZE_MIGRATE_CAPS_SET(check_caps_background_snapshot,
 152     MIGRATION_CAPABILITY_POSTCOPY_RAM,
 153     MIGRATION_CAPABILITY_DIRTY_BITMAPS,
 154     MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME,
 155     MIGRATION_CAPABILITY_LATE_BLOCK_ACTIVATE,
 156     MIGRATION_CAPABILITY_RETURN_PATH,
 157     MIGRATION_CAPABILITY_MULTIFD,
 158     MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER,
 159     MIGRATION_CAPABILITY_AUTO_CONVERGE,
 160     MIGRATION_CAPABILITY_RELEASE_RAM,
 161     MIGRATION_CAPABILITY_RDMA_PIN_ALL,
 162     MIGRATION_CAPABILITY_COMPRESS,
 163     MIGRATION_CAPABILITY_XBZRLE,
 164     MIGRATION_CAPABILITY_X_COLO,
 165     MIGRATION_CAPABILITY_VALIDATE_UUID);
 166
 167 /* When we add fault tolerance, we could have several
 168    migrations at once.  For now we don't need to add
 169    dynamic creation of migration */
 170
 171 static MigrationState *current_migration;
 172 static MigrationIncomingState *current_incoming;
 173
 174 static GSList *migration_blockers;
 175
 176 static bool migration_object_check(MigrationState *ms, Error **errp);
 177 static int migration_maybe_pause(MigrationState *s,
 178                                  int *current_active_state,
 179                                  int new_state);
 180 static void migrate_fd_cancel(MigrationState *s);
 181
 182 static gint page_request_addr_cmp(gconstpointer ap, gconstpointer bp)
 183 {
 184     uintptr_t a = (uintptr_t) ap, b = (uintptr_t) bp;
 185
 186     return (a > b) - (a < b);
 187 }
 188
 189 void migration_object_init(void)
 190 {
 191     /* This can only be called once. */
 192     assert(!current_migration);
 193     current_migration = MIGRATION_OBJ(object_new(TYPE_MIGRATION));
 194
 195     /*
 196      * Init the migrate incoming object as well no matter whether
 197      * we'll use it or not.
 198      */
 199     assert(!current_incoming);
 200     current_incoming = g_new0(MigrationIncomingState, 1);
 201     current_incoming->state = MIGRATION_STATUS_NONE;
 202     current_incoming->postcopy_remote_fds =
 203         g_array_new(FALSE, TRUE, sizeof(struct PostCopyFD));
 204     qemu_mutex_init(&current_incoming->rp_mutex);
 205     qemu_event_init(&current_incoming->main_thread_load_event, false);
 206     qemu_sem_init(&current_incoming->postcopy_pause_sem_dst, 0);
 207     qemu_sem_init(&current_incoming->postcopy_pause_sem_fault, 0);
 208     qemu_mutex_init(&current_incoming->page_request_mutex);
 209     current_incoming->page_requested = g_tree_new(page_request_addr_cmp);
 210
 211     migration_object_check(current_migration, &error_fatal);
 212
 213     blk_mig_init();
 214     ram_mig_init();
 215     dirty_bitmap_mig_init();
 216 }
 217
 218 void migration_cancel(const Error *error)
 219 {
 220     if (error) {
 221         migrate_set_error(current_migration, error);
 222     }
 223     migrate_fd_cancel(current_migration);
 224 }
 225
 226 void migration_shutdown(void)
 227 {
 228     /*
 229      * When the QEMU main thread exit, the COLO thread
 230      * may wait a semaphore. So, we should wakeup the
 231      * COLO thread before migration shutdown.
 232      */
 233     colo_shutdown();
 234     /*
 235      * Cancel the current migration - that will (eventually)
 236      * stop the migration using this structure
 237      */
 238     migration_cancel(NULL);
 239     object_unref(OBJECT(current_migration));
 240
 241     /*
 242      * Cancel outgoing migration of dirty bitmaps. It should
 243      * at least unref used block nodes.
 244      */
 245     dirty_bitmap_mig_cancel_outgoing();
 246
 247     /*
 248      * Cancel incoming migration of dirty bitmaps. Dirty bitmaps
 249      * are non-critical data, and their loss never considered as
 250      * something serious.
 251      */
 252     dirty_bitmap_mig_cancel_incoming();
 253 }
 254
 255 /* For outgoing */
 256 MigrationState *migrate_get_current(void)
 257 {
 258     /* This can only be called after the object created. */
 259     assert(current_migration);
 260     return current_migration;
 261 }
 262
 263 MigrationIncomingState *migration_incoming_get_current(void)
 264 {
 265     assert(current_incoming);
 266     return current_incoming;
 267 }
 268
 269 void migration_incoming_state_destroy(void)
 270 {
 271     struct MigrationIncomingState *mis = migration_incoming_get_current();
 272
 273     if (mis->to_src_file) {
 274         /* Tell source that we are done */
 275         migrate_send_rp_shut(mis, qemu_file_get_error(mis->from_src_file) != 0);
 276         qemu_fclose(mis->to_src_file);
 277         mis->to_src_file = NULL;
 278     }
 279
 280     if (mis->from_src_file) {
 281         migration_ioc_unregister_yank_from_file(mis->from_src_file);
 282         qemu_fclose(mis->from_src_file);
 283         mis->from_src_file = NULL;
 284     }
 285     if (mis->postcopy_remote_fds) {
 286         g_array_free(mis->postcopy_remote_fds, TRUE);
 287         mis->postcopy_remote_fds = NULL;
 288     }
 289     if (mis->transport_cleanup) {
 290         mis->transport_cleanup(mis->transport_data);
 291     }
 292
 293     qemu_event_reset(&mis->main_thread_load_event);
 294
 295     if (mis->page_requested) {
 296         g_tree_destroy(mis->page_requested);
 297         mis->page_requested = NULL;
 298     }
 299
 300     if (mis->socket_address_list) {
 301         qapi_free_SocketAddressList(mis->socket_address_list);
 302         mis->socket_address_list = NULL;
 303     }
 304
 305     yank_unregister_instance(MIGRATION_YANK_INSTANCE);
 306 }
 307
 308 static void migrate_generate_event(int new_state)
 309 {
 310     if (migrate_use_events()) {
 311         qapi_event_send_migration(new_state);
 312     }
 313 }
 314
 315 static bool migrate_late_block_activate(void)
 316 {
 317     MigrationState *s;
 318
 319     s = migrate_get_current();
 320
 321     return s->enabled_capabilities[
 322         MIGRATION_CAPABILITY_LATE_BLOCK_ACTIVATE];
 323 }
 324
 325 /*
 326  * Send a message on the return channel back to the source
 327  * of the migration.
 328  */
 329 static int migrate_send_rp_message(MigrationIncomingState *mis,
 330                                    enum mig_rp_message_type message_type,
 331                                    uint16_t len, void *data)
 332 {
 333     int ret = 0;
 334
 335     trace_migrate_send_rp_message((int)message_type, len);
 336     QEMU_LOCK_GUARD(&mis->rp_mutex);
 337
 338     /*
 339      * It's possible that the file handle got lost due to network
 340      * failures.
 341      */
 342     if (!mis->to_src_file) {
 343         ret = -EIO;
 344         return ret;
 345     }
 346
 347     qemu_put_be16(mis->to_src_file, (unsigned int)message_type);
 348     qemu_put_be16(mis->to_src_file, len);
 349     qemu_put_buffer(mis->to_src_file, data, len);
 350     qemu_fflush(mis->to_src_file);
 351
 352     /* It's possible that qemu file got error during sending */
 353     ret = qemu_file_get_error(mis->to_src_file);
 354
 355     return ret;
 356 }
 357
 358 /* Request one page from the source VM at the given start address.
 359  *   rb: the RAMBlock to request the page in
 360  *   Start: Address offset within the RB
 361  *   Len: Length in bytes required - must be a multiple of pagesize
 362  */
 363 int migrate_send_rp_message_req_pages(MigrationIncomingState *mis,
 364                                       RAMBlock *rb, ram_addr_t start)
 365 {
 366     uint8_t bufc[12 + 1 + 255]; /* start (8), len (4), rbname up to 256 */
 367     size_t msglen = 12; /* start + len */
 368     size_t len = qemu_ram_pagesize(rb);
 369     enum mig_rp_message_type msg_type;
 370     const char *rbname;
 371     int rbname_len;
 372
 373     *(uint64_t *)bufc = cpu_to_be64((uint64_t)start);
 374     *(uint32_t *)(bufc + 8) = cpu_to_be32((uint32_t)len);
 375
 376     /*
 377      * We maintain the last ramblock that we requested for page.  Note that we
 378      * don't need locking because this function will only be called within the
 379      * postcopy ram fault thread.
 380      */
 381     if (rb != mis->last_rb) {
 382         mis->last_rb = rb;
 383
 384         rbname = qemu_ram_get_idstr(rb);
 385         rbname_len = strlen(rbname);
 386
 387         assert(rbname_len < 256);
 388
 389         bufc[msglen++] = rbname_len;
 390         memcpy(bufc + msglen, rbname, rbname_len);
 391         msglen += rbname_len;
 392         msg_type = MIG_RP_MSG_REQ_PAGES_ID;
 393     } else {
 394         msg_type = MIG_RP_MSG_REQ_PAGES;
 395     }
 396
 397     return migrate_send_rp_message(mis, msg_type, msglen, bufc);
 398 }
 399
 400 int migrate_send_rp_req_pages(MigrationIncomingState *mis,
 401                               RAMBlock *rb, ram_addr_t start, uint64_t haddr)
 402 {
 403     void *aligned = (void *)(uintptr_t)ROUND_DOWN(haddr, qemu_ram_pagesize(rb));
 404     bool received = false;
 405
 406     WITH_QEMU_LOCK_GUARD(&mis->page_request_mutex) {
 407         received = ramblock_recv_bitmap_test_byte_offset(rb, start);
 408         if (!received && !g_tree_lookup(mis->page_requested, aligned)) {
 409             /*
 410              * The page has not been received, and it's not yet in the page
 411              * request list.  Queue it.  Set the value of element to 1, so that
 412              * things like g_tree_lookup() will return TRUE (1) when found.
 413              */
 414             g_tree_insert(mis->page_requested, aligned, (gpointer)1);
 415             mis->page_requested_count++;
 416             trace_postcopy_page_req_add(aligned, mis->page_requested_count);
 417         }
 418     }
 419
 420     /*
 421      * If the page is there, skip sending the message.  We don't even need the
 422      * lock because as long as the page arrived, it'll be there forever.
 423      */
 424     if (received) {
 425         return 0;
 426     }
 427
 428     return migrate_send_rp_message_req_pages(mis, rb, start);
 429 }
 430
 431 static bool migration_colo_enabled;
 432 bool migration_incoming_colo_enabled(void)
 433 {
 434     return migration_colo_enabled;
 435 }
 436
 437 void migration_incoming_disable_colo(void)
 438 {
 439     ram_block_discard_disable(false);
 440     migration_colo_enabled = false;
 441 }
 442
 443 int migration_incoming_enable_colo(void)
 444 {
 445     if (ram_block_discard_disable(true)) {
 446         error_report("COLO: cannot disable RAM discard");
 447         return -EBUSY;
 448     }
 449     migration_colo_enabled = true;
 450     return 0;
 451 }
 452
 453 void migrate_add_address(SocketAddress *address)
 454 {
 455     MigrationIncomingState *mis = migration_incoming_get_current();
 456
 457     QAPI_LIST_PREPEND(mis->socket_address_list,
 458                       QAPI_CLONE(SocketAddress, address));
 459 }
 460
 461 static void qemu_start_incoming_migration(const char *uri, Error **errp)
 462 {
 463     const char *p = NULL;
 464
 465     migrate_protocol_allow_multifd(false); /* reset it anyway */
 466     qapi_event_send_migration(MIGRATION_STATUS_SETUP);
 467     if (strstart(uri, "tcp:", &p) ||
 468         strstart(uri, "unix:", NULL) ||
 469         strstart(uri, "vsock:", NULL)) {
 470         migrate_protocol_allow_multifd(true);
 471         socket_start_incoming_migration(p ? p : uri, errp);
 472 #ifdef CONFIG_RDMA
 473     } else if (strstart(uri, "rdma:", &p)) {
 474         rdma_start_incoming_migration(p, errp);
 475 #endif
 476     } else if (strstart(uri, "exec:", &p)) {
 477         exec_start_incoming_migration(p, errp);
 478     } else if (strstart(uri, "fd:", &p)) {
 479         fd_start_incoming_migration(p, errp);
 480     } else {
 481         error_setg(errp, "unknown migration protocol: %s", uri);
 482     }
 483 }
 484
 485 static void process_incoming_migration_bh(void *opaque)
 486 {
 487     Error *local_err = NULL;
 488     MigrationIncomingState *mis = opaque;
 489
 490     /* If capability late_block_activate is set:
 491      * Only fire up the block code now if we're going to restart the
 492      * VM, else 'cont' will do it.
 493      * This causes file locking to happen; so we don't want it to happen
 494      * unless we really are starting the VM.
 495      */
 496     if (!migrate_late_block_activate() ||
 497          (autostart && (!global_state_received() ||
 498             global_state_get_runstate() == RUN_STATE_RUNNING))) {
 499         /* Make sure all file formats flush their mutable metadata.
 500          * If we get an error here, just don't restart the VM yet. */
 501         bdrv_invalidate_cache_all(&local_err);
 502         if (local_err) {
 503             error_report_err(local_err);
 504             local_err = NULL;
 505             autostart = false;
 506         }
 507     }
 508
 509     /*
 510      * This must happen after all error conditions are dealt with and
 511      * we're sure the VM is going to be running on this host.
 512      */
 513     qemu_announce_self(&mis->announce_timer, migrate_announce_params());
 514
 515     if (multifd_load_cleanup(&local_err) != 0) {
 516         error_report_err(local_err);
 517         autostart = false;
 518     }
 519     /* If global state section was not received or we are in running
 520        state, we need to obey autostart. Any other state is set with
 521        runstate_set. */
 522
 523     dirty_bitmap_mig_before_vm_start();
 524
 525     if (!global_state_received() ||
 526         global_state_get_runstate() == RUN_STATE_RUNNING) {
 527         if (autostart) {
 528             vm_start();
 529         } else {
 530             runstate_set(RUN_STATE_PAUSED);
 531         }
 532     } else if (migration_incoming_colo_enabled()) {
 533         migration_incoming_disable_colo();
 534         vm_start();
 535     } else {
 536         runstate_set(global_state_get_runstate());
 537     }
 538     /*
 539      * This must happen after any state changes since as soon as an external
 540      * observer sees this event they might start to prod at the VM assuming
 541      * it's ready to use.
 542      */
 543     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
 544                       MIGRATION_STATUS_COMPLETED);
 545     qemu_bh_delete(mis->bh);
 546     migration_incoming_state_destroy();
 547 }
 548
 549 static void process_incoming_migration_co(void *opaque)
 550 {
 551     MigrationIncomingState *mis = migration_incoming_get_current();
 552     PostcopyState ps;
 553     int ret;
 554     Error *local_err = NULL;
 555
 556     assert(mis->from_src_file);
 557     mis->migration_incoming_co = qemu_coroutine_self();
 558     mis->largest_page_size = qemu_ram_pagesize_largest();
 559     postcopy_state_set(POSTCOPY_INCOMING_NONE);
 560     migrate_set_state(&mis->state, MIGRATION_STATUS_NONE,
 561                       MIGRATION_STATUS_ACTIVE);
 562     ret = qemu_loadvm_state(mis->from_src_file);
 563
 564     ps = postcopy_state_get();
 565     trace_process_incoming_migration_co_end(ret, ps);
 566     if (ps != POSTCOPY_INCOMING_NONE) {
 567         if (ps == POSTCOPY_INCOMING_ADVISE) {
 568             /*
 569              * Where a migration had postcopy enabled (and thus went to advise)
 570              * but managed to complete within the precopy period, we can use
 571              * the normal exit.
 572              */
 573             postcopy_ram_incoming_cleanup(mis);
 574         } else if (ret >= 0) {
 575             /*
 576              * Postcopy was started, cleanup should happen at the end of the
 577              * postcopy thread.
 578              */
 579             trace_process_incoming_migration_co_postcopy_end_main();
 580             return;
 581         }
 582         /* Else if something went wrong then just fall out of the normal exit */
 583     }
 584
 585     /* we get COLO info, and know if we are in COLO mode */
 586     if (!ret && migration_incoming_colo_enabled()) {
 587         /* Make sure all file formats flush their mutable metadata */
 588         bdrv_invalidate_cache_all(&local_err);
 589         if (local_err) {
 590             error_report_err(local_err);
 591             goto fail;
 592         }
 593
 594         qemu_thread_create(&mis->colo_incoming_thread, "COLO incoming",
 595              colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE);
 596         mis->have_colo_incoming_thread = true;
 597         qemu_coroutine_yield();
 598
 599         qemu_mutex_unlock_iothread();
 600         /* Wait checkpoint incoming thread exit before free resource */
 601         qemu_thread_join(&mis->colo_incoming_thread);
 602         qemu_mutex_lock_iothread();
 603         /* We hold the global iothread lock, so it is safe here */
 604         colo_release_ram_cache();
 605     }
 606
 607     if (ret < 0) {
 608         error_report("load of migration failed: %s", strerror(-ret));
 609         goto fail;
 610     }
 611     mis->bh = qemu_bh_new(process_incoming_migration_bh, mis);
 612     qemu_bh_schedule(mis->bh);
 613     mis->migration_incoming_co = NULL;
 614     return;
 615 fail:
 616     local_err = NULL;
 617     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
 618                       MIGRATION_STATUS_FAILED);
 619     qemu_fclose(mis->from_src_file);
 620     if (multifd_load_cleanup(&local_err) != 0) {
 621         error_report_err(local_err);
 622     }
 623     exit(EXIT_FAILURE);
 624 }
 625
 626 /**
 627  * migration_incoming_setup: Setup incoming migration
 628  * @f: file for main migration channel
 629  * @errp: where to put errors
 630  *
 631  * Returns: %true on success, %false on error.
 632  */
 633 static bool migration_incoming_setup(QEMUFile *f, Error **errp)
 634 {
 635     MigrationIncomingState *mis = migration_incoming_get_current();
 636
 637     if (multifd_load_setup(errp) != 0) {
 638         return false;
 639     }
 640
 641     if (!mis->from_src_file) {
 642         mis->from_src_file = f;
 643     }
 644     qemu_file_set_blocking(f, false);
 645     return true;
 646 }
 647
 648 void migration_incoming_process(void)
 649 {
 650     Coroutine *co = qemu_coroutine_create(process_incoming_migration_co, NULL);
 651     qemu_coroutine_enter(co);
 652 }
 653
 654 /* Returns true if recovered from a paused migration, otherwise false */
 655 static bool postcopy_try_recover(QEMUFile *f)
 656 {
 657     MigrationIncomingState *mis = migration_incoming_get_current();
 658
 659     if (mis->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
 660         /* Resumed from a paused postcopy migration */
 661
 662         mis->from_src_file = f;
 663         /* Postcopy has standalone thread to do vm load */
 664         qemu_file_set_blocking(f, true);
 665
 666         /* Re-configure the return path */
 667         mis->to_src_file = qemu_file_get_return_path(f);
 668
 669         migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
 670                           MIGRATION_STATUS_POSTCOPY_RECOVER);
 671
 672         /*
 673          * Here, we only wake up the main loading thread (while the
 674          * fault thread will still be waiting), so that we can receive
 675          * commands from source now, and answer it if needed. The
 676          * fault thread will be woken up afterwards until we are sure
 677          * that source is ready to reply to page requests.
 678          */
 679         qemu_sem_post(&mis->postcopy_pause_sem_dst);
 680         return true;
 681     }
 682
 683     return false;
 684 }
 685
 686 void migration_fd_process_incoming(QEMUFile *f, Error **errp)
 687 {
 688     if (postcopy_try_recover(f)) {
 689         return;
 690     }
 691
 692     if (!migration_incoming_setup(f, errp)) {
 693         return;
 694     }
 695     migration_incoming_process();
 696 }
 697
 698 void migration_ioc_process_incoming(QIOChannel *ioc, Error **errp)
 699 {
 700     MigrationIncomingState *mis = migration_incoming_get_current();
 701     Error *local_err = NULL;
 702     bool start_migration;
 703
 704     if (!mis->from_src_file) {
 705         /* The first connection (multifd may have multiple) */
 706         QEMUFile *f = qemu_fopen_channel_input(ioc);
 707
 708         /* If it's a recovery, we're done */
 709         if (postcopy_try_recover(f)) {
 710             return;
 711         }
 712
 713         if (!migration_incoming_setup(f, errp)) {
 714             return;
 715         }
 716
 717         /*
 718          * Common migration only needs one channel, so we can start
 719          * right now.  Multifd needs more than one channel, we wait.
 720          */
 721         start_migration = !migrate_use_multifd();
 722     } else {
 723         /* Multiple connections */
 724         assert(migrate_use_multifd());
 725         start_migration = multifd_recv_new_channel(ioc, &local_err);
 726         if (local_err) {
 727             error_propagate(errp, local_err);
 728             return;
 729         }
 730     }
 731
 732     if (start_migration) {
 733         migration_incoming_process();
 734     }
 735 }
 736
 737 /**
 738  * @migration_has_all_channels: We have received all channels that we need
 739  *
 740  * Returns true when we have got connections to all the channels that
 741  * we need for migration.
 742  */
 743 bool migration_has_all_channels(void)
 744 {
 745     MigrationIncomingState *mis = migration_incoming_get_current();
 746     bool all_channels;
 747
 748     all_channels = multifd_recv_all_channels_created();
 749
 750     return all_channels && mis->from_src_file != NULL;
 751 }
 752
 753 /*
 754  * Send a 'SHUT' message on the return channel with the given value
 755  * to indicate that we've finished with the RP.  Non-0 value indicates
 756  * error.
 757  */
 758 void migrate_send_rp_shut(MigrationIncomingState *mis,
 759                           uint32_t value)
 760 {
 761     uint32_t buf;
 762
 763     buf = cpu_to_be32(value);
 764     migrate_send_rp_message(mis, MIG_RP_MSG_SHUT, sizeof(buf), &buf);
 765 }
 766
 767 /*
 768  * Send a 'PONG' message on the return channel with the given value
 769  * (normally in response to a 'PING')
 770  */
 771 void migrate_send_rp_pong(MigrationIncomingState *mis,
 772                           uint32_t value)
 773 {
 774     uint32_t buf;
 775
 776     buf = cpu_to_be32(value);
 777     migrate_send_rp_message(mis, MIG_RP_MSG_PONG, sizeof(buf), &buf);
 778 }
 779
 780 void migrate_send_rp_recv_bitmap(MigrationIncomingState *mis,
 781                                  char *block_name)
 782 {
 783     char buf[512];
 784     int len;
 785     int64_t res;
 786
 787     /*
 788      * First, we send the header part. It contains only the len of
 789      * idstr, and the idstr itself.
 790      */
 791     len = strlen(block_name);
 792     buf[0] = len;
 793     memcpy(buf + 1, block_name, len);
 794
 795     if (mis->state != MIGRATION_STATUS_POSTCOPY_RECOVER) {
 796         error_report("%s: MSG_RP_RECV_BITMAP only used for recovery",
 797                      __func__);
 798         return;
 799     }
 800
 801     migrate_send_rp_message(mis, MIG_RP_MSG_RECV_BITMAP, len + 1, buf);
 802
 803     /*
 804      * Next, we dump the received bitmap to the stream.
 805      *
 806      * TODO: currently we are safe since we are the only one that is
 807      * using the to_src_file handle (fault thread is still paused),
 808      * and it's ok even not taking the mutex. However the best way is
 809      * to take the lock before sending the message header, and release
 810      * the lock after sending the bitmap.
 811      */
 812     qemu_mutex_lock(&mis->rp_mutex);
 813     res = ramblock_recv_bitmap_send(mis->to_src_file, block_name);
 814     qemu_mutex_unlock(&mis->rp_mutex);
 815
 816     trace_migrate_send_rp_recv_bitmap(block_name, res);
 817 }
 818
 819 void migrate_send_rp_resume_ack(MigrationIncomingState *mis, uint32_t value)
 820 {
 821     uint32_t buf;
 822
 823     buf = cpu_to_be32(value);
 824     migrate_send_rp_message(mis, MIG_RP_MSG_RESUME_ACK, sizeof(buf), &buf);
 825 }
 826
 827 MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp)
 828 {
 829     MigrationCapabilityStatusList *head = NULL, **tail = &head;
 830     MigrationCapabilityStatus *caps;
 831     MigrationState *s = migrate_get_current();
 832     int i;
 833
 834     for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
 835 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
 836         if (i == MIGRATION_CAPABILITY_BLOCK) {
 837             continue;
 838         }
 839 #endif
 840         caps = g_malloc0(sizeof(*caps));
 841         caps->capability = i;
 842         caps->state = s->enabled_capabilities[i];
 843         QAPI_LIST_APPEND(tail, caps);
 844     }
 845
 846     return head;
 847 }
 848
 849 MigrationParameters *qmp_query_migrate_parameters(Error **errp)
 850 {
 851     MigrationParameters *params;
 852     MigrationState *s = migrate_get_current();
 853
 854     /* TODO use QAPI_CLONE() instead of duplicating it inline */
 855     params = g_malloc0(sizeof(*params));
 856     params->has_compress_level = true;
 857     params->compress_level = s->parameters.compress_level;
 858     params->has_compress_threads = true;
 859     params->compress_threads = s->parameters.compress_threads;
 860     params->has_compress_wait_thread = true;
 861     params->compress_wait_thread = s->parameters.compress_wait_thread;
 862     params->has_decompress_threads = true;
 863     params->decompress_threads = s->parameters.decompress_threads;
 864     params->has_throttle_trigger_threshold = true;
 865     params->throttle_trigger_threshold = s->parameters.throttle_trigger_threshold;
 866     params->has_cpu_throttle_initial = true;
 867     params->cpu_throttle_initial = s->parameters.cpu_throttle_initial;
 868     params->has_cpu_throttle_increment = true;
 869     params->cpu_throttle_increment = s->parameters.cpu_throttle_increment;
 870     params->has_cpu_throttle_tailslow = true;
 871     params->cpu_throttle_tailslow = s->parameters.cpu_throttle_tailslow;
 872     params->has_tls_creds = true;
 873     params->tls_creds = g_strdup(s->parameters.tls_creds);
 874     params->has_tls_hostname = true;
 875     params->tls_hostname = g_strdup(s->parameters.tls_hostname);
 876     params->has_tls_authz = true;
 877     params->tls_authz = g_strdup(s->parameters.tls_authz ?
 878                                  s->parameters.tls_authz : "");
 879     params->has_max_bandwidth = true;
 880     params->max_bandwidth = s->parameters.max_bandwidth;
 881     params->has_downtime_limit = true;
 882     params->downtime_limit = s->parameters.downtime_limit;
 883     params->has_x_checkpoint_delay = true;
 884     params->x_checkpoint_delay = s->parameters.x_checkpoint_delay;
 885     params->has_block_incremental = true;
 886     params->block_incremental = s->parameters.block_incremental;
 887     params->has_multifd_channels = true;
 888     params->multifd_channels = s->parameters.multifd_channels;
 889     params->has_multifd_compression = true;
 890     params->multifd_compression = s->parameters.multifd_compression;
 891     params->has_multifd_zlib_level = true;
 892     params->multifd_zlib_level = s->parameters.multifd_zlib_level;
 893     params->has_multifd_zstd_level = true;
 894     params->multifd_zstd_level = s->parameters.multifd_zstd_level;
 895     params->has_xbzrle_cache_size = true;
 896     params->xbzrle_cache_size = s->parameters.xbzrle_cache_size;
 897     params->has_max_postcopy_bandwidth = true;
 898     params->max_postcopy_bandwidth = s->parameters.max_postcopy_bandwidth;
 899     params->has_max_cpu_throttle = true;
 900     params->max_cpu_throttle = s->parameters.max_cpu_throttle;
 901     params->has_announce_initial = true;
 902     params->announce_initial = s->parameters.announce_initial;
 903     params->has_announce_max = true;
 904     params->announce_max = s->parameters.announce_max;
 905     params->has_announce_rounds = true;
 906     params->announce_rounds = s->parameters.announce_rounds;
 907     params->has_announce_step = true;
 908     params->announce_step = s->parameters.announce_step;
 909
 910     if (s->parameters.has_block_bitmap_mapping) {
 911         params->has_block_bitmap_mapping = true;
 912         params->block_bitmap_mapping =
 913             QAPI_CLONE(BitmapMigrationNodeAliasList,
 914                        s->parameters.block_bitmap_mapping);
 915     }
 916
 917     return params;
 918 }
 919
 920 AnnounceParameters *migrate_announce_params(void)
 921 {
 922     static AnnounceParameters ap;
 923
 924     MigrationState *s = migrate_get_current();
 925
 926     ap.initial = s->parameters.announce_initial;
 927     ap.max = s->parameters.announce_max;
 928     ap.rounds = s->parameters.announce_rounds;
 929     ap.step = s->parameters.announce_step;
 930
 931     return &ap;
 932 }
 933
 934 /*
 935  * Return true if we're already in the middle of a migration
 936  * (i.e. any of the active or setup states)
 937  */
 938 bool migration_is_setup_or_active(int state)
 939 {
 940     switch (state) {
 941     case MIGRATION_STATUS_ACTIVE:
 942     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
 943     case MIGRATION_STATUS_POSTCOPY_PAUSED:
 944     case MIGRATION_STATUS_POSTCOPY_RECOVER:
 945     case MIGRATION_STATUS_SETUP:
 946     case MIGRATION_STATUS_PRE_SWITCHOVER:
 947     case MIGRATION_STATUS_DEVICE:
 948     case MIGRATION_STATUS_WAIT_UNPLUG:
 949     case MIGRATION_STATUS_COLO:
 950         return true;
 951
 952     default:
 953         return false;
 954
 955     }
 956 }
 957
 958 bool migration_is_running(int state)
 959 {
 960     switch (state) {
 961     case MIGRATION_STATUS_ACTIVE:
 962     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
 963     case MIGRATION_STATUS_POSTCOPY_PAUSED:
 964     case MIGRATION_STATUS_POSTCOPY_RECOVER:
 965     case MIGRATION_STATUS_SETUP:
 966     case MIGRATION_STATUS_PRE_SWITCHOVER:
 967     case MIGRATION_STATUS_DEVICE:
 968     case MIGRATION_STATUS_WAIT_UNPLUG:
 969     case MIGRATION_STATUS_CANCELLING:
 970         return true;
 971
 972     default:
 973         return false;
 974
 975     }
 976 }
 977
 978 static void populate_time_info(MigrationInfo *info, MigrationState *s)
 979 {
 980     info->has_status = true;
 981     info->has_setup_time = true;
 982     info->setup_time = s->setup_time;
 983     if (s->state == MIGRATION_STATUS_COMPLETED) {
 984         info->has_total_time = true;
 985         info->total_time = s->total_time;
 986         info->has_downtime = true;
 987         info->downtime = s->downtime;
 988     } else {
 989         info->has_total_time = true;
 990         info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) -
 991                            s->start_time;
 992         info->has_expected_downtime = true;
 993         info->expected_downtime = s->expected_downtime;
 994     }
 995 }
 996
 997 static void populate_ram_info(MigrationInfo *info, MigrationState *s)
 998 {
 999     size_t page_size = qemu_target_page_size();
1000
1001     info->has_ram = true;
1002     info->ram = g_malloc0(sizeof(*info->ram));
1003     info->ram->transferred = ram_counters.transferred;
1004     info->ram->total = ram_bytes_total();
1005     info->ram->duplicate = ram_counters.duplicate;
1006     /* legacy value.  It is not used anymore */
1007     info->ram->skipped = 0;
1008     info->ram->normal = ram_counters.normal;
1009     info->ram->normal_bytes = ram_counters.normal * page_size;
1010     info->ram->mbps = s->mbps;
1011     info->ram->dirty_sync_count = ram_counters.dirty_sync_count;
1012     info->ram->postcopy_requests = ram_counters.postcopy_requests;
1013     info->ram->page_size = page_size;
1014     info->ram->multifd_bytes = ram_counters.multifd_bytes;
1015     info->ram->pages_per_second = s->pages_per_second;
1016
1017     if (migrate_use_xbzrle()) {
1018         info->has_xbzrle_cache = true;
1019         info->xbzrle_cache = g_malloc0(sizeof(*info->xbzrle_cache));
1020         info->xbzrle_cache->cache_size = migrate_xbzrle_cache_size();
1021         info->xbzrle_cache->bytes = xbzrle_counters.bytes;
1022         info->xbzrle_cache->pages = xbzrle_counters.pages;
1023         info->xbzrle_cache->cache_miss = xbzrle_counters.cache_miss;
1024         info->xbzrle_cache->cache_miss_rate = xbzrle_counters.cache_miss_rate;
1025         info->xbzrle_cache->encoding_rate = xbzrle_counters.encoding_rate;
1026         info->xbzrle_cache->overflow = xbzrle_counters.overflow;
1027     }
1028
1029     if (migrate_use_compression()) {
1030         info->has_compression = true;
1031         info->compression = g_malloc0(sizeof(*info->compression));
1032         info->compression->pages = compression_counters.pages;
1033         info->compression->busy = compression_counters.busy;
1034         info->compression->busy_rate = compression_counters.busy_rate;
1035         info->compression->compressed_size =
1036                                     compression_counters.compressed_size;
1037         info->compression->compression_rate =
1038                                     compression_counters.compression_rate;
1039     }
1040
1041     if (cpu_throttle_active()) {
1042         info->has_cpu_throttle_percentage = true;
1043         info->cpu_throttle_percentage = cpu_throttle_get_percentage();
1044     }
1045
1046     if (s->state != MIGRATION_STATUS_COMPLETED) {
1047         info->ram->remaining = ram_bytes_remaining();
1048         info->ram->dirty_pages_rate = ram_counters.dirty_pages_rate;
1049     }
1050 }
1051
1052 static void populate_disk_info(MigrationInfo *info)
1053 {
1054     if (blk_mig_active()) {
1055         info->has_disk = true;
1056         info->disk = g_malloc0(sizeof(*info->disk));
1057         info->disk->transferred = blk_mig_bytes_transferred();
1058         info->disk->remaining = blk_mig_bytes_remaining();
1059         info->disk->total = blk_mig_bytes_total();
1060     }
1061 }
1062
1063 static void fill_source_migration_info(MigrationInfo *info)
1064 {
1065     MigrationState *s = migrate_get_current();
1066     GSList *cur_blocker = migration_blockers;
1067
1068     info->blocked_reasons = NULL;
1069
1070     /*
1071      * There are two types of reasons a migration might be blocked;
1072      * a) devices marked in VMState as non-migratable, and
1073      * b) Explicit migration blockers
1074      * We need to add both of them here.
1075      */
1076     qemu_savevm_non_migratable_list(&info->blocked_reasons);
1077
1078     while (cur_blocker) {
1079         QAPI_LIST_PREPEND(info->blocked_reasons,
1080                           g_strdup(error_get_pretty(cur_blocker->data)));
1081         cur_blocker = g_slist_next(cur_blocker);
1082     }
1083     info->has_blocked_reasons = info->blocked_reasons != NULL;
1084
1085     switch (s->state) {
1086     case MIGRATION_STATUS_NONE:
1087         /* no migration has happened ever */
1088         /* do not overwrite destination migration status */
1089         return;
1090     case MIGRATION_STATUS_SETUP:
1091         info->has_status = true;
1092         info->has_total_time = false;
1093         break;
1094     case MIGRATION_STATUS_ACTIVE:
1095     case MIGRATION_STATUS_CANCELLING:
1096     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1097     case MIGRATION_STATUS_PRE_SWITCHOVER:
1098     case MIGRATION_STATUS_DEVICE:
1099     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1100     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1101         /* TODO add some postcopy stats */
1102         populate_time_info(info, s);
1103         populate_ram_info(info, s);
1104         populate_disk_info(info);
1105         populate_vfio_info(info);
1106         break;
1107     case MIGRATION_STATUS_COLO:
1108         info->has_status = true;
1109         /* TODO: display COLO specific information (checkpoint info etc.) */
1110         break;
1111     case MIGRATION_STATUS_COMPLETED:
1112         populate_time_info(info, s);
1113         populate_ram_info(info, s);
1114         populate_vfio_info(info);
1115         break;
1116     case MIGRATION_STATUS_FAILED:
1117         info->has_status = true;
1118         if (s->error) {
1119             info->has_error_desc = true;
1120             info->error_desc = g_strdup(error_get_pretty(s->error));
1121         }
1122         break;
1123     case MIGRATION_STATUS_CANCELLED:
1124         info->has_status = true;
1125         break;
1126     case MIGRATION_STATUS_WAIT_UNPLUG:
1127         info->has_status = true;
1128         break;
1129     }
1130     info->status = s->state;
1131 }
1132
1133 typedef enum WriteTrackingSupport {
1134     WT_SUPPORT_UNKNOWN = 0,
1135     WT_SUPPORT_ABSENT,
1136     WT_SUPPORT_AVAILABLE,
1137     WT_SUPPORT_COMPATIBLE
1138 } WriteTrackingSupport;
1139
1140 static
1141 WriteTrackingSupport migrate_query_write_tracking(void)
1142 {
1143     /* Check if kernel supports required UFFD features */
1144     if (!ram_write_tracking_available()) {
1145         return WT_SUPPORT_ABSENT;
1146     }
1147     /*
1148      * Check if current memory configuration is
1149      * compatible with required UFFD features.
1150      */
1151     if (!ram_write_tracking_compatible()) {
1152         return WT_SUPPORT_AVAILABLE;
1153     }
1154
1155     return WT_SUPPORT_COMPATIBLE;
1156 }
1157
1158 /**
1159  * @migration_caps_check - check capability validity
1160  *
1161  * @cap_list: old capability list, array of bool
1162  * @params: new capabilities to be applied soon
1163  * @errp: set *errp if the check failed, with reason
1164  *
1165  * Returns true if check passed, otherwise false.
1166  */
1167 static bool migrate_caps_check(bool *cap_list,
1168                                MigrationCapabilityStatusList *params,
1169                                Error **errp)
1170 {
1171     MigrationCapabilityStatusList *cap;
1172     bool old_postcopy_cap;
1173     MigrationIncomingState *mis = migration_incoming_get_current();
1174
1175     old_postcopy_cap = cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM];
1176
1177     for (cap = params; cap; cap = cap->next) {
1178         cap_list[cap->value->capability] = cap->value->state;
1179     }
1180
1181 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
1182     if (cap_list[MIGRATION_CAPABILITY_BLOCK]) {
1183         error_setg(errp, "QEMU compiled without old-style (blk/-b, inc/-i) "
1184                    "block migration");
1185         error_append_hint(errp, "Use drive_mirror+NBD instead.\n");
1186         return false;
1187     }
1188 #endif
1189
1190 #ifndef CONFIG_REPLICATION
1191     if (cap_list[MIGRATION_CAPABILITY_X_COLO]) {
1192         error_setg(errp, "QEMU compiled without replication module"
1193                    " can't enable COLO");
1194         error_append_hint(errp, "Please enable replication before COLO.\n");
1195         return false;
1196     }
1197 #endif
1198
1199     if (cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM]) {
1200         /* This check is reasonably expensive, so only when it's being
1201          * set the first time, also it's only the destination that needs
1202          * special support.
1203          */
1204         if (!old_postcopy_cap && runstate_check(RUN_STATE_INMIGRATE) &&
1205             !postcopy_ram_supported_by_host(mis)) {
1206             /* postcopy_ram_supported_by_host will have emitted a more
1207              * detailed message
1208              */
1209             error_setg(errp, "Postcopy is not supported");
1210             return false;
1211         }
1212
1213         if (cap_list[MIGRATION_CAPABILITY_X_IGNORE_SHARED]) {
1214             error_setg(errp, "Postcopy is not compatible with ignore-shared");
1215             return false;
1216         }
1217     }
1218
1219     if (cap_list[MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT]) {
1220         WriteTrackingSupport wt_support;
1221         int idx;
1222         /*
1223          * Check if 'background-snapshot' capability is supported by
1224          * host kernel and compatible with guest memory configuration.
1225          */
1226         wt_support = migrate_query_write_tracking();
1227         if (wt_support < WT_SUPPORT_AVAILABLE) {
1228             error_setg(errp, "Background-snapshot is not supported by host kernel");
1229             return false;
1230         }
1231         if (wt_support < WT_SUPPORT_COMPATIBLE) {
1232             error_setg(errp, "Background-snapshot is not compatible "
1233                     "with guest memory configuration");
1234             return false;
1235         }
1236
1237         /*
1238          * Check if there are any migration capabilities
1239          * incompatible with 'background-snapshot'.
1240          */
1241         for (idx = 0; idx < check_caps_background_snapshot.size; idx++) {
1242             int incomp_cap = check_caps_background_snapshot.caps[idx];
1243             if (cap_list[incomp_cap]) {
1244                 error_setg(errp,
1245                         "Background-snapshot is not compatible with %s",
1246                         MigrationCapability_str(incomp_cap));
1247                 return false;
1248             }
1249         }
1250     }
1251
1252     /* incoming side only */
1253     if (runstate_check(RUN_STATE_INMIGRATE) &&
1254         !migrate_multifd_is_allowed() &&
1255         cap_list[MIGRATION_CAPABILITY_MULTIFD]) {
1256         error_setg(errp, "multifd is not supported by current protocol");
1257         return false;
1258     }
1259
1260     return true;
1261 }
1262
1263 static void fill_destination_migration_info(MigrationInfo *info)
1264 {
1265     MigrationIncomingState *mis = migration_incoming_get_current();
1266
1267     if (mis->socket_address_list) {
1268         info->has_socket_address = true;
1269         info->socket_address =
1270             QAPI_CLONE(SocketAddressList, mis->socket_address_list);
1271     }
1272
1273     switch (mis->state) {
1274     case MIGRATION_STATUS_NONE:
1275         return;
1276     case MIGRATION_STATUS_SETUP:
1277     case MIGRATION_STATUS_CANCELLING:
1278     case MIGRATION_STATUS_CANCELLED:
1279     case MIGRATION_STATUS_ACTIVE:
1280     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1281     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1282     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1283     case MIGRATION_STATUS_FAILED:
1284     case MIGRATION_STATUS_COLO:
1285         info->has_status = true;
1286         break;
1287     case MIGRATION_STATUS_COMPLETED:
1288         info->has_status = true;
1289         fill_destination_postcopy_migration_info(info);
1290         break;
1291     }
1292     info->status = mis->state;
1293 }
1294
1295 MigrationInfo *qmp_query_migrate(Error **errp)
1296 {
1297     MigrationInfo *info = g_malloc0(sizeof(*info));
1298
1299     fill_destination_migration_info(info);
1300     fill_source_migration_info(info);
1301
1302     return info;
1303 }
1304
1305 void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params,
1306                                   Error **errp)
1307 {
1308     MigrationState *s = migrate_get_current();
1309     MigrationCapabilityStatusList *cap;
1310     bool cap_list[MIGRATION_CAPABILITY__MAX];
1311
1312     if (migration_is_running(s->state)) {
1313         error_setg(errp, QERR_MIGRATION_ACTIVE);
1314         return;
1315     }
1316
1317     memcpy(cap_list, s->enabled_capabilities, sizeof(cap_list));
1318     if (!migrate_caps_check(cap_list, params, errp)) {
1319         return;
1320     }
1321
1322     for (cap = params; cap; cap = cap->next) {
1323         s->enabled_capabilities[cap->value->capability] = cap->value->state;
1324     }
1325 }
1326
1327 /*
1328  * Check whether the parameters are valid. Error will be put into errp
1329  * (if provided). Return true if valid, otherwise false.
1330  */
1331 static bool migrate_params_check(MigrationParameters *params, Error **errp)
1332 {
1333     if (params->has_compress_level &&
1334         (params->compress_level > 9)) {
1335         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "compress_level",
1336                    "a value between 0 and 9");
1337         return false;
1338     }
1339
1340     if (params->has_compress_threads && (params->compress_threads < 1)) {
1341         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1342                    "compress_threads",
1343                    "a value between 1 and 255");
1344         return false;
1345     }
1346
1347     if (params->has_decompress_threads && (params->decompress_threads < 1)) {
1348         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1349                    "decompress_threads",
1350                    "a value between 1 and 255");
1351         return false;
1352     }
1353
1354     if (params->has_throttle_trigger_threshold &&
1355         (params->throttle_trigger_threshold < 1 ||
1356          params->throttle_trigger_threshold > 100)) {
1357         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1358                    "throttle_trigger_threshold",
1359                    "an integer in the range of 1 to 100");
1360         return false;
1361     }
1362
1363     if (params->has_cpu_throttle_initial &&
1364         (params->cpu_throttle_initial < 1 ||
1365          params->cpu_throttle_initial > 99)) {
1366         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1367                    "cpu_throttle_initial",
1368                    "an integer in the range of 1 to 99");
1369         return false;
1370     }
1371
1372     if (params->has_cpu_throttle_increment &&
1373         (params->cpu_throttle_increment < 1 ||
1374          params->cpu_throttle_increment > 99)) {
1375         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1376                    "cpu_throttle_increment",
1377                    "an integer in the range of 1 to 99");
1378         return false;
1379     }
1380
1381     if (params->has_max_bandwidth && (params->max_bandwidth > SIZE_MAX)) {
1382         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1383                    "max_bandwidth",
1384                    "an integer in the range of 0 to "stringify(SIZE_MAX)
1385                    " bytes/second");
1386         return false;
1387     }
1388
1389     if (params->has_downtime_limit &&
1390         (params->downtime_limit > MAX_MIGRATE_DOWNTIME)) {
1391         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1392                    "downtime_limit",
1393                    "an integer in the range of 0 to "
1394                     stringify(MAX_MIGRATE_DOWNTIME)" ms");
1395         return false;
1396     }
1397
1398     /* x_checkpoint_delay is now always positive */
1399
1400     if (params->has_multifd_channels && (params->multifd_channels < 1)) {
1401         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1402                    "multifd_channels",
1403                    "a value between 1 and 255");
1404         return false;
1405     }
1406
1407     if (params->has_multifd_zlib_level &&
1408         (params->multifd_zlib_level > 9)) {
1409         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zlib_level",
1410                    "a value between 0 and 9");
1411         return false;
1412     }
1413
1414     if (params->has_multifd_zstd_level &&
1415         (params->multifd_zstd_level > 20)) {
1416         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zstd_level",
1417                    "a value between 0 and 20");
1418         return false;
1419     }
1420
1421     if (params->has_xbzrle_cache_size &&
1422         (params->xbzrle_cache_size < qemu_target_page_size() ||
1423          !is_power_of_2(params->xbzrle_cache_size))) {
1424         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1425                    "xbzrle_cache_size",
1426                    "a power of two no less than the target page size");
1427         return false;
1428     }
1429
1430     if (params->has_max_cpu_throttle &&
1431         (params->max_cpu_throttle < params->cpu_throttle_initial ||
1432          params->max_cpu_throttle > 99)) {
1433         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1434                    "max_cpu_throttle",
1435                    "an integer in the range of cpu_throttle_initial to 99");
1436         return false;
1437     }
1438
1439     if (params->has_announce_initial &&
1440         params->announce_initial > 100000) {
1441         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1442                    "announce_initial",
1443                    "a value between 0 and 100000");
1444         return false;
1445     }
1446     if (params->has_announce_max &&
1447         params->announce_max > 100000) {
1448         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1449                    "announce_max",
1450                    "a value between 0 and 100000");
1451        return false;
1452     }
1453     if (params->has_announce_rounds &&
1454         params->announce_rounds > 1000) {
1455         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1456                    "announce_rounds",
1457                    "a value between 0 and 1000");
1458        return false;
1459     }
1460     if (params->has_announce_step &&
1461         (params->announce_step < 1 ||
1462         params->announce_step > 10000)) {
1463         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1464                    "announce_step",
1465                    "a value between 0 and 10000");
1466        return false;
1467     }
1468
1469     if (params->has_block_bitmap_mapping &&
1470         !check_dirty_bitmap_mig_alias_map(params->block_bitmap_mapping, errp)) {
1471         error_prepend(errp, "Invalid mapping given for block-bitmap-mapping: ");
1472         return false;
1473     }
1474
1475     return true;
1476 }
1477
1478 static void migrate_params_test_apply(MigrateSetParameters *params,
1479                                       MigrationParameters *dest)
1480 {
1481     *dest = migrate_get_current()->parameters;
1482
1483     /* TODO use QAPI_CLONE() instead of duplicating it inline */
1484
1485     if (params->has_compress_level) {
1486         dest->compress_level = params->compress_level;
1487     }
1488
1489     if (params->has_compress_threads) {
1490         dest->compress_threads = params->compress_threads;
1491     }
1492
1493     if (params->has_compress_wait_thread) {
1494         dest->compress_wait_thread = params->compress_wait_thread;
1495     }
1496
1497     if (params->has_decompress_threads) {
1498         dest->decompress_threads = params->decompress_threads;
1499     }
1500
1501     if (params->has_throttle_trigger_threshold) {
1502         dest->throttle_trigger_threshold = params->throttle_trigger_threshold;
1503     }
1504
1505     if (params->has_cpu_throttle_initial) {
1506         dest->cpu_throttle_initial = params->cpu_throttle_initial;
1507     }
1508
1509     if (params->has_cpu_throttle_increment) {
1510         dest->cpu_throttle_increment = params->cpu_throttle_increment;
1511     }
1512
1513     if (params->has_cpu_throttle_tailslow) {
1514         dest->cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1515     }
1516
1517     if (params->has_tls_creds) {
1518         assert(params->tls_creds->type == QTYPE_QSTRING);
1519         dest->tls_creds = params->tls_creds->u.s;
1520     }
1521
1522     if (params->has_tls_hostname) {
1523         assert(params->tls_hostname->type == QTYPE_QSTRING);
1524         dest->tls_hostname = params->tls_hostname->u.s;
1525     }
1526
1527     if (params->has_max_bandwidth) {
1528         dest->max_bandwidth = params->max_bandwidth;
1529     }
1530
1531     if (params->has_downtime_limit) {
1532         dest->downtime_limit = params->downtime_limit;
1533     }
1534
1535     if (params->has_x_checkpoint_delay) {
1536         dest->x_checkpoint_delay = params->x_checkpoint_delay;
1537     }
1538
1539     if (params->has_block_incremental) {
1540         dest->block_incremental = params->block_incremental;
1541     }
1542     if (params->has_multifd_channels) {
1543         dest->multifd_channels = params->multifd_channels;
1544     }
1545     if (params->has_multifd_compression) {
1546         dest->multifd_compression = params->multifd_compression;
1547     }
1548     if (params->has_xbzrle_cache_size) {
1549         dest->xbzrle_cache_size = params->xbzrle_cache_size;
1550     }
1551     if (params->has_max_postcopy_bandwidth) {
1552         dest->max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1553     }
1554     if (params->has_max_cpu_throttle) {
1555         dest->max_cpu_throttle = params->max_cpu_throttle;
1556     }
1557     if (params->has_announce_initial) {
1558         dest->announce_initial = params->announce_initial;
1559     }
1560     if (params->has_announce_max) {
1561         dest->announce_max = params->announce_max;
1562     }
1563     if (params->has_announce_rounds) {
1564         dest->announce_rounds = params->announce_rounds;
1565     }
1566     if (params->has_announce_step) {
1567         dest->announce_step = params->announce_step;
1568     }
1569
1570     if (params->has_block_bitmap_mapping) {
1571         dest->has_block_bitmap_mapping = true;
1572         dest->block_bitmap_mapping = params->block_bitmap_mapping;
1573     }
1574 }
1575
1576 static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
1577 {
1578     MigrationState *s = migrate_get_current();
1579
1580     /* TODO use QAPI_CLONE() instead of duplicating it inline */
1581
1582     if (params->has_compress_level) {
1583         s->parameters.compress_level = params->compress_level;
1584     }
1585
1586     if (params->has_compress_threads) {
1587         s->parameters.compress_threads = params->compress_threads;
1588     }
1589
1590     if (params->has_compress_wait_thread) {
1591         s->parameters.compress_wait_thread = params->compress_wait_thread;
1592     }
1593
1594     if (params->has_decompress_threads) {
1595         s->parameters.decompress_threads = params->decompress_threads;
1596     }
1597
1598     if (params->has_throttle_trigger_threshold) {
1599         s->parameters.throttle_trigger_threshold = params->throttle_trigger_threshold;
1600     }
1601
1602     if (params->has_cpu_throttle_initial) {
1603         s->parameters.cpu_throttle_initial = params->cpu_throttle_initial;
1604     }
1605
1606     if (params->has_cpu_throttle_increment) {
1607         s->parameters.cpu_throttle_increment = params->cpu_throttle_increment;
1608     }
1609
1610     if (params->has_cpu_throttle_tailslow) {
1611         s->parameters.cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1612     }
1613
1614     if (params->has_tls_creds) {
1615         g_free(s->parameters.tls_creds);
1616         assert(params->tls_creds->type == QTYPE_QSTRING);
1617         s->parameters.tls_creds = g_strdup(params->tls_creds->u.s);
1618     }
1619
1620     if (params->has_tls_hostname) {
1621         g_free(s->parameters.tls_hostname);
1622         assert(params->tls_hostname->type == QTYPE_QSTRING);
1623         s->parameters.tls_hostname = g_strdup(params->tls_hostname->u.s);
1624     }
1625
1626     if (params->has_tls_authz) {
1627         g_free(s->parameters.tls_authz);
1628         assert(params->tls_authz->type == QTYPE_QSTRING);
1629         s->parameters.tls_authz = g_strdup(params->tls_authz->u.s);
1630     }
1631
1632     if (params->has_max_bandwidth) {
1633         s->parameters.max_bandwidth = params->max_bandwidth;
1634         if (s->to_dst_file && !migration_in_postcopy()) {
1635             qemu_file_set_rate_limit(s->to_dst_file,
1636                                 s->parameters.max_bandwidth / XFER_LIMIT_RATIO);
1637         }
1638     }
1639
1640     if (params->has_downtime_limit) {
1641         s->parameters.downtime_limit = params->downtime_limit;
1642     }
1643
1644     if (params->has_x_checkpoint_delay) {
1645         s->parameters.x_checkpoint_delay = params->x_checkpoint_delay;
1646         if (migration_in_colo_state()) {
1647             colo_checkpoint_notify(s);
1648         }
1649     }
1650
1651     if (params->has_block_incremental) {
1652         s->parameters.block_incremental = params->block_incremental;
1653     }
1654     if (params->has_multifd_channels) {
1655         s->parameters.multifd_channels = params->multifd_channels;
1656     }
1657     if (params->has_multifd_compression) {
1658         s->parameters.multifd_compression = params->multifd_compression;
1659     }
1660     if (params->has_xbzrle_cache_size) {
1661         s->parameters.xbzrle_cache_size = params->xbzrle_cache_size;
1662         xbzrle_cache_resize(params->xbzrle_cache_size, errp);
1663     }
1664     if (params->has_max_postcopy_bandwidth) {
1665         s->parameters.max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1666         if (s->to_dst_file && migration_in_postcopy()) {
1667             qemu_file_set_rate_limit(s->to_dst_file,
1668                     s->parameters.max_postcopy_bandwidth / XFER_LIMIT_RATIO);
1669         }
1670     }
1671     if (params->has_max_cpu_throttle) {
1672         s->parameters.max_cpu_throttle = params->max_cpu_throttle;
1673     }
1674     if (params->has_announce_initial) {
1675         s->parameters.announce_initial = params->announce_initial;
1676     }
1677     if (params->has_announce_max) {
1678         s->parameters.announce_max = params->announce_max;
1679     }
1680     if (params->has_announce_rounds) {
1681         s->parameters.announce_rounds = params->announce_rounds;
1682     }
1683     if (params->has_announce_step) {
1684         s->parameters.announce_step = params->announce_step;
1685     }
1686
1687     if (params->has_block_bitmap_mapping) {
1688         qapi_free_BitmapMigrationNodeAliasList(
1689             s->parameters.block_bitmap_mapping);
1690
1691         s->parameters.has_block_bitmap_mapping = true;
1692         s->parameters.block_bitmap_mapping =
1693             QAPI_CLONE(BitmapMigrationNodeAliasList,
1694                        params->block_bitmap_mapping);
1695     }
1696 }
1697
1698 void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp)
1699 {
1700     MigrationParameters tmp;
1701
1702     /* TODO Rewrite "" to null instead */
1703     if (params->has_tls_creds
1704         && params->tls_creds->type == QTYPE_QNULL) {
1705         qobject_unref(params->tls_creds->u.n);
1706         params->tls_creds->type = QTYPE_QSTRING;
1707         params->tls_creds->u.s = strdup("");
1708     }
1709     /* TODO Rewrite "" to null instead */
1710     if (params->has_tls_hostname
1711         && params->tls_hostname->type == QTYPE_QNULL) {
1712         qobject_unref(params->tls_hostname->u.n);
1713         params->tls_hostname->type = QTYPE_QSTRING;
1714         params->tls_hostname->u.s = strdup("");
1715     }
1716
1717     migrate_params_test_apply(params, &tmp);
1718
1719     if (!migrate_params_check(&tmp, errp)) {
1720         /* Invalid parameter */
1721         return;
1722     }
1723
1724     migrate_params_apply(params, errp);
1725 }
1726
1727
1728 void qmp_migrate_start_postcopy(Error **errp)
1729 {
1730     MigrationState *s = migrate_get_current();
1731
1732     if (!migrate_postcopy()) {
1733         error_setg(errp, "Enable postcopy with migrate_set_capability before"
1734                          " the start of migration");
1735         return;
1736     }
1737
1738     if (s->state == MIGRATION_STATUS_NONE) {
1739         error_setg(errp, "Postcopy must be started after migration has been"
1740                          " started");
1741         return;
1742     }
1743     /*
1744      * we don't error if migration has finished since that would be racy
1745      * with issuing this command.
1746      */
1747     qatomic_set(&s->start_postcopy, true);
1748 }
1749
1750 /* shared migration helpers */
1751
1752 void migrate_set_state(int *state, int old_state, int new_state)
1753 {
1754     assert(new_state < MIGRATION_STATUS__MAX);
1755     if (qatomic_cmpxchg(state, old_state, new_state) == old_state) {
1756         trace_migrate_set_state(MigrationStatus_str(new_state));
1757         migrate_generate_event(new_state);
1758     }
1759 }
1760
1761 static MigrationCapabilityStatus *migrate_cap_add(MigrationCapability index,
1762                                                   bool state)
1763 {
1764     MigrationCapabilityStatus *cap;
1765
1766     cap = g_new0(MigrationCapabilityStatus, 1);
1767     cap->capability = index;
1768     cap->state = state;
1769
1770     return cap;
1771 }
1772
1773 void migrate_set_block_enabled(bool value, Error **errp)
1774 {
1775     MigrationCapabilityStatusList *cap = NULL;
1776
1777     QAPI_LIST_PREPEND(cap, migrate_cap_add(MIGRATION_CAPABILITY_BLOCK, value));
1778     qmp_migrate_set_capabilities(cap, errp);
1779     qapi_free_MigrationCapabilityStatusList(cap);
1780 }
1781
1782 static void migrate_set_block_incremental(MigrationState *s, bool value)
1783 {
1784     s->parameters.block_incremental = value;
1785 }
1786
1787 static void block_cleanup_parameters(MigrationState *s)
1788 {
1789     if (s->must_remove_block_options) {
1790         /* setting to false can never fail */
1791         migrate_set_block_enabled(false, &error_abort);
1792         migrate_set_block_incremental(s, false);
1793         s->must_remove_block_options = false;
1794     }
1795 }
1796
1797 static void migrate_fd_cleanup(MigrationState *s)
1798 {
1799     qemu_bh_delete(s->cleanup_bh);
1800     s->cleanup_bh = NULL;
1801
1802     qemu_savevm_state_cleanup();
1803
1804     if (s->to_dst_file) {
1805         QEMUFile *tmp;
1806
1807         trace_migrate_fd_cleanup();
1808         qemu_mutex_unlock_iothread();
1809         if (s->migration_thread_running) {
1810             qemu_thread_join(&s->thread);
1811             s->migration_thread_running = false;
1812         }
1813         qemu_mutex_lock_iothread();
1814
1815         multifd_save_cleanup();
1816         qemu_mutex_lock(&s->qemu_file_lock);
1817         tmp = s->to_dst_file;
1818         s->to_dst_file = NULL;
1819         qemu_mutex_unlock(&s->qemu_file_lock);
1820         /*
1821          * Close the file handle without the lock to make sure the
1822          * critical section won't block for long.
1823          */
1824         migration_ioc_unregister_yank_from_file(tmp);
1825         qemu_fclose(tmp);
1826     }
1827
1828     assert(!migration_is_active(s));
1829
1830     if (s->state == MIGRATION_STATUS_CANCELLING) {
1831         migrate_set_state(&s->state, MIGRATION_STATUS_CANCELLING,
1832                           MIGRATION_STATUS_CANCELLED);
1833     }
1834
1835     if (s->error) {
1836         /* It is used on info migrate.  We can't free it */
1837         error_report_err(error_copy(s->error));
1838     }
1839     notifier_list_notify(&migration_state_notifiers, s);
1840     block_cleanup_parameters(s);
1841     yank_unregister_instance(MIGRATION_YANK_INSTANCE);
1842 }
1843
1844 static void migrate_fd_cleanup_schedule(MigrationState *s)
1845 {
1846     /*
1847      * Ref the state for bh, because it may be called when
1848      * there're already no other refs
1849      */
1850     object_ref(OBJECT(s));
1851     qemu_bh_schedule(s->cleanup_bh);
1852 }
1853
1854 static void migrate_fd_cleanup_bh(void *opaque)
1855 {
1856     MigrationState *s = opaque;
1857     migrate_fd_cleanup(s);
1858     object_unref(OBJECT(s));
1859 }
1860
1861 void migrate_set_error(MigrationState *s, const Error *error)
1862 {
1863     QEMU_LOCK_GUARD(&s->error_mutex);
1864     if (!s->error) {
1865         s->error = error_copy(error);
1866     }
1867 }
1868
1869 static void migrate_error_free(MigrationState *s)
1870 {
1871     QEMU_LOCK_GUARD(&s->error_mutex);
1872     if (s->error) {
1873         error_free(s->error);
1874         s->error = NULL;
1875     }
1876 }
1877
1878 void migrate_fd_error(MigrationState *s, const Error *error)
1879 {
1880     trace_migrate_fd_error(error_get_pretty(error));
1881     assert(s->to_dst_file == NULL);
1882     migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
1883                       MIGRATION_STATUS_FAILED);
1884     migrate_set_error(s, error);
1885 }
1886
1887 static void migrate_fd_cancel(MigrationState *s)
1888 {
1889     int old_state ;
1890     QEMUFile *f = migrate_get_current()->to_dst_file;
1891     trace_migrate_fd_cancel();
1892
1893     WITH_QEMU_LOCK_GUARD(&s->qemu_file_lock) {
1894         if (s->rp_state.from_dst_file) {
1895             /* shutdown the rp socket, so causing the rp thread to shutdown */
1896             qemu_file_shutdown(s->rp_state.from_dst_file);
1897         }
1898     }
1899
1900     do {
1901         old_state = s->state;
1902         if (!migration_is_running(old_state)) {
1903             break;
1904         }
1905         /* If the migration is paused, kick it out of the pause */
1906         if (old_state == MIGRATION_STATUS_PRE_SWITCHOVER) {
1907             qemu_sem_post(&s->pause_sem);
1908         }
1909         migrate_set_state(&s->state, old_state, MIGRATION_STATUS_CANCELLING);
1910     } while (s->state != MIGRATION_STATUS_CANCELLING);
1911
1912     /*
1913      * If we're unlucky the migration code might be stuck somewhere in a
1914      * send/write while the network has failed and is waiting to timeout;
1915      * if we've got shutdown(2) available then we can force it to quit.
1916      * The outgoing qemu file gets closed in migrate_fd_cleanup that is
1917      * called in a bh, so there is no race against this cancel.
1918      */
1919     if (s->state == MIGRATION_STATUS_CANCELLING && f) {
1920         qemu_file_shutdown(f);
1921     }
1922     if (s->state == MIGRATION_STATUS_CANCELLING && s->block_inactive) {
1923         Error *local_err = NULL;
1924
1925         bdrv_invalidate_cache_all(&local_err);
1926         if (local_err) {
1927             error_report_err(local_err);
1928         } else {
1929             s->block_inactive = false;
1930         }
1931     }
1932 }
1933
1934 void add_migration_state_change_notifier(Notifier *notify)
1935 {
1936     notifier_list_add(&migration_state_notifiers, notify);
1937 }
1938
1939 void remove_migration_state_change_notifier(Notifier *notify)
1940 {
1941     notifier_remove(notify);
1942 }
1943
1944 bool migration_in_setup(MigrationState *s)
1945 {
1946     return s->state == MIGRATION_STATUS_SETUP;
1947 }
1948
1949 bool migration_has_finished(MigrationState *s)
1950 {
1951     return s->state == MIGRATION_STATUS_COMPLETED;
1952 }
1953
1954 bool migration_has_failed(MigrationState *s)
1955 {
1956     return (s->state == MIGRATION_STATUS_CANCELLED ||
1957             s->state == MIGRATION_STATUS_FAILED);
1958 }
1959
1960 bool migration_in_postcopy(void)
1961 {
1962     MigrationState *s = migrate_get_current();
1963
1964     switch (s->state) {
1965     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1966     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1967     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1968         return true;
1969     default:
1970         return false;
1971     }
1972 }
1973
1974 bool migration_in_postcopy_after_devices(MigrationState *s)
1975 {
1976     return migration_in_postcopy() && s->postcopy_after_devices;
1977 }
1978
1979 bool migration_in_incoming_postcopy(void)
1980 {
1981     PostcopyState ps = postcopy_state_get();
1982
1983     return ps >= POSTCOPY_INCOMING_DISCARD && ps < POSTCOPY_INCOMING_END;
1984 }
1985
1986 bool migration_in_bg_snapshot(void)
1987 {
1988     MigrationState *s = migrate_get_current();
1989
1990     return migrate_background_snapshot() &&
1991             migration_is_setup_or_active(s->state);
1992 }
1993
1994 bool migration_is_idle(void)
1995 {
1996     MigrationState *s = current_migration;
1997
1998     if (!s) {
1999         return true;
2000     }
2001
2002     switch (s->state) {
2003     case MIGRATION_STATUS_NONE:
2004     case MIGRATION_STATUS_CANCELLED:
2005     case MIGRATION_STATUS_COMPLETED:
2006     case MIGRATION_STATUS_FAILED:
2007         return true;
2008     case MIGRATION_STATUS_SETUP:
2009     case MIGRATION_STATUS_CANCELLING:
2010     case MIGRATION_STATUS_ACTIVE:
2011     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
2012     case MIGRATION_STATUS_COLO:
2013     case MIGRATION_STATUS_PRE_SWITCHOVER:
2014     case MIGRATION_STATUS_DEVICE:
2015     case MIGRATION_STATUS_WAIT_UNPLUG:
2016         return false;
2017     case MIGRATION_STATUS__MAX:
2018         g_assert_not_reached();
2019     }
2020
2021     return false;
2022 }
2023
2024 bool migration_is_active(MigrationState *s)
2025 {
2026     return (s->state == MIGRATION_STATUS_ACTIVE ||
2027             s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
2028 }
2029
2030 void migrate_init(MigrationState *s)
2031 {
2032     /*
2033      * Reinitialise all migration state, except
2034      * parameters/capabilities that the user set, and
2035      * locks.
2036      */
2037     s->cleanup_bh = 0;
2038     s->vm_start_bh = 0;
2039     s->to_dst_file = NULL;
2040     s->state = MIGRATION_STATUS_NONE;
2041     s->rp_state.from_dst_file = NULL;
2042     s->rp_state.error = false;
2043     s->mbps = 0.0;
2044     s->pages_per_second = 0.0;
2045     s->downtime = 0;
2046     s->expected_downtime = 0;
2047     s->setup_time = 0;
2048     s->start_postcopy = false;
2049     s->postcopy_after_devices = false;
2050     s->migration_thread_running = false;
2051     error_free(s->error);
2052     s->error = NULL;
2053     s->hostname = NULL;
2054
2055     migrate_set_state(&s->state, MIGRATION_STATUS_NONE, MIGRATION_STATUS_SETUP);
2056
2057     s->start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
2058     s->total_time = 0;
2059     s->vm_was_running = false;
2060     s->iteration_initial_bytes = 0;
2061     s->threshold_size = 0;
2062 }
2063
2064 int migrate_add_blocker_internal(Error *reason, Error **errp)
2065 {
2066     /* Snapshots are similar to migrations, so check RUN_STATE_SAVE_VM too. */
2067     if (runstate_check(RUN_STATE_SAVE_VM) || !migration_is_idle()) {
2068         error_propagate_prepend(errp, error_copy(reason),
2069                                 "disallowing migration blocker "
2070                                 "(migration/snapshot in progress) for: ");
2071         return -EBUSY;
2072     }
2073
2074     migration_blockers = g_slist_prepend(migration_blockers, reason);
2075     return 0;
2076 }
2077
2078 int migrate_add_blocker(Error *reason, Error **errp)
2079 {
2080     if (only_migratable) {
2081         error_propagate_prepend(errp, error_copy(reason),
2082                                 "disallowing migration blocker "
2083                                 "(--only-migratable) for: ");
2084         return -EACCES;
2085     }
2086
2087     return migrate_add_blocker_internal(reason, errp);
2088 }
2089
2090 void migrate_del_blocker(Error *reason)
2091 {
2092     migration_blockers = g_slist_remove(migration_blockers, reason);
2093 }
2094
2095 void qmp_migrate_incoming(const char *uri, Error **errp)
2096 {
2097     Error *local_err = NULL;
2098     static bool once = true;
2099
2100     if (!once) {
2101         error_setg(errp, "The incoming migration has already been started");
2102         return;
2103     }
2104     if (!runstate_check(RUN_STATE_INMIGRATE)) {
2105         error_setg(errp, "'-incoming' was not specified on the command line");
2106         return;
2107     }
2108
2109     if (!yank_register_instance(MIGRATION_YANK_INSTANCE, errp)) {
2110         return;
2111     }
2112
2113     qemu_start_incoming_migration(uri, &local_err);
2114
2115     if (local_err) {
2116         yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2117         error_propagate(errp, local_err);
2118         return;
2119     }
2120
2121     once = false;
2122 }
2123
2124 void qmp_migrate_recover(const char *uri, Error **errp)
2125 {
2126     MigrationIncomingState *mis = migration_incoming_get_current();
2127
2128     /*
2129      * Don't even bother to use ERRP_GUARD() as it _must_ always be set by
2130      * callers (no one should ignore a recover failure); if there is, it's a
2131      * programming error.
2132      */
2133     assert(errp);
2134
2135     if (mis->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
2136         error_setg(errp, "Migrate recover can only be run "
2137                    "when postcopy is paused.");
2138         return;
2139     }
2140
2141     if (qatomic_cmpxchg(&mis->postcopy_recover_triggered,
2142                        false, true) == true) {
2143         error_setg(errp, "Migrate recovery is triggered already");
2144         return;
2145     }
2146
2147     /*
2148      * Note that this call will never start a real migration; it will
2149      * only re-setup the migration stream and poke existing migration
2150      * to continue using that newly established channel.
2151      */
2152     qemu_start_incoming_migration(uri, errp);
2153
2154     /* Safe to dereference with the assert above */
2155     if (*errp) {
2156         /* Reset the flag so user could still retry */
2157         qatomic_set(&mis->postcopy_recover_triggered, false);
2158     }
2159 }
2160
2161 void qmp_migrate_pause(Error **errp)
2162 {
2163     MigrationState *ms = migrate_get_current();
2164     MigrationIncomingState *mis = migration_incoming_get_current();
2165     int ret;
2166
2167     if (ms->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
2168         /* Source side, during postcopy */
2169         qemu_mutex_lock(&ms->qemu_file_lock);
2170         ret = qemu_file_shutdown(ms->to_dst_file);
2171         qemu_mutex_unlock(&ms->qemu_file_lock);
2172         if (ret) {
2173             error_setg(errp, "Failed to pause source migration");
2174         }
2175         return;
2176     }
2177
2178     if (mis->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
2179         ret = qemu_file_shutdown(mis->from_src_file);
2180         if (ret) {
2181             error_setg(errp, "Failed to pause destination migration");
2182         }
2183         return;
2184     }
2185
2186     error_setg(errp, "migrate-pause is currently only supported "
2187                "during postcopy-active state");
2188 }
2189
2190 bool migration_is_blocked(Error **errp)
2191 {
2192     if (qemu_savevm_state_blocked(errp)) {
2193         return true;
2194     }
2195
2196     if (migration_blockers) {
2197         error_propagate(errp, error_copy(migration_blockers->data));
2198         return true;
2199     }
2200
2201     return false;
2202 }
2203
2204 /* Returns true if continue to migrate, or false if error detected */
2205 static bool migrate_prepare(MigrationState *s, bool blk, bool blk_inc,
2206                             bool resume, Error **errp)
2207 {
2208     Error *local_err = NULL;
2209
2210     if (resume) {
2211         if (s->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
2212             error_setg(errp, "Cannot resume if there is no "
2213                        "paused migration");
2214             return false;
2215         }
2216
2217         /*
2218          * Postcopy recovery won't work well with release-ram
2219          * capability since release-ram will drop the page buffer as
2220          * long as the page is put into the send buffer.  So if there
2221          * is a network failure happened, any page buffers that have
2222          * not yet reached the destination VM but have already been
2223          * sent from the source VM will be lost forever.  Let's refuse
2224          * the client from resuming such a postcopy migration.
2225          * Luckily release-ram was designed to only be used when src
2226          * and destination VMs are on the same host, so it should be
2227          * fine.
2228          */
2229         if (migrate_release_ram()) {
2230             error_setg(errp, "Postcopy recovery cannot work "
2231                        "when release-ram capability is set");
2232             return false;
2233         }
2234
2235         /* This is a resume, skip init status */
2236         return true;
2237     }
2238
2239     if (migration_is_running(s->state)) {
2240         error_setg(errp, QERR_MIGRATION_ACTIVE);
2241         return false;
2242     }
2243
2244     if (runstate_check(RUN_STATE_INMIGRATE)) {
2245         error_setg(errp, "Guest is waiting for an incoming migration");
2246         return false;
2247     }
2248
2249     if (runstate_check(RUN_STATE_POSTMIGRATE)) {
2250         error_setg(errp, "Can't migrate the vm that was paused due to "
2251                    "previous migration");
2252         return false;
2253     }
2254
2255     if (migration_is_blocked(errp)) {
2256         return false;
2257     }
2258
2259     if (blk || blk_inc) {
2260         if (migrate_colo_enabled()) {
2261             error_setg(errp, "No disk migration is required in COLO mode");
2262             return false;
2263         }
2264         if (migrate_use_block() || migrate_use_block_incremental()) {
2265             error_setg(errp, "Command options are incompatible with "
2266                        "current migration capabilities");
2267             return false;
2268         }
2269         migrate_set_block_enabled(true, &local_err);
2270         if (local_err) {
2271             error_propagate(errp, local_err);
2272             return false;
2273         }
2274         s->must_remove_block_options = true;
2275     }
2276
2277     if (blk_inc) {
2278         migrate_set_block_incremental(s, true);
2279     }
2280
2281     migrate_init(s);
2282     /*
2283      * set ram_counters compression_counters memory to zero for a
2284      * new migration
2285      */
2286     memset(&ram_counters, 0, sizeof(ram_counters));
2287     memset(&compression_counters, 0, sizeof(compression_counters));
2288
2289     return true;
2290 }
2291
2292 void qmp_migrate(const char *uri, bool has_blk, bool blk,
2293                  bool has_inc, bool inc, bool has_detach, bool detach,
2294                  bool has_resume, bool resume, Error **errp)
2295 {
2296     Error *local_err = NULL;
2297     MigrationState *s = migrate_get_current();
2298     const char *p = NULL;
2299
2300     if (!migrate_prepare(s, has_blk && blk, has_inc && inc,
2301                          has_resume && resume, errp)) {
2302         /* Error detected, put into errp */
2303         return;
2304     }
2305
2306     if (!(has_resume && resume)) {
2307         if (!yank_register_instance(MIGRATION_YANK_INSTANCE, errp)) {
2308             return;
2309         }
2310     }
2311
2312     migrate_protocol_allow_multifd(false);
2313     if (strstart(uri, "tcp:", &p) ||
2314         strstart(uri, "unix:", NULL) ||
2315         strstart(uri, "vsock:", NULL)) {
2316         migrate_protocol_allow_multifd(true);
2317         socket_start_outgoing_migration(s, p ? p : uri, &local_err);
2318 #ifdef CONFIG_RDMA
2319     } else if (strstart(uri, "rdma:", &p)) {
2320         rdma_start_outgoing_migration(s, p, &local_err);
2321 #endif
2322     } else if (strstart(uri, "exec:", &p)) {
2323         exec_start_outgoing_migration(s, p, &local_err);
2324     } else if (strstart(uri, "fd:", &p)) {
2325         fd_start_outgoing_migration(s, p, &local_err);
2326     } else {
2327         if (!(has_resume && resume)) {
2328             yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2329         }
2330         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "uri",
2331                    "a valid migration protocol");
2332         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
2333                           MIGRATION_STATUS_FAILED);
2334         block_cleanup_parameters(s);
2335         return;
2336     }
2337
2338     if (local_err) {
2339         if (!(has_resume && resume)) {
2340             yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2341         }
2342         migrate_fd_error(s, local_err);
2343         error_propagate(errp, local_err);
2344         return;
2345     }
2346 }
2347
2348 void qmp_migrate_cancel(Error **errp)
2349 {
2350     migration_cancel(NULL);
2351 }
2352
2353 void qmp_migrate_continue(MigrationStatus state, Error **errp)
2354 {
2355     MigrationState *s = migrate_get_current();
2356     if (s->state != state) {
2357         error_setg(errp,  "Migration not in expected state: %s",
2358                    MigrationStatus_str(s->state));
2359         return;
2360     }
2361     qemu_sem_post(&s->pause_sem);
2362 }
2363
2364 bool migrate_release_ram(void)
2365 {
2366     MigrationState *s;
2367
2368     s = migrate_get_current();
2369
2370     return s->enabled_capabilities[MIGRATION_CAPABILITY_RELEASE_RAM];
2371 }
2372
2373 bool migrate_postcopy_ram(void)
2374 {
2375     MigrationState *s;
2376
2377     s = migrate_get_current();
2378
2379     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_RAM];
2380 }
2381
2382 bool migrate_postcopy(void)
2383 {
2384     return migrate_postcopy_ram() || migrate_dirty_bitmaps();
2385 }
2386
2387 bool migrate_auto_converge(void)
2388 {
2389     MigrationState *s;
2390
2391     s = migrate_get_current();
2392
2393     return s->enabled_capabilities[MIGRATION_CAPABILITY_AUTO_CONVERGE];
2394 }
2395
2396 bool migrate_zero_blocks(void)
2397 {
2398     MigrationState *s;
2399
2400     s = migrate_get_current();
2401
2402     return s->enabled_capabilities[MIGRATION_CAPABILITY_ZERO_BLOCKS];
2403 }
2404
2405 bool migrate_postcopy_blocktime(void)
2406 {
2407     MigrationState *s;
2408
2409     s = migrate_get_current();
2410
2411     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME];
2412 }
2413
2414 bool migrate_use_compression(void)
2415 {
2416     MigrationState *s;
2417
2418     s = migrate_get_current();
2419
2420     return s->enabled_capabilities[MIGRATION_CAPABILITY_COMPRESS];
2421 }
2422
2423 int migrate_compress_level(void)
2424 {
2425     MigrationState *s;
2426
2427     s = migrate_get_current();
2428
2429     return s->parameters.compress_level;
2430 }
2431
2432 int migrate_compress_threads(void)
2433 {
2434     MigrationState *s;
2435
2436     s = migrate_get_current();
2437
2438     return s->parameters.compress_threads;
2439 }
2440
2441 int migrate_compress_wait_thread(void)
2442 {
2443     MigrationState *s;
2444
2445     s = migrate_get_current();
2446
2447     return s->parameters.compress_wait_thread;
2448 }
2449
2450 int migrate_decompress_threads(void)
2451 {
2452     MigrationState *s;
2453
2454     s = migrate_get_current();
2455
2456     return s->parameters.decompress_threads;
2457 }
2458
2459 bool migrate_dirty_bitmaps(void)
2460 {
2461     MigrationState *s;
2462
2463     s = migrate_get_current();
2464
2465     return s->enabled_capabilities[MIGRATION_CAPABILITY_DIRTY_BITMAPS];
2466 }
2467
2468 bool migrate_ignore_shared(void)
2469 {
2470     MigrationState *s;
2471
2472     s = migrate_get_current();
2473
2474     return s->enabled_capabilities[MIGRATION_CAPABILITY_X_IGNORE_SHARED];
2475 }
2476
2477 bool migrate_validate_uuid(void)
2478 {
2479     MigrationState *s;
2480
2481     s = migrate_get_current();
2482
2483     return s->enabled_capabilities[MIGRATION_CAPABILITY_VALIDATE_UUID];
2484 }
2485
2486 bool migrate_use_events(void)
2487 {
2488     MigrationState *s;
2489
2490     s = migrate_get_current();
2491
2492     return s->enabled_capabilities[MIGRATION_CAPABILITY_EVENTS];
2493 }
2494
2495 bool migrate_use_multifd(void)
2496 {
2497     MigrationState *s;
2498
2499     s = migrate_get_current();
2500
2501     return s->enabled_capabilities[MIGRATION_CAPABILITY_MULTIFD];
2502 }
2503
2504 bool migrate_pause_before_switchover(void)
2505 {
2506     MigrationState *s;
2507
2508     s = migrate_get_current();
2509
2510     return s->enabled_capabilities[
2511         MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER];
2512 }
2513
2514 int migrate_multifd_channels(void)
2515 {
2516     MigrationState *s;
2517
2518     s = migrate_get_current();
2519
2520     return s->parameters.multifd_channels;
2521 }
2522
2523 MultiFDCompression migrate_multifd_compression(void)
2524 {
2525     MigrationState *s;
2526
2527     s = migrate_get_current();
2528
2529     return s->parameters.multifd_compression;
2530 }
2531
2532 int migrate_multifd_zlib_level(void)
2533 {
2534     MigrationState *s;
2535
2536     s = migrate_get_current();
2537
2538     return s->parameters.multifd_zlib_level;
2539 }
2540
2541 int migrate_multifd_zstd_level(void)
2542 {
2543     MigrationState *s;
2544
2545     s = migrate_get_current();
2546
2547     return s->parameters.multifd_zstd_level;
2548 }
2549
2550 int migrate_use_xbzrle(void)
2551 {
2552     MigrationState *s;
2553
2554     s = migrate_get_current();
2555
2556     return s->enabled_capabilities[MIGRATION_CAPABILITY_XBZRLE];
2557 }
2558
2559 uint64_t migrate_xbzrle_cache_size(void)
2560 {
2561     MigrationState *s;
2562
2563     s = migrate_get_current();
2564
2565     return s->parameters.xbzrle_cache_size;
2566 }
2567
2568 static int64_t migrate_max_postcopy_bandwidth(void)
2569 {
2570     MigrationState *s;
2571
2572     s = migrate_get_current();
2573
2574     return s->parameters.max_postcopy_bandwidth;
2575 }
2576
2577 bool migrate_use_block(void)
2578 {
2579     MigrationState *s;
2580
2581     s = migrate_get_current();
2582
2583     return s->enabled_capabilities[MIGRATION_CAPABILITY_BLOCK];
2584 }
2585
2586 bool migrate_use_return_path(void)
2587 {
2588     MigrationState *s;
2589
2590     s = migrate_get_current();
2591
2592     return s->enabled_capabilities[MIGRATION_CAPABILITY_RETURN_PATH];
2593 }
2594
2595 bool migrate_use_block_incremental(void)
2596 {
2597     MigrationState *s;
2598
2599     s = migrate_get_current();
2600
2601     return s->parameters.block_incremental;
2602 }
2603
2604 bool migrate_background_snapshot(void)
2605 {
2606     MigrationState *s;
2607
2608     s = migrate_get_current();
2609
2610     return s->enabled_capabilities[MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT];
2611 }
2612
2613 /* migration thread support */
2614 /*
2615  * Something bad happened to the RP stream, mark an error
2616  * The caller shall print or trace something to indicate why
2617  */
2618 static void mark_source_rp_bad(MigrationState *s)
2619 {
2620     s->rp_state.error = true;
2621 }
2622
2623 static struct rp_cmd_args {
2624     ssize_t     len; /* -1 = variable */
2625     const char *name;
2626 } rp_cmd_args[] = {
2627     [MIG_RP_MSG_INVALID]        = { .len = -1, .name = "INVALID" },
2628     [MIG_RP_MSG_SHUT]           = { .len =  4, .name = "SHUT" },
2629     [MIG_RP_MSG_PONG]           = { .len =  4, .name = "PONG" },
2630     [MIG_RP_MSG_REQ_PAGES]      = { .len = 12, .name = "REQ_PAGES" },
2631     [MIG_RP_MSG_REQ_PAGES_ID]   = { .len = -1, .name = "REQ_PAGES_ID" },
2632     [MIG_RP_MSG_RECV_BITMAP]    = { .len = -1, .name = "RECV_BITMAP" },
2633     [MIG_RP_MSG_RESUME_ACK]     = { .len =  4, .name = "RESUME_ACK" },
2634     [MIG_RP_MSG_MAX]            = { .len = -1, .name = "MAX" },
2635 };
2636
2637 /*
2638  * Process a request for pages received on the return path,
2639  * We're allowed to send more than requested (e.g. to round to our page size)
2640  * and we don't need to send pages that have already been sent.
2641  */
2642 static void migrate_handle_rp_req_pages(MigrationState *ms, const char* rbname,
2643                                        ram_addr_t start, size_t len)
2644 {
2645     long our_host_ps = qemu_real_host_page_size;
2646
2647     trace_migrate_handle_rp_req_pages(rbname, start, len);
2648
2649     /*
2650      * Since we currently insist on matching page sizes, just sanity check
2651      * we're being asked for whole host pages.
2652      */
2653     if (!QEMU_IS_ALIGNED(start, our_host_ps) ||
2654         !QEMU_IS_ALIGNED(len, our_host_ps)) {
2655         error_report("%s: Misaligned page request, start: " RAM_ADDR_FMT
2656                      " len: %zd", __func__, start, len);
2657         mark_source_rp_bad(ms);
2658         return;
2659     }
2660
2661     if (ram_save_queue_pages(rbname, start, len)) {
2662         mark_source_rp_bad(ms);
2663     }
2664 }
2665
2666 /* Return true to retry, false to quit */
2667 static bool postcopy_pause_return_path_thread(MigrationState *s)
2668 {
2669     trace_postcopy_pause_return_path();
2670
2671     qemu_sem_wait(&s->postcopy_pause_rp_sem);
2672
2673     trace_postcopy_pause_return_path_continued();
2674
2675     return true;
2676 }
2677
2678 static int migrate_handle_rp_recv_bitmap(MigrationState *s, char *block_name)
2679 {
2680     RAMBlock *block = qemu_ram_block_by_name(block_name);
2681
2682     if (!block) {
2683         error_report("%s: invalid block name '%s'", __func__, block_name);
2684         return -EINVAL;
2685     }
2686
2687     /* Fetch the received bitmap and refresh the dirty bitmap */
2688     return ram_dirty_bitmap_reload(s, block);
2689 }
2690
2691 static int migrate_handle_rp_resume_ack(MigrationState *s, uint32_t value)
2692 {
2693     trace_source_return_path_thread_resume_ack(value);
2694
2695     if (value != MIGRATION_RESUME_ACK_VALUE) {
2696         error_report("%s: illegal resume_ack value %"PRIu32,
2697                      __func__, value);
2698         return -1;
2699     }
2700
2701     /* Now both sides are active. */
2702     migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_RECOVER,
2703                       MIGRATION_STATUS_POSTCOPY_ACTIVE);
2704
2705     /* Notify send thread that time to continue send pages */
2706     qemu_sem_post(&s->rp_state.rp_sem);
2707
2708     return 0;
2709 }
2710
2711 /* Release ms->rp_state.from_dst_file in a safe way */
2712 static void migration_release_from_dst_file(MigrationState *ms)
2713 {
2714     QEMUFile *file;
2715
2716     WITH_QEMU_LOCK_GUARD(&ms->qemu_file_lock) {
2717         /*
2718          * Reset the from_dst_file pointer first before releasing it, as we
2719          * can't block within lock section
2720          */
2721         file = ms->rp_state.from_dst_file;
2722         ms->rp_state.from_dst_file = NULL;
2723     }
2724
2725     qemu_fclose(file);
2726 }
2727
2728 /*
2729  * Handles messages sent on the return path towards the source VM
2730  *
2731  */
2732 static void *source_return_path_thread(void *opaque)
2733 {
2734     MigrationState *ms = opaque;
2735     QEMUFile *rp = ms->rp_state.from_dst_file;
2736     uint16_t header_len, header_type;
2737     uint8_t buf[512];
2738     uint32_t tmp32, sibling_error;
2739     ram_addr_t start = 0; /* =0 to silence warning */
2740     size_t  len = 0, expected_len;
2741     int res;
2742
2743     trace_source_return_path_thread_entry();
2744     rcu_register_thread();
2745
2746 retry:
2747     while (!ms->rp_state.error && !qemu_file_get_error(rp) &&
2748            migration_is_setup_or_active(ms->state)) {
2749         trace_source_return_path_thread_loop_top();
2750         header_type = qemu_get_be16(rp);
2751         header_len = qemu_get_be16(rp);
2752
2753         if (qemu_file_get_error(rp)) {
2754             mark_source_rp_bad(ms);
2755             goto out;
2756         }
2757
2758         if (header_type >= MIG_RP_MSG_MAX ||
2759             header_type == MIG_RP_MSG_INVALID) {
2760             error_report("RP: Received invalid message 0x%04x length 0x%04x",
2761                          header_type, header_len);
2762             mark_source_rp_bad(ms);
2763             goto out;
2764         }
2765
2766         if ((rp_cmd_args[header_type].len != -1 &&
2767             header_len != rp_cmd_args[header_type].len) ||
2768             header_len > sizeof(buf)) {
2769             error_report("RP: Received '%s' message (0x%04x) with"
2770                          "incorrect length %d expecting %zu",
2771                          rp_cmd_args[header_type].name, header_type, header_len,
2772                          (size_t)rp_cmd_args[header_type].len);
2773             mark_source_rp_bad(ms);
2774             goto out;
2775         }
2776
2777         /* We know we've got a valid header by this point */
2778         res = qemu_get_buffer(rp, buf, header_len);
2779         if (res != header_len) {
2780             error_report("RP: Failed reading data for message 0x%04x"
2781                          " read %d expected %d",
2782                          header_type, res, header_len);
2783             mark_source_rp_bad(ms);
2784             goto out;
2785         }
2786
2787         /* OK, we have the message and the data */
2788         switch (header_type) {
2789         case MIG_RP_MSG_SHUT:
2790             sibling_error = ldl_be_p(buf);
2791             trace_source_return_path_thread_shut(sibling_error);
2792             if (sibling_error) {
2793                 error_report("RP: Sibling indicated error %d", sibling_error);
2794                 mark_source_rp_bad(ms);
2795             }
2796             /*
2797              * We'll let the main thread deal with closing the RP
2798              * we could do a shutdown(2) on it, but we're the only user
2799              * anyway, so there's nothing gained.
2800              */
2801             goto out;
2802
2803         case MIG_RP_MSG_PONG:
2804             tmp32 = ldl_be_p(buf);
2805             trace_source_return_path_thread_pong(tmp32);
2806             break;
2807
2808         case MIG_RP_MSG_REQ_PAGES:
2809             start = ldq_be_p(buf);
2810             len = ldl_be_p(buf + 8);
2811             migrate_handle_rp_req_pages(ms, NULL, start, len);
2812             break;
2813
2814         case MIG_RP_MSG_REQ_PAGES_ID:
2815             expected_len = 12 + 1; /* header + termination */
2816
2817             if (header_len >= expected_len) {
2818                 start = ldq_be_p(buf);
2819                 len = ldl_be_p(buf + 8);
2820                 /* Now we expect an idstr */
2821                 tmp32 = buf[12]; /* Length of the following idstr */
2822                 buf[13 + tmp32] = '\0';
2823                 expected_len += tmp32;
2824             }
2825             if (header_len != expected_len) {
2826                 error_report("RP: Req_Page_id with length %d expecting %zd",
2827                              header_len, expected_len);
2828                 mark_source_rp_bad(ms);
2829                 goto out;
2830             }
2831             migrate_handle_rp_req_pages(ms, (char *)&buf[13], start, len);
2832             break;
2833
2834         case MIG_RP_MSG_RECV_BITMAP:
2835             if (header_len < 1) {
2836                 error_report("%s: missing block name", __func__);
2837                 mark_source_rp_bad(ms);
2838                 goto out;
2839             }
2840             /* Format: len (1B) + idstr (<255B). This ends the idstr. */
2841             buf[buf[0] + 1] = '\0';
2842             if (migrate_handle_rp_recv_bitmap(ms, (char *)(buf + 1))) {
2843                 mark_source_rp_bad(ms);
2844                 goto out;
2845             }
2846             break;
2847
2848         case MIG_RP_MSG_RESUME_ACK:
2849             tmp32 = ldl_be_p(buf);
2850             if (migrate_handle_rp_resume_ack(ms, tmp32)) {
2851                 mark_source_rp_bad(ms);
2852                 goto out;
2853             }
2854             break;
2855
2856         default:
2857             break;
2858         }
2859     }
2860
2861 out:
2862     res = qemu_file_get_error(rp);
2863     if (res) {
2864         if (res == -EIO && migration_in_postcopy()) {
2865             /*
2866              * Maybe there is something we can do: it looks like a
2867              * network down issue, and we pause for a recovery.
2868              */
2869             migration_release_from_dst_file(ms);
2870             rp = NULL;
2871             if (postcopy_pause_return_path_thread(ms)) {
2872                 /*
2873                  * Reload rp, reset the rest.  Referencing it is safe since
2874                  * it's reset only by us above, or when migration completes
2875                  */
2876                 rp = ms->rp_state.from_dst_file;
2877                 ms->rp_state.error = false;
2878                 goto retry;
2879             }
2880         }
2881
2882         trace_source_return_path_thread_bad_end();
2883         mark_source_rp_bad(ms);
2884     }
2885
2886     trace_source_return_path_thread_end();
2887     migration_release_from_dst_file(ms);
2888     rcu_unregister_thread();
2889     return NULL;
2890 }
2891
2892 static int open_return_path_on_source(MigrationState *ms,
2893                                       bool create_thread)
2894 {
2895     ms->rp_state.from_dst_file = qemu_file_get_return_path(ms->to_dst_file);
2896     if (!ms->rp_state.from_dst_file) {
2897         return -1;
2898     }
2899
2900     trace_open_return_path_on_source();
2901
2902     if (!create_thread) {
2903         /* We're done */
2904         return 0;
2905     }
2906
2907     qemu_thread_create(&ms->rp_state.rp_thread, "return path",
2908                        source_return_path_thread, ms, QEMU_THREAD_JOINABLE);
2909     ms->rp_state.rp_thread_created = true;
2910
2911     trace_open_return_path_on_source_continue();
2912
2913     return 0;
2914 }
2915
2916 /* Returns 0 if the RP was ok, otherwise there was an error on the RP */
2917 static int await_return_path_close_on_source(MigrationState *ms)
2918 {
2919     /*
2920      * If this is a normal exit then the destination will send a SHUT and the
2921      * rp_thread will exit, however if there's an error we need to cause
2922      * it to exit.
2923      */
2924     if (qemu_file_get_error(ms->to_dst_file) && ms->rp_state.from_dst_file) {
2925         /*
2926          * shutdown(2), if we have it, will cause it to unblock if it's stuck
2927          * waiting for the destination.
2928          */
2929         qemu_file_shutdown(ms->rp_state.from_dst_file);
2930         mark_source_rp_bad(ms);
2931     }
2932     trace_await_return_path_close_on_source_joining();
2933     qemu_thread_join(&ms->rp_state.rp_thread);
2934     ms->rp_state.rp_thread_created = false;
2935     trace_await_return_path_close_on_source_close();
2936     return ms->rp_state.error;
2937 }
2938
2939 /*
2940  * Switch from normal iteration to postcopy
2941  * Returns non-0 on error
2942  */
2943 static int postcopy_start(MigrationState *ms)
2944 {
2945     int ret;
2946     QIOChannelBuffer *bioc;
2947     QEMUFile *fb;
2948     int64_t time_at_stop = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
2949     int64_t bandwidth = migrate_max_postcopy_bandwidth();
2950     bool restart_block = false;
2951     int cur_state = MIGRATION_STATUS_ACTIVE;
2952     if (!migrate_pause_before_switchover()) {
2953         migrate_set_state(&ms->state, MIGRATION_STATUS_ACTIVE,
2954                           MIGRATION_STATUS_POSTCOPY_ACTIVE);
2955     }
2956
2957     trace_postcopy_start();
2958     qemu_mutex_lock_iothread();
2959     trace_postcopy_start_set_run();
2960
2961     qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
2962     global_state_store();
2963     ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
2964     if (ret < 0) {
2965         goto fail;
2966     }
2967
2968     ret = migration_maybe_pause(ms, &cur_state,
2969                                 MIGRATION_STATUS_POSTCOPY_ACTIVE);
2970     if (ret < 0) {
2971         goto fail;
2972     }
2973
2974     ret = bdrv_inactivate_all();
2975     if (ret < 0) {
2976         goto fail;
2977     }
2978     restart_block = true;
2979
2980     /*
2981      * Cause any non-postcopiable, but iterative devices to
2982      * send out their final data.
2983      */
2984     qemu_savevm_state_complete_precopy(ms->to_dst_file, true, false);
2985
2986     /*
2987      * in Finish migrate and with the io-lock held everything should
2988      * be quiet, but we've potentially still got dirty pages and we
2989      * need to tell the destination to throw any pages it's already received
2990      * that are dirty
2991      */
2992     if (migrate_postcopy_ram()) {
2993         if (ram_postcopy_send_discard_bitmap(ms)) {
2994             error_report("postcopy send discard bitmap failed");
2995             goto fail;
2996         }
2997     }
2998
2999     /*
3000      * send rest of state - note things that are doing postcopy
3001      * will notice we're in POSTCOPY_ACTIVE and not actually
3002      * wrap their state up here
3003      */
3004     /* 0 max-postcopy-bandwidth means unlimited */
3005     if (!bandwidth) {
3006         qemu_file_set_rate_limit(ms->to_dst_file, INT64_MAX);
3007     } else {
3008         qemu_file_set_rate_limit(ms->to_dst_file, bandwidth / XFER_LIMIT_RATIO);
3009     }
3010     if (migrate_postcopy_ram()) {
3011         /* Ping just for debugging, helps line traces up */
3012         qemu_savevm_send_ping(ms->to_dst_file, 2);
3013     }
3014
3015     /*
3016      * While loading the device state we may trigger page transfer
3017      * requests and the fd must be free to process those, and thus
3018      * the destination must read the whole device state off the fd before
3019      * it starts processing it.  Unfortunately the ad-hoc migration format
3020      * doesn't allow the destination to know the size to read without fully
3021      * parsing it through each devices load-state code (especially the open
3022      * coded devices that use get/put).
3023      * So we wrap the device state up in a package with a length at the start;
3024      * to do this we use a qemu_buf to hold the whole of the device state.
3025      */
3026     bioc = qio_channel_buffer_new(4096);
3027     qio_channel_set_name(QIO_CHANNEL(bioc), "migration-postcopy-buffer");
3028     fb = qemu_fopen_channel_output(QIO_CHANNEL(bioc));
3029     object_unref(OBJECT(bioc));
3030
3031     /*
3032      * Make sure the receiver can get incoming pages before we send the rest
3033      * of the state
3034      */
3035     qemu_savevm_send_postcopy_listen(fb);
3036
3037     qemu_savevm_state_complete_precopy(fb, false, false);
3038     if (migrate_postcopy_ram()) {
3039         qemu_savevm_send_ping(fb, 3);
3040     }
3041
3042     qemu_savevm_send_postcopy_run(fb);
3043
3044     /* <><> end of stuff going into the package */
3045
3046     /* Last point of recovery; as soon as we send the package the destination
3047      * can open devices and potentially start running.
3048      * Lets just check again we've not got any errors.
3049      */
3050     ret = qemu_file_get_error(ms->to_dst_file);
3051     if (ret) {
3052         error_report("postcopy_start: Migration stream errored (pre package)");
3053         goto fail_closefb;
3054     }
3055
3056     restart_block = false;
3057
3058     /* Now send that blob */
3059     if (qemu_savevm_send_packaged(ms->to_dst_file, bioc->data, bioc->usage)) {
3060         goto fail_closefb;
3061     }
3062     qemu_fclose(fb);
3063
3064     /* Send a notify to give a chance for anything that needs to happen
3065      * at the transition to postcopy and after the device state; in particular
3066      * spice needs to trigger a transition now
3067      */
3068     ms->postcopy_after_devices = true;
3069     notifier_list_notify(&migration_state_notifiers, ms);
3070
3071     ms->downtime =  qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - time_at_stop;
3072
3073     qemu_mutex_unlock_iothread();
3074
3075     if (migrate_postcopy_ram()) {
3076         /*
3077          * Although this ping is just for debug, it could potentially be
3078          * used for getting a better measurement of downtime at the source.
3079          */
3080         qemu_savevm_send_ping(ms->to_dst_file, 4);
3081     }
3082
3083     if (migrate_release_ram()) {
3084         ram_postcopy_migrated_memory_release(ms);
3085     }
3086
3087     ret = qemu_file_get_error(ms->to_dst_file);
3088     if (ret) {
3089         error_report("postcopy_start: Migration stream errored");
3090         migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
3091                               MIGRATION_STATUS_FAILED);
3092     }
3093
3094     return ret;
3095
3096 fail_closefb:
3097     qemu_fclose(fb);
3098 fail:
3099     migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
3100                           MIGRATION_STATUS_FAILED);
3101     if (restart_block) {
3102         /* A failure happened early enough that we know the destination hasn't
3103          * accessed block devices, so we're safe to recover.
3104          */
3105         Error *local_err = NULL;
3106
3107         bdrv_invalidate_cache_all(&local_err);
3108         if (local_err) {
3109             error_report_err(local_err);
3110         }
3111     }
3112     qemu_mutex_unlock_iothread();
3113     return -1;
3114 }
3115
3116 /**
3117  * migration_maybe_pause: Pause if required to by
3118  * migrate_pause_before_switchover called with the iothread locked
3119  * Returns: 0 on success
3120  */
3121 static int migration_maybe_pause(MigrationState *s,
3122                                  int *current_active_state,
3123                                  int new_state)
3124 {
3125     if (!migrate_pause_before_switchover()) {
3126         return 0;
3127     }
3128
3129     /* Since leaving this state is not atomic with posting the semaphore
3130      * it's possible that someone could have issued multiple migrate_continue
3131      * and the semaphore is incorrectly positive at this point;
3132      * the docs say it's undefined to reinit a semaphore that's already
3133      * init'd, so use timedwait to eat up any existing posts.
3134      */
3135     while (qemu_sem_timedwait(&s->pause_sem, 1) == 0) {
3136         /* This block intentionally left blank */
3137     }
3138
3139     /*
3140      * If the migration is cancelled when it is in the completion phase,
3141      * the migration state is set to MIGRATION_STATUS_CANCELLING.
3142      * So we don't need to wait a semaphore, otherwise we would always
3143      * wait for the 'pause_sem' semaphore.
3144      */
3145     if (s->state != MIGRATION_STATUS_CANCELLING) {
3146         qemu_mutex_unlock_iothread();
3147         migrate_set_state(&s->state, *current_active_state,
3148                           MIGRATION_STATUS_PRE_SWITCHOVER);
3149         qemu_sem_wait(&s->pause_sem);
3150         migrate_set_state(&s->state, MIGRATION_STATUS_PRE_SWITCHOVER,
3151                           new_state);
3152         *current_active_state = new_state;
3153         qemu_mutex_lock_iothread();
3154     }
3155
3156     return s->state == new_state ? 0 : -EINVAL;
3157 }
3158
3159 /**
3160  * migration_completion: Used by migration_thread when there's not much left.
3161  *   The caller 'breaks' the loop when this returns.
3162  *
3163  * @s: Current migration state
3164  */
3165 static void migration_completion(MigrationState *s)
3166 {
3167     int ret;
3168     int current_active_state = s->state;
3169
3170     if (s->state == MIGRATION_STATUS_ACTIVE) {
3171         qemu_mutex_lock_iothread();
3172         s->downtime_start = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3173         qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
3174         s->vm_was_running = runstate_is_running();
3175         ret = global_state_store();
3176
3177         if (!ret) {
3178             bool inactivate = !migrate_colo_enabled();
3179             ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
3180             trace_migration_completion_vm_stop(ret);
3181             if (ret >= 0) {
3182                 ret = migration_maybe_pause(s, &current_active_state,
3183                                             MIGRATION_STATUS_DEVICE);
3184             }
3185             if (ret >= 0) {
3186                 qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX);
3187                 ret = qemu_savevm_state_complete_precopy(s->to_dst_file, false,
3188                                                          inactivate);
3189             }
3190             if (inactivate && ret >= 0) {
3191                 s->block_inactive = true;
3192             }
3193         }
3194         qemu_mutex_unlock_iothread();
3195
3196         if (ret < 0) {
3197             goto fail;
3198         }
3199     } else if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
3200         trace_migration_completion_postcopy_end();
3201
3202         qemu_mutex_lock_iothread();
3203         qemu_savevm_state_complete_postcopy(s->to_dst_file);
3204         qemu_mutex_unlock_iothread();
3205
3206         trace_migration_completion_postcopy_end_after_complete();
3207     } else if (s->state == MIGRATION_STATUS_CANCELLING) {
3208         goto fail;
3209     }
3210
3211     /*
3212      * If rp was opened we must clean up the thread before
3213      * cleaning everything else up (since if there are no failures
3214      * it will wait for the destination to send it's status in
3215      * a SHUT command).
3216      */
3217     if (s->rp_state.rp_thread_created) {
3218         int rp_error;
3219         trace_migration_return_path_end_before();
3220         rp_error = await_return_path_close_on_source(s);
3221         trace_migration_return_path_end_after(rp_error);
3222         if (rp_error) {
3223             goto fail_invalidate;
3224         }
3225     }
3226
3227     if (qemu_file_get_error(s->to_dst_file)) {
3228         trace_migration_completion_file_err();
3229         goto fail_invalidate;
3230     }
3231
3232     if (!migrate_colo_enabled()) {
3233         migrate_set_state(&s->state, current_active_state,
3234                           MIGRATION_STATUS_COMPLETED);
3235     }
3236
3237     return;
3238
3239 fail_invalidate:
3240     /* If not doing postcopy, vm_start() will be called: let's regain
3241      * control on images.
3242      */
3243     if (s->state == MIGRATION_STATUS_ACTIVE ||
3244         s->state == MIGRATION_STATUS_DEVICE) {
3245         Error *local_err = NULL;
3246
3247         qemu_mutex_lock_iothread();
3248         bdrv_invalidate_cache_all(&local_err);
3249         if (local_err) {
3250             error_report_err(local_err);
3251         } else {
3252             s->block_inactive = false;
3253         }
3254         qemu_mutex_unlock_iothread();
3255     }
3256
3257 fail:
3258     migrate_set_state(&s->state, current_active_state,
3259                       MIGRATION_STATUS_FAILED);
3260 }
3261
3262 /**
3263  * bg_migration_completion: Used by bg_migration_thread when after all the
3264  *   RAM has been saved. The caller 'breaks' the loop when this returns.
3265  *
3266  * @s: Current migration state
3267  */
3268 static void bg_migration_completion(MigrationState *s)
3269 {
3270     int current_active_state = s->state;
3271
3272     /*
3273      * Stop tracking RAM writes - un-protect memory, un-register UFFD
3274      * memory ranges, flush kernel wait queues and wake up threads
3275      * waiting for write fault to be resolved.
3276      */
3277     ram_write_tracking_stop();
3278
3279     if (s->state == MIGRATION_STATUS_ACTIVE) {
3280         /*
3281          * By this moment we have RAM content saved into the migration stream.
3282          * The next step is to flush the non-RAM content (device state)
3283          * right after the ram content. The device state has been stored into
3284          * the temporary buffer before RAM saving started.
3285          */
3286         qemu_put_buffer(s->to_dst_file, s->bioc->data, s->bioc->usage);
3287         qemu_fflush(s->to_dst_file);
3288     } else if (s->state == MIGRATION_STATUS_CANCELLING) {
3289         goto fail;
3290     }
3291
3292     if (qemu_file_get_error(s->to_dst_file)) {
3293         trace_migration_completion_file_err();
3294         goto fail;
3295     }
3296
3297     migrate_set_state(&s->state, current_active_state,
3298                       MIGRATION_STATUS_COMPLETED);
3299     return;
3300
3301 fail:
3302     migrate_set_state(&s->state, current_active_state,
3303                       MIGRATION_STATUS_FAILED);
3304 }
3305
3306 bool migrate_colo_enabled(void)
3307 {
3308     MigrationState *s = migrate_get_current();
3309     return s->enabled_capabilities[MIGRATION_CAPABILITY_X_COLO];
3310 }
3311
3312 typedef enum MigThrError {
3313     /* No error detected */
3314     MIG_THR_ERR_NONE = 0,
3315     /* Detected error, but resumed successfully */
3316     MIG_THR_ERR_RECOVERED = 1,
3317     /* Detected fatal error, need to exit */
3318     MIG_THR_ERR_FATAL = 2,
3319 } MigThrError;
3320
3321 static int postcopy_resume_handshake(MigrationState *s)
3322 {
3323     qemu_savevm_send_postcopy_resume(s->to_dst_file);
3324
3325     while (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3326         qemu_sem_wait(&s->rp_state.rp_sem);
3327     }
3328
3329     if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
3330         return 0;
3331     }
3332
3333     return -1;
3334 }
3335
3336 /* Return zero if success, or <0 for error */
3337 static int postcopy_do_resume(MigrationState *s)
3338 {
3339     int ret;
3340
3341     /*
3342      * Call all the resume_prepare() hooks, so that modules can be
3343      * ready for the migration resume.
3344      */
3345     ret = qemu_savevm_state_resume_prepare(s);
3346     if (ret) {
3347         error_report("%s: resume_prepare() failure detected: %d",
3348                      __func__, ret);
3349         return ret;
3350     }
3351
3352     /*
3353      * Last handshake with destination on the resume (destination will
3354      * switch to postcopy-active afterwards)
3355      */
3356     ret = postcopy_resume_handshake(s);
3357     if (ret) {
3358         error_report("%s: handshake failed: %d", __func__, ret);
3359         return ret;
3360     }
3361
3362     return 0;
3363 }
3364
3365 /*
3366  * We don't return until we are in a safe state to continue current
3367  * postcopy migration.  Returns MIG_THR_ERR_RECOVERED if recovered, or
3368  * MIG_THR_ERR_FATAL if unrecovery failure happened.
3369  */
3370 static MigThrError postcopy_pause(MigrationState *s)
3371 {
3372     assert(s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
3373
3374     while (true) {
3375         QEMUFile *file;
3376
3377         /*
3378          * Current channel is possibly broken. Release it.  Note that this is
3379          * guaranteed even without lock because to_dst_file should only be
3380          * modified by the migration thread.  That also guarantees that the
3381          * unregister of yank is safe too without the lock.  It should be safe
3382          * even to be within the qemu_file_lock, but we didn't do that to avoid
3383          * taking more mutex (yank_lock) within qemu_file_lock.  TL;DR: we make
3384          * the qemu_file_lock critical section as small as possible.
3385          */
3386         assert(s->to_dst_file);
3387         migration_ioc_unregister_yank_from_file(s->to_dst_file);
3388         qemu_mutex_lock(&s->qemu_file_lock);
3389         file = s->to_dst_file;
3390         s->to_dst_file = NULL;
3391         qemu_mutex_unlock(&s->qemu_file_lock);
3392
3393         qemu_file_shutdown(file);
3394         qemu_fclose(file);
3395
3396         migrate_set_state(&s->state, s->state,
3397                           MIGRATION_STATUS_POSTCOPY_PAUSED);
3398
3399         error_report("Detected IO failure for postcopy. "
3400                      "Migration paused.");
3401
3402         /*
3403          * We wait until things fixed up. Then someone will setup the
3404          * status back for us.
3405          */
3406         while (s->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
3407             qemu_sem_wait(&s->postcopy_pause_sem);
3408         }
3409
3410         if (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3411             /* Woken up by a recover procedure. Give it a shot */
3412
3413             /*
3414              * Firstly, let's wake up the return path now, with a new
3415              * return path channel.
3416              */
3417             qemu_sem_post(&s->postcopy_pause_rp_sem);
3418
3419             /* Do the resume logic */
3420             if (postcopy_do_resume(s) == 0) {
3421                 /* Let's continue! */
3422                 trace_postcopy_pause_continued();
3423                 return MIG_THR_ERR_RECOVERED;
3424             } else {
3425                 /*
3426                  * Something wrong happened during the recovery, let's
3427                  * pause again. Pause is always better than throwing
3428                  * data away.
3429                  */
3430                 continue;
3431             }
3432         } else {
3433             /* This is not right... Time to quit. */
3434             return MIG_THR_ERR_FATAL;
3435         }
3436     }
3437 }
3438
3439 static MigThrError migration_detect_error(MigrationState *s)
3440 {
3441     int ret;
3442     int state = s->state;
3443     Error *local_error = NULL;
3444
3445     if (state == MIGRATION_STATUS_CANCELLING ||
3446         state == MIGRATION_STATUS_CANCELLED) {
3447         /* End the migration, but don't set the state to failed */
3448         return MIG_THR_ERR_FATAL;
3449     }
3450
3451     /* Try to detect any file errors */
3452     ret = qemu_file_get_error_obj(s->to_dst_file, &local_error);
3453     if (!ret) {
3454         /* Everything is fine */
3455         assert(!local_error);
3456         return MIG_THR_ERR_NONE;
3457     }
3458
3459     if (local_error) {
3460         migrate_set_error(s, local_error);
3461         error_free(local_error);
3462     }
3463
3464     if (state == MIGRATION_STATUS_POSTCOPY_ACTIVE && ret == -EIO) {
3465         /*
3466          * For postcopy, we allow the network to be down for a
3467          * while. After that, it can be continued by a
3468          * recovery phase.
3469          */
3470         return postcopy_pause(s);
3471     } else {
3472         /*
3473          * For precopy (or postcopy with error outside IO), we fail
3474          * with no time.
3475          */
3476         migrate_set_state(&s->state, state, MIGRATION_STATUS_FAILED);
3477         trace_migration_thread_file_err();
3478
3479         /* Time to stop the migration, now. */
3480         return MIG_THR_ERR_FATAL;
3481     }
3482 }
3483
3484 /* How many bytes have we transferred since the beginning of the migration */
3485 static uint64_t migration_total_bytes(MigrationState *s)
3486 {
3487     return qemu_ftell(s->to_dst_file) + ram_counters.multifd_bytes;
3488 }
3489
3490 static void migration_calculate_complete(MigrationState *s)
3491 {
3492     uint64_t bytes = migration_total_bytes(s);
3493     int64_t end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3494     int64_t transfer_time;
3495
3496     s->total_time = end_time - s->start_time;
3497     if (!s->downtime) {
3498         /*
3499          * It's still not set, so we are precopy migration.  For
3500          * postcopy, downtime is calculated during postcopy_start().
3501          */
3502         s->downtime = end_time - s->downtime_start;
3503     }
3504
3505     transfer_time = s->total_time - s->setup_time;
3506     if (transfer_time) {
3507         s->mbps = ((double) bytes * 8.0) / transfer_time / 1000;
3508     }
3509 }
3510
3511 static void update_iteration_initial_status(MigrationState *s)
3512 {
3513     /*
3514      * Update these three fields at the same time to avoid mismatch info lead
3515      * wrong speed calculation.
3516      */
3517     s->iteration_start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3518     s->iteration_initial_bytes = migration_total_bytes(s);
3519     s->iteration_initial_pages = ram_get_total_transferred_pages();
3520 }
3521
3522 static void migration_update_counters(MigrationState *s,
3523                                       int64_t current_time)
3524 {
3525     uint64_t transferred, transferred_pages, time_spent;
3526     uint64_t current_bytes; /* bytes transferred since the beginning */
3527     double bandwidth;
3528
3529     if (current_time < s->iteration_start_time + BUFFER_DELAY) {
3530         return;
3531     }
3532
3533     current_bytes = migration_total_bytes(s);
3534     transferred = current_bytes - s->iteration_initial_bytes;
3535     time_spent = current_time - s->iteration_start_time;
3536     bandwidth = (double)transferred / time_spent;
3537     s->threshold_size = bandwidth * s->parameters.downtime_limit;
3538
3539     s->mbps = (((double) transferred * 8.0) /
3540                ((double) time_spent / 1000.0)) / 1000.0 / 1000.0;
3541
3542     transferred_pages = ram_get_total_transferred_pages() -
3543                             s->iteration_initial_pages;
3544     s->pages_per_second = (double) transferred_pages /
3545                              (((double) time_spent / 1000.0));
3546
3547     /*
3548      * if we haven't sent anything, we don't want to
3549      * recalculate. 10000 is a small enough number for our purposes
3550      */
3551     if (ram_counters.dirty_pages_rate && transferred > 10000) {
3552         s->expected_downtime = ram_counters.remaining / bandwidth;
3553     }
3554
3555     qemu_file_reset_rate_limit(s->to_dst_file);
3556
3557     update_iteration_initial_status(s);
3558
3559     trace_migrate_transferred(transferred, time_spent,
3560                               bandwidth, s->threshold_size);
3561 }
3562
3563 /* Migration thread iteration status */
3564 typedef enum {
3565     MIG_ITERATE_RESUME,         /* Resume current iteration */
3566     MIG_ITERATE_SKIP,           /* Skip current iteration */
3567     MIG_ITERATE_BREAK,          /* Break the loop */
3568 } MigIterateState;
3569
3570 /*
3571  * Return true if continue to the next iteration directly, false
3572  * otherwise.
3573  */
3574 static MigIterateState migration_iteration_run(MigrationState *s)
3575 {
3576     uint64_t pending_size, pend_pre, pend_compat, pend_post;
3577     bool in_postcopy = s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE;
3578
3579     qemu_savevm_state_pending(s->to_dst_file, s->threshold_size, &pend_pre,
3580                               &pend_compat, &pend_post);
3581     pending_size = pend_pre + pend_compat + pend_post;
3582
3583     trace_migrate_pending(pending_size, s->threshold_size,
3584                           pend_pre, pend_compat, pend_post);
3585
3586     if (pending_size && pending_size >= s->threshold_size) {
3587         /* Still a significant amount to transfer */
3588         if (!in_postcopy && pend_pre <= s->threshold_size &&
3589             qatomic_read(&s->start_postcopy)) {
3590             if (postcopy_start(s)) {
3591                 error_report("%s: postcopy failed to start", __func__);
3592             }
3593             return MIG_ITERATE_SKIP;
3594         }
3595         /* Just another iteration step */
3596         qemu_savevm_state_iterate(s->to_dst_file, in_postcopy);
3597     } else {
3598         trace_migration_thread_low_pending(pending_size);
3599         migration_completion(s);
3600         return MIG_ITERATE_BREAK;
3601     }
3602
3603     return MIG_ITERATE_RESUME;
3604 }
3605
3606 static void migration_iteration_finish(MigrationState *s)
3607 {
3608     /* If we enabled cpu throttling for auto-converge, turn it off. */
3609     cpu_throttle_stop();
3610
3611     qemu_mutex_lock_iothread();
3612     switch (s->state) {
3613     case MIGRATION_STATUS_COMPLETED:
3614         migration_calculate_complete(s);
3615         runstate_set(RUN_STATE_POSTMIGRATE);
3616         break;
3617     case MIGRATION_STATUS_COLO:
3618         if (!migrate_colo_enabled()) {
3619             error_report("%s: critical error: calling COLO code without "
3620                          "COLO enabled", __func__);
3621         }
3622         migrate_start_colo_process(s);
3623         /*
3624          * Fixme: we will run VM in COLO no matter its old running state.
3625          * After exited COLO, we will keep running.
3626          */
3627          /* Fallthrough */
3628     case MIGRATION_STATUS_ACTIVE:
3629         /*
3630          * We should really assert here, but since it's during
3631          * migration, let's try to reduce the usage of assertions.
3632          */
3633         s->vm_was_running = true;
3634         /* Fallthrough */
3635     case MIGRATION_STATUS_FAILED:
3636     case MIGRATION_STATUS_CANCELLED:
3637     case MIGRATION_STATUS_CANCELLING:
3638         if (s->vm_was_running) {
3639             if (!runstate_check(RUN_STATE_SHUTDOWN)) {
3640                 vm_start();
3641             }
3642         } else {
3643             if (runstate_check(RUN_STATE_FINISH_MIGRATE)) {
3644                 runstate_set(RUN_STATE_POSTMIGRATE);
3645             }
3646         }
3647         break;
3648
3649     default:
3650         /* Should not reach here, but if so, forgive the VM. */
3651         error_report("%s: Unknown ending state %d", __func__, s->state);
3652         break;
3653     }
3654     migrate_fd_cleanup_schedule(s);
3655     qemu_mutex_unlock_iothread();
3656 }
3657
3658 static void bg_migration_iteration_finish(MigrationState *s)
3659 {
3660     qemu_mutex_lock_iothread();
3661     switch (s->state) {
3662     case MIGRATION_STATUS_COMPLETED:
3663         migration_calculate_complete(s);
3664         break;
3665
3666     case MIGRATION_STATUS_ACTIVE:
3667     case MIGRATION_STATUS_FAILED:
3668     case MIGRATION_STATUS_CANCELLED:
3669     case MIGRATION_STATUS_CANCELLING:
3670         break;
3671
3672     default:
3673         /* Should not reach here, but if so, forgive the VM. */
3674         error_report("%s: Unknown ending state %d", __func__, s->state);
3675         break;
3676     }
3677
3678     migrate_fd_cleanup_schedule(s);
3679     qemu_mutex_unlock_iothread();
3680 }
3681
3682 /*
3683  * Return true if continue to the next iteration directly, false
3684  * otherwise.
3685  */
3686 static MigIterateState bg_migration_iteration_run(MigrationState *s)
3687 {
3688     int res;
3689
3690     res = qemu_savevm_state_iterate(s->to_dst_file, false);
3691     if (res > 0) {
3692         bg_migration_completion(s);
3693         return MIG_ITERATE_BREAK;
3694     }
3695
3696     return MIG_ITERATE_RESUME;
3697 }
3698
3699 void migration_make_urgent_request(void)
3700 {
3701     qemu_sem_post(&migrate_get_current()->rate_limit_sem);
3702 }
3703
3704 void migration_consume_urgent_request(void)
3705 {
3706     qemu_sem_wait(&migrate_get_current()->rate_limit_sem);
3707 }
3708
3709 /* Returns true if the rate limiting was broken by an urgent request */
3710 bool migration_rate_limit(void)
3711 {
3712     int64_t now = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3713     MigrationState *s = migrate_get_current();
3714
3715     bool urgent = false;
3716     migration_update_counters(s, now);
3717     if (qemu_file_rate_limit(s->to_dst_file)) {
3718
3719         if (qemu_file_get_error(s->to_dst_file)) {
3720             return false;
3721         }
3722         /*
3723          * Wait for a delay to do rate limiting OR
3724          * something urgent to post the semaphore.
3725          */
3726         int ms = s->iteration_start_time + BUFFER_DELAY - now;
3727         trace_migration_rate_limit_pre(ms);
3728         if (qemu_sem_timedwait(&s->rate_limit_sem, ms) == 0) {
3729             /*
3730              * We were woken by one or more urgent things but
3731              * the timedwait will have consumed one of them.
3732              * The service routine for the urgent wake will dec
3733              * the semaphore itself for each item it consumes,
3734              * so add this one we just eat back.
3735              */
3736             qemu_sem_post(&s->rate_limit_sem);
3737             urgent = true;
3738         }
3739         trace_migration_rate_limit_post(urgent);
3740     }
3741     return urgent;
3742 }
3743
3744 /*
3745  * if failover devices are present, wait they are completely
3746  * unplugged
3747  */
3748
3749 static void qemu_savevm_wait_unplug(MigrationState *s, int old_state,
3750                                     int new_state)
3751 {
3752     if (qemu_savevm_state_guest_unplug_pending()) {
3753         migrate_set_state(&s->state, old_state, MIGRATION_STATUS_WAIT_UNPLUG);
3754
3755         while (s->state == MIGRATION_STATUS_WAIT_UNPLUG &&
3756                qemu_savevm_state_guest_unplug_pending()) {
3757             qemu_sem_timedwait(&s->wait_unplug_sem, 250);
3758         }
3759         if (s->state != MIGRATION_STATUS_WAIT_UNPLUG) {
3760             int timeout = 120; /* 30 seconds */
3761             /*
3762              * migration has been canceled
3763              * but as we have started an unplug we must wait the end
3764              * to be able to plug back the card
3765              */
3766             while (timeout-- && qemu_savevm_state_guest_unplug_pending()) {
3767                 qemu_sem_timedwait(&s->wait_unplug_sem, 250);
3768             }
3769             if (qemu_savevm_state_guest_unplug_pending()) {
3770                 warn_report("migration: partially unplugged device on "
3771                             "failure");
3772             }
3773         }
3774
3775         migrate_set_state(&s->state, MIGRATION_STATUS_WAIT_UNPLUG, new_state);
3776     } else {
3777         migrate_set_state(&s->state, old_state, new_state);
3778     }
3779 }
3780
3781 /*
3782  * Master migration thread on the source VM.
3783  * It drives the migration and pumps the data down the outgoing channel.
3784  */
3785 static void *migration_thread(void *opaque)
3786 {
3787     MigrationState *s = opaque;
3788     int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
3789     MigThrError thr_error;
3790     bool urgent = false;
3791
3792     rcu_register_thread();
3793
3794     object_ref(OBJECT(s));
3795     update_iteration_initial_status(s);
3796
3797     qemu_savevm_state_header(s->to_dst_file);
3798
3799     /*
3800      * If we opened the return path, we need to make sure dst has it
3801      * opened as well.
3802      */
3803     if (s->rp_state.rp_thread_created) {
3804         /* Now tell the dest that it should open its end so it can reply */
3805         qemu_savevm_send_open_return_path(s->to_dst_file);
3806
3807         /* And do a ping that will make stuff easier to debug */
3808         qemu_savevm_send_ping(s->to_dst_file, 1);
3809     }
3810
3811     if (migrate_postcopy()) {
3812         /*
3813          * Tell the destination that we *might* want to do postcopy later;
3814          * if the other end can't do postcopy it should fail now, nice and
3815          * early.
3816          */
3817         qemu_savevm_send_postcopy_advise(s->to_dst_file);
3818     }
3819
3820     if (migrate_colo_enabled()) {
3821         /* Notify migration destination that we enable COLO */
3822         qemu_savevm_send_colo_enable(s->to_dst_file);
3823     }
3824
3825     qemu_savevm_state_setup(s->to_dst_file);
3826
3827     qemu_savevm_wait_unplug(s, MIGRATION_STATUS_SETUP,
3828                                MIGRATION_STATUS_ACTIVE);
3829
3830     s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
3831
3832     trace_migration_thread_setup_complete();
3833
3834     while (migration_is_active(s)) {
3835         if (urgent || !qemu_file_rate_limit(s->to_dst_file)) {
3836             MigIterateState iter_state = migration_iteration_run(s);
3837             if (iter_state == MIG_ITERATE_SKIP) {
3838                 continue;
3839             } else if (iter_state == MIG_ITERATE_BREAK) {
3840                 break;
3841             }
3842         }
3843
3844         /*
3845          * Try to detect any kind of failures, and see whether we
3846          * should stop the migration now.
3847          */
3848         thr_error = migration_detect_error(s);
3849         if (thr_error == MIG_THR_ERR_FATAL) {
3850             /* Stop migration */
3851             break;
3852         } else if (thr_error == MIG_THR_ERR_RECOVERED) {
3853             /*
3854              * Just recovered from a e.g. network failure, reset all
3855              * the local variables. This is important to avoid
3856              * breaking transferred_bytes and bandwidth calculation
3857              */
3858             update_iteration_initial_status(s);
3859         }
3860
3861         urgent = migration_rate_limit();
3862     }
3863
3864     trace_migration_thread_after_loop();
3865     migration_iteration_finish(s);
3866     object_unref(OBJECT(s));
3867     rcu_unregister_thread();
3868     return NULL;
3869 }
3870
3871 static void bg_migration_vm_start_bh(void *opaque)
3872 {
3873     MigrationState *s = opaque;
3874
3875     qemu_bh_delete(s->vm_start_bh);
3876     s->vm_start_bh = NULL;
3877
3878     vm_start();
3879     s->downtime = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - s->downtime_start;
3880 }
3881
3882 /**
3883  * Background snapshot thread, based on live migration code.
3884  * This is an alternative implementation of live migration mechanism
3885  * introduced specifically to support background snapshots.
3886  *
3887  * It takes advantage of userfault_fd write protection mechanism introduced
3888  * in v5.7 kernel. Compared to existing dirty page logging migration much
3889  * lesser stream traffic is produced resulting in smaller snapshot images,
3890  * simply cause of no page duplicates can get into the stream.
3891  *
3892  * Another key point is that generated vmstate stream reflects machine state
3893  * 'frozen' at the beginning of snapshot creation compared to dirty page logging
3894  * mechanism, which effectively results in that saved snapshot is the state of VM
3895  * at the end of the process.
3896  */
3897 static void *bg_migration_thread(void *opaque)
3898 {
3899     MigrationState *s = opaque;
3900     int64_t setup_start;
3901     MigThrError thr_error;
3902     QEMUFile *fb;
3903     bool early_fail = true;
3904
3905     rcu_register_thread();
3906     object_ref(OBJECT(s));
3907
3908     qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX);
3909
3910     setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
3911     /*
3912      * We want to save vmstate for the moment when migration has been
3913      * initiated but also we want to save RAM content while VM is running.
3914      * The RAM content should appear first in the vmstate. So, we first
3915      * stash the non-RAM part of the vmstate to the temporary buffer,
3916      * then write RAM part of the vmstate to the migration stream
3917      * with vCPUs running and, finally, write stashed non-RAM part of
3918      * the vmstate from the buffer to the migration stream.
3919      */
3920     s->bioc = qio_channel_buffer_new(512 * 1024);
3921     qio_channel_set_name(QIO_CHANNEL(s->bioc), "vmstate-buffer");
3922     fb = qemu_fopen_channel_output(QIO_CHANNEL(s->bioc));
3923     object_unref(OBJECT(s->bioc));
3924
3925     update_iteration_initial_status(s);
3926
3927     /*
3928      * Prepare for tracking memory writes with UFFD-WP - populate
3929      * RAM pages before protecting.
3930      */
3931 #ifdef __linux__
3932     ram_write_tracking_prepare();
3933 #endif
3934
3935     qemu_savevm_state_header(s->to_dst_file);
3936     qemu_savevm_state_setup(s->to_dst_file);
3937
3938     qemu_savevm_wait_unplug(s, MIGRATION_STATUS_SETUP,
3939                                MIGRATION_STATUS_ACTIVE);
3940
3941     s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
3942
3943     trace_migration_thread_setup_complete();
3944     s->downtime_start = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3945
3946     qemu_mutex_lock_iothread();
3947
3948     /*
3949      * If VM is currently in suspended state, then, to make a valid runstate
3950      * transition in vm_stop_force_state() we need to wakeup it up.
3951      */
3952     qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
3953     s->vm_was_running = runstate_is_running();
3954
3955     if (global_state_store()) {
3956         goto fail;
3957     }
3958     /* Forcibly stop VM before saving state of vCPUs and devices */
3959     if (vm_stop_force_state(RUN_STATE_PAUSED)) {
3960         goto fail;
3961     }
3962     /*
3963      * Put vCPUs in sync with shadow context structures, then
3964      * save their state to channel-buffer along with devices.
3965      */
3966     cpu_synchronize_all_states();
3967     if (qemu_savevm_state_complete_precopy_non_iterable(fb, false, false)) {
3968         goto fail;
3969     }
3970     /*
3971      * Since we are going to get non-iterable state data directly
3972      * from s->bioc->data, explicit flush is needed here.
3973      */
3974     qemu_fflush(fb);
3975
3976     /* Now initialize UFFD context and start tracking RAM writes */
3977     if (ram_write_tracking_start()) {
3978         goto fail;
3979     }
3980     early_fail = false;
3981
3982     /*
3983      * Start VM from BH handler to avoid write-fault lock here.
3984      * UFFD-WP protection for the whole RAM is already enabled so
3985      * calling VM state change notifiers from vm_start() would initiate
3986      * writes to virtio VQs memory which is in write-protected region.
3987      */
3988     s->vm_start_bh = qemu_bh_new(bg_migration_vm_start_bh, s);
3989     qemu_bh_schedule(s->vm_start_bh);
3990
3991     qemu_mutex_unlock_iothread();
3992
3993     while (migration_is_active(s)) {
3994         MigIterateState iter_state = bg_migration_iteration_run(s);
3995         if (iter_state == MIG_ITERATE_SKIP) {
3996             continue;
3997         } else if (iter_state == MIG_ITERATE_BREAK) {
3998             break;
3999         }
4000
4001         /*
4002          * Try to detect any kind of failures, and see whether we
4003          * should stop the migration now.
4004          */
4005         thr_error = migration_detect_error(s);
4006         if (thr_error == MIG_THR_ERR_FATAL) {
4007             /* Stop migration */
4008             break;
4009         }
4010
4011         migration_update_counters(s, qemu_clock_get_ms(QEMU_CLOCK_REALTIME));
4012     }
4013
4014     trace_migration_thread_after_loop();
4015
4016 fail:
4017     if (early_fail) {
4018         migrate_set_state(&s->state, MIGRATION_STATUS_ACTIVE,
4019                 MIGRATION_STATUS_FAILED);
4020         qemu_mutex_unlock_iothread();
4021     }
4022
4023     bg_migration_iteration_finish(s);
4024
4025     qemu_fclose(fb);
4026     object_unref(OBJECT(s));
4027     rcu_unregister_thread();
4028
4029     return NULL;
4030 }
4031
4032 void migrate_fd_connect(MigrationState *s, Error *error_in)
4033 {
4034     Error *local_err = NULL;
4035     int64_t rate_limit;
4036     bool resume = s->state == MIGRATION_STATUS_POSTCOPY_PAUSED;
4037
4038     /*
4039      * If there's a previous error, free it and prepare for another one.
4040      * Meanwhile if migration completes successfully, there won't have an error
4041      * dumped when calling migrate_fd_cleanup().
4042      */
4043     migrate_error_free(s);
4044
4045     s->expected_downtime = s->parameters.downtime_limit;
4046     if (resume) {
4047         assert(s->cleanup_bh);
4048     } else {
4049         assert(!s->cleanup_bh);
4050         s->cleanup_bh = qemu_bh_new(migrate_fd_cleanup_bh, s);
4051     }
4052     if (error_in) {
4053         migrate_fd_error(s, error_in);
4054         if (resume) {
4055             /*
4056              * Don't do cleanup for resume if channel is invalid, but only dump
4057              * the error.  We wait for another channel connect from the user.
4058              * The error_report still gives HMP user a hint on what failed.
4059              * It's normally done in migrate_fd_cleanup(), but call it here
4060              * explicitly.
4061              */
4062             error_report_err(error_copy(s->error));
4063         } else {
4064             migrate_fd_cleanup(s);
4065         }
4066         return;
4067     }
4068
4069     if (resume) {
4070         /* This is a resumed migration */
4071         rate_limit = s->parameters.max_postcopy_bandwidth /
4072             XFER_LIMIT_RATIO;
4073     } else {
4074         /* This is a fresh new migration */
4075         rate_limit = s->parameters.max_bandwidth / XFER_LIMIT_RATIO;
4076
4077         /* Notify before starting migration thread */
4078         notifier_list_notify(&migration_state_notifiers, s);
4079     }
4080
4081     qemu_file_set_rate_limit(s->to_dst_file, rate_limit);
4082     qemu_file_set_blocking(s->to_dst_file, true);
4083
4084     /*
4085      * Open the return path. For postcopy, it is used exclusively. For
4086      * precopy, only if user specified "return-path" capability would
4087      * QEMU uses the return path.
4088      */
4089     if (migrate_postcopy_ram() || migrate_use_return_path()) {
4090         if (open_return_path_on_source(s, !resume)) {
4091             error_report("Unable to open return-path for postcopy");
4092             migrate_set_state(&s->state, s->state, MIGRATION_STATUS_FAILED);
4093             migrate_fd_cleanup(s);
4094             return;
4095         }
4096     }
4097
4098     if (resume) {
4099         /* Wakeup the main migration thread to do the recovery */
4100         migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
4101                           MIGRATION_STATUS_POSTCOPY_RECOVER);
4102         qemu_sem_post(&s->postcopy_pause_sem);
4103         return;
4104     }
4105
4106     if (multifd_save_setup(&local_err) != 0) {
4107         error_report_err(local_err);
4108         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
4109                           MIGRATION_STATUS_FAILED);
4110         migrate_fd_cleanup(s);
4111         return;
4112     }
4113
4114     if (migrate_background_snapshot()) {
4115         qemu_thread_create(&s->thread, "bg_snapshot",
4116                 bg_migration_thread, s, QEMU_THREAD_JOINABLE);
4117     } else {
4118         qemu_thread_create(&s->thread, "live_migration",
4119                 migration_thread, s, QEMU_THREAD_JOINABLE);
4120     }
4121     s->migration_thread_running = true;
4122 }
4123
4124 void migration_global_dump(Monitor *mon)
4125 {
4126     MigrationState *ms = migrate_get_current();
4127
4128     monitor_printf(mon, "globals:\n");
4129     monitor_printf(mon, "store-global-state: %s\n",
4130                    ms->store_global_state ? "on" : "off");
4131     monitor_printf(mon, "only-migratable: %s\n",
4132                    only_migratable ? "on" : "off");
4133     monitor_printf(mon, "send-configuration: %s\n",
4134                    ms->send_configuration ? "on" : "off");
4135     monitor_printf(mon, "send-section-footer: %s\n",
4136                    ms->send_section_footer ? "on" : "off");
4137     monitor_printf(mon, "decompress-error-check: %s\n",
4138                    ms->decompress_error_check ? "on" : "off");
4139     monitor_printf(mon, "clear-bitmap-shift: %u\n",
4140                    ms->clear_bitmap_shift);
4141 }
4142
4143 #define DEFINE_PROP_MIG_CAP(name, x)             \
4144     DEFINE_PROP_BOOL(name, MigrationState, enabled_capabilities[x], false)
4145
4146 static Property migration_properties[] = {
4147     DEFINE_PROP_BOOL("store-global-state", MigrationState,
4148                      store_global_state, true),
4149     DEFINE_PROP_BOOL("send-configuration", MigrationState,
4150                      send_configuration, true),
4151     DEFINE_PROP_BOOL("send-section-footer", MigrationState,
4152                      send_section_footer, true),
4153     DEFINE_PROP_BOOL("decompress-error-check", MigrationState,
4154                       decompress_error_check, true),
4155     DEFINE_PROP_UINT8("x-clear-bitmap-shift", MigrationState,
4156                       clear_bitmap_shift, CLEAR_BITMAP_SHIFT_DEFAULT),
4157
4158     /* Migration parameters */
4159     DEFINE_PROP_UINT8("x-compress-level", MigrationState,
4160                       parameters.compress_level,
4161                       DEFAULT_MIGRATE_COMPRESS_LEVEL),
4162     DEFINE_PROP_UINT8("x-compress-threads", MigrationState,
4163                       parameters.compress_threads,
4164                       DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT),
4165     DEFINE_PROP_BOOL("x-compress-wait-thread", MigrationState,
4166                       parameters.compress_wait_thread, true),
4167     DEFINE_PROP_UINT8("x-decompress-threads", MigrationState,
4168                       parameters.decompress_threads,
4169                       DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT),
4170     DEFINE_PROP_UINT8("x-throttle-trigger-threshold", MigrationState,
4171                       parameters.throttle_trigger_threshold,
4172                       DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD),
4173     DEFINE_PROP_UINT8("x-cpu-throttle-initial", MigrationState,
4174                       parameters.cpu_throttle_initial,
4175                       DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL),
4176     DEFINE_PROP_UINT8("x-cpu-throttle-increment", MigrationState,
4177                       parameters.cpu_throttle_increment,
4178                       DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT),
4179     DEFINE_PROP_BOOL("x-cpu-throttle-tailslow", MigrationState,
4180                       parameters.cpu_throttle_tailslow, false),
4181     DEFINE_PROP_SIZE("x-max-bandwidth", MigrationState,
4182                       parameters.max_bandwidth, MAX_THROTTLE),
4183     DEFINE_PROP_UINT64("x-downtime-limit", MigrationState,
4184                       parameters.downtime_limit,
4185                       DEFAULT_MIGRATE_SET_DOWNTIME),
4186     DEFINE_PROP_UINT32("x-checkpoint-delay", MigrationState,
4187                       parameters.x_checkpoint_delay,
4188                       DEFAULT_MIGRATE_X_CHECKPOINT_DELAY),
4189     DEFINE_PROP_UINT8("multifd-channels", MigrationState,
4190                       parameters.multifd_channels,
4191                       DEFAULT_MIGRATE_MULTIFD_CHANNELS),
4192     DEFINE_PROP_MULTIFD_COMPRESSION("multifd-compression", MigrationState,
4193                       parameters.multifd_compression,
4194                       DEFAULT_MIGRATE_MULTIFD_COMPRESSION),
4195     DEFINE_PROP_UINT8("multifd-zlib-level", MigrationState,
4196                       parameters.multifd_zlib_level,
4197                       DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL),
4198     DEFINE_PROP_UINT8("multifd-zstd-level", MigrationState,
4199                       parameters.multifd_zstd_level,
4200                       DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL),
4201     DEFINE_PROP_SIZE("xbzrle-cache-size", MigrationState,
4202                       parameters.xbzrle_cache_size,
4203                       DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE),
4204     DEFINE_PROP_SIZE("max-postcopy-bandwidth", MigrationState,
4205                       parameters.max_postcopy_bandwidth,
4206                       DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH),
4207     DEFINE_PROP_UINT8("max-cpu-throttle", MigrationState,
4208                       parameters.max_cpu_throttle,
4209                       DEFAULT_MIGRATE_MAX_CPU_THROTTLE),
4210     DEFINE_PROP_SIZE("announce-initial", MigrationState,
4211                       parameters.announce_initial,
4212                       DEFAULT_MIGRATE_ANNOUNCE_INITIAL),
4213     DEFINE_PROP_SIZE("announce-max", MigrationState,
4214                       parameters.announce_max,
4215                       DEFAULT_MIGRATE_ANNOUNCE_MAX),
4216     DEFINE_PROP_SIZE("announce-rounds", MigrationState,
4217                       parameters.announce_rounds,
4218                       DEFAULT_MIGRATE_ANNOUNCE_ROUNDS),
4219     DEFINE_PROP_SIZE("announce-step", MigrationState,
4220                       parameters.announce_step,
4221                       DEFAULT_MIGRATE_ANNOUNCE_STEP),
4222
4223     /* Migration capabilities */
4224     DEFINE_PROP_MIG_CAP("x-xbzrle", MIGRATION_CAPABILITY_XBZRLE),
4225     DEFINE_PROP_MIG_CAP("x-rdma-pin-all", MIGRATION_CAPABILITY_RDMA_PIN_ALL),
4226     DEFINE_PROP_MIG_CAP("x-auto-converge", MIGRATION_CAPABILITY_AUTO_CONVERGE),
4227     DEFINE_PROP_MIG_CAP("x-zero-blocks", MIGRATION_CAPABILITY_ZERO_BLOCKS),
4228     DEFINE_PROP_MIG_CAP("x-compress", MIGRATION_CAPABILITY_COMPRESS),
4229     DEFINE_PROP_MIG_CAP("x-events", MIGRATION_CAPABILITY_EVENTS),
4230     DEFINE_PROP_MIG_CAP("x-postcopy-ram", MIGRATION_CAPABILITY_POSTCOPY_RAM),
4231     DEFINE_PROP_MIG_CAP("x-colo", MIGRATION_CAPABILITY_X_COLO),
4232     DEFINE_PROP_MIG_CAP("x-release-ram", MIGRATION_CAPABILITY_RELEASE_RAM),
4233     DEFINE_PROP_MIG_CAP("x-block", MIGRATION_CAPABILITY_BLOCK),
4234     DEFINE_PROP_MIG_CAP("x-return-path", MIGRATION_CAPABILITY_RETURN_PATH),
4235     DEFINE_PROP_MIG_CAP("x-multifd", MIGRATION_CAPABILITY_MULTIFD),
4236     DEFINE_PROP_MIG_CAP("x-background-snapshot",
4237             MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT),
4238
4239     DEFINE_PROP_END_OF_LIST(),
4240 };
4241
4242 static void migration_class_init(ObjectClass *klass, void *data)
4243 {
4244     DeviceClass *dc = DEVICE_CLASS(klass);
4245
4246     dc->user_creatable = false;
4247     device_class_set_props(dc, migration_properties);
4248 }
4249
4250 static void migration_instance_finalize(Object *obj)
4251 {
4252     MigrationState *ms = MIGRATION_OBJ(obj);
4253     MigrationParameters *params = &ms->parameters;
4254
4255     qemu_mutex_destroy(&ms->error_mutex);
4256     qemu_mutex_destroy(&ms->qemu_file_lock);
4257     g_free(params->tls_hostname);
4258     g_free(params->tls_creds);
4259     qemu_sem_destroy(&ms->wait_unplug_sem);
4260     qemu_sem_destroy(&ms->rate_limit_sem);
4261     qemu_sem_destroy(&ms->pause_sem);
4262     qemu_sem_destroy(&ms->postcopy_pause_sem);
4263     qemu_sem_destroy(&ms->postcopy_pause_rp_sem);
4264     qemu_sem_destroy(&ms->rp_state.rp_sem);
4265     error_free(ms->error);
4266 }
4267
4268 static void migration_instance_init(Object *obj)
4269 {
4270     MigrationState *ms = MIGRATION_OBJ(obj);
4271     MigrationParameters *params = &ms->parameters;
4272
4273     ms->state = MIGRATION_STATUS_NONE;
4274     ms->mbps = -1;
4275     ms->pages_per_second = -1;
4276     qemu_sem_init(&ms->pause_sem, 0);
4277     qemu_mutex_init(&ms->error_mutex);
4278
4279     params->tls_hostname = g_strdup("");
4280     params->tls_creds = g_strdup("");
4281
4282     /* Set has_* up only for parameter checks */
4283     params->has_compress_level = true;
4284     params->has_compress_threads = true;
4285     params->has_decompress_threads = true;
4286     params->has_throttle_trigger_threshold = true;
4287     params->has_cpu_throttle_initial = true;
4288     params->has_cpu_throttle_increment = true;
4289     params->has_cpu_throttle_tailslow = true;
4290     params->has_max_bandwidth = true;
4291     params->has_downtime_limit = true;
4292     params->has_x_checkpoint_delay = true;
4293     params->has_block_incremental = true;
4294     params->has_multifd_channels = true;
4295     params->has_multifd_compression = true;
4296     params->has_multifd_zlib_level = true;
4297     params->has_multifd_zstd_level = true;
4298     params->has_xbzrle_cache_size = true;
4299     params->has_max_postcopy_bandwidth = true;
4300     params->has_max_cpu_throttle = true;
4301     params->has_announce_initial = true;
4302     params->has_announce_max = true;
4303     params->has_announce_rounds = true;
4304     params->has_announce_step = true;
4305
4306     qemu_sem_init(&ms->postcopy_pause_sem, 0);
4307     qemu_sem_init(&ms->postcopy_pause_rp_sem, 0);
4308     qemu_sem_init(&ms->rp_state.rp_sem, 0);
4309     qemu_sem_init(&ms->rate_limit_sem, 0);
4310     qemu_sem_init(&ms->wait_unplug_sem, 0);
4311     qemu_mutex_init(&ms->qemu_file_lock);
4312 }
4313
4314 /*
4315  * Return true if check pass, false otherwise. Error will be put
4316  * inside errp if provided.
4317  */
4318 static bool migration_object_check(MigrationState *ms, Error **errp)
4319 {
4320     MigrationCapabilityStatusList *head = NULL;
4321     /* Assuming all off */
4322     bool cap_list[MIGRATION_CAPABILITY__MAX] = { 0 }, ret;
4323     int i;
4324
4325     if (!migrate_params_check(&ms->parameters, errp)) {
4326         return false;
4327     }
4328
4329     for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
4330         if (ms->enabled_capabilities[i]) {
4331             QAPI_LIST_PREPEND(head, migrate_cap_add(i, true));
4332         }
4333     }
4334
4335     ret = migrate_caps_check(cap_list, head, errp);
4336
4337     /* It works with head == NULL */
4338     qapi_free_MigrationCapabilityStatusList(head);
4339
4340     return ret;
4341 }
4342
4343 static const TypeInfo migration_type = {
4344     .name = TYPE_MIGRATION,
4345     /*
4346      * NOTE: TYPE_MIGRATION is not really a device, as the object is
4347      * not created using qdev_new(), it is not attached to the qdev
4348      * device tree, and it is never realized.
4349      *
4350      * TODO: Make this TYPE_OBJECT once QOM provides something like
4351      * TYPE_DEVICE's "-global" properties.
4352      */
4353     .parent = TYPE_DEVICE,
4354     .class_init = migration_class_init,
4355     .class_size = sizeof(MigrationClass),
4356     .instance_size = sizeof(MigrationState),
4357     .instance_init = migration_instance_init,
4358     .instance_finalize = migration_instance_finalize,
4359 };
4360
4361 static void register_migration_types(void)
4362 {
4363     type_register_static(&migration_type);
4364 }
4365
4366 type_init(register_migration_types);