arch_init.c

   1 /*
   2  * QEMU System Emulator
   3  *
   4  * Copyright (c) 2003-2008 Fabrice Bellard
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to deal
   8  * in the Software without restriction, including without limitation the rights
   9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10  * copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22  * THE SOFTWARE.
  23  */
  24 #include <stdint.h>
  25 #include <stdarg.h>
  26 #include <stdlib.h>
  27 #include <zlib.h>
  28 #ifndef _WIN32
  29 #include <sys/types.h>
  30 #include <sys/mman.h>
  31 #endif
  32 #include "config.h"
  33 #include "monitor/monitor.h"
  34 #include "sysemu/sysemu.h"
  35 #include "qemu/bitops.h"
  36 #include "qemu/bitmap.h"
  37 #include "sysemu/arch_init.h"
  38 #include "audio/audio.h"
  39 #include "hw/i386/pc.h"
  40 #include "hw/pci/pci.h"
  41 #include "hw/audio/audio.h"
  42 #include "sysemu/kvm.h"
  43 #include "migration/migration.h"
  44 #include "hw/i386/smbios.h"
  45 #include "exec/address-spaces.h"
  46 #include "hw/audio/pcspk.h"
  47 #include "migration/page_cache.h"
  48 #include "qemu/config-file.h"
  49 #include "qemu/error-report.h"
  50 #include "qmp-commands.h"
  51 #include "trace.h"
  52 #include "exec/cpu-all.h"
  53 #include "exec/ram_addr.h"
  54 #include "hw/acpi/acpi.h"
  55 #include "qemu/host-utils.h"
  56 #include "qemu/rcu_queue.h"
  57
  58 #ifdef DEBUG_ARCH_INIT
  59 #define DPRINTF(fmt, ...) \
  60     do { fprintf(stdout, "arch_init: " fmt, ## __VA_ARGS__); } while (0)
  61 #else
  62 #define DPRINTF(fmt, ...) \
  63     do { } while (0)
  64 #endif
  65
  66 #ifdef TARGET_SPARC
  67 int graphic_width = 1024;
  68 int graphic_height = 768;
  69 int graphic_depth = 8;
  70 #else
  71 int graphic_width = 800;
  72 int graphic_height = 600;
  73 int graphic_depth = 32;
  74 #endif
  75
  76
  77 #if defined(TARGET_ALPHA)
  78 #define QEMU_ARCH QEMU_ARCH_ALPHA
  79 #elif defined(TARGET_ARM)
  80 #define QEMU_ARCH QEMU_ARCH_ARM
  81 #elif defined(TARGET_CRIS)
  82 #define QEMU_ARCH QEMU_ARCH_CRIS
  83 #elif defined(TARGET_I386)
  84 #define QEMU_ARCH QEMU_ARCH_I386
  85 #elif defined(TARGET_M68K)
  86 #define QEMU_ARCH QEMU_ARCH_M68K
  87 #elif defined(TARGET_LM32)
  88 #define QEMU_ARCH QEMU_ARCH_LM32
  89 #elif defined(TARGET_MICROBLAZE)
  90 #define QEMU_ARCH QEMU_ARCH_MICROBLAZE
  91 #elif defined(TARGET_MIPS)
  92 #define QEMU_ARCH QEMU_ARCH_MIPS
  93 #elif defined(TARGET_MOXIE)
  94 #define QEMU_ARCH QEMU_ARCH_MOXIE
  95 #elif defined(TARGET_OPENRISC)
  96 #define QEMU_ARCH QEMU_ARCH_OPENRISC
  97 #elif defined(TARGET_PPC)
  98 #define QEMU_ARCH QEMU_ARCH_PPC
  99 #elif defined(TARGET_S390X)
 100 #define QEMU_ARCH QEMU_ARCH_S390X
 101 #elif defined(TARGET_SH4)
 102 #define QEMU_ARCH QEMU_ARCH_SH4
 103 #elif defined(TARGET_SPARC)
 104 #define QEMU_ARCH QEMU_ARCH_SPARC
 105 #elif defined(TARGET_XTENSA)
 106 #define QEMU_ARCH QEMU_ARCH_XTENSA
 107 #elif defined(TARGET_UNICORE32)
 108 #define QEMU_ARCH QEMU_ARCH_UNICORE32
 109 #elif defined(TARGET_TRICORE)
 110 #define QEMU_ARCH QEMU_ARCH_TRICORE
 111 #endif
 112
 113 const uint32_t arch_type = QEMU_ARCH;
 114 static bool mig_throttle_on;
 115 static int dirty_rate_high_cnt;
 116 static void check_guest_throttling(void);
 117
 118 static uint64_t bitmap_sync_count;
 119
 120 /***********************************************************/
 121 /* ram save/restore */
 122
 123 #define RAM_SAVE_FLAG_FULL     0x01 /* Obsolete, not used anymore */
 124 #define RAM_SAVE_FLAG_COMPRESS 0x02
 125 #define RAM_SAVE_FLAG_MEM_SIZE 0x04
 126 #define RAM_SAVE_FLAG_PAGE     0x08
 127 #define RAM_SAVE_FLAG_EOS      0x10
 128 #define RAM_SAVE_FLAG_CONTINUE 0x20
 129 #define RAM_SAVE_FLAG_XBZRLE   0x40
 130 /* 0x80 is reserved in migration.h start with 0x100 next */
 131 #define RAM_SAVE_FLAG_COMPRESS_PAGE    0x100
 132
 133 static struct defconfig_file {
 134     const char *filename;
 135     /* Indicates it is an user config file (disabled by -no-user-config) */
 136     bool userconfig;
 137 } default_config_files[] = {
 138     { CONFIG_QEMU_CONFDIR "/qemu.conf",                   true },
 139     { NULL }, /* end of list */
 140 };
 141
 142 static const uint8_t ZERO_TARGET_PAGE[TARGET_PAGE_SIZE];
 143
 144 int qemu_read_default_config_files(bool userconfig)
 145 {
 146     int ret;
 147     struct defconfig_file *f;
 148
 149     for (f = default_config_files; f->filename; f++) {
 150         if (!userconfig && f->userconfig) {
 151             continue;
 152         }
 153         ret = qemu_read_config_file(f->filename);
 154         if (ret < 0 && ret != -ENOENT) {
 155             return ret;
 156         }
 157     }
 158
 159     return 0;
 160 }
 161
 162 static inline bool is_zero_range(uint8_t *p, uint64_t size)
 163 {
 164     return buffer_find_nonzero_offset(p, size) == size;
 165 }
 166
 167 /* struct contains XBZRLE cache and a static page
 168    used by the compression */
 169 static struct {
 170     /* buffer used for XBZRLE encoding */
 171     uint8_t *encoded_buf;
 172     /* buffer for storing page content */
 173     uint8_t *current_buf;
 174     /* Cache for XBZRLE, Protected by lock. */
 175     PageCache *cache;
 176     QemuMutex lock;
 177 } XBZRLE;
 178
 179 /* buffer used for XBZRLE decoding */
 180 static uint8_t *xbzrle_decoded_buf;
 181
 182 static void XBZRLE_cache_lock(void)
 183 {
 184     if (migrate_use_xbzrle())
 185         qemu_mutex_lock(&XBZRLE.lock);
 186 }
 187
 188 static void XBZRLE_cache_unlock(void)
 189 {
 190     if (migrate_use_xbzrle())
 191         qemu_mutex_unlock(&XBZRLE.lock);
 192 }
 193
 194 /*
 195  * called from qmp_migrate_set_cache_size in main thread, possibly while
 196  * a migration is in progress.
 197  * A running migration maybe using the cache and might finish during this
 198  * call, hence changes to the cache are protected by XBZRLE.lock().
 199  */
 200 int64_t xbzrle_cache_resize(int64_t new_size)
 201 {
 202     PageCache *new_cache;
 203     int64_t ret;
 204
 205     if (new_size < TARGET_PAGE_SIZE) {
 206         return -1;
 207     }
 208
 209     XBZRLE_cache_lock();
 210
 211     if (XBZRLE.cache != NULL) {
 212         if (pow2floor(new_size) == migrate_xbzrle_cache_size()) {
 213             goto out_new_size;
 214         }
 215         new_cache = cache_init(new_size / TARGET_PAGE_SIZE,
 216                                         TARGET_PAGE_SIZE);
 217         if (!new_cache) {
 218             error_report("Error creating cache");
 219             ret = -1;
 220             goto out;
 221         }
 222
 223         cache_fini(XBZRLE.cache);
 224         XBZRLE.cache = new_cache;
 225     }
 226
 227 out_new_size:
 228     ret = pow2floor(new_size);
 229 out:
 230     XBZRLE_cache_unlock();
 231     return ret;
 232 }
 233
 234 /* accounting for migration statistics */
 235 typedef struct AccountingInfo {
 236     uint64_t dup_pages;
 237     uint64_t skipped_pages;
 238     uint64_t norm_pages;
 239     uint64_t iterations;
 240     uint64_t xbzrle_bytes;
 241     uint64_t xbzrle_pages;
 242     uint64_t xbzrle_cache_miss;
 243     double xbzrle_cache_miss_rate;
 244     uint64_t xbzrle_overflows;
 245 } AccountingInfo;
 246
 247 static AccountingInfo acct_info;
 248
 249 static void acct_clear(void)
 250 {
 251     memset(&acct_info, 0, sizeof(acct_info));
 252 }
 253
 254 uint64_t dup_mig_bytes_transferred(void)
 255 {
 256     return acct_info.dup_pages * TARGET_PAGE_SIZE;
 257 }
 258
 259 uint64_t dup_mig_pages_transferred(void)
 260 {
 261     return acct_info.dup_pages;
 262 }
 263
 264 uint64_t skipped_mig_bytes_transferred(void)
 265 {
 266     return acct_info.skipped_pages * TARGET_PAGE_SIZE;
 267 }
 268
 269 uint64_t skipped_mig_pages_transferred(void)
 270 {
 271     return acct_info.skipped_pages;
 272 }
 273
 274 uint64_t norm_mig_bytes_transferred(void)
 275 {
 276     return acct_info.norm_pages * TARGET_PAGE_SIZE;
 277 }
 278
 279 uint64_t norm_mig_pages_transferred(void)
 280 {
 281     return acct_info.norm_pages;
 282 }
 283
 284 uint64_t xbzrle_mig_bytes_transferred(void)
 285 {
 286     return acct_info.xbzrle_bytes;
 287 }
 288
 289 uint64_t xbzrle_mig_pages_transferred(void)
 290 {
 291     return acct_info.xbzrle_pages;
 292 }
 293
 294 uint64_t xbzrle_mig_pages_cache_miss(void)
 295 {
 296     return acct_info.xbzrle_cache_miss;
 297 }
 298
 299 double xbzrle_mig_cache_miss_rate(void)
 300 {
 301     return acct_info.xbzrle_cache_miss_rate;
 302 }
 303
 304 uint64_t xbzrle_mig_pages_overflow(void)
 305 {
 306     return acct_info.xbzrle_overflows;
 307 }
 308
 309 /* This is the last block that we have visited serching for dirty pages
 310  */
 311 static RAMBlock *last_seen_block;
 312 /* This is the last block from where we have sent data */
 313 static RAMBlock *last_sent_block;
 314 static ram_addr_t last_offset;
 315 static unsigned long *migration_bitmap;
 316 static uint64_t migration_dirty_pages;
 317 static uint32_t last_version;
 318 static bool ram_bulk_stage;
 319
 320 struct CompressParam {
 321     bool start;
 322     bool done;
 323     QEMUFile *file;
 324     QemuMutex mutex;
 325     QemuCond cond;
 326     RAMBlock *block;
 327     ram_addr_t offset;
 328 };
 329 typedef struct CompressParam CompressParam;
 330
 331 struct DecompressParam {
 332     bool start;
 333     QemuMutex mutex;
 334     QemuCond cond;
 335     void *des;
 336     uint8 *compbuf;
 337     int len;
 338 };
 339 typedef struct DecompressParam DecompressParam;
 340
 341 static CompressParam *comp_param;
 342 static QemuThread *compress_threads;
 343 /* comp_done_cond is used to wake up the migration thread when
 344  * one of the compression threads has finished the compression.
 345  * comp_done_lock is used to co-work with comp_done_cond.
 346  */
 347 static QemuMutex *comp_done_lock;
 348 static QemuCond *comp_done_cond;
 349 /* The empty QEMUFileOps will be used by file in CompressParam */
 350 static const QEMUFileOps empty_ops = { };
 351
 352 static bool compression_switch;
 353 static bool quit_comp_thread;
 354 static bool quit_decomp_thread;
 355 static DecompressParam *decomp_param;
 356 static QemuThread *decompress_threads;
 357 static uint8_t *compressed_data_buf;
 358
 359 static int do_compress_ram_page(CompressParam *param);
 360
 361 static void *do_data_compress(void *opaque)
 362 {
 363     CompressParam *param = opaque;
 364
 365     while (!quit_comp_thread) {
 366         qemu_mutex_lock(&param->mutex);
 367         /* Re-check the quit_comp_thread in case of
 368          * terminate_compression_threads is called just before
 369          * qemu_mutex_lock(&param->mutex) and after
 370          * while(!quit_comp_thread), re-check it here can make
 371          * sure the compression thread terminate as expected.
 372          */
 373         while (!param->start && !quit_comp_thread) {
 374             qemu_cond_wait(&param->cond, &param->mutex);
 375         }
 376         if (!quit_comp_thread) {
 377             do_compress_ram_page(param);
 378         }
 379         param->start = false;
 380         qemu_mutex_unlock(&param->mutex);
 381
 382         qemu_mutex_lock(comp_done_lock);
 383         param->done = true;
 384         qemu_cond_signal(comp_done_cond);
 385         qemu_mutex_unlock(comp_done_lock);
 386     }
 387
 388     return NULL;
 389 }
 390
 391 static inline void terminate_compression_threads(void)
 392 {
 393     int idx, thread_count;
 394
 395     thread_count = migrate_compress_threads();
 396     quit_comp_thread = true;
 397     for (idx = 0; idx < thread_count; idx++) {
 398         qemu_mutex_lock(&comp_param[idx].mutex);
 399         qemu_cond_signal(&comp_param[idx].cond);
 400         qemu_mutex_unlock(&comp_param[idx].mutex);
 401     }
 402 }
 403
 404 void migrate_compress_threads_join(void)
 405 {
 406     int i, thread_count;
 407
 408     if (!migrate_use_compression()) {
 409         return;
 410     }
 411     terminate_compression_threads();
 412     thread_count = migrate_compress_threads();
 413     for (i = 0; i < thread_count; i++) {
 414         qemu_thread_join(compress_threads + i);
 415         qemu_fclose(comp_param[i].file);
 416         qemu_mutex_destroy(&comp_param[i].mutex);
 417         qemu_cond_destroy(&comp_param[i].cond);
 418     }
 419     qemu_mutex_destroy(comp_done_lock);
 420     qemu_cond_destroy(comp_done_cond);
 421     g_free(compress_threads);
 422     g_free(comp_param);
 423     g_free(comp_done_cond);
 424     g_free(comp_done_lock);
 425     compress_threads = NULL;
 426     comp_param = NULL;
 427     comp_done_cond = NULL;
 428     comp_done_lock = NULL;
 429 }
 430
 431 void migrate_compress_threads_create(void)
 432 {
 433     int i, thread_count;
 434
 435     if (!migrate_use_compression()) {
 436         return;
 437     }
 438     quit_comp_thread = false;
 439     compression_switch = true;
 440     thread_count = migrate_compress_threads();
 441     compress_threads = g_new0(QemuThread, thread_count);
 442     comp_param = g_new0(CompressParam, thread_count);
 443     comp_done_cond = g_new0(QemuCond, 1);
 444     comp_done_lock = g_new0(QemuMutex, 1);
 445     qemu_cond_init(comp_done_cond);
 446     qemu_mutex_init(comp_done_lock);
 447     for (i = 0; i < thread_count; i++) {
 448         /* com_param[i].file is just used as a dummy buffer to save data, set
 449          * it's ops to empty.
 450          */
 451         comp_param[i].file = qemu_fopen_ops(NULL, &empty_ops);
 452         comp_param[i].done = true;
 453         qemu_mutex_init(&comp_param[i].mutex);
 454         qemu_cond_init(&comp_param[i].cond);
 455         qemu_thread_create(compress_threads + i, "compress",
 456                            do_data_compress, comp_param + i,
 457                            QEMU_THREAD_JOINABLE);
 458     }
 459 }
 460
 461 /**
 462  * save_page_header: Write page header to wire
 463  *
 464  * If this is the 1st block, it also writes the block identification
 465  *
 466  * Returns: Number of bytes written
 467  *
 468  * @f: QEMUFile where to send the data
 469  * @block: block that contains the page we want to send
 470  * @offset: offset inside the block for the page
 471  *          in the lower bits, it contains flags
 472  */
 473 static size_t save_page_header(QEMUFile *f, RAMBlock *block, ram_addr_t offset)
 474 {
 475     size_t size;
 476
 477     qemu_put_be64(f, offset);
 478     size = 8;
 479
 480     if (!(offset & RAM_SAVE_FLAG_CONTINUE)) {
 481         qemu_put_byte(f, strlen(block->idstr));
 482         qemu_put_buffer(f, (uint8_t *)block->idstr,
 483                         strlen(block->idstr));
 484         size += 1 + strlen(block->idstr);
 485     }
 486     return size;
 487 }
 488
 489 /* Update the xbzrle cache to reflect a page that's been sent as all 0.
 490  * The important thing is that a stale (not-yet-0'd) page be replaced
 491  * by the new data.
 492  * As a bonus, if the page wasn't in the cache it gets added so that
 493  * when a small write is made into the 0'd page it gets XBZRLE sent
 494  */
 495 static void xbzrle_cache_zero_page(ram_addr_t current_addr)
 496 {
 497     if (ram_bulk_stage || !migrate_use_xbzrle()) {
 498         return;
 499     }
 500
 501     /* We don't care if this fails to allocate a new cache page
 502      * as long as it updated an old one */
 503     cache_insert(XBZRLE.cache, current_addr, ZERO_TARGET_PAGE,
 504                  bitmap_sync_count);
 505 }
 506
 507 #define ENCODING_FLAG_XBZRLE 0x1
 508
 509 /**
 510  * save_xbzrle_page: compress and send current page
 511  *
 512  * Returns: 1 means that we wrote the page
 513  *          0 means that page is identical to the one already sent
 514  *          -1 means that xbzrle would be longer than normal
 515  *
 516  * @f: QEMUFile where to send the data
 517  * @current_data:
 518  * @current_addr:
 519  * @block: block that contains the page we want to send
 520  * @offset: offset inside the block for the page
 521  * @last_stage: if we are at the completion stage
 522  * @bytes_transferred: increase it with the number of transferred bytes
 523  */
 524 static int save_xbzrle_page(QEMUFile *f, uint8_t **current_data,
 525                             ram_addr_t current_addr, RAMBlock *block,
 526                             ram_addr_t offset, bool last_stage,
 527                             uint64_t *bytes_transferred)
 528 {
 529     int encoded_len = 0, bytes_xbzrle;
 530     uint8_t *prev_cached_page;
 531
 532     if (!cache_is_cached(XBZRLE.cache, current_addr, bitmap_sync_count)) {
 533         acct_info.xbzrle_cache_miss++;
 534         if (!last_stage) {
 535             if (cache_insert(XBZRLE.cache, current_addr, *current_data,
 536                              bitmap_sync_count) == -1) {
 537                 return -1;
 538             } else {
 539                 /* update *current_data when the page has been
 540                    inserted into cache */
 541                 *current_data = get_cached_data(XBZRLE.cache, current_addr);
 542             }
 543         }
 544         return -1;
 545     }
 546
 547     prev_cached_page = get_cached_data(XBZRLE.cache, current_addr);
 548
 549     /* save current buffer into memory */
 550     memcpy(XBZRLE.current_buf, *current_data, TARGET_PAGE_SIZE);
 551
 552     /* XBZRLE encoding (if there is no overflow) */
 553     encoded_len = xbzrle_encode_buffer(prev_cached_page, XBZRLE.current_buf,
 554                                        TARGET_PAGE_SIZE, XBZRLE.encoded_buf,
 555                                        TARGET_PAGE_SIZE);
 556     if (encoded_len == 0) {
 557         DPRINTF("Skipping unmodified page\n");
 558         return 0;
 559     } else if (encoded_len == -1) {
 560         DPRINTF("Overflow\n");
 561         acct_info.xbzrle_overflows++;
 562         /* update data in the cache */
 563         if (!last_stage) {
 564             memcpy(prev_cached_page, *current_data, TARGET_PAGE_SIZE);
 565             *current_data = prev_cached_page;
 566         }
 567         return -1;
 568     }
 569
 570     /* we need to update the data in the cache, in order to get the same data */
 571     if (!last_stage) {
 572         memcpy(prev_cached_page, XBZRLE.current_buf, TARGET_PAGE_SIZE);
 573     }
 574
 575     /* Send XBZRLE based compressed page */
 576     bytes_xbzrle = save_page_header(f, block, offset | RAM_SAVE_FLAG_XBZRLE);
 577     qemu_put_byte(f, ENCODING_FLAG_XBZRLE);
 578     qemu_put_be16(f, encoded_len);
 579     qemu_put_buffer(f, XBZRLE.encoded_buf, encoded_len);
 580     bytes_xbzrle += encoded_len + 1 + 2;
 581     acct_info.xbzrle_pages++;
 582     acct_info.xbzrle_bytes += bytes_xbzrle;
 583     *bytes_transferred += bytes_xbzrle;
 584
 585     return 1;
 586 }
 587
 588 static inline
 589 ram_addr_t migration_bitmap_find_and_reset_dirty(MemoryRegion *mr,
 590                                                  ram_addr_t start)
 591 {
 592     unsigned long base = mr->ram_addr >> TARGET_PAGE_BITS;
 593     unsigned long nr = base + (start >> TARGET_PAGE_BITS);
 594     uint64_t mr_size = TARGET_PAGE_ALIGN(memory_region_size(mr));
 595     unsigned long size = base + (mr_size >> TARGET_PAGE_BITS);
 596
 597     unsigned long next;
 598
 599     if (ram_bulk_stage && nr > base) {
 600         next = nr + 1;
 601     } else {
 602         next = find_next_bit(migration_bitmap, size, nr);
 603     }
 604
 605     if (next < size) {
 606         clear_bit(next, migration_bitmap);
 607         migration_dirty_pages--;
 608     }
 609     return (next - base) << TARGET_PAGE_BITS;
 610 }
 611
 612 static void migration_bitmap_sync_range(ram_addr_t start, ram_addr_t length)
 613 {
 614     migration_dirty_pages +=
 615         cpu_physical_memory_sync_dirty_bitmap(migration_bitmap, start, length);
 616 }
 617
 618
 619 /* Fix me: there are too many global variables used in migration process. */
 620 static int64_t start_time;
 621 static int64_t bytes_xfer_prev;
 622 static int64_t num_dirty_pages_period;
 623 static uint64_t xbzrle_cache_miss_prev;
 624 static uint64_t iterations_prev;
 625
 626 static void migration_bitmap_sync_init(void)
 627 {
 628     start_time = 0;
 629     bytes_xfer_prev = 0;
 630     num_dirty_pages_period = 0;
 631     xbzrle_cache_miss_prev = 0;
 632     iterations_prev = 0;
 633 }
 634
 635 /* Called with iothread lock held, to protect ram_list.dirty_memory[] */
 636 static void migration_bitmap_sync(void)
 637 {
 638     RAMBlock *block;
 639     uint64_t num_dirty_pages_init = migration_dirty_pages;
 640     MigrationState *s = migrate_get_current();
 641     int64_t end_time;
 642     int64_t bytes_xfer_now;
 643
 644     bitmap_sync_count++;
 645
 646     if (!bytes_xfer_prev) {
 647         bytes_xfer_prev = ram_bytes_transferred();
 648     }
 649
 650     if (!start_time) {
 651         start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
 652     }
 653
 654     trace_migration_bitmap_sync_start();
 655     address_space_sync_dirty_bitmap(&address_space_memory);
 656
 657     rcu_read_lock();
 658     QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
 659         migration_bitmap_sync_range(block->mr->ram_addr, block->used_length);
 660     }
 661     rcu_read_unlock();
 662
 663     trace_migration_bitmap_sync_end(migration_dirty_pages
 664                                     - num_dirty_pages_init);
 665     num_dirty_pages_period += migration_dirty_pages - num_dirty_pages_init;
 666     end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
 667
 668     /* more than 1 second = 1000 millisecons */
 669     if (end_time > start_time + 1000) {
 670         if (migrate_auto_converge()) {
 671             /* The following detection logic can be refined later. For now:
 672                Check to see if the dirtied bytes is 50% more than the approx.
 673                amount of bytes that just got transferred since the last time we
 674                were in this routine. If that happens >N times (for now N==4)
 675                we turn on the throttle down logic */
 676             bytes_xfer_now = ram_bytes_transferred();
 677             if (s->dirty_pages_rate &&
 678                (num_dirty_pages_period * TARGET_PAGE_SIZE >
 679                    (bytes_xfer_now - bytes_xfer_prev)/2) &&
 680                (dirty_rate_high_cnt++ > 4)) {
 681                     trace_migration_throttle();
 682                     mig_throttle_on = true;
 683                     dirty_rate_high_cnt = 0;
 684              }
 685              bytes_xfer_prev = bytes_xfer_now;
 686         } else {
 687              mig_throttle_on = false;
 688         }
 689         if (migrate_use_xbzrle()) {
 690             if (iterations_prev != acct_info.iterations) {
 691                 acct_info.xbzrle_cache_miss_rate =
 692                    (double)(acct_info.xbzrle_cache_miss -
 693                             xbzrle_cache_miss_prev) /
 694                    (acct_info.iterations - iterations_prev);
 695             }
 696             iterations_prev = acct_info.iterations;
 697             xbzrle_cache_miss_prev = acct_info.xbzrle_cache_miss;
 698         }
 699         s->dirty_pages_rate = num_dirty_pages_period * 1000
 700             / (end_time - start_time);
 701         s->dirty_bytes_rate = s->dirty_pages_rate * TARGET_PAGE_SIZE;
 702         start_time = end_time;
 703         num_dirty_pages_period = 0;
 704     }
 705     s->dirty_sync_count = bitmap_sync_count;
 706 }
 707
 708 /**
 709  * save_zero_page: Send the zero page to the stream
 710  *
 711  * Returns: Number of pages written.
 712  *
 713  * @f: QEMUFile where to send the data
 714  * @block: block that contains the page we want to send
 715  * @offset: offset inside the block for the page
 716  * @p: pointer to the page
 717  * @bytes_transferred: increase it with the number of transferred bytes
 718  */
 719 static int save_zero_page(QEMUFile *f, RAMBlock *block, ram_addr_t offset,
 720                           uint8_t *p, uint64_t *bytes_transferred)
 721 {
 722     int pages = -1;
 723
 724     if (is_zero_range(p, TARGET_PAGE_SIZE)) {
 725         acct_info.dup_pages++;
 726         *bytes_transferred += save_page_header(f, block,
 727                                                offset | RAM_SAVE_FLAG_COMPRESS);
 728         qemu_put_byte(f, 0);
 729         *bytes_transferred += 1;
 730         pages = 1;
 731     }
 732
 733     return pages;
 734 }
 735
 736 /**
 737  * ram_save_page: Send the given page to the stream
 738  *
 739  * Returns: Number of pages written.
 740  *
 741  * @f: QEMUFile where to send the data
 742  * @block: block that contains the page we want to send
 743  * @offset: offset inside the block for the page
 744  * @last_stage: if we are at the completion stage
 745  * @bytes_transferred: increase it with the number of transferred bytes
 746  */
 747 static int ram_save_page(QEMUFile *f, RAMBlock* block, ram_addr_t offset,
 748                          bool last_stage, uint64_t *bytes_transferred)
 749 {
 750     int pages = -1;
 751     uint64_t bytes_xmit;
 752     ram_addr_t current_addr;
 753     MemoryRegion *mr = block->mr;
 754     uint8_t *p;
 755     int ret;
 756     bool send_async = true;
 757
 758     p = memory_region_get_ram_ptr(mr) + offset;
 759
 760     /* In doubt sent page as normal */
 761     bytes_xmit = 0;
 762     ret = ram_control_save_page(f, block->offset,
 763                            offset, TARGET_PAGE_SIZE, &bytes_xmit);
 764     if (bytes_xmit) {
 765         *bytes_transferred += bytes_xmit;
 766         pages = 1;
 767     }
 768
 769     XBZRLE_cache_lock();
 770
 771     current_addr = block->offset + offset;
 772
 773     if (block == last_sent_block) {
 774         offset |= RAM_SAVE_FLAG_CONTINUE;
 775     }
 776     if (ret != RAM_SAVE_CONTROL_NOT_SUPP) {
 777         if (ret != RAM_SAVE_CONTROL_DELAYED) {
 778             if (bytes_xmit > 0) {
 779                 acct_info.norm_pages++;
 780             } else if (bytes_xmit == 0) {
 781                 acct_info.dup_pages++;
 782             }
 783         }
 784     } else {
 785         pages = save_zero_page(f, block, offset, p, bytes_transferred);
 786         if (pages > 0) {
 787             /* Must let xbzrle know, otherwise a previous (now 0'd) cached
 788              * page would be stale
 789              */
 790             xbzrle_cache_zero_page(current_addr);
 791         } else if (!ram_bulk_stage && migrate_use_xbzrle()) {
 792             pages = save_xbzrle_page(f, &p, current_addr, block,
 793                                      offset, last_stage, bytes_transferred);
 794             if (!last_stage) {
 795                 /* Can't send this cached data async, since the cache page
 796                  * might get updated before it gets to the wire
 797                  */
 798                 send_async = false;
 799             }
 800         }
 801     }
 802
 803     /* XBZRLE overflow or normal page */
 804     if (pages == -1) {
 805         *bytes_transferred += save_page_header(f, block,
 806                                                offset | RAM_SAVE_FLAG_PAGE);
 807         if (send_async) {
 808             qemu_put_buffer_async(f, p, TARGET_PAGE_SIZE);
 809         } else {
 810             qemu_put_buffer(f, p, TARGET_PAGE_SIZE);
 811         }
 812         *bytes_transferred += TARGET_PAGE_SIZE;
 813         pages = 1;
 814         acct_info.norm_pages++;
 815     }
 816
 817     XBZRLE_cache_unlock();
 818
 819     return pages;
 820 }
 821
 822 static int do_compress_ram_page(CompressParam *param)
 823 {
 824     int bytes_sent, blen;
 825     uint8_t *p;
 826     RAMBlock *block = param->block;
 827     ram_addr_t offset = param->offset;
 828
 829     p = memory_region_get_ram_ptr(block->mr) + (offset & TARGET_PAGE_MASK);
 830
 831     bytes_sent = save_page_header(param->file, block, offset |
 832                                   RAM_SAVE_FLAG_COMPRESS_PAGE);
 833     blen = qemu_put_compression_data(param->file, p, TARGET_PAGE_SIZE,
 834                                      migrate_compress_level());
 835     bytes_sent += blen;
 836
 837     return bytes_sent;
 838 }
 839
 840 static inline void start_compression(CompressParam *param)
 841 {
 842     param->done = false;
 843     qemu_mutex_lock(&param->mutex);
 844     param->start = true;
 845     qemu_cond_signal(&param->cond);
 846     qemu_mutex_unlock(&param->mutex);
 847 }
 848
 849 static inline void start_decompression(DecompressParam *param)
 850 {
 851     qemu_mutex_lock(&param->mutex);
 852     param->start = true;
 853     qemu_cond_signal(&param->cond);
 854     qemu_mutex_unlock(&param->mutex);
 855 }
 856
 857 static uint64_t bytes_transferred;
 858
 859 static void flush_compressed_data(QEMUFile *f)
 860 {
 861     int idx, len, thread_count;
 862
 863     if (!migrate_use_compression()) {
 864         return;
 865     }
 866     thread_count = migrate_compress_threads();
 867     for (idx = 0; idx < thread_count; idx++) {
 868         if (!comp_param[idx].done) {
 869             qemu_mutex_lock(comp_done_lock);
 870             while (!comp_param[idx].done && !quit_comp_thread) {
 871                 qemu_cond_wait(comp_done_cond, comp_done_lock);
 872             }
 873             qemu_mutex_unlock(comp_done_lock);
 874         }
 875         if (!quit_comp_thread) {
 876             len = qemu_put_qemu_file(f, comp_param[idx].file);
 877             bytes_transferred += len;
 878         }
 879     }
 880 }
 881
 882 static inline void set_compress_params(CompressParam *param, RAMBlock *block,
 883                                        ram_addr_t offset)
 884 {
 885     param->block = block;
 886     param->offset = offset;
 887 }
 888
 889 static int compress_page_with_multi_thread(QEMUFile *f, RAMBlock *block,
 890                                            ram_addr_t offset,
 891                                            uint64_t *bytes_transferred)
 892 {
 893     int idx, thread_count, bytes_xmit = -1, pages = -1;
 894
 895     thread_count = migrate_compress_threads();
 896     qemu_mutex_lock(comp_done_lock);
 897     while (true) {
 898         for (idx = 0; idx < thread_count; idx++) {
 899             if (comp_param[idx].done) {
 900                 bytes_xmit = qemu_put_qemu_file(f, comp_param[idx].file);
 901                 set_compress_params(&comp_param[idx], block, offset);
 902                 start_compression(&comp_param[idx]);
 903                 pages = 1;
 904                 acct_info.norm_pages++;
 905                 *bytes_transferred += bytes_xmit;
 906                 break;
 907             }
 908         }
 909         if (pages > 0) {
 910             break;
 911         } else {
 912             qemu_cond_wait(comp_done_cond, comp_done_lock);
 913         }
 914     }
 915     qemu_mutex_unlock(comp_done_lock);
 916
 917     return pages;
 918 }
 919
 920 /**
 921  * ram_save_compressed_page: compress the given page and send it to the stream
 922  *
 923  * Returns: Number of pages written.
 924  *
 925  * @f: QEMUFile where to send the data
 926  * @block: block that contains the page we want to send
 927  * @offset: offset inside the block for the page
 928  * @last_stage: if we are at the completion stage
 929  * @bytes_transferred: increase it with the number of transferred bytes
 930  */
 931 static int ram_save_compressed_page(QEMUFile *f, RAMBlock *block,
 932                                     ram_addr_t offset, bool last_stage,
 933                                     uint64_t *bytes_transferred)
 934 {
 935     int pages = -1;
 936     uint64_t bytes_xmit;
 937     MemoryRegion *mr = block->mr;
 938     uint8_t *p;
 939     int ret;
 940
 941     p = memory_region_get_ram_ptr(mr) + offset;
 942
 943     bytes_xmit = 0;
 944     ret = ram_control_save_page(f, block->offset,
 945                                 offset, TARGET_PAGE_SIZE, &bytes_xmit);
 946     if (bytes_xmit) {
 947         *bytes_transferred += bytes_xmit;
 948         pages = 1;
 949     }
 950     if (block == last_sent_block) {
 951         offset |= RAM_SAVE_FLAG_CONTINUE;
 952     }
 953     if (ret != RAM_SAVE_CONTROL_NOT_SUPP) {
 954         if (ret != RAM_SAVE_CONTROL_DELAYED) {
 955             if (bytes_xmit > 0) {
 956                 acct_info.norm_pages++;
 957             } else if (bytes_xmit == 0) {
 958                 acct_info.dup_pages++;
 959             }
 960         }
 961     } else {
 962         /* When starting the process of a new block, the first page of
 963          * the block should be sent out before other pages in the same
 964          * block, and all the pages in last block should have been sent
 965          * out, keeping this order is important, because the 'cont' flag
 966          * is used to avoid resending the block name.
 967          */
 968         if (block != last_sent_block) {
 969             flush_compressed_data(f);
 970             pages = save_zero_page(f, block, offset, p, bytes_transferred);
 971             if (pages == -1) {
 972                 set_compress_params(&comp_param[0], block, offset);
 973                 /* Use the qemu thread to compress the data to make sure the
 974                  * first page is sent out before other pages
 975                  */
 976                 bytes_xmit = do_compress_ram_page(&comp_param[0]);
 977                 acct_info.norm_pages++;
 978                 qemu_put_qemu_file(f, comp_param[0].file);
 979                 *bytes_transferred += bytes_xmit;
 980                 pages = 1;
 981             }
 982         } else {
 983             pages = save_zero_page(f, block, offset, p, bytes_transferred);
 984             if (pages == -1) {
 985                 pages = compress_page_with_multi_thread(f, block, offset,
 986                                                         bytes_transferred);
 987             }
 988         }
 989     }
 990
 991     return pages;
 992 }
 993
 994 /**
 995  * ram_find_and_save_block: Finds a dirty page and sends it to f
 996  *
 997  * Called within an RCU critical section.
 998  *
 999  * Returns:  The number of pages written
1000  *           0 means no dirty pages
1001  *
1002  * @f: QEMUFile where to send the data
1003  * @last_stage: if we are at the completion stage
1004  * @bytes_transferred: increase it with the number of transferred bytes
1005  */
1006
1007 static int ram_find_and_save_block(QEMUFile *f, bool last_stage,
1008                                    uint64_t *bytes_transferred)
1009 {
1010     RAMBlock *block = last_seen_block;
1011     ram_addr_t offset = last_offset;
1012     bool complete_round = false;
1013     int pages = 0;
1014     MemoryRegion *mr;
1015
1016     if (!block)
1017         block = QLIST_FIRST_RCU(&ram_list.blocks);
1018
1019     while (true) {
1020         mr = block->mr;
1021         offset = migration_bitmap_find_and_reset_dirty(mr, offset);
1022         if (complete_round && block == last_seen_block &&
1023             offset >= last_offset) {
1024             break;
1025         }
1026         if (offset >= block->used_length) {
1027             offset = 0;
1028             block = QLIST_NEXT_RCU(block, next);
1029             if (!block) {
1030                 block = QLIST_FIRST_RCU(&ram_list.blocks);
1031                 complete_round = true;
1032                 ram_bulk_stage = false;
1033                 if (migrate_use_xbzrle()) {
1034                     /* If xbzrle is on, stop using the data compression at this
1035                      * point. In theory, xbzrle can do better than compression.
1036                      */
1037                     flush_compressed_data(f);
1038                     compression_switch = false;
1039                 }
1040             }
1041         } else {
1042             if (compression_switch && migrate_use_compression()) {
1043                 pages = ram_save_compressed_page(f, block, offset, last_stage,
1044                                                  bytes_transferred);
1045             } else {
1046                 pages = ram_save_page(f, block, offset, last_stage,
1047                                       bytes_transferred);
1048             }
1049
1050             /* if page is unmodified, continue to the next */
1051             if (pages > 0) {
1052                 last_sent_block = block;
1053                 break;
1054             }
1055         }
1056     }
1057
1058     last_seen_block = block;
1059     last_offset = offset;
1060
1061     return pages;
1062 }
1063
1064 void acct_update_position(QEMUFile *f, size_t size, bool zero)
1065 {
1066     uint64_t pages = size / TARGET_PAGE_SIZE;
1067     if (zero) {
1068         acct_info.dup_pages += pages;
1069     } else {
1070         acct_info.norm_pages += pages;
1071         bytes_transferred += size;
1072         qemu_update_position(f, size);
1073     }
1074 }
1075
1076 static ram_addr_t ram_save_remaining(void)
1077 {
1078     return migration_dirty_pages;
1079 }
1080
1081 uint64_t ram_bytes_remaining(void)
1082 {
1083     return ram_save_remaining() * TARGET_PAGE_SIZE;
1084 }
1085
1086 uint64_t ram_bytes_transferred(void)
1087 {
1088     return bytes_transferred;
1089 }
1090
1091 uint64_t ram_bytes_total(void)
1092 {
1093     RAMBlock *block;
1094     uint64_t total = 0;
1095
1096     rcu_read_lock();
1097     QLIST_FOREACH_RCU(block, &ram_list.blocks, next)
1098         total += block->used_length;
1099     rcu_read_unlock();
1100     return total;
1101 }
1102
1103 void free_xbzrle_decoded_buf(void)
1104 {
1105     g_free(xbzrle_decoded_buf);
1106     xbzrle_decoded_buf = NULL;
1107 }
1108
1109 static void migration_end(void)
1110 {
1111     if (migration_bitmap) {
1112         memory_global_dirty_log_stop();
1113         g_free(migration_bitmap);
1114         migration_bitmap = NULL;
1115     }
1116
1117     XBZRLE_cache_lock();
1118     if (XBZRLE.cache) {
1119         cache_fini(XBZRLE.cache);
1120         g_free(XBZRLE.encoded_buf);
1121         g_free(XBZRLE.current_buf);
1122         XBZRLE.cache = NULL;
1123         XBZRLE.encoded_buf = NULL;
1124         XBZRLE.current_buf = NULL;
1125     }
1126     XBZRLE_cache_unlock();
1127 }
1128
1129 static void ram_migration_cancel(void *opaque)
1130 {
1131     migration_end();
1132 }
1133
1134 static void reset_ram_globals(void)
1135 {
1136     last_seen_block = NULL;
1137     last_sent_block = NULL;
1138     last_offset = 0;
1139     last_version = ram_list.version;
1140     ram_bulk_stage = true;
1141 }
1142
1143 #define MAX_WAIT 50 /* ms, half buffered_file limit */
1144
1145
1146 /* Each of ram_save_setup, ram_save_iterate and ram_save_complete has
1147  * long-running RCU critical section.  When rcu-reclaims in the code
1148  * start to become numerous it will be necessary to reduce the
1149  * granularity of these critical sections.
1150  */
1151
1152 static int ram_save_setup(QEMUFile *f, void *opaque)
1153 {
1154     RAMBlock *block;
1155     int64_t ram_bitmap_pages; /* Size of bitmap in pages, including gaps */
1156
1157     mig_throttle_on = false;
1158     dirty_rate_high_cnt = 0;
1159     bitmap_sync_count = 0;
1160     migration_bitmap_sync_init();
1161
1162     if (migrate_use_xbzrle()) {
1163         XBZRLE_cache_lock();
1164         XBZRLE.cache = cache_init(migrate_xbzrle_cache_size() /
1165                                   TARGET_PAGE_SIZE,
1166                                   TARGET_PAGE_SIZE);
1167         if (!XBZRLE.cache) {
1168             XBZRLE_cache_unlock();
1169             error_report("Error creating cache");
1170             return -1;
1171         }
1172         XBZRLE_cache_unlock();
1173
1174         /* We prefer not to abort if there is no memory */
1175         XBZRLE.encoded_buf = g_try_malloc0(TARGET_PAGE_SIZE);
1176         if (!XBZRLE.encoded_buf) {
1177             error_report("Error allocating encoded_buf");
1178             return -1;
1179         }
1180
1181         XBZRLE.current_buf = g_try_malloc(TARGET_PAGE_SIZE);
1182         if (!XBZRLE.current_buf) {
1183             error_report("Error allocating current_buf");
1184             g_free(XBZRLE.encoded_buf);
1185             XBZRLE.encoded_buf = NULL;
1186             return -1;
1187         }
1188
1189         acct_clear();
1190     }
1191
1192     /* iothread lock needed for ram_list.dirty_memory[] */
1193     qemu_mutex_lock_iothread();
1194     qemu_mutex_lock_ramlist();
1195     rcu_read_lock();
1196     bytes_transferred = 0;
1197     reset_ram_globals();
1198
1199     ram_bitmap_pages = last_ram_offset() >> TARGET_PAGE_BITS;
1200     migration_bitmap = bitmap_new(ram_bitmap_pages);
1201     bitmap_set(migration_bitmap, 0, ram_bitmap_pages);
1202
1203     /*
1204      * Count the total number of pages used by ram blocks not including any
1205      * gaps due to alignment or unplugs.
1206      */
1207     migration_dirty_pages = ram_bytes_total() >> TARGET_PAGE_BITS;
1208
1209     memory_global_dirty_log_start();
1210     migration_bitmap_sync();
1211     qemu_mutex_unlock_ramlist();
1212     qemu_mutex_unlock_iothread();
1213
1214     qemu_put_be64(f, ram_bytes_total() | RAM_SAVE_FLAG_MEM_SIZE);
1215
1216     QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1217         qemu_put_byte(f, strlen(block->idstr));
1218         qemu_put_buffer(f, (uint8_t *)block->idstr, strlen(block->idstr));
1219         qemu_put_be64(f, block->used_length);
1220     }
1221
1222     rcu_read_unlock();
1223
1224     ram_control_before_iterate(f, RAM_CONTROL_SETUP);
1225     ram_control_after_iterate(f, RAM_CONTROL_SETUP);
1226
1227     qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
1228
1229     return 0;
1230 }
1231
1232 static int ram_save_iterate(QEMUFile *f, void *opaque)
1233 {
1234     int ret;
1235     int i;
1236     int64_t t0;
1237     int pages_sent = 0;
1238
1239     rcu_read_lock();
1240     if (ram_list.version != last_version) {
1241         reset_ram_globals();
1242     }
1243
1244     /* Read version before ram_list.blocks */
1245     smp_rmb();
1246
1247     ram_control_before_iterate(f, RAM_CONTROL_ROUND);
1248
1249     t0 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
1250     i = 0;
1251     while ((ret = qemu_file_rate_limit(f)) == 0) {
1252         int pages;
1253
1254         pages = ram_find_and_save_block(f, false, &bytes_transferred);
1255         /* no more pages to sent */
1256         if (pages == 0) {
1257             break;
1258         }
1259         pages_sent += pages;
1260         acct_info.iterations++;
1261         check_guest_throttling();
1262         /* we want to check in the 1st loop, just in case it was the 1st time
1263            and we had to sync the dirty bitmap.
1264            qemu_get_clock_ns() is a bit expensive, so we only check each some
1265            iterations
1266         */
1267         if ((i & 63) == 0) {
1268             uint64_t t1 = (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - t0) / 1000000;
1269             if (t1 > MAX_WAIT) {
1270                 DPRINTF("big wait: %" PRIu64 " milliseconds, %d iterations\n",
1271                         t1, i);
1272                 break;
1273             }
1274         }
1275         i++;
1276     }
1277     flush_compressed_data(f);
1278     rcu_read_unlock();
1279
1280     /*
1281      * Must occur before EOS (or any QEMUFile operation)
1282      * because of RDMA protocol.
1283      */
1284     ram_control_after_iterate(f, RAM_CONTROL_ROUND);
1285
1286     qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
1287     bytes_transferred += 8;
1288
1289     ret = qemu_file_get_error(f);
1290     if (ret < 0) {
1291         return ret;
1292     }
1293
1294     return pages_sent;
1295 }
1296
1297 /* Called with iothread lock */
1298 static int ram_save_complete(QEMUFile *f, void *opaque)
1299 {
1300     rcu_read_lock();
1301
1302     migration_bitmap_sync();
1303
1304     ram_control_before_iterate(f, RAM_CONTROL_FINISH);
1305
1306     /* try transferring iterative blocks of memory */
1307
1308     /* flush all remaining blocks regardless of rate limiting */
1309     while (true) {
1310         int pages;
1311
1312         pages = ram_find_and_save_block(f, true, &bytes_transferred);
1313         /* no more blocks to sent */
1314         if (pages == 0) {
1315             break;
1316         }
1317     }
1318
1319     flush_compressed_data(f);
1320     ram_control_after_iterate(f, RAM_CONTROL_FINISH);
1321     migration_end();
1322
1323     rcu_read_unlock();
1324     qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
1325
1326     return 0;
1327 }
1328
1329 static uint64_t ram_save_pending(QEMUFile *f, void *opaque, uint64_t max_size)
1330 {
1331     uint64_t remaining_size;
1332
1333     remaining_size = ram_save_remaining() * TARGET_PAGE_SIZE;
1334
1335     if (remaining_size < max_size) {
1336         qemu_mutex_lock_iothread();
1337         rcu_read_lock();
1338         migration_bitmap_sync();
1339         rcu_read_unlock();
1340         qemu_mutex_unlock_iothread();
1341         remaining_size = ram_save_remaining() * TARGET_PAGE_SIZE;
1342     }
1343     return remaining_size;
1344 }
1345
1346 static int load_xbzrle(QEMUFile *f, ram_addr_t addr, void *host)
1347 {
1348     unsigned int xh_len;
1349     int xh_flags;
1350
1351     if (!xbzrle_decoded_buf) {
1352         xbzrle_decoded_buf = g_malloc(TARGET_PAGE_SIZE);
1353     }
1354
1355     /* extract RLE header */
1356     xh_flags = qemu_get_byte(f);
1357     xh_len = qemu_get_be16(f);
1358
1359     if (xh_flags != ENCODING_FLAG_XBZRLE) {
1360         error_report("Failed to load XBZRLE page - wrong compression!");
1361         return -1;
1362     }
1363
1364     if (xh_len > TARGET_PAGE_SIZE) {
1365         error_report("Failed to load XBZRLE page - len overflow!");
1366         return -1;
1367     }
1368     /* load data and decode */
1369     qemu_get_buffer(f, xbzrle_decoded_buf, xh_len);
1370
1371     /* decode RLE */
1372     if (xbzrle_decode_buffer(xbzrle_decoded_buf, xh_len, host,
1373                              TARGET_PAGE_SIZE) == -1) {
1374         error_report("Failed to load XBZRLE page - decode error!");
1375         return -1;
1376     }
1377
1378     return 0;
1379 }
1380
1381 /* Must be called from within a rcu critical section.
1382  * Returns a pointer from within the RCU-protected ram_list.
1383  */
1384 static inline void *host_from_stream_offset(QEMUFile *f,
1385                                             ram_addr_t offset,
1386                                             int flags)
1387 {
1388     static RAMBlock *block = NULL;
1389     char id[256];
1390     uint8_t len;
1391
1392     if (flags & RAM_SAVE_FLAG_CONTINUE) {
1393         if (!block || block->max_length <= offset) {
1394             error_report("Ack, bad migration stream!");
1395             return NULL;
1396         }
1397
1398         return memory_region_get_ram_ptr(block->mr) + offset;
1399     }
1400
1401     len = qemu_get_byte(f);
1402     qemu_get_buffer(f, (uint8_t *)id, len);
1403     id[len] = 0;
1404
1405     QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1406         if (!strncmp(id, block->idstr, sizeof(id)) &&
1407             block->max_length > offset) {
1408             return memory_region_get_ram_ptr(block->mr) + offset;
1409         }
1410     }
1411
1412     error_report("Can't find block %s!", id);
1413     return NULL;
1414 }
1415
1416 /*
1417  * If a page (or a whole RDMA chunk) has been
1418  * determined to be zero, then zap it.
1419  */
1420 void ram_handle_compressed(void *host, uint8_t ch, uint64_t size)
1421 {
1422     if (ch != 0 || !is_zero_range(host, size)) {
1423         memset(host, ch, size);
1424     }
1425 }
1426
1427 static void *do_data_decompress(void *opaque)
1428 {
1429     DecompressParam *param = opaque;
1430     unsigned long pagesize;
1431
1432     while (!quit_decomp_thread) {
1433         qemu_mutex_lock(&param->mutex);
1434         while (!param->start && !quit_decomp_thread) {
1435             qemu_cond_wait(&param->cond, &param->mutex);
1436             pagesize = TARGET_PAGE_SIZE;
1437             if (!quit_decomp_thread) {
1438                 /* uncompress() will return failed in some case, especially
1439                  * when the page is dirted when doing the compression, it's
1440                  * not a problem because the dirty page will be retransferred
1441                  * and uncompress() won't break the data in other pages.
1442                  */
1443                 uncompress((Bytef *)param->des, &pagesize,
1444                            (const Bytef *)param->compbuf, param->len);
1445             }
1446             param->start = false;
1447         }
1448         qemu_mutex_unlock(&param->mutex);
1449     }
1450
1451     return NULL;
1452 }
1453
1454 void migrate_decompress_threads_create(void)
1455 {
1456     int i, thread_count;
1457
1458     thread_count = migrate_decompress_threads();
1459     decompress_threads = g_new0(QemuThread, thread_count);
1460     decomp_param = g_new0(DecompressParam, thread_count);
1461     compressed_data_buf = g_malloc0(compressBound(TARGET_PAGE_SIZE));
1462     quit_decomp_thread = false;
1463     for (i = 0; i < thread_count; i++) {
1464         qemu_mutex_init(&decomp_param[i].mutex);
1465         qemu_cond_init(&decomp_param[i].cond);
1466         decomp_param[i].compbuf = g_malloc0(compressBound(TARGET_PAGE_SIZE));
1467         qemu_thread_create(decompress_threads + i, "decompress",
1468                            do_data_decompress, decomp_param + i,
1469                            QEMU_THREAD_JOINABLE);
1470     }
1471 }
1472
1473 void migrate_decompress_threads_join(void)
1474 {
1475     int i, thread_count;
1476
1477     quit_decomp_thread = true;
1478     thread_count = migrate_decompress_threads();
1479     for (i = 0; i < thread_count; i++) {
1480         qemu_mutex_lock(&decomp_param[i].mutex);
1481         qemu_cond_signal(&decomp_param[i].cond);
1482         qemu_mutex_unlock(&decomp_param[i].mutex);
1483     }
1484     for (i = 0; i < thread_count; i++) {
1485         qemu_thread_join(decompress_threads + i);
1486         qemu_mutex_destroy(&decomp_param[i].mutex);
1487         qemu_cond_destroy(&decomp_param[i].cond);
1488         g_free(decomp_param[i].compbuf);
1489     }
1490     g_free(decompress_threads);
1491     g_free(decomp_param);
1492     g_free(compressed_data_buf);
1493     decompress_threads = NULL;
1494     decomp_param = NULL;
1495     compressed_data_buf = NULL;
1496 }
1497
1498 static void decompress_data_with_multi_threads(uint8_t *compbuf,
1499                                                void *host, int len)
1500 {
1501     int idx, thread_count;
1502
1503     thread_count = migrate_decompress_threads();
1504     while (true) {
1505         for (idx = 0; idx < thread_count; idx++) {
1506             if (!decomp_param[idx].start) {
1507                 memcpy(decomp_param[idx].compbuf, compbuf, len);
1508                 decomp_param[idx].des = host;
1509                 decomp_param[idx].len = len;
1510                 start_decompression(&decomp_param[idx]);
1511                 break;
1512             }
1513         }
1514         if (idx < thread_count) {
1515             break;
1516         }
1517     }
1518 }
1519
1520 static int ram_load(QEMUFile *f, void *opaque, int version_id)
1521 {
1522     int flags = 0, ret = 0;
1523     static uint64_t seq_iter;
1524     int len = 0;
1525
1526     seq_iter++;
1527
1528     if (version_id != 4) {
1529         ret = -EINVAL;
1530     }
1531
1532     /* This RCU critical section can be very long running.
1533      * When RCU reclaims in the code start to become numerous,
1534      * it will be necessary to reduce the granularity of this
1535      * critical section.
1536      */
1537     rcu_read_lock();
1538     while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) {
1539         ram_addr_t addr, total_ram_bytes;
1540         void *host;
1541         uint8_t ch;
1542
1543         addr = qemu_get_be64(f);
1544         flags = addr & ~TARGET_PAGE_MASK;
1545         addr &= TARGET_PAGE_MASK;
1546
1547         switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
1548         case RAM_SAVE_FLAG_MEM_SIZE:
1549             /* Synchronize RAM block list */
1550             total_ram_bytes = addr;
1551             while (!ret && total_ram_bytes) {
1552                 RAMBlock *block;
1553                 uint8_t len;
1554                 char id[256];
1555                 ram_addr_t length;
1556
1557                 len = qemu_get_byte(f);
1558                 qemu_get_buffer(f, (uint8_t *)id, len);
1559                 id[len] = 0;
1560                 length = qemu_get_be64(f);
1561
1562                 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1563                     if (!strncmp(id, block->idstr, sizeof(id))) {
1564                         if (length != block->used_length) {
1565                             Error *local_err = NULL;
1566
1567                             ret = qemu_ram_resize(block->offset, length, &local_err);
1568                             if (local_err) {
1569                                 error_report_err(local_err);
1570                             }
1571                         }
1572                         break;
1573                     }
1574                 }
1575
1576                 if (!block) {
1577                     error_report("Unknown ramblock \"%s\", cannot "
1578                                  "accept migration", id);
1579                     ret = -EINVAL;
1580                 }
1581
1582                 total_ram_bytes -= length;
1583             }
1584             break;
1585         case RAM_SAVE_FLAG_COMPRESS:
1586             host = host_from_stream_offset(f, addr, flags);
1587             if (!host) {
1588                 error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
1589                 ret = -EINVAL;
1590                 break;
1591             }
1592             ch = qemu_get_byte(f);
1593             ram_handle_compressed(host, ch, TARGET_PAGE_SIZE);
1594             break;
1595         case RAM_SAVE_FLAG_PAGE:
1596             host = host_from_stream_offset(f, addr, flags);
1597             if (!host) {
1598                 error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
1599                 ret = -EINVAL;
1600                 break;
1601             }
1602             qemu_get_buffer(f, host, TARGET_PAGE_SIZE);
1603             break;
1604         case RAM_SAVE_FLAG_COMPRESS_PAGE:
1605             host = host_from_stream_offset(f, addr, flags);
1606             if (!host) {
1607                 error_report("Invalid RAM offset " RAM_ADDR_FMT, addr);
1608                 ret = -EINVAL;
1609                 break;
1610             }
1611
1612             len = qemu_get_be32(f);
1613             if (len < 0 || len > compressBound(TARGET_PAGE_SIZE)) {
1614                 error_report("Invalid compressed data length: %d", len);
1615                 ret = -EINVAL;
1616                 break;
1617             }
1618             qemu_get_buffer(f, compressed_data_buf, len);
1619             decompress_data_with_multi_threads(compressed_data_buf, host, len);
1620             break;
1621         case RAM_SAVE_FLAG_XBZRLE:
1622             host = host_from_stream_offset(f, addr, flags);
1623             if (!host) {
1624                 error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
1625                 ret = -EINVAL;
1626                 break;
1627             }
1628             if (load_xbzrle(f, addr, host) < 0) {
1629                 error_report("Failed to decompress XBZRLE page at "
1630                              RAM_ADDR_FMT, addr);
1631                 ret = -EINVAL;
1632                 break;
1633             }
1634             break;
1635         case RAM_SAVE_FLAG_EOS:
1636             /* normal exit */
1637             break;
1638         default:
1639             if (flags & RAM_SAVE_FLAG_HOOK) {
1640                 ram_control_load_hook(f, flags);
1641             } else {
1642                 error_report("Unknown combination of migration flags: %#x",
1643                              flags);
1644                 ret = -EINVAL;
1645             }
1646         }
1647         if (!ret) {
1648             ret = qemu_file_get_error(f);
1649         }
1650     }
1651
1652     rcu_read_unlock();
1653     DPRINTF("Completed load of VM with exit code %d seq iteration "
1654             "%" PRIu64 "\n", ret, seq_iter);
1655     return ret;
1656 }
1657
1658 static SaveVMHandlers savevm_ram_handlers = {
1659     .save_live_setup = ram_save_setup,
1660     .save_live_iterate = ram_save_iterate,
1661     .save_live_complete = ram_save_complete,
1662     .save_live_pending = ram_save_pending,
1663     .load_state = ram_load,
1664     .cancel = ram_migration_cancel,
1665 };
1666
1667 void ram_mig_init(void)
1668 {
1669     qemu_mutex_init(&XBZRLE.lock);
1670     register_savevm_live(NULL, "ram", 0, 4, &savevm_ram_handlers, NULL);
1671 }
1672
1673 struct soundhw {
1674     const char *name;
1675     const char *descr;
1676     int enabled;
1677     int isa;
1678     union {
1679         int (*init_isa) (ISABus *bus);
1680         int (*init_pci) (PCIBus *bus);
1681     } init;
1682 };
1683
1684 static struct soundhw soundhw[9];
1685 static int soundhw_count;
1686
1687 void isa_register_soundhw(const char *name, const char *descr,
1688                           int (*init_isa)(ISABus *bus))
1689 {
1690     assert(soundhw_count < ARRAY_SIZE(soundhw) - 1);
1691     soundhw[soundhw_count].name = name;
1692     soundhw[soundhw_count].descr = descr;
1693     soundhw[soundhw_count].isa = 1;
1694     soundhw[soundhw_count].init.init_isa = init_isa;
1695     soundhw_count++;
1696 }
1697
1698 void pci_register_soundhw(const char *name, const char *descr,
1699                           int (*init_pci)(PCIBus *bus))
1700 {
1701     assert(soundhw_count < ARRAY_SIZE(soundhw) - 1);
1702     soundhw[soundhw_count].name = name;
1703     soundhw[soundhw_count].descr = descr;
1704     soundhw[soundhw_count].isa = 0;
1705     soundhw[soundhw_count].init.init_pci = init_pci;
1706     soundhw_count++;
1707 }
1708
1709 void select_soundhw(const char *optarg)
1710 {
1711     struct soundhw *c;
1712
1713     if (is_help_option(optarg)) {
1714     show_valid_cards:
1715
1716         if (soundhw_count) {
1717              printf("Valid sound card names (comma separated):\n");
1718              for (c = soundhw; c->name; ++c) {
1719                  printf ("%-11s %s\n", c->name, c->descr);
1720              }
1721              printf("\n-soundhw all will enable all of the above\n");
1722         } else {
1723              printf("Machine has no user-selectable audio hardware "
1724                     "(it may or may not have always-present audio hardware).\n");
1725         }
1726         exit(!is_help_option(optarg));
1727     }
1728     else {
1729         size_t l;
1730         const char *p;
1731         char *e;
1732         int bad_card = 0;
1733
1734         if (!strcmp(optarg, "all")) {
1735             for (c = soundhw; c->name; ++c) {
1736                 c->enabled = 1;
1737             }
1738             return;
1739         }
1740
1741         p = optarg;
1742         while (*p) {
1743             e = strchr(p, ',');
1744             l = !e ? strlen(p) : (size_t) (e - p);
1745
1746             for (c = soundhw; c->name; ++c) {
1747                 if (!strncmp(c->name, p, l) && !c->name[l]) {
1748                     c->enabled = 1;
1749                     break;
1750                 }
1751             }
1752
1753             if (!c->name) {
1754                 if (l > 80) {
1755                     error_report("Unknown sound card name (too big to show)");
1756                 }
1757                 else {
1758                     error_report("Unknown sound card name `%.*s'",
1759                                  (int) l, p);
1760                 }
1761                 bad_card = 1;
1762             }
1763             p += l + (e != NULL);
1764         }
1765
1766         if (bad_card) {
1767             goto show_valid_cards;
1768         }
1769     }
1770 }
1771
1772 void audio_init(void)
1773 {
1774     struct soundhw *c;
1775     ISABus *isa_bus = (ISABus *) object_resolve_path_type("", TYPE_ISA_BUS, NULL);
1776     PCIBus *pci_bus = (PCIBus *) object_resolve_path_type("", TYPE_PCI_BUS, NULL);
1777
1778     for (c = soundhw; c->name; ++c) {
1779         if (c->enabled) {
1780             if (c->isa) {
1781                 if (!isa_bus) {
1782                     error_report("ISA bus not available for %s", c->name);
1783                     exit(1);
1784                 }
1785                 c->init.init_isa(isa_bus);
1786             } else {
1787                 if (!pci_bus) {
1788                     error_report("PCI bus not available for %s", c->name);
1789                     exit(1);
1790                 }
1791                 c->init.init_pci(pci_bus);
1792             }
1793         }
1794     }
1795 }
1796
1797 int qemu_uuid_parse(const char *str, uint8_t *uuid)
1798 {
1799     int ret;
1800
1801     if (strlen(str) != 36) {
1802         return -1;
1803     }
1804
1805     ret = sscanf(str, UUID_FMT, &uuid[0], &uuid[1], &uuid[2], &uuid[3],
1806                  &uuid[4], &uuid[5], &uuid[6], &uuid[7], &uuid[8], &uuid[9],
1807                  &uuid[10], &uuid[11], &uuid[12], &uuid[13], &uuid[14],
1808                  &uuid[15]);
1809
1810     if (ret != 16) {
1811         return -1;
1812     }
1813     return 0;
1814 }
1815
1816 void do_acpitable_option(const QemuOpts *opts)
1817 {
1818 #ifdef TARGET_I386
1819     Error *err = NULL;
1820
1821     acpi_table_add(opts, &err);
1822     if (err) {
1823         error_report("Wrong acpi table provided: %s",
1824                      error_get_pretty(err));
1825         error_free(err);
1826         exit(1);
1827     }
1828 #endif
1829 }
1830
1831 void do_smbios_option(QemuOpts *opts)
1832 {
1833 #ifdef TARGET_I386
1834     smbios_entry_add(opts);
1835 #endif
1836 }
1837
1838 void cpudef_init(void)
1839 {
1840 #if defined(cpudef_setup)
1841     cpudef_setup(); /* parse cpu definitions in target config file */
1842 #endif
1843 }
1844
1845 int kvm_available(void)
1846 {
1847 #ifdef CONFIG_KVM
1848     return 1;
1849 #else
1850     return 0;
1851 #endif
1852 }
1853
1854 int xen_available(void)
1855 {
1856 #ifdef CONFIG_XEN
1857     return 1;
1858 #else
1859     return 0;
1860 #endif
1861 }
1862
1863
1864 TargetInfo *qmp_query_target(Error **errp)
1865 {
1866     TargetInfo *info = g_malloc0(sizeof(*info));
1867
1868     info->arch = g_strdup(TARGET_NAME);
1869
1870     return info;
1871 }
1872
1873 /* Stub function that's gets run on the vcpu when its brought out of the
1874    VM to run inside qemu via async_run_on_cpu()*/
1875 static void mig_sleep_cpu(void *opq)
1876 {
1877     qemu_mutex_unlock_iothread();
1878     g_usleep(30*1000);
1879     qemu_mutex_lock_iothread();
1880 }
1881
1882 /* To reduce the dirty rate explicitly disallow the VCPUs from spending
1883    much time in the VM. The migration thread will try to catchup.
1884    Workload will experience a performance drop.
1885 */
1886 static void mig_throttle_guest_down(void)
1887 {
1888     CPUState *cpu;
1889
1890     qemu_mutex_lock_iothread();
1891     CPU_FOREACH(cpu) {
1892         async_run_on_cpu(cpu, mig_sleep_cpu, NULL);
1893     }
1894     qemu_mutex_unlock_iothread();
1895 }
1896
1897 static void check_guest_throttling(void)
1898 {
1899     static int64_t t0;
1900     int64_t        t1;
1901
1902     if (!mig_throttle_on) {
1903         return;
1904     }
1905
1906     if (!t0)  {
1907         t0 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
1908         return;
1909     }
1910
1911     t1 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
1912
1913     /* If it has been more than 40 ms since the last time the guest
1914      * was throttled then do it again.
1915      */
1916     if (40 < (t1-t0)/1000000) {
1917         mig_throttle_guest_down();
1918         t0 = t1;
1919     }
1920 }