#include "qemu/bitops.h"
#include "qemu/bitmap.h"
#include "qemu/main-loop.h"
+#include "qemu/pmem.h"
#include "xbzrle.h"
#include "ram.h"
#include "migration.h"
+#include "socket.h"
#include "migration/register.h"
#include "migration/misc.h"
#include "qemu-file.h"
#include "postcopy-ram.h"
-#include "migration/page_cache.h"
+#include "page_cache.h"
#include "qemu/error-report.h"
#include "qapi/error.h"
#include "qapi/qapi-events-migration.h"
#include "exec/target_page.h"
#include "qemu/rcu_queue.h"
#include "migration/colo.h"
-#include "migration/block.h"
+#include "block.h"
+#include "sysemu/sysemu.h"
+#include "qemu/uuid.h"
+#include "savevm.h"
+#include "qemu/iov.h"
/***********************************************************/
/* ram save/restore */
return ret;
}
+static bool ramblock_is_ignored(RAMBlock *block)
+{
+ return !qemu_ram_is_migratable(block) ||
+ (migrate_ignore_shared() && qemu_ram_is_shared(block));
+}
+
+/* Should be holding either ram_list.mutex, or the RCU lock. */
+#define RAMBLOCK_FOREACH_NOT_IGNORED(block) \
+ INTERNAL_RAMBLOCK_FOREACH(block) \
+ if (ramblock_is_ignored(block)) {} else
+
+#define RAMBLOCK_FOREACH_MIGRATABLE(block) \
+ INTERNAL_RAMBLOCK_FOREACH(block) \
+ if (!qemu_ram_is_migratable(block)) {} else
+
+#undef RAMBLOCK_FOREACH
+
+int foreach_not_ignored_block(RAMBlockIterFunc func, void *opaque)
+{
+ RAMBlock *block;
+ int ret = 0;
+
+ rcu_read_lock();
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
+ ret = func(block, opaque);
+ if (ret) {
+ break;
+ }
+ }
+ rcu_read_unlock();
+ return ret;
+}
+
static void ramblock_recv_map_init(void)
{
RAMBlock *rb;
- RAMBLOCK_FOREACH(rb) {
+ RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
assert(!rb->receivedmap);
rb->receivedmap = bitmap_new(rb->max_length >> qemu_target_page_bits());
}
rb->receivedmap);
}
+bool ramblock_recv_bitmap_test_byte_offset(RAMBlock *rb, uint64_t byte_offset)
+{
+ return test_bit(byte_offset >> TARGET_PAGE_BITS, rb->receivedmap);
+}
+
void ramblock_recv_bitmap_set(RAMBlock *rb, void *host_addr)
{
set_bit_atomic(ramblock_recv_bitmap_offset(host_addr, rb), rb->receivedmap);
nr);
}
+#define RAMBLOCK_RECV_BITMAP_ENDING (0x0123456789abcdefULL)
+
+/*
+ * Format: bitmap_size (8 bytes) + whole_bitmap (N bytes).
+ *
+ * Returns >0 if success with sent bytes, or <0 if error.
+ */
+int64_t ramblock_recv_bitmap_send(QEMUFile *file,
+ const char *block_name)
+{
+ RAMBlock *block = qemu_ram_block_by_name(block_name);
+ unsigned long *le_bitmap, nbits;
+ uint64_t size;
+
+ if (!block) {
+ error_report("%s: invalid block name: %s", __func__, block_name);
+ return -1;
+ }
+
+ nbits = block->used_length >> TARGET_PAGE_BITS;
+
+ /*
+ * Make sure the tmp bitmap buffer is big enough, e.g., on 32bit
+ * machines we may need 4 more bytes for padding (see below
+ * comment). So extend it a bit before hand.
+ */
+ le_bitmap = bitmap_new(nbits + BITS_PER_LONG);
+
+ /*
+ * Always use little endian when sending the bitmap. This is
+ * required that when source and destination VMs are not using the
+ * same endianess. (Note: big endian won't work.)
+ */
+ bitmap_to_le(le_bitmap, block->receivedmap, nbits);
+
+ /* Size of the bitmap, in bytes */
+ size = DIV_ROUND_UP(nbits, 8);
+
+ /*
+ * size is always aligned to 8 bytes for 64bit machines, but it
+ * may not be true for 32bit machines. We need this padding to
+ * make sure the migration can survive even between 32bit and
+ * 64bit machines.
+ */
+ size = ROUND_UP(size, 8);
+
+ qemu_put_be64(file, size);
+ qemu_put_buffer(file, (const uint8_t *)le_bitmap, size);
+ /*
+ * Mark as an end, in case the middle part is screwed up due to
+ * some "misterious" reason.
+ */
+ qemu_put_be64(file, RAMBLOCK_RECV_BITMAP_ENDING);
+ qemu_fflush(file);
+
+ g_free(le_bitmap);
+
+ if (qemu_file_get_error(file)) {
+ return qemu_file_get_error(file);
+ }
+
+ return size + sizeof(size);
+}
+
/*
* An outstanding page request, on the source, having been received
* and queued
uint32_t last_version;
/* We are in the first round */
bool ram_bulk_stage;
+ /* The free page optimization is enabled */
+ bool fpo_enabled;
/* How many times we have dirty too many pages */
int dirty_rate_high_cnt;
/* these variables are used for bitmap sync */
uint64_t num_dirty_pages_period;
/* xbzrle misses since the beginning of the period */
uint64_t xbzrle_cache_miss_prev;
- /* number of iterations at the beginning of period */
- uint64_t iterations_prev;
- /* Iterations since start */
- uint64_t iterations;
+
+ /* compression statistics since the beginning of the period */
+ /* amount of count that no free thread to compress data */
+ uint64_t compress_thread_busy_prev;
+ /* amount bytes after compression */
+ uint64_t compressed_size_prev;
+ /* amount of compressed pages */
+ uint64_t compress_pages_prev;
+
+ /* total handled target pages at the beginning of period */
+ uint64_t target_page_count_prev;
+ /* total handled target pages since start */
+ uint64_t target_page_count;
/* number of dirty bits in the bitmap */
uint64_t migration_dirty_pages;
- /* protects modification of the bitmap */
+ /* Protects modification of the bitmap and migration dirty pages */
QemuMutex bitmap_mutex;
/* The RAMBlock used in the last src_page_requests */
RAMBlock *last_req_rb;
/* Queue of outstanding page requests from the destination */
QemuMutex src_page_req_mutex;
- QSIMPLEQ_HEAD(src_page_requests, RAMSrcPageRequest) src_page_requests;
+ QSIMPLEQ_HEAD(, RAMSrcPageRequest) src_page_requests;
};
typedef struct RAMState RAMState;
static RAMState *ram_state;
+static NotifierWithReturnList precopy_notifier_list;
+
+void precopy_infrastructure_init(void)
+{
+ notifier_with_return_list_init(&precopy_notifier_list);
+}
+
+void precopy_add_notifier(NotifierWithReturn *n)
+{
+ notifier_with_return_list_add(&precopy_notifier_list, n);
+}
+
+void precopy_remove_notifier(NotifierWithReturn *n)
+{
+ notifier_with_return_remove(n);
+}
+
+int precopy_notify(PrecopyNotifyReason reason, Error **errp)
+{
+ PrecopyNotifyData pnd;
+ pnd.reason = reason;
+ pnd.errp = errp;
+
+ return notifier_with_return_list_notify(&precopy_notifier_list, &pnd);
+}
+
+void precopy_enable_free_page_optimization(void)
+{
+ if (!ram_state) {
+ return;
+ }
+
+ ram_state->fpo_enabled = true;
+}
+
uint64_t ram_bytes_remaining(void)
{
return ram_state ? (ram_state->migration_dirty_pages * TARGET_PAGE_SIZE) :
};
typedef struct PageSearchStatus PageSearchStatus;
+CompressionStats compression_counters;
+
struct CompressParam {
bool done;
bool quit;
+ bool zero_page;
QEMUFile *file;
QemuMutex mutex;
QemuCond cond;
RAMBlock *block;
ram_addr_t offset;
+
+ /* internally used fields */
+ z_stream stream;
+ uint8_t *originbuf;
};
typedef struct CompressParam CompressParam;
void *des;
uint8_t *compbuf;
int len;
+ z_stream stream;
};
typedef struct DecompressParam DecompressParam;
/* The empty QEMUFileOps will be used by file in CompressParam */
static const QEMUFileOps empty_ops = { };
+static QEMUFile *decomp_file;
static DecompressParam *decomp_param;
static QemuThread *decompress_threads;
static QemuMutex decomp_done_lock;
static QemuCond decomp_done_cond;
-static int do_compress_ram_page(QEMUFile *f, RAMBlock *block,
- ram_addr_t offset);
+static bool do_compress_ram_page(QEMUFile *f, z_stream *stream, RAMBlock *block,
+ ram_addr_t offset, uint8_t *source_buf);
static void *do_data_compress(void *opaque)
{
CompressParam *param = opaque;
RAMBlock *block;
ram_addr_t offset;
+ bool zero_page;
qemu_mutex_lock(¶m->mutex);
while (!param->quit) {
param->block = NULL;
qemu_mutex_unlock(¶m->mutex);
- do_compress_ram_page(param->file, block, offset);
+ zero_page = do_compress_ram_page(param->file, ¶m->stream,
+ block, offset, param->originbuf);
qemu_mutex_lock(&comp_done_lock);
param->done = true;
+ param->zero_page = zero_page;
qemu_cond_signal(&comp_done_cond);
qemu_mutex_unlock(&comp_done_lock);
return NULL;
}
-static inline void terminate_compression_threads(void)
-{
- int idx, thread_count;
-
- thread_count = migrate_compress_threads();
-
- for (idx = 0; idx < thread_count; idx++) {
- qemu_mutex_lock(&comp_param[idx].mutex);
- comp_param[idx].quit = true;
- qemu_cond_signal(&comp_param[idx].cond);
- qemu_mutex_unlock(&comp_param[idx].mutex);
- }
-}
-
static void compress_threads_save_cleanup(void)
{
int i, thread_count;
- if (!migrate_use_compression()) {
+ if (!migrate_use_compression() || !comp_param) {
return;
}
- terminate_compression_threads();
+
thread_count = migrate_compress_threads();
for (i = 0; i < thread_count; i++) {
+ /*
+ * we use it as a indicator which shows if the thread is
+ * properly init'd or not
+ */
+ if (!comp_param[i].file) {
+ break;
+ }
+
+ qemu_mutex_lock(&comp_param[i].mutex);
+ comp_param[i].quit = true;
+ qemu_cond_signal(&comp_param[i].cond);
+ qemu_mutex_unlock(&comp_param[i].mutex);
+
qemu_thread_join(compress_threads + i);
- qemu_fclose(comp_param[i].file);
qemu_mutex_destroy(&comp_param[i].mutex);
qemu_cond_destroy(&comp_param[i].cond);
+ deflateEnd(&comp_param[i].stream);
+ g_free(comp_param[i].originbuf);
+ qemu_fclose(comp_param[i].file);
+ comp_param[i].file = NULL;
}
qemu_mutex_destroy(&comp_done_lock);
qemu_cond_destroy(&comp_done_cond);
comp_param = NULL;
}
-static void compress_threads_save_setup(void)
+static int compress_threads_save_setup(void)
{
int i, thread_count;
if (!migrate_use_compression()) {
- return;
+ return 0;
}
thread_count = migrate_compress_threads();
compress_threads = g_new0(QemuThread, thread_count);
qemu_cond_init(&comp_done_cond);
qemu_mutex_init(&comp_done_lock);
for (i = 0; i < thread_count; i++) {
+ comp_param[i].originbuf = g_try_malloc(TARGET_PAGE_SIZE);
+ if (!comp_param[i].originbuf) {
+ goto exit;
+ }
+
+ if (deflateInit(&comp_param[i].stream,
+ migrate_compress_level()) != Z_OK) {
+ g_free(comp_param[i].originbuf);
+ goto exit;
+ }
+
/* comp_param[i].file is just used as a dummy buffer to save data,
* set its ops to empty.
*/
do_data_compress, comp_param + i,
QEMU_THREAD_JOINABLE);
}
+ return 0;
+
+exit:
+ compress_threads_save_cleanup();
+ return -1;
}
/* Multiple fd's */
-struct MultiFDSendParams {
+#define MULTIFD_MAGIC 0x11223344U
+#define MULTIFD_VERSION 1
+
+#define MULTIFD_FLAG_SYNC (1 << 0)
+
+/* This value needs to be a multiple of qemu_target_page_size() */
+#define MULTIFD_PACKET_SIZE (512 * 1024)
+
+typedef struct {
+ uint32_t magic;
+ uint32_t version;
+ unsigned char uuid[16]; /* QemuUUID */
uint8_t id;
+ uint8_t unused1[7]; /* Reserved for future use */
+ uint64_t unused2[4]; /* Reserved for future use */
+} __attribute__((packed)) MultiFDInit_t;
+
+typedef struct {
+ uint32_t magic;
+ uint32_t version;
+ uint32_t flags;
+ /* maximum number of allocated pages */
+ uint32_t pages_alloc;
+ uint32_t pages_used;
+ /* size of the next packet that contains pages */
+ uint32_t next_packet_size;
+ uint64_t packet_num;
+ uint64_t unused[4]; /* Reserved for future use */
+ char ramblock[256];
+ uint64_t offset[];
+} __attribute__((packed)) MultiFDPacket_t;
+
+typedef struct {
+ /* number of used pages */
+ uint32_t used;
+ /* number of allocated pages */
+ uint32_t allocated;
+ /* global number of generated multifd packets */
+ uint64_t packet_num;
+ /* offset of each page */
+ ram_addr_t *offset;
+ /* pointer to each page */
+ struct iovec *iov;
+ RAMBlock *block;
+} MultiFDPages_t;
+
+typedef struct {
+ /* this fields are not changed once the thread is created */
+ /* channel number */
+ uint8_t id;
+ /* channel thread name */
char *name;
+ /* channel thread id */
QemuThread thread;
+ /* communication channel */
+ QIOChannel *c;
+ /* sem where to wait for more work */
QemuSemaphore sem;
+ /* this mutex protects the following parameters */
QemuMutex mutex;
+ /* is this channel thread running */
+ bool running;
+ /* should this thread finish */
bool quit;
-};
-typedef struct MultiFDSendParams MultiFDSendParams;
+ /* thread has work to do */
+ int pending_job;
+ /* array of pages to sent */
+ MultiFDPages_t *pages;
+ /* packet allocated len */
+ uint32_t packet_len;
+ /* pointer to the packet */
+ MultiFDPacket_t *packet;
+ /* multifd flags for each packet */
+ uint32_t flags;
+ /* size of the next packet that contains pages */
+ uint32_t next_packet_size;
+ /* global number of generated multifd packets */
+ uint64_t packet_num;
+ /* thread local variables */
+ /* packets sent through this channel */
+ uint64_t num_packets;
+ /* pages sent through this channel */
+ uint64_t num_pages;
+} MultiFDSendParams;
+
+typedef struct {
+ /* this fields are not changed once the thread is created */
+ /* channel number */
+ uint8_t id;
+ /* channel thread name */
+ char *name;
+ /* channel thread id */
+ QemuThread thread;
+ /* communication channel */
+ QIOChannel *c;
+ /* this mutex protects the following parameters */
+ QemuMutex mutex;
+ /* is this channel thread running */
+ bool running;
+ /* should this thread finish */
+ bool quit;
+ /* array of pages to receive */
+ MultiFDPages_t *pages;
+ /* packet allocated len */
+ uint32_t packet_len;
+ /* pointer to the packet */
+ MultiFDPacket_t *packet;
+ /* multifd flags for each packet */
+ uint32_t flags;
+ /* global number of generated multifd packets */
+ uint64_t packet_num;
+ /* thread local variables */
+ /* size of the next packet that contains pages */
+ uint32_t next_packet_size;
+ /* packets sent through this channel */
+ uint64_t num_packets;
+ /* pages sent through this channel */
+ uint64_t num_pages;
+ /* syncs main thread and channels */
+ QemuSemaphore sem_sync;
+} MultiFDRecvParams;
+
+static int multifd_send_initial_packet(MultiFDSendParams *p, Error **errp)
+{
+ MultiFDInit_t msg;
+ int ret;
+
+ msg.magic = cpu_to_be32(MULTIFD_MAGIC);
+ msg.version = cpu_to_be32(MULTIFD_VERSION);
+ msg.id = p->id;
+ memcpy(msg.uuid, &qemu_uuid.data, sizeof(msg.uuid));
+
+ ret = qio_channel_write_all(p->c, (char *)&msg, sizeof(msg), errp);
+ if (ret != 0) {
+ return -1;
+ }
+ return 0;
+}
+
+static int multifd_recv_initial_packet(QIOChannel *c, Error **errp)
+{
+ MultiFDInit_t msg;
+ int ret;
+
+ ret = qio_channel_read_all(c, (char *)&msg, sizeof(msg), errp);
+ if (ret != 0) {
+ return -1;
+ }
+
+ msg.magic = be32_to_cpu(msg.magic);
+ msg.version = be32_to_cpu(msg.version);
+
+ if (msg.magic != MULTIFD_MAGIC) {
+ error_setg(errp, "multifd: received packet magic %x "
+ "expected %x", msg.magic, MULTIFD_MAGIC);
+ return -1;
+ }
+
+ if (msg.version != MULTIFD_VERSION) {
+ error_setg(errp, "multifd: received packet version %d "
+ "expected %d", msg.version, MULTIFD_VERSION);
+ return -1;
+ }
+
+ if (memcmp(msg.uuid, &qemu_uuid, sizeof(qemu_uuid))) {
+ char *uuid = qemu_uuid_unparse_strdup(&qemu_uuid);
+ char *msg_uuid = qemu_uuid_unparse_strdup((const QemuUUID *)msg.uuid);
+
+ error_setg(errp, "multifd: received uuid '%s' and expected "
+ "uuid '%s' for channel %hhd", msg_uuid, uuid, msg.id);
+ g_free(uuid);
+ g_free(msg_uuid);
+ return -1;
+ }
+
+ if (msg.id > migrate_multifd_channels()) {
+ error_setg(errp, "multifd: received channel version %d "
+ "expected %d", msg.version, MULTIFD_VERSION);
+ return -1;
+ }
+
+ return msg.id;
+}
+
+static MultiFDPages_t *multifd_pages_init(size_t size)
+{
+ MultiFDPages_t *pages = g_new0(MultiFDPages_t, 1);
+
+ pages->allocated = size;
+ pages->iov = g_new0(struct iovec, size);
+ pages->offset = g_new0(ram_addr_t, size);
+
+ return pages;
+}
+
+static void multifd_pages_clear(MultiFDPages_t *pages)
+{
+ pages->used = 0;
+ pages->allocated = 0;
+ pages->packet_num = 0;
+ pages->block = NULL;
+ g_free(pages->iov);
+ pages->iov = NULL;
+ g_free(pages->offset);
+ pages->offset = NULL;
+ g_free(pages);
+}
+
+static void multifd_send_fill_packet(MultiFDSendParams *p)
+{
+ MultiFDPacket_t *packet = p->packet;
+ uint32_t page_max = MULTIFD_PACKET_SIZE / qemu_target_page_size();
+ int i;
+
+ packet->magic = cpu_to_be32(MULTIFD_MAGIC);
+ packet->version = cpu_to_be32(MULTIFD_VERSION);
+ packet->flags = cpu_to_be32(p->flags);
+ packet->pages_alloc = cpu_to_be32(page_max);
+ packet->pages_used = cpu_to_be32(p->pages->used);
+ packet->next_packet_size = cpu_to_be32(p->next_packet_size);
+ packet->packet_num = cpu_to_be64(p->packet_num);
+
+ if (p->pages->block) {
+ strncpy(packet->ramblock, p->pages->block->idstr, 256);
+ }
+
+ for (i = 0; i < p->pages->used; i++) {
+ packet->offset[i] = cpu_to_be64(p->pages->offset[i]);
+ }
+}
+
+static int multifd_recv_unfill_packet(MultiFDRecvParams *p, Error **errp)
+{
+ MultiFDPacket_t *packet = p->packet;
+ uint32_t pages_max = MULTIFD_PACKET_SIZE / qemu_target_page_size();
+ RAMBlock *block;
+ int i;
+
+ packet->magic = be32_to_cpu(packet->magic);
+ if (packet->magic != MULTIFD_MAGIC) {
+ error_setg(errp, "multifd: received packet "
+ "magic %x and expected magic %x",
+ packet->magic, MULTIFD_MAGIC);
+ return -1;
+ }
+
+ packet->version = be32_to_cpu(packet->version);
+ if (packet->version != MULTIFD_VERSION) {
+ error_setg(errp, "multifd: received packet "
+ "version %d and expected version %d",
+ packet->version, MULTIFD_VERSION);
+ return -1;
+ }
+
+ p->flags = be32_to_cpu(packet->flags);
+
+ packet->pages_alloc = be32_to_cpu(packet->pages_alloc);
+ /*
+ * If we recevied a packet that is 100 times bigger than expected
+ * just stop migration. It is a magic number.
+ */
+ if (packet->pages_alloc > pages_max * 100) {
+ error_setg(errp, "multifd: received packet "
+ "with size %d and expected a maximum size of %d",
+ packet->pages_alloc, pages_max * 100) ;
+ return -1;
+ }
+ /*
+ * We received a packet that is bigger than expected but inside
+ * reasonable limits (see previous comment). Just reallocate.
+ */
+ if (packet->pages_alloc > p->pages->allocated) {
+ multifd_pages_clear(p->pages);
+ p->pages = multifd_pages_init(packet->pages_alloc);
+ }
+
+ p->pages->used = be32_to_cpu(packet->pages_used);
+ if (p->pages->used > packet->pages_alloc) {
+ error_setg(errp, "multifd: received packet "
+ "with %d pages and expected maximum pages are %d",
+ p->pages->used, packet->pages_alloc) ;
+ return -1;
+ }
+
+ p->next_packet_size = be32_to_cpu(packet->next_packet_size);
+ p->packet_num = be64_to_cpu(packet->packet_num);
+
+ if (p->pages->used) {
+ /* make sure that ramblock is 0 terminated */
+ packet->ramblock[255] = 0;
+ block = qemu_ram_block_by_name(packet->ramblock);
+ if (!block) {
+ error_setg(errp, "multifd: unknown ram block %s",
+ packet->ramblock);
+ return -1;
+ }
+ }
+
+ for (i = 0; i < p->pages->used; i++) {
+ ram_addr_t offset = be64_to_cpu(packet->offset[i]);
+
+ if (offset > (block->used_length - TARGET_PAGE_SIZE)) {
+ error_setg(errp, "multifd: offset too long " RAM_ADDR_FMT
+ " (max " RAM_ADDR_FMT ")",
+ offset, block->max_length);
+ return -1;
+ }
+ p->pages->iov[i].iov_base = block->host + offset;
+ p->pages->iov[i].iov_len = TARGET_PAGE_SIZE;
+ }
+
+ return 0;
+}
struct {
MultiFDSendParams *params;
- /* number of created threads */
- int count;
+ /* array of pages to sent */
+ MultiFDPages_t *pages;
+ /* syncs main thread and channels */
+ QemuSemaphore sem_sync;
+ /* global number of generated multifd packets */
+ uint64_t packet_num;
+ /* send channels ready */
+ QemuSemaphore channels_ready;
} *multifd_send_state;
-static void terminate_multifd_send_threads(Error *errp)
+/*
+ * How we use multifd_send_state->pages and channel->pages?
+ *
+ * We create a pages for each channel, and a main one. Each time that
+ * we need to send a batch of pages we interchange the ones between
+ * multifd_send_state and the channel that is sending it. There are
+ * two reasons for that:
+ * - to not have to do so many mallocs during migration
+ * - to make easier to know what to free at the end of migration
+ *
+ * This way we always know who is the owner of each "pages" struct,
+ * and we don't need any locking. It belongs to the migration thread
+ * or to the channel thread. Switching is safe because the migration
+ * thread is using the channel mutex when changing it, and the channel
+ * have to had finish with its own, otherwise pending_job can't be
+ * false.
+ */
+
+static int multifd_send_pages(RAMState *rs)
{
int i;
+ static int next_channel;
+ MultiFDSendParams *p = NULL; /* make happy gcc */
+ MultiFDPages_t *pages = multifd_send_state->pages;
+ uint64_t transferred;
+
+ qemu_sem_wait(&multifd_send_state->channels_ready);
+ for (i = next_channel;; i = (i + 1) % migrate_multifd_channels()) {
+ p = &multifd_send_state->params[i];
+
+ qemu_mutex_lock(&p->mutex);
+ if (p->quit) {
+ error_report("%s: channel %d has already quit!", __func__, i);
+ qemu_mutex_unlock(&p->mutex);
+ return -1;
+ }
+ if (!p->pending_job) {
+ p->pending_job++;
+ next_channel = (i + 1) % migrate_multifd_channels();
+ break;
+ }
+ qemu_mutex_unlock(&p->mutex);
+ }
+ p->pages->used = 0;
+
+ p->packet_num = multifd_send_state->packet_num++;
+ p->pages->block = NULL;
+ multifd_send_state->pages = p->pages;
+ p->pages = pages;
+ transferred = ((uint64_t) pages->used) * TARGET_PAGE_SIZE + p->packet_len;
+ qemu_file_update_transfer(rs->f, transferred);
+ ram_counters.multifd_bytes += transferred;
+ ram_counters.transferred += transferred;;
+ qemu_mutex_unlock(&p->mutex);
+ qemu_sem_post(&p->sem);
- for (i = 0; i < multifd_send_state->count; i++) {
+ return 1;
+}
+
+static int multifd_queue_page(RAMState *rs, RAMBlock *block, ram_addr_t offset)
+{
+ MultiFDPages_t *pages = multifd_send_state->pages;
+
+ if (!pages->block) {
+ pages->block = block;
+ }
+
+ if (pages->block == block) {
+ pages->offset[pages->used] = offset;
+ pages->iov[pages->used].iov_base = block->host + offset;
+ pages->iov[pages->used].iov_len = TARGET_PAGE_SIZE;
+ pages->used++;
+
+ if (pages->used < pages->allocated) {
+ return 1;
+ }
+ }
+
+ if (multifd_send_pages(rs) < 0) {
+ return -1;
+ }
+
+ if (pages->block != block) {
+ return multifd_queue_page(rs, block, offset);
+ }
+
+ return 1;
+}
+
+static void multifd_send_terminate_threads(Error *err)
+{
+ int i;
+
+ if (err) {
+ MigrationState *s = migrate_get_current();
+ migrate_set_error(s, err);
+ if (s->state == MIGRATION_STATUS_SETUP ||
+ s->state == MIGRATION_STATUS_PRE_SWITCHOVER ||
+ s->state == MIGRATION_STATUS_DEVICE ||
+ s->state == MIGRATION_STATUS_ACTIVE) {
+ migrate_set_state(&s->state, s->state,
+ MIGRATION_STATUS_FAILED);
+ }
+ }
+
+ for (i = 0; i < migrate_multifd_channels(); i++) {
MultiFDSendParams *p = &multifd_send_state->params[i];
qemu_mutex_lock(&p->mutex);
}
}
-int multifd_save_cleanup(Error **errp)
+void multifd_save_cleanup(void)
{
int i;
- int ret = 0;
if (!migrate_use_multifd()) {
- return 0;
+ return;
}
- terminate_multifd_send_threads(NULL);
- for (i = 0; i < multifd_send_state->count; i++) {
+ multifd_send_terminate_threads(NULL);
+ for (i = 0; i < migrate_multifd_channels(); i++) {
MultiFDSendParams *p = &multifd_send_state->params[i];
- qemu_thread_join(&p->thread);
+ if (p->running) {
+ qemu_thread_join(&p->thread);
+ }
+ socket_send_channel_destroy(p->c);
+ p->c = NULL;
qemu_mutex_destroy(&p->mutex);
qemu_sem_destroy(&p->sem);
g_free(p->name);
p->name = NULL;
+ multifd_pages_clear(p->pages);
+ p->pages = NULL;
+ p->packet_len = 0;
+ g_free(p->packet);
+ p->packet = NULL;
}
+ qemu_sem_destroy(&multifd_send_state->channels_ready);
+ qemu_sem_destroy(&multifd_send_state->sem_sync);
g_free(multifd_send_state->params);
multifd_send_state->params = NULL;
+ multifd_pages_clear(multifd_send_state->pages);
+ multifd_send_state->pages = NULL;
g_free(multifd_send_state);
multifd_send_state = NULL;
- return ret;
+}
+
+static void multifd_send_sync_main(RAMState *rs)
+{
+ int i;
+
+ if (!migrate_use_multifd()) {
+ return;
+ }
+ if (multifd_send_state->pages->used) {
+ if (multifd_send_pages(rs) < 0) {
+ error_report("%s: multifd_send_pages fail", __func__);
+ return;
+ }
+ }
+ for (i = 0; i < migrate_multifd_channels(); i++) {
+ MultiFDSendParams *p = &multifd_send_state->params[i];
+
+ trace_multifd_send_sync_main_signal(p->id);
+
+ qemu_mutex_lock(&p->mutex);
+
+ if (p->quit) {
+ error_report("%s: channel %d has already quit", __func__, i);
+ qemu_mutex_unlock(&p->mutex);
+ return;
+ }
+
+ p->packet_num = multifd_send_state->packet_num++;
+ p->flags |= MULTIFD_FLAG_SYNC;
+ p->pending_job++;
+ qemu_file_update_transfer(rs->f, p->packet_len);
+ ram_counters.multifd_bytes += p->packet_len;
+ ram_counters.transferred += p->packet_len;
+ qemu_mutex_unlock(&p->mutex);
+ qemu_sem_post(&p->sem);
+ }
+ for (i = 0; i < migrate_multifd_channels(); i++) {
+ MultiFDSendParams *p = &multifd_send_state->params[i];
+
+ trace_multifd_send_sync_main_wait(p->id);
+ qemu_sem_wait(&multifd_send_state->sem_sync);
+ }
+ trace_multifd_send_sync_main(multifd_send_state->packet_num);
}
static void *multifd_send_thread(void *opaque)
{
MultiFDSendParams *p = opaque;
+ Error *local_err = NULL;
+ int ret = 0;
+ uint32_t flags = 0;
+
+ trace_multifd_send_thread_start(p->id);
+ rcu_register_thread();
+
+ if (multifd_send_initial_packet(p, &local_err) < 0) {
+ goto out;
+ }
+ /* initial packet */
+ p->num_packets = 1;
while (true) {
+ qemu_sem_wait(&p->sem);
qemu_mutex_lock(&p->mutex);
- if (p->quit) {
+
+ if (p->pending_job) {
+ uint32_t used = p->pages->used;
+ uint64_t packet_num = p->packet_num;
+ flags = p->flags;
+
+ p->next_packet_size = used * qemu_target_page_size();
+ multifd_send_fill_packet(p);
+ p->flags = 0;
+ p->num_packets++;
+ p->num_pages += used;
+ p->pages->used = 0;
+ qemu_mutex_unlock(&p->mutex);
+
+ trace_multifd_send(p->id, packet_num, used, flags,
+ p->next_packet_size);
+
+ ret = qio_channel_write_all(p->c, (void *)p->packet,
+ p->packet_len, &local_err);
+ if (ret != 0) {
+ break;
+ }
+
+ if (used) {
+ ret = qio_channel_writev_all(p->c, p->pages->iov,
+ used, &local_err);
+ if (ret != 0) {
+ break;
+ }
+ }
+
+ qemu_mutex_lock(&p->mutex);
+ p->pending_job--;
+ qemu_mutex_unlock(&p->mutex);
+
+ if (flags & MULTIFD_FLAG_SYNC) {
+ qemu_sem_post(&multifd_send_state->sem_sync);
+ }
+ qemu_sem_post(&multifd_send_state->channels_ready);
+ } else if (p->quit) {
qemu_mutex_unlock(&p->mutex);
break;
+ } else {
+ qemu_mutex_unlock(&p->mutex);
+ /* sometimes there are spurious wakeups */
}
- qemu_mutex_unlock(&p->mutex);
- qemu_sem_wait(&p->sem);
}
+out:
+ if (local_err) {
+ multifd_send_terminate_threads(local_err);
+ }
+
+ /*
+ * Error happen, I will exit, but I can't just leave, tell
+ * who pay attention to me.
+ */
+ if (ret != 0) {
+ if (flags & MULTIFD_FLAG_SYNC) {
+ qemu_sem_post(&multifd_send_state->sem_sync);
+ }
+ qemu_sem_post(&multifd_send_state->channels_ready);
+ }
+
+ qemu_mutex_lock(&p->mutex);
+ p->running = false;
+ qemu_mutex_unlock(&p->mutex);
+
+ rcu_unregister_thread();
+ trace_multifd_send_thread_end(p->id, p->num_packets, p->num_pages);
+
return NULL;
}
+static void multifd_new_send_channel_async(QIOTask *task, gpointer opaque)
+{
+ MultiFDSendParams *p = opaque;
+ QIOChannel *sioc = QIO_CHANNEL(qio_task_get_source(task));
+ Error *local_err = NULL;
+
+ if (qio_task_propagate_error(task, &local_err)) {
+ migrate_set_error(migrate_get_current(), local_err);
+ multifd_save_cleanup();
+ } else {
+ p->c = QIO_CHANNEL(sioc);
+ qio_channel_set_delay(p->c, false);
+ p->running = true;
+ qemu_thread_create(&p->thread, p->name, multifd_send_thread, p,
+ QEMU_THREAD_JOINABLE);
+ }
+}
+
int multifd_save_setup(void)
{
int thread_count;
+ uint32_t page_count = MULTIFD_PACKET_SIZE / qemu_target_page_size();
uint8_t i;
if (!migrate_use_multifd()) {
thread_count = migrate_multifd_channels();
multifd_send_state = g_malloc0(sizeof(*multifd_send_state));
multifd_send_state->params = g_new0(MultiFDSendParams, thread_count);
- multifd_send_state->count = 0;
+ multifd_send_state->pages = multifd_pages_init(page_count);
+ qemu_sem_init(&multifd_send_state->sem_sync, 0);
+ qemu_sem_init(&multifd_send_state->channels_ready, 0);
+
for (i = 0; i < thread_count; i++) {
MultiFDSendParams *p = &multifd_send_state->params[i];
qemu_mutex_init(&p->mutex);
qemu_sem_init(&p->sem, 0);
p->quit = false;
+ p->pending_job = 0;
p->id = i;
+ p->pages = multifd_pages_init(page_count);
+ p->packet_len = sizeof(MultiFDPacket_t)
+ + sizeof(ram_addr_t) * page_count;
+ p->packet = g_malloc0(p->packet_len);
p->name = g_strdup_printf("multifdsend_%d", i);
- qemu_thread_create(&p->thread, p->name, multifd_send_thread, p,
- QEMU_THREAD_JOINABLE);
-
- multifd_send_state->count++;
+ socket_send_channel_create(multifd_new_send_channel_async, p);
}
return 0;
}
-struct MultiFDRecvParams {
- uint8_t id;
- char *name;
- QemuThread thread;
- QemuSemaphore sem;
- QemuMutex mutex;
- bool quit;
-};
-typedef struct MultiFDRecvParams MultiFDRecvParams;
-
struct {
MultiFDRecvParams *params;
/* number of created threads */
int count;
+ /* syncs main thread and channels */
+ QemuSemaphore sem_sync;
+ /* global number of generated multifd packets */
+ uint64_t packet_num;
} *multifd_recv_state;
-static void terminate_multifd_recv_threads(Error *errp)
+static void multifd_recv_terminate_threads(Error *err)
{
int i;
- for (i = 0; i < multifd_recv_state->count; i++) {
+ if (err) {
+ MigrationState *s = migrate_get_current();
+ migrate_set_error(s, err);
+ if (s->state == MIGRATION_STATUS_SETUP ||
+ s->state == MIGRATION_STATUS_ACTIVE) {
+ migrate_set_state(&s->state, s->state,
+ MIGRATION_STATUS_FAILED);
+ }
+ }
+
+ for (i = 0; i < migrate_multifd_channels(); i++) {
MultiFDRecvParams *p = &multifd_recv_state->params[i];
qemu_mutex_lock(&p->mutex);
p->quit = true;
- qemu_sem_post(&p->sem);
+ /* We could arrive here for two reasons:
+ - normal quit, i.e. everything went fine, just finished
+ - error quit: We close the channels so the channel threads
+ finish the qio_channel_read_all_eof() */
+ qio_channel_shutdown(p->c, QIO_CHANNEL_SHUTDOWN_BOTH, NULL);
qemu_mutex_unlock(&p->mutex);
}
}
if (!migrate_use_multifd()) {
return 0;
}
- terminate_multifd_recv_threads(NULL);
- for (i = 0; i < multifd_recv_state->count; i++) {
+ multifd_recv_terminate_threads(NULL);
+ for (i = 0; i < migrate_multifd_channels(); i++) {
MultiFDRecvParams *p = &multifd_recv_state->params[i];
- qemu_thread_join(&p->thread);
+ if (p->running) {
+ p->quit = true;
+ /*
+ * multifd_recv_thread may hung at MULTIFD_FLAG_SYNC handle code,
+ * however try to wakeup it without harm in cleanup phase.
+ */
+ qemu_sem_post(&p->sem_sync);
+ qemu_thread_join(&p->thread);
+ }
+ object_unref(OBJECT(p->c));
+ p->c = NULL;
qemu_mutex_destroy(&p->mutex);
- qemu_sem_destroy(&p->sem);
+ qemu_sem_destroy(&p->sem_sync);
g_free(p->name);
p->name = NULL;
+ multifd_pages_clear(p->pages);
+ p->pages = NULL;
+ p->packet_len = 0;
+ g_free(p->packet);
+ p->packet = NULL;
}
+ qemu_sem_destroy(&multifd_recv_state->sem_sync);
g_free(multifd_recv_state->params);
multifd_recv_state->params = NULL;
g_free(multifd_recv_state);
return ret;
}
+static void multifd_recv_sync_main(void)
+{
+ int i;
+
+ if (!migrate_use_multifd()) {
+ return;
+ }
+ for (i = 0; i < migrate_multifd_channels(); i++) {
+ MultiFDRecvParams *p = &multifd_recv_state->params[i];
+
+ trace_multifd_recv_sync_main_wait(p->id);
+ qemu_sem_wait(&multifd_recv_state->sem_sync);
+ }
+ for (i = 0; i < migrate_multifd_channels(); i++) {
+ MultiFDRecvParams *p = &multifd_recv_state->params[i];
+
+ qemu_mutex_lock(&p->mutex);
+ if (multifd_recv_state->packet_num < p->packet_num) {
+ multifd_recv_state->packet_num = p->packet_num;
+ }
+ qemu_mutex_unlock(&p->mutex);
+ trace_multifd_recv_sync_main_signal(p->id);
+ qemu_sem_post(&p->sem_sync);
+ }
+ trace_multifd_recv_sync_main(multifd_recv_state->packet_num);
+}
+
static void *multifd_recv_thread(void *opaque)
{
MultiFDRecvParams *p = opaque;
+ Error *local_err = NULL;
+ int ret;
+
+ trace_multifd_recv_thread_start(p->id);
+ rcu_register_thread();
while (true) {
- qemu_mutex_lock(&p->mutex);
+ uint32_t used;
+ uint32_t flags;
+
if (p->quit) {
+ break;
+ }
+
+ ret = qio_channel_read_all_eof(p->c, (void *)p->packet,
+ p->packet_len, &local_err);
+ if (ret == 0) { /* EOF */
+ break;
+ }
+ if (ret == -1) { /* Error */
+ break;
+ }
+
+ qemu_mutex_lock(&p->mutex);
+ ret = multifd_recv_unfill_packet(p, &local_err);
+ if (ret) {
qemu_mutex_unlock(&p->mutex);
break;
}
+
+ used = p->pages->used;
+ flags = p->flags;
+ trace_multifd_recv(p->id, p->packet_num, used, flags,
+ p->next_packet_size);
+ p->num_packets++;
+ p->num_pages += used;
qemu_mutex_unlock(&p->mutex);
- qemu_sem_wait(&p->sem);
+
+ if (used) {
+ ret = qio_channel_readv_all(p->c, p->pages->iov,
+ used, &local_err);
+ if (ret != 0) {
+ break;
+ }
+ }
+
+ if (flags & MULTIFD_FLAG_SYNC) {
+ qemu_sem_post(&multifd_recv_state->sem_sync);
+ qemu_sem_wait(&p->sem_sync);
+ }
+ }
+
+ if (local_err) {
+ multifd_recv_terminate_threads(local_err);
}
+ qemu_mutex_lock(&p->mutex);
+ p->running = false;
+ qemu_mutex_unlock(&p->mutex);
+
+ rcu_unregister_thread();
+ trace_multifd_recv_thread_end(p->id, p->num_packets, p->num_pages);
return NULL;
}
int multifd_load_setup(void)
{
int thread_count;
+ uint32_t page_count = MULTIFD_PACKET_SIZE / qemu_target_page_size();
uint8_t i;
if (!migrate_use_multifd()) {
thread_count = migrate_multifd_channels();
multifd_recv_state = g_malloc0(sizeof(*multifd_recv_state));
multifd_recv_state->params = g_new0(MultiFDRecvParams, thread_count);
- multifd_recv_state->count = 0;
+ atomic_set(&multifd_recv_state->count, 0);
+ qemu_sem_init(&multifd_recv_state->sem_sync, 0);
+
for (i = 0; i < thread_count; i++) {
MultiFDRecvParams *p = &multifd_recv_state->params[i];
qemu_mutex_init(&p->mutex);
- qemu_sem_init(&p->sem, 0);
+ qemu_sem_init(&p->sem_sync, 0);
p->quit = false;
p->id = i;
+ p->pages = multifd_pages_init(page_count);
+ p->packet_len = sizeof(MultiFDPacket_t)
+ + sizeof(ram_addr_t) * page_count;
+ p->packet = g_malloc0(p->packet_len);
p->name = g_strdup_printf("multifdrecv_%d", i);
- qemu_thread_create(&p->thread, p->name, multifd_recv_thread, p,
- QEMU_THREAD_JOINABLE);
- multifd_recv_state->count++;
}
return 0;
}
+bool multifd_recv_all_channels_created(void)
+{
+ int thread_count = migrate_multifd_channels();
+
+ if (!migrate_use_multifd()) {
+ return true;
+ }
+
+ return thread_count == atomic_read(&multifd_recv_state->count);
+}
+
+/*
+ * Try to receive all multifd channels to get ready for the migration.
+ * - Return true and do not set @errp when correctly receving all channels;
+ * - Return false and do not set @errp when correctly receiving the current one;
+ * - Return false and set @errp when failing to receive the current channel.
+ */
+bool multifd_recv_new_channel(QIOChannel *ioc, Error **errp)
+{
+ MultiFDRecvParams *p;
+ Error *local_err = NULL;
+ int id;
+
+ id = multifd_recv_initial_packet(ioc, &local_err);
+ if (id < 0) {
+ multifd_recv_terminate_threads(local_err);
+ error_propagate_prepend(errp, local_err,
+ "failed to receive packet"
+ " via multifd channel %d: ",
+ atomic_read(&multifd_recv_state->count));
+ return false;
+ }
+
+ p = &multifd_recv_state->params[id];
+ if (p->c != NULL) {
+ error_setg(&local_err, "multifd: received id '%d' already setup'",
+ id);
+ multifd_recv_terminate_threads(local_err);
+ error_propagate(errp, local_err);
+ return false;
+ }
+ p->c = ioc;
+ object_ref(OBJECT(ioc));
+ /* initial packet */
+ p->num_packets = 1;
+
+ p->running = true;
+ qemu_thread_create(&p->thread, p->name, multifd_recv_thread, p,
+ QEMU_THREAD_JOINABLE);
+ atomic_inc(&multifd_recv_state->count);
+ return atomic_read(&multifd_recv_state->count) ==
+ migrate_multifd_channels();
+}
+
/**
* save_page_header: write page header to wire
*
MigrationState *s = migrate_get_current();
uint64_t pct_initial = s->parameters.cpu_throttle_initial;
uint64_t pct_icrement = s->parameters.cpu_throttle_increment;
+ int pct_max = s->parameters.max_cpu_throttle;
/* We have not started throttling yet. Let's start it. */
if (!cpu_throttle_active()) {
cpu_throttle_set(pct_initial);
} else {
/* Throttling already on, just increase the rate */
- cpu_throttle_set(cpu_throttle_get_percentage() + pct_icrement);
+ cpu_throttle_set(MIN(cpu_throttle_get_percentage() + pct_icrement,
+ pct_max));
}
}
encoded_len = xbzrle_encode_buffer(prev_cached_page, XBZRLE.current_buf,
TARGET_PAGE_SIZE, XBZRLE.encoded_buf,
TARGET_PAGE_SIZE);
+
+ /*
+ * Update the cache contents, so that it corresponds to the data
+ * sent, in all cases except where we skip the page.
+ */
+ if (!last_stage && encoded_len != 0) {
+ memcpy(prev_cached_page, XBZRLE.current_buf, TARGET_PAGE_SIZE);
+ /*
+ * In the case where we couldn't compress, ensure that the caller
+ * sends the data from the cache, since the guest might have
+ * changed the RAM since we copied it.
+ */
+ *current_data = prev_cached_page;
+ }
+
if (encoded_len == 0) {
trace_save_xbzrle_page_skipping();
return 0;
} else if (encoded_len == -1) {
trace_save_xbzrle_page_overflow();
xbzrle_counters.overflow++;
- /* update data in the cache */
- if (!last_stage) {
- memcpy(prev_cached_page, *current_data, TARGET_PAGE_SIZE);
- *current_data = prev_cached_page;
- }
return -1;
}
- /* we need to update the data in the cache, in order to get the same data */
- if (!last_stage) {
- memcpy(prev_cached_page, XBZRLE.current_buf, TARGET_PAGE_SIZE);
- }
-
/* Send XBZRLE based compressed page */
bytes_xbzrle = save_page_header(rs, rs->f, block,
offset | RAM_SAVE_FLAG_XBZRLE);
/**
* migration_bitmap_find_dirty: find the next dirty page from start
*
- * Called with rcu_read_lock() to protect migration_bitmap
- *
- * Returns the byte offset within memory region of the start of a dirty page
+ * Returns the page offset within memory region of the start of a dirty page
*
* @rs: current RAM state
* @rb: RAMBlock where to search for dirty pages
unsigned long *bitmap = rb->bmap;
unsigned long next;
- if (rs->ram_bulk_stage && start > 0) {
+ if (ramblock_is_ignored(rb)) {
+ return size;
+ }
+
+ /*
+ * When the free page optimization is enabled, we need to check the bitmap
+ * to send the non-free pages rather than all the pages in the bulk stage.
+ */
+ if (!rs->fpo_enabled && rs->ram_bulk_stage && start > 0) {
next = start + 1;
} else {
next = find_next_bit(bitmap, size, start);
{
bool ret;
+ qemu_mutex_lock(&rs->bitmap_mutex);
+
+ /*
+ * Clear dirty bitmap if needed. This _must_ be called before we
+ * send any of the page in the chunk because we need to make sure
+ * we can capture further page content changes when we sync dirty
+ * log the next time. So as long as we are going to send any of
+ * the page in the chunk we clear the remote dirty bitmap for all.
+ * Clearing it earlier won't be a problem, but too late will.
+ */
+ if (rb->clear_bmap && clear_bmap_test_and_clear(rb, page)) {
+ uint8_t shift = rb->clear_bmap_shift;
+ hwaddr size = 1ULL << (TARGET_PAGE_BITS + shift);
+ hwaddr start = (page << TARGET_PAGE_BITS) & (-size);
+
+ /*
+ * CLEAR_BITMAP_SHIFT_MIN should always guarantee this... this
+ * can make things easier sometimes since then start address
+ * of the small chunk will always be 64 pages aligned so the
+ * bitmap will always be aligned to unsigned long. We should
+ * even be able to remove this restriction but I'm simply
+ * keeping it.
+ */
+ assert(shift >= 6);
+ trace_migration_bitmap_clear_dirty(rb->idstr, start, size, page);
+ memory_region_clear_dirty_bitmap(rb->mr, start, size);
+ }
+
ret = test_and_clear_bit(page, rb->bmap);
if (ret) {
rs->migration_dirty_pages--;
}
+ qemu_mutex_unlock(&rs->bitmap_mutex);
+
return ret;
}
-static void migration_bitmap_sync_range(RAMState *rs, RAMBlock *rb,
- ram_addr_t start, ram_addr_t length)
+/* Called with RCU critical section */
+static void ramblock_sync_dirty_bitmap(RAMState *rs, RAMBlock *rb)
{
rs->migration_dirty_pages +=
- cpu_physical_memory_sync_dirty_bitmap(rb, start, length,
+ cpu_physical_memory_sync_dirty_bitmap(rb, 0, rb->used_length,
&rs->num_dirty_pages_period);
}
RAMBlock *block;
uint64_t summary = 0;
- RAMBLOCK_FOREACH(block) {
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
summary |= block->page_size;
}
return summary;
}
+uint64_t ram_get_total_transferred_pages(void)
+{
+ return ram_counters.normal + ram_counters.duplicate +
+ compression_counters.pages + xbzrle_counters.pages;
+}
+
+static void migration_update_rates(RAMState *rs, int64_t end_time)
+{
+ uint64_t page_count = rs->target_page_count - rs->target_page_count_prev;
+ double compressed_size;
+
+ /* calculate period counters */
+ ram_counters.dirty_pages_rate = rs->num_dirty_pages_period * 1000
+ / (end_time - rs->time_last_bitmap_sync);
+
+ if (!page_count) {
+ return;
+ }
+
+ if (migrate_use_xbzrle()) {
+ xbzrle_counters.cache_miss_rate = (double)(xbzrle_counters.cache_miss -
+ rs->xbzrle_cache_miss_prev) / page_count;
+ rs->xbzrle_cache_miss_prev = xbzrle_counters.cache_miss;
+ }
+
+ if (migrate_use_compression()) {
+ compression_counters.busy_rate = (double)(compression_counters.busy -
+ rs->compress_thread_busy_prev) / page_count;
+ rs->compress_thread_busy_prev = compression_counters.busy;
+
+ compressed_size = compression_counters.compressed_size -
+ rs->compressed_size_prev;
+ if (compressed_size) {
+ double uncompressed_size = (compression_counters.pages -
+ rs->compress_pages_prev) * TARGET_PAGE_SIZE;
+
+ /* Compression-Ratio = Uncompressed-size / Compressed-size */
+ compression_counters.compression_rate =
+ uncompressed_size / compressed_size;
+
+ rs->compress_pages_prev = compression_counters.pages;
+ rs->compressed_size_prev = compression_counters.compressed_size;
+ }
+ }
+}
+
static void migration_bitmap_sync(RAMState *rs)
{
RAMBlock *block;
qemu_mutex_lock(&rs->bitmap_mutex);
rcu_read_lock();
- RAMBLOCK_FOREACH(block) {
- migration_bitmap_sync_range(rs, block, 0, block->used_length);
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
+ ramblock_sync_dirty_bitmap(rs, block);
}
+ ram_counters.remaining = ram_bytes_remaining();
rcu_read_unlock();
qemu_mutex_unlock(&rs->bitmap_mutex);
/* more than 1 second = 1000 millisecons */
if (end_time > rs->time_last_bitmap_sync + 1000) {
- /* calculate period counters */
- ram_counters.dirty_pages_rate = rs->num_dirty_pages_period * 1000
- / (end_time - rs->time_last_bitmap_sync);
bytes_xfer_now = ram_counters.transferred;
/* During block migration the auto-converge logic incorrectly detects
}
}
- if (migrate_use_xbzrle()) {
- if (rs->iterations_prev != rs->iterations) {
- xbzrle_counters.cache_miss_rate =
- (double)(xbzrle_counters.cache_miss -
- rs->xbzrle_cache_miss_prev) /
- (rs->iterations - rs->iterations_prev);
- }
- rs->iterations_prev = rs->iterations;
- rs->xbzrle_cache_miss_prev = xbzrle_counters.cache_miss;
- }
+ migration_update_rates(rs, end_time);
+
+ rs->target_page_count_prev = rs->target_page_count;
/* reset period counters */
rs->time_last_bitmap_sync = end_time;
rs->bytes_xfer_prev = bytes_xfer_now;
}
if (migrate_use_events()) {
- qapi_event_send_migration_pass(ram_counters.dirty_sync_count, NULL);
+ qapi_event_send_migration_pass(ram_counters.dirty_sync_count);
+ }
+}
+
+static void migration_bitmap_sync_precopy(RAMState *rs)
+{
+ Error *local_err = NULL;
+
+ /*
+ * The current notifier usage is just an optimization to migration, so we
+ * don't stop the normal migration process in the error case.
+ */
+ if (precopy_notify(PRECOPY_NOTIFY_BEFORE_BITMAP_SYNC, &local_err)) {
+ error_report_err(local_err);
+ }
+
+ migration_bitmap_sync(rs);
+
+ if (precopy_notify(PRECOPY_NOTIFY_AFTER_BITMAP_SYNC, &local_err)) {
+ error_report_err(local_err);
+ }
+}
+
+/**
+ * save_zero_page_to_file: send the zero page to the file
+ *
+ * Returns the size of data written to the file, 0 means the page is not
+ * a zero page
+ *
+ * @rs: current RAM state
+ * @file: the file where the data is saved
+ * @block: block that contains the page we want to send
+ * @offset: offset inside the block for the page
+ */
+static int save_zero_page_to_file(RAMState *rs, QEMUFile *file,
+ RAMBlock *block, ram_addr_t offset)
+{
+ uint8_t *p = block->host + offset;
+ int len = 0;
+
+ if (is_zero_range(p, TARGET_PAGE_SIZE)) {
+ len += save_page_header(rs, file, block, offset | RAM_SAVE_FLAG_ZERO);
+ qemu_put_byte(file, 0);
+ len += 1;
}
+ return len;
}
/**
*/
static int save_zero_page(RAMState *rs, RAMBlock *block, ram_addr_t offset)
{
- uint8_t *p = block->host + offset;
- int pages = -1;
+ int len = save_zero_page_to_file(rs, rs->f, block, offset);
- if (is_zero_range(p, TARGET_PAGE_SIZE)) {
+ if (len) {
ram_counters.duplicate++;
- ram_counters.transferred +=
- save_page_header(rs, rs->f, block, offset | RAM_SAVE_FLAG_ZERO);
- qemu_put_byte(rs->f, 0);
- ram_counters.transferred += 1;
- pages = 1;
+ ram_counters.transferred += len;
+ return 1;
}
-
- return pages;
+ return -1;
}
static void ram_release_pages(const char *rbname, uint64_t offset, int pages)
ram_discard_range(rbname, offset, pages << TARGET_PAGE_BITS);
}
+/*
+ * @pages: the number of pages written by the control path,
+ * < 0 - error
+ * > 0 - number of pages written
+ *
+ * Return true if the pages has been saved, otherwise false is returned.
+ */
+static bool control_save_page(RAMState *rs, RAMBlock *block, ram_addr_t offset,
+ int *pages)
+{
+ uint64_t bytes_xmit = 0;
+ int ret;
+
+ *pages = -1;
+ ret = ram_control_save_page(rs->f, block->offset, offset, TARGET_PAGE_SIZE,
+ &bytes_xmit);
+ if (ret == RAM_SAVE_CONTROL_NOT_SUPP) {
+ return false;
+ }
+
+ if (bytes_xmit) {
+ ram_counters.transferred += bytes_xmit;
+ *pages = 1;
+ }
+
+ if (ret == RAM_SAVE_CONTROL_DELAYED) {
+ return true;
+ }
+
+ if (bytes_xmit > 0) {
+ ram_counters.normal++;
+ } else if (bytes_xmit == 0) {
+ ram_counters.duplicate++;
+ }
+
+ return true;
+}
+
+/*
+ * directly send the page to the stream
+ *
+ * Returns the number of pages written.
+ *
+ * @rs: current RAM state
+ * @block: block that contains the page we want to send
+ * @offset: offset inside the block for the page
+ * @buf: the page to be sent
+ * @async: send to page asyncly
+ */
+static int save_normal_page(RAMState *rs, RAMBlock *block, ram_addr_t offset,
+ uint8_t *buf, bool async)
+{
+ ram_counters.transferred += save_page_header(rs, rs->f, block,
+ offset | RAM_SAVE_FLAG_PAGE);
+ if (async) {
+ qemu_put_buffer_async(rs->f, buf, TARGET_PAGE_SIZE,
+ migrate_release_ram() &
+ migration_in_postcopy());
+ } else {
+ qemu_put_buffer(rs->f, buf, TARGET_PAGE_SIZE);
+ }
+ ram_counters.transferred += TARGET_PAGE_SIZE;
+ ram_counters.normal++;
+ return 1;
+}
+
/**
* ram_save_page: send the given page to the stream
*
static int ram_save_page(RAMState *rs, PageSearchStatus *pss, bool last_stage)
{
int pages = -1;
- uint64_t bytes_xmit;
- ram_addr_t current_addr;
uint8_t *p;
- int ret;
bool send_async = true;
RAMBlock *block = pss->block;
ram_addr_t offset = pss->page << TARGET_PAGE_BITS;
+ ram_addr_t current_addr = block->offset + offset;
p = block->host + offset;
trace_ram_save_page(block->idstr, (uint64_t)offset, p);
- /* In doubt sent page as normal */
- bytes_xmit = 0;
- ret = ram_control_save_page(rs->f, block->offset,
- offset, TARGET_PAGE_SIZE, &bytes_xmit);
- if (bytes_xmit) {
- ram_counters.transferred += bytes_xmit;
- pages = 1;
- }
-
XBZRLE_cache_lock();
-
- current_addr = block->offset + offset;
-
- if (ret != RAM_SAVE_CONTROL_NOT_SUPP) {
- if (ret != RAM_SAVE_CONTROL_DELAYED) {
- if (bytes_xmit > 0) {
- ram_counters.normal++;
- } else if (bytes_xmit == 0) {
- ram_counters.duplicate++;
- }
- }
- } else {
- pages = save_zero_page(rs, block, offset);
- if (pages > 0) {
- /* Must let xbzrle know, otherwise a previous (now 0'd) cached
- * page would be stale
+ if (!rs->ram_bulk_stage && !migration_in_postcopy() &&
+ migrate_use_xbzrle()) {
+ pages = save_xbzrle_page(rs, &p, current_addr, block,
+ offset, last_stage);
+ if (!last_stage) {
+ /* Can't send this cached data async, since the cache page
+ * might get updated before it gets to the wire
*/
- xbzrle_cache_zero_page(rs, current_addr);
- ram_release_pages(block->idstr, offset, pages);
- } else if (!rs->ram_bulk_stage &&
- !migration_in_postcopy() && migrate_use_xbzrle()) {
- pages = save_xbzrle_page(rs, &p, current_addr, block,
- offset, last_stage);
- if (!last_stage) {
- /* Can't send this cached data async, since the cache page
- * might get updated before it gets to the wire
- */
- send_async = false;
- }
+ send_async = false;
}
}
/* XBZRLE overflow or normal page */
if (pages == -1) {
- ram_counters.transferred +=
- save_page_header(rs, rs->f, block, offset | RAM_SAVE_FLAG_PAGE);
- if (send_async) {
- qemu_put_buffer_async(rs->f, p, TARGET_PAGE_SIZE,
- migrate_release_ram() &
- migration_in_postcopy());
- } else {
- qemu_put_buffer(rs->f, p, TARGET_PAGE_SIZE);
- }
- ram_counters.transferred += TARGET_PAGE_SIZE;
- pages = 1;
- ram_counters.normal++;
+ pages = save_normal_page(rs, block, offset, p, send_async);
}
XBZRLE_cache_unlock();
return pages;
}
-static int do_compress_ram_page(QEMUFile *f, RAMBlock *block,
- ram_addr_t offset)
+static int ram_save_multifd_page(RAMState *rs, RAMBlock *block,
+ ram_addr_t offset)
+{
+ if (multifd_queue_page(rs, block, offset) < 0) {
+ return -1;
+ }
+ ram_counters.normal++;
+
+ return 1;
+}
+
+static bool do_compress_ram_page(QEMUFile *f, z_stream *stream, RAMBlock *block,
+ ram_addr_t offset, uint8_t *source_buf)
{
RAMState *rs = ram_state;
- int bytes_sent, blen;
uint8_t *p = block->host + (offset & TARGET_PAGE_MASK);
+ bool zero_page = false;
+ int ret;
- bytes_sent = save_page_header(rs, f, block, offset |
- RAM_SAVE_FLAG_COMPRESS_PAGE);
- blen = qemu_put_compression_data(f, p, TARGET_PAGE_SIZE,
- migrate_compress_level());
- if (blen < 0) {
- bytes_sent = 0;
- qemu_file_set_error(migrate_get_current()->to_dst_file, blen);
+ if (save_zero_page_to_file(rs, f, block, offset)) {
+ zero_page = true;
+ goto exit;
+ }
+
+ save_page_header(rs, f, block, offset | RAM_SAVE_FLAG_COMPRESS_PAGE);
+
+ /*
+ * copy it to a internal buffer to avoid it being modified by VM
+ * so that we can catch up the error during compression and
+ * decompression
+ */
+ memcpy(source_buf, p, TARGET_PAGE_SIZE);
+ ret = qemu_put_compression_data(f, stream, source_buf, TARGET_PAGE_SIZE);
+ if (ret < 0) {
+ qemu_file_set_error(migrate_get_current()->to_dst_file, ret);
error_report("compressed data failed!");
- } else {
- bytes_sent += blen;
- ram_release_pages(block->idstr, offset & TARGET_PAGE_MASK, 1);
+ return false;
+ }
+
+exit:
+ ram_release_pages(block->idstr, offset & TARGET_PAGE_MASK, 1);
+ return zero_page;
+}
+
+static void
+update_compress_thread_counts(const CompressParam *param, int bytes_xmit)
+{
+ ram_counters.transferred += bytes_xmit;
+
+ if (param->zero_page) {
+ ram_counters.duplicate++;
+ return;
}
- return bytes_sent;
+ /* 8 means a header with RAM_SAVE_FLAG_CONTINUE. */
+ compression_counters.compressed_size += bytes_xmit - 8;
+ compression_counters.pages++;
}
+static bool save_page_use_compression(RAMState *rs);
+
static void flush_compressed_data(RAMState *rs)
{
int idx, len, thread_count;
- if (!migrate_use_compression()) {
+ if (!save_page_use_compression(rs)) {
return;
}
thread_count = migrate_compress_threads();
qemu_mutex_lock(&comp_param[idx].mutex);
if (!comp_param[idx].quit) {
len = qemu_put_qemu_file(rs->f, comp_param[idx].file);
- ram_counters.transferred += len;
+ /*
+ * it's safe to fetch zero_page without holding comp_done_lock
+ * as there is no further request submitted to the thread,
+ * i.e, the thread should be waiting for a request at this point.
+ */
+ update_compress_thread_counts(&comp_param[idx], len);
}
qemu_mutex_unlock(&comp_param[idx].mutex);
}
-}
-
-static inline void set_compress_params(CompressParam *param, RAMBlock *block,
- ram_addr_t offset)
-{
- param->block = block;
- param->offset = offset;
-}
-
-static int compress_page_with_multi_thread(RAMState *rs, RAMBlock *block,
- ram_addr_t offset)
-{
- int idx, thread_count, bytes_xmit = -1, pages = -1;
-
- thread_count = migrate_compress_threads();
- qemu_mutex_lock(&comp_done_lock);
- while (true) {
- for (idx = 0; idx < thread_count; idx++) {
- if (comp_param[idx].done) {
- comp_param[idx].done = false;
- bytes_xmit = qemu_put_qemu_file(rs->f, comp_param[idx].file);
- qemu_mutex_lock(&comp_param[idx].mutex);
- set_compress_params(&comp_param[idx], block, offset);
- qemu_cond_signal(&comp_param[idx].cond);
- qemu_mutex_unlock(&comp_param[idx].mutex);
- pages = 1;
- ram_counters.normal++;
- ram_counters.transferred += bytes_xmit;
- break;
- }
- }
- if (pages > 0) {
- break;
- } else {
- qemu_cond_wait(&comp_done_cond, &comp_done_lock);
- }
- }
- qemu_mutex_unlock(&comp_done_lock);
-
- return pages;
-}
-
-/**
- * ram_save_compressed_page: compress the given page and send it to the stream
- *
- * Returns the number of pages written.
- *
- * @rs: current RAM state
- * @block: block that contains the page we want to send
- * @offset: offset inside the block for the page
- * @last_stage: if we are at the completion stage
- */
-static int ram_save_compressed_page(RAMState *rs, PageSearchStatus *pss,
- bool last_stage)
+}
+
+static inline void set_compress_params(CompressParam *param, RAMBlock *block,
+ ram_addr_t offset)
{
- int pages = -1;
- uint64_t bytes_xmit = 0;
- uint8_t *p;
- int ret, blen;
- RAMBlock *block = pss->block;
- ram_addr_t offset = pss->page << TARGET_PAGE_BITS;
+ param->block = block;
+ param->offset = offset;
+}
- p = block->host + offset;
+static int compress_page_with_multi_thread(RAMState *rs, RAMBlock *block,
+ ram_addr_t offset)
+{
+ int idx, thread_count, bytes_xmit = -1, pages = -1;
+ bool wait = migrate_compress_wait_thread();
- ret = ram_control_save_page(rs->f, block->offset,
- offset, TARGET_PAGE_SIZE, &bytes_xmit);
- if (bytes_xmit) {
- ram_counters.transferred += bytes_xmit;
- pages = 1;
- }
- if (ret != RAM_SAVE_CONTROL_NOT_SUPP) {
- if (ret != RAM_SAVE_CONTROL_DELAYED) {
- if (bytes_xmit > 0) {
- ram_counters.normal++;
- } else if (bytes_xmit == 0) {
- ram_counters.duplicate++;
- }
- }
- } else {
- /* When starting the process of a new block, the first page of
- * the block should be sent out before other pages in the same
- * block, and all the pages in last block should have been sent
- * out, keeping this order is important, because the 'cont' flag
- * is used to avoid resending the block name.
- */
- if (block != rs->last_sent_block) {
- flush_compressed_data(rs);
- pages = save_zero_page(rs, block, offset);
- if (pages == -1) {
- /* Make sure the first page is sent out before other pages */
- bytes_xmit = save_page_header(rs, rs->f, block, offset |
- RAM_SAVE_FLAG_COMPRESS_PAGE);
- blen = qemu_put_compression_data(rs->f, p, TARGET_PAGE_SIZE,
- migrate_compress_level());
- if (blen > 0) {
- ram_counters.transferred += bytes_xmit + blen;
- ram_counters.normal++;
- pages = 1;
- } else {
- qemu_file_set_error(rs->f, blen);
- error_report("compressed data failed!");
- }
- }
- if (pages > 0) {
- ram_release_pages(block->idstr, offset, pages);
- }
- } else {
- pages = save_zero_page(rs, block, offset);
- if (pages == -1) {
- pages = compress_page_with_multi_thread(rs, block, offset);
- } else {
- ram_release_pages(block->idstr, offset, pages);
- }
+ thread_count = migrate_compress_threads();
+ qemu_mutex_lock(&comp_done_lock);
+retry:
+ for (idx = 0; idx < thread_count; idx++) {
+ if (comp_param[idx].done) {
+ comp_param[idx].done = false;
+ bytes_xmit = qemu_put_qemu_file(rs->f, comp_param[idx].file);
+ qemu_mutex_lock(&comp_param[idx].mutex);
+ set_compress_params(&comp_param[idx], block, offset);
+ qemu_cond_signal(&comp_param[idx].cond);
+ qemu_mutex_unlock(&comp_param[idx].mutex);
+ pages = 1;
+ update_compress_thread_counts(&comp_param[idx], bytes_xmit);
+ break;
}
}
+ /*
+ * wait for the free thread if the user specifies 'compress-wait-thread',
+ * otherwise we will post the page out in the main thread as normal page.
+ */
+ if (pages < 0 && wait) {
+ qemu_cond_wait(&comp_done_cond, &comp_done_lock);
+ goto retry;
+ }
+ qemu_mutex_unlock(&comp_done_lock);
+
return pages;
}
* find_dirty_block: find the next dirty page and update any state
* associated with the search process.
*
- * Returns if a page is found
+ * Returns true if a page is found
*
* @rs: current RAM state
* @pss: data about the state of the current dirty page scan
pss->page = 0;
pss->block = QLIST_NEXT_RCU(pss->block, next);
if (!pss->block) {
+ /*
+ * If memory migration starts over, we will meet a dirtied page
+ * which may still exists in compression threads's ring, so we
+ * should flush the compressed data to make sure the new page
+ * is not overwritten by the old one in the destination.
+ *
+ * Also If xbzrle is on, stop using the data compression at this
+ * point. In theory, xbzrle can do better than compression.
+ */
+ flush_compressed_data(rs);
+
/* Hit the end of the list */
pss->block = QLIST_FIRST_RCU(&ram_list.blocks);
/* Flag that we've looped */
pss->complete_round = true;
rs->ram_bulk_stage = false;
- if (migrate_use_xbzrle()) {
- /* If xbzrle is on, stop using the data compression at this
- * point. In theory, xbzrle can do better than compression.
- */
- flush_compressed_data(rs);
- }
}
/* Didn't find anything this time, but try again on the new block */
*again = true;
{
RAMBlock *block = NULL;
+ if (QSIMPLEQ_EMPTY_ATOMIC(&rs->src_page_requests)) {
+ return NULL;
+ }
+
qemu_mutex_lock(&rs->src_page_req_mutex);
if (!QSIMPLEQ_EMPTY(&rs->src_page_requests)) {
struct RAMSrcPageRequest *entry =
memory_region_unref(block->mr);
QSIMPLEQ_REMOVE_HEAD(&rs->src_page_requests, next_req);
g_free(entry);
+ migration_consume_urgent_request();
}
}
qemu_mutex_unlock(&rs->src_page_req_mutex);
}
/**
- * get_queued_page: unqueue a page from the postocpy requests
+ * get_queued_page: unqueue a page from the postcopy requests
*
* Skips pages that are already sent (!dirty)
*
- * Returns if a queued page is found
+ * Returns true if a queued page is found
*
* @rs: current RAM state
* @pss: data about the state of the current dirty page scan
*/
pss->block = block;
pss->page = offset >> TARGET_PAGE_BITS;
+
+ /*
+ * This unqueued page would break the "one round" check, even is
+ * really rare.
+ */
+ pss->complete_round = false;
}
return !!block;
memory_region_ref(ramblock->mr);
qemu_mutex_lock(&rs->src_page_req_mutex);
QSIMPLEQ_INSERT_TAIL(&rs->src_page_requests, new_entry, next_req);
+ migration_make_urgent_request();
qemu_mutex_unlock(&rs->src_page_req_mutex);
rcu_read_unlock();
return -1;
}
+static bool save_page_use_compression(RAMState *rs)
+{
+ if (!migrate_use_compression()) {
+ return false;
+ }
+
+ /*
+ * If xbzrle is on, stop using the data compression after first
+ * round of migration even if compression is enabled. In theory,
+ * xbzrle can do better than compression.
+ */
+ if (rs->ram_bulk_stage || !migrate_use_xbzrle()) {
+ return true;
+ }
+
+ return false;
+}
+
+/*
+ * try to compress the page before posting it out, return true if the page
+ * has been properly handled by compression, otherwise needs other
+ * paths to handle it
+ */
+static bool save_compress_page(RAMState *rs, RAMBlock *block, ram_addr_t offset)
+{
+ if (!save_page_use_compression(rs)) {
+ return false;
+ }
+
+ /*
+ * When starting the process of a new block, the first page of
+ * the block should be sent out before other pages in the same
+ * block, and all the pages in last block should have been sent
+ * out, keeping this order is important, because the 'cont' flag
+ * is used to avoid resending the block name.
+ *
+ * We post the fist page as normal page as compression will take
+ * much CPU resource.
+ */
+ if (block != rs->last_sent_block) {
+ flush_compressed_data(rs);
+ return false;
+ }
+
+ if (compress_page_with_multi_thread(rs, block, offset) > 0) {
+ return true;
+ }
+
+ compression_counters.busy++;
+ return false;
+}
+
/**
* ram_save_target_page: save one target page
*
* Returns the number of pages written
*
* @rs: current RAM state
- * @ms: current migration state
* @pss: data about the page we want to send
* @last_stage: if we are at the completion stage
*/
static int ram_save_target_page(RAMState *rs, PageSearchStatus *pss,
bool last_stage)
{
- int res = 0;
+ RAMBlock *block = pss->block;
+ ram_addr_t offset = pss->page << TARGET_PAGE_BITS;
+ int res;
- /* Check the pages is dirty and if it is send it */
- if (migration_bitmap_clear_dirty(rs, pss->block, pss->page)) {
- /*
- * If xbzrle is on, stop using the data compression after first
- * round of migration even if compression is enabled. In theory,
- * xbzrle can do better than compression.
+ if (control_save_page(rs, block, offset, &res)) {
+ return res;
+ }
+
+ if (save_compress_page(rs, block, offset)) {
+ return 1;
+ }
+
+ res = save_zero_page(rs, block, offset);
+ if (res > 0) {
+ /* Must let xbzrle know, otherwise a previous (now 0'd) cached
+ * page would be stale
*/
- if (migrate_use_compression() &&
- (rs->ram_bulk_stage || !migrate_use_xbzrle())) {
- res = ram_save_compressed_page(rs, pss, last_stage);
- } else {
- res = ram_save_page(rs, pss, last_stage);
+ if (!save_page_use_compression(rs)) {
+ XBZRLE_cache_lock();
+ xbzrle_cache_zero_page(rs, block->offset + offset);
+ XBZRLE_cache_unlock();
}
+ ram_release_pages(block->idstr, offset, res);
+ return res;
+ }
- if (res < 0) {
- return res;
- }
- if (pss->block->unsentmap) {
- clear_bit(pss->page, pss->block->unsentmap);
- }
+ /*
+ * do not use multifd for compression as the first page in the new
+ * block should be posted out before sending the compressed page
+ */
+ if (!save_page_use_compression(rs) && migrate_use_multifd()) {
+ return ram_save_multifd_page(rs, block, offset);
}
- return res;
+ return ram_save_page(rs, pss, last_stage);
}
/**
size_t pagesize_bits =
qemu_ram_pagesize(pss->block) >> TARGET_PAGE_BITS;
+ if (ramblock_is_ignored(pss->block)) {
+ error_report("block %s should not be migrated !", pss->block->idstr);
+ return 0;
+ }
+
do {
+ /* Check the pages is dirty and if it is send it */
+ if (!migration_bitmap_clear_dirty(rs, pss->block, pss->page)) {
+ pss->page++;
+ continue;
+ }
+
tmppages = ram_save_target_page(rs, pss, last_stage);
if (tmppages < 0) {
return tmppages;
}
pages += tmppages;
+ if (pss->block->unsentmap) {
+ clear_bit(pss->page, pss->block->unsentmap);
+ }
+
pss->page++;
} while ((pss->page & (pagesize_bits - 1)) &&
offset_in_ramblock(pss->block, pss->page << TARGET_PAGE_BITS));
*
* Called within an RCU critical section.
*
- * Returns the number of pages written where zero means no dirty pages
+ * Returns the number of pages written where zero means no dirty pages,
+ * or negative on error
*
* @rs: current RAM state
* @last_stage: if we are at the completion stage
}
}
-uint64_t ram_bytes_total(void)
+static uint64_t ram_bytes_total_common(bool count_ignored)
{
RAMBlock *block;
uint64_t total = 0;
rcu_read_lock();
- RAMBLOCK_FOREACH(block) {
- total += block->used_length;
+ if (count_ignored) {
+ RAMBLOCK_FOREACH_MIGRATABLE(block) {
+ total += block->used_length;
+ }
+ } else {
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
+ total += block->used_length;
+ }
}
rcu_read_unlock();
return total;
}
+uint64_t ram_bytes_total(void)
+{
+ return ram_bytes_total_common(false);
+}
+
static void xbzrle_load_setup(void)
{
XBZRLE.decoded_buf = g_malloc(TARGET_PAGE_SIZE);
RAMBlock *block;
/* caller have hold iothread lock or is in a bh, so there is
- * no writing race against this migration_bitmap
+ * no writing race against the migration bitmap
*/
memory_global_dirty_log_stop();
- QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
+ g_free(block->clear_bmap);
+ block->clear_bmap = NULL;
g_free(block->bmap);
block->bmap = NULL;
g_free(block->unsentmap);
rs->last_page = 0;
rs->last_version = ram_list.version;
rs->ram_bulk_stage = true;
+ rs->fpo_enabled = false;
}
#define MAX_WAIT 50 /* ms, half buffered_file limit */
{
struct RAMBlock *block;
- RAMBLOCK_FOREACH(block) {
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
unsigned long *bitmap = block->bmap;
unsigned long range = block->used_length >> TARGET_PAGE_BITS;
unsigned long run_start = find_next_zero_bit(bitmap, range, 0);
* with the dirtymap; so a '1' means it's either dirty or unsent.
*
* @ms: current migration state
- * @pds: state for postcopy
- * @start: RAMBlock starting page
- * @length: RAMBlock size
+ * @block: RAMBlock to discard
*/
-static int postcopy_send_discard_bm_ram(MigrationState *ms,
- PostcopyDiscardState *pds,
- RAMBlock *block)
+static int postcopy_send_discard_bm_ram(MigrationState *ms, RAMBlock *block)
{
unsigned long end = block->used_length >> TARGET_PAGE_BITS;
unsigned long current;
for (current = 0; current < end; ) {
unsigned long one = find_next_bit(unsentmap, end, current);
+ unsigned long zero, discard_length;
- if (one <= end) {
- unsigned long zero = find_next_zero_bit(unsentmap, end, one + 1);
- unsigned long discard_length;
+ if (one >= end) {
+ break;
+ }
- if (zero >= end) {
- discard_length = end - one;
- } else {
- discard_length = zero - one;
- }
- if (discard_length) {
- postcopy_discard_send_range(ms, pds, one, discard_length);
- }
- current = one + discard_length;
+ zero = find_next_zero_bit(unsentmap, end, one + 1);
+
+ if (zero >= end) {
+ discard_length = end - one;
} else {
- current = one;
+ discard_length = zero - one;
}
+ postcopy_discard_send_range(ms, one, discard_length);
+ current = one + discard_length;
}
return 0;
struct RAMBlock *block;
int ret;
- RAMBLOCK_FOREACH(block) {
- PostcopyDiscardState *pds =
- postcopy_discard_send_init(ms, block->idstr);
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
+ postcopy_discard_send_init(ms, block->idstr);
/*
* Postcopy sends chunks of bitmap over the wire, but it
* just needs indexes at this point, avoids it having
* target page specific code.
*/
- ret = postcopy_send_discard_bm_ram(ms, pds, block);
- postcopy_discard_send_finish(ms, pds);
+ ret = postcopy_send_discard_bm_ram(ms, block);
+ postcopy_discard_send_finish(ms);
if (ret) {
return ret;
}
* @unsent_pass: if true we need to canonicalize partially unsent host pages
* otherwise we need to canonicalize partially dirty host pages
* @block: block that contains the page we want to canonicalize
- * @pds: state for postcopy
*/
static void postcopy_chunk_hostpages_pass(MigrationState *ms, bool unsent_pass,
- RAMBlock *block,
- PostcopyDiscardState *pds)
+ RAMBlock *block)
{
RAMState *rs = ram_state;
unsigned long *bitmap = block->bmap;
}
while (run_start < pages) {
- bool do_fixup = false;
- unsigned long fixup_start_addr;
- unsigned long host_offset;
/*
* If the start of this run of pages is in the middle of a host
* page, then we need to fixup this host page.
*/
- host_offset = run_start % host_ratio;
- if (host_offset) {
- do_fixup = true;
- run_start -= host_offset;
- fixup_start_addr = run_start;
- /* For the next pass */
- run_start = run_start + host_ratio;
- } else {
+ if (QEMU_IS_ALIGNED(run_start, host_ratio)) {
/* Find the end of this run */
- unsigned long run_end;
if (unsent_pass) {
- run_end = find_next_bit(unsentmap, pages, run_start + 1);
+ run_start = find_next_bit(unsentmap, pages, run_start + 1);
} else {
- run_end = find_next_zero_bit(bitmap, pages, run_start + 1);
+ run_start = find_next_zero_bit(bitmap, pages, run_start + 1);
}
/*
* If the end isn't at the start of a host page, then the
* run doesn't finish at the end of a host page
* and we need to discard.
*/
- host_offset = run_end % host_ratio;
- if (host_offset) {
- do_fixup = true;
- fixup_start_addr = run_end - host_offset;
- /*
- * This host page has gone, the next loop iteration starts
- * from after the fixup
- */
- run_start = fixup_start_addr + host_ratio;
- } else {
- /*
- * No discards on this iteration, next loop starts from
- * next sent/dirty page
- */
- run_start = run_end + 1;
- }
}
- if (do_fixup) {
+ if (!QEMU_IS_ALIGNED(run_start, host_ratio)) {
unsigned long page;
+ unsigned long fixup_start_addr = QEMU_ALIGN_DOWN(run_start,
+ host_ratio);
+ run_start = QEMU_ALIGN_UP(run_start, host_ratio);
/* Tell the destination to discard this page */
if (unsent_pass || !test_bit(fixup_start_addr, unsentmap)) {
* (any partially sent pages were already discarded
* by the previous unsent_pass)
*/
- postcopy_discard_send_range(ms, pds, fixup_start_addr,
- host_ratio);
+ postcopy_discard_send_range(ms, fixup_start_addr, host_ratio);
}
/* Clean up the bitmap */
}
/**
- * postcopy_chuck_hostpages: discrad any partially sent host page
+ * postcopy_chunk_hostpages: discard any partially sent host page
*
* Utility for the outgoing postcopy code.
*
*/
static int postcopy_chunk_hostpages(MigrationState *ms, RAMBlock *block)
{
- PostcopyDiscardState *pds =
- postcopy_discard_send_init(ms, block->idstr);
+ postcopy_discard_send_init(ms, block->idstr);
/* First pass: Discard all partially sent host pages */
- postcopy_chunk_hostpages_pass(ms, true, block, pds);
+ postcopy_chunk_hostpages_pass(ms, true, block);
/*
* Second pass: Ensure that all partially dirty host pages are made
* fully dirty.
*/
- postcopy_chunk_hostpages_pass(ms, false, block, pds);
+ postcopy_chunk_hostpages_pass(ms, false, block);
- postcopy_discard_send_finish(ms, pds);
+ postcopy_discard_send_finish(ms);
return 0;
}
rs->last_sent_block = NULL;
rs->last_page = 0;
- QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
unsigned long pages = block->used_length >> TARGET_PAGE_BITS;
unsigned long *bitmap = block->bmap;
unsigned long *unsentmap = block->unsentmap;
goto err;
}
- bitmap_clear(rb->receivedmap, start >> qemu_target_page_bits(),
- length >> qemu_target_page_bits());
+ /*
+ * On source VM, we don't need to update the received bitmap since
+ * we don't even have one.
+ */
+ if (rb->receivedmap) {
+ bitmap_clear(rb->receivedmap, start >> qemu_target_page_bits(),
+ length >> qemu_target_page_bits());
+ }
+
ret = ram_block_discard_range(rb, start, length);
err:
/*
* Count the total number of pages used by ram blocks not including any
* gaps due to alignment or unplugs.
+ * This must match with the initial values of dirty bitmap.
*/
(*rsp)->migration_dirty_pages = ram_bytes_total() >> TARGET_PAGE_BITS;
-
ram_state_reset(*rsp);
return 0;
static void ram_list_init_bitmaps(void)
{
+ MigrationState *ms = migrate_get_current();
RAMBlock *block;
unsigned long pages;
+ uint8_t shift;
/* Skip setting bitmap if there is no RAM */
if (ram_bytes_total()) {
- QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+ shift = ms->clear_bitmap_shift;
+ if (shift > CLEAR_BITMAP_SHIFT_MAX) {
+ error_report("clear_bitmap_shift (%u) too big, using "
+ "max value (%u)", shift, CLEAR_BITMAP_SHIFT_MAX);
+ shift = CLEAR_BITMAP_SHIFT_MAX;
+ } else if (shift < CLEAR_BITMAP_SHIFT_MIN) {
+ error_report("clear_bitmap_shift (%u) too small, using "
+ "min value (%u)", shift, CLEAR_BITMAP_SHIFT_MIN);
+ shift = CLEAR_BITMAP_SHIFT_MIN;
+ }
+
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
pages = block->max_length >> TARGET_PAGE_BITS;
+ /*
+ * The initial dirty bitmap for migration must be set with all
+ * ones to make sure we'll migrate every guest RAM page to
+ * destination.
+ * Here we set RAMBlock.bmap all to 1 because when rebegin a
+ * new migration after a failed migration, ram_list.
+ * dirty_memory[DIRTY_MEMORY_MIGRATION] don't include the whole
+ * guest memory.
+ */
block->bmap = bitmap_new(pages);
bitmap_set(block->bmap, 0, pages);
+ block->clear_bmap_shift = shift;
+ block->clear_bmap = bitmap_new(clear_bmap_size(pages, shift));
if (migrate_postcopy_ram()) {
block->unsentmap = bitmap_new(pages);
bitmap_set(block->unsentmap, 0, pages);
ram_list_init_bitmaps();
memory_global_dirty_log_start();
- migration_bitmap_sync(rs);
+ migration_bitmap_sync_precopy(rs);
rcu_read_unlock();
qemu_mutex_unlock_ramlist();
return 0;
}
+static void ram_state_resume_prepare(RAMState *rs, QEMUFile *out)
+{
+ RAMBlock *block;
+ uint64_t pages = 0;
+
+ /*
+ * Postcopy is not using xbzrle/compression, so no need for that.
+ * Also, since source are already halted, we don't need to care
+ * about dirty page logging as well.
+ */
+
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
+ pages += bitmap_count_one(block->bmap,
+ block->used_length >> TARGET_PAGE_BITS);
+ }
+
+ /* This may not be aligned with current bitmaps. Recalculate. */
+ rs->migration_dirty_pages = pages;
+
+ rs->last_seen_block = NULL;
+ rs->last_sent_block = NULL;
+ rs->last_page = 0;
+ rs->last_version = ram_list.version;
+ /*
+ * Disable the bulk stage, otherwise we'll resend the whole RAM no
+ * matter what we have sent.
+ */
+ rs->ram_bulk_stage = false;
+
+ /* Update RAMState cache of output QEMUFile */
+ rs->f = out;
+
+ trace_ram_state_resume_prepare(pages);
+}
+
+/*
+ * This function clears bits of the free pages reported by the caller from the
+ * migration dirty bitmap. @addr is the host address corresponding to the
+ * start of the continuous guest free pages, and @len is the total bytes of
+ * those pages.
+ */
+void qemu_guest_free_page_hint(void *addr, size_t len)
+{
+ RAMBlock *block;
+ ram_addr_t offset;
+ size_t used_len, start, npages;
+ MigrationState *s = migrate_get_current();
+
+ /* This function is currently expected to be used during live migration */
+ if (!migration_is_setup_or_active(s->state)) {
+ return;
+ }
+
+ for (; len > 0; len -= used_len, addr += used_len) {
+ block = qemu_ram_block_from_host(addr, false, &offset);
+ if (unlikely(!block || offset >= block->used_length)) {
+ /*
+ * The implementation might not support RAMBlock resize during
+ * live migration, but it could happen in theory with future
+ * updates. So we add a check here to capture that case.
+ */
+ error_report_once("%s unexpected error", __func__);
+ return;
+ }
+
+ if (len <= block->used_length - offset) {
+ used_len = len;
+ } else {
+ used_len = block->used_length - offset;
+ }
+
+ start = offset >> TARGET_PAGE_BITS;
+ npages = used_len >> TARGET_PAGE_BITS;
+
+ qemu_mutex_lock(&ram_state->bitmap_mutex);
+ ram_state->migration_dirty_pages -=
+ bitmap_count_one_with_offset(block->bmap, start, npages);
+ bitmap_clear(block->bmap, start, npages);
+ qemu_mutex_unlock(&ram_state->bitmap_mutex);
+ }
+}
+
/*
* Each of ram_save_setup, ram_save_iterate and ram_save_complete has
* long-running RCU critical section. When rcu-reclaims in the code
RAMState **rsp = opaque;
RAMBlock *block;
+ if (compress_threads_save_setup()) {
+ return -1;
+ }
+
/* migration has already setup the bitmap, reuse it. */
if (!migration_in_colo_state()) {
if (ram_init_all(rsp) != 0) {
+ compress_threads_save_cleanup();
return -1;
}
}
rcu_read_lock();
- qemu_put_be64(f, ram_bytes_total() | RAM_SAVE_FLAG_MEM_SIZE);
+ qemu_put_be64(f, ram_bytes_total_common(true) | RAM_SAVE_FLAG_MEM_SIZE);
- RAMBLOCK_FOREACH(block) {
+ RAMBLOCK_FOREACH_MIGRATABLE(block) {
qemu_put_byte(f, strlen(block->idstr));
qemu_put_buffer(f, (uint8_t *)block->idstr, strlen(block->idstr));
qemu_put_be64(f, block->used_length);
if (migrate_postcopy_ram() && block->page_size != qemu_host_page_size) {
qemu_put_be64(f, block->page_size);
}
+ if (migrate_ignore_shared()) {
+ qemu_put_be64(f, block->mr->addr);
+ }
}
rcu_read_unlock();
- compress_threads_save_setup();
ram_control_before_iterate(f, RAM_CONTROL_SETUP);
ram_control_after_iterate(f, RAM_CONTROL_SETUP);
+ multifd_send_sync_main(*rsp);
qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
+ qemu_fflush(f);
return 0;
}
t0 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
i = 0;
- while ((ret = qemu_file_rate_limit(f)) == 0) {
+ while ((ret = qemu_file_rate_limit(f)) == 0 ||
+ !QSIMPLEQ_EMPTY(&rs->src_page_requests)) {
int pages;
+ if (qemu_file_get_error(f)) {
+ break;
+ }
+
pages = ram_find_and_save_block(rs, false);
/* no more pages to sent */
if (pages == 0) {
done = 1;
break;
}
- rs->iterations++;
+
+ if (pages < 0) {
+ qemu_file_set_error(f, pages);
+ break;
+ }
+
+ rs->target_page_count += pages;
/* we want to check in the 1st loop, just in case it was the 1st time
and we had to sync the dirty bitmap.
- qemu_get_clock_ns() is a bit expensive, so we only check each some
+ qemu_clock_get_ns() is a bit expensive, so we only check each some
iterations
*/
if ((i & 63) == 0) {
}
i++;
}
- flush_compressed_data(rs);
rcu_read_unlock();
/*
ram_control_after_iterate(f, RAM_CONTROL_ROUND);
out:
+ multifd_send_sync_main(rs);
qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
+ qemu_fflush(f);
ram_counters.transferred += 8;
ret = qemu_file_get_error(f);
/**
* ram_save_complete: function called to send the remaining amount of ram
*
- * Returns zero to indicate success
+ * Returns zero to indicate success or negative on error
*
* Called with iothread lock
*
{
RAMState **temp = opaque;
RAMState *rs = *temp;
+ int ret = 0;
rcu_read_lock();
if (!migration_in_postcopy()) {
- migration_bitmap_sync(rs);
+ migration_bitmap_sync_precopy(rs);
}
ram_control_before_iterate(f, RAM_CONTROL_FINISH);
if (pages == 0) {
break;
}
+ if (pages < 0) {
+ ret = pages;
+ break;
+ }
}
flush_compressed_data(rs);
rcu_read_unlock();
+ multifd_send_sync_main(rs);
qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
+ qemu_fflush(f);
- return 0;
+ return ret;
}
static void ram_save_pending(QEMUFile *f, void *opaque, uint64_t max_size,
- uint64_t *non_postcopiable_pending,
- uint64_t *postcopiable_pending)
+ uint64_t *res_precopy_only,
+ uint64_t *res_compatible,
+ uint64_t *res_postcopy_only)
{
RAMState **temp = opaque;
RAMState *rs = *temp;
remaining_size < max_size) {
qemu_mutex_lock_iothread();
rcu_read_lock();
- migration_bitmap_sync(rs);
+ migration_bitmap_sync_precopy(rs);
rcu_read_unlock();
qemu_mutex_unlock_iothread();
remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE;
if (migrate_postcopy_ram()) {
/* We can do postcopy, and all the data is postcopiable */
- *postcopiable_pending += remaining_size;
+ *res_compatible += remaining_size;
} else {
- *non_postcopiable_pending += remaining_size;
+ *res_precopy_only += remaining_size;
}
}
return NULL;
}
+ if (ramblock_is_ignored(block)) {
+ error_report("block %s should not be migrated !", id);
+ return NULL;
+ }
+
return block;
}
return block->host + offset;
}
+static inline void *colo_cache_from_block_offset(RAMBlock *block,
+ ram_addr_t offset)
+{
+ if (!offset_in_ramblock(block, offset)) {
+ return NULL;
+ }
+ if (!block->colo_cache) {
+ error_report("%s: colo_cache is NULL in block :%s",
+ __func__, block->idstr);
+ return NULL;
+ }
+
+ /*
+ * During colo checkpoint, we need bitmap of these migrated pages.
+ * It help us to decide which pages in ram cache should be flushed
+ * into VM's RAM later.
+ */
+ if (!test_and_set_bit(offset >> TARGET_PAGE_BITS, block->bmap)) {
+ ram_state->migration_dirty_pages++;
+ }
+ return block->colo_cache + offset;
+}
+
/**
* ram_handle_compressed: handle the zero page case
*
}
}
+/* return the size after decompression, or negative value on error */
+static int
+qemu_uncompress_data(z_stream *stream, uint8_t *dest, size_t dest_len,
+ const uint8_t *source, size_t source_len)
+{
+ int err;
+
+ err = inflateReset(stream);
+ if (err != Z_OK) {
+ return -1;
+ }
+
+ stream->avail_in = source_len;
+ stream->next_in = (uint8_t *)source;
+ stream->avail_out = dest_len;
+ stream->next_out = dest;
+
+ err = inflate(stream, Z_NO_FLUSH);
+ if (err != Z_STREAM_END) {
+ return -1;
+ }
+
+ return stream->total_out;
+}
+
static void *do_data_decompress(void *opaque)
{
DecompressParam *param = opaque;
unsigned long pagesize;
uint8_t *des;
- int len;
+ int len, ret;
qemu_mutex_lock(¶m->mutex);
while (!param->quit) {
qemu_mutex_unlock(¶m->mutex);
pagesize = TARGET_PAGE_SIZE;
- /* uncompress() will return failed in some case, especially
- * when the page is dirted when doing the compression, it's
- * not a problem because the dirty page will be retransferred
- * and uncompress() won't break the data in other pages.
- */
- uncompress((Bytef *)des, &pagesize,
- (const Bytef *)param->compbuf, len);
+
+ ret = qemu_uncompress_data(¶m->stream, des, pagesize,
+ param->compbuf, len);
+ if (ret < 0 && migrate_get_current()->decompress_error_check) {
+ error_report("decompress data failed");
+ qemu_file_set_error(decomp_file, ret);
+ }
qemu_mutex_lock(&decomp_done_lock);
param->done = true;
return NULL;
}
-static void wait_for_decompress_done(void)
+static int wait_for_decompress_done(void)
{
int idx, thread_count;
if (!migrate_use_compression()) {
- return;
+ return 0;
}
thread_count = migrate_decompress_threads();
qemu_mutex_lock(&decomp_done_lock);
for (idx = 0; idx < thread_count; idx++) {
- while (!decomp_param[idx].done) {
- qemu_cond_wait(&decomp_done_cond, &decomp_done_lock);
- }
- }
- qemu_mutex_unlock(&decomp_done_lock);
-}
-
-static void compress_threads_load_setup(void)
-{
- int i, thread_count;
-
- if (!migrate_use_compression()) {
- return;
- }
- thread_count = migrate_decompress_threads();
- decompress_threads = g_new0(QemuThread, thread_count);
- decomp_param = g_new0(DecompressParam, thread_count);
- qemu_mutex_init(&decomp_done_lock);
- qemu_cond_init(&decomp_done_cond);
- for (i = 0; i < thread_count; i++) {
- qemu_mutex_init(&decomp_param[i].mutex);
- qemu_cond_init(&decomp_param[i].cond);
- decomp_param[i].compbuf = g_malloc0(compressBound(TARGET_PAGE_SIZE));
- decomp_param[i].done = true;
- decomp_param[i].quit = false;
- qemu_thread_create(decompress_threads + i, "decompress",
- do_data_decompress, decomp_param + i,
- QEMU_THREAD_JOINABLE);
+ while (!decomp_param[idx].done) {
+ qemu_cond_wait(&decomp_done_cond, &decomp_done_lock);
+ }
}
+ qemu_mutex_unlock(&decomp_done_lock);
+ return qemu_file_get_error(decomp_file);
}
static void compress_threads_load_cleanup(void)
}
thread_count = migrate_decompress_threads();
for (i = 0; i < thread_count; i++) {
+ /*
+ * we use it as a indicator which shows if the thread is
+ * properly init'd or not
+ */
+ if (!decomp_param[i].compbuf) {
+ break;
+ }
+
qemu_mutex_lock(&decomp_param[i].mutex);
decomp_param[i].quit = true;
qemu_cond_signal(&decomp_param[i].cond);
qemu_mutex_unlock(&decomp_param[i].mutex);
}
for (i = 0; i < thread_count; i++) {
+ if (!decomp_param[i].compbuf) {
+ break;
+ }
+
qemu_thread_join(decompress_threads + i);
qemu_mutex_destroy(&decomp_param[i].mutex);
qemu_cond_destroy(&decomp_param[i].cond);
+ inflateEnd(&decomp_param[i].stream);
g_free(decomp_param[i].compbuf);
+ decomp_param[i].compbuf = NULL;
}
g_free(decompress_threads);
g_free(decomp_param);
decompress_threads = NULL;
decomp_param = NULL;
+ decomp_file = NULL;
+}
+
+static int compress_threads_load_setup(QEMUFile *f)
+{
+ int i, thread_count;
+
+ if (!migrate_use_compression()) {
+ return 0;
+ }
+
+ thread_count = migrate_decompress_threads();
+ decompress_threads = g_new0(QemuThread, thread_count);
+ decomp_param = g_new0(DecompressParam, thread_count);
+ qemu_mutex_init(&decomp_done_lock);
+ qemu_cond_init(&decomp_done_cond);
+ decomp_file = f;
+ for (i = 0; i < thread_count; i++) {
+ if (inflateInit(&decomp_param[i].stream) != Z_OK) {
+ goto exit;
+ }
+
+ decomp_param[i].compbuf = g_malloc0(compressBound(TARGET_PAGE_SIZE));
+ qemu_mutex_init(&decomp_param[i].mutex);
+ qemu_cond_init(&decomp_param[i].cond);
+ decomp_param[i].done = true;
+ decomp_param[i].quit = false;
+ qemu_thread_create(decompress_threads + i, "decompress",
+ do_data_decompress, decomp_param + i,
+ QEMU_THREAD_JOINABLE);
+ }
+ return 0;
+exit:
+ compress_threads_load_cleanup();
+ return -1;
}
static void decompress_data_with_multi_threads(QEMUFile *f,
qemu_mutex_unlock(&decomp_done_lock);
}
+/*
+ * colo cache: this is for secondary VM, we cache the whole
+ * memory of the secondary VM, it is need to hold the global lock
+ * to call this helper.
+ */
+int colo_init_ram_cache(void)
+{
+ RAMBlock *block;
+
+ rcu_read_lock();
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
+ block->colo_cache = qemu_anon_ram_alloc(block->used_length,
+ NULL,
+ false);
+ if (!block->colo_cache) {
+ error_report("%s: Can't alloc memory for COLO cache of block %s,"
+ "size 0x" RAM_ADDR_FMT, __func__, block->idstr,
+ block->used_length);
+ goto out_locked;
+ }
+ memcpy(block->colo_cache, block->host, block->used_length);
+ }
+ rcu_read_unlock();
+ /*
+ * Record the dirty pages that sent by PVM, we use this dirty bitmap together
+ * with to decide which page in cache should be flushed into SVM's RAM. Here
+ * we use the same name 'ram_bitmap' as for migration.
+ */
+ if (ram_bytes_total()) {
+ RAMBlock *block;
+
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
+ unsigned long pages = block->max_length >> TARGET_PAGE_BITS;
+
+ block->bmap = bitmap_new(pages);
+ bitmap_set(block->bmap, 0, pages);
+ }
+ }
+ ram_state = g_new0(RAMState, 1);
+ ram_state->migration_dirty_pages = 0;
+ qemu_mutex_init(&ram_state->bitmap_mutex);
+ memory_global_dirty_log_start();
+
+ return 0;
+
+out_locked:
+
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
+ if (block->colo_cache) {
+ qemu_anon_ram_free(block->colo_cache, block->used_length);
+ block->colo_cache = NULL;
+ }
+ }
+
+ rcu_read_unlock();
+ return -errno;
+}
+
+/* It is need to hold the global lock to call this helper */
+void colo_release_ram_cache(void)
+{
+ RAMBlock *block;
+
+ memory_global_dirty_log_stop();
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
+ g_free(block->bmap);
+ block->bmap = NULL;
+ }
+
+ rcu_read_lock();
+
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
+ if (block->colo_cache) {
+ qemu_anon_ram_free(block->colo_cache, block->used_length);
+ block->colo_cache = NULL;
+ }
+ }
+
+ rcu_read_unlock();
+ qemu_mutex_destroy(&ram_state->bitmap_mutex);
+ g_free(ram_state);
+ ram_state = NULL;
+}
+
/**
* ram_load_setup: Setup RAM for migration incoming side
*
*/
static int ram_load_setup(QEMUFile *f, void *opaque)
{
+ if (compress_threads_load_setup(f)) {
+ return -1;
+ }
+
xbzrle_load_setup();
- compress_threads_load_setup();
ramblock_recv_map_init();
+
return 0;
}
static int ram_load_cleanup(void *opaque)
{
RAMBlock *rb;
+
+ RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
+ if (ramblock_is_pmem(rb)) {
+ pmem_persist(rb->host, rb->used_length);
+ }
+ }
+
xbzrle_load_cleanup();
compress_threads_load_cleanup();
- RAMBLOCK_FOREACH(rb) {
+ RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
g_free(rb->receivedmap);
rb->receivedmap = NULL;
}
+
return 0;
}
*/
int ram_postcopy_incoming_init(MigrationIncomingState *mis)
{
- unsigned long ram_pages = last_ram_page();
-
- return postcopy_ram_incoming_init(mis, ram_pages);
+ return postcopy_ram_incoming_init(mis);
}
/**
{
int flags = 0, ret = 0;
bool place_needed = false;
- bool matching_page_sizes = false;
+ bool matches_target_page_size = false;
MigrationIncomingState *mis = migration_incoming_get_current();
/* Temporary page that is later 'placed' */
void *postcopy_host_page = postcopy_get_tmp_page(mis);
ret = -EINVAL;
break;
}
- matching_page_sizes = block->page_size == TARGET_PAGE_SIZE;
+ matches_target_page_size = block->page_size == TARGET_PAGE_SIZE;
/*
* Postcopy requires that we place whole host pages atomically;
* these may be huge pages for RAMBlocks that are backed by
case RAM_SAVE_FLAG_PAGE:
all_zero = false;
- if (!place_needed || !matching_page_sizes) {
+ if (!matches_target_page_size) {
+ /* For huge pages, we always use temporary buffer */
qemu_get_buffer(f, page_buffer, TARGET_PAGE_SIZE);
} else {
- /* Avoids the qemu_file copy during postcopy, which is
- * going to do a copy later; can only do it when we
- * do this read in one go (matching page sizes)
+ /*
+ * For small pages that matches target page size, we
+ * avoid the qemu_file copy. Instead we directly use
+ * the buffer of QEMUFile to place the page. Note: we
+ * cannot do any QEMUFile operation before using that
+ * buffer to make sure the buffer is valid when
+ * placing the page.
*/
qemu_get_buffer_in_place(f, (uint8_t **)&place_source,
TARGET_PAGE_SIZE);
break;
case RAM_SAVE_FLAG_EOS:
/* normal exit */
+ multifd_recv_sync_main();
break;
default:
error_report("Unknown combination of migration flags: %#x"
return ps >= POSTCOPY_INCOMING_LISTENING && ps < POSTCOPY_INCOMING_END;
}
-static int ram_load(QEMUFile *f, void *opaque, int version_id)
+/*
+ * Flush content of RAM cache into SVM's memory.
+ * Only flush the pages that be dirtied by PVM or SVM or both.
+ */
+static void colo_flush_ram_cache(void)
{
- int flags = 0, ret = 0, invalid_flags = 0;
- static uint64_t seq_iter;
- int len = 0;
- /*
- * If system is running in postcopy mode, page inserts to host memory must
- * be atomic
- */
- bool postcopy_running = postcopy_is_running();
- /* ADVISE is earlier, it shows the source has the postcopy capability on */
- bool postcopy_advised = postcopy_is_advised();
+ RAMBlock *block = NULL;
+ void *dst_host;
+ void *src_host;
+ unsigned long offset = 0;
- seq_iter++;
+ memory_global_dirty_log_sync();
+ rcu_read_lock();
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
+ ramblock_sync_dirty_bitmap(ram_state, block);
+ }
+ rcu_read_unlock();
- if (version_id != 4) {
- ret = -EINVAL;
+ trace_colo_flush_ram_cache_begin(ram_state->migration_dirty_pages);
+ rcu_read_lock();
+ block = QLIST_FIRST_RCU(&ram_list.blocks);
+
+ while (block) {
+ offset = migration_bitmap_find_dirty(ram_state, block, offset);
+
+ if (offset << TARGET_PAGE_BITS >= block->used_length) {
+ offset = 0;
+ block = QLIST_NEXT_RCU(block, next);
+ } else {
+ migration_bitmap_clear_dirty(ram_state, block, offset);
+ dst_host = block->host + (offset << TARGET_PAGE_BITS);
+ src_host = block->colo_cache + (offset << TARGET_PAGE_BITS);
+ memcpy(dst_host, src_host, TARGET_PAGE_SIZE);
+ }
}
+ rcu_read_unlock();
+ trace_colo_flush_ram_cache_end();
+}
+
+/**
+ * ram_load_precopy: load pages in precopy case
+ *
+ * Returns 0 for success or -errno in case of error
+ *
+ * Called in precopy mode by ram_load().
+ * rcu_read_lock is taken prior to this being called.
+ *
+ * @f: QEMUFile where to send the data
+ */
+static int ram_load_precopy(QEMUFile *f)
+{
+ int flags = 0, ret = 0, invalid_flags = 0, len = 0;
+ /* ADVISE is earlier, it shows the source has the postcopy capability on */
+ bool postcopy_advised = postcopy_is_advised();
if (!migrate_use_compression()) {
invalid_flags |= RAM_SAVE_FLAG_COMPRESS_PAGE;
}
- /* This RCU critical section can be very long running.
- * When RCU reclaims in the code start to become numerous,
- * it will be necessary to reduce the granularity of this
- * critical section.
- */
- rcu_read_lock();
-
- if (postcopy_running) {
- ret = ram_load_postcopy(f);
- }
- while (!postcopy_running && !ret && !(flags & RAM_SAVE_FLAG_EOS)) {
+ while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) {
ram_addr_t addr, total_ram_bytes;
void *host = NULL;
uint8_t ch;
RAM_SAVE_FLAG_COMPRESS_PAGE | RAM_SAVE_FLAG_XBZRLE)) {
RAMBlock *block = ram_block_from_stream(f, flags);
- host = host_from_ram_block_offset(block, addr);
+ /*
+ * After going into COLO, we should load the Page into colo_cache.
+ */
+ if (migration_incoming_in_colo_state()) {
+ host = colo_cache_from_block_offset(block, addr);
+ } else {
+ host = host_from_ram_block_offset(block, addr);
+ }
if (!host) {
error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
ret = -EINVAL;
break;
}
- ramblock_recv_bitmap_set(block, host);
+
+ if (!migration_incoming_in_colo_state()) {
+ ramblock_recv_bitmap_set(block, host);
+ }
+
trace_ram_load_loop(block->idstr, (uint64_t)addr, flags, host);
}
length = qemu_get_be64(f);
block = qemu_ram_block_by_name(id);
- if (block) {
+ if (block && !qemu_ram_is_migratable(block)) {
+ error_report("block %s should not be migrated !", id);
+ ret = -EINVAL;
+ } else if (block) {
if (length != block->used_length) {
Error *local_err = NULL;
ret = -EINVAL;
}
}
+ if (migrate_ignore_shared()) {
+ hwaddr addr = qemu_get_be64(f);
+ if (ramblock_is_ignored(block) &&
+ block->mr->addr != addr) {
+ error_report("Mismatched GPAs for block %s "
+ "%" PRId64 "!= %" PRId64,
+ id, (uint64_t)addr,
+ (uint64_t)block->mr->addr);
+ ret = -EINVAL;
+ }
+ }
ram_control_load_hook(f, RAM_CONTROL_BLOCK_REG,
block->idstr);
} else {
break;
case RAM_SAVE_FLAG_EOS:
/* normal exit */
+ multifd_recv_sync_main();
break;
default:
if (flags & RAM_SAVE_FLAG_HOOK) {
}
}
- wait_for_decompress_done();
+ return ret;
+}
+
+static int ram_load(QEMUFile *f, void *opaque, int version_id)
+{
+ int ret = 0;
+ static uint64_t seq_iter;
+ /*
+ * If system is running in postcopy mode, page inserts to host memory must
+ * be atomic
+ */
+ bool postcopy_running = postcopy_is_running();
+
+ seq_iter++;
+
+ if (version_id != 4) {
+ return -EINVAL;
+ }
+
+ /*
+ * This RCU critical section can be very long running.
+ * When RCU reclaims in the code start to become numerous,
+ * it will be necessary to reduce the granularity of this
+ * critical section.
+ */
+ rcu_read_lock();
+
+ if (postcopy_running) {
+ ret = ram_load_postcopy(f);
+ } else {
+ ret = ram_load_precopy(f);
+ }
+
+ ret |= wait_for_decompress_done();
rcu_read_unlock();
trace_ram_load_complete(ret, seq_iter);
+
+ if (!ret && migration_incoming_in_colo_state()) {
+ colo_flush_ram_cache();
+ }
return ret;
}
static bool ram_has_postcopy(void *opaque)
{
+ RAMBlock *rb;
+ RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
+ if (ramblock_is_pmem(rb)) {
+ info_report("Block: %s, host: %p is a nvdimm memory, postcopy"
+ "is not supported now!", rb->idstr, rb->host);
+ return false;
+ }
+ }
+
return migrate_postcopy_ram();
}
+/* Sync all the dirty bitmap with destination VM. */
+static int ram_dirty_bitmap_sync_all(MigrationState *s, RAMState *rs)
+{
+ RAMBlock *block;
+ QEMUFile *file = s->to_dst_file;
+ int ramblock_count = 0;
+
+ trace_ram_dirty_bitmap_sync_start();
+
+ RAMBLOCK_FOREACH_NOT_IGNORED(block) {
+ qemu_savevm_send_recv_bitmap(file, block->idstr);
+ trace_ram_dirty_bitmap_request(block->idstr);
+ ramblock_count++;
+ }
+
+ trace_ram_dirty_bitmap_sync_wait();
+
+ /* Wait until all the ramblocks' dirty bitmap synced */
+ while (ramblock_count--) {
+ qemu_sem_wait(&s->rp_state.rp_sem);
+ }
+
+ trace_ram_dirty_bitmap_sync_complete();
+
+ return 0;
+}
+
+static void ram_dirty_bitmap_reload_notify(MigrationState *s)
+{
+ qemu_sem_post(&s->rp_state.rp_sem);
+}
+
+/*
+ * Read the received bitmap, revert it as the initial dirty bitmap.
+ * This is only used when the postcopy migration is paused but wants
+ * to resume from a middle point.
+ */
+int ram_dirty_bitmap_reload(MigrationState *s, RAMBlock *block)
+{
+ int ret = -EINVAL;
+ QEMUFile *file = s->rp_state.from_dst_file;
+ unsigned long *le_bitmap, nbits = block->used_length >> TARGET_PAGE_BITS;
+ uint64_t local_size = DIV_ROUND_UP(nbits, 8);
+ uint64_t size, end_mark;
+
+ trace_ram_dirty_bitmap_reload_begin(block->idstr);
+
+ if (s->state != MIGRATION_STATUS_POSTCOPY_RECOVER) {
+ error_report("%s: incorrect state %s", __func__,
+ MigrationStatus_str(s->state));
+ return -EINVAL;
+ }
+
+ /*
+ * Note: see comments in ramblock_recv_bitmap_send() on why we
+ * need the endianess convertion, and the paddings.
+ */
+ local_size = ROUND_UP(local_size, 8);
+
+ /* Add paddings */
+ le_bitmap = bitmap_new(nbits + BITS_PER_LONG);
+
+ size = qemu_get_be64(file);
+
+ /* The size of the bitmap should match with our ramblock */
+ if (size != local_size) {
+ error_report("%s: ramblock '%s' bitmap size mismatch "
+ "(0x%"PRIx64" != 0x%"PRIx64")", __func__,
+ block->idstr, size, local_size);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ size = qemu_get_buffer(file, (uint8_t *)le_bitmap, local_size);
+ end_mark = qemu_get_be64(file);
+
+ ret = qemu_file_get_error(file);
+ if (ret || size != local_size) {
+ error_report("%s: read bitmap failed for ramblock '%s': %d"
+ " (size 0x%"PRIx64", got: 0x%"PRIx64")",
+ __func__, block->idstr, ret, local_size, size);
+ ret = -EIO;
+ goto out;
+ }
+
+ if (end_mark != RAMBLOCK_RECV_BITMAP_ENDING) {
+ error_report("%s: ramblock '%s' end mark incorrect: 0x%"PRIu64,
+ __func__, block->idstr, end_mark);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /*
+ * Endianess convertion. We are during postcopy (though paused).
+ * The dirty bitmap won't change. We can directly modify it.
+ */
+ bitmap_from_le(block->bmap, le_bitmap, nbits);
+
+ /*
+ * What we received is "received bitmap". Revert it as the initial
+ * dirty bitmap for this ramblock.
+ */
+ bitmap_complement(block->bmap, block->bmap, nbits);
+
+ trace_ram_dirty_bitmap_reload_complete(block->idstr);
+
+ /*
+ * We succeeded to sync bitmap for current ramblock. If this is
+ * the last one to sync, we need to notify the main send thread.
+ */
+ ram_dirty_bitmap_reload_notify(s);
+
+ ret = 0;
+out:
+ g_free(le_bitmap);
+ return ret;
+}
+
+static int ram_resume_prepare(MigrationState *s, void *opaque)
+{
+ RAMState *rs = *(RAMState **)opaque;
+ int ret;
+
+ ret = ram_dirty_bitmap_sync_all(s, rs);
+ if (ret) {
+ return ret;
+ }
+
+ ram_state_resume_prepare(rs, s->to_dst_file);
+
+ return 0;
+}
+
static SaveVMHandlers savevm_ram_handlers = {
.save_setup = ram_save_setup,
.save_live_iterate = ram_save_iterate,
.save_cleanup = ram_save_cleanup,
.load_setup = ram_load_setup,
.load_cleanup = ram_load_cleanup,
+ .resume_prepare = ram_resume_prepare,
};
void ram_mig_init(void)
projects / qemu.git / blobdiff