4 * Copyright IBM, Corp. 2008
7 * Anthony Liguori <aliguori@us.ibm.com>
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
12 * Contributions after 2012-01-13 are licensed under the terms of the
13 * GNU GPL, version 2 or (at your option) any later version.
16 #include "qemu-common.h"
17 #include "migration/migration.h"
18 #include "monitor/monitor.h"
19 #include "migration/qemu-file.h"
20 #include "sysemu/sysemu.h"
21 #include "block/block.h"
22 #include "qemu/sockets.h"
23 #include "migration/block.h"
24 #include "qemu/thread.h"
25 #include "qmp-commands.h"
28 //#define DEBUG_MIGRATION
30 #ifdef DEBUG_MIGRATION
31 #define DPRINTF(fmt, ...) \
32 do { printf("migration: " fmt, ## __VA_ARGS__); } while (0)
34 #define DPRINTF(fmt, ...) \
46 #define MAX_THROTTLE (32 << 20) /* Migration speed throttling */
48 /* Amount of time to allocate to each "chunk" of bandwidth-throttled
50 #define BUFFER_DELAY 100
51 #define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY)
53 /* Migration XBZRLE default cache size */
54 #define DEFAULT_MIGRATE_CACHE_SIZE (64 * 1024 * 1024)
56 static NotifierList migration_state_notifiers =
57 NOTIFIER_LIST_INITIALIZER(migration_state_notifiers);
59 /* When we add fault tolerance, we could have several
60 migrations at once. For now we don't need to add
61 dynamic creation of migration */
63 MigrationState *migrate_get_current(void)
65 static MigrationState current_migration = {
66 .state = MIG_STATE_SETUP,
67 .bandwidth_limit = MAX_THROTTLE,
68 .xbzrle_cache_size = DEFAULT_MIGRATE_CACHE_SIZE,
72 return ¤t_migration;
75 void qemu_start_incoming_migration(const char *uri, Error **errp)
79 if (strstart(uri, "tcp:", &p))
80 tcp_start_incoming_migration(p, errp);
82 else if (strstart(uri, "exec:", &p))
83 exec_start_incoming_migration(p, errp);
84 else if (strstart(uri, "unix:", &p))
85 unix_start_incoming_migration(p, errp);
86 else if (strstart(uri, "fd:", &p))
87 fd_start_incoming_migration(p, errp);
90 error_setg(errp, "unknown migration protocol: %s", uri);
94 static void process_incoming_migration_co(void *opaque)
99 ret = qemu_loadvm_state(f);
102 fprintf(stderr, "load of migration failed\n");
105 qemu_announce_self();
106 DPRINTF("successfully loaded vm state\n");
108 bdrv_clear_incoming_migration_all();
109 /* Make sure all file formats flush their mutable metadata */
110 bdrv_invalidate_cache_all();
115 runstate_set(RUN_STATE_PAUSED);
119 void process_incoming_migration(QEMUFile *f)
121 Coroutine *co = qemu_coroutine_create(process_incoming_migration_co);
122 int fd = qemu_get_fd(f);
125 qemu_set_nonblock(fd);
126 qemu_coroutine_enter(co, f);
129 /* amount of nanoseconds we are willing to wait for migration to be down.
130 * the choice of nanoseconds is because it is the maximum resolution that
131 * get_clock() can achieve. It is an internal measure. All user-visible
132 * units must be in seconds */
133 static uint64_t max_downtime = 30000000;
135 uint64_t migrate_max_downtime(void)
140 MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp)
142 MigrationCapabilityStatusList *head = NULL;
143 MigrationCapabilityStatusList *caps;
144 MigrationState *s = migrate_get_current();
147 for (i = 0; i < MIGRATION_CAPABILITY_MAX; i++) {
149 head = g_malloc0(sizeof(*caps));
152 caps->next = g_malloc0(sizeof(*caps));
156 g_malloc(sizeof(*caps->value));
157 caps->value->capability = i;
158 caps->value->state = s->enabled_capabilities[i];
164 static void get_xbzrle_cache_stats(MigrationInfo *info)
166 if (migrate_use_xbzrle()) {
167 info->has_xbzrle_cache = true;
168 info->xbzrle_cache = g_malloc0(sizeof(*info->xbzrle_cache));
169 info->xbzrle_cache->cache_size = migrate_xbzrle_cache_size();
170 info->xbzrle_cache->bytes = xbzrle_mig_bytes_transferred();
171 info->xbzrle_cache->pages = xbzrle_mig_pages_transferred();
172 info->xbzrle_cache->cache_miss = xbzrle_mig_pages_cache_miss();
173 info->xbzrle_cache->overflow = xbzrle_mig_pages_overflow();
177 MigrationInfo *qmp_query_migrate(Error **errp)
179 MigrationInfo *info = g_malloc0(sizeof(*info));
180 MigrationState *s = migrate_get_current();
183 case MIG_STATE_SETUP:
184 /* no migration has happened ever */
186 case MIG_STATE_ACTIVE:
187 info->has_status = true;
188 info->status = g_strdup("active");
189 info->has_total_time = true;
190 info->total_time = qemu_get_clock_ms(rt_clock)
192 info->has_expected_downtime = true;
193 info->expected_downtime = s->expected_downtime;
195 info->has_ram = true;
196 info->ram = g_malloc0(sizeof(*info->ram));
197 info->ram->transferred = ram_bytes_transferred();
198 info->ram->remaining = ram_bytes_remaining();
199 info->ram->total = ram_bytes_total();
200 info->ram->duplicate = dup_mig_pages_transferred();
201 info->ram->skipped = skipped_mig_pages_transferred();
202 info->ram->normal = norm_mig_pages_transferred();
203 info->ram->normal_bytes = norm_mig_bytes_transferred();
204 info->ram->dirty_pages_rate = s->dirty_pages_rate;
205 info->ram->mbps = s->mbps;
207 if (blk_mig_active()) {
208 info->has_disk = true;
209 info->disk = g_malloc0(sizeof(*info->disk));
210 info->disk->transferred = blk_mig_bytes_transferred();
211 info->disk->remaining = blk_mig_bytes_remaining();
212 info->disk->total = blk_mig_bytes_total();
215 get_xbzrle_cache_stats(info);
217 case MIG_STATE_COMPLETED:
218 get_xbzrle_cache_stats(info);
220 info->has_status = true;
221 info->status = g_strdup("completed");
222 info->total_time = s->total_time;
223 info->has_downtime = true;
224 info->downtime = s->downtime;
226 info->has_ram = true;
227 info->ram = g_malloc0(sizeof(*info->ram));
228 info->ram->transferred = ram_bytes_transferred();
229 info->ram->remaining = 0;
230 info->ram->total = ram_bytes_total();
231 info->ram->duplicate = dup_mig_pages_transferred();
232 info->ram->skipped = skipped_mig_pages_transferred();
233 info->ram->normal = norm_mig_pages_transferred();
234 info->ram->normal_bytes = norm_mig_bytes_transferred();
235 info->ram->mbps = s->mbps;
237 case MIG_STATE_ERROR:
238 info->has_status = true;
239 info->status = g_strdup("failed");
241 case MIG_STATE_CANCELLED:
242 info->has_status = true;
243 info->status = g_strdup("cancelled");
250 void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params,
253 MigrationState *s = migrate_get_current();
254 MigrationCapabilityStatusList *cap;
256 if (s->state == MIG_STATE_ACTIVE) {
257 error_set(errp, QERR_MIGRATION_ACTIVE);
261 for (cap = params; cap; cap = cap->next) {
262 s->enabled_capabilities[cap->value->capability] = cap->value->state;
266 /* shared migration helpers */
268 static void migrate_fd_cleanup(void *opaque)
270 MigrationState *s = opaque;
272 qemu_bh_delete(s->cleanup_bh);
273 s->cleanup_bh = NULL;
276 DPRINTF("closing file\n");
277 qemu_mutex_unlock_iothread();
278 qemu_thread_join(&s->thread);
279 qemu_mutex_lock_iothread();
281 qemu_fclose(s->file);
285 assert(s->state != MIG_STATE_ACTIVE);
287 if (s->state != MIG_STATE_COMPLETED) {
288 qemu_savevm_state_cancel();
291 notifier_list_notify(&migration_state_notifiers, s);
294 static void migrate_finish_set_state(MigrationState *s, int new_state)
296 if (atomic_cmpxchg(&s->state, MIG_STATE_ACTIVE, new_state) == new_state) {
297 trace_migrate_set_state(new_state);
301 void migrate_fd_error(MigrationState *s)
303 DPRINTF("setting error state\n");
304 assert(s->file == NULL);
305 s->state = MIG_STATE_ERROR;
306 trace_migrate_set_state(MIG_STATE_ERROR);
307 notifier_list_notify(&migration_state_notifiers, s);
310 static void migrate_fd_cancel(MigrationState *s)
312 DPRINTF("cancelling migration\n");
314 migrate_finish_set_state(s, MIG_STATE_CANCELLED);
317 void add_migration_state_change_notifier(Notifier *notify)
319 notifier_list_add(&migration_state_notifiers, notify);
322 void remove_migration_state_change_notifier(Notifier *notify)
324 notifier_remove(notify);
327 bool migration_is_active(MigrationState *s)
329 return s->state == MIG_STATE_ACTIVE;
332 bool migration_has_finished(MigrationState *s)
334 return s->state == MIG_STATE_COMPLETED;
337 bool migration_has_failed(MigrationState *s)
339 return (s->state == MIG_STATE_CANCELLED ||
340 s->state == MIG_STATE_ERROR);
343 static MigrationState *migrate_init(const MigrationParams *params)
345 MigrationState *s = migrate_get_current();
346 int64_t bandwidth_limit = s->bandwidth_limit;
347 bool enabled_capabilities[MIGRATION_CAPABILITY_MAX];
348 int64_t xbzrle_cache_size = s->xbzrle_cache_size;
350 memcpy(enabled_capabilities, s->enabled_capabilities,
351 sizeof(enabled_capabilities));
353 memset(s, 0, sizeof(*s));
355 memcpy(s->enabled_capabilities, enabled_capabilities,
356 sizeof(enabled_capabilities));
357 s->xbzrle_cache_size = xbzrle_cache_size;
359 s->bandwidth_limit = bandwidth_limit;
360 s->state = MIG_STATE_SETUP;
361 trace_migrate_set_state(MIG_STATE_SETUP);
363 s->total_time = qemu_get_clock_ms(rt_clock);
367 static GSList *migration_blockers;
369 void migrate_add_blocker(Error *reason)
371 migration_blockers = g_slist_prepend(migration_blockers, reason);
374 void migrate_del_blocker(Error *reason)
376 migration_blockers = g_slist_remove(migration_blockers, reason);
379 void qmp_migrate(const char *uri, bool has_blk, bool blk,
380 bool has_inc, bool inc, bool has_detach, bool detach,
383 Error *local_err = NULL;
384 MigrationState *s = migrate_get_current();
385 MigrationParams params;
391 if (s->state == MIG_STATE_ACTIVE) {
392 error_set(errp, QERR_MIGRATION_ACTIVE);
396 if (qemu_savevm_state_blocked(errp)) {
400 if (migration_blockers) {
401 *errp = error_copy(migration_blockers->data);
405 s = migrate_init(¶ms);
407 if (strstart(uri, "tcp:", &p)) {
408 tcp_start_outgoing_migration(s, p, &local_err);
410 } else if (strstart(uri, "exec:", &p)) {
411 exec_start_outgoing_migration(s, p, &local_err);
412 } else if (strstart(uri, "unix:", &p)) {
413 unix_start_outgoing_migration(s, p, &local_err);
414 } else if (strstart(uri, "fd:", &p)) {
415 fd_start_outgoing_migration(s, p, &local_err);
418 error_set(errp, QERR_INVALID_PARAMETER_VALUE, "uri", "a valid migration protocol");
424 error_propagate(errp, local_err);
429 void qmp_migrate_cancel(Error **errp)
431 migrate_fd_cancel(migrate_get_current());
434 void qmp_migrate_set_cache_size(int64_t value, Error **errp)
436 MigrationState *s = migrate_get_current();
438 /* Check for truncation */
439 if (value != (size_t)value) {
440 error_set(errp, QERR_INVALID_PARAMETER_VALUE, "cache size",
441 "exceeding address space");
445 s->xbzrle_cache_size = xbzrle_cache_resize(value);
448 int64_t qmp_query_migrate_cache_size(Error **errp)
450 return migrate_xbzrle_cache_size();
453 void qmp_migrate_set_speed(int64_t value, Error **errp)
460 if (value > SIZE_MAX) {
464 s = migrate_get_current();
465 s->bandwidth_limit = value;
467 qemu_file_set_rate_limit(s->file, s->bandwidth_limit / XFER_LIMIT_RATIO);
471 void qmp_migrate_set_downtime(double value, Error **errp)
474 value = MAX(0, MIN(UINT64_MAX, value));
475 max_downtime = (uint64_t)value;
478 bool migrate_rdma_pin_all(void)
482 s = migrate_get_current();
484 return s->enabled_capabilities[MIGRATION_CAPABILITY_X_RDMA_PIN_ALL];
487 bool migrate_auto_converge(void)
491 s = migrate_get_current();
493 return s->enabled_capabilities[MIGRATION_CAPABILITY_AUTO_CONVERGE];
496 bool migrate_zero_blocks(void)
500 s = migrate_get_current();
502 return s->enabled_capabilities[MIGRATION_CAPABILITY_ZERO_BLOCKS];
505 int migrate_use_xbzrle(void)
509 s = migrate_get_current();
511 return s->enabled_capabilities[MIGRATION_CAPABILITY_XBZRLE];
514 int64_t migrate_xbzrle_cache_size(void)
518 s = migrate_get_current();
520 return s->xbzrle_cache_size;
523 /* migration thread support */
525 static void *migration_thread(void *opaque)
527 MigrationState *s = opaque;
528 int64_t initial_time = qemu_get_clock_ms(rt_clock);
529 int64_t initial_bytes = 0;
530 int64_t max_size = 0;
531 int64_t start_time = initial_time;
532 bool old_vm_running = false;
534 DPRINTF("beginning savevm\n");
535 qemu_savevm_state_begin(s->file, &s->params);
537 while (s->state == MIG_STATE_ACTIVE) {
538 int64_t current_time;
539 uint64_t pending_size;
541 if (!qemu_file_rate_limit(s->file)) {
542 DPRINTF("iterate\n");
543 pending_size = qemu_savevm_state_pending(s->file, max_size);
544 DPRINTF("pending size %lu max %lu\n", pending_size, max_size);
545 if (pending_size && pending_size >= max_size) {
546 qemu_savevm_state_iterate(s->file);
550 DPRINTF("done iterating\n");
551 qemu_mutex_lock_iothread();
552 start_time = qemu_get_clock_ms(rt_clock);
553 qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
554 old_vm_running = runstate_is_running();
556 ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
558 qemu_file_set_rate_limit(s->file, INT_MAX);
559 qemu_savevm_state_complete(s->file);
561 qemu_mutex_unlock_iothread();
564 migrate_finish_set_state(s, MIG_STATE_ERROR);
568 if (!qemu_file_get_error(s->file)) {
569 migrate_finish_set_state(s, MIG_STATE_COMPLETED);
575 if (qemu_file_get_error(s->file)) {
576 migrate_finish_set_state(s, MIG_STATE_ERROR);
579 current_time = qemu_get_clock_ms(rt_clock);
580 if (current_time >= initial_time + BUFFER_DELAY) {
581 uint64_t transferred_bytes = qemu_ftell(s->file) - initial_bytes;
582 uint64_t time_spent = current_time - initial_time;
583 double bandwidth = transferred_bytes / time_spent;
584 max_size = bandwidth * migrate_max_downtime() / 1000000;
586 s->mbps = time_spent ? (((double) transferred_bytes * 8.0) /
587 ((double) time_spent / 1000.0)) / 1000.0 / 1000.0 : -1;
589 DPRINTF("transferred %" PRIu64 " time_spent %" PRIu64
590 " bandwidth %g max_size %" PRId64 "\n",
591 transferred_bytes, time_spent, bandwidth, max_size);
592 /* if we haven't sent anything, we don't want to recalculate
593 10000 is a small enough number for our purposes */
594 if (s->dirty_bytes_rate && transferred_bytes > 10000) {
595 s->expected_downtime = s->dirty_bytes_rate / bandwidth;
598 qemu_file_reset_rate_limit(s->file);
599 initial_time = current_time;
600 initial_bytes = qemu_ftell(s->file);
602 if (qemu_file_rate_limit(s->file)) {
603 /* usleep expects microseconds */
604 g_usleep((initial_time + BUFFER_DELAY - current_time)*1000);
608 qemu_mutex_lock_iothread();
609 if (s->state == MIG_STATE_COMPLETED) {
610 int64_t end_time = qemu_get_clock_ms(rt_clock);
611 s->total_time = end_time - s->total_time;
612 s->downtime = end_time - start_time;
613 runstate_set(RUN_STATE_POSTMIGRATE);
615 if (old_vm_running) {
619 qemu_bh_schedule(s->cleanup_bh);
620 qemu_mutex_unlock_iothread();
625 void migrate_fd_connect(MigrationState *s)
627 s->state = MIG_STATE_ACTIVE;
628 trace_migrate_set_state(MIG_STATE_ACTIVE);
630 /* This is a best 1st approximation. ns to ms */
631 s->expected_downtime = max_downtime/1000000;
632 s->cleanup_bh = qemu_bh_new(migrate_fd_cleanup, s);
634 qemu_file_set_rate_limit(s->file,
635 s->bandwidth_limit / XFER_LIMIT_RATIO);
637 qemu_thread_create(&s->thread, migration_thread, s,
638 QEMU_THREAD_JOINABLE);
639 notifier_list_notify(&migration_state_notifiers, s);