4 * Copyright (c) 2003-2008 Fabrice Bellard
5 * Copyright (c) 2011-2015 Red Hat Inc
8 * Juan Quintela <quintela@redhat.com>
10 * Permission is hereby granted, free of charge, to any person obtaining a copy
11 * of this software and associated documentation files (the "Software"), to deal
12 * in the Software without restriction, including without limitation the rights
13 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the Software is
15 * furnished to do so, subject to the following conditions:
17 * The above copyright notice and this permission notice shall be included in
18 * all copies or substantial portions of the Software.
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
29 #include "qemu/osdep.h"
30 #include "qemu/cutils.h"
31 #include "qemu/bitops.h"
32 #include "qemu/bitmap.h"
33 #include "qemu/madvise.h"
34 #include "qemu/main-loop.h"
35 #include "io/channel-null.h"
38 #include "migration.h"
39 #include "migration/register.h"
40 #include "migration/misc.h"
41 #include "qemu-file.h"
42 #include "postcopy-ram.h"
43 #include "page_cache.h"
44 #include "qemu/error-report.h"
45 #include "qapi/error.h"
46 #include "qapi/qapi-types-migration.h"
47 #include "qapi/qapi-events-migration.h"
48 #include "qapi/qmp/qerror.h"
50 #include "exec/ram_addr.h"
51 #include "exec/target_page.h"
52 #include "qemu/rcu_queue.h"
53 #include "migration/colo.h"
55 #include "sysemu/cpu-throttle.h"
59 #include "sysemu/runstate.h"
61 #include "hw/boards.h" /* for machine_dump_guest_core() */
63 #if defined(__linux__)
64 #include "qemu/userfaultfd.h"
65 #endif /* defined(__linux__) */
67 /***********************************************************/
68 /* ram save/restore */
70 /* RAM_SAVE_FLAG_ZERO used to be named RAM_SAVE_FLAG_COMPRESS, it
71 * worked for pages that where filled with the same char. We switched
72 * it to only search for the zero value. And to avoid confusion with
73 * RAM_SSAVE_FLAG_COMPRESS_PAGE just rename it.
76 #define RAM_SAVE_FLAG_FULL 0x01 /* Obsolete, not used anymore */
77 #define RAM_SAVE_FLAG_ZERO 0x02
78 #define RAM_SAVE_FLAG_MEM_SIZE 0x04
79 #define RAM_SAVE_FLAG_PAGE 0x08
80 #define RAM_SAVE_FLAG_EOS 0x10
81 #define RAM_SAVE_FLAG_CONTINUE 0x20
82 #define RAM_SAVE_FLAG_XBZRLE 0x40
83 /* 0x80 is reserved in migration.h start with 0x100 next */
84 #define RAM_SAVE_FLAG_COMPRESS_PAGE 0x100
86 int (*xbzrle_encode_buffer_func)(uint8_t *, uint8_t *, int,
87 uint8_t *, int) = xbzrle_encode_buffer;
88 #if defined(CONFIG_AVX512BW_OPT)
89 #include "qemu/cpuid.h"
90 static void __attribute__((constructor)) init_cpu_flag(void)
92 unsigned max = __get_cpuid_max(0, NULL);
95 __cpuid(1, a, b, c, d);
96 /* We must check that AVX is not just available, but usable. */
97 if ((c & bit_OSXSAVE) && (c & bit_AVX) && max >= 7) {
99 __asm("xgetbv" : "=a"(bv), "=d"(d) : "c"(0));
100 __cpuid_count(7, 0, a, b, c, d);
102 * XCR0[7:5] = 111b (OPMASK state, upper 256-bit of ZMM0-ZMM15
103 * and ZMM16-ZMM31 state are enabled by OS)
104 * XCR0[2:1] = 11b (XMM state and YMM state are enabled by OS)
106 if ((bv & 0xe6) == 0xe6 && (b & bit_AVX512BW)) {
107 xbzrle_encode_buffer_func = xbzrle_encode_buffer_avx512;
114 XBZRLECacheStats xbzrle_counters;
116 /* used by the search for pages to send */
117 struct PageSearchStatus {
118 /* The migration channel used for a specific host page */
119 QEMUFile *pss_channel;
120 /* Last block from where we have sent data */
121 RAMBlock *last_sent_block;
122 /* Current block being searched */
124 /* Current page to search from */
126 /* Set once we wrap around */
128 /* Whether we're sending a host page */
129 bool host_page_sending;
130 /* The start/end of current host page. Invalid if host_page_sending==false */
131 unsigned long host_page_start;
132 unsigned long host_page_end;
134 typedef struct PageSearchStatus PageSearchStatus;
136 /* struct contains XBZRLE cache and a static page
137 used by the compression */
139 /* buffer used for XBZRLE encoding */
140 uint8_t *encoded_buf;
141 /* buffer for storing page content */
142 uint8_t *current_buf;
143 /* Cache for XBZRLE, Protected by lock. */
146 /* it will store a page full of zeros */
147 uint8_t *zero_target_page;
148 /* buffer used for XBZRLE decoding */
149 uint8_t *decoded_buf;
152 static void XBZRLE_cache_lock(void)
154 if (migrate_use_xbzrle()) {
155 qemu_mutex_lock(&XBZRLE.lock);
159 static void XBZRLE_cache_unlock(void)
161 if (migrate_use_xbzrle()) {
162 qemu_mutex_unlock(&XBZRLE.lock);
167 * xbzrle_cache_resize: resize the xbzrle cache
169 * This function is called from migrate_params_apply in main
170 * thread, possibly while a migration is in progress. A running
171 * migration may be using the cache and might finish during this call,
172 * hence changes to the cache are protected by XBZRLE.lock().
174 * Returns 0 for success or -1 for error
176 * @new_size: new cache size
177 * @errp: set *errp if the check failed, with reason
179 int xbzrle_cache_resize(uint64_t new_size, Error **errp)
181 PageCache *new_cache;
184 /* Check for truncation */
185 if (new_size != (size_t)new_size) {
186 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cache size",
187 "exceeding address space");
191 if (new_size == migrate_xbzrle_cache_size()) {
198 if (XBZRLE.cache != NULL) {
199 new_cache = cache_init(new_size, TARGET_PAGE_SIZE, errp);
205 cache_fini(XBZRLE.cache);
206 XBZRLE.cache = new_cache;
209 XBZRLE_cache_unlock();
213 static bool postcopy_preempt_active(void)
215 return migrate_postcopy_preempt() && migration_in_postcopy();
218 bool ramblock_is_ignored(RAMBlock *block)
220 return !qemu_ram_is_migratable(block) ||
221 (migrate_ignore_shared() && qemu_ram_is_shared(block));
224 #undef RAMBLOCK_FOREACH
226 int foreach_not_ignored_block(RAMBlockIterFunc func, void *opaque)
231 RCU_READ_LOCK_GUARD();
233 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
234 ret = func(block, opaque);
242 static void ramblock_recv_map_init(void)
246 RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
247 assert(!rb->receivedmap);
248 rb->receivedmap = bitmap_new(rb->max_length >> qemu_target_page_bits());
252 int ramblock_recv_bitmap_test(RAMBlock *rb, void *host_addr)
254 return test_bit(ramblock_recv_bitmap_offset(host_addr, rb),
258 bool ramblock_recv_bitmap_test_byte_offset(RAMBlock *rb, uint64_t byte_offset)
260 return test_bit(byte_offset >> TARGET_PAGE_BITS, rb->receivedmap);
263 void ramblock_recv_bitmap_set(RAMBlock *rb, void *host_addr)
265 set_bit_atomic(ramblock_recv_bitmap_offset(host_addr, rb), rb->receivedmap);
268 void ramblock_recv_bitmap_set_range(RAMBlock *rb, void *host_addr,
271 bitmap_set_atomic(rb->receivedmap,
272 ramblock_recv_bitmap_offset(host_addr, rb),
276 #define RAMBLOCK_RECV_BITMAP_ENDING (0x0123456789abcdefULL)
279 * Format: bitmap_size (8 bytes) + whole_bitmap (N bytes).
281 * Returns >0 if success with sent bytes, or <0 if error.
283 int64_t ramblock_recv_bitmap_send(QEMUFile *file,
284 const char *block_name)
286 RAMBlock *block = qemu_ram_block_by_name(block_name);
287 unsigned long *le_bitmap, nbits;
291 error_report("%s: invalid block name: %s", __func__, block_name);
295 nbits = block->postcopy_length >> TARGET_PAGE_BITS;
298 * Make sure the tmp bitmap buffer is big enough, e.g., on 32bit
299 * machines we may need 4 more bytes for padding (see below
300 * comment). So extend it a bit before hand.
302 le_bitmap = bitmap_new(nbits + BITS_PER_LONG);
305 * Always use little endian when sending the bitmap. This is
306 * required that when source and destination VMs are not using the
307 * same endianness. (Note: big endian won't work.)
309 bitmap_to_le(le_bitmap, block->receivedmap, nbits);
311 /* Size of the bitmap, in bytes */
312 size = DIV_ROUND_UP(nbits, 8);
315 * size is always aligned to 8 bytes for 64bit machines, but it
316 * may not be true for 32bit machines. We need this padding to
317 * make sure the migration can survive even between 32bit and
320 size = ROUND_UP(size, 8);
322 qemu_put_be64(file, size);
323 qemu_put_buffer(file, (const uint8_t *)le_bitmap, size);
325 * Mark as an end, in case the middle part is screwed up due to
326 * some "mysterious" reason.
328 qemu_put_be64(file, RAMBLOCK_RECV_BITMAP_ENDING);
333 if (qemu_file_get_error(file)) {
334 return qemu_file_get_error(file);
337 return size + sizeof(size);
341 * An outstanding page request, on the source, having been received
344 struct RAMSrcPageRequest {
349 QSIMPLEQ_ENTRY(RAMSrcPageRequest) next_req;
352 /* State of RAM for migration */
355 * PageSearchStatus structures for the channels when send pages.
356 * Protected by the bitmap_mutex.
358 PageSearchStatus pss[RAM_CHANNEL_MAX];
359 /* UFFD file descriptor, used in 'write-tracking' migration */
361 /* total ram size in bytes */
362 uint64_t ram_bytes_total;
363 /* Last block that we have visited searching for dirty pages */
364 RAMBlock *last_seen_block;
365 /* Last dirty target page we have sent */
366 ram_addr_t last_page;
367 /* last ram version we have seen */
368 uint32_t last_version;
369 /* How many times we have dirty too many pages */
370 int dirty_rate_high_cnt;
371 /* these variables are used for bitmap sync */
372 /* last time we did a full bitmap_sync */
373 int64_t time_last_bitmap_sync;
374 /* bytes transferred at start_time */
375 uint64_t bytes_xfer_prev;
376 /* number of dirty pages since start_time */
377 uint64_t num_dirty_pages_period;
378 /* xbzrle misses since the beginning of the period */
379 uint64_t xbzrle_cache_miss_prev;
380 /* Amount of xbzrle pages since the beginning of the period */
381 uint64_t xbzrle_pages_prev;
382 /* Amount of xbzrle encoded bytes since the beginning of the period */
383 uint64_t xbzrle_bytes_prev;
384 /* Start using XBZRLE (e.g., after the first round). */
386 /* Are we on the last stage of migration */
388 /* compression statistics since the beginning of the period */
389 /* amount of count that no free thread to compress data */
390 uint64_t compress_thread_busy_prev;
391 /* amount bytes after compression */
392 uint64_t compressed_size_prev;
393 /* amount of compressed pages */
394 uint64_t compress_pages_prev;
396 /* total handled target pages at the beginning of period */
397 uint64_t target_page_count_prev;
398 /* total handled target pages since start */
399 uint64_t target_page_count;
400 /* number of dirty bits in the bitmap */
401 uint64_t migration_dirty_pages;
404 * - dirty/clear bitmap
405 * - migration_dirty_pages
408 QemuMutex bitmap_mutex;
409 /* The RAMBlock used in the last src_page_requests */
410 RAMBlock *last_req_rb;
411 /* Queue of outstanding page requests from the destination */
412 QemuMutex src_page_req_mutex;
413 QSIMPLEQ_HEAD(, RAMSrcPageRequest) src_page_requests;
415 typedef struct RAMState RAMState;
417 static RAMState *ram_state;
419 static NotifierWithReturnList precopy_notifier_list;
421 /* Whether postcopy has queued requests? */
422 static bool postcopy_has_request(RAMState *rs)
424 return !QSIMPLEQ_EMPTY_ATOMIC(&rs->src_page_requests);
427 void precopy_infrastructure_init(void)
429 notifier_with_return_list_init(&precopy_notifier_list);
432 void precopy_add_notifier(NotifierWithReturn *n)
434 notifier_with_return_list_add(&precopy_notifier_list, n);
437 void precopy_remove_notifier(NotifierWithReturn *n)
439 notifier_with_return_remove(n);
442 int precopy_notify(PrecopyNotifyReason reason, Error **errp)
444 PrecopyNotifyData pnd;
448 return notifier_with_return_list_notify(&precopy_notifier_list, &pnd);
451 uint64_t ram_bytes_remaining(void)
453 return ram_state ? (ram_state->migration_dirty_pages * TARGET_PAGE_SIZE) :
458 * NOTE: not all stats in ram_counters are used in reality. See comments
459 * for struct MigrationAtomicStats. The ultimate result of ram migration
460 * counters will be a merged version with both ram_counters and the atomic
461 * fields in ram_atomic_counters.
463 MigrationStats ram_counters;
464 MigrationAtomicStats ram_atomic_counters;
466 void ram_transferred_add(uint64_t bytes)
468 if (runstate_is_running()) {
469 ram_counters.precopy_bytes += bytes;
470 } else if (migration_in_postcopy()) {
471 stat64_add(&ram_atomic_counters.postcopy_bytes, bytes);
473 ram_counters.downtime_bytes += bytes;
475 stat64_add(&ram_atomic_counters.transferred, bytes);
478 void dirty_sync_missed_zero_copy(void)
480 ram_counters.dirty_sync_missed_zero_copy++;
483 struct MigrationOps {
484 int (*ram_save_target_page)(RAMState *rs, PageSearchStatus *pss);
486 typedef struct MigrationOps MigrationOps;
488 MigrationOps *migration_ops;
490 CompressionStats compression_counters;
492 struct CompressParam {
502 /* internally used fields */
506 typedef struct CompressParam CompressParam;
508 struct DecompressParam {
518 typedef struct DecompressParam DecompressParam;
520 static CompressParam *comp_param;
521 static QemuThread *compress_threads;
522 /* comp_done_cond is used to wake up the migration thread when
523 * one of the compression threads has finished the compression.
524 * comp_done_lock is used to co-work with comp_done_cond.
526 static QemuMutex comp_done_lock;
527 static QemuCond comp_done_cond;
529 static QEMUFile *decomp_file;
530 static DecompressParam *decomp_param;
531 static QemuThread *decompress_threads;
532 static QemuMutex decomp_done_lock;
533 static QemuCond decomp_done_cond;
535 static int ram_save_host_page_urgent(PageSearchStatus *pss);
537 static bool do_compress_ram_page(QEMUFile *f, z_stream *stream, RAMBlock *block,
538 ram_addr_t offset, uint8_t *source_buf);
540 /* NOTE: page is the PFN not real ram_addr_t. */
541 static void pss_init(PageSearchStatus *pss, RAMBlock *rb, ram_addr_t page)
545 pss->complete_round = false;
549 * Check whether two PSSs are actively sending the same page. Return true
550 * if it is, false otherwise.
552 static bool pss_overlap(PageSearchStatus *pss1, PageSearchStatus *pss2)
554 return pss1->host_page_sending && pss2->host_page_sending &&
555 (pss1->host_page_start == pss2->host_page_start);
558 static void *do_data_compress(void *opaque)
560 CompressParam *param = opaque;
565 qemu_mutex_lock(¶m->mutex);
566 while (!param->quit) {
568 block = param->block;
569 offset = param->offset;
571 qemu_mutex_unlock(¶m->mutex);
573 zero_page = do_compress_ram_page(param->file, ¶m->stream,
574 block, offset, param->originbuf);
576 qemu_mutex_lock(&comp_done_lock);
578 param->zero_page = zero_page;
579 qemu_cond_signal(&comp_done_cond);
580 qemu_mutex_unlock(&comp_done_lock);
582 qemu_mutex_lock(¶m->mutex);
584 qemu_cond_wait(¶m->cond, ¶m->mutex);
587 qemu_mutex_unlock(¶m->mutex);
592 static void compress_threads_save_cleanup(void)
596 if (!migrate_use_compression() || !comp_param) {
600 thread_count = migrate_compress_threads();
601 for (i = 0; i < thread_count; i++) {
603 * we use it as a indicator which shows if the thread is
604 * properly init'd or not
606 if (!comp_param[i].file) {
610 qemu_mutex_lock(&comp_param[i].mutex);
611 comp_param[i].quit = true;
612 qemu_cond_signal(&comp_param[i].cond);
613 qemu_mutex_unlock(&comp_param[i].mutex);
615 qemu_thread_join(compress_threads + i);
616 qemu_mutex_destroy(&comp_param[i].mutex);
617 qemu_cond_destroy(&comp_param[i].cond);
618 deflateEnd(&comp_param[i].stream);
619 g_free(comp_param[i].originbuf);
620 qemu_fclose(comp_param[i].file);
621 comp_param[i].file = NULL;
623 qemu_mutex_destroy(&comp_done_lock);
624 qemu_cond_destroy(&comp_done_cond);
625 g_free(compress_threads);
627 compress_threads = NULL;
631 static int compress_threads_save_setup(void)
635 if (!migrate_use_compression()) {
638 thread_count = migrate_compress_threads();
639 compress_threads = g_new0(QemuThread, thread_count);
640 comp_param = g_new0(CompressParam, thread_count);
641 qemu_cond_init(&comp_done_cond);
642 qemu_mutex_init(&comp_done_lock);
643 for (i = 0; i < thread_count; i++) {
644 comp_param[i].originbuf = g_try_malloc(TARGET_PAGE_SIZE);
645 if (!comp_param[i].originbuf) {
649 if (deflateInit(&comp_param[i].stream,
650 migrate_compress_level()) != Z_OK) {
651 g_free(comp_param[i].originbuf);
655 /* comp_param[i].file is just used as a dummy buffer to save data,
656 * set its ops to empty.
658 comp_param[i].file = qemu_file_new_output(
659 QIO_CHANNEL(qio_channel_null_new()));
660 comp_param[i].done = true;
661 comp_param[i].quit = false;
662 qemu_mutex_init(&comp_param[i].mutex);
663 qemu_cond_init(&comp_param[i].cond);
664 qemu_thread_create(compress_threads + i, "compress",
665 do_data_compress, comp_param + i,
666 QEMU_THREAD_JOINABLE);
671 compress_threads_save_cleanup();
676 * save_page_header: write page header to wire
678 * If this is the 1st block, it also writes the block identification
680 * Returns the number of bytes written
682 * @pss: current PSS channel status
683 * @block: block that contains the page we want to send
684 * @offset: offset inside the block for the page
685 * in the lower bits, it contains flags
687 static size_t save_page_header(PageSearchStatus *pss, RAMBlock *block,
691 bool same_block = (block == pss->last_sent_block);
692 QEMUFile *f = pss->pss_channel;
695 offset |= RAM_SAVE_FLAG_CONTINUE;
697 qemu_put_be64(f, offset);
701 len = strlen(block->idstr);
702 qemu_put_byte(f, len);
703 qemu_put_buffer(f, (uint8_t *)block->idstr, len);
705 pss->last_sent_block = block;
711 * mig_throttle_guest_down: throttle down the guest
713 * Reduce amount of guest cpu execution to hopefully slow down memory
714 * writes. If guest dirty memory rate is reduced below the rate at
715 * which we can transfer pages to the destination then we should be
716 * able to complete migration. Some workloads dirty memory way too
717 * fast and will not effectively converge, even with auto-converge.
719 static void mig_throttle_guest_down(uint64_t bytes_dirty_period,
720 uint64_t bytes_dirty_threshold)
722 MigrationState *s = migrate_get_current();
723 uint64_t pct_initial = s->parameters.cpu_throttle_initial;
724 uint64_t pct_increment = s->parameters.cpu_throttle_increment;
725 bool pct_tailslow = s->parameters.cpu_throttle_tailslow;
726 int pct_max = s->parameters.max_cpu_throttle;
728 uint64_t throttle_now = cpu_throttle_get_percentage();
729 uint64_t cpu_now, cpu_ideal, throttle_inc;
731 /* We have not started throttling yet. Let's start it. */
732 if (!cpu_throttle_active()) {
733 cpu_throttle_set(pct_initial);
735 /* Throttling already on, just increase the rate */
737 throttle_inc = pct_increment;
739 /* Compute the ideal CPU percentage used by Guest, which may
740 * make the dirty rate match the dirty rate threshold. */
741 cpu_now = 100 - throttle_now;
742 cpu_ideal = cpu_now * (bytes_dirty_threshold * 1.0 /
744 throttle_inc = MIN(cpu_now - cpu_ideal, pct_increment);
746 cpu_throttle_set(MIN(throttle_now + throttle_inc, pct_max));
750 void mig_throttle_counter_reset(void)
752 RAMState *rs = ram_state;
754 rs->time_last_bitmap_sync = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
755 rs->num_dirty_pages_period = 0;
756 rs->bytes_xfer_prev = stat64_get(&ram_atomic_counters.transferred);
760 * xbzrle_cache_zero_page: insert a zero page in the XBZRLE cache
762 * @rs: current RAM state
763 * @current_addr: address for the zero page
765 * Update the xbzrle cache to reflect a page that's been sent as all 0.
766 * The important thing is that a stale (not-yet-0'd) page be replaced
768 * As a bonus, if the page wasn't in the cache it gets added so that
769 * when a small write is made into the 0'd page it gets XBZRLE sent.
771 static void xbzrle_cache_zero_page(RAMState *rs, ram_addr_t current_addr)
773 /* We don't care if this fails to allocate a new cache page
774 * as long as it updated an old one */
775 cache_insert(XBZRLE.cache, current_addr, XBZRLE.zero_target_page,
776 ram_counters.dirty_sync_count);
779 #define ENCODING_FLAG_XBZRLE 0x1
782 * save_xbzrle_page: compress and send current page
784 * Returns: 1 means that we wrote the page
785 * 0 means that page is identical to the one already sent
786 * -1 means that xbzrle would be longer than normal
788 * @rs: current RAM state
789 * @pss: current PSS channel
790 * @current_data: pointer to the address of the page contents
791 * @current_addr: addr of the page
792 * @block: block that contains the page we want to send
793 * @offset: offset inside the block for the page
795 static int save_xbzrle_page(RAMState *rs, PageSearchStatus *pss,
796 uint8_t **current_data, ram_addr_t current_addr,
797 RAMBlock *block, ram_addr_t offset)
799 int encoded_len = 0, bytes_xbzrle;
800 uint8_t *prev_cached_page;
801 QEMUFile *file = pss->pss_channel;
803 if (!cache_is_cached(XBZRLE.cache, current_addr,
804 ram_counters.dirty_sync_count)) {
805 xbzrle_counters.cache_miss++;
806 if (!rs->last_stage) {
807 if (cache_insert(XBZRLE.cache, current_addr, *current_data,
808 ram_counters.dirty_sync_count) == -1) {
811 /* update *current_data when the page has been
812 inserted into cache */
813 *current_data = get_cached_data(XBZRLE.cache, current_addr);
820 * Reaching here means the page has hit the xbzrle cache, no matter what
821 * encoding result it is (normal encoding, overflow or skipping the page),
822 * count the page as encoded. This is used to calculate the encoding rate.
824 * Example: 2 pages (8KB) being encoded, first page encoding generates 2KB,
825 * 2nd page turns out to be skipped (i.e. no new bytes written to the
826 * page), the overall encoding rate will be 8KB / 2KB = 4, which has the
827 * skipped page included. In this way, the encoding rate can tell if the
828 * guest page is good for xbzrle encoding.
830 xbzrle_counters.pages++;
831 prev_cached_page = get_cached_data(XBZRLE.cache, current_addr);
833 /* save current buffer into memory */
834 memcpy(XBZRLE.current_buf, *current_data, TARGET_PAGE_SIZE);
836 /* XBZRLE encoding (if there is no overflow) */
837 encoded_len = xbzrle_encode_buffer_func(prev_cached_page, XBZRLE.current_buf,
838 TARGET_PAGE_SIZE, XBZRLE.encoded_buf,
842 * Update the cache contents, so that it corresponds to the data
843 * sent, in all cases except where we skip the page.
845 if (!rs->last_stage && encoded_len != 0) {
846 memcpy(prev_cached_page, XBZRLE.current_buf, TARGET_PAGE_SIZE);
848 * In the case where we couldn't compress, ensure that the caller
849 * sends the data from the cache, since the guest might have
850 * changed the RAM since we copied it.
852 *current_data = prev_cached_page;
855 if (encoded_len == 0) {
856 trace_save_xbzrle_page_skipping();
858 } else if (encoded_len == -1) {
859 trace_save_xbzrle_page_overflow();
860 xbzrle_counters.overflow++;
861 xbzrle_counters.bytes += TARGET_PAGE_SIZE;
865 /* Send XBZRLE based compressed page */
866 bytes_xbzrle = save_page_header(pss, block,
867 offset | RAM_SAVE_FLAG_XBZRLE);
868 qemu_put_byte(file, ENCODING_FLAG_XBZRLE);
869 qemu_put_be16(file, encoded_len);
870 qemu_put_buffer(file, XBZRLE.encoded_buf, encoded_len);
871 bytes_xbzrle += encoded_len + 1 + 2;
873 * Like compressed_size (please see update_compress_thread_counts),
874 * the xbzrle encoded bytes don't count the 8 byte header with
875 * RAM_SAVE_FLAG_CONTINUE.
877 xbzrle_counters.bytes += bytes_xbzrle - 8;
878 ram_transferred_add(bytes_xbzrle);
884 * pss_find_next_dirty: find the next dirty page of current ramblock
886 * This function updates pss->page to point to the next dirty page index
887 * within the ramblock to migrate, or the end of ramblock when nothing
888 * found. Note that when pss->host_page_sending==true it means we're
889 * during sending a host page, so we won't look for dirty page that is
890 * outside the host page boundary.
892 * @pss: the current page search status
894 static void pss_find_next_dirty(PageSearchStatus *pss)
896 RAMBlock *rb = pss->block;
897 unsigned long size = rb->used_length >> TARGET_PAGE_BITS;
898 unsigned long *bitmap = rb->bmap;
900 if (ramblock_is_ignored(rb)) {
901 /* Points directly to the end, so we know no dirty page */
907 * If during sending a host page, only look for dirty pages within the
908 * current host page being send.
910 if (pss->host_page_sending) {
911 assert(pss->host_page_end);
912 size = MIN(size, pss->host_page_end);
915 pss->page = find_next_bit(bitmap, size, pss->page);
918 static void migration_clear_memory_region_dirty_bitmap(RAMBlock *rb,
924 if (!rb->clear_bmap || !clear_bmap_test_and_clear(rb, page)) {
928 shift = rb->clear_bmap_shift;
930 * CLEAR_BITMAP_SHIFT_MIN should always guarantee this... this
931 * can make things easier sometimes since then start address
932 * of the small chunk will always be 64 pages aligned so the
933 * bitmap will always be aligned to unsigned long. We should
934 * even be able to remove this restriction but I'm simply
939 size = 1ULL << (TARGET_PAGE_BITS + shift);
940 start = QEMU_ALIGN_DOWN((ram_addr_t)page << TARGET_PAGE_BITS, size);
941 trace_migration_bitmap_clear_dirty(rb->idstr, start, size, page);
942 memory_region_clear_dirty_bitmap(rb->mr, start, size);
946 migration_clear_memory_region_dirty_bitmap_range(RAMBlock *rb,
948 unsigned long npages)
950 unsigned long i, chunk_pages = 1UL << rb->clear_bmap_shift;
951 unsigned long chunk_start = QEMU_ALIGN_DOWN(start, chunk_pages);
952 unsigned long chunk_end = QEMU_ALIGN_UP(start + npages, chunk_pages);
955 * Clear pages from start to start + npages - 1, so the end boundary is
958 for (i = chunk_start; i < chunk_end; i += chunk_pages) {
959 migration_clear_memory_region_dirty_bitmap(rb, i);
964 * colo_bitmap_find_diry:find contiguous dirty pages from start
966 * Returns the page offset within memory region of the start of the contiguout
969 * @rs: current RAM state
970 * @rb: RAMBlock where to search for dirty pages
971 * @start: page where we start the search
972 * @num: the number of contiguous dirty pages
975 unsigned long colo_bitmap_find_dirty(RAMState *rs, RAMBlock *rb,
976 unsigned long start, unsigned long *num)
978 unsigned long size = rb->used_length >> TARGET_PAGE_BITS;
979 unsigned long *bitmap = rb->bmap;
980 unsigned long first, next;
984 if (ramblock_is_ignored(rb)) {
988 first = find_next_bit(bitmap, size, start);
992 next = find_next_zero_bit(bitmap, size, first + 1);
993 assert(next >= first);
998 static inline bool migration_bitmap_clear_dirty(RAMState *rs,
1005 * Clear dirty bitmap if needed. This _must_ be called before we
1006 * send any of the page in the chunk because we need to make sure
1007 * we can capture further page content changes when we sync dirty
1008 * log the next time. So as long as we are going to send any of
1009 * the page in the chunk we clear the remote dirty bitmap for all.
1010 * Clearing it earlier won't be a problem, but too late will.
1012 migration_clear_memory_region_dirty_bitmap(rb, page);
1014 ret = test_and_clear_bit(page, rb->bmap);
1016 rs->migration_dirty_pages--;
1022 static void dirty_bitmap_clear_section(MemoryRegionSection *section,
1025 const hwaddr offset = section->offset_within_region;
1026 const hwaddr size = int128_get64(section->size);
1027 const unsigned long start = offset >> TARGET_PAGE_BITS;
1028 const unsigned long npages = size >> TARGET_PAGE_BITS;
1029 RAMBlock *rb = section->mr->ram_block;
1030 uint64_t *cleared_bits = opaque;
1033 * We don't grab ram_state->bitmap_mutex because we expect to run
1034 * only when starting migration or during postcopy recovery where
1035 * we don't have concurrent access.
1037 if (!migration_in_postcopy() && !migrate_background_snapshot()) {
1038 migration_clear_memory_region_dirty_bitmap_range(rb, start, npages);
1040 *cleared_bits += bitmap_count_one_with_offset(rb->bmap, start, npages);
1041 bitmap_clear(rb->bmap, start, npages);
1045 * Exclude all dirty pages from migration that fall into a discarded range as
1046 * managed by a RamDiscardManager responsible for the mapped memory region of
1047 * the RAMBlock. Clear the corresponding bits in the dirty bitmaps.
1049 * Discarded pages ("logically unplugged") have undefined content and must
1050 * not get migrated, because even reading these pages for migration might
1051 * result in undesired behavior.
1053 * Returns the number of cleared bits in the RAMBlock dirty bitmap.
1055 * Note: The result is only stable while migrating (precopy/postcopy).
1057 static uint64_t ramblock_dirty_bitmap_clear_discarded_pages(RAMBlock *rb)
1059 uint64_t cleared_bits = 0;
1061 if (rb->mr && rb->bmap && memory_region_has_ram_discard_manager(rb->mr)) {
1062 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
1063 MemoryRegionSection section = {
1065 .offset_within_region = 0,
1066 .size = int128_make64(qemu_ram_get_used_length(rb)),
1069 ram_discard_manager_replay_discarded(rdm, §ion,
1070 dirty_bitmap_clear_section,
1073 return cleared_bits;
1077 * Check if a host-page aligned page falls into a discarded range as managed by
1078 * a RamDiscardManager responsible for the mapped memory region of the RAMBlock.
1080 * Note: The result is only stable while migrating (precopy/postcopy).
1082 bool ramblock_page_is_discarded(RAMBlock *rb, ram_addr_t start)
1084 if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) {
1085 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
1086 MemoryRegionSection section = {
1088 .offset_within_region = start,
1089 .size = int128_make64(qemu_ram_pagesize(rb)),
1092 return !ram_discard_manager_is_populated(rdm, §ion);
1097 /* Called with RCU critical section */
1098 static void ramblock_sync_dirty_bitmap(RAMState *rs, RAMBlock *rb)
1100 uint64_t new_dirty_pages =
1101 cpu_physical_memory_sync_dirty_bitmap(rb, 0, rb->used_length);
1103 rs->migration_dirty_pages += new_dirty_pages;
1104 rs->num_dirty_pages_period += new_dirty_pages;
1108 * ram_pagesize_summary: calculate all the pagesizes of a VM
1110 * Returns a summary bitmap of the page sizes of all RAMBlocks
1112 * For VMs with just normal pages this is equivalent to the host page
1113 * size. If it's got some huge pages then it's the OR of all the
1114 * different page sizes.
1116 uint64_t ram_pagesize_summary(void)
1119 uint64_t summary = 0;
1121 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1122 summary |= block->page_size;
1128 uint64_t ram_get_total_transferred_pages(void)
1130 return stat64_get(&ram_atomic_counters.normal) +
1131 stat64_get(&ram_atomic_counters.duplicate) +
1132 compression_counters.pages + xbzrle_counters.pages;
1135 static void migration_update_rates(RAMState *rs, int64_t end_time)
1137 uint64_t page_count = rs->target_page_count - rs->target_page_count_prev;
1138 double compressed_size;
1140 /* calculate period counters */
1141 ram_counters.dirty_pages_rate = rs->num_dirty_pages_period * 1000
1142 / (end_time - rs->time_last_bitmap_sync);
1148 if (migrate_use_xbzrle()) {
1149 double encoded_size, unencoded_size;
1151 xbzrle_counters.cache_miss_rate = (double)(xbzrle_counters.cache_miss -
1152 rs->xbzrle_cache_miss_prev) / page_count;
1153 rs->xbzrle_cache_miss_prev = xbzrle_counters.cache_miss;
1154 unencoded_size = (xbzrle_counters.pages - rs->xbzrle_pages_prev) *
1156 encoded_size = xbzrle_counters.bytes - rs->xbzrle_bytes_prev;
1157 if (xbzrle_counters.pages == rs->xbzrle_pages_prev || !encoded_size) {
1158 xbzrle_counters.encoding_rate = 0;
1160 xbzrle_counters.encoding_rate = unencoded_size / encoded_size;
1162 rs->xbzrle_pages_prev = xbzrle_counters.pages;
1163 rs->xbzrle_bytes_prev = xbzrle_counters.bytes;
1166 if (migrate_use_compression()) {
1167 compression_counters.busy_rate = (double)(compression_counters.busy -
1168 rs->compress_thread_busy_prev) / page_count;
1169 rs->compress_thread_busy_prev = compression_counters.busy;
1171 compressed_size = compression_counters.compressed_size -
1172 rs->compressed_size_prev;
1173 if (compressed_size) {
1174 double uncompressed_size = (compression_counters.pages -
1175 rs->compress_pages_prev) * TARGET_PAGE_SIZE;
1177 /* Compression-Ratio = Uncompressed-size / Compressed-size */
1178 compression_counters.compression_rate =
1179 uncompressed_size / compressed_size;
1181 rs->compress_pages_prev = compression_counters.pages;
1182 rs->compressed_size_prev = compression_counters.compressed_size;
1187 static void migration_trigger_throttle(RAMState *rs)
1189 MigrationState *s = migrate_get_current();
1190 uint64_t threshold = s->parameters.throttle_trigger_threshold;
1191 uint64_t bytes_xfer_period =
1192 stat64_get(&ram_atomic_counters.transferred) - rs->bytes_xfer_prev;
1193 uint64_t bytes_dirty_period = rs->num_dirty_pages_period * TARGET_PAGE_SIZE;
1194 uint64_t bytes_dirty_threshold = bytes_xfer_period * threshold / 100;
1196 /* During block migration the auto-converge logic incorrectly detects
1197 * that ram migration makes no progress. Avoid this by disabling the
1198 * throttling logic during the bulk phase of block migration. */
1199 if (migrate_auto_converge() && !blk_mig_bulk_active()) {
1200 /* The following detection logic can be refined later. For now:
1201 Check to see if the ratio between dirtied bytes and the approx.
1202 amount of bytes that just got transferred since the last time
1203 we were in this routine reaches the threshold. If that happens
1204 twice, start or increase throttling. */
1206 if ((bytes_dirty_period > bytes_dirty_threshold) &&
1207 (++rs->dirty_rate_high_cnt >= 2)) {
1208 trace_migration_throttle();
1209 rs->dirty_rate_high_cnt = 0;
1210 mig_throttle_guest_down(bytes_dirty_period,
1211 bytes_dirty_threshold);
1216 static void migration_bitmap_sync(RAMState *rs)
1221 ram_counters.dirty_sync_count++;
1223 if (!rs->time_last_bitmap_sync) {
1224 rs->time_last_bitmap_sync = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
1227 trace_migration_bitmap_sync_start();
1228 memory_global_dirty_log_sync();
1230 qemu_mutex_lock(&rs->bitmap_mutex);
1231 WITH_RCU_READ_LOCK_GUARD() {
1232 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1233 ramblock_sync_dirty_bitmap(rs, block);
1235 ram_counters.remaining = ram_bytes_remaining();
1237 qemu_mutex_unlock(&rs->bitmap_mutex);
1239 memory_global_after_dirty_log_sync();
1240 trace_migration_bitmap_sync_end(rs->num_dirty_pages_period);
1242 end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
1244 /* more than 1 second = 1000 millisecons */
1245 if (end_time > rs->time_last_bitmap_sync + 1000) {
1246 migration_trigger_throttle(rs);
1248 migration_update_rates(rs, end_time);
1250 rs->target_page_count_prev = rs->target_page_count;
1252 /* reset period counters */
1253 rs->time_last_bitmap_sync = end_time;
1254 rs->num_dirty_pages_period = 0;
1255 rs->bytes_xfer_prev = stat64_get(&ram_atomic_counters.transferred);
1257 if (migrate_use_events()) {
1258 qapi_event_send_migration_pass(ram_counters.dirty_sync_count);
1262 static void migration_bitmap_sync_precopy(RAMState *rs)
1264 Error *local_err = NULL;
1267 * The current notifier usage is just an optimization to migration, so we
1268 * don't stop the normal migration process in the error case.
1270 if (precopy_notify(PRECOPY_NOTIFY_BEFORE_BITMAP_SYNC, &local_err)) {
1271 error_report_err(local_err);
1275 migration_bitmap_sync(rs);
1277 if (precopy_notify(PRECOPY_NOTIFY_AFTER_BITMAP_SYNC, &local_err)) {
1278 error_report_err(local_err);
1282 void ram_release_page(const char *rbname, uint64_t offset)
1284 if (!migrate_release_ram() || !migration_in_postcopy()) {
1288 ram_discard_range(rbname, offset, TARGET_PAGE_SIZE);
1292 * save_zero_page_to_file: send the zero page to the file
1294 * Returns the size of data written to the file, 0 means the page is not
1297 * @pss: current PSS channel
1298 * @block: block that contains the page we want to send
1299 * @offset: offset inside the block for the page
1301 static int save_zero_page_to_file(PageSearchStatus *pss,
1302 RAMBlock *block, ram_addr_t offset)
1304 uint8_t *p = block->host + offset;
1305 QEMUFile *file = pss->pss_channel;
1308 if (buffer_is_zero(p, TARGET_PAGE_SIZE)) {
1309 len += save_page_header(pss, block, offset | RAM_SAVE_FLAG_ZERO);
1310 qemu_put_byte(file, 0);
1312 ram_release_page(block->idstr, offset);
1318 * save_zero_page: send the zero page to the stream
1320 * Returns the number of pages written.
1322 * @pss: current PSS channel
1323 * @block: block that contains the page we want to send
1324 * @offset: offset inside the block for the page
1326 static int save_zero_page(PageSearchStatus *pss, RAMBlock *block,
1329 int len = save_zero_page_to_file(pss, block, offset);
1332 stat64_add(&ram_atomic_counters.duplicate, 1);
1333 ram_transferred_add(len);
1340 * @pages: the number of pages written by the control path,
1342 * > 0 - number of pages written
1344 * Return true if the pages has been saved, otherwise false is returned.
1346 static bool control_save_page(PageSearchStatus *pss, RAMBlock *block,
1347 ram_addr_t offset, int *pages)
1349 uint64_t bytes_xmit = 0;
1353 ret = ram_control_save_page(pss->pss_channel, block->offset, offset,
1354 TARGET_PAGE_SIZE, &bytes_xmit);
1355 if (ret == RAM_SAVE_CONTROL_NOT_SUPP) {
1360 ram_transferred_add(bytes_xmit);
1364 if (ret == RAM_SAVE_CONTROL_DELAYED) {
1368 if (bytes_xmit > 0) {
1369 stat64_add(&ram_atomic_counters.normal, 1);
1370 } else if (bytes_xmit == 0) {
1371 stat64_add(&ram_atomic_counters.duplicate, 1);
1378 * directly send the page to the stream
1380 * Returns the number of pages written.
1382 * @pss: current PSS channel
1383 * @block: block that contains the page we want to send
1384 * @offset: offset inside the block for the page
1385 * @buf: the page to be sent
1386 * @async: send to page asyncly
1388 static int save_normal_page(PageSearchStatus *pss, RAMBlock *block,
1389 ram_addr_t offset, uint8_t *buf, bool async)
1391 QEMUFile *file = pss->pss_channel;
1393 ram_transferred_add(save_page_header(pss, block,
1394 offset | RAM_SAVE_FLAG_PAGE));
1396 qemu_put_buffer_async(file, buf, TARGET_PAGE_SIZE,
1397 migrate_release_ram() &&
1398 migration_in_postcopy());
1400 qemu_put_buffer(file, buf, TARGET_PAGE_SIZE);
1402 ram_transferred_add(TARGET_PAGE_SIZE);
1403 stat64_add(&ram_atomic_counters.normal, 1);
1408 * ram_save_page: send the given page to the stream
1410 * Returns the number of pages written.
1412 * >=0 - Number of pages written - this might legally be 0
1413 * if xbzrle noticed the page was the same.
1415 * @rs: current RAM state
1416 * @block: block that contains the page we want to send
1417 * @offset: offset inside the block for the page
1419 static int ram_save_page(RAMState *rs, PageSearchStatus *pss)
1423 bool send_async = true;
1424 RAMBlock *block = pss->block;
1425 ram_addr_t offset = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS;
1426 ram_addr_t current_addr = block->offset + offset;
1428 p = block->host + offset;
1429 trace_ram_save_page(block->idstr, (uint64_t)offset, p);
1431 XBZRLE_cache_lock();
1432 if (rs->xbzrle_enabled && !migration_in_postcopy()) {
1433 pages = save_xbzrle_page(rs, pss, &p, current_addr,
1435 if (!rs->last_stage) {
1436 /* Can't send this cached data async, since the cache page
1437 * might get updated before it gets to the wire
1443 /* XBZRLE overflow or normal page */
1445 pages = save_normal_page(pss, block, offset, p, send_async);
1448 XBZRLE_cache_unlock();
1453 static int ram_save_multifd_page(QEMUFile *file, RAMBlock *block,
1456 if (multifd_queue_page(file, block, offset) < 0) {
1459 stat64_add(&ram_atomic_counters.normal, 1);
1464 static bool do_compress_ram_page(QEMUFile *f, z_stream *stream, RAMBlock *block,
1465 ram_addr_t offset, uint8_t *source_buf)
1467 RAMState *rs = ram_state;
1468 PageSearchStatus *pss = &rs->pss[RAM_CHANNEL_PRECOPY];
1469 uint8_t *p = block->host + offset;
1472 if (save_zero_page_to_file(pss, block, offset)) {
1476 save_page_header(pss, block, offset | RAM_SAVE_FLAG_COMPRESS_PAGE);
1479 * copy it to a internal buffer to avoid it being modified by VM
1480 * so that we can catch up the error during compression and
1483 memcpy(source_buf, p, TARGET_PAGE_SIZE);
1484 ret = qemu_put_compression_data(f, stream, source_buf, TARGET_PAGE_SIZE);
1486 qemu_file_set_error(migrate_get_current()->to_dst_file, ret);
1487 error_report("compressed data failed!");
1493 update_compress_thread_counts(const CompressParam *param, int bytes_xmit)
1495 ram_transferred_add(bytes_xmit);
1497 if (param->zero_page) {
1498 stat64_add(&ram_atomic_counters.duplicate, 1);
1502 /* 8 means a header with RAM_SAVE_FLAG_CONTINUE. */
1503 compression_counters.compressed_size += bytes_xmit - 8;
1504 compression_counters.pages++;
1507 static bool save_page_use_compression(RAMState *rs);
1509 static void flush_compressed_data(RAMState *rs)
1511 MigrationState *ms = migrate_get_current();
1512 int idx, len, thread_count;
1514 if (!save_page_use_compression(rs)) {
1517 thread_count = migrate_compress_threads();
1519 qemu_mutex_lock(&comp_done_lock);
1520 for (idx = 0; idx < thread_count; idx++) {
1521 while (!comp_param[idx].done) {
1522 qemu_cond_wait(&comp_done_cond, &comp_done_lock);
1525 qemu_mutex_unlock(&comp_done_lock);
1527 for (idx = 0; idx < thread_count; idx++) {
1528 qemu_mutex_lock(&comp_param[idx].mutex);
1529 if (!comp_param[idx].quit) {
1530 len = qemu_put_qemu_file(ms->to_dst_file, comp_param[idx].file);
1532 * it's safe to fetch zero_page without holding comp_done_lock
1533 * as there is no further request submitted to the thread,
1534 * i.e, the thread should be waiting for a request at this point.
1536 update_compress_thread_counts(&comp_param[idx], len);
1538 qemu_mutex_unlock(&comp_param[idx].mutex);
1542 static inline void set_compress_params(CompressParam *param, RAMBlock *block,
1545 param->block = block;
1546 param->offset = offset;
1549 static int compress_page_with_multi_thread(RAMBlock *block, ram_addr_t offset)
1551 int idx, thread_count, bytes_xmit = -1, pages = -1;
1552 bool wait = migrate_compress_wait_thread();
1553 MigrationState *ms = migrate_get_current();
1555 thread_count = migrate_compress_threads();
1556 qemu_mutex_lock(&comp_done_lock);
1558 for (idx = 0; idx < thread_count; idx++) {
1559 if (comp_param[idx].done) {
1560 comp_param[idx].done = false;
1561 bytes_xmit = qemu_put_qemu_file(ms->to_dst_file,
1562 comp_param[idx].file);
1563 qemu_mutex_lock(&comp_param[idx].mutex);
1564 set_compress_params(&comp_param[idx], block, offset);
1565 qemu_cond_signal(&comp_param[idx].cond);
1566 qemu_mutex_unlock(&comp_param[idx].mutex);
1568 update_compress_thread_counts(&comp_param[idx], bytes_xmit);
1574 * wait for the free thread if the user specifies 'compress-wait-thread',
1575 * otherwise we will post the page out in the main thread as normal page.
1577 if (pages < 0 && wait) {
1578 qemu_cond_wait(&comp_done_cond, &comp_done_lock);
1581 qemu_mutex_unlock(&comp_done_lock);
1586 #define PAGE_ALL_CLEAN 0
1587 #define PAGE_TRY_AGAIN 1
1588 #define PAGE_DIRTY_FOUND 2
1590 * find_dirty_block: find the next dirty page and update any state
1591 * associated with the search process.
1594 * PAGE_ALL_CLEAN: no dirty page found, give up
1595 * PAGE_TRY_AGAIN: no dirty page found, retry for next block
1596 * PAGE_DIRTY_FOUND: dirty page found
1598 * @rs: current RAM state
1599 * @pss: data about the state of the current dirty page scan
1600 * @again: set to false if the search has scanned the whole of RAM
1602 static int find_dirty_block(RAMState *rs, PageSearchStatus *pss)
1604 /* Update pss->page for the next dirty bit in ramblock */
1605 pss_find_next_dirty(pss);
1607 if (pss->complete_round && pss->block == rs->last_seen_block &&
1608 pss->page >= rs->last_page) {
1610 * We've been once around the RAM and haven't found anything.
1613 return PAGE_ALL_CLEAN;
1615 if (!offset_in_ramblock(pss->block,
1616 ((ram_addr_t)pss->page) << TARGET_PAGE_BITS)) {
1617 /* Didn't find anything in this RAM Block */
1619 pss->block = QLIST_NEXT_RCU(pss->block, next);
1622 * If memory migration starts over, we will meet a dirtied page
1623 * which may still exists in compression threads's ring, so we
1624 * should flush the compressed data to make sure the new page
1625 * is not overwritten by the old one in the destination.
1627 * Also If xbzrle is on, stop using the data compression at this
1628 * point. In theory, xbzrle can do better than compression.
1630 flush_compressed_data(rs);
1632 /* Hit the end of the list */
1633 pss->block = QLIST_FIRST_RCU(&ram_list.blocks);
1634 /* Flag that we've looped */
1635 pss->complete_round = true;
1636 /* After the first round, enable XBZRLE. */
1637 if (migrate_use_xbzrle()) {
1638 rs->xbzrle_enabled = true;
1641 /* Didn't find anything this time, but try again on the new block */
1642 return PAGE_TRY_AGAIN;
1644 /* We've found something */
1645 return PAGE_DIRTY_FOUND;
1650 * unqueue_page: gets a page of the queue
1652 * Helper for 'get_queued_page' - gets a page off the queue
1654 * Returns the block of the page (or NULL if none available)
1656 * @rs: current RAM state
1657 * @offset: used to return the offset within the RAMBlock
1659 static RAMBlock *unqueue_page(RAMState *rs, ram_addr_t *offset)
1661 struct RAMSrcPageRequest *entry;
1662 RAMBlock *block = NULL;
1664 if (!postcopy_has_request(rs)) {
1668 QEMU_LOCK_GUARD(&rs->src_page_req_mutex);
1671 * This should _never_ change even after we take the lock, because no one
1672 * should be taking anything off the request list other than us.
1674 assert(postcopy_has_request(rs));
1676 entry = QSIMPLEQ_FIRST(&rs->src_page_requests);
1678 *offset = entry->offset;
1680 if (entry->len > TARGET_PAGE_SIZE) {
1681 entry->len -= TARGET_PAGE_SIZE;
1682 entry->offset += TARGET_PAGE_SIZE;
1684 memory_region_unref(block->mr);
1685 QSIMPLEQ_REMOVE_HEAD(&rs->src_page_requests, next_req);
1687 migration_consume_urgent_request();
1693 #if defined(__linux__)
1695 * poll_fault_page: try to get next UFFD write fault page and, if pending fault
1696 * is found, return RAM block pointer and page offset
1698 * Returns pointer to the RAMBlock containing faulting page,
1699 * NULL if no write faults are pending
1701 * @rs: current RAM state
1702 * @offset: page offset from the beginning of the block
1704 static RAMBlock *poll_fault_page(RAMState *rs, ram_addr_t *offset)
1706 struct uffd_msg uffd_msg;
1711 if (!migrate_background_snapshot()) {
1715 res = uffd_read_events(rs->uffdio_fd, &uffd_msg, 1);
1720 page_address = (void *)(uintptr_t) uffd_msg.arg.pagefault.address;
1721 block = qemu_ram_block_from_host(page_address, false, offset);
1722 assert(block && (block->flags & RAM_UF_WRITEPROTECT) != 0);
1727 * ram_save_release_protection: release UFFD write protection after
1728 * a range of pages has been saved
1730 * @rs: current RAM state
1731 * @pss: page-search-status structure
1732 * @start_page: index of the first page in the range relative to pss->block
1734 * Returns 0 on success, negative value in case of an error
1736 static int ram_save_release_protection(RAMState *rs, PageSearchStatus *pss,
1737 unsigned long start_page)
1741 /* Check if page is from UFFD-managed region. */
1742 if (pss->block->flags & RAM_UF_WRITEPROTECT) {
1743 void *page_address = pss->block->host + (start_page << TARGET_PAGE_BITS);
1744 uint64_t run_length = (pss->page - start_page) << TARGET_PAGE_BITS;
1746 /* Flush async buffers before un-protect. */
1747 qemu_fflush(pss->pss_channel);
1748 /* Un-protect memory range. */
1749 res = uffd_change_protection(rs->uffdio_fd, page_address, run_length,
1756 /* ram_write_tracking_available: check if kernel supports required UFFD features
1758 * Returns true if supports, false otherwise
1760 bool ram_write_tracking_available(void)
1762 uint64_t uffd_features;
1765 res = uffd_query_features(&uffd_features);
1767 (uffd_features & UFFD_FEATURE_PAGEFAULT_FLAG_WP) != 0);
1770 /* ram_write_tracking_compatible: check if guest configuration is
1771 * compatible with 'write-tracking'
1773 * Returns true if compatible, false otherwise
1775 bool ram_write_tracking_compatible(void)
1777 const uint64_t uffd_ioctls_mask = BIT(_UFFDIO_WRITEPROTECT);
1782 /* Open UFFD file descriptor */
1783 uffd_fd = uffd_create_fd(UFFD_FEATURE_PAGEFAULT_FLAG_WP, false);
1788 RCU_READ_LOCK_GUARD();
1790 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1791 uint64_t uffd_ioctls;
1793 /* Nothing to do with read-only and MMIO-writable regions */
1794 if (block->mr->readonly || block->mr->rom_device) {
1797 /* Try to register block memory via UFFD-IO to track writes */
1798 if (uffd_register_memory(uffd_fd, block->host, block->max_length,
1799 UFFDIO_REGISTER_MODE_WP, &uffd_ioctls)) {
1802 if ((uffd_ioctls & uffd_ioctls_mask) != uffd_ioctls_mask) {
1809 uffd_close_fd(uffd_fd);
1813 static inline void populate_read_range(RAMBlock *block, ram_addr_t offset,
1816 const ram_addr_t end = offset + size;
1819 * We read one byte of each page; this will preallocate page tables if
1820 * required and populate the shared zeropage on MAP_PRIVATE anonymous memory
1821 * where no page was populated yet. This might require adaption when
1822 * supporting other mappings, like shmem.
1824 for (; offset < end; offset += block->page_size) {
1825 char tmp = *((char *)block->host + offset);
1827 /* Don't optimize the read out */
1828 asm volatile("" : "+r" (tmp));
1832 static inline int populate_read_section(MemoryRegionSection *section,
1835 const hwaddr size = int128_get64(section->size);
1836 hwaddr offset = section->offset_within_region;
1837 RAMBlock *block = section->mr->ram_block;
1839 populate_read_range(block, offset, size);
1844 * ram_block_populate_read: preallocate page tables and populate pages in the
1845 * RAM block by reading a byte of each page.
1847 * Since it's solely used for userfault_fd WP feature, here we just
1848 * hardcode page size to qemu_real_host_page_size.
1850 * @block: RAM block to populate
1852 static void ram_block_populate_read(RAMBlock *rb)
1855 * Skip populating all pages that fall into a discarded range as managed by
1856 * a RamDiscardManager responsible for the mapped memory region of the
1857 * RAMBlock. Such discarded ("logically unplugged") parts of a RAMBlock
1858 * must not get populated automatically. We don't have to track
1859 * modifications via userfaultfd WP reliably, because these pages will
1860 * not be part of the migration stream either way -- see
1861 * ramblock_dirty_bitmap_exclude_discarded_pages().
1863 * Note: The result is only stable while migrating (precopy/postcopy).
1865 if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) {
1866 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
1867 MemoryRegionSection section = {
1869 .offset_within_region = 0,
1870 .size = rb->mr->size,
1873 ram_discard_manager_replay_populated(rdm, §ion,
1874 populate_read_section, NULL);
1876 populate_read_range(rb, 0, rb->used_length);
1881 * ram_write_tracking_prepare: prepare for UFFD-WP memory tracking
1883 void ram_write_tracking_prepare(void)
1887 RCU_READ_LOCK_GUARD();
1889 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1890 /* Nothing to do with read-only and MMIO-writable regions */
1891 if (block->mr->readonly || block->mr->rom_device) {
1896 * Populate pages of the RAM block before enabling userfault_fd
1899 * This stage is required since ioctl(UFFDIO_WRITEPROTECT) with
1900 * UFFDIO_WRITEPROTECT_MODE_WP mode setting would silently skip
1901 * pages with pte_none() entries in page table.
1903 ram_block_populate_read(block);
1907 static inline int uffd_protect_section(MemoryRegionSection *section,
1910 const hwaddr size = int128_get64(section->size);
1911 const hwaddr offset = section->offset_within_region;
1912 RAMBlock *rb = section->mr->ram_block;
1913 int uffd_fd = (uintptr_t)opaque;
1915 return uffd_change_protection(uffd_fd, rb->host + offset, size, true,
1919 static int ram_block_uffd_protect(RAMBlock *rb, int uffd_fd)
1921 assert(rb->flags & RAM_UF_WRITEPROTECT);
1923 /* See ram_block_populate_read() */
1924 if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) {
1925 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
1926 MemoryRegionSection section = {
1928 .offset_within_region = 0,
1929 .size = rb->mr->size,
1932 return ram_discard_manager_replay_populated(rdm, §ion,
1933 uffd_protect_section,
1934 (void *)(uintptr_t)uffd_fd);
1936 return uffd_change_protection(uffd_fd, rb->host,
1937 rb->used_length, true, false);
1941 * ram_write_tracking_start: start UFFD-WP memory tracking
1943 * Returns 0 for success or negative value in case of error
1945 int ram_write_tracking_start(void)
1948 RAMState *rs = ram_state;
1951 /* Open UFFD file descriptor */
1952 uffd_fd = uffd_create_fd(UFFD_FEATURE_PAGEFAULT_FLAG_WP, true);
1956 rs->uffdio_fd = uffd_fd;
1958 RCU_READ_LOCK_GUARD();
1960 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1961 /* Nothing to do with read-only and MMIO-writable regions */
1962 if (block->mr->readonly || block->mr->rom_device) {
1966 /* Register block memory with UFFD to track writes */
1967 if (uffd_register_memory(rs->uffdio_fd, block->host,
1968 block->max_length, UFFDIO_REGISTER_MODE_WP, NULL)) {
1971 block->flags |= RAM_UF_WRITEPROTECT;
1972 memory_region_ref(block->mr);
1974 /* Apply UFFD write protection to the block memory range */
1975 if (ram_block_uffd_protect(block, uffd_fd)) {
1979 trace_ram_write_tracking_ramblock_start(block->idstr, block->page_size,
1980 block->host, block->max_length);
1986 error_report("ram_write_tracking_start() failed: restoring initial memory state");
1988 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1989 if ((block->flags & RAM_UF_WRITEPROTECT) == 0) {
1992 uffd_unregister_memory(rs->uffdio_fd, block->host, block->max_length);
1993 /* Cleanup flags and remove reference */
1994 block->flags &= ~RAM_UF_WRITEPROTECT;
1995 memory_region_unref(block->mr);
1998 uffd_close_fd(uffd_fd);
2004 * ram_write_tracking_stop: stop UFFD-WP memory tracking and remove protection
2006 void ram_write_tracking_stop(void)
2008 RAMState *rs = ram_state;
2011 RCU_READ_LOCK_GUARD();
2013 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2014 if ((block->flags & RAM_UF_WRITEPROTECT) == 0) {
2017 uffd_unregister_memory(rs->uffdio_fd, block->host, block->max_length);
2019 trace_ram_write_tracking_ramblock_stop(block->idstr, block->page_size,
2020 block->host, block->max_length);
2022 /* Cleanup flags and remove reference */
2023 block->flags &= ~RAM_UF_WRITEPROTECT;
2024 memory_region_unref(block->mr);
2027 /* Finally close UFFD file descriptor */
2028 uffd_close_fd(rs->uffdio_fd);
2033 /* No target OS support, stubs just fail or ignore */
2035 static RAMBlock *poll_fault_page(RAMState *rs, ram_addr_t *offset)
2043 static int ram_save_release_protection(RAMState *rs, PageSearchStatus *pss,
2044 unsigned long start_page)
2053 bool ram_write_tracking_available(void)
2058 bool ram_write_tracking_compatible(void)
2064 int ram_write_tracking_start(void)
2070 void ram_write_tracking_stop(void)
2074 #endif /* defined(__linux__) */
2077 * get_queued_page: unqueue a page from the postcopy requests
2079 * Skips pages that are already sent (!dirty)
2081 * Returns true if a queued page is found
2083 * @rs: current RAM state
2084 * @pss: data about the state of the current dirty page scan
2086 static bool get_queued_page(RAMState *rs, PageSearchStatus *pss)
2093 block = unqueue_page(rs, &offset);
2095 * We're sending this page, and since it's postcopy nothing else
2096 * will dirty it, and we must make sure it doesn't get sent again
2097 * even if this queue request was received after the background
2098 * search already sent it.
2103 page = offset >> TARGET_PAGE_BITS;
2104 dirty = test_bit(page, block->bmap);
2106 trace_get_queued_page_not_dirty(block->idstr, (uint64_t)offset,
2109 trace_get_queued_page(block->idstr, (uint64_t)offset, page);
2113 } while (block && !dirty);
2117 * Poll write faults too if background snapshot is enabled; that's
2118 * when we have vcpus got blocked by the write protected pages.
2120 block = poll_fault_page(rs, &offset);
2125 * We want the background search to continue from the queued page
2126 * since the guest is likely to want other pages near to the page
2127 * it just requested.
2130 pss->page = offset >> TARGET_PAGE_BITS;
2133 * This unqueued page would break the "one round" check, even is
2136 pss->complete_round = false;
2143 * migration_page_queue_free: drop any remaining pages in the ram
2146 * It should be empty at the end anyway, but in error cases there may
2147 * be some left. in case that there is any page left, we drop it.
2150 static void migration_page_queue_free(RAMState *rs)
2152 struct RAMSrcPageRequest *mspr, *next_mspr;
2153 /* This queue generally should be empty - but in the case of a failed
2154 * migration might have some droppings in.
2156 RCU_READ_LOCK_GUARD();
2157 QSIMPLEQ_FOREACH_SAFE(mspr, &rs->src_page_requests, next_req, next_mspr) {
2158 memory_region_unref(mspr->rb->mr);
2159 QSIMPLEQ_REMOVE_HEAD(&rs->src_page_requests, next_req);
2165 * ram_save_queue_pages: queue the page for transmission
2167 * A request from postcopy destination for example.
2169 * Returns zero on success or negative on error
2171 * @rbname: Name of the RAMBLock of the request. NULL means the
2172 * same that last one.
2173 * @start: starting address from the start of the RAMBlock
2174 * @len: length (in bytes) to send
2176 int ram_save_queue_pages(const char *rbname, ram_addr_t start, ram_addr_t len)
2179 RAMState *rs = ram_state;
2181 ram_counters.postcopy_requests++;
2182 RCU_READ_LOCK_GUARD();
2185 /* Reuse last RAMBlock */
2186 ramblock = rs->last_req_rb;
2190 * Shouldn't happen, we can't reuse the last RAMBlock if
2191 * it's the 1st request.
2193 error_report("ram_save_queue_pages no previous block");
2197 ramblock = qemu_ram_block_by_name(rbname);
2200 /* We shouldn't be asked for a non-existent RAMBlock */
2201 error_report("ram_save_queue_pages no block '%s'", rbname);
2204 rs->last_req_rb = ramblock;
2206 trace_ram_save_queue_pages(ramblock->idstr, start, len);
2207 if (!offset_in_ramblock(ramblock, start + len - 1)) {
2208 error_report("%s request overrun start=" RAM_ADDR_FMT " len="
2209 RAM_ADDR_FMT " blocklen=" RAM_ADDR_FMT,
2210 __func__, start, len, ramblock->used_length);
2215 * When with postcopy preempt, we send back the page directly in the
2218 if (postcopy_preempt_active()) {
2219 ram_addr_t page_start = start >> TARGET_PAGE_BITS;
2220 size_t page_size = qemu_ram_pagesize(ramblock);
2221 PageSearchStatus *pss = &ram_state->pss[RAM_CHANNEL_POSTCOPY];
2224 qemu_mutex_lock(&rs->bitmap_mutex);
2226 pss_init(pss, ramblock, page_start);
2228 * Always use the preempt channel, and make sure it's there. It's
2229 * safe to access without lock, because when rp-thread is running
2230 * we should be the only one who operates on the qemufile
2232 pss->pss_channel = migrate_get_current()->postcopy_qemufile_src;
2233 assert(pss->pss_channel);
2236 * It must be either one or multiple of host page size. Just
2237 * assert; if something wrong we're mostly split brain anyway.
2239 assert(len % page_size == 0);
2241 if (ram_save_host_page_urgent(pss)) {
2242 error_report("%s: ram_save_host_page_urgent() failed: "
2243 "ramblock=%s, start_addr=0x"RAM_ADDR_FMT,
2244 __func__, ramblock->idstr, start);
2249 * NOTE: after ram_save_host_page_urgent() succeeded, pss->page
2250 * will automatically be moved and point to the next host page
2251 * we're going to send, so no need to update here.
2253 * Normally QEMU never sends >1 host page in requests, so
2254 * logically we don't even need that as the loop should only
2255 * run once, but just to be consistent.
2259 qemu_mutex_unlock(&rs->bitmap_mutex);
2264 struct RAMSrcPageRequest *new_entry =
2265 g_new0(struct RAMSrcPageRequest, 1);
2266 new_entry->rb = ramblock;
2267 new_entry->offset = start;
2268 new_entry->len = len;
2270 memory_region_ref(ramblock->mr);
2271 qemu_mutex_lock(&rs->src_page_req_mutex);
2272 QSIMPLEQ_INSERT_TAIL(&rs->src_page_requests, new_entry, next_req);
2273 migration_make_urgent_request();
2274 qemu_mutex_unlock(&rs->src_page_req_mutex);
2279 static bool save_page_use_compression(RAMState *rs)
2281 if (!migrate_use_compression()) {
2286 * If xbzrle is enabled (e.g., after first round of migration), stop
2287 * using the data compression. In theory, xbzrle can do better than
2290 if (rs->xbzrle_enabled) {
2298 * try to compress the page before posting it out, return true if the page
2299 * has been properly handled by compression, otherwise needs other
2300 * paths to handle it
2302 static bool save_compress_page(RAMState *rs, PageSearchStatus *pss,
2303 RAMBlock *block, ram_addr_t offset)
2305 if (!save_page_use_compression(rs)) {
2310 * When starting the process of a new block, the first page of
2311 * the block should be sent out before other pages in the same
2312 * block, and all the pages in last block should have been sent
2313 * out, keeping this order is important, because the 'cont' flag
2314 * is used to avoid resending the block name.
2316 * We post the fist page as normal page as compression will take
2317 * much CPU resource.
2319 if (block != pss->last_sent_block) {
2320 flush_compressed_data(rs);
2324 if (compress_page_with_multi_thread(block, offset) > 0) {
2328 compression_counters.busy++;
2333 * ram_save_target_page_legacy: save one target page
2335 * Returns the number of pages written
2337 * @rs: current RAM state
2338 * @pss: data about the page we want to send
2340 static int ram_save_target_page_legacy(RAMState *rs, PageSearchStatus *pss)
2342 RAMBlock *block = pss->block;
2343 ram_addr_t offset = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS;
2346 if (control_save_page(pss, block, offset, &res)) {
2350 if (save_compress_page(rs, pss, block, offset)) {
2354 res = save_zero_page(pss, block, offset);
2356 /* Must let xbzrle know, otherwise a previous (now 0'd) cached
2357 * page would be stale
2359 if (rs->xbzrle_enabled) {
2360 XBZRLE_cache_lock();
2361 xbzrle_cache_zero_page(rs, block->offset + offset);
2362 XBZRLE_cache_unlock();
2368 * Do not use multifd in postcopy as one whole host page should be
2369 * placed. Meanwhile postcopy requires atomic update of pages, so even
2370 * if host page size == guest page size the dest guest during run may
2371 * still see partially copied pages which is data corruption.
2373 if (migrate_use_multifd() && !migration_in_postcopy()) {
2374 return ram_save_multifd_page(pss->pss_channel, block, offset);
2377 return ram_save_page(rs, pss);
2380 /* Should be called before sending a host page */
2381 static void pss_host_page_prepare(PageSearchStatus *pss)
2383 /* How many guest pages are there in one host page? */
2384 size_t guest_pfns = qemu_ram_pagesize(pss->block) >> TARGET_PAGE_BITS;
2386 pss->host_page_sending = true;
2387 if (guest_pfns <= 1) {
2389 * This covers both when guest psize == host psize, or when guest
2390 * has larger psize than the host (guest_pfns==0).
2392 * For the latter, we always send one whole guest page per
2393 * iteration of the host page (example: an Alpha VM on x86 host
2394 * will have guest psize 8K while host psize 4K).
2396 pss->host_page_start = pss->page;
2397 pss->host_page_end = pss->page + 1;
2400 * The host page spans over multiple guest pages, we send them
2401 * within the same host page iteration.
2403 pss->host_page_start = ROUND_DOWN(pss->page, guest_pfns);
2404 pss->host_page_end = ROUND_UP(pss->page + 1, guest_pfns);
2409 * Whether the page pointed by PSS is within the host page being sent.
2410 * Must be called after a previous pss_host_page_prepare().
2412 static bool pss_within_range(PageSearchStatus *pss)
2414 ram_addr_t ram_addr;
2416 assert(pss->host_page_sending);
2418 /* Over host-page boundary? */
2419 if (pss->page >= pss->host_page_end) {
2423 ram_addr = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS;
2425 return offset_in_ramblock(pss->block, ram_addr);
2428 static void pss_host_page_finish(PageSearchStatus *pss)
2430 pss->host_page_sending = false;
2431 /* This is not needed, but just to reset it */
2432 pss->host_page_start = pss->host_page_end = 0;
2436 * Send an urgent host page specified by `pss'. Need to be called with
2437 * bitmap_mutex held.
2439 * Returns 0 if save host page succeeded, false otherwise.
2441 static int ram_save_host_page_urgent(PageSearchStatus *pss)
2443 bool page_dirty, sent = false;
2444 RAMState *rs = ram_state;
2447 trace_postcopy_preempt_send_host_page(pss->block->idstr, pss->page);
2448 pss_host_page_prepare(pss);
2451 * If precopy is sending the same page, let it be done in precopy, or
2452 * we could send the same page in two channels and none of them will
2453 * receive the whole page.
2455 if (pss_overlap(pss, &ram_state->pss[RAM_CHANNEL_PRECOPY])) {
2456 trace_postcopy_preempt_hit(pss->block->idstr,
2457 pss->page << TARGET_PAGE_BITS);
2462 page_dirty = migration_bitmap_clear_dirty(rs, pss->block, pss->page);
2465 /* Be strict to return code; it must be 1, or what else? */
2466 if (migration_ops->ram_save_target_page(rs, pss) != 1) {
2467 error_report_once("%s: ram_save_target_page failed", __func__);
2473 pss_find_next_dirty(pss);
2474 } while (pss_within_range(pss));
2476 pss_host_page_finish(pss);
2477 /* For urgent requests, flush immediately if sent */
2479 qemu_fflush(pss->pss_channel);
2485 * ram_save_host_page: save a whole host page
2487 * Starting at *offset send pages up to the end of the current host
2488 * page. It's valid for the initial offset to point into the middle of
2489 * a host page in which case the remainder of the hostpage is sent.
2490 * Only dirty target pages are sent. Note that the host page size may
2491 * be a huge page for this block.
2493 * The saving stops at the boundary of the used_length of the block
2494 * if the RAMBlock isn't a multiple of the host page size.
2496 * The caller must be with ram_state.bitmap_mutex held to call this
2497 * function. Note that this function can temporarily release the lock, but
2498 * when the function is returned it'll make sure the lock is still held.
2500 * Returns the number of pages written or negative on error
2502 * @rs: current RAM state
2503 * @pss: data about the page we want to send
2505 static int ram_save_host_page(RAMState *rs, PageSearchStatus *pss)
2507 bool page_dirty, preempt_active = postcopy_preempt_active();
2508 int tmppages, pages = 0;
2509 size_t pagesize_bits =
2510 qemu_ram_pagesize(pss->block) >> TARGET_PAGE_BITS;
2511 unsigned long start_page = pss->page;
2514 if (ramblock_is_ignored(pss->block)) {
2515 error_report("block %s should not be migrated !", pss->block->idstr);
2519 /* Update host page boundary information */
2520 pss_host_page_prepare(pss);
2523 page_dirty = migration_bitmap_clear_dirty(rs, pss->block, pss->page);
2525 /* Check the pages is dirty and if it is send it */
2528 * Properly yield the lock only in postcopy preempt mode
2529 * because both migration thread and rp-return thread can
2530 * operate on the bitmaps.
2532 if (preempt_active) {
2533 qemu_mutex_unlock(&rs->bitmap_mutex);
2535 tmppages = migration_ops->ram_save_target_page(rs, pss);
2536 if (tmppages >= 0) {
2539 * Allow rate limiting to happen in the middle of huge pages if
2540 * something is sent in the current iteration.
2542 if (pagesize_bits > 1 && tmppages > 0) {
2543 migration_rate_limit();
2546 if (preempt_active) {
2547 qemu_mutex_lock(&rs->bitmap_mutex);
2554 pss_host_page_finish(pss);
2558 pss_find_next_dirty(pss);
2559 } while (pss_within_range(pss));
2561 pss_host_page_finish(pss);
2563 res = ram_save_release_protection(rs, pss, start_page);
2564 return (res < 0 ? res : pages);
2568 * ram_find_and_save_block: finds a dirty page and sends it to f
2570 * Called within an RCU critical section.
2572 * Returns the number of pages written where zero means no dirty pages,
2573 * or negative on error
2575 * @rs: current RAM state
2577 * On systems where host-page-size > target-page-size it will send all the
2578 * pages in a host page that are dirty.
2580 static int ram_find_and_save_block(RAMState *rs)
2582 PageSearchStatus *pss = &rs->pss[RAM_CHANNEL_PRECOPY];
2585 /* No dirty page as there is zero RAM */
2586 if (!rs->ram_bytes_total) {
2591 * Always keep last_seen_block/last_page valid during this procedure,
2592 * because find_dirty_block() relies on these values (e.g., we compare
2593 * last_seen_block with pss.block to see whether we searched all the
2594 * ramblocks) to detect the completion of migration. Having NULL value
2595 * of last_seen_block can conditionally cause below loop to run forever.
2597 if (!rs->last_seen_block) {
2598 rs->last_seen_block = QLIST_FIRST_RCU(&ram_list.blocks);
2602 pss_init(pss, rs->last_seen_block, rs->last_page);
2605 if (!get_queued_page(rs, pss)) {
2606 /* priority queue empty, so just search for something dirty */
2607 int res = find_dirty_block(rs, pss);
2608 if (res != PAGE_DIRTY_FOUND) {
2609 if (res == PAGE_ALL_CLEAN) {
2611 } else if (res == PAGE_TRY_AGAIN) {
2616 pages = ram_save_host_page(rs, pss);
2622 rs->last_seen_block = pss->block;
2623 rs->last_page = pss->page;
2628 void acct_update_position(QEMUFile *f, size_t size, bool zero)
2630 uint64_t pages = size / TARGET_PAGE_SIZE;
2633 stat64_add(&ram_atomic_counters.duplicate, pages);
2635 stat64_add(&ram_atomic_counters.normal, pages);
2636 ram_transferred_add(size);
2637 qemu_file_credit_transfer(f, size);
2641 static uint64_t ram_bytes_total_with_ignored(void)
2646 RCU_READ_LOCK_GUARD();
2648 RAMBLOCK_FOREACH_MIGRATABLE(block) {
2649 total += block->used_length;
2654 uint64_t ram_bytes_total(void)
2659 RCU_READ_LOCK_GUARD();
2661 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2662 total += block->used_length;
2667 static void xbzrle_load_setup(void)
2669 XBZRLE.decoded_buf = g_malloc(TARGET_PAGE_SIZE);
2672 static void xbzrle_load_cleanup(void)
2674 g_free(XBZRLE.decoded_buf);
2675 XBZRLE.decoded_buf = NULL;
2678 static void ram_state_cleanup(RAMState **rsp)
2681 migration_page_queue_free(*rsp);
2682 qemu_mutex_destroy(&(*rsp)->bitmap_mutex);
2683 qemu_mutex_destroy(&(*rsp)->src_page_req_mutex);
2689 static void xbzrle_cleanup(void)
2691 XBZRLE_cache_lock();
2693 cache_fini(XBZRLE.cache);
2694 g_free(XBZRLE.encoded_buf);
2695 g_free(XBZRLE.current_buf);
2696 g_free(XBZRLE.zero_target_page);
2697 XBZRLE.cache = NULL;
2698 XBZRLE.encoded_buf = NULL;
2699 XBZRLE.current_buf = NULL;
2700 XBZRLE.zero_target_page = NULL;
2702 XBZRLE_cache_unlock();
2705 static void ram_save_cleanup(void *opaque)
2707 RAMState **rsp = opaque;
2710 /* We don't use dirty log with background snapshots */
2711 if (!migrate_background_snapshot()) {
2712 /* caller have hold iothread lock or is in a bh, so there is
2713 * no writing race against the migration bitmap
2715 if (global_dirty_tracking & GLOBAL_DIRTY_MIGRATION) {
2717 * do not stop dirty log without starting it, since
2718 * memory_global_dirty_log_stop will assert that
2719 * memory_global_dirty_log_start/stop used in pairs
2721 memory_global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION);
2725 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2726 g_free(block->clear_bmap);
2727 block->clear_bmap = NULL;
2728 g_free(block->bmap);
2733 compress_threads_save_cleanup();
2734 ram_state_cleanup(rsp);
2735 g_free(migration_ops);
2736 migration_ops = NULL;
2739 static void ram_state_reset(RAMState *rs)
2743 for (i = 0; i < RAM_CHANNEL_MAX; i++) {
2744 rs->pss[i].last_sent_block = NULL;
2747 rs->last_seen_block = NULL;
2749 rs->last_version = ram_list.version;
2750 rs->xbzrle_enabled = false;
2753 #define MAX_WAIT 50 /* ms, half buffered_file limit */
2755 /* **** functions for postcopy ***** */
2757 void ram_postcopy_migrated_memory_release(MigrationState *ms)
2759 struct RAMBlock *block;
2761 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2762 unsigned long *bitmap = block->bmap;
2763 unsigned long range = block->used_length >> TARGET_PAGE_BITS;
2764 unsigned long run_start = find_next_zero_bit(bitmap, range, 0);
2766 while (run_start < range) {
2767 unsigned long run_end = find_next_bit(bitmap, range, run_start + 1);
2768 ram_discard_range(block->idstr,
2769 ((ram_addr_t)run_start) << TARGET_PAGE_BITS,
2770 ((ram_addr_t)(run_end - run_start))
2771 << TARGET_PAGE_BITS);
2772 run_start = find_next_zero_bit(bitmap, range, run_end + 1);
2778 * postcopy_send_discard_bm_ram: discard a RAMBlock
2780 * Callback from postcopy_each_ram_send_discard for each RAMBlock
2782 * @ms: current migration state
2783 * @block: RAMBlock to discard
2785 static void postcopy_send_discard_bm_ram(MigrationState *ms, RAMBlock *block)
2787 unsigned long end = block->used_length >> TARGET_PAGE_BITS;
2788 unsigned long current;
2789 unsigned long *bitmap = block->bmap;
2791 for (current = 0; current < end; ) {
2792 unsigned long one = find_next_bit(bitmap, end, current);
2793 unsigned long zero, discard_length;
2799 zero = find_next_zero_bit(bitmap, end, one + 1);
2802 discard_length = end - one;
2804 discard_length = zero - one;
2806 postcopy_discard_send_range(ms, one, discard_length);
2807 current = one + discard_length;
2811 static void postcopy_chunk_hostpages_pass(MigrationState *ms, RAMBlock *block);
2814 * postcopy_each_ram_send_discard: discard all RAMBlocks
2816 * Utility for the outgoing postcopy code.
2817 * Calls postcopy_send_discard_bm_ram for each RAMBlock
2818 * passing it bitmap indexes and name.
2819 * (qemu_ram_foreach_block ends up passing unscaled lengths
2820 * which would mean postcopy code would have to deal with target page)
2822 * @ms: current migration state
2824 static void postcopy_each_ram_send_discard(MigrationState *ms)
2826 struct RAMBlock *block;
2828 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2829 postcopy_discard_send_init(ms, block->idstr);
2832 * Deal with TPS != HPS and huge pages. It discard any partially sent
2833 * host-page size chunks, mark any partially dirty host-page size
2834 * chunks as all dirty. In this case the host-page is the host-page
2835 * for the particular RAMBlock, i.e. it might be a huge page.
2837 postcopy_chunk_hostpages_pass(ms, block);
2840 * Postcopy sends chunks of bitmap over the wire, but it
2841 * just needs indexes at this point, avoids it having
2842 * target page specific code.
2844 postcopy_send_discard_bm_ram(ms, block);
2845 postcopy_discard_send_finish(ms);
2850 * postcopy_chunk_hostpages_pass: canonicalize bitmap in hostpages
2852 * Helper for postcopy_chunk_hostpages; it's called twice to
2853 * canonicalize the two bitmaps, that are similar, but one is
2856 * Postcopy requires that all target pages in a hostpage are dirty or
2857 * clean, not a mix. This function canonicalizes the bitmaps.
2859 * @ms: current migration state
2860 * @block: block that contains the page we want to canonicalize
2862 static void postcopy_chunk_hostpages_pass(MigrationState *ms, RAMBlock *block)
2864 RAMState *rs = ram_state;
2865 unsigned long *bitmap = block->bmap;
2866 unsigned int host_ratio = block->page_size / TARGET_PAGE_SIZE;
2867 unsigned long pages = block->used_length >> TARGET_PAGE_BITS;
2868 unsigned long run_start;
2870 if (block->page_size == TARGET_PAGE_SIZE) {
2871 /* Easy case - TPS==HPS for a non-huge page RAMBlock */
2875 /* Find a dirty page */
2876 run_start = find_next_bit(bitmap, pages, 0);
2878 while (run_start < pages) {
2881 * If the start of this run of pages is in the middle of a host
2882 * page, then we need to fixup this host page.
2884 if (QEMU_IS_ALIGNED(run_start, host_ratio)) {
2885 /* Find the end of this run */
2886 run_start = find_next_zero_bit(bitmap, pages, run_start + 1);
2888 * If the end isn't at the start of a host page, then the
2889 * run doesn't finish at the end of a host page
2890 * and we need to discard.
2894 if (!QEMU_IS_ALIGNED(run_start, host_ratio)) {
2896 unsigned long fixup_start_addr = QEMU_ALIGN_DOWN(run_start,
2898 run_start = QEMU_ALIGN_UP(run_start, host_ratio);
2900 /* Clean up the bitmap */
2901 for (page = fixup_start_addr;
2902 page < fixup_start_addr + host_ratio; page++) {
2904 * Remark them as dirty, updating the count for any pages
2905 * that weren't previously dirty.
2907 rs->migration_dirty_pages += !test_and_set_bit(page, bitmap);
2911 /* Find the next dirty page for the next iteration */
2912 run_start = find_next_bit(bitmap, pages, run_start);
2917 * ram_postcopy_send_discard_bitmap: transmit the discard bitmap
2919 * Transmit the set of pages to be discarded after precopy to the target
2920 * these are pages that:
2921 * a) Have been previously transmitted but are now dirty again
2922 * b) Pages that have never been transmitted, this ensures that
2923 * any pages on the destination that have been mapped by background
2924 * tasks get discarded (transparent huge pages is the specific concern)
2925 * Hopefully this is pretty sparse
2927 * @ms: current migration state
2929 void ram_postcopy_send_discard_bitmap(MigrationState *ms)
2931 RAMState *rs = ram_state;
2933 RCU_READ_LOCK_GUARD();
2935 /* This should be our last sync, the src is now paused */
2936 migration_bitmap_sync(rs);
2938 /* Easiest way to make sure we don't resume in the middle of a host-page */
2939 rs->pss[RAM_CHANNEL_PRECOPY].last_sent_block = NULL;
2940 rs->last_seen_block = NULL;
2943 postcopy_each_ram_send_discard(ms);
2945 trace_ram_postcopy_send_discard_bitmap();
2949 * ram_discard_range: discard dirtied pages at the beginning of postcopy
2951 * Returns zero on success
2953 * @rbname: name of the RAMBlock of the request. NULL means the
2954 * same that last one.
2955 * @start: RAMBlock starting page
2956 * @length: RAMBlock size
2958 int ram_discard_range(const char *rbname, uint64_t start, size_t length)
2960 trace_ram_discard_range(rbname, start, length);
2962 RCU_READ_LOCK_GUARD();
2963 RAMBlock *rb = qemu_ram_block_by_name(rbname);
2966 error_report("ram_discard_range: Failed to find block '%s'", rbname);
2971 * On source VM, we don't need to update the received bitmap since
2972 * we don't even have one.
2974 if (rb->receivedmap) {
2975 bitmap_clear(rb->receivedmap, start >> qemu_target_page_bits(),
2976 length >> qemu_target_page_bits());
2979 return ram_block_discard_range(rb, start, length);
2983 * For every allocation, we will try not to crash the VM if the
2984 * allocation failed.
2986 static int xbzrle_init(void)
2988 Error *local_err = NULL;
2990 if (!migrate_use_xbzrle()) {
2994 XBZRLE_cache_lock();
2996 XBZRLE.zero_target_page = g_try_malloc0(TARGET_PAGE_SIZE);
2997 if (!XBZRLE.zero_target_page) {
2998 error_report("%s: Error allocating zero page", __func__);
3002 XBZRLE.cache = cache_init(migrate_xbzrle_cache_size(),
3003 TARGET_PAGE_SIZE, &local_err);
3004 if (!XBZRLE.cache) {
3005 error_report_err(local_err);
3006 goto free_zero_page;
3009 XBZRLE.encoded_buf = g_try_malloc0(TARGET_PAGE_SIZE);
3010 if (!XBZRLE.encoded_buf) {
3011 error_report("%s: Error allocating encoded_buf", __func__);
3015 XBZRLE.current_buf = g_try_malloc(TARGET_PAGE_SIZE);
3016 if (!XBZRLE.current_buf) {
3017 error_report("%s: Error allocating current_buf", __func__);
3018 goto free_encoded_buf;
3021 /* We are all good */
3022 XBZRLE_cache_unlock();
3026 g_free(XBZRLE.encoded_buf);
3027 XBZRLE.encoded_buf = NULL;
3029 cache_fini(XBZRLE.cache);
3030 XBZRLE.cache = NULL;
3032 g_free(XBZRLE.zero_target_page);
3033 XBZRLE.zero_target_page = NULL;
3035 XBZRLE_cache_unlock();
3039 static int ram_state_init(RAMState **rsp)
3041 *rsp = g_try_new0(RAMState, 1);
3044 error_report("%s: Init ramstate fail", __func__);
3048 qemu_mutex_init(&(*rsp)->bitmap_mutex);
3049 qemu_mutex_init(&(*rsp)->src_page_req_mutex);
3050 QSIMPLEQ_INIT(&(*rsp)->src_page_requests);
3051 (*rsp)->ram_bytes_total = ram_bytes_total();
3054 * Count the total number of pages used by ram blocks not including any
3055 * gaps due to alignment or unplugs.
3056 * This must match with the initial values of dirty bitmap.
3058 (*rsp)->migration_dirty_pages = (*rsp)->ram_bytes_total >> TARGET_PAGE_BITS;
3059 ram_state_reset(*rsp);
3064 static void ram_list_init_bitmaps(void)
3066 MigrationState *ms = migrate_get_current();
3068 unsigned long pages;
3071 /* Skip setting bitmap if there is no RAM */
3072 if (ram_bytes_total()) {
3073 shift = ms->clear_bitmap_shift;
3074 if (shift > CLEAR_BITMAP_SHIFT_MAX) {
3075 error_report("clear_bitmap_shift (%u) too big, using "
3076 "max value (%u)", shift, CLEAR_BITMAP_SHIFT_MAX);
3077 shift = CLEAR_BITMAP_SHIFT_MAX;
3078 } else if (shift < CLEAR_BITMAP_SHIFT_MIN) {
3079 error_report("clear_bitmap_shift (%u) too small, using "
3080 "min value (%u)", shift, CLEAR_BITMAP_SHIFT_MIN);
3081 shift = CLEAR_BITMAP_SHIFT_MIN;
3084 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3085 pages = block->max_length >> TARGET_PAGE_BITS;
3087 * The initial dirty bitmap for migration must be set with all
3088 * ones to make sure we'll migrate every guest RAM page to
3090 * Here we set RAMBlock.bmap all to 1 because when rebegin a
3091 * new migration after a failed migration, ram_list.
3092 * dirty_memory[DIRTY_MEMORY_MIGRATION] don't include the whole
3095 block->bmap = bitmap_new(pages);
3096 bitmap_set(block->bmap, 0, pages);
3097 block->clear_bmap_shift = shift;
3098 block->clear_bmap = bitmap_new(clear_bmap_size(pages, shift));
3103 static void migration_bitmap_clear_discarded_pages(RAMState *rs)
3105 unsigned long pages;
3108 RCU_READ_LOCK_GUARD();
3110 RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
3111 pages = ramblock_dirty_bitmap_clear_discarded_pages(rb);
3112 rs->migration_dirty_pages -= pages;
3116 static void ram_init_bitmaps(RAMState *rs)
3118 /* For memory_global_dirty_log_start below. */
3119 qemu_mutex_lock_iothread();
3120 qemu_mutex_lock_ramlist();
3122 WITH_RCU_READ_LOCK_GUARD() {
3123 ram_list_init_bitmaps();
3124 /* We don't use dirty log with background snapshots */
3125 if (!migrate_background_snapshot()) {
3126 memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION);
3127 migration_bitmap_sync_precopy(rs);
3130 qemu_mutex_unlock_ramlist();
3131 qemu_mutex_unlock_iothread();
3134 * After an eventual first bitmap sync, fixup the initial bitmap
3135 * containing all 1s to exclude any discarded pages from migration.
3137 migration_bitmap_clear_discarded_pages(rs);
3140 static int ram_init_all(RAMState **rsp)
3142 if (ram_state_init(rsp)) {
3146 if (xbzrle_init()) {
3147 ram_state_cleanup(rsp);
3151 ram_init_bitmaps(*rsp);
3156 static void ram_state_resume_prepare(RAMState *rs, QEMUFile *out)
3162 * Postcopy is not using xbzrle/compression, so no need for that.
3163 * Also, since source are already halted, we don't need to care
3164 * about dirty page logging as well.
3167 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3168 pages += bitmap_count_one(block->bmap,
3169 block->used_length >> TARGET_PAGE_BITS);
3172 /* This may not be aligned with current bitmaps. Recalculate. */
3173 rs->migration_dirty_pages = pages;
3175 ram_state_reset(rs);
3177 /* Update RAMState cache of output QEMUFile */
3178 rs->pss[RAM_CHANNEL_PRECOPY].pss_channel = out;
3180 trace_ram_state_resume_prepare(pages);
3184 * This function clears bits of the free pages reported by the caller from the
3185 * migration dirty bitmap. @addr is the host address corresponding to the
3186 * start of the continuous guest free pages, and @len is the total bytes of
3189 void qemu_guest_free_page_hint(void *addr, size_t len)
3193 size_t used_len, start, npages;
3194 MigrationState *s = migrate_get_current();
3196 /* This function is currently expected to be used during live migration */
3197 if (!migration_is_setup_or_active(s->state)) {
3201 for (; len > 0; len -= used_len, addr += used_len) {
3202 block = qemu_ram_block_from_host(addr, false, &offset);
3203 if (unlikely(!block || offset >= block->used_length)) {
3205 * The implementation might not support RAMBlock resize during
3206 * live migration, but it could happen in theory with future
3207 * updates. So we add a check here to capture that case.
3209 error_report_once("%s unexpected error", __func__);
3213 if (len <= block->used_length - offset) {
3216 used_len = block->used_length - offset;
3219 start = offset >> TARGET_PAGE_BITS;
3220 npages = used_len >> TARGET_PAGE_BITS;
3222 qemu_mutex_lock(&ram_state->bitmap_mutex);
3224 * The skipped free pages are equavalent to be sent from clear_bmap's
3225 * perspective, so clear the bits from the memory region bitmap which
3226 * are initially set. Otherwise those skipped pages will be sent in
3227 * the next round after syncing from the memory region bitmap.
3229 migration_clear_memory_region_dirty_bitmap_range(block, start, npages);
3230 ram_state->migration_dirty_pages -=
3231 bitmap_count_one_with_offset(block->bmap, start, npages);
3232 bitmap_clear(block->bmap, start, npages);
3233 qemu_mutex_unlock(&ram_state->bitmap_mutex);
3238 * Each of ram_save_setup, ram_save_iterate and ram_save_complete has
3239 * long-running RCU critical section. When rcu-reclaims in the code
3240 * start to become numerous it will be necessary to reduce the
3241 * granularity of these critical sections.
3245 * ram_save_setup: Setup RAM for migration
3247 * Returns zero to indicate success and negative for error
3249 * @f: QEMUFile where to send the data
3250 * @opaque: RAMState pointer
3252 static int ram_save_setup(QEMUFile *f, void *opaque)
3254 RAMState **rsp = opaque;
3258 if (compress_threads_save_setup()) {
3262 /* migration has already setup the bitmap, reuse it. */
3263 if (!migration_in_colo_state()) {
3264 if (ram_init_all(rsp) != 0) {
3265 compress_threads_save_cleanup();
3269 (*rsp)->pss[RAM_CHANNEL_PRECOPY].pss_channel = f;
3271 WITH_RCU_READ_LOCK_GUARD() {
3272 qemu_put_be64(f, ram_bytes_total_with_ignored()
3273 | RAM_SAVE_FLAG_MEM_SIZE);
3275 RAMBLOCK_FOREACH_MIGRATABLE(block) {
3276 qemu_put_byte(f, strlen(block->idstr));
3277 qemu_put_buffer(f, (uint8_t *)block->idstr, strlen(block->idstr));
3278 qemu_put_be64(f, block->used_length);
3279 if (migrate_postcopy_ram() && block->page_size !=
3280 qemu_host_page_size) {
3281 qemu_put_be64(f, block->page_size);
3283 if (migrate_ignore_shared()) {
3284 qemu_put_be64(f, block->mr->addr);
3289 ram_control_before_iterate(f, RAM_CONTROL_SETUP);
3290 ram_control_after_iterate(f, RAM_CONTROL_SETUP);
3292 migration_ops = g_malloc0(sizeof(MigrationOps));
3293 migration_ops->ram_save_target_page = ram_save_target_page_legacy;
3294 ret = multifd_send_sync_main(f);
3299 qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
3306 * ram_save_iterate: iterative stage for migration
3308 * Returns zero to indicate success and negative for error
3310 * @f: QEMUFile where to send the data
3311 * @opaque: RAMState pointer
3313 static int ram_save_iterate(QEMUFile *f, void *opaque)
3315 RAMState **temp = opaque;
3316 RAMState *rs = *temp;
3322 if (blk_mig_bulk_active()) {
3323 /* Avoid transferring ram during bulk phase of block migration as
3324 * the bulk phase will usually take a long time and transferring
3325 * ram updates during that time is pointless. */
3330 * We'll take this lock a little bit long, but it's okay for two reasons.
3331 * Firstly, the only possible other thread to take it is who calls
3332 * qemu_guest_free_page_hint(), which should be rare; secondly, see
3333 * MAX_WAIT (if curious, further see commit 4508bd9ed8053ce) below, which
3334 * guarantees that we'll at least released it in a regular basis.
3336 qemu_mutex_lock(&rs->bitmap_mutex);
3337 WITH_RCU_READ_LOCK_GUARD() {
3338 if (ram_list.version != rs->last_version) {
3339 ram_state_reset(rs);
3342 /* Read version before ram_list.blocks */
3345 ram_control_before_iterate(f, RAM_CONTROL_ROUND);
3347 t0 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
3349 while ((ret = qemu_file_rate_limit(f)) == 0 ||
3350 postcopy_has_request(rs)) {
3353 if (qemu_file_get_error(f)) {
3357 pages = ram_find_and_save_block(rs);
3358 /* no more pages to sent */
3365 qemu_file_set_error(f, pages);
3369 rs->target_page_count += pages;
3372 * During postcopy, it is necessary to make sure one whole host
3373 * page is sent in one chunk.
3375 if (migrate_postcopy_ram()) {
3376 flush_compressed_data(rs);
3380 * we want to check in the 1st loop, just in case it was the 1st
3381 * time and we had to sync the dirty bitmap.
3382 * qemu_clock_get_ns() is a bit expensive, so we only check each
3385 if ((i & 63) == 0) {
3386 uint64_t t1 = (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - t0) /
3388 if (t1 > MAX_WAIT) {
3389 trace_ram_save_iterate_big_wait(t1, i);
3396 qemu_mutex_unlock(&rs->bitmap_mutex);
3399 * Must occur before EOS (or any QEMUFile operation)
3400 * because of RDMA protocol.
3402 ram_control_after_iterate(f, RAM_CONTROL_ROUND);
3406 && migration_is_setup_or_active(migrate_get_current()->state)) {
3407 ret = multifd_send_sync_main(rs->pss[RAM_CHANNEL_PRECOPY].pss_channel);
3412 qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
3414 ram_transferred_add(8);
3416 ret = qemu_file_get_error(f);
3426 * ram_save_complete: function called to send the remaining amount of ram
3428 * Returns zero to indicate success or negative on error
3430 * Called with iothread lock
3432 * @f: QEMUFile where to send the data
3433 * @opaque: RAMState pointer
3435 static int ram_save_complete(QEMUFile *f, void *opaque)
3437 RAMState **temp = opaque;
3438 RAMState *rs = *temp;
3441 rs->last_stage = !migration_in_colo_state();
3443 WITH_RCU_READ_LOCK_GUARD() {
3444 if (!migration_in_postcopy()) {
3445 migration_bitmap_sync_precopy(rs);
3448 ram_control_before_iterate(f, RAM_CONTROL_FINISH);
3450 /* try transferring iterative blocks of memory */
3452 /* flush all remaining blocks regardless of rate limiting */
3453 qemu_mutex_lock(&rs->bitmap_mutex);
3457 pages = ram_find_and_save_block(rs);
3458 /* no more blocks to sent */
3467 qemu_mutex_unlock(&rs->bitmap_mutex);
3469 flush_compressed_data(rs);
3470 ram_control_after_iterate(f, RAM_CONTROL_FINISH);
3477 ret = multifd_send_sync_main(rs->pss[RAM_CHANNEL_PRECOPY].pss_channel);
3482 qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
3488 static void ram_state_pending_estimate(void *opaque,
3489 uint64_t *res_precopy_only,
3490 uint64_t *res_compatible,
3491 uint64_t *res_postcopy_only)
3493 RAMState **temp = opaque;
3494 RAMState *rs = *temp;
3496 uint64_t remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE;
3498 if (migrate_postcopy_ram()) {
3499 /* We can do postcopy, and all the data is postcopiable */
3500 *res_postcopy_only += remaining_size;
3502 *res_precopy_only += remaining_size;
3506 static void ram_state_pending_exact(void *opaque,
3507 uint64_t *res_precopy_only,
3508 uint64_t *res_compatible,
3509 uint64_t *res_postcopy_only)
3511 RAMState **temp = opaque;
3512 RAMState *rs = *temp;
3514 uint64_t remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE;
3516 if (!migration_in_postcopy()) {
3517 qemu_mutex_lock_iothread();
3518 WITH_RCU_READ_LOCK_GUARD() {
3519 migration_bitmap_sync_precopy(rs);
3521 qemu_mutex_unlock_iothread();
3522 remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE;
3525 if (migrate_postcopy_ram()) {
3526 /* We can do postcopy, and all the data is postcopiable */
3527 *res_compatible += remaining_size;
3529 *res_precopy_only += remaining_size;
3533 static int load_xbzrle(QEMUFile *f, ram_addr_t addr, void *host)
3535 unsigned int xh_len;
3537 uint8_t *loaded_data;
3539 /* extract RLE header */
3540 xh_flags = qemu_get_byte(f);
3541 xh_len = qemu_get_be16(f);
3543 if (xh_flags != ENCODING_FLAG_XBZRLE) {
3544 error_report("Failed to load XBZRLE page - wrong compression!");
3548 if (xh_len > TARGET_PAGE_SIZE) {
3549 error_report("Failed to load XBZRLE page - len overflow!");
3552 loaded_data = XBZRLE.decoded_buf;
3553 /* load data and decode */
3554 /* it can change loaded_data to point to an internal buffer */
3555 qemu_get_buffer_in_place(f, &loaded_data, xh_len);
3558 if (xbzrle_decode_buffer(loaded_data, xh_len, host,
3559 TARGET_PAGE_SIZE) == -1) {
3560 error_report("Failed to load XBZRLE page - decode error!");
3568 * ram_block_from_stream: read a RAMBlock id from the migration stream
3570 * Must be called from within a rcu critical section.
3572 * Returns a pointer from within the RCU-protected ram_list.
3574 * @mis: the migration incoming state pointer
3575 * @f: QEMUFile where to read the data from
3576 * @flags: Page flags (mostly to see if it's a continuation of previous block)
3577 * @channel: the channel we're using
3579 static inline RAMBlock *ram_block_from_stream(MigrationIncomingState *mis,
3580 QEMUFile *f, int flags,
3583 RAMBlock *block = mis->last_recv_block[channel];
3587 if (flags & RAM_SAVE_FLAG_CONTINUE) {
3589 error_report("Ack, bad migration stream!");
3595 len = qemu_get_byte(f);
3596 qemu_get_buffer(f, (uint8_t *)id, len);
3599 block = qemu_ram_block_by_name(id);
3601 error_report("Can't find block %s", id);
3605 if (ramblock_is_ignored(block)) {
3606 error_report("block %s should not be migrated !", id);
3610 mis->last_recv_block[channel] = block;
3615 static inline void *host_from_ram_block_offset(RAMBlock *block,
3618 if (!offset_in_ramblock(block, offset)) {
3622 return block->host + offset;
3625 static void *host_page_from_ram_block_offset(RAMBlock *block,
3628 /* Note: Explicitly no check against offset_in_ramblock(). */
3629 return (void *)QEMU_ALIGN_DOWN((uintptr_t)(block->host + offset),
3633 static ram_addr_t host_page_offset_from_ram_block_offset(RAMBlock *block,
3636 return ((uintptr_t)block->host + offset) & (block->page_size - 1);
3639 static inline void *colo_cache_from_block_offset(RAMBlock *block,
3640 ram_addr_t offset, bool record_bitmap)
3642 if (!offset_in_ramblock(block, offset)) {
3645 if (!block->colo_cache) {
3646 error_report("%s: colo_cache is NULL in block :%s",
3647 __func__, block->idstr);
3652 * During colo checkpoint, we need bitmap of these migrated pages.
3653 * It help us to decide which pages in ram cache should be flushed
3654 * into VM's RAM later.
3656 if (record_bitmap &&
3657 !test_and_set_bit(offset >> TARGET_PAGE_BITS, block->bmap)) {
3658 ram_state->migration_dirty_pages++;
3660 return block->colo_cache + offset;
3664 * ram_handle_compressed: handle the zero page case
3666 * If a page (or a whole RDMA chunk) has been
3667 * determined to be zero, then zap it.
3669 * @host: host address for the zero page
3670 * @ch: what the page is filled from. We only support zero
3671 * @size: size of the zero page
3673 void ram_handle_compressed(void *host, uint8_t ch, uint64_t size)
3675 if (ch != 0 || !buffer_is_zero(host, size)) {
3676 memset(host, ch, size);
3680 /* return the size after decompression, or negative value on error */
3682 qemu_uncompress_data(z_stream *stream, uint8_t *dest, size_t dest_len,
3683 const uint8_t *source, size_t source_len)
3687 err = inflateReset(stream);
3692 stream->avail_in = source_len;
3693 stream->next_in = (uint8_t *)source;
3694 stream->avail_out = dest_len;
3695 stream->next_out = dest;
3697 err = inflate(stream, Z_NO_FLUSH);
3698 if (err != Z_STREAM_END) {
3702 return stream->total_out;
3705 static void *do_data_decompress(void *opaque)
3707 DecompressParam *param = opaque;
3708 unsigned long pagesize;
3712 qemu_mutex_lock(¶m->mutex);
3713 while (!param->quit) {
3718 qemu_mutex_unlock(¶m->mutex);
3720 pagesize = TARGET_PAGE_SIZE;
3722 ret = qemu_uncompress_data(¶m->stream, des, pagesize,
3723 param->compbuf, len);
3724 if (ret < 0 && migrate_get_current()->decompress_error_check) {
3725 error_report("decompress data failed");
3726 qemu_file_set_error(decomp_file, ret);
3729 qemu_mutex_lock(&decomp_done_lock);
3731 qemu_cond_signal(&decomp_done_cond);
3732 qemu_mutex_unlock(&decomp_done_lock);
3734 qemu_mutex_lock(¶m->mutex);
3736 qemu_cond_wait(¶m->cond, ¶m->mutex);
3739 qemu_mutex_unlock(¶m->mutex);
3744 static int wait_for_decompress_done(void)
3746 int idx, thread_count;
3748 if (!migrate_use_compression()) {
3752 thread_count = migrate_decompress_threads();
3753 qemu_mutex_lock(&decomp_done_lock);
3754 for (idx = 0; idx < thread_count; idx++) {
3755 while (!decomp_param[idx].done) {
3756 qemu_cond_wait(&decomp_done_cond, &decomp_done_lock);
3759 qemu_mutex_unlock(&decomp_done_lock);
3760 return qemu_file_get_error(decomp_file);
3763 static void compress_threads_load_cleanup(void)
3765 int i, thread_count;
3767 if (!migrate_use_compression()) {
3770 thread_count = migrate_decompress_threads();
3771 for (i = 0; i < thread_count; i++) {
3773 * we use it as a indicator which shows if the thread is
3774 * properly init'd or not
3776 if (!decomp_param[i].compbuf) {
3780 qemu_mutex_lock(&decomp_param[i].mutex);
3781 decomp_param[i].quit = true;
3782 qemu_cond_signal(&decomp_param[i].cond);
3783 qemu_mutex_unlock(&decomp_param[i].mutex);
3785 for (i = 0; i < thread_count; i++) {
3786 if (!decomp_param[i].compbuf) {
3790 qemu_thread_join(decompress_threads + i);
3791 qemu_mutex_destroy(&decomp_param[i].mutex);
3792 qemu_cond_destroy(&decomp_param[i].cond);
3793 inflateEnd(&decomp_param[i].stream);
3794 g_free(decomp_param[i].compbuf);
3795 decomp_param[i].compbuf = NULL;
3797 g_free(decompress_threads);
3798 g_free(decomp_param);
3799 decompress_threads = NULL;
3800 decomp_param = NULL;
3804 static int compress_threads_load_setup(QEMUFile *f)
3806 int i, thread_count;
3808 if (!migrate_use_compression()) {
3812 thread_count = migrate_decompress_threads();
3813 decompress_threads = g_new0(QemuThread, thread_count);
3814 decomp_param = g_new0(DecompressParam, thread_count);
3815 qemu_mutex_init(&decomp_done_lock);
3816 qemu_cond_init(&decomp_done_cond);
3818 for (i = 0; i < thread_count; i++) {
3819 if (inflateInit(&decomp_param[i].stream) != Z_OK) {
3823 decomp_param[i].compbuf = g_malloc0(compressBound(TARGET_PAGE_SIZE));
3824 qemu_mutex_init(&decomp_param[i].mutex);
3825 qemu_cond_init(&decomp_param[i].cond);
3826 decomp_param[i].done = true;
3827 decomp_param[i].quit = false;
3828 qemu_thread_create(decompress_threads + i, "decompress",
3829 do_data_decompress, decomp_param + i,
3830 QEMU_THREAD_JOINABLE);
3834 compress_threads_load_cleanup();
3838 static void decompress_data_with_multi_threads(QEMUFile *f,
3839 void *host, int len)
3841 int idx, thread_count;
3843 thread_count = migrate_decompress_threads();
3844 QEMU_LOCK_GUARD(&decomp_done_lock);
3846 for (idx = 0; idx < thread_count; idx++) {
3847 if (decomp_param[idx].done) {
3848 decomp_param[idx].done = false;
3849 qemu_mutex_lock(&decomp_param[idx].mutex);
3850 qemu_get_buffer(f, decomp_param[idx].compbuf, len);
3851 decomp_param[idx].des = host;
3852 decomp_param[idx].len = len;
3853 qemu_cond_signal(&decomp_param[idx].cond);
3854 qemu_mutex_unlock(&decomp_param[idx].mutex);
3858 if (idx < thread_count) {
3861 qemu_cond_wait(&decomp_done_cond, &decomp_done_lock);
3866 static void colo_init_ram_state(void)
3868 ram_state_init(&ram_state);
3872 * colo cache: this is for secondary VM, we cache the whole
3873 * memory of the secondary VM, it is need to hold the global lock
3874 * to call this helper.
3876 int colo_init_ram_cache(void)
3880 WITH_RCU_READ_LOCK_GUARD() {
3881 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3882 block->colo_cache = qemu_anon_ram_alloc(block->used_length,
3883 NULL, false, false);
3884 if (!block->colo_cache) {
3885 error_report("%s: Can't alloc memory for COLO cache of block %s,"
3886 "size 0x" RAM_ADDR_FMT, __func__, block->idstr,
3887 block->used_length);
3888 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3889 if (block->colo_cache) {
3890 qemu_anon_ram_free(block->colo_cache, block->used_length);
3891 block->colo_cache = NULL;
3896 if (!machine_dump_guest_core(current_machine)) {
3897 qemu_madvise(block->colo_cache, block->used_length,
3898 QEMU_MADV_DONTDUMP);
3904 * Record the dirty pages that sent by PVM, we use this dirty bitmap together
3905 * with to decide which page in cache should be flushed into SVM's RAM. Here
3906 * we use the same name 'ram_bitmap' as for migration.
3908 if (ram_bytes_total()) {
3911 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3912 unsigned long pages = block->max_length >> TARGET_PAGE_BITS;
3913 block->bmap = bitmap_new(pages);
3917 colo_init_ram_state();
3921 /* TODO: duplicated with ram_init_bitmaps */
3922 void colo_incoming_start_dirty_log(void)
3924 RAMBlock *block = NULL;
3925 /* For memory_global_dirty_log_start below. */
3926 qemu_mutex_lock_iothread();
3927 qemu_mutex_lock_ramlist();
3929 memory_global_dirty_log_sync();
3930 WITH_RCU_READ_LOCK_GUARD() {
3931 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3932 ramblock_sync_dirty_bitmap(ram_state, block);
3933 /* Discard this dirty bitmap record */
3934 bitmap_zero(block->bmap, block->max_length >> TARGET_PAGE_BITS);
3936 memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION);
3938 ram_state->migration_dirty_pages = 0;
3939 qemu_mutex_unlock_ramlist();
3940 qemu_mutex_unlock_iothread();
3943 /* It is need to hold the global lock to call this helper */
3944 void colo_release_ram_cache(void)
3948 memory_global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION);
3949 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3950 g_free(block->bmap);
3954 WITH_RCU_READ_LOCK_GUARD() {
3955 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3956 if (block->colo_cache) {
3957 qemu_anon_ram_free(block->colo_cache, block->used_length);
3958 block->colo_cache = NULL;
3962 ram_state_cleanup(&ram_state);
3966 * ram_load_setup: Setup RAM for migration incoming side
3968 * Returns zero to indicate success and negative for error
3970 * @f: QEMUFile where to receive the data
3971 * @opaque: RAMState pointer
3973 static int ram_load_setup(QEMUFile *f, void *opaque)
3975 if (compress_threads_load_setup(f)) {
3979 xbzrle_load_setup();
3980 ramblock_recv_map_init();
3985 static int ram_load_cleanup(void *opaque)
3989 RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
3990 qemu_ram_block_writeback(rb);
3993 xbzrle_load_cleanup();
3994 compress_threads_load_cleanup();
3996 RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
3997 g_free(rb->receivedmap);
3998 rb->receivedmap = NULL;
4005 * ram_postcopy_incoming_init: allocate postcopy data structures
4007 * Returns 0 for success and negative if there was one error
4009 * @mis: current migration incoming state
4011 * Allocate data structures etc needed by incoming migration with
4012 * postcopy-ram. postcopy-ram's similarly names
4013 * postcopy_ram_incoming_init does the work.
4015 int ram_postcopy_incoming_init(MigrationIncomingState *mis)
4017 return postcopy_ram_incoming_init(mis);
4021 * ram_load_postcopy: load a page in postcopy case
4023 * Returns 0 for success or -errno in case of error
4025 * Called in postcopy mode by ram_load().
4026 * rcu_read_lock is taken prior to this being called.
4028 * @f: QEMUFile where to send the data
4029 * @channel: the channel to use for loading
4031 int ram_load_postcopy(QEMUFile *f, int channel)
4033 int flags = 0, ret = 0;
4034 bool place_needed = false;
4035 bool matches_target_page_size = false;
4036 MigrationIncomingState *mis = migration_incoming_get_current();
4037 PostcopyTmpPage *tmp_page = &mis->postcopy_tmp_pages[channel];
4039 while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) {
4041 void *page_buffer = NULL;
4042 void *place_source = NULL;
4043 RAMBlock *block = NULL;
4047 addr = qemu_get_be64(f);
4050 * If qemu file error, we should stop here, and then "addr"
4053 ret = qemu_file_get_error(f);
4058 flags = addr & ~TARGET_PAGE_MASK;
4059 addr &= TARGET_PAGE_MASK;
4061 trace_ram_load_postcopy_loop(channel, (uint64_t)addr, flags);
4062 if (flags & (RAM_SAVE_FLAG_ZERO | RAM_SAVE_FLAG_PAGE |
4063 RAM_SAVE_FLAG_COMPRESS_PAGE)) {
4064 block = ram_block_from_stream(mis, f, flags, channel);
4071 * Relying on used_length is racy and can result in false positives.
4072 * We might place pages beyond used_length in case RAM was shrunk
4073 * while in postcopy, which is fine - trying to place via
4074 * UFFDIO_COPY/UFFDIO_ZEROPAGE will never segfault.
4076 if (!block->host || addr >= block->postcopy_length) {
4077 error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
4081 tmp_page->target_pages++;
4082 matches_target_page_size = block->page_size == TARGET_PAGE_SIZE;
4084 * Postcopy requires that we place whole host pages atomically;
4085 * these may be huge pages for RAMBlocks that are backed by
4087 * To make it atomic, the data is read into a temporary page
4088 * that's moved into place later.
4089 * The migration protocol uses, possibly smaller, target-pages
4090 * however the source ensures it always sends all the components
4091 * of a host page in one chunk.
4093 page_buffer = tmp_page->tmp_huge_page +
4094 host_page_offset_from_ram_block_offset(block, addr);
4095 /* If all TP are zero then we can optimise the place */
4096 if (tmp_page->target_pages == 1) {
4097 tmp_page->host_addr =
4098 host_page_from_ram_block_offset(block, addr);
4099 } else if (tmp_page->host_addr !=
4100 host_page_from_ram_block_offset(block, addr)) {
4101 /* not the 1st TP within the HP */
4102 error_report("Non-same host page detected on channel %d: "
4103 "Target host page %p, received host page %p "
4104 "(rb %s offset 0x"RAM_ADDR_FMT" target_pages %d)",
4105 channel, tmp_page->host_addr,
4106 host_page_from_ram_block_offset(block, addr),
4107 block->idstr, addr, tmp_page->target_pages);
4113 * If it's the last part of a host page then we place the host
4116 if (tmp_page->target_pages ==
4117 (block->page_size / TARGET_PAGE_SIZE)) {
4118 place_needed = true;
4120 place_source = tmp_page->tmp_huge_page;
4123 switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
4124 case RAM_SAVE_FLAG_ZERO:
4125 ch = qemu_get_byte(f);
4127 * Can skip to set page_buffer when
4128 * this is a zero page and (block->page_size == TARGET_PAGE_SIZE).
4130 if (ch || !matches_target_page_size) {
4131 memset(page_buffer, ch, TARGET_PAGE_SIZE);
4134 tmp_page->all_zero = false;
4138 case RAM_SAVE_FLAG_PAGE:
4139 tmp_page->all_zero = false;
4140 if (!matches_target_page_size) {
4141 /* For huge pages, we always use temporary buffer */
4142 qemu_get_buffer(f, page_buffer, TARGET_PAGE_SIZE);
4145 * For small pages that matches target page size, we
4146 * avoid the qemu_file copy. Instead we directly use
4147 * the buffer of QEMUFile to place the page. Note: we
4148 * cannot do any QEMUFile operation before using that
4149 * buffer to make sure the buffer is valid when
4152 qemu_get_buffer_in_place(f, (uint8_t **)&place_source,
4156 case RAM_SAVE_FLAG_COMPRESS_PAGE:
4157 tmp_page->all_zero = false;
4158 len = qemu_get_be32(f);
4159 if (len < 0 || len > compressBound(TARGET_PAGE_SIZE)) {
4160 error_report("Invalid compressed data length: %d", len);
4164 decompress_data_with_multi_threads(f, page_buffer, len);
4167 case RAM_SAVE_FLAG_EOS:
4169 multifd_recv_sync_main();
4172 error_report("Unknown combination of migration flags: 0x%x"
4173 " (postcopy mode)", flags);
4178 /* Got the whole host page, wait for decompress before placing. */
4180 ret |= wait_for_decompress_done();
4183 /* Detect for any possible file errors */
4184 if (!ret && qemu_file_get_error(f)) {
4185 ret = qemu_file_get_error(f);
4188 if (!ret && place_needed) {
4189 if (tmp_page->all_zero) {
4190 ret = postcopy_place_page_zero(mis, tmp_page->host_addr, block);
4192 ret = postcopy_place_page(mis, tmp_page->host_addr,
4193 place_source, block);
4195 place_needed = false;
4196 postcopy_temp_page_reset(tmp_page);
4203 static bool postcopy_is_running(void)
4205 PostcopyState ps = postcopy_state_get();
4206 return ps >= POSTCOPY_INCOMING_LISTENING && ps < POSTCOPY_INCOMING_END;
4210 * Flush content of RAM cache into SVM's memory.
4211 * Only flush the pages that be dirtied by PVM or SVM or both.
4213 void colo_flush_ram_cache(void)
4215 RAMBlock *block = NULL;
4218 unsigned long offset = 0;
4220 memory_global_dirty_log_sync();
4221 WITH_RCU_READ_LOCK_GUARD() {
4222 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
4223 ramblock_sync_dirty_bitmap(ram_state, block);
4227 trace_colo_flush_ram_cache_begin(ram_state->migration_dirty_pages);
4228 WITH_RCU_READ_LOCK_GUARD() {
4229 block = QLIST_FIRST_RCU(&ram_list.blocks);
4232 unsigned long num = 0;
4234 offset = colo_bitmap_find_dirty(ram_state, block, offset, &num);
4235 if (!offset_in_ramblock(block,
4236 ((ram_addr_t)offset) << TARGET_PAGE_BITS)) {
4239 block = QLIST_NEXT_RCU(block, next);
4241 unsigned long i = 0;
4243 for (i = 0; i < num; i++) {
4244 migration_bitmap_clear_dirty(ram_state, block, offset + i);
4246 dst_host = block->host
4247 + (((ram_addr_t)offset) << TARGET_PAGE_BITS);
4248 src_host = block->colo_cache
4249 + (((ram_addr_t)offset) << TARGET_PAGE_BITS);
4250 memcpy(dst_host, src_host, TARGET_PAGE_SIZE * num);
4255 trace_colo_flush_ram_cache_end();
4259 * ram_load_precopy: load pages in precopy case
4261 * Returns 0 for success or -errno in case of error
4263 * Called in precopy mode by ram_load().
4264 * rcu_read_lock is taken prior to this being called.
4266 * @f: QEMUFile where to send the data
4268 static int ram_load_precopy(QEMUFile *f)
4270 MigrationIncomingState *mis = migration_incoming_get_current();
4271 int flags = 0, ret = 0, invalid_flags = 0, len = 0, i = 0;
4272 /* ADVISE is earlier, it shows the source has the postcopy capability on */
4273 bool postcopy_advised = migration_incoming_postcopy_advised();
4274 if (!migrate_use_compression()) {
4275 invalid_flags |= RAM_SAVE_FLAG_COMPRESS_PAGE;
4278 while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) {
4279 ram_addr_t addr, total_ram_bytes;
4280 void *host = NULL, *host_bak = NULL;
4284 * Yield periodically to let main loop run, but an iteration of
4285 * the main loop is expensive, so do it each some iterations
4287 if ((i & 32767) == 0 && qemu_in_coroutine()) {
4288 aio_co_schedule(qemu_get_current_aio_context(),
4289 qemu_coroutine_self());
4290 qemu_coroutine_yield();
4294 addr = qemu_get_be64(f);
4295 flags = addr & ~TARGET_PAGE_MASK;
4296 addr &= TARGET_PAGE_MASK;
4298 if (flags & invalid_flags) {
4299 if (flags & invalid_flags & RAM_SAVE_FLAG_COMPRESS_PAGE) {
4300 error_report("Received an unexpected compressed page");
4307 if (flags & (RAM_SAVE_FLAG_ZERO | RAM_SAVE_FLAG_PAGE |
4308 RAM_SAVE_FLAG_COMPRESS_PAGE | RAM_SAVE_FLAG_XBZRLE)) {
4309 RAMBlock *block = ram_block_from_stream(mis, f, flags,
4310 RAM_CHANNEL_PRECOPY);
4312 host = host_from_ram_block_offset(block, addr);
4314 * After going into COLO stage, we should not load the page
4315 * into SVM's memory directly, we put them into colo_cache firstly.
4316 * NOTE: We need to keep a copy of SVM's ram in colo_cache.
4317 * Previously, we copied all these memory in preparing stage of COLO
4318 * while we need to stop VM, which is a time-consuming process.
4319 * Here we optimize it by a trick, back-up every page while in
4320 * migration process while COLO is enabled, though it affects the
4321 * speed of the migration, but it obviously reduce the downtime of
4322 * back-up all SVM'S memory in COLO preparing stage.
4324 if (migration_incoming_colo_enabled()) {
4325 if (migration_incoming_in_colo_state()) {
4326 /* In COLO stage, put all pages into cache temporarily */
4327 host = colo_cache_from_block_offset(block, addr, true);
4330 * In migration stage but before COLO stage,
4331 * Put all pages into both cache and SVM's memory.
4333 host_bak = colo_cache_from_block_offset(block, addr, false);
4337 error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
4341 if (!migration_incoming_in_colo_state()) {
4342 ramblock_recv_bitmap_set(block, host);
4345 trace_ram_load_loop(block->idstr, (uint64_t)addr, flags, host);
4348 switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
4349 case RAM_SAVE_FLAG_MEM_SIZE:
4350 /* Synchronize RAM block list */
4351 total_ram_bytes = addr;
4352 while (!ret && total_ram_bytes) {
4357 len = qemu_get_byte(f);
4358 qemu_get_buffer(f, (uint8_t *)id, len);
4360 length = qemu_get_be64(f);
4362 block = qemu_ram_block_by_name(id);
4363 if (block && !qemu_ram_is_migratable(block)) {
4364 error_report("block %s should not be migrated !", id);
4367 if (length != block->used_length) {
4368 Error *local_err = NULL;
4370 ret = qemu_ram_resize(block, length,
4373 error_report_err(local_err);
4376 /* For postcopy we need to check hugepage sizes match */
4377 if (postcopy_advised && migrate_postcopy_ram() &&
4378 block->page_size != qemu_host_page_size) {
4379 uint64_t remote_page_size = qemu_get_be64(f);
4380 if (remote_page_size != block->page_size) {
4381 error_report("Mismatched RAM page size %s "
4382 "(local) %zd != %" PRId64,
4383 id, block->page_size,
4388 if (migrate_ignore_shared()) {
4389 hwaddr addr = qemu_get_be64(f);
4390 if (ramblock_is_ignored(block) &&
4391 block->mr->addr != addr) {
4392 error_report("Mismatched GPAs for block %s "
4393 "%" PRId64 "!= %" PRId64,
4395 (uint64_t)block->mr->addr);
4399 ram_control_load_hook(f, RAM_CONTROL_BLOCK_REG,
4402 error_report("Unknown ramblock \"%s\", cannot "
4403 "accept migration", id);
4407 total_ram_bytes -= length;
4411 case RAM_SAVE_FLAG_ZERO:
4412 ch = qemu_get_byte(f);
4413 ram_handle_compressed(host, ch, TARGET_PAGE_SIZE);
4416 case RAM_SAVE_FLAG_PAGE:
4417 qemu_get_buffer(f, host, TARGET_PAGE_SIZE);
4420 case RAM_SAVE_FLAG_COMPRESS_PAGE:
4421 len = qemu_get_be32(f);
4422 if (len < 0 || len > compressBound(TARGET_PAGE_SIZE)) {
4423 error_report("Invalid compressed data length: %d", len);
4427 decompress_data_with_multi_threads(f, host, len);
4430 case RAM_SAVE_FLAG_XBZRLE:
4431 if (load_xbzrle(f, addr, host) < 0) {
4432 error_report("Failed to decompress XBZRLE page at "
4433 RAM_ADDR_FMT, addr);
4438 case RAM_SAVE_FLAG_EOS:
4440 multifd_recv_sync_main();
4443 if (flags & RAM_SAVE_FLAG_HOOK) {
4444 ram_control_load_hook(f, RAM_CONTROL_HOOK, NULL);
4446 error_report("Unknown combination of migration flags: 0x%x",
4452 ret = qemu_file_get_error(f);
4454 if (!ret && host_bak) {
4455 memcpy(host_bak, host, TARGET_PAGE_SIZE);
4459 ret |= wait_for_decompress_done();
4463 static int ram_load(QEMUFile *f, void *opaque, int version_id)
4466 static uint64_t seq_iter;
4468 * If system is running in postcopy mode, page inserts to host memory must
4471 bool postcopy_running = postcopy_is_running();
4475 if (version_id != 4) {
4480 * This RCU critical section can be very long running.
4481 * When RCU reclaims in the code start to become numerous,
4482 * it will be necessary to reduce the granularity of this
4485 WITH_RCU_READ_LOCK_GUARD() {
4486 if (postcopy_running) {
4488 * Note! Here RAM_CHANNEL_PRECOPY is the precopy channel of
4489 * postcopy migration, we have another RAM_CHANNEL_POSTCOPY to
4490 * service fast page faults.
4492 ret = ram_load_postcopy(f, RAM_CHANNEL_PRECOPY);
4494 ret = ram_load_precopy(f);
4497 trace_ram_load_complete(ret, seq_iter);
4502 static bool ram_has_postcopy(void *opaque)
4505 RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
4506 if (ramblock_is_pmem(rb)) {
4507 info_report("Block: %s, host: %p is a nvdimm memory, postcopy"
4508 "is not supported now!", rb->idstr, rb->host);
4513 return migrate_postcopy_ram();
4516 /* Sync all the dirty bitmap with destination VM. */
4517 static int ram_dirty_bitmap_sync_all(MigrationState *s, RAMState *rs)
4520 QEMUFile *file = s->to_dst_file;
4521 int ramblock_count = 0;
4523 trace_ram_dirty_bitmap_sync_start();
4525 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
4526 qemu_savevm_send_recv_bitmap(file, block->idstr);
4527 trace_ram_dirty_bitmap_request(block->idstr);
4531 trace_ram_dirty_bitmap_sync_wait();
4533 /* Wait until all the ramblocks' dirty bitmap synced */
4534 while (ramblock_count--) {
4535 qemu_sem_wait(&s->rp_state.rp_sem);
4538 trace_ram_dirty_bitmap_sync_complete();
4543 static void ram_dirty_bitmap_reload_notify(MigrationState *s)
4545 qemu_sem_post(&s->rp_state.rp_sem);
4549 * Read the received bitmap, revert it as the initial dirty bitmap.
4550 * This is only used when the postcopy migration is paused but wants
4551 * to resume from a middle point.
4553 int ram_dirty_bitmap_reload(MigrationState *s, RAMBlock *block)
4556 /* from_dst_file is always valid because we're within rp_thread */
4557 QEMUFile *file = s->rp_state.from_dst_file;
4558 unsigned long *le_bitmap, nbits = block->used_length >> TARGET_PAGE_BITS;
4559 uint64_t local_size = DIV_ROUND_UP(nbits, 8);
4560 uint64_t size, end_mark;
4562 trace_ram_dirty_bitmap_reload_begin(block->idstr);
4564 if (s->state != MIGRATION_STATUS_POSTCOPY_RECOVER) {
4565 error_report("%s: incorrect state %s", __func__,
4566 MigrationStatus_str(s->state));
4571 * Note: see comments in ramblock_recv_bitmap_send() on why we
4572 * need the endianness conversion, and the paddings.
4574 local_size = ROUND_UP(local_size, 8);
4577 le_bitmap = bitmap_new(nbits + BITS_PER_LONG);
4579 size = qemu_get_be64(file);
4581 /* The size of the bitmap should match with our ramblock */
4582 if (size != local_size) {
4583 error_report("%s: ramblock '%s' bitmap size mismatch "
4584 "(0x%"PRIx64" != 0x%"PRIx64")", __func__,
4585 block->idstr, size, local_size);
4590 size = qemu_get_buffer(file, (uint8_t *)le_bitmap, local_size);
4591 end_mark = qemu_get_be64(file);
4593 ret = qemu_file_get_error(file);
4594 if (ret || size != local_size) {
4595 error_report("%s: read bitmap failed for ramblock '%s': %d"
4596 " (size 0x%"PRIx64", got: 0x%"PRIx64")",
4597 __func__, block->idstr, ret, local_size, size);
4602 if (end_mark != RAMBLOCK_RECV_BITMAP_ENDING) {
4603 error_report("%s: ramblock '%s' end mark incorrect: 0x%"PRIx64,
4604 __func__, block->idstr, end_mark);
4610 * Endianness conversion. We are during postcopy (though paused).
4611 * The dirty bitmap won't change. We can directly modify it.
4613 bitmap_from_le(block->bmap, le_bitmap, nbits);
4616 * What we received is "received bitmap". Revert it as the initial
4617 * dirty bitmap for this ramblock.
4619 bitmap_complement(block->bmap, block->bmap, nbits);
4621 /* Clear dirty bits of discarded ranges that we don't want to migrate. */
4622 ramblock_dirty_bitmap_clear_discarded_pages(block);
4624 /* We'll recalculate migration_dirty_pages in ram_state_resume_prepare(). */
4625 trace_ram_dirty_bitmap_reload_complete(block->idstr);
4628 * We succeeded to sync bitmap for current ramblock. If this is
4629 * the last one to sync, we need to notify the main send thread.
4631 ram_dirty_bitmap_reload_notify(s);
4639 static int ram_resume_prepare(MigrationState *s, void *opaque)
4641 RAMState *rs = *(RAMState **)opaque;
4644 ret = ram_dirty_bitmap_sync_all(s, rs);
4649 ram_state_resume_prepare(rs, s->to_dst_file);
4654 void postcopy_preempt_shutdown_file(MigrationState *s)
4656 qemu_put_be64(s->postcopy_qemufile_src, RAM_SAVE_FLAG_EOS);
4657 qemu_fflush(s->postcopy_qemufile_src);
4660 static SaveVMHandlers savevm_ram_handlers = {
4661 .save_setup = ram_save_setup,
4662 .save_live_iterate = ram_save_iterate,
4663 .save_live_complete_postcopy = ram_save_complete,
4664 .save_live_complete_precopy = ram_save_complete,
4665 .has_postcopy = ram_has_postcopy,
4666 .state_pending_exact = ram_state_pending_exact,
4667 .state_pending_estimate = ram_state_pending_estimate,
4668 .load_state = ram_load,
4669 .save_cleanup = ram_save_cleanup,
4670 .load_setup = ram_load_setup,
4671 .load_cleanup = ram_load_cleanup,
4672 .resume_prepare = ram_resume_prepare,
4675 static void ram_mig_ram_block_resized(RAMBlockNotifier *n, void *host,
4676 size_t old_size, size_t new_size)
4678 PostcopyState ps = postcopy_state_get();
4680 RAMBlock *rb = qemu_ram_block_from_host(host, false, &offset);
4683 if (ramblock_is_ignored(rb)) {
4687 if (!migration_is_idle()) {
4689 * Precopy code on the source cannot deal with the size of RAM blocks
4690 * changing at random points in time - especially after sending the
4691 * RAM block sizes in the migration stream, they must no longer change.
4692 * Abort and indicate a proper reason.
4694 error_setg(&err, "RAM block '%s' resized during precopy.", rb->idstr);
4695 migration_cancel(err);
4700 case POSTCOPY_INCOMING_ADVISE:
4702 * Update what ram_postcopy_incoming_init()->init_range() does at the
4703 * time postcopy was advised. Syncing RAM blocks with the source will
4704 * result in RAM resizes.
4706 if (old_size < new_size) {
4707 if (ram_discard_range(rb->idstr, old_size, new_size - old_size)) {
4708 error_report("RAM block '%s' discard of resized RAM failed",
4712 rb->postcopy_length = new_size;
4714 case POSTCOPY_INCOMING_NONE:
4715 case POSTCOPY_INCOMING_RUNNING:
4716 case POSTCOPY_INCOMING_END:
4718 * Once our guest is running, postcopy does no longer care about
4719 * resizes. When growing, the new memory was not available on the
4720 * source, no handler needed.
4724 error_report("RAM block '%s' resized during postcopy state: %d",
4730 static RAMBlockNotifier ram_mig_ram_notifier = {
4731 .ram_block_resized = ram_mig_ram_block_resized,
4734 void ram_mig_init(void)
4736 qemu_mutex_init(&XBZRLE.lock);
4737 register_savevm_live("ram", 0, 4, &savevm_ram_handlers, &ram_state);
4738 ram_block_notifier_add(&ram_mig_ram_notifier);