4 * Copyright (c) 2003-2008 Fabrice Bellard
5 * Copyright (c) 2011-2015 Red Hat Inc
8 * Juan Quintela <quintela@redhat.com>
10 * Permission is hereby granted, free of charge, to any person obtaining a copy
11 * of this software and associated documentation files (the "Software"), to deal
12 * in the Software without restriction, including without limitation the rights
13 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the Software is
15 * furnished to do so, subject to the following conditions:
17 * The above copyright notice and this permission notice shall be included in
18 * all copies or substantial portions of the Software.
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
29 #include "qemu/osdep.h"
30 #include "qemu/cutils.h"
31 #include "qemu/bitops.h"
32 #include "qemu/bitmap.h"
33 #include "qemu/madvise.h"
34 #include "qemu/main-loop.h"
36 #include "ram-compress.h"
38 #include "migration.h"
39 #include "migration-stats.h"
40 #include "migration/register.h"
41 #include "migration/misc.h"
42 #include "qemu-file.h"
43 #include "postcopy-ram.h"
44 #include "page_cache.h"
45 #include "qemu/error-report.h"
46 #include "qapi/error.h"
47 #include "qapi/qapi-types-migration.h"
48 #include "qapi/qapi-events-migration.h"
49 #include "qapi/qapi-commands-migration.h"
50 #include "qapi/qmp/qerror.h"
52 #include "exec/ram_addr.h"
53 #include "exec/target_page.h"
54 #include "qemu/rcu_queue.h"
55 #include "migration/colo.h"
57 #include "sysemu/cpu-throttle.h"
61 #include "sysemu/runstate.h"
63 #include "sysemu/dirtylimit.h"
64 #include "sysemu/kvm.h"
66 #include "hw/boards.h" /* for machine_dump_guest_core() */
68 #if defined(__linux__)
69 #include "qemu/userfaultfd.h"
70 #endif /* defined(__linux__) */
72 /***********************************************************/
73 /* ram save/restore */
76 * RAM_SAVE_FLAG_ZERO used to be named RAM_SAVE_FLAG_COMPRESS, it
77 * worked for pages that were filled with the same char. We switched
78 * it to only search for the zero value. And to avoid confusion with
79 * RAM_SAVE_FLAG_COMPRESS_PAGE just rename it.
82 * RAM_SAVE_FLAG_FULL was obsoleted in 2009, it can be reused now
84 #define RAM_SAVE_FLAG_FULL 0x01
85 #define RAM_SAVE_FLAG_ZERO 0x02
86 #define RAM_SAVE_FLAG_MEM_SIZE 0x04
87 #define RAM_SAVE_FLAG_PAGE 0x08
88 #define RAM_SAVE_FLAG_EOS 0x10
89 #define RAM_SAVE_FLAG_CONTINUE 0x20
90 #define RAM_SAVE_FLAG_XBZRLE 0x40
91 /* 0x80 is reserved in qemu-file.h for RAM_SAVE_FLAG_HOOK */
92 #define RAM_SAVE_FLAG_COMPRESS_PAGE 0x100
93 #define RAM_SAVE_FLAG_MULTIFD_FLUSH 0x200
94 /* We can't use any flag that is bigger than 0x200 */
96 XBZRLECacheStats xbzrle_counters;
98 /* used by the search for pages to send */
99 struct PageSearchStatus {
100 /* The migration channel used for a specific host page */
101 QEMUFile *pss_channel;
102 /* Last block from where we have sent data */
103 RAMBlock *last_sent_block;
104 /* Current block being searched */
106 /* Current page to search from */
108 /* Set once we wrap around */
110 /* Whether we're sending a host page */
111 bool host_page_sending;
112 /* The start/end of current host page. Invalid if host_page_sending==false */
113 unsigned long host_page_start;
114 unsigned long host_page_end;
116 typedef struct PageSearchStatus PageSearchStatus;
118 /* struct contains XBZRLE cache and a static page
119 used by the compression */
121 /* buffer used for XBZRLE encoding */
122 uint8_t *encoded_buf;
123 /* buffer for storing page content */
124 uint8_t *current_buf;
125 /* Cache for XBZRLE, Protected by lock. */
128 /* it will store a page full of zeros */
129 uint8_t *zero_target_page;
130 /* buffer used for XBZRLE decoding */
131 uint8_t *decoded_buf;
134 static void XBZRLE_cache_lock(void)
136 if (migrate_xbzrle()) {
137 qemu_mutex_lock(&XBZRLE.lock);
141 static void XBZRLE_cache_unlock(void)
143 if (migrate_xbzrle()) {
144 qemu_mutex_unlock(&XBZRLE.lock);
149 * xbzrle_cache_resize: resize the xbzrle cache
151 * This function is called from migrate_params_apply in main
152 * thread, possibly while a migration is in progress. A running
153 * migration may be using the cache and might finish during this call,
154 * hence changes to the cache are protected by XBZRLE.lock().
156 * Returns 0 for success or -1 for error
158 * @new_size: new cache size
159 * @errp: set *errp if the check failed, with reason
161 int xbzrle_cache_resize(uint64_t new_size, Error **errp)
163 PageCache *new_cache;
166 /* Check for truncation */
167 if (new_size != (size_t)new_size) {
168 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cache size",
169 "exceeding address space");
173 if (new_size == migrate_xbzrle_cache_size()) {
180 if (XBZRLE.cache != NULL) {
181 new_cache = cache_init(new_size, TARGET_PAGE_SIZE, errp);
187 cache_fini(XBZRLE.cache);
188 XBZRLE.cache = new_cache;
191 XBZRLE_cache_unlock();
195 static bool postcopy_preempt_active(void)
197 return migrate_postcopy_preempt() && migration_in_postcopy();
200 bool migrate_ram_is_ignored(RAMBlock *block)
202 return !qemu_ram_is_migratable(block) ||
203 (migrate_ignore_shared() && qemu_ram_is_shared(block)
204 && qemu_ram_is_named_file(block));
207 #undef RAMBLOCK_FOREACH
209 int foreach_not_ignored_block(RAMBlockIterFunc func, void *opaque)
214 RCU_READ_LOCK_GUARD();
216 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
217 ret = func(block, opaque);
225 static void ramblock_recv_map_init(void)
229 RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
230 assert(!rb->receivedmap);
231 rb->receivedmap = bitmap_new(rb->max_length >> qemu_target_page_bits());
235 int ramblock_recv_bitmap_test(RAMBlock *rb, void *host_addr)
237 return test_bit(ramblock_recv_bitmap_offset(host_addr, rb),
241 bool ramblock_recv_bitmap_test_byte_offset(RAMBlock *rb, uint64_t byte_offset)
243 return test_bit(byte_offset >> TARGET_PAGE_BITS, rb->receivedmap);
246 void ramblock_recv_bitmap_set(RAMBlock *rb, void *host_addr)
248 set_bit_atomic(ramblock_recv_bitmap_offset(host_addr, rb), rb->receivedmap);
251 void ramblock_recv_bitmap_set_range(RAMBlock *rb, void *host_addr,
254 bitmap_set_atomic(rb->receivedmap,
255 ramblock_recv_bitmap_offset(host_addr, rb),
259 #define RAMBLOCK_RECV_BITMAP_ENDING (0x0123456789abcdefULL)
262 * Format: bitmap_size (8 bytes) + whole_bitmap (N bytes).
264 * Returns >0 if success with sent bytes, or <0 if error.
266 int64_t ramblock_recv_bitmap_send(QEMUFile *file,
267 const char *block_name)
269 RAMBlock *block = qemu_ram_block_by_name(block_name);
270 unsigned long *le_bitmap, nbits;
274 error_report("%s: invalid block name: %s", __func__, block_name);
278 nbits = block->postcopy_length >> TARGET_PAGE_BITS;
281 * Make sure the tmp bitmap buffer is big enough, e.g., on 32bit
282 * machines we may need 4 more bytes for padding (see below
283 * comment). So extend it a bit before hand.
285 le_bitmap = bitmap_new(nbits + BITS_PER_LONG);
288 * Always use little endian when sending the bitmap. This is
289 * required that when source and destination VMs are not using the
290 * same endianness. (Note: big endian won't work.)
292 bitmap_to_le(le_bitmap, block->receivedmap, nbits);
294 /* Size of the bitmap, in bytes */
295 size = DIV_ROUND_UP(nbits, 8);
298 * size is always aligned to 8 bytes for 64bit machines, but it
299 * may not be true for 32bit machines. We need this padding to
300 * make sure the migration can survive even between 32bit and
303 size = ROUND_UP(size, 8);
305 qemu_put_be64(file, size);
306 qemu_put_buffer(file, (const uint8_t *)le_bitmap, size);
308 * Mark as an end, in case the middle part is screwed up due to
309 * some "mysterious" reason.
311 qemu_put_be64(file, RAMBLOCK_RECV_BITMAP_ENDING);
316 if (qemu_file_get_error(file)) {
317 return qemu_file_get_error(file);
320 return size + sizeof(size);
324 * An outstanding page request, on the source, having been received
327 struct RAMSrcPageRequest {
332 QSIMPLEQ_ENTRY(RAMSrcPageRequest) next_req;
335 /* State of RAM for migration */
338 * PageSearchStatus structures for the channels when send pages.
339 * Protected by the bitmap_mutex.
341 PageSearchStatus pss[RAM_CHANNEL_MAX];
342 /* UFFD file descriptor, used in 'write-tracking' migration */
344 /* total ram size in bytes */
345 uint64_t ram_bytes_total;
346 /* Last block that we have visited searching for dirty pages */
347 RAMBlock *last_seen_block;
348 /* Last dirty target page we have sent */
349 ram_addr_t last_page;
350 /* last ram version we have seen */
351 uint32_t last_version;
352 /* How many times we have dirty too many pages */
353 int dirty_rate_high_cnt;
354 /* these variables are used for bitmap sync */
355 /* last time we did a full bitmap_sync */
356 int64_t time_last_bitmap_sync;
357 /* bytes transferred at start_time */
358 uint64_t bytes_xfer_prev;
359 /* number of dirty pages since start_time */
360 uint64_t num_dirty_pages_period;
361 /* xbzrle misses since the beginning of the period */
362 uint64_t xbzrle_cache_miss_prev;
363 /* Amount of xbzrle pages since the beginning of the period */
364 uint64_t xbzrle_pages_prev;
365 /* Amount of xbzrle encoded bytes since the beginning of the period */
366 uint64_t xbzrle_bytes_prev;
367 /* Are we really using XBZRLE (e.g., after the first round). */
369 /* Are we on the last stage of migration */
371 /* compression statistics since the beginning of the period */
372 /* amount of count that no free thread to compress data */
373 uint64_t compress_thread_busy_prev;
374 /* amount bytes after compression */
375 uint64_t compressed_size_prev;
376 /* amount of compressed pages */
377 uint64_t compress_pages_prev;
379 /* total handled target pages at the beginning of period */
380 uint64_t target_page_count_prev;
381 /* total handled target pages since start */
382 uint64_t target_page_count;
383 /* number of dirty bits in the bitmap */
384 uint64_t migration_dirty_pages;
387 * - dirty/clear bitmap
388 * - migration_dirty_pages
391 QemuMutex bitmap_mutex;
392 /* The RAMBlock used in the last src_page_requests */
393 RAMBlock *last_req_rb;
394 /* Queue of outstanding page requests from the destination */
395 QemuMutex src_page_req_mutex;
396 QSIMPLEQ_HEAD(, RAMSrcPageRequest) src_page_requests;
398 typedef struct RAMState RAMState;
400 static RAMState *ram_state;
402 static NotifierWithReturnList precopy_notifier_list;
404 /* Whether postcopy has queued requests? */
405 static bool postcopy_has_request(RAMState *rs)
407 return !QSIMPLEQ_EMPTY_ATOMIC(&rs->src_page_requests);
410 void precopy_infrastructure_init(void)
412 notifier_with_return_list_init(&precopy_notifier_list);
415 void precopy_add_notifier(NotifierWithReturn *n)
417 notifier_with_return_list_add(&precopy_notifier_list, n);
420 void precopy_remove_notifier(NotifierWithReturn *n)
422 notifier_with_return_remove(n);
425 int precopy_notify(PrecopyNotifyReason reason, Error **errp)
427 PrecopyNotifyData pnd;
431 return notifier_with_return_list_notify(&precopy_notifier_list, &pnd);
434 uint64_t ram_bytes_remaining(void)
436 return ram_state ? (ram_state->migration_dirty_pages * TARGET_PAGE_SIZE) :
440 void ram_transferred_add(uint64_t bytes)
442 if (runstate_is_running()) {
443 stat64_add(&mig_stats.precopy_bytes, bytes);
444 } else if (migration_in_postcopy()) {
445 stat64_add(&mig_stats.postcopy_bytes, bytes);
447 stat64_add(&mig_stats.downtime_bytes, bytes);
449 stat64_add(&mig_stats.transferred, bytes);
452 struct MigrationOps {
453 int (*ram_save_target_page)(RAMState *rs, PageSearchStatus *pss);
455 typedef struct MigrationOps MigrationOps;
457 MigrationOps *migration_ops;
459 static int ram_save_host_page_urgent(PageSearchStatus *pss);
461 /* NOTE: page is the PFN not real ram_addr_t. */
462 static void pss_init(PageSearchStatus *pss, RAMBlock *rb, ram_addr_t page)
466 pss->complete_round = false;
470 * Check whether two PSSs are actively sending the same page. Return true
471 * if it is, false otherwise.
473 static bool pss_overlap(PageSearchStatus *pss1, PageSearchStatus *pss2)
475 return pss1->host_page_sending && pss2->host_page_sending &&
476 (pss1->host_page_start == pss2->host_page_start);
480 * save_page_header: write page header to wire
482 * If this is the 1st block, it also writes the block identification
484 * Returns the number of bytes written
486 * @pss: current PSS channel status
487 * @block: block that contains the page we want to send
488 * @offset: offset inside the block for the page
489 * in the lower bits, it contains flags
491 static size_t save_page_header(PageSearchStatus *pss, QEMUFile *f,
492 RAMBlock *block, ram_addr_t offset)
495 bool same_block = (block == pss->last_sent_block);
498 offset |= RAM_SAVE_FLAG_CONTINUE;
500 qemu_put_be64(f, offset);
504 len = strlen(block->idstr);
505 qemu_put_byte(f, len);
506 qemu_put_buffer(f, (uint8_t *)block->idstr, len);
508 pss->last_sent_block = block;
514 * mig_throttle_guest_down: throttle down the guest
516 * Reduce amount of guest cpu execution to hopefully slow down memory
517 * writes. If guest dirty memory rate is reduced below the rate at
518 * which we can transfer pages to the destination then we should be
519 * able to complete migration. Some workloads dirty memory way too
520 * fast and will not effectively converge, even with auto-converge.
522 static void mig_throttle_guest_down(uint64_t bytes_dirty_period,
523 uint64_t bytes_dirty_threshold)
525 uint64_t pct_initial = migrate_cpu_throttle_initial();
526 uint64_t pct_increment = migrate_cpu_throttle_increment();
527 bool pct_tailslow = migrate_cpu_throttle_tailslow();
528 int pct_max = migrate_max_cpu_throttle();
530 uint64_t throttle_now = cpu_throttle_get_percentage();
531 uint64_t cpu_now, cpu_ideal, throttle_inc;
533 /* We have not started throttling yet. Let's start it. */
534 if (!cpu_throttle_active()) {
535 cpu_throttle_set(pct_initial);
537 /* Throttling already on, just increase the rate */
539 throttle_inc = pct_increment;
541 /* Compute the ideal CPU percentage used by Guest, which may
542 * make the dirty rate match the dirty rate threshold. */
543 cpu_now = 100 - throttle_now;
544 cpu_ideal = cpu_now * (bytes_dirty_threshold * 1.0 /
546 throttle_inc = MIN(cpu_now - cpu_ideal, pct_increment);
548 cpu_throttle_set(MIN(throttle_now + throttle_inc, pct_max));
552 void mig_throttle_counter_reset(void)
554 RAMState *rs = ram_state;
556 rs->time_last_bitmap_sync = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
557 rs->num_dirty_pages_period = 0;
558 rs->bytes_xfer_prev = stat64_get(&mig_stats.transferred);
562 * xbzrle_cache_zero_page: insert a zero page in the XBZRLE cache
564 * @rs: current RAM state
565 * @current_addr: address for the zero page
567 * Update the xbzrle cache to reflect a page that's been sent as all 0.
568 * The important thing is that a stale (not-yet-0'd) page be replaced
570 * As a bonus, if the page wasn't in the cache it gets added so that
571 * when a small write is made into the 0'd page it gets XBZRLE sent.
573 static void xbzrle_cache_zero_page(RAMState *rs, ram_addr_t current_addr)
575 /* We don't care if this fails to allocate a new cache page
576 * as long as it updated an old one */
577 cache_insert(XBZRLE.cache, current_addr, XBZRLE.zero_target_page,
578 stat64_get(&mig_stats.dirty_sync_count));
581 #define ENCODING_FLAG_XBZRLE 0x1
584 * save_xbzrle_page: compress and send current page
586 * Returns: 1 means that we wrote the page
587 * 0 means that page is identical to the one already sent
588 * -1 means that xbzrle would be longer than normal
590 * @rs: current RAM state
591 * @pss: current PSS channel
592 * @current_data: pointer to the address of the page contents
593 * @current_addr: addr of the page
594 * @block: block that contains the page we want to send
595 * @offset: offset inside the block for the page
597 static int save_xbzrle_page(RAMState *rs, PageSearchStatus *pss,
598 uint8_t **current_data, ram_addr_t current_addr,
599 RAMBlock *block, ram_addr_t offset)
601 int encoded_len = 0, bytes_xbzrle;
602 uint8_t *prev_cached_page;
603 QEMUFile *file = pss->pss_channel;
604 uint64_t generation = stat64_get(&mig_stats.dirty_sync_count);
606 if (!cache_is_cached(XBZRLE.cache, current_addr, generation)) {
607 xbzrle_counters.cache_miss++;
608 if (!rs->last_stage) {
609 if (cache_insert(XBZRLE.cache, current_addr, *current_data,
613 /* update *current_data when the page has been
614 inserted into cache */
615 *current_data = get_cached_data(XBZRLE.cache, current_addr);
622 * Reaching here means the page has hit the xbzrle cache, no matter what
623 * encoding result it is (normal encoding, overflow or skipping the page),
624 * count the page as encoded. This is used to calculate the encoding rate.
626 * Example: 2 pages (8KB) being encoded, first page encoding generates 2KB,
627 * 2nd page turns out to be skipped (i.e. no new bytes written to the
628 * page), the overall encoding rate will be 8KB / 2KB = 4, which has the
629 * skipped page included. In this way, the encoding rate can tell if the
630 * guest page is good for xbzrle encoding.
632 xbzrle_counters.pages++;
633 prev_cached_page = get_cached_data(XBZRLE.cache, current_addr);
635 /* save current buffer into memory */
636 memcpy(XBZRLE.current_buf, *current_data, TARGET_PAGE_SIZE);
638 /* XBZRLE encoding (if there is no overflow) */
639 encoded_len = xbzrle_encode_buffer(prev_cached_page, XBZRLE.current_buf,
640 TARGET_PAGE_SIZE, XBZRLE.encoded_buf,
644 * Update the cache contents, so that it corresponds to the data
645 * sent, in all cases except where we skip the page.
647 if (!rs->last_stage && encoded_len != 0) {
648 memcpy(prev_cached_page, XBZRLE.current_buf, TARGET_PAGE_SIZE);
650 * In the case where we couldn't compress, ensure that the caller
651 * sends the data from the cache, since the guest might have
652 * changed the RAM since we copied it.
654 *current_data = prev_cached_page;
657 if (encoded_len == 0) {
658 trace_save_xbzrle_page_skipping();
660 } else if (encoded_len == -1) {
661 trace_save_xbzrle_page_overflow();
662 xbzrle_counters.overflow++;
663 xbzrle_counters.bytes += TARGET_PAGE_SIZE;
667 /* Send XBZRLE based compressed page */
668 bytes_xbzrle = save_page_header(pss, pss->pss_channel, block,
669 offset | RAM_SAVE_FLAG_XBZRLE);
670 qemu_put_byte(file, ENCODING_FLAG_XBZRLE);
671 qemu_put_be16(file, encoded_len);
672 qemu_put_buffer(file, XBZRLE.encoded_buf, encoded_len);
673 bytes_xbzrle += encoded_len + 1 + 2;
675 * Like compressed_size (please see update_compress_thread_counts),
676 * the xbzrle encoded bytes don't count the 8 byte header with
677 * RAM_SAVE_FLAG_CONTINUE.
679 xbzrle_counters.bytes += bytes_xbzrle - 8;
680 ram_transferred_add(bytes_xbzrle);
686 * pss_find_next_dirty: find the next dirty page of current ramblock
688 * This function updates pss->page to point to the next dirty page index
689 * within the ramblock to migrate, or the end of ramblock when nothing
690 * found. Note that when pss->host_page_sending==true it means we're
691 * during sending a host page, so we won't look for dirty page that is
692 * outside the host page boundary.
694 * @pss: the current page search status
696 static void pss_find_next_dirty(PageSearchStatus *pss)
698 RAMBlock *rb = pss->block;
699 unsigned long size = rb->used_length >> TARGET_PAGE_BITS;
700 unsigned long *bitmap = rb->bmap;
702 if (migrate_ram_is_ignored(rb)) {
703 /* Points directly to the end, so we know no dirty page */
709 * If during sending a host page, only look for dirty pages within the
710 * current host page being send.
712 if (pss->host_page_sending) {
713 assert(pss->host_page_end);
714 size = MIN(size, pss->host_page_end);
717 pss->page = find_next_bit(bitmap, size, pss->page);
720 static void migration_clear_memory_region_dirty_bitmap(RAMBlock *rb,
726 if (!rb->clear_bmap || !clear_bmap_test_and_clear(rb, page)) {
730 shift = rb->clear_bmap_shift;
732 * CLEAR_BITMAP_SHIFT_MIN should always guarantee this... this
733 * can make things easier sometimes since then start address
734 * of the small chunk will always be 64 pages aligned so the
735 * bitmap will always be aligned to unsigned long. We should
736 * even be able to remove this restriction but I'm simply
741 size = 1ULL << (TARGET_PAGE_BITS + shift);
742 start = QEMU_ALIGN_DOWN((ram_addr_t)page << TARGET_PAGE_BITS, size);
743 trace_migration_bitmap_clear_dirty(rb->idstr, start, size, page);
744 memory_region_clear_dirty_bitmap(rb->mr, start, size);
748 migration_clear_memory_region_dirty_bitmap_range(RAMBlock *rb,
750 unsigned long npages)
752 unsigned long i, chunk_pages = 1UL << rb->clear_bmap_shift;
753 unsigned long chunk_start = QEMU_ALIGN_DOWN(start, chunk_pages);
754 unsigned long chunk_end = QEMU_ALIGN_UP(start + npages, chunk_pages);
757 * Clear pages from start to start + npages - 1, so the end boundary is
760 for (i = chunk_start; i < chunk_end; i += chunk_pages) {
761 migration_clear_memory_region_dirty_bitmap(rb, i);
766 * colo_bitmap_find_diry:find contiguous dirty pages from start
768 * Returns the page offset within memory region of the start of the contiguout
771 * @rs: current RAM state
772 * @rb: RAMBlock where to search for dirty pages
773 * @start: page where we start the search
774 * @num: the number of contiguous dirty pages
777 unsigned long colo_bitmap_find_dirty(RAMState *rs, RAMBlock *rb,
778 unsigned long start, unsigned long *num)
780 unsigned long size = rb->used_length >> TARGET_PAGE_BITS;
781 unsigned long *bitmap = rb->bmap;
782 unsigned long first, next;
786 if (migrate_ram_is_ignored(rb)) {
790 first = find_next_bit(bitmap, size, start);
794 next = find_next_zero_bit(bitmap, size, first + 1);
795 assert(next >= first);
800 static inline bool migration_bitmap_clear_dirty(RAMState *rs,
807 * Clear dirty bitmap if needed. This _must_ be called before we
808 * send any of the page in the chunk because we need to make sure
809 * we can capture further page content changes when we sync dirty
810 * log the next time. So as long as we are going to send any of
811 * the page in the chunk we clear the remote dirty bitmap for all.
812 * Clearing it earlier won't be a problem, but too late will.
814 migration_clear_memory_region_dirty_bitmap(rb, page);
816 ret = test_and_clear_bit(page, rb->bmap);
818 rs->migration_dirty_pages--;
824 static void dirty_bitmap_clear_section(MemoryRegionSection *section,
827 const hwaddr offset = section->offset_within_region;
828 const hwaddr size = int128_get64(section->size);
829 const unsigned long start = offset >> TARGET_PAGE_BITS;
830 const unsigned long npages = size >> TARGET_PAGE_BITS;
831 RAMBlock *rb = section->mr->ram_block;
832 uint64_t *cleared_bits = opaque;
835 * We don't grab ram_state->bitmap_mutex because we expect to run
836 * only when starting migration or during postcopy recovery where
837 * we don't have concurrent access.
839 if (!migration_in_postcopy() && !migrate_background_snapshot()) {
840 migration_clear_memory_region_dirty_bitmap_range(rb, start, npages);
842 *cleared_bits += bitmap_count_one_with_offset(rb->bmap, start, npages);
843 bitmap_clear(rb->bmap, start, npages);
847 * Exclude all dirty pages from migration that fall into a discarded range as
848 * managed by a RamDiscardManager responsible for the mapped memory region of
849 * the RAMBlock. Clear the corresponding bits in the dirty bitmaps.
851 * Discarded pages ("logically unplugged") have undefined content and must
852 * not get migrated, because even reading these pages for migration might
853 * result in undesired behavior.
855 * Returns the number of cleared bits in the RAMBlock dirty bitmap.
857 * Note: The result is only stable while migrating (precopy/postcopy).
859 static uint64_t ramblock_dirty_bitmap_clear_discarded_pages(RAMBlock *rb)
861 uint64_t cleared_bits = 0;
863 if (rb->mr && rb->bmap && memory_region_has_ram_discard_manager(rb->mr)) {
864 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
865 MemoryRegionSection section = {
867 .offset_within_region = 0,
868 .size = int128_make64(qemu_ram_get_used_length(rb)),
871 ram_discard_manager_replay_discarded(rdm, §ion,
872 dirty_bitmap_clear_section,
879 * Check if a host-page aligned page falls into a discarded range as managed by
880 * a RamDiscardManager responsible for the mapped memory region of the RAMBlock.
882 * Note: The result is only stable while migrating (precopy/postcopy).
884 bool ramblock_page_is_discarded(RAMBlock *rb, ram_addr_t start)
886 if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) {
887 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
888 MemoryRegionSection section = {
890 .offset_within_region = start,
891 .size = int128_make64(qemu_ram_pagesize(rb)),
894 return !ram_discard_manager_is_populated(rdm, §ion);
899 /* Called with RCU critical section */
900 static void ramblock_sync_dirty_bitmap(RAMState *rs, RAMBlock *rb)
902 uint64_t new_dirty_pages =
903 cpu_physical_memory_sync_dirty_bitmap(rb, 0, rb->used_length);
905 rs->migration_dirty_pages += new_dirty_pages;
906 rs->num_dirty_pages_period += new_dirty_pages;
910 * ram_pagesize_summary: calculate all the pagesizes of a VM
912 * Returns a summary bitmap of the page sizes of all RAMBlocks
914 * For VMs with just normal pages this is equivalent to the host page
915 * size. If it's got some huge pages then it's the OR of all the
916 * different page sizes.
918 uint64_t ram_pagesize_summary(void)
921 uint64_t summary = 0;
923 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
924 summary |= block->page_size;
930 uint64_t ram_get_total_transferred_pages(void)
932 return stat64_get(&mig_stats.normal_pages) +
933 stat64_get(&mig_stats.zero_pages) +
934 compression_counters.pages + xbzrle_counters.pages;
937 static void migration_update_rates(RAMState *rs, int64_t end_time)
939 uint64_t page_count = rs->target_page_count - rs->target_page_count_prev;
940 double compressed_size;
942 /* calculate period counters */
943 stat64_set(&mig_stats.dirty_pages_rate,
944 rs->num_dirty_pages_period * 1000 /
945 (end_time - rs->time_last_bitmap_sync));
951 if (migrate_xbzrle()) {
952 double encoded_size, unencoded_size;
954 xbzrle_counters.cache_miss_rate = (double)(xbzrle_counters.cache_miss -
955 rs->xbzrle_cache_miss_prev) / page_count;
956 rs->xbzrle_cache_miss_prev = xbzrle_counters.cache_miss;
957 unencoded_size = (xbzrle_counters.pages - rs->xbzrle_pages_prev) *
959 encoded_size = xbzrle_counters.bytes - rs->xbzrle_bytes_prev;
960 if (xbzrle_counters.pages == rs->xbzrle_pages_prev || !encoded_size) {
961 xbzrle_counters.encoding_rate = 0;
963 xbzrle_counters.encoding_rate = unencoded_size / encoded_size;
965 rs->xbzrle_pages_prev = xbzrle_counters.pages;
966 rs->xbzrle_bytes_prev = xbzrle_counters.bytes;
969 if (migrate_compress()) {
970 compression_counters.busy_rate = (double)(compression_counters.busy -
971 rs->compress_thread_busy_prev) / page_count;
972 rs->compress_thread_busy_prev = compression_counters.busy;
974 compressed_size = compression_counters.compressed_size -
975 rs->compressed_size_prev;
976 if (compressed_size) {
977 double uncompressed_size = (compression_counters.pages -
978 rs->compress_pages_prev) * TARGET_PAGE_SIZE;
980 /* Compression-Ratio = Uncompressed-size / Compressed-size */
981 compression_counters.compression_rate =
982 uncompressed_size / compressed_size;
984 rs->compress_pages_prev = compression_counters.pages;
985 rs->compressed_size_prev = compression_counters.compressed_size;
991 * Enable dirty-limit to throttle down the guest
993 static void migration_dirty_limit_guest(void)
996 * dirty page rate quota for all vCPUs fetched from
997 * migration parameter 'vcpu_dirty_limit'
999 static int64_t quota_dirtyrate;
1000 MigrationState *s = migrate_get_current();
1003 * If dirty limit already enabled and migration parameter
1004 * vcpu-dirty-limit untouched.
1006 if (dirtylimit_in_service() &&
1007 quota_dirtyrate == s->parameters.vcpu_dirty_limit) {
1011 quota_dirtyrate = s->parameters.vcpu_dirty_limit;
1014 * Set all vCPU a quota dirtyrate, note that the second
1015 * parameter will be ignored if setting all vCPU for the vm
1017 qmp_set_vcpu_dirty_limit(false, -1, quota_dirtyrate, NULL);
1018 trace_migration_dirty_limit_guest(quota_dirtyrate);
1021 static void migration_trigger_throttle(RAMState *rs)
1023 uint64_t threshold = migrate_throttle_trigger_threshold();
1024 uint64_t bytes_xfer_period =
1025 stat64_get(&mig_stats.transferred) - rs->bytes_xfer_prev;
1026 uint64_t bytes_dirty_period = rs->num_dirty_pages_period * TARGET_PAGE_SIZE;
1027 uint64_t bytes_dirty_threshold = bytes_xfer_period * threshold / 100;
1029 /* During block migration the auto-converge logic incorrectly detects
1030 * that ram migration makes no progress. Avoid this by disabling the
1031 * throttling logic during the bulk phase of block migration. */
1032 if (blk_mig_bulk_active()) {
1037 * The following detection logic can be refined later. For now:
1038 * Check to see if the ratio between dirtied bytes and the approx.
1039 * amount of bytes that just got transferred since the last time
1040 * we were in this routine reaches the threshold. If that happens
1041 * twice, start or increase throttling.
1043 if ((bytes_dirty_period > bytes_dirty_threshold) &&
1044 (++rs->dirty_rate_high_cnt >= 2)) {
1045 rs->dirty_rate_high_cnt = 0;
1046 if (migrate_auto_converge()) {
1047 trace_migration_throttle();
1048 mig_throttle_guest_down(bytes_dirty_period,
1049 bytes_dirty_threshold);
1050 } else if (migrate_dirty_limit()) {
1051 migration_dirty_limit_guest();
1056 static void migration_bitmap_sync(RAMState *rs, bool last_stage)
1061 stat64_add(&mig_stats.dirty_sync_count, 1);
1063 if (!rs->time_last_bitmap_sync) {
1064 rs->time_last_bitmap_sync = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
1067 trace_migration_bitmap_sync_start();
1068 memory_global_dirty_log_sync(last_stage);
1070 qemu_mutex_lock(&rs->bitmap_mutex);
1071 WITH_RCU_READ_LOCK_GUARD() {
1072 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1073 ramblock_sync_dirty_bitmap(rs, block);
1075 stat64_set(&mig_stats.dirty_bytes_last_sync, ram_bytes_remaining());
1077 qemu_mutex_unlock(&rs->bitmap_mutex);
1079 memory_global_after_dirty_log_sync();
1080 trace_migration_bitmap_sync_end(rs->num_dirty_pages_period);
1082 end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
1084 /* more than 1 second = 1000 millisecons */
1085 if (end_time > rs->time_last_bitmap_sync + 1000) {
1086 migration_trigger_throttle(rs);
1088 migration_update_rates(rs, end_time);
1090 rs->target_page_count_prev = rs->target_page_count;
1092 /* reset period counters */
1093 rs->time_last_bitmap_sync = end_time;
1094 rs->num_dirty_pages_period = 0;
1095 rs->bytes_xfer_prev = stat64_get(&mig_stats.transferred);
1097 if (migrate_events()) {
1098 uint64_t generation = stat64_get(&mig_stats.dirty_sync_count);
1099 qapi_event_send_migration_pass(generation);
1103 static void migration_bitmap_sync_precopy(RAMState *rs, bool last_stage)
1105 Error *local_err = NULL;
1108 * The current notifier usage is just an optimization to migration, so we
1109 * don't stop the normal migration process in the error case.
1111 if (precopy_notify(PRECOPY_NOTIFY_BEFORE_BITMAP_SYNC, &local_err)) {
1112 error_report_err(local_err);
1116 migration_bitmap_sync(rs, last_stage);
1118 if (precopy_notify(PRECOPY_NOTIFY_AFTER_BITMAP_SYNC, &local_err)) {
1119 error_report_err(local_err);
1123 void ram_release_page(const char *rbname, uint64_t offset)
1125 if (!migrate_release_ram() || !migration_in_postcopy()) {
1129 ram_discard_range(rbname, offset, TARGET_PAGE_SIZE);
1133 * save_zero_page_to_file: send the zero page to the file
1135 * Returns the size of data written to the file, 0 means the page is not
1138 * @pss: current PSS channel
1139 * @block: block that contains the page we want to send
1140 * @offset: offset inside the block for the page
1142 static int save_zero_page_to_file(PageSearchStatus *pss, QEMUFile *file,
1143 RAMBlock *block, ram_addr_t offset)
1145 uint8_t *p = block->host + offset;
1148 if (buffer_is_zero(p, TARGET_PAGE_SIZE)) {
1149 len += save_page_header(pss, file, block, offset | RAM_SAVE_FLAG_ZERO);
1150 qemu_put_byte(file, 0);
1152 ram_release_page(block->idstr, offset);
1158 * save_zero_page: send the zero page to the stream
1160 * Returns the number of pages written.
1162 * @pss: current PSS channel
1163 * @block: block that contains the page we want to send
1164 * @offset: offset inside the block for the page
1166 static int save_zero_page(PageSearchStatus *pss, QEMUFile *f, RAMBlock *block,
1169 int len = save_zero_page_to_file(pss, f, block, offset);
1172 stat64_add(&mig_stats.zero_pages, 1);
1173 ram_transferred_add(len);
1180 * @pages: the number of pages written by the control path,
1182 * > 0 - number of pages written
1184 * Return true if the pages has been saved, otherwise false is returned.
1186 static bool control_save_page(PageSearchStatus *pss, RAMBlock *block,
1187 ram_addr_t offset, int *pages)
1189 uint64_t bytes_xmit = 0;
1193 ret = ram_control_save_page(pss->pss_channel, block->offset, offset,
1194 TARGET_PAGE_SIZE, &bytes_xmit);
1195 if (ret == RAM_SAVE_CONTROL_NOT_SUPP) {
1200 ram_transferred_add(bytes_xmit);
1204 if (ret == RAM_SAVE_CONTROL_DELAYED) {
1208 if (bytes_xmit > 0) {
1209 stat64_add(&mig_stats.normal_pages, 1);
1210 } else if (bytes_xmit == 0) {
1211 stat64_add(&mig_stats.zero_pages, 1);
1218 * directly send the page to the stream
1220 * Returns the number of pages written.
1222 * @pss: current PSS channel
1223 * @block: block that contains the page we want to send
1224 * @offset: offset inside the block for the page
1225 * @buf: the page to be sent
1226 * @async: send to page asyncly
1228 static int save_normal_page(PageSearchStatus *pss, RAMBlock *block,
1229 ram_addr_t offset, uint8_t *buf, bool async)
1231 QEMUFile *file = pss->pss_channel;
1233 ram_transferred_add(save_page_header(pss, pss->pss_channel, block,
1234 offset | RAM_SAVE_FLAG_PAGE));
1236 qemu_put_buffer_async(file, buf, TARGET_PAGE_SIZE,
1237 migrate_release_ram() &&
1238 migration_in_postcopy());
1240 qemu_put_buffer(file, buf, TARGET_PAGE_SIZE);
1242 ram_transferred_add(TARGET_PAGE_SIZE);
1243 stat64_add(&mig_stats.normal_pages, 1);
1248 * ram_save_page: send the given page to the stream
1250 * Returns the number of pages written.
1252 * >=0 - Number of pages written - this might legally be 0
1253 * if xbzrle noticed the page was the same.
1255 * @rs: current RAM state
1256 * @block: block that contains the page we want to send
1257 * @offset: offset inside the block for the page
1259 static int ram_save_page(RAMState *rs, PageSearchStatus *pss)
1263 bool send_async = true;
1264 RAMBlock *block = pss->block;
1265 ram_addr_t offset = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS;
1266 ram_addr_t current_addr = block->offset + offset;
1268 p = block->host + offset;
1269 trace_ram_save_page(block->idstr, (uint64_t)offset, p);
1271 XBZRLE_cache_lock();
1272 if (rs->xbzrle_started && !migration_in_postcopy()) {
1273 pages = save_xbzrle_page(rs, pss, &p, current_addr,
1275 if (!rs->last_stage) {
1276 /* Can't send this cached data async, since the cache page
1277 * might get updated before it gets to the wire
1283 /* XBZRLE overflow or normal page */
1285 pages = save_normal_page(pss, block, offset, p, send_async);
1288 XBZRLE_cache_unlock();
1293 static int ram_save_multifd_page(QEMUFile *file, RAMBlock *block,
1296 if (multifd_queue_page(file, block, offset) < 0) {
1299 stat64_add(&mig_stats.normal_pages, 1);
1305 update_compress_thread_counts(const CompressParam *param, int bytes_xmit)
1307 ram_transferred_add(bytes_xmit);
1309 if (param->result == RES_ZEROPAGE) {
1310 stat64_add(&mig_stats.zero_pages, 1);
1314 /* 8 means a header with RAM_SAVE_FLAG_CONTINUE. */
1315 compression_counters.compressed_size += bytes_xmit - 8;
1316 compression_counters.pages++;
1319 static bool save_page_use_compression(RAMState *rs);
1321 static int send_queued_data(CompressParam *param)
1323 PageSearchStatus *pss = &ram_state->pss[RAM_CHANNEL_PRECOPY];
1324 MigrationState *ms = migrate_get_current();
1325 QEMUFile *file = ms->to_dst_file;
1328 RAMBlock *block = param->block;
1329 ram_addr_t offset = param->offset;
1331 if (param->result == RES_NONE) {
1335 assert(block == pss->last_sent_block);
1337 if (param->result == RES_ZEROPAGE) {
1338 assert(qemu_file_buffer_empty(param->file));
1339 len += save_page_header(pss, file, block, offset | RAM_SAVE_FLAG_ZERO);
1340 qemu_put_byte(file, 0);
1342 ram_release_page(block->idstr, offset);
1343 } else if (param->result == RES_COMPRESS) {
1344 assert(!qemu_file_buffer_empty(param->file));
1345 len += save_page_header(pss, file, block,
1346 offset | RAM_SAVE_FLAG_COMPRESS_PAGE);
1347 len += qemu_put_qemu_file(file, param->file);
1352 update_compress_thread_counts(param, len);
1357 static void ram_flush_compressed_data(RAMState *rs)
1359 if (!save_page_use_compression(rs)) {
1363 flush_compressed_data(send_queued_data);
1366 #define PAGE_ALL_CLEAN 0
1367 #define PAGE_TRY_AGAIN 1
1368 #define PAGE_DIRTY_FOUND 2
1370 * find_dirty_block: find the next dirty page and update any state
1371 * associated with the search process.
1374 * <0: An error happened
1375 * PAGE_ALL_CLEAN: no dirty page found, give up
1376 * PAGE_TRY_AGAIN: no dirty page found, retry for next block
1377 * PAGE_DIRTY_FOUND: dirty page found
1379 * @rs: current RAM state
1380 * @pss: data about the state of the current dirty page scan
1381 * @again: set to false if the search has scanned the whole of RAM
1383 static int find_dirty_block(RAMState *rs, PageSearchStatus *pss)
1385 /* Update pss->page for the next dirty bit in ramblock */
1386 pss_find_next_dirty(pss);
1388 if (pss->complete_round && pss->block == rs->last_seen_block &&
1389 pss->page >= rs->last_page) {
1391 * We've been once around the RAM and haven't found anything.
1394 return PAGE_ALL_CLEAN;
1396 if (!offset_in_ramblock(pss->block,
1397 ((ram_addr_t)pss->page) << TARGET_PAGE_BITS)) {
1398 /* Didn't find anything in this RAM Block */
1400 pss->block = QLIST_NEXT_RCU(pss->block, next);
1402 if (!migrate_multifd_flush_after_each_section()) {
1403 QEMUFile *f = rs->pss[RAM_CHANNEL_PRECOPY].pss_channel;
1404 int ret = multifd_send_sync_main(f);
1408 qemu_put_be64(f, RAM_SAVE_FLAG_MULTIFD_FLUSH);
1412 * If memory migration starts over, we will meet a dirtied page
1413 * which may still exists in compression threads's ring, so we
1414 * should flush the compressed data to make sure the new page
1415 * is not overwritten by the old one in the destination.
1417 * Also If xbzrle is on, stop using the data compression at this
1418 * point. In theory, xbzrle can do better than compression.
1420 ram_flush_compressed_data(rs);
1422 /* Hit the end of the list */
1423 pss->block = QLIST_FIRST_RCU(&ram_list.blocks);
1424 /* Flag that we've looped */
1425 pss->complete_round = true;
1426 /* After the first round, enable XBZRLE. */
1427 if (migrate_xbzrle()) {
1428 rs->xbzrle_started = true;
1431 /* Didn't find anything this time, but try again on the new block */
1432 return PAGE_TRY_AGAIN;
1434 /* We've found something */
1435 return PAGE_DIRTY_FOUND;
1440 * unqueue_page: gets a page of the queue
1442 * Helper for 'get_queued_page' - gets a page off the queue
1444 * Returns the block of the page (or NULL if none available)
1446 * @rs: current RAM state
1447 * @offset: used to return the offset within the RAMBlock
1449 static RAMBlock *unqueue_page(RAMState *rs, ram_addr_t *offset)
1451 struct RAMSrcPageRequest *entry;
1452 RAMBlock *block = NULL;
1454 if (!postcopy_has_request(rs)) {
1458 QEMU_LOCK_GUARD(&rs->src_page_req_mutex);
1461 * This should _never_ change even after we take the lock, because no one
1462 * should be taking anything off the request list other than us.
1464 assert(postcopy_has_request(rs));
1466 entry = QSIMPLEQ_FIRST(&rs->src_page_requests);
1468 *offset = entry->offset;
1470 if (entry->len > TARGET_PAGE_SIZE) {
1471 entry->len -= TARGET_PAGE_SIZE;
1472 entry->offset += TARGET_PAGE_SIZE;
1474 memory_region_unref(block->mr);
1475 QSIMPLEQ_REMOVE_HEAD(&rs->src_page_requests, next_req);
1477 migration_consume_urgent_request();
1483 #if defined(__linux__)
1485 * poll_fault_page: try to get next UFFD write fault page and, if pending fault
1486 * is found, return RAM block pointer and page offset
1488 * Returns pointer to the RAMBlock containing faulting page,
1489 * NULL if no write faults are pending
1491 * @rs: current RAM state
1492 * @offset: page offset from the beginning of the block
1494 static RAMBlock *poll_fault_page(RAMState *rs, ram_addr_t *offset)
1496 struct uffd_msg uffd_msg;
1501 if (!migrate_background_snapshot()) {
1505 res = uffd_read_events(rs->uffdio_fd, &uffd_msg, 1);
1510 page_address = (void *)(uintptr_t) uffd_msg.arg.pagefault.address;
1511 block = qemu_ram_block_from_host(page_address, false, offset);
1512 assert(block && (block->flags & RAM_UF_WRITEPROTECT) != 0);
1517 * ram_save_release_protection: release UFFD write protection after
1518 * a range of pages has been saved
1520 * @rs: current RAM state
1521 * @pss: page-search-status structure
1522 * @start_page: index of the first page in the range relative to pss->block
1524 * Returns 0 on success, negative value in case of an error
1526 static int ram_save_release_protection(RAMState *rs, PageSearchStatus *pss,
1527 unsigned long start_page)
1531 /* Check if page is from UFFD-managed region. */
1532 if (pss->block->flags & RAM_UF_WRITEPROTECT) {
1533 void *page_address = pss->block->host + (start_page << TARGET_PAGE_BITS);
1534 uint64_t run_length = (pss->page - start_page) << TARGET_PAGE_BITS;
1536 /* Flush async buffers before un-protect. */
1537 qemu_fflush(pss->pss_channel);
1538 /* Un-protect memory range. */
1539 res = uffd_change_protection(rs->uffdio_fd, page_address, run_length,
1546 /* ram_write_tracking_available: check if kernel supports required UFFD features
1548 * Returns true if supports, false otherwise
1550 bool ram_write_tracking_available(void)
1552 uint64_t uffd_features;
1555 res = uffd_query_features(&uffd_features);
1557 (uffd_features & UFFD_FEATURE_PAGEFAULT_FLAG_WP) != 0);
1560 /* ram_write_tracking_compatible: check if guest configuration is
1561 * compatible with 'write-tracking'
1563 * Returns true if compatible, false otherwise
1565 bool ram_write_tracking_compatible(void)
1567 const uint64_t uffd_ioctls_mask = BIT(_UFFDIO_WRITEPROTECT);
1572 /* Open UFFD file descriptor */
1573 uffd_fd = uffd_create_fd(UFFD_FEATURE_PAGEFAULT_FLAG_WP, false);
1578 RCU_READ_LOCK_GUARD();
1580 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1581 uint64_t uffd_ioctls;
1583 /* Nothing to do with read-only and MMIO-writable regions */
1584 if (block->mr->readonly || block->mr->rom_device) {
1587 /* Try to register block memory via UFFD-IO to track writes */
1588 if (uffd_register_memory(uffd_fd, block->host, block->max_length,
1589 UFFDIO_REGISTER_MODE_WP, &uffd_ioctls)) {
1592 if ((uffd_ioctls & uffd_ioctls_mask) != uffd_ioctls_mask) {
1599 uffd_close_fd(uffd_fd);
1603 static inline void populate_read_range(RAMBlock *block, ram_addr_t offset,
1606 const ram_addr_t end = offset + size;
1609 * We read one byte of each page; this will preallocate page tables if
1610 * required and populate the shared zeropage on MAP_PRIVATE anonymous memory
1611 * where no page was populated yet. This might require adaption when
1612 * supporting other mappings, like shmem.
1614 for (; offset < end; offset += block->page_size) {
1615 char tmp = *((char *)block->host + offset);
1617 /* Don't optimize the read out */
1618 asm volatile("" : "+r" (tmp));
1622 static inline int populate_read_section(MemoryRegionSection *section,
1625 const hwaddr size = int128_get64(section->size);
1626 hwaddr offset = section->offset_within_region;
1627 RAMBlock *block = section->mr->ram_block;
1629 populate_read_range(block, offset, size);
1634 * ram_block_populate_read: preallocate page tables and populate pages in the
1635 * RAM block by reading a byte of each page.
1637 * Since it's solely used for userfault_fd WP feature, here we just
1638 * hardcode page size to qemu_real_host_page_size.
1640 * @block: RAM block to populate
1642 static void ram_block_populate_read(RAMBlock *rb)
1645 * Skip populating all pages that fall into a discarded range as managed by
1646 * a RamDiscardManager responsible for the mapped memory region of the
1647 * RAMBlock. Such discarded ("logically unplugged") parts of a RAMBlock
1648 * must not get populated automatically. We don't have to track
1649 * modifications via userfaultfd WP reliably, because these pages will
1650 * not be part of the migration stream either way -- see
1651 * ramblock_dirty_bitmap_exclude_discarded_pages().
1653 * Note: The result is only stable while migrating (precopy/postcopy).
1655 if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) {
1656 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
1657 MemoryRegionSection section = {
1659 .offset_within_region = 0,
1660 .size = rb->mr->size,
1663 ram_discard_manager_replay_populated(rdm, §ion,
1664 populate_read_section, NULL);
1666 populate_read_range(rb, 0, rb->used_length);
1671 * ram_write_tracking_prepare: prepare for UFFD-WP memory tracking
1673 void ram_write_tracking_prepare(void)
1677 RCU_READ_LOCK_GUARD();
1679 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1680 /* Nothing to do with read-only and MMIO-writable regions */
1681 if (block->mr->readonly || block->mr->rom_device) {
1686 * Populate pages of the RAM block before enabling userfault_fd
1689 * This stage is required since ioctl(UFFDIO_WRITEPROTECT) with
1690 * UFFDIO_WRITEPROTECT_MODE_WP mode setting would silently skip
1691 * pages with pte_none() entries in page table.
1693 ram_block_populate_read(block);
1697 static inline int uffd_protect_section(MemoryRegionSection *section,
1700 const hwaddr size = int128_get64(section->size);
1701 const hwaddr offset = section->offset_within_region;
1702 RAMBlock *rb = section->mr->ram_block;
1703 int uffd_fd = (uintptr_t)opaque;
1705 return uffd_change_protection(uffd_fd, rb->host + offset, size, true,
1709 static int ram_block_uffd_protect(RAMBlock *rb, int uffd_fd)
1711 assert(rb->flags & RAM_UF_WRITEPROTECT);
1713 /* See ram_block_populate_read() */
1714 if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) {
1715 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
1716 MemoryRegionSection section = {
1718 .offset_within_region = 0,
1719 .size = rb->mr->size,
1722 return ram_discard_manager_replay_populated(rdm, §ion,
1723 uffd_protect_section,
1724 (void *)(uintptr_t)uffd_fd);
1726 return uffd_change_protection(uffd_fd, rb->host,
1727 rb->used_length, true, false);
1731 * ram_write_tracking_start: start UFFD-WP memory tracking
1733 * Returns 0 for success or negative value in case of error
1735 int ram_write_tracking_start(void)
1738 RAMState *rs = ram_state;
1741 /* Open UFFD file descriptor */
1742 uffd_fd = uffd_create_fd(UFFD_FEATURE_PAGEFAULT_FLAG_WP, true);
1746 rs->uffdio_fd = uffd_fd;
1748 RCU_READ_LOCK_GUARD();
1750 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1751 /* Nothing to do with read-only and MMIO-writable regions */
1752 if (block->mr->readonly || block->mr->rom_device) {
1756 /* Register block memory with UFFD to track writes */
1757 if (uffd_register_memory(rs->uffdio_fd, block->host,
1758 block->max_length, UFFDIO_REGISTER_MODE_WP, NULL)) {
1761 block->flags |= RAM_UF_WRITEPROTECT;
1762 memory_region_ref(block->mr);
1764 /* Apply UFFD write protection to the block memory range */
1765 if (ram_block_uffd_protect(block, uffd_fd)) {
1769 trace_ram_write_tracking_ramblock_start(block->idstr, block->page_size,
1770 block->host, block->max_length);
1776 error_report("ram_write_tracking_start() failed: restoring initial memory state");
1778 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1779 if ((block->flags & RAM_UF_WRITEPROTECT) == 0) {
1782 uffd_unregister_memory(rs->uffdio_fd, block->host, block->max_length);
1783 /* Cleanup flags and remove reference */
1784 block->flags &= ~RAM_UF_WRITEPROTECT;
1785 memory_region_unref(block->mr);
1788 uffd_close_fd(uffd_fd);
1794 * ram_write_tracking_stop: stop UFFD-WP memory tracking and remove protection
1796 void ram_write_tracking_stop(void)
1798 RAMState *rs = ram_state;
1801 RCU_READ_LOCK_GUARD();
1803 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1804 if ((block->flags & RAM_UF_WRITEPROTECT) == 0) {
1807 uffd_unregister_memory(rs->uffdio_fd, block->host, block->max_length);
1809 trace_ram_write_tracking_ramblock_stop(block->idstr, block->page_size,
1810 block->host, block->max_length);
1812 /* Cleanup flags and remove reference */
1813 block->flags &= ~RAM_UF_WRITEPROTECT;
1814 memory_region_unref(block->mr);
1817 /* Finally close UFFD file descriptor */
1818 uffd_close_fd(rs->uffdio_fd);
1823 /* No target OS support, stubs just fail or ignore */
1825 static RAMBlock *poll_fault_page(RAMState *rs, ram_addr_t *offset)
1833 static int ram_save_release_protection(RAMState *rs, PageSearchStatus *pss,
1834 unsigned long start_page)
1843 bool ram_write_tracking_available(void)
1848 bool ram_write_tracking_compatible(void)
1854 int ram_write_tracking_start(void)
1860 void ram_write_tracking_stop(void)
1864 #endif /* defined(__linux__) */
1867 * get_queued_page: unqueue a page from the postcopy requests
1869 * Skips pages that are already sent (!dirty)
1871 * Returns true if a queued page is found
1873 * @rs: current RAM state
1874 * @pss: data about the state of the current dirty page scan
1876 static bool get_queued_page(RAMState *rs, PageSearchStatus *pss)
1883 block = unqueue_page(rs, &offset);
1885 * We're sending this page, and since it's postcopy nothing else
1886 * will dirty it, and we must make sure it doesn't get sent again
1887 * even if this queue request was received after the background
1888 * search already sent it.
1893 page = offset >> TARGET_PAGE_BITS;
1894 dirty = test_bit(page, block->bmap);
1896 trace_get_queued_page_not_dirty(block->idstr, (uint64_t)offset,
1899 trace_get_queued_page(block->idstr, (uint64_t)offset, page);
1903 } while (block && !dirty);
1907 * Poll write faults too if background snapshot is enabled; that's
1908 * when we have vcpus got blocked by the write protected pages.
1910 block = poll_fault_page(rs, &offset);
1915 * We want the background search to continue from the queued page
1916 * since the guest is likely to want other pages near to the page
1917 * it just requested.
1920 pss->page = offset >> TARGET_PAGE_BITS;
1923 * This unqueued page would break the "one round" check, even is
1926 pss->complete_round = false;
1933 * migration_page_queue_free: drop any remaining pages in the ram
1936 * It should be empty at the end anyway, but in error cases there may
1937 * be some left. in case that there is any page left, we drop it.
1940 static void migration_page_queue_free(RAMState *rs)
1942 struct RAMSrcPageRequest *mspr, *next_mspr;
1943 /* This queue generally should be empty - but in the case of a failed
1944 * migration might have some droppings in.
1946 RCU_READ_LOCK_GUARD();
1947 QSIMPLEQ_FOREACH_SAFE(mspr, &rs->src_page_requests, next_req, next_mspr) {
1948 memory_region_unref(mspr->rb->mr);
1949 QSIMPLEQ_REMOVE_HEAD(&rs->src_page_requests, next_req);
1955 * ram_save_queue_pages: queue the page for transmission
1957 * A request from postcopy destination for example.
1959 * Returns zero on success or negative on error
1961 * @rbname: Name of the RAMBLock of the request. NULL means the
1962 * same that last one.
1963 * @start: starting address from the start of the RAMBlock
1964 * @len: length (in bytes) to send
1966 int ram_save_queue_pages(const char *rbname, ram_addr_t start, ram_addr_t len)
1969 RAMState *rs = ram_state;
1971 stat64_add(&mig_stats.postcopy_requests, 1);
1972 RCU_READ_LOCK_GUARD();
1975 /* Reuse last RAMBlock */
1976 ramblock = rs->last_req_rb;
1980 * Shouldn't happen, we can't reuse the last RAMBlock if
1981 * it's the 1st request.
1983 error_report("ram_save_queue_pages no previous block");
1987 ramblock = qemu_ram_block_by_name(rbname);
1990 /* We shouldn't be asked for a non-existent RAMBlock */
1991 error_report("ram_save_queue_pages no block '%s'", rbname);
1994 rs->last_req_rb = ramblock;
1996 trace_ram_save_queue_pages(ramblock->idstr, start, len);
1997 if (!offset_in_ramblock(ramblock, start + len - 1)) {
1998 error_report("%s request overrun start=" RAM_ADDR_FMT " len="
1999 RAM_ADDR_FMT " blocklen=" RAM_ADDR_FMT,
2000 __func__, start, len, ramblock->used_length);
2005 * When with postcopy preempt, we send back the page directly in the
2008 if (postcopy_preempt_active()) {
2009 ram_addr_t page_start = start >> TARGET_PAGE_BITS;
2010 size_t page_size = qemu_ram_pagesize(ramblock);
2011 PageSearchStatus *pss = &ram_state->pss[RAM_CHANNEL_POSTCOPY];
2014 qemu_mutex_lock(&rs->bitmap_mutex);
2016 pss_init(pss, ramblock, page_start);
2018 * Always use the preempt channel, and make sure it's there. It's
2019 * safe to access without lock, because when rp-thread is running
2020 * we should be the only one who operates on the qemufile
2022 pss->pss_channel = migrate_get_current()->postcopy_qemufile_src;
2023 assert(pss->pss_channel);
2026 * It must be either one or multiple of host page size. Just
2027 * assert; if something wrong we're mostly split brain anyway.
2029 assert(len % page_size == 0);
2031 if (ram_save_host_page_urgent(pss)) {
2032 error_report("%s: ram_save_host_page_urgent() failed: "
2033 "ramblock=%s, start_addr=0x"RAM_ADDR_FMT,
2034 __func__, ramblock->idstr, start);
2039 * NOTE: after ram_save_host_page_urgent() succeeded, pss->page
2040 * will automatically be moved and point to the next host page
2041 * we're going to send, so no need to update here.
2043 * Normally QEMU never sends >1 host page in requests, so
2044 * logically we don't even need that as the loop should only
2045 * run once, but just to be consistent.
2049 qemu_mutex_unlock(&rs->bitmap_mutex);
2054 struct RAMSrcPageRequest *new_entry =
2055 g_new0(struct RAMSrcPageRequest, 1);
2056 new_entry->rb = ramblock;
2057 new_entry->offset = start;
2058 new_entry->len = len;
2060 memory_region_ref(ramblock->mr);
2061 qemu_mutex_lock(&rs->src_page_req_mutex);
2062 QSIMPLEQ_INSERT_TAIL(&rs->src_page_requests, new_entry, next_req);
2063 migration_make_urgent_request();
2064 qemu_mutex_unlock(&rs->src_page_req_mutex);
2069 static bool save_page_use_compression(RAMState *rs)
2071 if (!migrate_compress()) {
2076 * If xbzrle is enabled (e.g., after first round of migration), stop
2077 * using the data compression. In theory, xbzrle can do better than
2080 if (rs->xbzrle_started) {
2088 * try to compress the page before posting it out, return true if the page
2089 * has been properly handled by compression, otherwise needs other
2090 * paths to handle it
2092 static bool save_compress_page(RAMState *rs, PageSearchStatus *pss,
2093 RAMBlock *block, ram_addr_t offset)
2095 if (!save_page_use_compression(rs)) {
2100 * When starting the process of a new block, the first page of
2101 * the block should be sent out before other pages in the same
2102 * block, and all the pages in last block should have been sent
2103 * out, keeping this order is important, because the 'cont' flag
2104 * is used to avoid resending the block name.
2106 * We post the fist page as normal page as compression will take
2107 * much CPU resource.
2109 if (block != pss->last_sent_block) {
2110 ram_flush_compressed_data(rs);
2114 if (compress_page_with_multi_thread(block, offset, send_queued_data) > 0) {
2118 compression_counters.busy++;
2123 * ram_save_target_page_legacy: save one target page
2125 * Returns the number of pages written
2127 * @rs: current RAM state
2128 * @pss: data about the page we want to send
2130 static int ram_save_target_page_legacy(RAMState *rs, PageSearchStatus *pss)
2132 RAMBlock *block = pss->block;
2133 ram_addr_t offset = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS;
2136 if (control_save_page(pss, block, offset, &res)) {
2140 if (save_compress_page(rs, pss, block, offset)) {
2144 res = save_zero_page(pss, pss->pss_channel, block, offset);
2146 /* Must let xbzrle know, otherwise a previous (now 0'd) cached
2147 * page would be stale
2149 if (rs->xbzrle_started) {
2150 XBZRLE_cache_lock();
2151 xbzrle_cache_zero_page(rs, block->offset + offset);
2152 XBZRLE_cache_unlock();
2158 * Do not use multifd in postcopy as one whole host page should be
2159 * placed. Meanwhile postcopy requires atomic update of pages, so even
2160 * if host page size == guest page size the dest guest during run may
2161 * still see partially copied pages which is data corruption.
2163 if (migrate_multifd() && !migration_in_postcopy()) {
2164 return ram_save_multifd_page(pss->pss_channel, block, offset);
2167 return ram_save_page(rs, pss);
2170 /* Should be called before sending a host page */
2171 static void pss_host_page_prepare(PageSearchStatus *pss)
2173 /* How many guest pages are there in one host page? */
2174 size_t guest_pfns = qemu_ram_pagesize(pss->block) >> TARGET_PAGE_BITS;
2176 pss->host_page_sending = true;
2177 if (guest_pfns <= 1) {
2179 * This covers both when guest psize == host psize, or when guest
2180 * has larger psize than the host (guest_pfns==0).
2182 * For the latter, we always send one whole guest page per
2183 * iteration of the host page (example: an Alpha VM on x86 host
2184 * will have guest psize 8K while host psize 4K).
2186 pss->host_page_start = pss->page;
2187 pss->host_page_end = pss->page + 1;
2190 * The host page spans over multiple guest pages, we send them
2191 * within the same host page iteration.
2193 pss->host_page_start = ROUND_DOWN(pss->page, guest_pfns);
2194 pss->host_page_end = ROUND_UP(pss->page + 1, guest_pfns);
2199 * Whether the page pointed by PSS is within the host page being sent.
2200 * Must be called after a previous pss_host_page_prepare().
2202 static bool pss_within_range(PageSearchStatus *pss)
2204 ram_addr_t ram_addr;
2206 assert(pss->host_page_sending);
2208 /* Over host-page boundary? */
2209 if (pss->page >= pss->host_page_end) {
2213 ram_addr = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS;
2215 return offset_in_ramblock(pss->block, ram_addr);
2218 static void pss_host_page_finish(PageSearchStatus *pss)
2220 pss->host_page_sending = false;
2221 /* This is not needed, but just to reset it */
2222 pss->host_page_start = pss->host_page_end = 0;
2226 * Send an urgent host page specified by `pss'. Need to be called with
2227 * bitmap_mutex held.
2229 * Returns 0 if save host page succeeded, false otherwise.
2231 static int ram_save_host_page_urgent(PageSearchStatus *pss)
2233 bool page_dirty, sent = false;
2234 RAMState *rs = ram_state;
2237 trace_postcopy_preempt_send_host_page(pss->block->idstr, pss->page);
2238 pss_host_page_prepare(pss);
2241 * If precopy is sending the same page, let it be done in precopy, or
2242 * we could send the same page in two channels and none of them will
2243 * receive the whole page.
2245 if (pss_overlap(pss, &ram_state->pss[RAM_CHANNEL_PRECOPY])) {
2246 trace_postcopy_preempt_hit(pss->block->idstr,
2247 pss->page << TARGET_PAGE_BITS);
2252 page_dirty = migration_bitmap_clear_dirty(rs, pss->block, pss->page);
2255 /* Be strict to return code; it must be 1, or what else? */
2256 if (migration_ops->ram_save_target_page(rs, pss) != 1) {
2257 error_report_once("%s: ram_save_target_page failed", __func__);
2263 pss_find_next_dirty(pss);
2264 } while (pss_within_range(pss));
2266 pss_host_page_finish(pss);
2267 /* For urgent requests, flush immediately if sent */
2269 qemu_fflush(pss->pss_channel);
2275 * ram_save_host_page: save a whole host page
2277 * Starting at *offset send pages up to the end of the current host
2278 * page. It's valid for the initial offset to point into the middle of
2279 * a host page in which case the remainder of the hostpage is sent.
2280 * Only dirty target pages are sent. Note that the host page size may
2281 * be a huge page for this block.
2283 * The saving stops at the boundary of the used_length of the block
2284 * if the RAMBlock isn't a multiple of the host page size.
2286 * The caller must be with ram_state.bitmap_mutex held to call this
2287 * function. Note that this function can temporarily release the lock, but
2288 * when the function is returned it'll make sure the lock is still held.
2290 * Returns the number of pages written or negative on error
2292 * @rs: current RAM state
2293 * @pss: data about the page we want to send
2295 static int ram_save_host_page(RAMState *rs, PageSearchStatus *pss)
2297 bool page_dirty, preempt_active = postcopy_preempt_active();
2298 int tmppages, pages = 0;
2299 size_t pagesize_bits =
2300 qemu_ram_pagesize(pss->block) >> TARGET_PAGE_BITS;
2301 unsigned long start_page = pss->page;
2304 if (migrate_ram_is_ignored(pss->block)) {
2305 error_report("block %s should not be migrated !", pss->block->idstr);
2309 /* Update host page boundary information */
2310 pss_host_page_prepare(pss);
2313 page_dirty = migration_bitmap_clear_dirty(rs, pss->block, pss->page);
2315 /* Check the pages is dirty and if it is send it */
2318 * Properly yield the lock only in postcopy preempt mode
2319 * because both migration thread and rp-return thread can
2320 * operate on the bitmaps.
2322 if (preempt_active) {
2323 qemu_mutex_unlock(&rs->bitmap_mutex);
2325 tmppages = migration_ops->ram_save_target_page(rs, pss);
2326 if (tmppages >= 0) {
2329 * Allow rate limiting to happen in the middle of huge pages if
2330 * something is sent in the current iteration.
2332 if (pagesize_bits > 1 && tmppages > 0) {
2333 migration_rate_limit();
2336 if (preempt_active) {
2337 qemu_mutex_lock(&rs->bitmap_mutex);
2344 pss_host_page_finish(pss);
2348 pss_find_next_dirty(pss);
2349 } while (pss_within_range(pss));
2351 pss_host_page_finish(pss);
2353 res = ram_save_release_protection(rs, pss, start_page);
2354 return (res < 0 ? res : pages);
2358 * ram_find_and_save_block: finds a dirty page and sends it to f
2360 * Called within an RCU critical section.
2362 * Returns the number of pages written where zero means no dirty pages,
2363 * or negative on error
2365 * @rs: current RAM state
2367 * On systems where host-page-size > target-page-size it will send all the
2368 * pages in a host page that are dirty.
2370 static int ram_find_and_save_block(RAMState *rs)
2372 PageSearchStatus *pss = &rs->pss[RAM_CHANNEL_PRECOPY];
2375 /* No dirty page as there is zero RAM */
2376 if (!rs->ram_bytes_total) {
2381 * Always keep last_seen_block/last_page valid during this procedure,
2382 * because find_dirty_block() relies on these values (e.g., we compare
2383 * last_seen_block with pss.block to see whether we searched all the
2384 * ramblocks) to detect the completion of migration. Having NULL value
2385 * of last_seen_block can conditionally cause below loop to run forever.
2387 if (!rs->last_seen_block) {
2388 rs->last_seen_block = QLIST_FIRST_RCU(&ram_list.blocks);
2392 pss_init(pss, rs->last_seen_block, rs->last_page);
2395 if (!get_queued_page(rs, pss)) {
2396 /* priority queue empty, so just search for something dirty */
2397 int res = find_dirty_block(rs, pss);
2398 if (res != PAGE_DIRTY_FOUND) {
2399 if (res == PAGE_ALL_CLEAN) {
2401 } else if (res == PAGE_TRY_AGAIN) {
2403 } else if (res < 0) {
2409 pages = ram_save_host_page(rs, pss);
2415 rs->last_seen_block = pss->block;
2416 rs->last_page = pss->page;
2421 static uint64_t ram_bytes_total_with_ignored(void)
2426 RCU_READ_LOCK_GUARD();
2428 RAMBLOCK_FOREACH_MIGRATABLE(block) {
2429 total += block->used_length;
2434 uint64_t ram_bytes_total(void)
2439 RCU_READ_LOCK_GUARD();
2441 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2442 total += block->used_length;
2447 static void xbzrle_load_setup(void)
2449 XBZRLE.decoded_buf = g_malloc(TARGET_PAGE_SIZE);
2452 static void xbzrle_load_cleanup(void)
2454 g_free(XBZRLE.decoded_buf);
2455 XBZRLE.decoded_buf = NULL;
2458 static void ram_state_cleanup(RAMState **rsp)
2461 migration_page_queue_free(*rsp);
2462 qemu_mutex_destroy(&(*rsp)->bitmap_mutex);
2463 qemu_mutex_destroy(&(*rsp)->src_page_req_mutex);
2469 static void xbzrle_cleanup(void)
2471 XBZRLE_cache_lock();
2473 cache_fini(XBZRLE.cache);
2474 g_free(XBZRLE.encoded_buf);
2475 g_free(XBZRLE.current_buf);
2476 g_free(XBZRLE.zero_target_page);
2477 XBZRLE.cache = NULL;
2478 XBZRLE.encoded_buf = NULL;
2479 XBZRLE.current_buf = NULL;
2480 XBZRLE.zero_target_page = NULL;
2482 XBZRLE_cache_unlock();
2485 static void ram_save_cleanup(void *opaque)
2487 RAMState **rsp = opaque;
2490 /* We don't use dirty log with background snapshots */
2491 if (!migrate_background_snapshot()) {
2492 /* caller have hold iothread lock or is in a bh, so there is
2493 * no writing race against the migration bitmap
2495 if (global_dirty_tracking & GLOBAL_DIRTY_MIGRATION) {
2497 * do not stop dirty log without starting it, since
2498 * memory_global_dirty_log_stop will assert that
2499 * memory_global_dirty_log_start/stop used in pairs
2501 memory_global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION);
2505 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2506 g_free(block->clear_bmap);
2507 block->clear_bmap = NULL;
2508 g_free(block->bmap);
2513 compress_threads_save_cleanup();
2514 ram_state_cleanup(rsp);
2515 g_free(migration_ops);
2516 migration_ops = NULL;
2519 static void ram_state_reset(RAMState *rs)
2523 for (i = 0; i < RAM_CHANNEL_MAX; i++) {
2524 rs->pss[i].last_sent_block = NULL;
2527 rs->last_seen_block = NULL;
2529 rs->last_version = ram_list.version;
2530 rs->xbzrle_started = false;
2533 #define MAX_WAIT 50 /* ms, half buffered_file limit */
2535 /* **** functions for postcopy ***** */
2537 void ram_postcopy_migrated_memory_release(MigrationState *ms)
2539 struct RAMBlock *block;
2541 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2542 unsigned long *bitmap = block->bmap;
2543 unsigned long range = block->used_length >> TARGET_PAGE_BITS;
2544 unsigned long run_start = find_next_zero_bit(bitmap, range, 0);
2546 while (run_start < range) {
2547 unsigned long run_end = find_next_bit(bitmap, range, run_start + 1);
2548 ram_discard_range(block->idstr,
2549 ((ram_addr_t)run_start) << TARGET_PAGE_BITS,
2550 ((ram_addr_t)(run_end - run_start))
2551 << TARGET_PAGE_BITS);
2552 run_start = find_next_zero_bit(bitmap, range, run_end + 1);
2558 * postcopy_send_discard_bm_ram: discard a RAMBlock
2560 * Callback from postcopy_each_ram_send_discard for each RAMBlock
2562 * @ms: current migration state
2563 * @block: RAMBlock to discard
2565 static void postcopy_send_discard_bm_ram(MigrationState *ms, RAMBlock *block)
2567 unsigned long end = block->used_length >> TARGET_PAGE_BITS;
2568 unsigned long current;
2569 unsigned long *bitmap = block->bmap;
2571 for (current = 0; current < end; ) {
2572 unsigned long one = find_next_bit(bitmap, end, current);
2573 unsigned long zero, discard_length;
2579 zero = find_next_zero_bit(bitmap, end, one + 1);
2582 discard_length = end - one;
2584 discard_length = zero - one;
2586 postcopy_discard_send_range(ms, one, discard_length);
2587 current = one + discard_length;
2591 static void postcopy_chunk_hostpages_pass(MigrationState *ms, RAMBlock *block);
2594 * postcopy_each_ram_send_discard: discard all RAMBlocks
2596 * Utility for the outgoing postcopy code.
2597 * Calls postcopy_send_discard_bm_ram for each RAMBlock
2598 * passing it bitmap indexes and name.
2599 * (qemu_ram_foreach_block ends up passing unscaled lengths
2600 * which would mean postcopy code would have to deal with target page)
2602 * @ms: current migration state
2604 static void postcopy_each_ram_send_discard(MigrationState *ms)
2606 struct RAMBlock *block;
2608 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2609 postcopy_discard_send_init(ms, block->idstr);
2612 * Deal with TPS != HPS and huge pages. It discard any partially sent
2613 * host-page size chunks, mark any partially dirty host-page size
2614 * chunks as all dirty. In this case the host-page is the host-page
2615 * for the particular RAMBlock, i.e. it might be a huge page.
2617 postcopy_chunk_hostpages_pass(ms, block);
2620 * Postcopy sends chunks of bitmap over the wire, but it
2621 * just needs indexes at this point, avoids it having
2622 * target page specific code.
2624 postcopy_send_discard_bm_ram(ms, block);
2625 postcopy_discard_send_finish(ms);
2630 * postcopy_chunk_hostpages_pass: canonicalize bitmap in hostpages
2632 * Helper for postcopy_chunk_hostpages; it's called twice to
2633 * canonicalize the two bitmaps, that are similar, but one is
2636 * Postcopy requires that all target pages in a hostpage are dirty or
2637 * clean, not a mix. This function canonicalizes the bitmaps.
2639 * @ms: current migration state
2640 * @block: block that contains the page we want to canonicalize
2642 static void postcopy_chunk_hostpages_pass(MigrationState *ms, RAMBlock *block)
2644 RAMState *rs = ram_state;
2645 unsigned long *bitmap = block->bmap;
2646 unsigned int host_ratio = block->page_size / TARGET_PAGE_SIZE;
2647 unsigned long pages = block->used_length >> TARGET_PAGE_BITS;
2648 unsigned long run_start;
2650 if (block->page_size == TARGET_PAGE_SIZE) {
2651 /* Easy case - TPS==HPS for a non-huge page RAMBlock */
2655 /* Find a dirty page */
2656 run_start = find_next_bit(bitmap, pages, 0);
2658 while (run_start < pages) {
2661 * If the start of this run of pages is in the middle of a host
2662 * page, then we need to fixup this host page.
2664 if (QEMU_IS_ALIGNED(run_start, host_ratio)) {
2665 /* Find the end of this run */
2666 run_start = find_next_zero_bit(bitmap, pages, run_start + 1);
2668 * If the end isn't at the start of a host page, then the
2669 * run doesn't finish at the end of a host page
2670 * and we need to discard.
2674 if (!QEMU_IS_ALIGNED(run_start, host_ratio)) {
2676 unsigned long fixup_start_addr = QEMU_ALIGN_DOWN(run_start,
2678 run_start = QEMU_ALIGN_UP(run_start, host_ratio);
2680 /* Clean up the bitmap */
2681 for (page = fixup_start_addr;
2682 page < fixup_start_addr + host_ratio; page++) {
2684 * Remark them as dirty, updating the count for any pages
2685 * that weren't previously dirty.
2687 rs->migration_dirty_pages += !test_and_set_bit(page, bitmap);
2691 /* Find the next dirty page for the next iteration */
2692 run_start = find_next_bit(bitmap, pages, run_start);
2697 * ram_postcopy_send_discard_bitmap: transmit the discard bitmap
2699 * Transmit the set of pages to be discarded after precopy to the target
2700 * these are pages that:
2701 * a) Have been previously transmitted but are now dirty again
2702 * b) Pages that have never been transmitted, this ensures that
2703 * any pages on the destination that have been mapped by background
2704 * tasks get discarded (transparent huge pages is the specific concern)
2705 * Hopefully this is pretty sparse
2707 * @ms: current migration state
2709 void ram_postcopy_send_discard_bitmap(MigrationState *ms)
2711 RAMState *rs = ram_state;
2713 RCU_READ_LOCK_GUARD();
2715 /* This should be our last sync, the src is now paused */
2716 migration_bitmap_sync(rs, false);
2718 /* Easiest way to make sure we don't resume in the middle of a host-page */
2719 rs->pss[RAM_CHANNEL_PRECOPY].last_sent_block = NULL;
2720 rs->last_seen_block = NULL;
2723 postcopy_each_ram_send_discard(ms);
2725 trace_ram_postcopy_send_discard_bitmap();
2729 * ram_discard_range: discard dirtied pages at the beginning of postcopy
2731 * Returns zero on success
2733 * @rbname: name of the RAMBlock of the request. NULL means the
2734 * same that last one.
2735 * @start: RAMBlock starting page
2736 * @length: RAMBlock size
2738 int ram_discard_range(const char *rbname, uint64_t start, size_t length)
2740 trace_ram_discard_range(rbname, start, length);
2742 RCU_READ_LOCK_GUARD();
2743 RAMBlock *rb = qemu_ram_block_by_name(rbname);
2746 error_report("ram_discard_range: Failed to find block '%s'", rbname);
2751 * On source VM, we don't need to update the received bitmap since
2752 * we don't even have one.
2754 if (rb->receivedmap) {
2755 bitmap_clear(rb->receivedmap, start >> qemu_target_page_bits(),
2756 length >> qemu_target_page_bits());
2759 return ram_block_discard_range(rb, start, length);
2763 * For every allocation, we will try not to crash the VM if the
2764 * allocation failed.
2766 static int xbzrle_init(void)
2768 Error *local_err = NULL;
2770 if (!migrate_xbzrle()) {
2774 XBZRLE_cache_lock();
2776 XBZRLE.zero_target_page = g_try_malloc0(TARGET_PAGE_SIZE);
2777 if (!XBZRLE.zero_target_page) {
2778 error_report("%s: Error allocating zero page", __func__);
2782 XBZRLE.cache = cache_init(migrate_xbzrle_cache_size(),
2783 TARGET_PAGE_SIZE, &local_err);
2784 if (!XBZRLE.cache) {
2785 error_report_err(local_err);
2786 goto free_zero_page;
2789 XBZRLE.encoded_buf = g_try_malloc0(TARGET_PAGE_SIZE);
2790 if (!XBZRLE.encoded_buf) {
2791 error_report("%s: Error allocating encoded_buf", __func__);
2795 XBZRLE.current_buf = g_try_malloc(TARGET_PAGE_SIZE);
2796 if (!XBZRLE.current_buf) {
2797 error_report("%s: Error allocating current_buf", __func__);
2798 goto free_encoded_buf;
2801 /* We are all good */
2802 XBZRLE_cache_unlock();
2806 g_free(XBZRLE.encoded_buf);
2807 XBZRLE.encoded_buf = NULL;
2809 cache_fini(XBZRLE.cache);
2810 XBZRLE.cache = NULL;
2812 g_free(XBZRLE.zero_target_page);
2813 XBZRLE.zero_target_page = NULL;
2815 XBZRLE_cache_unlock();
2819 static int ram_state_init(RAMState **rsp)
2821 *rsp = g_try_new0(RAMState, 1);
2824 error_report("%s: Init ramstate fail", __func__);
2828 qemu_mutex_init(&(*rsp)->bitmap_mutex);
2829 qemu_mutex_init(&(*rsp)->src_page_req_mutex);
2830 QSIMPLEQ_INIT(&(*rsp)->src_page_requests);
2831 (*rsp)->ram_bytes_total = ram_bytes_total();
2834 * Count the total number of pages used by ram blocks not including any
2835 * gaps due to alignment or unplugs.
2836 * This must match with the initial values of dirty bitmap.
2838 (*rsp)->migration_dirty_pages = (*rsp)->ram_bytes_total >> TARGET_PAGE_BITS;
2839 ram_state_reset(*rsp);
2844 static void ram_list_init_bitmaps(void)
2846 MigrationState *ms = migrate_get_current();
2848 unsigned long pages;
2851 /* Skip setting bitmap if there is no RAM */
2852 if (ram_bytes_total()) {
2853 shift = ms->clear_bitmap_shift;
2854 if (shift > CLEAR_BITMAP_SHIFT_MAX) {
2855 error_report("clear_bitmap_shift (%u) too big, using "
2856 "max value (%u)", shift, CLEAR_BITMAP_SHIFT_MAX);
2857 shift = CLEAR_BITMAP_SHIFT_MAX;
2858 } else if (shift < CLEAR_BITMAP_SHIFT_MIN) {
2859 error_report("clear_bitmap_shift (%u) too small, using "
2860 "min value (%u)", shift, CLEAR_BITMAP_SHIFT_MIN);
2861 shift = CLEAR_BITMAP_SHIFT_MIN;
2864 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2865 pages = block->max_length >> TARGET_PAGE_BITS;
2867 * The initial dirty bitmap for migration must be set with all
2868 * ones to make sure we'll migrate every guest RAM page to
2870 * Here we set RAMBlock.bmap all to 1 because when rebegin a
2871 * new migration after a failed migration, ram_list.
2872 * dirty_memory[DIRTY_MEMORY_MIGRATION] don't include the whole
2875 block->bmap = bitmap_new(pages);
2876 bitmap_set(block->bmap, 0, pages);
2877 block->clear_bmap_shift = shift;
2878 block->clear_bmap = bitmap_new(clear_bmap_size(pages, shift));
2883 static void migration_bitmap_clear_discarded_pages(RAMState *rs)
2885 unsigned long pages;
2888 RCU_READ_LOCK_GUARD();
2890 RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
2891 pages = ramblock_dirty_bitmap_clear_discarded_pages(rb);
2892 rs->migration_dirty_pages -= pages;
2896 static void ram_init_bitmaps(RAMState *rs)
2898 /* For memory_global_dirty_log_start below. */
2899 qemu_mutex_lock_iothread();
2900 qemu_mutex_lock_ramlist();
2902 WITH_RCU_READ_LOCK_GUARD() {
2903 ram_list_init_bitmaps();
2904 /* We don't use dirty log with background snapshots */
2905 if (!migrate_background_snapshot()) {
2906 memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION);
2907 migration_bitmap_sync_precopy(rs, false);
2910 qemu_mutex_unlock_ramlist();
2911 qemu_mutex_unlock_iothread();
2914 * After an eventual first bitmap sync, fixup the initial bitmap
2915 * containing all 1s to exclude any discarded pages from migration.
2917 migration_bitmap_clear_discarded_pages(rs);
2920 static int ram_init_all(RAMState **rsp)
2922 if (ram_state_init(rsp)) {
2926 if (xbzrle_init()) {
2927 ram_state_cleanup(rsp);
2931 ram_init_bitmaps(*rsp);
2936 static void ram_state_resume_prepare(RAMState *rs, QEMUFile *out)
2942 * Postcopy is not using xbzrle/compression, so no need for that.
2943 * Also, since source are already halted, we don't need to care
2944 * about dirty page logging as well.
2947 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2948 pages += bitmap_count_one(block->bmap,
2949 block->used_length >> TARGET_PAGE_BITS);
2952 /* This may not be aligned with current bitmaps. Recalculate. */
2953 rs->migration_dirty_pages = pages;
2955 ram_state_reset(rs);
2957 /* Update RAMState cache of output QEMUFile */
2958 rs->pss[RAM_CHANNEL_PRECOPY].pss_channel = out;
2960 trace_ram_state_resume_prepare(pages);
2964 * This function clears bits of the free pages reported by the caller from the
2965 * migration dirty bitmap. @addr is the host address corresponding to the
2966 * start of the continuous guest free pages, and @len is the total bytes of
2969 void qemu_guest_free_page_hint(void *addr, size_t len)
2973 size_t used_len, start, npages;
2974 MigrationState *s = migrate_get_current();
2976 /* This function is currently expected to be used during live migration */
2977 if (!migration_is_setup_or_active(s->state)) {
2981 for (; len > 0; len -= used_len, addr += used_len) {
2982 block = qemu_ram_block_from_host(addr, false, &offset);
2983 if (unlikely(!block || offset >= block->used_length)) {
2985 * The implementation might not support RAMBlock resize during
2986 * live migration, but it could happen in theory with future
2987 * updates. So we add a check here to capture that case.
2989 error_report_once("%s unexpected error", __func__);
2993 if (len <= block->used_length - offset) {
2996 used_len = block->used_length - offset;
2999 start = offset >> TARGET_PAGE_BITS;
3000 npages = used_len >> TARGET_PAGE_BITS;
3002 qemu_mutex_lock(&ram_state->bitmap_mutex);
3004 * The skipped free pages are equavalent to be sent from clear_bmap's
3005 * perspective, so clear the bits from the memory region bitmap which
3006 * are initially set. Otherwise those skipped pages will be sent in
3007 * the next round after syncing from the memory region bitmap.
3009 migration_clear_memory_region_dirty_bitmap_range(block, start, npages);
3010 ram_state->migration_dirty_pages -=
3011 bitmap_count_one_with_offset(block->bmap, start, npages);
3012 bitmap_clear(block->bmap, start, npages);
3013 qemu_mutex_unlock(&ram_state->bitmap_mutex);
3018 * Each of ram_save_setup, ram_save_iterate and ram_save_complete has
3019 * long-running RCU critical section. When rcu-reclaims in the code
3020 * start to become numerous it will be necessary to reduce the
3021 * granularity of these critical sections.
3025 * ram_save_setup: Setup RAM for migration
3027 * Returns zero to indicate success and negative for error
3029 * @f: QEMUFile where to send the data
3030 * @opaque: RAMState pointer
3032 static int ram_save_setup(QEMUFile *f, void *opaque)
3034 RAMState **rsp = opaque;
3038 if (compress_threads_save_setup()) {
3042 /* migration has already setup the bitmap, reuse it. */
3043 if (!migration_in_colo_state()) {
3044 if (ram_init_all(rsp) != 0) {
3045 compress_threads_save_cleanup();
3049 (*rsp)->pss[RAM_CHANNEL_PRECOPY].pss_channel = f;
3051 WITH_RCU_READ_LOCK_GUARD() {
3052 qemu_put_be64(f, ram_bytes_total_with_ignored()
3053 | RAM_SAVE_FLAG_MEM_SIZE);
3055 RAMBLOCK_FOREACH_MIGRATABLE(block) {
3056 qemu_put_byte(f, strlen(block->idstr));
3057 qemu_put_buffer(f, (uint8_t *)block->idstr, strlen(block->idstr));
3058 qemu_put_be64(f, block->used_length);
3059 if (migrate_postcopy_ram() && block->page_size !=
3060 qemu_host_page_size) {
3061 qemu_put_be64(f, block->page_size);
3063 if (migrate_ignore_shared()) {
3064 qemu_put_be64(f, block->mr->addr);
3069 ram_control_before_iterate(f, RAM_CONTROL_SETUP);
3070 ram_control_after_iterate(f, RAM_CONTROL_SETUP);
3072 migration_ops = g_malloc0(sizeof(MigrationOps));
3073 migration_ops->ram_save_target_page = ram_save_target_page_legacy;
3074 ret = multifd_send_sync_main(f);
3079 if (!migrate_multifd_flush_after_each_section()) {
3080 qemu_put_be64(f, RAM_SAVE_FLAG_MULTIFD_FLUSH);
3083 qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
3090 * ram_save_iterate: iterative stage for migration
3092 * Returns zero to indicate success and negative for error
3094 * @f: QEMUFile where to send the data
3095 * @opaque: RAMState pointer
3097 static int ram_save_iterate(QEMUFile *f, void *opaque)
3099 RAMState **temp = opaque;
3100 RAMState *rs = *temp;
3106 if (blk_mig_bulk_active()) {
3107 /* Avoid transferring ram during bulk phase of block migration as
3108 * the bulk phase will usually take a long time and transferring
3109 * ram updates during that time is pointless. */
3114 * We'll take this lock a little bit long, but it's okay for two reasons.
3115 * Firstly, the only possible other thread to take it is who calls
3116 * qemu_guest_free_page_hint(), which should be rare; secondly, see
3117 * MAX_WAIT (if curious, further see commit 4508bd9ed8053ce) below, which
3118 * guarantees that we'll at least released it in a regular basis.
3120 qemu_mutex_lock(&rs->bitmap_mutex);
3121 WITH_RCU_READ_LOCK_GUARD() {
3122 if (ram_list.version != rs->last_version) {
3123 ram_state_reset(rs);
3126 /* Read version before ram_list.blocks */
3129 ram_control_before_iterate(f, RAM_CONTROL_ROUND);
3131 t0 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
3133 while ((ret = migration_rate_exceeded(f)) == 0 ||
3134 postcopy_has_request(rs)) {
3137 if (qemu_file_get_error(f)) {
3141 pages = ram_find_and_save_block(rs);
3142 /* no more pages to sent */
3149 qemu_file_set_error(f, pages);
3153 rs->target_page_count += pages;
3156 * During postcopy, it is necessary to make sure one whole host
3157 * page is sent in one chunk.
3159 if (migrate_postcopy_ram()) {
3160 ram_flush_compressed_data(rs);
3164 * we want to check in the 1st loop, just in case it was the 1st
3165 * time and we had to sync the dirty bitmap.
3166 * qemu_clock_get_ns() is a bit expensive, so we only check each
3169 if ((i & 63) == 0) {
3170 uint64_t t1 = (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - t0) /
3172 if (t1 > MAX_WAIT) {
3173 trace_ram_save_iterate_big_wait(t1, i);
3180 qemu_mutex_unlock(&rs->bitmap_mutex);
3183 * Must occur before EOS (or any QEMUFile operation)
3184 * because of RDMA protocol.
3186 ram_control_after_iterate(f, RAM_CONTROL_ROUND);
3190 && migration_is_setup_or_active(migrate_get_current()->state)) {
3191 if (migrate_multifd_flush_after_each_section()) {
3192 ret = multifd_send_sync_main(rs->pss[RAM_CHANNEL_PRECOPY].pss_channel);
3198 qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
3200 ram_transferred_add(8);
3202 ret = qemu_file_get_error(f);
3212 * ram_save_complete: function called to send the remaining amount of ram
3214 * Returns zero to indicate success or negative on error
3216 * Called with iothread lock
3218 * @f: QEMUFile where to send the data
3219 * @opaque: RAMState pointer
3221 static int ram_save_complete(QEMUFile *f, void *opaque)
3223 RAMState **temp = opaque;
3224 RAMState *rs = *temp;
3227 rs->last_stage = !migration_in_colo_state();
3229 WITH_RCU_READ_LOCK_GUARD() {
3230 if (!migration_in_postcopy()) {
3231 migration_bitmap_sync_precopy(rs, true);
3234 ram_control_before_iterate(f, RAM_CONTROL_FINISH);
3236 /* try transferring iterative blocks of memory */
3238 /* flush all remaining blocks regardless of rate limiting */
3239 qemu_mutex_lock(&rs->bitmap_mutex);
3243 pages = ram_find_and_save_block(rs);
3244 /* no more blocks to sent */
3253 qemu_mutex_unlock(&rs->bitmap_mutex);
3255 ram_flush_compressed_data(rs);
3256 ram_control_after_iterate(f, RAM_CONTROL_FINISH);
3263 ret = multifd_send_sync_main(rs->pss[RAM_CHANNEL_PRECOPY].pss_channel);
3268 if (!migrate_multifd_flush_after_each_section()) {
3269 qemu_put_be64(f, RAM_SAVE_FLAG_MULTIFD_FLUSH);
3271 qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
3277 static void ram_state_pending_estimate(void *opaque, uint64_t *must_precopy,
3278 uint64_t *can_postcopy)
3280 RAMState **temp = opaque;
3281 RAMState *rs = *temp;
3283 uint64_t remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE;
3285 if (migrate_postcopy_ram()) {
3286 /* We can do postcopy, and all the data is postcopiable */
3287 *can_postcopy += remaining_size;
3289 *must_precopy += remaining_size;
3293 static void ram_state_pending_exact(void *opaque, uint64_t *must_precopy,
3294 uint64_t *can_postcopy)
3296 MigrationState *s = migrate_get_current();
3297 RAMState **temp = opaque;
3298 RAMState *rs = *temp;
3300 uint64_t remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE;
3302 if (!migration_in_postcopy() && remaining_size < s->threshold_size) {
3303 qemu_mutex_lock_iothread();
3304 WITH_RCU_READ_LOCK_GUARD() {
3305 migration_bitmap_sync_precopy(rs, false);
3307 qemu_mutex_unlock_iothread();
3308 remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE;
3311 if (migrate_postcopy_ram()) {
3312 /* We can do postcopy, and all the data is postcopiable */
3313 *can_postcopy += remaining_size;
3315 *must_precopy += remaining_size;
3319 static int load_xbzrle(QEMUFile *f, ram_addr_t addr, void *host)
3321 unsigned int xh_len;
3323 uint8_t *loaded_data;
3325 /* extract RLE header */
3326 xh_flags = qemu_get_byte(f);
3327 xh_len = qemu_get_be16(f);
3329 if (xh_flags != ENCODING_FLAG_XBZRLE) {
3330 error_report("Failed to load XBZRLE page - wrong compression!");
3334 if (xh_len > TARGET_PAGE_SIZE) {
3335 error_report("Failed to load XBZRLE page - len overflow!");
3338 loaded_data = XBZRLE.decoded_buf;
3339 /* load data and decode */
3340 /* it can change loaded_data to point to an internal buffer */
3341 qemu_get_buffer_in_place(f, &loaded_data, xh_len);
3344 if (xbzrle_decode_buffer(loaded_data, xh_len, host,
3345 TARGET_PAGE_SIZE) == -1) {
3346 error_report("Failed to load XBZRLE page - decode error!");
3354 * ram_block_from_stream: read a RAMBlock id from the migration stream
3356 * Must be called from within a rcu critical section.
3358 * Returns a pointer from within the RCU-protected ram_list.
3360 * @mis: the migration incoming state pointer
3361 * @f: QEMUFile where to read the data from
3362 * @flags: Page flags (mostly to see if it's a continuation of previous block)
3363 * @channel: the channel we're using
3365 static inline RAMBlock *ram_block_from_stream(MigrationIncomingState *mis,
3366 QEMUFile *f, int flags,
3369 RAMBlock *block = mis->last_recv_block[channel];
3373 if (flags & RAM_SAVE_FLAG_CONTINUE) {
3375 error_report("Ack, bad migration stream!");
3381 len = qemu_get_byte(f);
3382 qemu_get_buffer(f, (uint8_t *)id, len);
3385 block = qemu_ram_block_by_name(id);
3387 error_report("Can't find block %s", id);
3391 if (migrate_ram_is_ignored(block)) {
3392 error_report("block %s should not be migrated !", id);
3396 mis->last_recv_block[channel] = block;
3401 static inline void *host_from_ram_block_offset(RAMBlock *block,
3404 if (!offset_in_ramblock(block, offset)) {
3408 return block->host + offset;
3411 static void *host_page_from_ram_block_offset(RAMBlock *block,
3414 /* Note: Explicitly no check against offset_in_ramblock(). */
3415 return (void *)QEMU_ALIGN_DOWN((uintptr_t)(block->host + offset),
3419 static ram_addr_t host_page_offset_from_ram_block_offset(RAMBlock *block,
3422 return ((uintptr_t)block->host + offset) & (block->page_size - 1);
3425 void colo_record_bitmap(RAMBlock *block, ram_addr_t *normal, uint32_t pages)
3427 qemu_mutex_lock(&ram_state->bitmap_mutex);
3428 for (int i = 0; i < pages; i++) {
3429 ram_addr_t offset = normal[i];
3430 ram_state->migration_dirty_pages += !test_and_set_bit(
3431 offset >> TARGET_PAGE_BITS,
3434 qemu_mutex_unlock(&ram_state->bitmap_mutex);
3437 static inline void *colo_cache_from_block_offset(RAMBlock *block,
3438 ram_addr_t offset, bool record_bitmap)
3440 if (!offset_in_ramblock(block, offset)) {
3443 if (!block->colo_cache) {
3444 error_report("%s: colo_cache is NULL in block :%s",
3445 __func__, block->idstr);
3450 * During colo checkpoint, we need bitmap of these migrated pages.
3451 * It help us to decide which pages in ram cache should be flushed
3452 * into VM's RAM later.
3454 if (record_bitmap) {
3455 colo_record_bitmap(block, &offset, 1);
3457 return block->colo_cache + offset;
3461 * ram_handle_compressed: handle the zero page case
3463 * If a page (or a whole RDMA chunk) has been
3464 * determined to be zero, then zap it.
3466 * @host: host address for the zero page
3467 * @ch: what the page is filled from. We only support zero
3468 * @size: size of the zero page
3470 void ram_handle_compressed(void *host, uint8_t ch, uint64_t size)
3472 if (ch != 0 || !buffer_is_zero(host, size)) {
3473 memset(host, ch, size);
3477 static void colo_init_ram_state(void)
3479 ram_state_init(&ram_state);
3483 * colo cache: this is for secondary VM, we cache the whole
3484 * memory of the secondary VM, it is need to hold the global lock
3485 * to call this helper.
3487 int colo_init_ram_cache(void)
3491 WITH_RCU_READ_LOCK_GUARD() {
3492 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3493 block->colo_cache = qemu_anon_ram_alloc(block->used_length,
3494 NULL, false, false);
3495 if (!block->colo_cache) {
3496 error_report("%s: Can't alloc memory for COLO cache of block %s,"
3497 "size 0x" RAM_ADDR_FMT, __func__, block->idstr,
3498 block->used_length);
3499 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3500 if (block->colo_cache) {
3501 qemu_anon_ram_free(block->colo_cache, block->used_length);
3502 block->colo_cache = NULL;
3507 if (!machine_dump_guest_core(current_machine)) {
3508 qemu_madvise(block->colo_cache, block->used_length,
3509 QEMU_MADV_DONTDUMP);
3515 * Record the dirty pages that sent by PVM, we use this dirty bitmap together
3516 * with to decide which page in cache should be flushed into SVM's RAM. Here
3517 * we use the same name 'ram_bitmap' as for migration.
3519 if (ram_bytes_total()) {
3522 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3523 unsigned long pages = block->max_length >> TARGET_PAGE_BITS;
3524 block->bmap = bitmap_new(pages);
3528 colo_init_ram_state();
3532 /* TODO: duplicated with ram_init_bitmaps */
3533 void colo_incoming_start_dirty_log(void)
3535 RAMBlock *block = NULL;
3536 /* For memory_global_dirty_log_start below. */
3537 qemu_mutex_lock_iothread();
3538 qemu_mutex_lock_ramlist();
3540 memory_global_dirty_log_sync(false);
3541 WITH_RCU_READ_LOCK_GUARD() {
3542 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3543 ramblock_sync_dirty_bitmap(ram_state, block);
3544 /* Discard this dirty bitmap record */
3545 bitmap_zero(block->bmap, block->max_length >> TARGET_PAGE_BITS);
3547 memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION);
3549 ram_state->migration_dirty_pages = 0;
3550 qemu_mutex_unlock_ramlist();
3551 qemu_mutex_unlock_iothread();
3554 /* It is need to hold the global lock to call this helper */
3555 void colo_release_ram_cache(void)
3559 memory_global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION);
3560 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3561 g_free(block->bmap);
3565 WITH_RCU_READ_LOCK_GUARD() {
3566 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3567 if (block->colo_cache) {
3568 qemu_anon_ram_free(block->colo_cache, block->used_length);
3569 block->colo_cache = NULL;
3573 ram_state_cleanup(&ram_state);
3577 * ram_load_setup: Setup RAM for migration incoming side
3579 * Returns zero to indicate success and negative for error
3581 * @f: QEMUFile where to receive the data
3582 * @opaque: RAMState pointer
3584 static int ram_load_setup(QEMUFile *f, void *opaque)
3586 xbzrle_load_setup();
3587 ramblock_recv_map_init();
3592 static int ram_load_cleanup(void *opaque)
3596 RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
3597 qemu_ram_block_writeback(rb);
3600 xbzrle_load_cleanup();
3602 RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
3603 g_free(rb->receivedmap);
3604 rb->receivedmap = NULL;
3611 * ram_postcopy_incoming_init: allocate postcopy data structures
3613 * Returns 0 for success and negative if there was one error
3615 * @mis: current migration incoming state
3617 * Allocate data structures etc needed by incoming migration with
3618 * postcopy-ram. postcopy-ram's similarly names
3619 * postcopy_ram_incoming_init does the work.
3621 int ram_postcopy_incoming_init(MigrationIncomingState *mis)
3623 return postcopy_ram_incoming_init(mis);
3627 * ram_load_postcopy: load a page in postcopy case
3629 * Returns 0 for success or -errno in case of error
3631 * Called in postcopy mode by ram_load().
3632 * rcu_read_lock is taken prior to this being called.
3634 * @f: QEMUFile where to send the data
3635 * @channel: the channel to use for loading
3637 int ram_load_postcopy(QEMUFile *f, int channel)
3639 int flags = 0, ret = 0;
3640 bool place_needed = false;
3641 bool matches_target_page_size = false;
3642 MigrationIncomingState *mis = migration_incoming_get_current();
3643 PostcopyTmpPage *tmp_page = &mis->postcopy_tmp_pages[channel];
3645 while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) {
3647 void *page_buffer = NULL;
3648 void *place_source = NULL;
3649 RAMBlock *block = NULL;
3653 addr = qemu_get_be64(f);
3656 * If qemu file error, we should stop here, and then "addr"
3659 ret = qemu_file_get_error(f);
3664 flags = addr & ~TARGET_PAGE_MASK;
3665 addr &= TARGET_PAGE_MASK;
3667 trace_ram_load_postcopy_loop(channel, (uint64_t)addr, flags);
3668 if (flags & (RAM_SAVE_FLAG_ZERO | RAM_SAVE_FLAG_PAGE |
3669 RAM_SAVE_FLAG_COMPRESS_PAGE)) {
3670 block = ram_block_from_stream(mis, f, flags, channel);
3677 * Relying on used_length is racy and can result in false positives.
3678 * We might place pages beyond used_length in case RAM was shrunk
3679 * while in postcopy, which is fine - trying to place via
3680 * UFFDIO_COPY/UFFDIO_ZEROPAGE will never segfault.
3682 if (!block->host || addr >= block->postcopy_length) {
3683 error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
3687 tmp_page->target_pages++;
3688 matches_target_page_size = block->page_size == TARGET_PAGE_SIZE;
3690 * Postcopy requires that we place whole host pages atomically;
3691 * these may be huge pages for RAMBlocks that are backed by
3693 * To make it atomic, the data is read into a temporary page
3694 * that's moved into place later.
3695 * The migration protocol uses, possibly smaller, target-pages
3696 * however the source ensures it always sends all the components
3697 * of a host page in one chunk.
3699 page_buffer = tmp_page->tmp_huge_page +
3700 host_page_offset_from_ram_block_offset(block, addr);
3701 /* If all TP are zero then we can optimise the place */
3702 if (tmp_page->target_pages == 1) {
3703 tmp_page->host_addr =
3704 host_page_from_ram_block_offset(block, addr);
3705 } else if (tmp_page->host_addr !=
3706 host_page_from_ram_block_offset(block, addr)) {
3707 /* not the 1st TP within the HP */
3708 error_report("Non-same host page detected on channel %d: "
3709 "Target host page %p, received host page %p "
3710 "(rb %s offset 0x"RAM_ADDR_FMT" target_pages %d)",
3711 channel, tmp_page->host_addr,
3712 host_page_from_ram_block_offset(block, addr),
3713 block->idstr, addr, tmp_page->target_pages);
3719 * If it's the last part of a host page then we place the host
3722 if (tmp_page->target_pages ==
3723 (block->page_size / TARGET_PAGE_SIZE)) {
3724 place_needed = true;
3726 place_source = tmp_page->tmp_huge_page;
3729 switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
3730 case RAM_SAVE_FLAG_ZERO:
3731 ch = qemu_get_byte(f);
3733 * Can skip to set page_buffer when
3734 * this is a zero page and (block->page_size == TARGET_PAGE_SIZE).
3736 if (ch || !matches_target_page_size) {
3737 memset(page_buffer, ch, TARGET_PAGE_SIZE);
3740 tmp_page->all_zero = false;
3744 case RAM_SAVE_FLAG_PAGE:
3745 tmp_page->all_zero = false;
3746 if (!matches_target_page_size) {
3747 /* For huge pages, we always use temporary buffer */
3748 qemu_get_buffer(f, page_buffer, TARGET_PAGE_SIZE);
3751 * For small pages that matches target page size, we
3752 * avoid the qemu_file copy. Instead we directly use
3753 * the buffer of QEMUFile to place the page. Note: we
3754 * cannot do any QEMUFile operation before using that
3755 * buffer to make sure the buffer is valid when
3758 qemu_get_buffer_in_place(f, (uint8_t **)&place_source,
3762 case RAM_SAVE_FLAG_COMPRESS_PAGE:
3763 tmp_page->all_zero = false;
3764 len = qemu_get_be32(f);
3765 if (len < 0 || len > compressBound(TARGET_PAGE_SIZE)) {
3766 error_report("Invalid compressed data length: %d", len);
3770 decompress_data_with_multi_threads(f, page_buffer, len);
3772 case RAM_SAVE_FLAG_MULTIFD_FLUSH:
3773 multifd_recv_sync_main();
3775 case RAM_SAVE_FLAG_EOS:
3777 if (migrate_multifd_flush_after_each_section()) {
3778 multifd_recv_sync_main();
3782 error_report("Unknown combination of migration flags: 0x%x"
3783 " (postcopy mode)", flags);
3788 /* Got the whole host page, wait for decompress before placing. */
3790 ret |= wait_for_decompress_done();
3793 /* Detect for any possible file errors */
3794 if (!ret && qemu_file_get_error(f)) {
3795 ret = qemu_file_get_error(f);
3798 if (!ret && place_needed) {
3799 if (tmp_page->all_zero) {
3800 ret = postcopy_place_page_zero(mis, tmp_page->host_addr, block);
3802 ret = postcopy_place_page(mis, tmp_page->host_addr,
3803 place_source, block);
3805 place_needed = false;
3806 postcopy_temp_page_reset(tmp_page);
3813 static bool postcopy_is_running(void)
3815 PostcopyState ps = postcopy_state_get();
3816 return ps >= POSTCOPY_INCOMING_LISTENING && ps < POSTCOPY_INCOMING_END;
3820 * Flush content of RAM cache into SVM's memory.
3821 * Only flush the pages that be dirtied by PVM or SVM or both.
3823 void colo_flush_ram_cache(void)
3825 RAMBlock *block = NULL;
3828 unsigned long offset = 0;
3830 memory_global_dirty_log_sync(false);
3831 qemu_mutex_lock(&ram_state->bitmap_mutex);
3832 WITH_RCU_READ_LOCK_GUARD() {
3833 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3834 ramblock_sync_dirty_bitmap(ram_state, block);
3838 trace_colo_flush_ram_cache_begin(ram_state->migration_dirty_pages);
3839 WITH_RCU_READ_LOCK_GUARD() {
3840 block = QLIST_FIRST_RCU(&ram_list.blocks);
3843 unsigned long num = 0;
3845 offset = colo_bitmap_find_dirty(ram_state, block, offset, &num);
3846 if (!offset_in_ramblock(block,
3847 ((ram_addr_t)offset) << TARGET_PAGE_BITS)) {
3850 block = QLIST_NEXT_RCU(block, next);
3852 unsigned long i = 0;
3854 for (i = 0; i < num; i++) {
3855 migration_bitmap_clear_dirty(ram_state, block, offset + i);
3857 dst_host = block->host
3858 + (((ram_addr_t)offset) << TARGET_PAGE_BITS);
3859 src_host = block->colo_cache
3860 + (((ram_addr_t)offset) << TARGET_PAGE_BITS);
3861 memcpy(dst_host, src_host, TARGET_PAGE_SIZE * num);
3866 qemu_mutex_unlock(&ram_state->bitmap_mutex);
3867 trace_colo_flush_ram_cache_end();
3871 * ram_load_precopy: load pages in precopy case
3873 * Returns 0 for success or -errno in case of error
3875 * Called in precopy mode by ram_load().
3876 * rcu_read_lock is taken prior to this being called.
3878 * @f: QEMUFile where to send the data
3880 static int ram_load_precopy(QEMUFile *f)
3882 MigrationIncomingState *mis = migration_incoming_get_current();
3883 int flags = 0, ret = 0, invalid_flags = 0, len = 0, i = 0;
3884 /* ADVISE is earlier, it shows the source has the postcopy capability on */
3885 bool postcopy_advised = migration_incoming_postcopy_advised();
3886 if (!migrate_compress()) {
3887 invalid_flags |= RAM_SAVE_FLAG_COMPRESS_PAGE;
3890 while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) {
3891 ram_addr_t addr, total_ram_bytes;
3892 void *host = NULL, *host_bak = NULL;
3896 * Yield periodically to let main loop run, but an iteration of
3897 * the main loop is expensive, so do it each some iterations
3899 if ((i & 32767) == 0 && qemu_in_coroutine()) {
3900 aio_co_schedule(qemu_get_current_aio_context(),
3901 qemu_coroutine_self());
3902 qemu_coroutine_yield();
3906 addr = qemu_get_be64(f);
3907 flags = addr & ~TARGET_PAGE_MASK;
3908 addr &= TARGET_PAGE_MASK;
3910 if (flags & invalid_flags) {
3911 if (flags & invalid_flags & RAM_SAVE_FLAG_COMPRESS_PAGE) {
3912 error_report("Received an unexpected compressed page");
3919 if (flags & (RAM_SAVE_FLAG_ZERO | RAM_SAVE_FLAG_PAGE |
3920 RAM_SAVE_FLAG_COMPRESS_PAGE | RAM_SAVE_FLAG_XBZRLE)) {
3921 RAMBlock *block = ram_block_from_stream(mis, f, flags,
3922 RAM_CHANNEL_PRECOPY);
3924 host = host_from_ram_block_offset(block, addr);
3926 * After going into COLO stage, we should not load the page
3927 * into SVM's memory directly, we put them into colo_cache firstly.
3928 * NOTE: We need to keep a copy of SVM's ram in colo_cache.
3929 * Previously, we copied all these memory in preparing stage of COLO
3930 * while we need to stop VM, which is a time-consuming process.
3931 * Here we optimize it by a trick, back-up every page while in
3932 * migration process while COLO is enabled, though it affects the
3933 * speed of the migration, but it obviously reduce the downtime of
3934 * back-up all SVM'S memory in COLO preparing stage.
3936 if (migration_incoming_colo_enabled()) {
3937 if (migration_incoming_in_colo_state()) {
3938 /* In COLO stage, put all pages into cache temporarily */
3939 host = colo_cache_from_block_offset(block, addr, true);
3942 * In migration stage but before COLO stage,
3943 * Put all pages into both cache and SVM's memory.
3945 host_bak = colo_cache_from_block_offset(block, addr, false);
3949 error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
3953 if (!migration_incoming_in_colo_state()) {
3954 ramblock_recv_bitmap_set(block, host);
3957 trace_ram_load_loop(block->idstr, (uint64_t)addr, flags, host);
3960 switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
3961 case RAM_SAVE_FLAG_MEM_SIZE:
3962 /* Synchronize RAM block list */
3963 total_ram_bytes = addr;
3964 while (!ret && total_ram_bytes) {
3969 len = qemu_get_byte(f);
3970 qemu_get_buffer(f, (uint8_t *)id, len);
3972 length = qemu_get_be64(f);
3974 block = qemu_ram_block_by_name(id);
3975 if (block && !qemu_ram_is_migratable(block)) {
3976 error_report("block %s should not be migrated !", id);
3979 if (length != block->used_length) {
3980 Error *local_err = NULL;
3982 ret = qemu_ram_resize(block, length,
3985 error_report_err(local_err);
3988 /* For postcopy we need to check hugepage sizes match */
3989 if (postcopy_advised && migrate_postcopy_ram() &&
3990 block->page_size != qemu_host_page_size) {
3991 uint64_t remote_page_size = qemu_get_be64(f);
3992 if (remote_page_size != block->page_size) {
3993 error_report("Mismatched RAM page size %s "
3994 "(local) %zd != %" PRId64,
3995 id, block->page_size,
4000 if (migrate_ignore_shared()) {
4001 hwaddr addr = qemu_get_be64(f);
4002 if (migrate_ram_is_ignored(block) &&
4003 block->mr->addr != addr) {
4004 error_report("Mismatched GPAs for block %s "
4005 "%" PRId64 "!= %" PRId64,
4007 (uint64_t)block->mr->addr);
4011 ram_control_load_hook(f, RAM_CONTROL_BLOCK_REG,
4014 error_report("Unknown ramblock \"%s\", cannot "
4015 "accept migration", id);
4019 total_ram_bytes -= length;
4023 case RAM_SAVE_FLAG_ZERO:
4024 ch = qemu_get_byte(f);
4025 ram_handle_compressed(host, ch, TARGET_PAGE_SIZE);
4028 case RAM_SAVE_FLAG_PAGE:
4029 qemu_get_buffer(f, host, TARGET_PAGE_SIZE);
4032 case RAM_SAVE_FLAG_COMPRESS_PAGE:
4033 len = qemu_get_be32(f);
4034 if (len < 0 || len > compressBound(TARGET_PAGE_SIZE)) {
4035 error_report("Invalid compressed data length: %d", len);
4039 decompress_data_with_multi_threads(f, host, len);
4042 case RAM_SAVE_FLAG_XBZRLE:
4043 if (load_xbzrle(f, addr, host) < 0) {
4044 error_report("Failed to decompress XBZRLE page at "
4045 RAM_ADDR_FMT, addr);
4050 case RAM_SAVE_FLAG_MULTIFD_FLUSH:
4051 multifd_recv_sync_main();
4053 case RAM_SAVE_FLAG_EOS:
4055 if (migrate_multifd_flush_after_each_section()) {
4056 multifd_recv_sync_main();
4059 case RAM_SAVE_FLAG_HOOK:
4060 ram_control_load_hook(f, RAM_CONTROL_HOOK, NULL);
4063 error_report("Unknown combination of migration flags: 0x%x", flags);
4067 ret = qemu_file_get_error(f);
4069 if (!ret && host_bak) {
4070 memcpy(host_bak, host, TARGET_PAGE_SIZE);
4074 ret |= wait_for_decompress_done();
4078 static int ram_load(QEMUFile *f, void *opaque, int version_id)
4081 static uint64_t seq_iter;
4083 * If system is running in postcopy mode, page inserts to host memory must
4086 bool postcopy_running = postcopy_is_running();
4090 if (version_id != 4) {
4095 * This RCU critical section can be very long running.
4096 * When RCU reclaims in the code start to become numerous,
4097 * it will be necessary to reduce the granularity of this
4100 WITH_RCU_READ_LOCK_GUARD() {
4101 if (postcopy_running) {
4103 * Note! Here RAM_CHANNEL_PRECOPY is the precopy channel of
4104 * postcopy migration, we have another RAM_CHANNEL_POSTCOPY to
4105 * service fast page faults.
4107 ret = ram_load_postcopy(f, RAM_CHANNEL_PRECOPY);
4109 ret = ram_load_precopy(f);
4112 trace_ram_load_complete(ret, seq_iter);
4117 static bool ram_has_postcopy(void *opaque)
4120 RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
4121 if (ramblock_is_pmem(rb)) {
4122 info_report("Block: %s, host: %p is a nvdimm memory, postcopy"
4123 "is not supported now!", rb->idstr, rb->host);
4128 return migrate_postcopy_ram();
4131 /* Sync all the dirty bitmap with destination VM. */
4132 static int ram_dirty_bitmap_sync_all(MigrationState *s, RAMState *rs)
4135 QEMUFile *file = s->to_dst_file;
4136 int ramblock_count = 0;
4138 trace_ram_dirty_bitmap_sync_start();
4140 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
4141 qemu_savevm_send_recv_bitmap(file, block->idstr);
4142 trace_ram_dirty_bitmap_request(block->idstr);
4146 trace_ram_dirty_bitmap_sync_wait();
4148 /* Wait until all the ramblocks' dirty bitmap synced */
4149 while (ramblock_count--) {
4150 qemu_sem_wait(&s->rp_state.rp_sem);
4153 trace_ram_dirty_bitmap_sync_complete();
4158 static void ram_dirty_bitmap_reload_notify(MigrationState *s)
4160 qemu_sem_post(&s->rp_state.rp_sem);
4164 * Read the received bitmap, revert it as the initial dirty bitmap.
4165 * This is only used when the postcopy migration is paused but wants
4166 * to resume from a middle point.
4168 int ram_dirty_bitmap_reload(MigrationState *s, RAMBlock *block)
4171 /* from_dst_file is always valid because we're within rp_thread */
4172 QEMUFile *file = s->rp_state.from_dst_file;
4173 unsigned long *le_bitmap, nbits = block->used_length >> TARGET_PAGE_BITS;
4174 uint64_t local_size = DIV_ROUND_UP(nbits, 8);
4175 uint64_t size, end_mark;
4177 trace_ram_dirty_bitmap_reload_begin(block->idstr);
4179 if (s->state != MIGRATION_STATUS_POSTCOPY_RECOVER) {
4180 error_report("%s: incorrect state %s", __func__,
4181 MigrationStatus_str(s->state));
4186 * Note: see comments in ramblock_recv_bitmap_send() on why we
4187 * need the endianness conversion, and the paddings.
4189 local_size = ROUND_UP(local_size, 8);
4192 le_bitmap = bitmap_new(nbits + BITS_PER_LONG);
4194 size = qemu_get_be64(file);
4196 /* The size of the bitmap should match with our ramblock */
4197 if (size != local_size) {
4198 error_report("%s: ramblock '%s' bitmap size mismatch "
4199 "(0x%"PRIx64" != 0x%"PRIx64")", __func__,
4200 block->idstr, size, local_size);
4205 size = qemu_get_buffer(file, (uint8_t *)le_bitmap, local_size);
4206 end_mark = qemu_get_be64(file);
4208 ret = qemu_file_get_error(file);
4209 if (ret || size != local_size) {
4210 error_report("%s: read bitmap failed for ramblock '%s': %d"
4211 " (size 0x%"PRIx64", got: 0x%"PRIx64")",
4212 __func__, block->idstr, ret, local_size, size);
4217 if (end_mark != RAMBLOCK_RECV_BITMAP_ENDING) {
4218 error_report("%s: ramblock '%s' end mark incorrect: 0x%"PRIx64,
4219 __func__, block->idstr, end_mark);
4225 * Endianness conversion. We are during postcopy (though paused).
4226 * The dirty bitmap won't change. We can directly modify it.
4228 bitmap_from_le(block->bmap, le_bitmap, nbits);
4231 * What we received is "received bitmap". Revert it as the initial
4232 * dirty bitmap for this ramblock.
4234 bitmap_complement(block->bmap, block->bmap, nbits);
4236 /* Clear dirty bits of discarded ranges that we don't want to migrate. */
4237 ramblock_dirty_bitmap_clear_discarded_pages(block);
4239 /* We'll recalculate migration_dirty_pages in ram_state_resume_prepare(). */
4240 trace_ram_dirty_bitmap_reload_complete(block->idstr);
4243 * We succeeded to sync bitmap for current ramblock. If this is
4244 * the last one to sync, we need to notify the main send thread.
4246 ram_dirty_bitmap_reload_notify(s);
4254 static int ram_resume_prepare(MigrationState *s, void *opaque)
4256 RAMState *rs = *(RAMState **)opaque;
4259 ret = ram_dirty_bitmap_sync_all(s, rs);
4264 ram_state_resume_prepare(rs, s->to_dst_file);
4269 void postcopy_preempt_shutdown_file(MigrationState *s)
4271 qemu_put_be64(s->postcopy_qemufile_src, RAM_SAVE_FLAG_EOS);
4272 qemu_fflush(s->postcopy_qemufile_src);
4275 static SaveVMHandlers savevm_ram_handlers = {
4276 .save_setup = ram_save_setup,
4277 .save_live_iterate = ram_save_iterate,
4278 .save_live_complete_postcopy = ram_save_complete,
4279 .save_live_complete_precopy = ram_save_complete,
4280 .has_postcopy = ram_has_postcopy,
4281 .state_pending_exact = ram_state_pending_exact,
4282 .state_pending_estimate = ram_state_pending_estimate,
4283 .load_state = ram_load,
4284 .save_cleanup = ram_save_cleanup,
4285 .load_setup = ram_load_setup,
4286 .load_cleanup = ram_load_cleanup,
4287 .resume_prepare = ram_resume_prepare,
4290 static void ram_mig_ram_block_resized(RAMBlockNotifier *n, void *host,
4291 size_t old_size, size_t new_size)
4293 PostcopyState ps = postcopy_state_get();
4295 RAMBlock *rb = qemu_ram_block_from_host(host, false, &offset);
4298 if (migrate_ram_is_ignored(rb)) {
4302 if (!migration_is_idle()) {
4304 * Precopy code on the source cannot deal with the size of RAM blocks
4305 * changing at random points in time - especially after sending the
4306 * RAM block sizes in the migration stream, they must no longer change.
4307 * Abort and indicate a proper reason.
4309 error_setg(&err, "RAM block '%s' resized during precopy.", rb->idstr);
4310 migration_cancel(err);
4315 case POSTCOPY_INCOMING_ADVISE:
4317 * Update what ram_postcopy_incoming_init()->init_range() does at the
4318 * time postcopy was advised. Syncing RAM blocks with the source will
4319 * result in RAM resizes.
4321 if (old_size < new_size) {
4322 if (ram_discard_range(rb->idstr, old_size, new_size - old_size)) {
4323 error_report("RAM block '%s' discard of resized RAM failed",
4327 rb->postcopy_length = new_size;
4329 case POSTCOPY_INCOMING_NONE:
4330 case POSTCOPY_INCOMING_RUNNING:
4331 case POSTCOPY_INCOMING_END:
4333 * Once our guest is running, postcopy does no longer care about
4334 * resizes. When growing, the new memory was not available on the
4335 * source, no handler needed.
4339 error_report("RAM block '%s' resized during postcopy state: %d",
4345 static RAMBlockNotifier ram_mig_ram_notifier = {
4346 .ram_block_resized = ram_mig_ram_block_resized,
4349 void ram_mig_init(void)
4351 qemu_mutex_init(&XBZRLE.lock);
4352 register_savevm_live("ram", 0, 4, &savevm_ram_handlers, &ram_state);
4353 ram_block_notifier_add(&ram_mig_ram_notifier);