2 * Declarations for cpu physical memory functions
4 * Copyright 2011 Red Hat, Inc. and/or its affiliates
7 * Avi Kivity <avi@redhat.com>
9 * This work is licensed under the terms of the GNU GPL, version 2 or
10 * later. See the COPYING file in the top-level directory.
15 * This header is for use by exec.c and memory.c ONLY. Do not include it.
16 * The functions declared here will be removed soon.
22 #ifndef CONFIG_USER_ONLY
24 #include "sysemu/xen.h"
25 #include "sysemu/tcg.h"
26 #include "exec/ramlist.h"
27 #include "exec/ramblock.h"
29 extern uint64_t total_dirty_pages;
32 * clear_bmap_size: calculate clear bitmap size
34 * @pages: number of guest pages
35 * @shift: guest page number shift
37 * Returns: number of bits for the clear bitmap
39 static inline long clear_bmap_size(uint64_t pages, uint8_t shift)
41 return DIV_ROUND_UP(pages, 1UL << shift);
45 * clear_bmap_set: set clear bitmap for the page range. Must be with
48 * @rb: the ramblock to operate on
49 * @start: the start page number
50 * @size: number of pages to set in the bitmap
54 static inline void clear_bmap_set(RAMBlock *rb, uint64_t start,
57 uint8_t shift = rb->clear_bmap_shift;
59 bitmap_set(rb->clear_bmap, start >> shift, clear_bmap_size(npages, shift));
63 * clear_bmap_test_and_clear: test clear bitmap for the page, clear if set.
64 * Must be with bitmap_mutex held.
66 * @rb: the ramblock to operate on
67 * @page: the page number to check
69 * Returns: true if the bit was set, false otherwise
71 static inline bool clear_bmap_test_and_clear(RAMBlock *rb, uint64_t page)
73 uint8_t shift = rb->clear_bmap_shift;
75 return bitmap_test_and_clear(rb->clear_bmap, page >> shift, 1);
78 static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset)
80 return (b && b->host && offset < b->used_length) ? true : false;
83 static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset)
85 assert(offset_in_ramblock(block, offset));
86 return (char *)block->host + offset;
89 static inline unsigned long int ramblock_recv_bitmap_offset(void *host_addr,
92 uint64_t host_addr_offset =
93 (uint64_t)(uintptr_t)(host_addr - (void *)rb->host);
94 return host_addr_offset >> TARGET_PAGE_BITS;
97 bool ramblock_is_pmem(RAMBlock *rb);
99 long qemu_minrampagesize(void);
100 long qemu_maxrampagesize(void);
103 * qemu_ram_alloc_from_file,
104 * qemu_ram_alloc_from_fd: Allocate a ram block from the specified backing
108 * @size: the size in bytes of the ram block
109 * @mr: the memory region where the ram block is
110 * @ram_flags: RamBlock flags. Supported flags: RAM_SHARED, RAM_PMEM,
112 * @mem_path or @fd: specify the backing file or device
113 * @readonly: true to open @path for reading, false for read/write.
114 * @errp: pointer to Error*, to store an error if it happens
117 * On success, return a pointer to the ram block.
118 * On failure, return NULL.
120 RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
121 uint32_t ram_flags, const char *mem_path,
122 bool readonly, Error **errp);
123 RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr,
124 uint32_t ram_flags, int fd, off_t offset,
125 bool readonly, Error **errp);
127 RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
128 MemoryRegion *mr, Error **errp);
129 RAMBlock *qemu_ram_alloc(ram_addr_t size, uint32_t ram_flags, MemoryRegion *mr,
131 RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size,
132 void (*resized)(const char*,
135 MemoryRegion *mr, Error **errp);
136 void qemu_ram_free(RAMBlock *block);
138 int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp);
140 void qemu_ram_msync(RAMBlock *block, ram_addr_t start, ram_addr_t length);
142 /* Clear whole block of mem */
143 static inline void qemu_ram_block_writeback(RAMBlock *block)
145 qemu_ram_msync(block, 0, block->used_length);
148 #define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1)
149 #define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE))
151 static inline bool cpu_physical_memory_get_dirty(ram_addr_t start,
155 DirtyMemoryBlocks *blocks;
156 unsigned long end, page;
157 unsigned long idx, offset, base;
160 assert(client < DIRTY_MEMORY_NUM);
162 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
163 page = start >> TARGET_PAGE_BITS;
165 WITH_RCU_READ_LOCK_GUARD() {
166 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
168 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
169 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
170 base = page - offset;
172 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
173 unsigned long num = next - base;
174 unsigned long found = find_next_bit(blocks->blocks[idx],
184 base += DIRTY_MEMORY_BLOCK_SIZE;
191 static inline bool cpu_physical_memory_all_dirty(ram_addr_t start,
195 DirtyMemoryBlocks *blocks;
196 unsigned long end, page;
197 unsigned long idx, offset, base;
200 assert(client < DIRTY_MEMORY_NUM);
202 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
203 page = start >> TARGET_PAGE_BITS;
205 RCU_READ_LOCK_GUARD();
207 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
209 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
210 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
211 base = page - offset;
213 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
214 unsigned long num = next - base;
215 unsigned long found = find_next_zero_bit(blocks->blocks[idx], num, offset);
224 base += DIRTY_MEMORY_BLOCK_SIZE;
230 static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr,
233 return cpu_physical_memory_get_dirty(addr, 1, client);
236 static inline bool cpu_physical_memory_is_clean(ram_addr_t addr)
238 bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA);
239 bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE);
241 cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION);
242 return !(vga && code && migration);
245 static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start,
251 if (mask & (1 << DIRTY_MEMORY_VGA) &&
252 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) {
253 ret |= (1 << DIRTY_MEMORY_VGA);
255 if (mask & (1 << DIRTY_MEMORY_CODE) &&
256 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) {
257 ret |= (1 << DIRTY_MEMORY_CODE);
259 if (mask & (1 << DIRTY_MEMORY_MIGRATION) &&
260 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) {
261 ret |= (1 << DIRTY_MEMORY_MIGRATION);
266 static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr,
269 unsigned long page, idx, offset;
270 DirtyMemoryBlocks *blocks;
272 assert(client < DIRTY_MEMORY_NUM);
274 page = addr >> TARGET_PAGE_BITS;
275 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
276 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
278 RCU_READ_LOCK_GUARD();
280 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
282 set_bit_atomic(offset, blocks->blocks[idx]);
285 static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start,
289 DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM];
290 unsigned long end, page;
291 unsigned long idx, offset, base;
294 if (!mask && !xen_enabled()) {
298 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
299 page = start >> TARGET_PAGE_BITS;
301 WITH_RCU_READ_LOCK_GUARD() {
302 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
303 blocks[i] = qatomic_rcu_read(&ram_list.dirty_memory[i]);
306 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
307 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
308 base = page - offset;
310 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
312 if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) {
313 bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx],
314 offset, next - page);
316 if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) {
317 bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx],
318 offset, next - page);
320 if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) {
321 bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx],
322 offset, next - page);
328 base += DIRTY_MEMORY_BLOCK_SIZE;
332 xen_hvm_modified_memory(start, length);
336 static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap,
341 unsigned long page_number, c;
344 unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
345 unsigned long hpratio = qemu_real_host_page_size() / TARGET_PAGE_SIZE;
346 unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
348 /* start address is aligned at the start of a word? */
349 if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) &&
351 unsigned long **blocks[DIRTY_MEMORY_NUM];
353 unsigned long offset;
355 long nr = BITS_TO_LONGS(pages);
357 idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE;
358 offset = BIT_WORD((start >> TARGET_PAGE_BITS) %
359 DIRTY_MEMORY_BLOCK_SIZE);
361 WITH_RCU_READ_LOCK_GUARD() {
362 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
364 qatomic_rcu_read(&ram_list.dirty_memory[i])->blocks;
367 for (k = 0; k < nr; k++) {
369 unsigned long temp = leul_to_cpu(bitmap[k]);
371 qatomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp);
373 if (global_dirty_tracking) {
375 &blocks[DIRTY_MEMORY_MIGRATION][idx][offset],
378 global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) {
379 total_dirty_pages += ctpopl(temp);
384 qatomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset],
389 if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
396 xen_hvm_modified_memory(start, pages << TARGET_PAGE_BITS);
398 uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE;
400 if (!global_dirty_tracking) {
401 clients &= ~(1 << DIRTY_MEMORY_MIGRATION);
405 * bitmap-traveling is faster than memory-traveling (for addr...)
406 * especially when most of the memory is not dirty.
408 for (i = 0; i < len; i++) {
409 if (bitmap[i] != 0) {
410 c = leul_to_cpu(bitmap[i]);
411 if (unlikely(global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) {
412 total_dirty_pages += ctpopl(c);
417 page_number = (i * HOST_LONG_BITS + j) * hpratio;
418 addr = page_number * TARGET_PAGE_SIZE;
419 ram_addr = start + addr;
420 cpu_physical_memory_set_dirty_range(ram_addr,
421 TARGET_PAGE_SIZE * hpratio, clients);
427 #endif /* not _WIN32 */
429 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
433 DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty
434 (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client);
436 bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap,
440 static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start,
443 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION);
444 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA);
445 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE);
449 /* Called with RCU critical section */
451 uint64_t cpu_physical_memory_sync_dirty_bitmap(RAMBlock *rb,
456 unsigned long word = BIT_WORD((start + rb->offset) >> TARGET_PAGE_BITS);
457 uint64_t num_dirty = 0;
458 unsigned long *dest = rb->bmap;
460 /* start address and length is aligned at the start of a word? */
461 if (((word * BITS_PER_LONG) << TARGET_PAGE_BITS) ==
462 (start + rb->offset) &&
463 !(length & ((BITS_PER_LONG << TARGET_PAGE_BITS) - 1))) {
465 int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS);
466 unsigned long * const *src;
467 unsigned long idx = (word * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE;
468 unsigned long offset = BIT_WORD((word * BITS_PER_LONG) %
469 DIRTY_MEMORY_BLOCK_SIZE);
470 unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
472 src = qatomic_rcu_read(
473 &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks;
475 for (k = page; k < page + nr; k++) {
476 if (src[idx][offset]) {
477 unsigned long bits = qatomic_xchg(&src[idx][offset], 0);
478 unsigned long new_dirty;
479 new_dirty = ~dest[k];
482 num_dirty += ctpopl(new_dirty);
485 if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
491 if (rb->clear_bmap) {
493 * Postpone the dirty bitmap clear to the point before we
494 * really send the pages, also we will split the clear
495 * dirty procedure into smaller chunks.
497 clear_bmap_set(rb, start >> TARGET_PAGE_BITS,
498 length >> TARGET_PAGE_BITS);
500 /* Slow path - still do that in a huge chunk */
501 memory_region_clear_dirty_bitmap(rb->mr, start, length);
504 ram_addr_t offset = rb->offset;
506 for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) {
507 if (cpu_physical_memory_test_and_clear_dirty(
508 start + addr + offset,
510 DIRTY_MEMORY_MIGRATION)) {
511 long k = (start + addr) >> TARGET_PAGE_BITS;
512 if (!test_and_set_bit(k, dest)) {