2 * Dirtyrate implement code
4 * Copyright (c) 2020 HUAWEI TECHNOLOGIES CO.,LTD.
7 * Chuan Zheng <zhengchuan@huawei.com>
9 * This work is licensed under the terms of the GNU GPL, version 2 or later.
10 * See the COPYING file in the top-level directory.
13 #include "qemu/osdep.h"
15 #include "qapi/error.h"
17 #include "exec/ramblock.h"
18 #include "exec/ram_addr.h"
19 #include "qemu/rcu_queue.h"
20 #include "qemu/main-loop.h"
21 #include "qapi/qapi-commands-migration.h"
24 #include "dirtyrate.h"
25 #include "monitor/hmp.h"
26 #include "monitor/monitor.h"
27 #include "qapi/qmp/qdict.h"
28 #include "sysemu/kvm.h"
29 #include "sysemu/runstate.h"
30 #include "exec/memory.h"
33 * total_dirty_pages is procted by BQL and is used
34 * to stat dirty pages during the period of two
35 * memory_global_dirty_log_sync
37 uint64_t total_dirty_pages;
39 typedef struct DirtyPageRecord {
44 static int CalculatingState = DIRTY_RATE_STATUS_UNSTARTED;
45 static struct DirtyRateStat DirtyStat;
46 static DirtyRateMeasureMode dirtyrate_mode =
47 DIRTY_RATE_MEASURE_MODE_PAGE_SAMPLING;
49 static int64_t dirty_stat_wait(int64_t msec, int64_t initial_time)
53 current_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
54 if ((current_time - initial_time) >= msec) {
55 msec = current_time - initial_time;
57 g_usleep((msec + initial_time - current_time) * 1000);
63 static inline void record_dirtypages(DirtyPageRecord *dirty_pages,
64 CPUState *cpu, bool start)
67 dirty_pages[cpu->cpu_index].start_pages = cpu->dirty_pages;
69 dirty_pages[cpu->cpu_index].end_pages = cpu->dirty_pages;
73 static int64_t do_calculate_dirtyrate(DirtyPageRecord dirty_pages,
76 uint64_t memory_size_MB;
77 uint64_t increased_dirty_pages =
78 dirty_pages.end_pages - dirty_pages.start_pages;
80 memory_size_MB = (increased_dirty_pages * TARGET_PAGE_SIZE) >> 20;
82 return memory_size_MB * 1000 / calc_time_ms;
85 void global_dirty_log_change(unsigned int flag, bool start)
87 qemu_mutex_lock_iothread();
89 memory_global_dirty_log_start(flag);
91 memory_global_dirty_log_stop(flag);
93 qemu_mutex_unlock_iothread();
97 * global_dirty_log_sync
98 * 1. sync dirty log from kvm
99 * 2. stop dirty tracking if needed.
101 static void global_dirty_log_sync(unsigned int flag, bool one_shot)
103 qemu_mutex_lock_iothread();
104 memory_global_dirty_log_sync();
106 memory_global_dirty_log_stop(flag);
108 qemu_mutex_unlock_iothread();
111 static DirtyPageRecord *vcpu_dirty_stat_alloc(VcpuStat *stat)
114 DirtyPageRecord *records;
122 stat->rates = g_new0(DirtyRateVcpu, nvcpu);
124 records = g_new0(DirtyPageRecord, nvcpu);
129 static void vcpu_dirty_stat_collect(VcpuStat *stat,
130 DirtyPageRecord *records,
136 record_dirtypages(records, cpu, start);
140 int64_t vcpu_calculate_dirtyrate(int64_t calc_time_ms,
145 DirtyPageRecord *records;
146 int64_t init_time_ms;
153 init_time_ms = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
156 gen_id = cpu_list_generation_id_get();
157 records = vcpu_dirty_stat_alloc(stat);
158 vcpu_dirty_stat_collect(stat, records, true);
161 duration = dirty_stat_wait(calc_time_ms, init_time_ms);
163 global_dirty_log_sync(flag, one_shot);
166 if (gen_id != cpu_list_generation_id_get()) {
172 vcpu_dirty_stat_collect(stat, records, false);
175 for (i = 0; i < stat->nvcpu; i++) {
176 dirtyrate = do_calculate_dirtyrate(records[i], duration);
178 stat->rates[i].id = i;
179 stat->rates[i].dirty_rate = dirtyrate;
181 trace_dirtyrate_do_calculate_vcpu(i, dirtyrate);
189 static bool is_sample_period_valid(int64_t sec)
191 if (sec < MIN_FETCH_DIRTYRATE_TIME_SEC ||
192 sec > MAX_FETCH_DIRTYRATE_TIME_SEC) {
199 static bool is_sample_pages_valid(int64_t pages)
201 return pages >= MIN_SAMPLE_PAGE_COUNT &&
202 pages <= MAX_SAMPLE_PAGE_COUNT;
205 static int dirtyrate_set_state(int *state, int old_state, int new_state)
207 assert(new_state < DIRTY_RATE_STATUS__MAX);
208 trace_dirtyrate_set_state(DirtyRateStatus_str(new_state));
209 if (qatomic_cmpxchg(state, old_state, new_state) == old_state) {
216 static struct DirtyRateInfo *query_dirty_rate_info(void)
219 int64_t dirty_rate = DirtyStat.dirty_rate;
220 struct DirtyRateInfo *info = g_new0(DirtyRateInfo, 1);
221 DirtyRateVcpuList *head = NULL, **tail = &head;
223 info->status = CalculatingState;
224 info->start_time = DirtyStat.start_time;
225 info->calc_time = DirtyStat.calc_time;
226 info->sample_pages = DirtyStat.sample_pages;
227 info->mode = dirtyrate_mode;
229 if (qatomic_read(&CalculatingState) == DIRTY_RATE_STATUS_MEASURED) {
230 info->has_dirty_rate = true;
231 info->dirty_rate = dirty_rate;
233 if (dirtyrate_mode == DIRTY_RATE_MEASURE_MODE_DIRTY_RING) {
235 * set sample_pages with 0 to indicate page sampling
238 info->sample_pages = 0;
239 info->has_vcpu_dirty_rate = true;
240 for (i = 0; i < DirtyStat.dirty_ring.nvcpu; i++) {
241 DirtyRateVcpu *rate = g_new0(DirtyRateVcpu, 1);
242 rate->id = DirtyStat.dirty_ring.rates[i].id;
243 rate->dirty_rate = DirtyStat.dirty_ring.rates[i].dirty_rate;
244 QAPI_LIST_APPEND(tail, rate);
246 info->vcpu_dirty_rate = head;
249 if (dirtyrate_mode == DIRTY_RATE_MEASURE_MODE_DIRTY_BITMAP) {
250 info->sample_pages = 0;
254 trace_query_dirty_rate_info(DirtyRateStatus_str(CalculatingState));
259 static void init_dirtyrate_stat(int64_t start_time,
260 struct DirtyRateConfig config)
262 DirtyStat.dirty_rate = -1;
263 DirtyStat.start_time = start_time;
264 DirtyStat.calc_time = config.sample_period_seconds;
265 DirtyStat.sample_pages = config.sample_pages_per_gigabytes;
267 switch (config.mode) {
268 case DIRTY_RATE_MEASURE_MODE_PAGE_SAMPLING:
269 DirtyStat.page_sampling.total_dirty_samples = 0;
270 DirtyStat.page_sampling.total_sample_count = 0;
271 DirtyStat.page_sampling.total_block_mem_MB = 0;
273 case DIRTY_RATE_MEASURE_MODE_DIRTY_RING:
274 DirtyStat.dirty_ring.nvcpu = -1;
275 DirtyStat.dirty_ring.rates = NULL;
282 static void cleanup_dirtyrate_stat(struct DirtyRateConfig config)
284 /* last calc-dirty-rate qmp use dirty ring mode */
285 if (dirtyrate_mode == DIRTY_RATE_MEASURE_MODE_DIRTY_RING) {
286 free(DirtyStat.dirty_ring.rates);
287 DirtyStat.dirty_ring.rates = NULL;
291 static void update_dirtyrate_stat(struct RamblockDirtyInfo *info)
293 DirtyStat.page_sampling.total_dirty_samples += info->sample_dirty_count;
294 DirtyStat.page_sampling.total_sample_count += info->sample_pages_count;
295 /* size of total pages in MB */
296 DirtyStat.page_sampling.total_block_mem_MB += (info->ramblock_pages *
297 TARGET_PAGE_SIZE) >> 20;
300 static void update_dirtyrate(uint64_t msec)
303 uint64_t total_dirty_samples = DirtyStat.page_sampling.total_dirty_samples;
304 uint64_t total_sample_count = DirtyStat.page_sampling.total_sample_count;
305 uint64_t total_block_mem_MB = DirtyStat.page_sampling.total_block_mem_MB;
307 dirtyrate = total_dirty_samples * total_block_mem_MB *
308 1000 / (total_sample_count * msec);
310 DirtyStat.dirty_rate = dirtyrate;
314 * get hash result for the sampled memory with length of TARGET_PAGE_SIZE
315 * in ramblock, which starts from ramblock base address.
317 static uint32_t get_ramblock_vfn_hash(struct RamblockDirtyInfo *info,
322 crc = crc32(0, (info->ramblock_addr +
323 vfn * TARGET_PAGE_SIZE), TARGET_PAGE_SIZE);
325 trace_get_ramblock_vfn_hash(info->idstr, vfn, crc);
329 static bool save_ramblock_hash(struct RamblockDirtyInfo *info)
331 unsigned int sample_pages_count;
335 sample_pages_count = info->sample_pages_count;
337 /* ramblock size less than one page, return success to skip this ramblock */
338 if (unlikely(info->ramblock_pages == 0 || sample_pages_count == 0)) {
342 info->hash_result = g_try_malloc0_n(sample_pages_count,
344 if (!info->hash_result) {
348 info->sample_page_vfn = g_try_malloc0_n(sample_pages_count,
350 if (!info->sample_page_vfn) {
351 g_free(info->hash_result);
356 for (i = 0; i < sample_pages_count; i++) {
357 info->sample_page_vfn[i] = g_rand_int_range(rand, 0,
358 info->ramblock_pages - 1);
359 info->hash_result[i] = get_ramblock_vfn_hash(info,
360 info->sample_page_vfn[i]);
367 static void get_ramblock_dirty_info(RAMBlock *block,
368 struct RamblockDirtyInfo *info,
369 struct DirtyRateConfig *config)
371 uint64_t sample_pages_per_gigabytes = config->sample_pages_per_gigabytes;
373 /* Right shift 30 bits to calc ramblock size in GB */
374 info->sample_pages_count = (qemu_ram_get_used_length(block) *
375 sample_pages_per_gigabytes) >> 30;
376 /* Right shift TARGET_PAGE_BITS to calc page count */
377 info->ramblock_pages = qemu_ram_get_used_length(block) >>
379 info->ramblock_addr = qemu_ram_get_host_addr(block);
380 strcpy(info->idstr, qemu_ram_get_idstr(block));
383 static void free_ramblock_dirty_info(struct RamblockDirtyInfo *infos, int count)
391 for (i = 0; i < count; i++) {
392 g_free(infos[i].sample_page_vfn);
393 g_free(infos[i].hash_result);
398 static bool skip_sample_ramblock(RAMBlock *block)
401 * Sample only blocks larger than MIN_RAMBLOCK_SIZE.
403 if (qemu_ram_get_used_length(block) < (MIN_RAMBLOCK_SIZE << 10)) {
404 trace_skip_sample_ramblock(block->idstr,
405 qemu_ram_get_used_length(block));
412 static bool record_ramblock_hash_info(struct RamblockDirtyInfo **block_dinfo,
413 struct DirtyRateConfig config,
416 struct RamblockDirtyInfo *info = NULL;
417 struct RamblockDirtyInfo *dinfo = NULL;
418 RAMBlock *block = NULL;
423 RAMBLOCK_FOREACH_MIGRATABLE(block) {
424 if (skip_sample_ramblock(block)) {
430 dinfo = g_try_malloc0_n(total_count, sizeof(struct RamblockDirtyInfo));
435 RAMBLOCK_FOREACH_MIGRATABLE(block) {
436 if (skip_sample_ramblock(block)) {
439 if (index >= total_count) {
442 info = &dinfo[index];
443 get_ramblock_dirty_info(block, info, &config);
444 if (!save_ramblock_hash(info)) {
452 *block_count = index;
453 *block_dinfo = dinfo;
457 static void calc_page_dirty_rate(struct RamblockDirtyInfo *info)
462 for (i = 0; i < info->sample_pages_count; i++) {
463 crc = get_ramblock_vfn_hash(info, info->sample_page_vfn[i]);
464 if (crc != info->hash_result[i]) {
465 trace_calc_page_dirty_rate(info->idstr, crc, info->hash_result[i]);
466 info->sample_dirty_count++;
471 static struct RamblockDirtyInfo *
472 find_block_matched(RAMBlock *block, int count,
473 struct RamblockDirtyInfo *infos)
476 struct RamblockDirtyInfo *matched;
478 for (i = 0; i < count; i++) {
479 if (!strcmp(infos[i].idstr, qemu_ram_get_idstr(block))) {
488 if (infos[i].ramblock_addr != qemu_ram_get_host_addr(block) ||
489 infos[i].ramblock_pages !=
490 (qemu_ram_get_used_length(block) >> TARGET_PAGE_BITS)) {
491 trace_find_page_matched(block->idstr);
500 static bool compare_page_hash_info(struct RamblockDirtyInfo *info,
503 struct RamblockDirtyInfo *block_dinfo = NULL;
504 RAMBlock *block = NULL;
506 RAMBLOCK_FOREACH_MIGRATABLE(block) {
507 if (skip_sample_ramblock(block)) {
510 block_dinfo = find_block_matched(block, block_count, info);
511 if (block_dinfo == NULL) {
514 calc_page_dirty_rate(block_dinfo);
515 update_dirtyrate_stat(block_dinfo);
518 if (DirtyStat.page_sampling.total_sample_count == 0) {
525 static inline void record_dirtypages_bitmap(DirtyPageRecord *dirty_pages,
529 dirty_pages->start_pages = total_dirty_pages;
531 dirty_pages->end_pages = total_dirty_pages;
535 static inline void dirtyrate_manual_reset_protect(void)
537 RAMBlock *block = NULL;
539 WITH_RCU_READ_LOCK_GUARD() {
540 RAMBLOCK_FOREACH_MIGRATABLE(block) {
541 memory_region_clear_dirty_bitmap(block->mr, 0,
547 static void calculate_dirtyrate_dirty_bitmap(struct DirtyRateConfig config)
551 DirtyPageRecord dirty_pages;
553 qemu_mutex_lock_iothread();
554 memory_global_dirty_log_start(GLOBAL_DIRTY_DIRTY_RATE);
557 * 1'round of log sync may return all 1 bits with
558 * KVM_DIRTY_LOG_INITIALLY_SET enable
559 * skip it unconditionally and start dirty tracking
560 * from 2'round of log sync
562 memory_global_dirty_log_sync();
565 * reset page protect manually and unconditionally.
566 * this make sure kvm dirty log be cleared if
567 * KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE cap is enabled.
569 dirtyrate_manual_reset_protect();
570 qemu_mutex_unlock_iothread();
572 record_dirtypages_bitmap(&dirty_pages, true);
574 start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
575 DirtyStat.start_time = start_time / 1000;
577 msec = config.sample_period_seconds * 1000;
578 msec = dirty_stat_wait(msec, start_time);
579 DirtyStat.calc_time = msec / 1000;
583 * 1. fetch dirty bitmap from kvm
584 * 2. stop dirty tracking
586 global_dirty_log_sync(GLOBAL_DIRTY_DIRTY_RATE, true);
588 record_dirtypages_bitmap(&dirty_pages, false);
590 DirtyStat.dirty_rate = do_calculate_dirtyrate(dirty_pages, msec);
593 static void calculate_dirtyrate_dirty_ring(struct DirtyRateConfig config)
596 uint64_t dirtyrate = 0;
597 uint64_t dirtyrate_sum = 0;
601 global_dirty_log_change(GLOBAL_DIRTY_DIRTY_RATE, true);
603 DirtyStat.start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) / 1000;
605 /* calculate vcpu dirtyrate */
606 duration = vcpu_calculate_dirtyrate(config.sample_period_seconds * 1000,
607 &DirtyStat.dirty_ring,
608 GLOBAL_DIRTY_DIRTY_RATE,
611 DirtyStat.calc_time = duration / 1000;
613 /* calculate vm dirtyrate */
614 for (i = 0; i < DirtyStat.dirty_ring.nvcpu; i++) {
615 dirtyrate = DirtyStat.dirty_ring.rates[i].dirty_rate;
616 DirtyStat.dirty_ring.rates[i].dirty_rate = dirtyrate;
617 dirtyrate_sum += dirtyrate;
620 DirtyStat.dirty_rate = dirtyrate_sum;
623 static void calculate_dirtyrate_sample_vm(struct DirtyRateConfig config)
625 struct RamblockDirtyInfo *block_dinfo = NULL;
628 int64_t initial_time;
631 initial_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
632 if (!record_ramblock_hash_info(&block_dinfo, config, &block_count)) {
637 msec = config.sample_period_seconds * 1000;
638 msec = dirty_stat_wait(msec, initial_time);
639 DirtyStat.start_time = initial_time / 1000;
640 DirtyStat.calc_time = msec / 1000;
643 if (!compare_page_hash_info(block_dinfo, block_count)) {
647 update_dirtyrate(msec);
651 free_ramblock_dirty_info(block_dinfo, block_count);
654 static void calculate_dirtyrate(struct DirtyRateConfig config)
656 if (config.mode == DIRTY_RATE_MEASURE_MODE_DIRTY_BITMAP) {
657 calculate_dirtyrate_dirty_bitmap(config);
658 } else if (config.mode == DIRTY_RATE_MEASURE_MODE_DIRTY_RING) {
659 calculate_dirtyrate_dirty_ring(config);
661 calculate_dirtyrate_sample_vm(config);
664 trace_dirtyrate_calculate(DirtyStat.dirty_rate);
667 void *get_dirtyrate_thread(void *arg)
669 struct DirtyRateConfig config = *(struct DirtyRateConfig *)arg;
671 rcu_register_thread();
673 ret = dirtyrate_set_state(&CalculatingState, DIRTY_RATE_STATUS_UNSTARTED,
674 DIRTY_RATE_STATUS_MEASURING);
676 error_report("change dirtyrate state failed.");
680 calculate_dirtyrate(config);
682 ret = dirtyrate_set_state(&CalculatingState, DIRTY_RATE_STATUS_MEASURING,
683 DIRTY_RATE_STATUS_MEASURED);
685 error_report("change dirtyrate state failed.");
688 rcu_unregister_thread();
692 void qmp_calc_dirty_rate(int64_t calc_time,
693 bool has_sample_pages,
694 int64_t sample_pages,
696 DirtyRateMeasureMode mode,
699 static struct DirtyRateConfig config;
705 * If the dirty rate is already being measured, don't attempt to start.
707 if (qatomic_read(&CalculatingState) == DIRTY_RATE_STATUS_MEASURING) {
708 error_setg(errp, "the dirty rate is already being measured.");
712 if (!is_sample_period_valid(calc_time)) {
713 error_setg(errp, "calc-time is out of range[%d, %d].",
714 MIN_FETCH_DIRTYRATE_TIME_SEC,
715 MAX_FETCH_DIRTYRATE_TIME_SEC);
720 mode = DIRTY_RATE_MEASURE_MODE_PAGE_SAMPLING;
723 if (has_sample_pages && mode == DIRTY_RATE_MEASURE_MODE_DIRTY_RING) {
724 error_setg(errp, "either sample-pages or dirty-ring can be specified.");
728 if (has_sample_pages) {
729 if (!is_sample_pages_valid(sample_pages)) {
730 error_setg(errp, "sample-pages is out of range[%d, %d].",
731 MIN_SAMPLE_PAGE_COUNT,
732 MAX_SAMPLE_PAGE_COUNT);
736 sample_pages = DIRTYRATE_DEFAULT_SAMPLE_PAGES;
740 * dirty ring mode only works when kvm dirty ring is enabled.
741 * on the contrary, dirty bitmap mode is not.
743 if (((mode == DIRTY_RATE_MEASURE_MODE_DIRTY_RING) &&
744 !kvm_dirty_ring_enabled()) ||
745 ((mode == DIRTY_RATE_MEASURE_MODE_DIRTY_BITMAP) &&
746 kvm_dirty_ring_enabled())) {
747 error_setg(errp, "mode %s is not enabled, use other method instead.",
748 DirtyRateMeasureMode_str(mode));
753 * Init calculation state as unstarted.
755 ret = dirtyrate_set_state(&CalculatingState, CalculatingState,
756 DIRTY_RATE_STATUS_UNSTARTED);
758 error_setg(errp, "init dirty rate calculation state failed.");
762 config.sample_period_seconds = calc_time;
763 config.sample_pages_per_gigabytes = sample_pages;
766 cleanup_dirtyrate_stat(config);
769 * update dirty rate mode so that we can figure out what mode has
770 * been used in last calculation
772 dirtyrate_mode = mode;
774 start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) / 1000;
775 init_dirtyrate_stat(start_time, config);
777 qemu_thread_create(&thread, "get_dirtyrate", get_dirtyrate_thread,
778 (void *)&config, QEMU_THREAD_DETACHED);
781 struct DirtyRateInfo *qmp_query_dirty_rate(Error **errp)
783 return query_dirty_rate_info();
786 void hmp_info_dirty_rate(Monitor *mon, const QDict *qdict)
788 DirtyRateInfo *info = query_dirty_rate_info();
790 monitor_printf(mon, "Status: %s\n",
791 DirtyRateStatus_str(info->status));
792 monitor_printf(mon, "Start Time: %"PRIi64" (ms)\n",
794 monitor_printf(mon, "Sample Pages: %"PRIu64" (per GB)\n",
796 monitor_printf(mon, "Period: %"PRIi64" (sec)\n",
798 monitor_printf(mon, "Mode: %s\n",
799 DirtyRateMeasureMode_str(info->mode));
800 monitor_printf(mon, "Dirty rate: ");
801 if (info->has_dirty_rate) {
802 monitor_printf(mon, "%"PRIi64" (MB/s)\n", info->dirty_rate);
803 if (info->has_vcpu_dirty_rate) {
804 DirtyRateVcpuList *rate, *head = info->vcpu_dirty_rate;
805 for (rate = head; rate != NULL; rate = rate->next) {
806 monitor_printf(mon, "vcpu[%"PRIi64"], Dirty rate: %"PRIi64
807 " (MB/s)\n", rate->value->id,
808 rate->value->dirty_rate);
812 monitor_printf(mon, "(not ready)\n");
815 qapi_free_DirtyRateVcpuList(info->vcpu_dirty_rate);
819 void hmp_calc_dirty_rate(Monitor *mon, const QDict *qdict)
821 int64_t sec = qdict_get_try_int(qdict, "second", 0);
822 int64_t sample_pages = qdict_get_try_int(qdict, "sample_pages_per_GB", -1);
823 bool has_sample_pages = (sample_pages != -1);
824 bool dirty_ring = qdict_get_try_bool(qdict, "dirty_ring", false);
825 bool dirty_bitmap = qdict_get_try_bool(qdict, "dirty_bitmap", false);
826 DirtyRateMeasureMode mode = DIRTY_RATE_MEASURE_MODE_PAGE_SAMPLING;
830 monitor_printf(mon, "Incorrect period length specified!\n");
834 if (dirty_ring && dirty_bitmap) {
835 monitor_printf(mon, "Either dirty ring or dirty bitmap "
836 "can be specified!\n");
841 mode = DIRTY_RATE_MEASURE_MODE_DIRTY_BITMAP;
842 } else if (dirty_ring) {
843 mode = DIRTY_RATE_MEASURE_MODE_DIRTY_RING;
846 qmp_calc_dirty_rate(sec, has_sample_pages, sample_pages, true,
849 hmp_handle_error(mon, err);
853 monitor_printf(mon, "Starting dirty rate measurement with period %"PRIi64
855 monitor_printf(mon, "[Please use 'info dirty_rate' to check results]\n");