4 * Copyright (C) 2013 Proxmox Server Solutions
5 * Copyright (c) 2019 Virtuozzo International GmbH.
8 * Dietmar Maurer (dietmar@proxmox.com)
9 * Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
11 * This work is licensed under the terms of the GNU GPL, version 2 or later.
12 * See the COPYING file in the top-level directory.
15 #include "qemu/osdep.h"
18 #include "qapi/error.h"
19 #include "block/block-copy.h"
20 #include "block/reqlist.h"
21 #include "sysemu/block-backend.h"
22 #include "qemu/units.h"
23 #include "qemu/coroutine.h"
24 #include "block/aio_task.h"
25 #include "qemu/error-report.h"
26 #include "qemu/memalign.h"
28 #define BLOCK_COPY_MAX_COPY_RANGE (16 * MiB)
29 #define BLOCK_COPY_MAX_BUFFER (1 * MiB)
30 #define BLOCK_COPY_MAX_MEM (128 * MiB)
31 #define BLOCK_COPY_MAX_WORKERS 64
32 #define BLOCK_COPY_SLICE_TIME 100000000ULL /* ns */
33 #define BLOCK_COPY_CLUSTER_SIZE_DEFAULT (1 << 16)
36 COPY_READ_WRITE_CLUSTER,
43 static coroutine_fn int block_copy_task_entry(AioTask *task);
45 typedef struct BlockCopyCallState {
46 /* Fields initialized in block_copy_async() and never changed. */
52 bool ignore_ratelimit;
53 BlockCopyAsyncCallbackFunc cb;
55 /* Coroutine where async block-copy is running */
58 /* Fields whose state changes throughout the execution */
59 bool finished; /* atomic */
60 QemuCoSleep sleep; /* TODO: protect API with a lock */
61 bool cancelled; /* atomic */
62 /* To reference all call states from BlockCopyState */
63 QLIST_ENTRY(BlockCopyCallState) list;
66 * Fields that report information about return values and erros.
67 * Protected by lock in BlockCopyState.
71 * @ret is set concurrently by tasks under mutex. Only set once by first
72 * failed task (and untouched if no task failed).
73 * After finishing (call_state->finished is true), it is not modified
74 * anymore and may be safely read without mutex.
79 typedef struct BlockCopyTask {
83 * Fields initialized in block_copy_task_create()
87 BlockCopyCallState *call_state;
89 * @method can also be set again in the while loop of
90 * block_copy_dirty_clusters(), but it is never accessed concurrently
91 * because the only other function that reads it is
92 * block_copy_task_entry() and it is invoked afterwards in the same
95 BlockCopyMethod method;
98 * Generally, req is protected by lock in BlockCopyState, Still req.offset
99 * is only set on task creation, so may be read concurrently after creation.
100 * req.bytes is changed at most once, and need only protecting the case of
101 * parallel read while updating @bytes value in block_copy_task_shrink().
106 static int64_t task_end(BlockCopyTask *task)
108 return task->req.offset + task->req.bytes;
111 typedef struct BlockCopyState {
113 * BdrvChild objects are not owned or managed by block-copy. They are
114 * provided by block-copy user and user is responsible for appropriate
115 * permissions on these children.
121 * Fields initialized in block_copy_state_new()
124 int64_t cluster_size;
125 int64_t max_transfer;
127 BdrvRequestFlags write_flags;
130 * Fields whose state changes throughout the execution
134 int64_t in_flight_bytes;
135 BlockCopyMethod method;
137 QLIST_HEAD(, BlockCopyCallState) calls;
141 * Used by sync=top jobs, which first scan the source node for unallocated
142 * areas and clear them in the copy_bitmap. During this process, the bitmap
143 * is thus not fully initialized: It may still have bits set for areas that
144 * are unallocated and should actually not be copied.
146 * This is indicated by skip_unallocated.
148 * In this case, block_copy() will query the source’s allocation status,
149 * skip unallocated regions, clear them in the copy_bitmap, and invoke
150 * block_copy_reset_unallocated() every time it does.
152 bool skip_unallocated; /* atomic */
153 /* State fields that use a thread-safe API */
154 BdrvDirtyBitmap *copy_bitmap;
155 ProgressMeter *progress;
157 RateLimit rate_limit;
160 /* Called with lock held */
161 static int64_t block_copy_chunk_size(BlockCopyState *s)
164 case COPY_READ_WRITE_CLUSTER:
165 return s->cluster_size;
166 case COPY_READ_WRITE:
167 case COPY_RANGE_SMALL:
168 return MIN(MAX(s->cluster_size, BLOCK_COPY_MAX_BUFFER),
170 case COPY_RANGE_FULL:
171 return MIN(MAX(s->cluster_size, BLOCK_COPY_MAX_COPY_RANGE),
174 /* Cannot have COPY_WRITE_ZEROES here. */
180 * Search for the first dirty area in offset/bytes range and create task at
181 * the beginning of it.
183 static coroutine_fn BlockCopyTask *
184 block_copy_task_create(BlockCopyState *s, BlockCopyCallState *call_state,
185 int64_t offset, int64_t bytes)
190 QEMU_LOCK_GUARD(&s->lock);
191 max_chunk = MIN_NON_ZERO(block_copy_chunk_size(s), call_state->max_chunk);
192 if (!bdrv_dirty_bitmap_next_dirty_area(s->copy_bitmap,
193 offset, offset + bytes,
194 max_chunk, &offset, &bytes))
199 assert(QEMU_IS_ALIGNED(offset, s->cluster_size));
200 bytes = QEMU_ALIGN_UP(bytes, s->cluster_size);
202 /* region is dirty, so no existent tasks possible in it */
203 assert(!reqlist_find_conflict(&s->reqs, offset, bytes));
205 bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, bytes);
206 s->in_flight_bytes += bytes;
208 task = g_new(BlockCopyTask, 1);
209 *task = (BlockCopyTask) {
210 .task.func = block_copy_task_entry,
212 .call_state = call_state,
215 reqlist_init_req(&s->reqs, &task->req, offset, bytes);
221 * block_copy_task_shrink
223 * Drop the tail of the task to be handled later. Set dirty bits back and
224 * wake up all tasks waiting for us (may be some of them are not intersecting
227 static void coroutine_fn block_copy_task_shrink(BlockCopyTask *task,
230 QEMU_LOCK_GUARD(&task->s->lock);
231 if (new_bytes == task->req.bytes) {
235 assert(new_bytes > 0 && new_bytes < task->req.bytes);
237 task->s->in_flight_bytes -= task->req.bytes - new_bytes;
238 bdrv_set_dirty_bitmap(task->s->copy_bitmap,
239 task->req.offset + new_bytes,
240 task->req.bytes - new_bytes);
242 reqlist_shrink_req(&task->req, new_bytes);
245 static void coroutine_fn block_copy_task_end(BlockCopyTask *task, int ret)
247 QEMU_LOCK_GUARD(&task->s->lock);
248 task->s->in_flight_bytes -= task->req.bytes;
250 bdrv_set_dirty_bitmap(task->s->copy_bitmap, task->req.offset,
253 if (task->s->progress) {
254 progress_set_remaining(task->s->progress,
255 bdrv_get_dirty_count(task->s->copy_bitmap) +
256 task->s->in_flight_bytes);
258 reqlist_remove_req(&task->req);
261 void block_copy_state_free(BlockCopyState *s)
267 ratelimit_destroy(&s->rate_limit);
268 bdrv_release_dirty_bitmap(s->copy_bitmap);
269 shres_destroy(s->mem);
273 static uint32_t block_copy_max_transfer(BdrvChild *source, BdrvChild *target)
275 return MIN_NON_ZERO(INT_MAX,
276 MIN_NON_ZERO(source->bs->bl.max_transfer,
277 target->bs->bl.max_transfer));
280 void block_copy_set_copy_opts(BlockCopyState *s, bool use_copy_range,
283 /* Keep BDRV_REQ_SERIALISING set (or not set) in block_copy_state_new() */
284 s->write_flags = (s->write_flags & BDRV_REQ_SERIALISING) |
285 (compress ? BDRV_REQ_WRITE_COMPRESSED : 0);
287 if (s->max_transfer < s->cluster_size) {
289 * copy_range does not respect max_transfer. We don't want to bother
290 * with requests smaller than block-copy cluster size, so fallback to
291 * buffered copying (read and write respect max_transfer on their
294 s->method = COPY_READ_WRITE_CLUSTER;
295 } else if (compress) {
296 /* Compression supports only cluster-size writes and no copy-range. */
297 s->method = COPY_READ_WRITE_CLUSTER;
300 * If copy range enabled, start with COPY_RANGE_SMALL, until first
301 * successful copy_range (look at block_copy_do_copy).
303 s->method = use_copy_range ? COPY_RANGE_SMALL : COPY_READ_WRITE;
307 static int64_t block_copy_calculate_cluster_size(BlockDriverState *target,
312 bool target_does_cow = bdrv_backing_chain_next(target);
315 * If there is no backing file on the target, we cannot rely on COW if our
316 * backup cluster size is smaller than the target cluster size. Even for
317 * targets with a backing file, try to avoid COW if possible.
319 ret = bdrv_get_info(target, &bdi);
320 if (ret == -ENOTSUP && !target_does_cow) {
321 /* Cluster size is not defined */
322 warn_report("The target block device doesn't provide "
323 "information about the block size and it doesn't have a "
324 "backing file. The default block size of %u bytes is "
325 "used. If the actual block size of the target exceeds "
326 "this default, the backup may be unusable",
327 BLOCK_COPY_CLUSTER_SIZE_DEFAULT);
328 return BLOCK_COPY_CLUSTER_SIZE_DEFAULT;
329 } else if (ret < 0 && !target_does_cow) {
330 error_setg_errno(errp, -ret,
331 "Couldn't determine the cluster size of the target image, "
332 "which has no backing file");
333 error_append_hint(errp,
334 "Aborting, since this may create an unusable destination image\n");
336 } else if (ret < 0 && target_does_cow) {
337 /* Not fatal; just trudge on ahead. */
338 return BLOCK_COPY_CLUSTER_SIZE_DEFAULT;
341 return MAX(BLOCK_COPY_CLUSTER_SIZE_DEFAULT, bdi.cluster_size);
344 BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target,
345 const BdrvDirtyBitmap *bitmap,
350 int64_t cluster_size;
351 BdrvDirtyBitmap *copy_bitmap;
354 cluster_size = block_copy_calculate_cluster_size(target->bs, errp);
355 if (cluster_size < 0) {
359 copy_bitmap = bdrv_create_dirty_bitmap(source->bs, cluster_size, NULL,
364 bdrv_disable_dirty_bitmap(copy_bitmap);
366 if (!bdrv_merge_dirty_bitmap(copy_bitmap, bitmap, NULL, errp)) {
367 error_prepend(errp, "Failed to merge bitmap '%s' to internal "
368 "copy-bitmap: ", bdrv_dirty_bitmap_name(bitmap));
369 bdrv_release_dirty_bitmap(copy_bitmap);
373 bdrv_set_dirty_bitmap(copy_bitmap, 0,
374 bdrv_dirty_bitmap_size(copy_bitmap));
378 * If source is in backing chain of target assume that target is going to be
379 * used for "image fleecing", i.e. it should represent a kind of snapshot of
380 * source at backup-start point in time. And target is going to be read by
381 * somebody (for example, used as NBD export) during backup job.
383 * In this case, we need to add BDRV_REQ_SERIALISING write flag to avoid
384 * intersection of backup writes and third party reads from target,
385 * otherwise reading from target we may occasionally read already updated by
388 * For more information see commit f8d59dfb40bb and test
389 * tests/qemu-iotests/222
391 is_fleecing = bdrv_chain_contains(target->bs, source->bs);
393 s = g_new(BlockCopyState, 1);
394 *s = (BlockCopyState) {
397 .copy_bitmap = copy_bitmap,
398 .cluster_size = cluster_size,
399 .len = bdrv_dirty_bitmap_size(copy_bitmap),
400 .write_flags = (is_fleecing ? BDRV_REQ_SERIALISING : 0),
401 .mem = shres_create(BLOCK_COPY_MAX_MEM),
402 .max_transfer = QEMU_ALIGN_DOWN(
403 block_copy_max_transfer(source, target),
407 block_copy_set_copy_opts(s, false, false);
409 ratelimit_init(&s->rate_limit);
410 qemu_co_mutex_init(&s->lock);
411 QLIST_INIT(&s->reqs);
412 QLIST_INIT(&s->calls);
417 /* Only set before running the job, no need for locking. */
418 void block_copy_set_progress_meter(BlockCopyState *s, ProgressMeter *pm)
424 * Takes ownership of @task
426 * If pool is NULL directly run the task, otherwise schedule it into the pool.
428 * Returns: task.func return code if pool is NULL
429 * otherwise -ECANCELED if pool status is bad
430 * otherwise 0 (successfully scheduled)
432 static coroutine_fn int block_copy_task_run(AioTaskPool *pool,
436 int ret = task->task.func(&task->task);
442 aio_task_pool_wait_slot(pool);
443 if (aio_task_pool_status(pool) < 0) {
444 co_put_to_shres(task->s->mem, task->req.bytes);
445 block_copy_task_end(task, -ECANCELED);
450 aio_task_pool_start_task(pool, &task->task);
458 * Do copy of cluster-aligned chunk. Requested region is allowed to exceed
459 * s->len only to cover last cluster when s->len is not aligned to clusters.
461 * No sync here: nor bitmap neighter intersecting requests handling, only copy.
463 * @method is an in-out argument, so that copy_range can be either extended to
464 * a full-size buffer or disabled if the copy_range attempt fails. The output
465 * value of @method should be used for subsequent tasks.
466 * Returns 0 on success.
468 static int coroutine_fn block_copy_do_copy(BlockCopyState *s,
469 int64_t offset, int64_t bytes,
470 BlockCopyMethod *method,
474 int64_t nbytes = MIN(offset + bytes, s->len) - offset;
475 void *bounce_buffer = NULL;
477 assert(offset >= 0 && bytes > 0 && INT64_MAX - offset >= bytes);
478 assert(QEMU_IS_ALIGNED(offset, s->cluster_size));
479 assert(QEMU_IS_ALIGNED(bytes, s->cluster_size));
480 assert(offset < s->len);
481 assert(offset + bytes <= s->len ||
482 offset + bytes == QEMU_ALIGN_UP(s->len, s->cluster_size));
483 assert(nbytes < INT_MAX);
486 case COPY_WRITE_ZEROES:
487 ret = bdrv_co_pwrite_zeroes(s->target, offset, nbytes, s->write_flags &
488 ~BDRV_REQ_WRITE_COMPRESSED);
490 trace_block_copy_write_zeroes_fail(s, offset, ret);
491 *error_is_read = false;
495 case COPY_RANGE_SMALL:
496 case COPY_RANGE_FULL:
497 ret = bdrv_co_copy_range(s->source, offset, s->target, offset, nbytes,
500 /* Successful copy-range, increase chunk size. */
501 *method = COPY_RANGE_FULL;
505 trace_block_copy_copy_range_fail(s, offset, ret);
506 *method = COPY_READ_WRITE;
507 /* Fall through to read+write with allocated buffer */
509 case COPY_READ_WRITE_CLUSTER:
510 case COPY_READ_WRITE:
512 * In case of failed copy_range request above, we may proceed with
513 * buffered request larger than BLOCK_COPY_MAX_BUFFER.
514 * Still, further requests will be properly limited, so don't care too
515 * much. Moreover the most likely case (copy_range is unsupported for
516 * the configuration, so the very first copy_range request fails)
517 * is handled by setting large copy_size only after first successful
521 bounce_buffer = qemu_blockalign(s->source->bs, nbytes);
523 ret = bdrv_co_pread(s->source, offset, nbytes, bounce_buffer, 0);
525 trace_block_copy_read_fail(s, offset, ret);
526 *error_is_read = true;
530 ret = bdrv_co_pwrite(s->target, offset, nbytes, bounce_buffer,
533 trace_block_copy_write_fail(s, offset, ret);
534 *error_is_read = false;
539 qemu_vfree(bounce_buffer);
549 static coroutine_fn int block_copy_task_entry(AioTask *task)
551 BlockCopyTask *t = container_of(task, BlockCopyTask, task);
552 BlockCopyState *s = t->s;
553 bool error_is_read = false;
554 BlockCopyMethod method = t->method;
557 ret = block_copy_do_copy(s, t->req.offset, t->req.bytes, &method,
560 WITH_QEMU_LOCK_GUARD(&s->lock) {
561 if (s->method == t->method) {
566 if (!t->call_state->ret) {
567 t->call_state->ret = ret;
568 t->call_state->error_is_read = error_is_read;
570 } else if (s->progress) {
571 progress_work_done(s->progress, t->req.bytes);
574 co_put_to_shres(s->mem, t->req.bytes);
575 block_copy_task_end(t, ret);
580 static int block_copy_block_status(BlockCopyState *s, int64_t offset,
581 int64_t bytes, int64_t *pnum)
584 BlockDriverState *base;
587 if (qatomic_read(&s->skip_unallocated)) {
588 base = bdrv_backing_chain_next(s->source->bs);
593 ret = bdrv_block_status_above(s->source->bs, base, offset, bytes, &num,
595 if (ret < 0 || num < s->cluster_size) {
597 * On error or if failed to obtain large enough chunk just fallback to
600 num = s->cluster_size;
601 ret = BDRV_BLOCK_ALLOCATED | BDRV_BLOCK_DATA;
602 } else if (offset + num == s->len) {
603 num = QEMU_ALIGN_UP(num, s->cluster_size);
605 num = QEMU_ALIGN_DOWN(num, s->cluster_size);
613 * Check if the cluster starting at offset is allocated or not.
614 * return via pnum the number of contiguous clusters sharing this allocation.
616 static int block_copy_is_cluster_allocated(BlockCopyState *s, int64_t offset,
619 BlockDriverState *bs = s->source->bs;
620 int64_t count, total_count = 0;
621 int64_t bytes = s->len - offset;
624 assert(QEMU_IS_ALIGNED(offset, s->cluster_size));
627 ret = bdrv_is_allocated(bs, offset, bytes, &count);
632 total_count += count;
634 if (ret || count == 0) {
636 * ret: partial segment(s) are considered allocated.
637 * otherwise: unallocated tail is treated as an entire segment.
639 *pnum = DIV_ROUND_UP(total_count, s->cluster_size);
643 /* Unallocated segment(s) with uncertain following segment(s) */
644 if (total_count >= s->cluster_size) {
645 *pnum = total_count / s->cluster_size;
654 void block_copy_reset(BlockCopyState *s, int64_t offset, int64_t bytes)
656 QEMU_LOCK_GUARD(&s->lock);
658 bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, bytes);
660 progress_set_remaining(s->progress,
661 bdrv_get_dirty_count(s->copy_bitmap) +
667 * Reset bits in copy_bitmap starting at offset if they represent unallocated
668 * data in the image. May reset subsequent contiguous bits.
669 * @return 0 when the cluster at @offset was unallocated,
670 * 1 otherwise, and -ret on error.
672 int64_t block_copy_reset_unallocated(BlockCopyState *s,
673 int64_t offset, int64_t *count)
676 int64_t clusters, bytes;
678 ret = block_copy_is_cluster_allocated(s, offset, &clusters);
683 bytes = clusters * s->cluster_size;
686 block_copy_reset(s, offset, bytes);
694 * block_copy_dirty_clusters
696 * Copy dirty clusters in @offset/@bytes range.
697 * Returns 1 if dirty clusters found and successfully copied, 0 if no dirty
698 * clusters found and -errno on failure.
700 static int coroutine_fn
701 block_copy_dirty_clusters(BlockCopyCallState *call_state)
703 BlockCopyState *s = call_state->s;
704 int64_t offset = call_state->offset;
705 int64_t bytes = call_state->bytes;
708 bool found_dirty = false;
709 int64_t end = offset + bytes;
710 AioTaskPool *aio = NULL;
713 * block_copy() user is responsible for keeping source and target in same
716 assert(bdrv_get_aio_context(s->source->bs) ==
717 bdrv_get_aio_context(s->target->bs));
719 assert(QEMU_IS_ALIGNED(offset, s->cluster_size));
720 assert(QEMU_IS_ALIGNED(bytes, s->cluster_size));
722 while (bytes && aio_task_pool_status(aio) == 0 &&
723 !qatomic_read(&call_state->cancelled)) {
725 int64_t status_bytes;
727 task = block_copy_task_create(s, call_state, offset, bytes);
729 /* No more dirty bits in the bitmap */
730 trace_block_copy_skip_range(s, offset, bytes);
733 if (task->req.offset > offset) {
734 trace_block_copy_skip_range(s, offset, task->req.offset - offset);
739 ret = block_copy_block_status(s, task->req.offset, task->req.bytes,
741 assert(ret >= 0); /* never fail */
742 if (status_bytes < task->req.bytes) {
743 block_copy_task_shrink(task, status_bytes);
745 if (qatomic_read(&s->skip_unallocated) &&
746 !(ret & BDRV_BLOCK_ALLOCATED)) {
747 block_copy_task_end(task, 0);
748 trace_block_copy_skip_range(s, task->req.offset, task->req.bytes);
749 offset = task_end(task);
750 bytes = end - offset;
754 if (ret & BDRV_BLOCK_ZERO) {
755 task->method = COPY_WRITE_ZEROES;
758 if (!call_state->ignore_ratelimit) {
759 uint64_t ns = ratelimit_calculate_delay(&s->rate_limit, 0);
761 block_copy_task_end(task, -EAGAIN);
763 qemu_co_sleep_ns_wakeable(&call_state->sleep,
764 QEMU_CLOCK_REALTIME, ns);
769 ratelimit_calculate_delay(&s->rate_limit, task->req.bytes);
771 trace_block_copy_process(s, task->req.offset);
773 co_get_from_shres(s->mem, task->req.bytes);
775 offset = task_end(task);
776 bytes = end - offset;
779 aio = aio_task_pool_new(call_state->max_workers);
782 ret = block_copy_task_run(aio, task);
790 aio_task_pool_wait_all(aio);
793 * We are not really interested in -ECANCELED returned from
794 * block_copy_task_run. If it fails, it means some task already failed
795 * for real reason, let's return first failure.
796 * Still, assert that we don't rewrite failure by success.
798 * Note: ret may be positive here because of block-status result.
800 assert(ret >= 0 || aio_task_pool_status(aio) < 0);
801 ret = aio_task_pool_status(aio);
803 aio_task_pool_free(aio);
806 return ret < 0 ? ret : found_dirty;
809 void block_copy_kick(BlockCopyCallState *call_state)
811 qemu_co_sleep_wake(&call_state->sleep);
817 * Copy requested region, accordingly to dirty bitmap.
818 * Collaborate with parallel block_copy requests: if they succeed it will help
819 * us. If they fail, we will retry not-copied regions. So, if we return error,
820 * it means that some I/O operation failed in context of _this_ block_copy call,
821 * not some parallel operation.
823 static int coroutine_fn block_copy_common(BlockCopyCallState *call_state)
826 BlockCopyState *s = call_state->s;
828 qemu_co_mutex_lock(&s->lock);
829 QLIST_INSERT_HEAD(&s->calls, call_state, list);
830 qemu_co_mutex_unlock(&s->lock);
833 ret = block_copy_dirty_clusters(call_state);
835 if (ret == 0 && !qatomic_read(&call_state->cancelled)) {
836 WITH_QEMU_LOCK_GUARD(&s->lock) {
838 * Check that there is no task we still need to
841 ret = reqlist_wait_one(&s->reqs, call_state->offset,
842 call_state->bytes, &s->lock);
845 * No pending tasks, but check again the bitmap in this
846 * same critical section, since a task might have failed
847 * between this and the critical section in
848 * block_copy_dirty_clusters().
850 * reqlist_wait_one return value 0 also means that it
851 * didn't release the lock. So, we are still in the same
852 * critical section, not interrupted by any concurrent
855 ret = bdrv_dirty_bitmap_next_dirty(s->copy_bitmap,
857 call_state->bytes) >= 0;
863 * We retry in two cases:
864 * 1. Some progress done
865 * Something was copied, which means that there were yield points
866 * and some new dirty bits may have appeared (due to failed parallel
867 * block-copy requests).
868 * 2. We have waited for some intersecting block-copy request
869 * It may have failed and produced new dirty bits.
871 } while (ret > 0 && !qatomic_read(&call_state->cancelled));
873 qatomic_store_release(&call_state->finished, true);
875 if (call_state->cb) {
876 call_state->cb(call_state->cb_opaque);
879 qemu_co_mutex_lock(&s->lock);
880 QLIST_REMOVE(call_state, list);
881 qemu_co_mutex_unlock(&s->lock);
886 static void coroutine_fn block_copy_async_co_entry(void *opaque)
888 block_copy_common(opaque);
891 int coroutine_fn block_copy(BlockCopyState *s, int64_t start, int64_t bytes,
892 bool ignore_ratelimit, uint64_t timeout_ns,
893 BlockCopyAsyncCallbackFunc cb,
897 BlockCopyCallState *call_state = g_new(BlockCopyCallState, 1);
899 *call_state = (BlockCopyCallState) {
903 .ignore_ratelimit = ignore_ratelimit,
904 .max_workers = BLOCK_COPY_MAX_WORKERS,
906 .cb_opaque = cb_opaque,
909 ret = qemu_co_timeout(block_copy_async_co_entry, call_state, timeout_ns,
912 assert(ret == -ETIMEDOUT);
913 block_copy_call_cancel(call_state);
914 /* call_state will be freed by running coroutine. */
918 ret = call_state->ret;
924 BlockCopyCallState *block_copy_async(BlockCopyState *s,
925 int64_t offset, int64_t bytes,
926 int max_workers, int64_t max_chunk,
927 BlockCopyAsyncCallbackFunc cb,
930 BlockCopyCallState *call_state = g_new(BlockCopyCallState, 1);
932 *call_state = (BlockCopyCallState) {
936 .max_workers = max_workers,
937 .max_chunk = max_chunk,
939 .cb_opaque = cb_opaque,
941 .co = qemu_coroutine_create(block_copy_async_co_entry, call_state),
944 qemu_coroutine_enter(call_state->co);
949 void block_copy_call_free(BlockCopyCallState *call_state)
955 assert(qatomic_read(&call_state->finished));
959 bool block_copy_call_finished(BlockCopyCallState *call_state)
961 return qatomic_read(&call_state->finished);
964 bool block_copy_call_succeeded(BlockCopyCallState *call_state)
966 return qatomic_load_acquire(&call_state->finished) &&
967 !qatomic_read(&call_state->cancelled) &&
968 call_state->ret == 0;
971 bool block_copy_call_failed(BlockCopyCallState *call_state)
973 return qatomic_load_acquire(&call_state->finished) &&
974 !qatomic_read(&call_state->cancelled) &&
978 bool block_copy_call_cancelled(BlockCopyCallState *call_state)
980 return qatomic_read(&call_state->cancelled);
983 int block_copy_call_status(BlockCopyCallState *call_state, bool *error_is_read)
985 assert(qatomic_load_acquire(&call_state->finished));
987 *error_is_read = call_state->error_is_read;
989 return call_state->ret;
993 * Note that cancelling and finishing are racy.
994 * User can cancel a block-copy that is already finished.
996 void block_copy_call_cancel(BlockCopyCallState *call_state)
998 qatomic_set(&call_state->cancelled, true);
999 block_copy_kick(call_state);
1002 BdrvDirtyBitmap *block_copy_dirty_bitmap(BlockCopyState *s)
1004 return s->copy_bitmap;
1007 int64_t block_copy_cluster_size(BlockCopyState *s)
1009 return s->cluster_size;
1012 void block_copy_set_skip_unallocated(BlockCopyState *s, bool skip)
1014 qatomic_set(&s->skip_unallocated, skip);
1017 void block_copy_set_speed(BlockCopyState *s, uint64_t speed)
1019 ratelimit_set_speed(&s->rate_limit, speed, BLOCK_COPY_SLICE_TIME);
1022 * Note: it's good to kick all call states from here, but it should be done
1023 * only from a coroutine, to not crash if s->calls list changed while
1024 * entering one call. So for now, the only user of this function kicks its
1025 * only one call_state by hand.