1 /* SPDX-License-Identifier: LGPL-2.1-or-later */
5 * Copyright Red Hat, Inc.
8 * Stefan Hajnoczi <stefanha@redhat.com>
11 #include "qemu/osdep.h"
13 #include "block/block_int.h"
14 #include "exec/memory.h"
15 #include "exec/cpu-common.h" /* for qemu_ram_get_fd() */
16 #include "qemu/defer-call.h"
17 #include "qapi/error.h"
18 #include "qemu/error-report.h"
19 #include "qapi/qmp/qdict.h"
20 #include "qemu/module.h"
21 #include "sysemu/block-backend.h"
22 #include "exec/memory.h" /* for ram_block_discard_disable() */
24 #include "block/block-io.h"
27 * Allocated bounce buffers are kept in a list sorted by buffer address.
29 typedef struct BlkioBounceBuf {
30 QLIST_ENTRY(BlkioBounceBuf) next;
32 /* The bounce buffer */
38 * libblkio is not thread-safe so this lock protects ->blkio and
43 struct blkioq *blkioq; /* make this multi-queue in the future... */
47 * Polling fetches the next completion into this field.
49 * No lock is necessary since only one thread calls aio_poll() and invokes
50 * fd and poll handlers.
52 struct blkio_completion poll_completion;
55 * Protects ->bounce_pool, ->bounce_bufs, ->bounce_available.
57 * Lock ordering: ->bounce_lock before ->blkio_lock.
61 /* Bounce buffer pool */
62 struct blkio_mem_region bounce_pool;
64 /* Sorted list of allocated bounce buffers */
65 QLIST_HEAD(, BlkioBounceBuf) bounce_bufs;
67 /* Queue for coroutines waiting for bounce buffer space */
68 CoQueue bounce_available;
70 /* The value of the "mem-region-alignment" property */
71 size_t mem_region_alignment;
73 /* Can we skip adding/deleting blkio_mem_regions? */
74 bool needs_mem_regions;
76 /* Are file descriptors necessary for blkio_mem_regions? */
77 bool needs_mem_region_fd;
79 /* Are madvise(MADV_DONTNEED)-style operations unavailable? */
80 bool may_pin_mem_regions;
83 /* Called with s->bounce_lock held */
84 static int blkio_resize_bounce_pool(BDRVBlkioState *s, int64_t bytes)
86 /* There can be no allocated bounce buffers during resize */
87 assert(QLIST_EMPTY(&s->bounce_bufs));
89 /* Pad size to reduce frequency of resize calls */
92 WITH_QEMU_LOCK_GUARD(&s->blkio_lock) {
95 if (s->bounce_pool.addr) {
96 blkio_unmap_mem_region(s->blkio, &s->bounce_pool);
97 blkio_free_mem_region(s->blkio, &s->bounce_pool);
98 memset(&s->bounce_pool, 0, sizeof(s->bounce_pool));
101 /* Automatically freed when s->blkio is destroyed */
102 ret = blkio_alloc_mem_region(s->blkio, &s->bounce_pool, bytes);
107 ret = blkio_map_mem_region(s->blkio, &s->bounce_pool);
109 blkio_free_mem_region(s->blkio, &s->bounce_pool);
110 memset(&s->bounce_pool, 0, sizeof(s->bounce_pool));
118 /* Called with s->bounce_lock held */
120 blkio_do_alloc_bounce_buffer(BDRVBlkioState *s, BlkioBounceBuf *bounce,
123 void *addr = s->bounce_pool.addr;
124 BlkioBounceBuf *cur = NULL;
125 BlkioBounceBuf *prev = NULL;
129 * This is just a linear search over the holes between requests. An
130 * efficient allocator would be nice.
132 QLIST_FOREACH(cur, &s->bounce_bufs, next) {
133 space = cur->buf.iov_base - addr;
134 if (bytes <= space) {
135 QLIST_INSERT_BEFORE(cur, bounce, next);
136 bounce->buf.iov_base = addr;
137 bounce->buf.iov_len = bytes;
141 addr = cur->buf.iov_base + cur->buf.iov_len;
145 /* Is there space after the last request? */
146 space = s->bounce_pool.addr + s->bounce_pool.len - addr;
151 QLIST_INSERT_AFTER(prev, bounce, next);
153 QLIST_INSERT_HEAD(&s->bounce_bufs, bounce, next);
155 bounce->buf.iov_base = addr;
156 bounce->buf.iov_len = bytes;
160 static int coroutine_fn
161 blkio_alloc_bounce_buffer(BDRVBlkioState *s, BlkioBounceBuf *bounce,
165 * Ensure fairness: first time around we join the back of the queue,
166 * subsequently we join the front so we don't lose our place.
168 CoQueueWaitFlags wait_flags = 0;
170 QEMU_LOCK_GUARD(&s->bounce_lock);
172 /* Ensure fairness: don't even try if other requests are already waiting */
173 if (!qemu_co_queue_empty(&s->bounce_available)) {
174 qemu_co_queue_wait_flags(&s->bounce_available, &s->bounce_lock,
176 wait_flags = CO_QUEUE_WAIT_FRONT;
180 if (blkio_do_alloc_bounce_buffer(s, bounce, bytes)) {
181 /* Kick the next queued request since there may be space */
182 qemu_co_queue_next(&s->bounce_available);
187 * If there are no in-flight requests then the pool was simply too
190 if (QLIST_EMPTY(&s->bounce_bufs)) {
194 ret = blkio_resize_bounce_pool(s, bytes);
196 /* Kick the next queued request since that may fail too */
197 qemu_co_queue_next(&s->bounce_available);
201 ok = blkio_do_alloc_bounce_buffer(s, bounce, bytes);
202 assert(ok); /* must have space this time */
206 qemu_co_queue_wait_flags(&s->bounce_available, &s->bounce_lock,
208 wait_flags = CO_QUEUE_WAIT_FRONT;
212 static void coroutine_fn blkio_free_bounce_buffer(BDRVBlkioState *s,
213 BlkioBounceBuf *bounce)
215 QEMU_LOCK_GUARD(&s->bounce_lock);
217 QLIST_REMOVE(bounce, next);
219 /* Wake up waiting coroutines since space may now be available */
220 qemu_co_queue_next(&s->bounce_available);
223 /* For async to .bdrv_co_*() conversion */
225 Coroutine *coroutine;
229 static void blkio_completion_fd_read(void *opaque)
231 BlockDriverState *bs = opaque;
232 BDRVBlkioState *s = bs->opaque;
236 /* Polling may have already fetched a completion */
237 if (s->poll_completion.user_data != NULL) {
238 BlkioCoData *cod = s->poll_completion.user_data;
239 cod->ret = s->poll_completion.ret;
241 /* Clear it in case aio_co_wake() enters a nested event loop */
242 s->poll_completion.user_data = NULL;
244 aio_co_wake(cod->coroutine);
247 /* Reset completion fd status */
248 ret = read(s->completion_fd, &val, sizeof(val));
250 /* Ignore errors, there's nothing we can do */
254 * Reading one completion at a time makes nested event loop re-entrancy
255 * simple. Change this loop to get multiple completions in one go if it
256 * becomes a performance bottleneck.
259 struct blkio_completion completion;
261 WITH_QEMU_LOCK_GUARD(&s->blkio_lock) {
262 ret = blkioq_do_io(s->blkioq, &completion, 0, 1, NULL);
268 BlkioCoData *cod = completion.user_data;
269 cod->ret = completion.ret;
270 aio_co_wake(cod->coroutine);
274 static bool blkio_completion_fd_poll(void *opaque)
276 BlockDriverState *bs = opaque;
277 BDRVBlkioState *s = bs->opaque;
280 /* Just in case we already fetched a completion */
281 if (s->poll_completion.user_data != NULL) {
285 WITH_QEMU_LOCK_GUARD(&s->blkio_lock) {
286 ret = blkioq_do_io(s->blkioq, &s->poll_completion, 0, 1, NULL);
291 static void blkio_completion_fd_poll_ready(void *opaque)
293 blkio_completion_fd_read(opaque);
296 static void blkio_attach_aio_context(BlockDriverState *bs,
297 AioContext *new_context)
299 BDRVBlkioState *s = bs->opaque;
301 aio_set_fd_handler(new_context, s->completion_fd,
302 blkio_completion_fd_read, NULL,
303 blkio_completion_fd_poll,
304 blkio_completion_fd_poll_ready, bs);
307 static void blkio_detach_aio_context(BlockDriverState *bs)
309 BDRVBlkioState *s = bs->opaque;
311 aio_set_fd_handler(bdrv_get_aio_context(bs), s->completion_fd, NULL, NULL,
316 * Called by defer_call_end() or immediately if not in a deferred section.
317 * Called without blkio_lock.
319 static void blkio_deferred_fn(void *opaque)
321 BDRVBlkioState *s = opaque;
323 WITH_QEMU_LOCK_GUARD(&s->blkio_lock) {
324 blkioq_do_io(s->blkioq, NULL, 0, 0, NULL);
329 * Schedule I/O submission after enqueuing a new request. Called without
332 static void blkio_submit_io(BlockDriverState *bs)
334 BDRVBlkioState *s = bs->opaque;
336 defer_call(blkio_deferred_fn, s);
339 static int coroutine_fn
340 blkio_co_pdiscard(BlockDriverState *bs, int64_t offset, int64_t bytes)
342 BDRVBlkioState *s = bs->opaque;
344 .coroutine = qemu_coroutine_self(),
347 WITH_QEMU_LOCK_GUARD(&s->blkio_lock) {
348 blkioq_discard(s->blkioq, offset, bytes, &cod, 0);
352 qemu_coroutine_yield();
356 static int coroutine_fn
357 blkio_co_preadv(BlockDriverState *bs, int64_t offset, int64_t bytes,
358 QEMUIOVector *qiov, BdrvRequestFlags flags)
361 .coroutine = qemu_coroutine_self(),
363 BDRVBlkioState *s = bs->opaque;
364 bool use_bounce_buffer =
365 s->needs_mem_regions && !(flags & BDRV_REQ_REGISTERED_BUF);
366 BlkioBounceBuf bounce;
367 struct iovec *iov = qiov->iov;
368 int iovcnt = qiov->niov;
370 if (use_bounce_buffer) {
371 int ret = blkio_alloc_bounce_buffer(s, &bounce, bytes);
380 WITH_QEMU_LOCK_GUARD(&s->blkio_lock) {
381 blkioq_readv(s->blkioq, offset, iov, iovcnt, &cod, 0);
385 qemu_coroutine_yield();
387 if (use_bounce_buffer) {
389 qemu_iovec_from_buf(qiov, 0,
394 blkio_free_bounce_buffer(s, &bounce);
400 static int coroutine_fn blkio_co_pwritev(BlockDriverState *bs, int64_t offset,
401 int64_t bytes, QEMUIOVector *qiov, BdrvRequestFlags flags)
403 uint32_t blkio_flags = (flags & BDRV_REQ_FUA) ? BLKIO_REQ_FUA : 0;
405 .coroutine = qemu_coroutine_self(),
407 BDRVBlkioState *s = bs->opaque;
408 bool use_bounce_buffer =
409 s->needs_mem_regions && !(flags & BDRV_REQ_REGISTERED_BUF);
410 BlkioBounceBuf bounce;
411 struct iovec *iov = qiov->iov;
412 int iovcnt = qiov->niov;
414 if (use_bounce_buffer) {
415 int ret = blkio_alloc_bounce_buffer(s, &bounce, bytes);
420 qemu_iovec_to_buf(qiov, 0, bounce.buf.iov_base, bytes);
425 WITH_QEMU_LOCK_GUARD(&s->blkio_lock) {
426 blkioq_writev(s->blkioq, offset, iov, iovcnt, &cod, blkio_flags);
430 qemu_coroutine_yield();
432 if (use_bounce_buffer) {
433 blkio_free_bounce_buffer(s, &bounce);
439 static int coroutine_fn blkio_co_flush(BlockDriverState *bs)
441 BDRVBlkioState *s = bs->opaque;
443 .coroutine = qemu_coroutine_self(),
446 WITH_QEMU_LOCK_GUARD(&s->blkio_lock) {
447 blkioq_flush(s->blkioq, &cod, 0);
451 qemu_coroutine_yield();
455 static int coroutine_fn blkio_co_pwrite_zeroes(BlockDriverState *bs,
456 int64_t offset, int64_t bytes, BdrvRequestFlags flags)
458 BDRVBlkioState *s = bs->opaque;
460 .coroutine = qemu_coroutine_self(),
462 uint32_t blkio_flags = 0;
464 if (flags & BDRV_REQ_FUA) {
465 blkio_flags |= BLKIO_REQ_FUA;
467 if (!(flags & BDRV_REQ_MAY_UNMAP)) {
468 blkio_flags |= BLKIO_REQ_NO_UNMAP;
470 if (flags & BDRV_REQ_NO_FALLBACK) {
471 blkio_flags |= BLKIO_REQ_NO_FALLBACK;
474 WITH_QEMU_LOCK_GUARD(&s->blkio_lock) {
475 blkioq_write_zeroes(s->blkioq, offset, bytes, &cod, blkio_flags);
479 qemu_coroutine_yield();
487 } BlkioMemRegionResult;
490 * Produce a struct blkio_mem_region for a given address and size.
492 * This function produces identical results when called multiple times with the
493 * same arguments. This property is necessary because blkio_unmap_mem_region()
494 * must receive the same struct blkio_mem_region field values that were passed
495 * to blkio_map_mem_region().
497 static BlkioMemRegionResult
498 blkio_mem_region_from_host(BlockDriverState *bs,
499 void *host, size_t size,
500 struct blkio_mem_region *region,
503 BDRVBlkioState *s = bs->opaque;
505 ram_addr_t fd_offset = 0;
507 if (((uintptr_t)host | size) % s->mem_region_alignment) {
508 error_setg(errp, "unaligned buf %p with size %zu", host, size);
512 /* Attempt to find the fd for the underlying memory */
513 if (s->needs_mem_region_fd) {
519 * bdrv_register_buf() is called with the BQL held so mr lives at least
520 * until this function returns.
522 ram_block = qemu_ram_block_from_host(host, false, &fd_offset);
524 fd = qemu_ram_get_fd(ram_block);
528 * Ideally every RAMBlock would have an fd. pc-bios and other
529 * things don't. Luckily they are usually not I/O buffers and we
530 * can just ignore them.
535 /* Make sure the fd covers the entire range */
536 end_block = qemu_ram_block_from_host(host + size - 1, false, &offset);
537 if (ram_block != end_block) {
538 error_setg(errp, "registered buffer at %p with size %zu extends "
539 "beyond RAMBlock", host, size);
544 *region = (struct blkio_mem_region){
548 .fd_offset = fd_offset,
553 static bool blkio_register_buf(BlockDriverState *bs, void *host, size_t size,
556 BDRVBlkioState *s = bs->opaque;
557 struct blkio_mem_region region;
558 BlkioMemRegionResult region_result;
562 * Mapping memory regions conflicts with RAM discard (virtio-mem) when
563 * there is pinning, so only do it when necessary.
565 if (!s->needs_mem_regions && s->may_pin_mem_regions) {
569 region_result = blkio_mem_region_from_host(bs, host, size, ®ion, errp);
570 if (region_result == BMRR_SKIP) {
572 } else if (region_result != BMRR_OK) {
576 WITH_QEMU_LOCK_GUARD(&s->blkio_lock) {
577 ret = blkio_map_mem_region(s->blkio, ®ion);
581 error_setg(errp, "Failed to add blkio mem region %p with size %zu: %s",
582 host, size, blkio_get_error_msg());
588 static void blkio_unregister_buf(BlockDriverState *bs, void *host, size_t size)
590 BDRVBlkioState *s = bs->opaque;
591 struct blkio_mem_region region;
593 /* See blkio_register_buf() */
594 if (!s->needs_mem_regions && s->may_pin_mem_regions) {
598 if (blkio_mem_region_from_host(bs, host, size, ®ion, NULL) != BMRR_OK) {
602 WITH_QEMU_LOCK_GUARD(&s->blkio_lock) {
603 blkio_unmap_mem_region(s->blkio, ®ion);
607 static int blkio_io_uring_connect(BlockDriverState *bs, QDict *options,
608 int flags, Error **errp)
610 const char *filename = qdict_get_str(options, "filename");
611 BDRVBlkioState *s = bs->opaque;
614 ret = blkio_set_str(s->blkio, "path", filename);
615 qdict_del(options, "filename");
617 error_setg_errno(errp, -ret, "failed to set path: %s",
618 blkio_get_error_msg());
622 if (flags & BDRV_O_NOCACHE) {
623 ret = blkio_set_bool(s->blkio, "direct", true);
625 error_setg_errno(errp, -ret, "failed to set direct: %s",
626 blkio_get_error_msg());
631 ret = blkio_connect(s->blkio);
633 error_setg_errno(errp, -ret, "blkio_connect failed: %s",
634 blkio_get_error_msg());
641 static int blkio_nvme_io_uring_connect(BlockDriverState *bs, QDict *options,
642 int flags, Error **errp)
644 const char *path = qdict_get_try_str(options, "path");
645 BDRVBlkioState *s = bs->opaque;
649 error_setg(errp, "missing 'path' option");
653 ret = blkio_set_str(s->blkio, "path", path);
654 qdict_del(options, "path");
656 error_setg_errno(errp, -ret, "failed to set path: %s",
657 blkio_get_error_msg());
661 if (!(flags & BDRV_O_NOCACHE)) {
662 error_setg(errp, "cache.direct=off is not supported");
666 ret = blkio_connect(s->blkio);
668 error_setg_errno(errp, -ret, "blkio_connect failed: %s",
669 blkio_get_error_msg());
676 static int blkio_virtio_blk_connect(BlockDriverState *bs, QDict *options,
677 int flags, Error **errp)
679 const char *path = qdict_get_try_str(options, "path");
680 BDRVBlkioState *s = bs->opaque;
681 bool fd_supported = false;
685 error_setg(errp, "missing 'path' option");
689 if (!(flags & BDRV_O_NOCACHE)) {
690 error_setg(errp, "cache.direct=off is not supported");
694 if (blkio_set_int(s->blkio, "fd", -1) == 0) {
699 * If the libblkio driver supports fd passing, let's always use qemu_open()
700 * to open the `path`, so we can handle fd passing from the management
701 * layer through the "/dev/fdset/N" special path.
705 * `path` can contain the path of a character device
706 * (e.g. /dev/vhost-vdpa-0 or /dev/vfio/vfio) or a unix socket.
708 * So, we should always open it with O_RDWR flag, also if BDRV_O_RDWR
709 * is not set in the open flags, because the exchange of IOCTL commands
710 * for example will fail.
712 * In order to open the device read-only, we are using the `read-only`
713 * property of the libblkio driver in blkio_file_open().
715 fd = qemu_open(path, O_RDWR, NULL);
718 * qemu_open() can fail if the user specifies a path that is not
719 * a file or device, for example in the case of Unix Domain Socket
720 * for the virtio-blk-vhost-user driver. In such cases let's have
721 * libblkio open the path directly.
723 fd_supported = false;
725 ret = blkio_set_int(s->blkio, "fd", fd);
727 fd_supported = false;
735 ret = blkio_set_str(s->blkio, "path", path);
737 error_setg_errno(errp, -ret, "failed to set path: %s",
738 blkio_get_error_msg());
743 ret = blkio_connect(s->blkio);
744 if (ret < 0 && fd >= 0) {
745 /* Failed to give the FD to libblkio, close it */
751 * Before https://gitlab.com/libblkio/libblkio/-/merge_requests/208
752 * (libblkio <= v1.3.0), setting the `fd` property is not enough to check
753 * whether the driver supports the `fd` property or not. In that case,
754 * blkio_connect() will fail with -EINVAL.
755 * So let's try calling blkio_connect() again by directly setting `path`
756 * to cover this scenario.
758 if (fd_supported && ret == -EINVAL) {
760 * We need to clear the `fd` property we set previously by setting
763 ret = blkio_set_int(s->blkio, "fd", -1);
765 error_setg_errno(errp, -ret, "failed to set fd: %s",
766 blkio_get_error_msg());
770 ret = blkio_set_str(s->blkio, "path", path);
772 error_setg_errno(errp, -ret, "failed to set path: %s",
773 blkio_get_error_msg());
777 ret = blkio_connect(s->blkio);
781 error_setg_errno(errp, -ret, "blkio_connect failed: %s",
782 blkio_get_error_msg());
786 qdict_del(options, "path");
791 static int blkio_file_open(BlockDriverState *bs, QDict *options, int flags,
794 const char *blkio_driver = bs->drv->protocol_name;
795 BDRVBlkioState *s = bs->opaque;
798 ret = blkio_create(blkio_driver, &s->blkio);
800 error_setg_errno(errp, -ret, "blkio_create failed: %s",
801 blkio_get_error_msg());
805 if (!(flags & BDRV_O_RDWR)) {
806 ret = blkio_set_bool(s->blkio, "read-only", true);
808 error_setg_errno(errp, -ret, "failed to set read-only: %s",
809 blkio_get_error_msg());
810 blkio_destroy(&s->blkio);
815 if (strcmp(blkio_driver, "io_uring") == 0) {
816 ret = blkio_io_uring_connect(bs, options, flags, errp);
817 } else if (strcmp(blkio_driver, "nvme-io_uring") == 0) {
818 ret = blkio_nvme_io_uring_connect(bs, options, flags, errp);
819 } else if (strcmp(blkio_driver, "virtio-blk-vfio-pci") == 0) {
820 ret = blkio_virtio_blk_connect(bs, options, flags, errp);
821 } else if (strcmp(blkio_driver, "virtio-blk-vhost-user") == 0) {
822 ret = blkio_virtio_blk_connect(bs, options, flags, errp);
823 } else if (strcmp(blkio_driver, "virtio-blk-vhost-vdpa") == 0) {
824 ret = blkio_virtio_blk_connect(bs, options, flags, errp);
826 g_assert_not_reached();
829 blkio_destroy(&s->blkio);
833 ret = blkio_get_bool(s->blkio,
835 &s->needs_mem_regions);
837 error_setg_errno(errp, -ret,
838 "failed to get needs-mem-regions: %s",
839 blkio_get_error_msg());
840 blkio_destroy(&s->blkio);
844 ret = blkio_get_bool(s->blkio,
845 "needs-mem-region-fd",
846 &s->needs_mem_region_fd);
848 error_setg_errno(errp, -ret,
849 "failed to get needs-mem-region-fd: %s",
850 blkio_get_error_msg());
851 blkio_destroy(&s->blkio);
855 ret = blkio_get_uint64(s->blkio,
856 "mem-region-alignment",
857 &s->mem_region_alignment);
859 error_setg_errno(errp, -ret,
860 "failed to get mem-region-alignment: %s",
861 blkio_get_error_msg());
862 blkio_destroy(&s->blkio);
866 ret = blkio_get_bool(s->blkio,
867 "may-pin-mem-regions",
868 &s->may_pin_mem_regions);
870 /* Be conservative (assume pinning) if the property is not supported */
871 s->may_pin_mem_regions = s->needs_mem_regions;
875 * Notify if libblkio drivers pin memory and prevent features like
876 * virtio-mem from working.
878 if (s->may_pin_mem_regions) {
879 ret = ram_block_discard_disable(true);
881 error_setg_errno(errp, -ret, "ram_block_discard_disable() failed");
882 blkio_destroy(&s->blkio);
887 ret = blkio_start(s->blkio);
889 error_setg_errno(errp, -ret, "blkio_start failed: %s",
890 blkio_get_error_msg());
891 blkio_destroy(&s->blkio);
892 if (s->may_pin_mem_regions) {
893 ram_block_discard_disable(false);
898 bs->supported_write_flags = BDRV_REQ_FUA | BDRV_REQ_REGISTERED_BUF;
899 bs->supported_zero_flags = BDRV_REQ_FUA | BDRV_REQ_MAY_UNMAP |
900 BDRV_REQ_NO_FALLBACK;
902 qemu_mutex_init(&s->blkio_lock);
903 qemu_co_mutex_init(&s->bounce_lock);
904 qemu_co_queue_init(&s->bounce_available);
905 QLIST_INIT(&s->bounce_bufs);
906 s->blkioq = blkio_get_queue(s->blkio, 0);
907 s->completion_fd = blkioq_get_completion_fd(s->blkioq);
908 blkioq_set_completion_fd_enabled(s->blkioq, true);
910 blkio_attach_aio_context(bs, bdrv_get_aio_context(bs));
914 static void blkio_close(BlockDriverState *bs)
916 BDRVBlkioState *s = bs->opaque;
918 /* There is no destroy() API for s->bounce_lock */
920 qemu_mutex_destroy(&s->blkio_lock);
921 blkio_detach_aio_context(bs);
922 blkio_destroy(&s->blkio);
924 if (s->may_pin_mem_regions) {
925 ram_block_discard_disable(false);
929 static int64_t coroutine_fn blkio_co_getlength(BlockDriverState *bs)
931 BDRVBlkioState *s = bs->opaque;
935 WITH_QEMU_LOCK_GUARD(&s->blkio_lock) {
936 ret = blkio_get_uint64(s->blkio, "capacity", &capacity);
945 static int coroutine_fn blkio_truncate(BlockDriverState *bs, int64_t offset,
946 bool exact, PreallocMode prealloc,
947 BdrvRequestFlags flags, Error **errp)
949 int64_t current_length;
951 if (prealloc != PREALLOC_MODE_OFF) {
952 error_setg(errp, "Unsupported preallocation mode '%s'",
953 PreallocMode_str(prealloc));
957 current_length = blkio_co_getlength(bs);
959 if (offset > current_length) {
960 error_setg(errp, "Cannot grow device");
962 } else if (exact && offset != current_length) {
963 error_setg(errp, "Cannot resize device");
970 static int coroutine_fn
971 blkio_co_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
976 static void blkio_refresh_limits(BlockDriverState *bs, Error **errp)
978 BDRVBlkioState *s = bs->opaque;
979 QEMU_LOCK_GUARD(&s->blkio_lock);
983 ret = blkio_get_int(s->blkio, "request-alignment", &value);
985 error_setg_errno(errp, -ret, "failed to get \"request-alignment\": %s",
986 blkio_get_error_msg());
989 bs->bl.request_alignment = value;
990 if (bs->bl.request_alignment < 1 ||
991 bs->bl.request_alignment >= INT_MAX ||
992 !is_power_of_2(bs->bl.request_alignment)) {
993 error_setg(errp, "invalid \"request-alignment\" value %" PRIu32 ", "
994 "must be a power of 2 less than INT_MAX",
995 bs->bl.request_alignment);
999 ret = blkio_get_int(s->blkio, "optimal-io-size", &value);
1001 error_setg_errno(errp, -ret, "failed to get \"optimal-io-size\": %s",
1002 blkio_get_error_msg());
1005 bs->bl.opt_transfer = value;
1006 if (bs->bl.opt_transfer > INT_MAX ||
1007 (bs->bl.opt_transfer % bs->bl.request_alignment)) {
1008 error_setg(errp, "invalid \"optimal-io-size\" value %" PRIu32 ", must "
1009 "be a multiple of %" PRIu32, bs->bl.opt_transfer,
1010 bs->bl.request_alignment);
1014 ret = blkio_get_int(s->blkio, "max-transfer", &value);
1016 error_setg_errno(errp, -ret, "failed to get \"max-transfer\": %s",
1017 blkio_get_error_msg());
1020 bs->bl.max_transfer = value;
1021 if ((bs->bl.max_transfer % bs->bl.request_alignment) ||
1022 (bs->bl.opt_transfer && (bs->bl.max_transfer % bs->bl.opt_transfer))) {
1023 error_setg(errp, "invalid \"max-transfer\" value %" PRIu32 ", must be "
1024 "a multiple of %" PRIu32 " and %" PRIu32 " (if non-zero)",
1025 bs->bl.max_transfer, bs->bl.request_alignment,
1026 bs->bl.opt_transfer);
1030 ret = blkio_get_int(s->blkio, "buf-alignment", &value);
1032 error_setg_errno(errp, -ret, "failed to get \"buf-alignment\": %s",
1033 blkio_get_error_msg());
1037 error_setg(errp, "invalid \"buf-alignment\" value %d, must be "
1041 bs->bl.min_mem_alignment = value;
1043 ret = blkio_get_int(s->blkio, "optimal-buf-alignment", &value);
1045 error_setg_errno(errp, -ret,
1046 "failed to get \"optimal-buf-alignment\": %s",
1047 blkio_get_error_msg());
1051 error_setg(errp, "invalid \"optimal-buf-alignment\" value %d, "
1052 "must be positive", value);
1055 bs->bl.opt_mem_alignment = value;
1057 ret = blkio_get_int(s->blkio, "max-segments", &value);
1059 error_setg_errno(errp, -ret, "failed to get \"max-segments\": %s",
1060 blkio_get_error_msg());
1064 error_setg(errp, "invalid \"max-segments\" value %d, must be positive",
1068 bs->bl.max_iov = value;
1073 * Missing libblkio APIs:
1075 * - co_invalidate_cache
1083 * Do not include .format_name and .protocol_name because module_block.py
1084 * does not parse macros in the source code.
1086 #define BLKIO_DRIVER_COMMON \
1087 .instance_size = sizeof(BDRVBlkioState), \
1088 .bdrv_file_open = blkio_file_open, \
1089 .bdrv_close = blkio_close, \
1090 .bdrv_co_getlength = blkio_co_getlength, \
1091 .bdrv_co_truncate = blkio_truncate, \
1092 .bdrv_co_get_info = blkio_co_get_info, \
1093 .bdrv_attach_aio_context = blkio_attach_aio_context, \
1094 .bdrv_detach_aio_context = blkio_detach_aio_context, \
1095 .bdrv_co_pdiscard = blkio_co_pdiscard, \
1096 .bdrv_co_preadv = blkio_co_preadv, \
1097 .bdrv_co_pwritev = blkio_co_pwritev, \
1098 .bdrv_co_flush_to_disk = blkio_co_flush, \
1099 .bdrv_co_pwrite_zeroes = blkio_co_pwrite_zeroes, \
1100 .bdrv_refresh_limits = blkio_refresh_limits, \
1101 .bdrv_register_buf = blkio_register_buf, \
1102 .bdrv_unregister_buf = blkio_unregister_buf,
1105 * Use the same .format_name and .protocol_name as the libblkio driver name for
1109 static BlockDriver bdrv_io_uring = {
1110 .format_name = "io_uring",
1111 .protocol_name = "io_uring",
1112 .bdrv_needs_filename = true,
1116 static BlockDriver bdrv_nvme_io_uring = {
1117 .format_name = "nvme-io_uring",
1118 .protocol_name = "nvme-io_uring",
1122 static BlockDriver bdrv_virtio_blk_vfio_pci = {
1123 .format_name = "virtio-blk-vfio-pci",
1124 .protocol_name = "virtio-blk-vfio-pci",
1128 static BlockDriver bdrv_virtio_blk_vhost_user = {
1129 .format_name = "virtio-blk-vhost-user",
1130 .protocol_name = "virtio-blk-vhost-user",
1134 static BlockDriver bdrv_virtio_blk_vhost_vdpa = {
1135 .format_name = "virtio-blk-vhost-vdpa",
1136 .protocol_name = "virtio-blk-vhost-vdpa",
1140 static void bdrv_blkio_init(void)
1142 bdrv_register(&bdrv_io_uring);
1143 bdrv_register(&bdrv_nvme_io_uring);
1144 bdrv_register(&bdrv_virtio_blk_vfio_pci);
1145 bdrv_register(&bdrv_virtio_blk_vhost_user);
1146 bdrv_register(&bdrv_virtio_blk_vhost_vdpa);
1149 block_init(bdrv_blkio_init);