1 /* Virtio ring implementation.
3 * Copyright 2007 Rusty Russell IBM Corporation
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
19 #include <linux/virtio.h>
20 #include <linux/virtio_ring.h>
21 #include <linux/virtio_config.h>
22 #include <linux/device.h>
23 #include <linux/slab.h>
24 #include <linux/module.h>
25 #include <linux/hrtimer.h>
26 #include <linux/dma-mapping.h>
30 /* For development, we want to crash whenever the ring is screwed. */
31 #define BAD_RING(_vq, fmt, args...) \
33 dev_err(&(_vq)->vq.vdev->dev, \
34 "%s:"fmt, (_vq)->vq.name, ##args); \
37 /* Caller is supposed to guarantee no reentry. */
38 #define START_USE(_vq) \
41 panic("%s:in_use = %i\n", \
42 (_vq)->vq.name, (_vq)->in_use); \
43 (_vq)->in_use = __LINE__; \
45 #define END_USE(_vq) \
46 do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0)
47 #define LAST_ADD_TIME_UPDATE(_vq) \
49 ktime_t now = ktime_get(); \
51 /* No kick or get, with .1 second between? Warn. */ \
52 if ((_vq)->last_add_time_valid) \
53 WARN_ON(ktime_to_ms(ktime_sub(now, \
54 (_vq)->last_add_time)) > 100); \
55 (_vq)->last_add_time = now; \
56 (_vq)->last_add_time_valid = true; \
58 #define LAST_ADD_TIME_CHECK(_vq) \
60 if ((_vq)->last_add_time_valid) { \
61 WARN_ON(ktime_to_ms(ktime_sub(ktime_get(), \
62 (_vq)->last_add_time)) > 100); \
65 #define LAST_ADD_TIME_INVALID(_vq) \
66 ((_vq)->last_add_time_valid = false)
68 #define BAD_RING(_vq, fmt, args...) \
70 dev_err(&_vq->vq.vdev->dev, \
71 "%s:"fmt, (_vq)->vq.name, ##args); \
72 (_vq)->broken = true; \
76 #define LAST_ADD_TIME_UPDATE(vq)
77 #define LAST_ADD_TIME_CHECK(vq)
78 #define LAST_ADD_TIME_INVALID(vq)
81 struct vring_desc_state_split {
82 void *data; /* Data for callback. */
83 struct vring_desc *indir_desc; /* Indirect descriptor, if any. */
86 struct vring_desc_state_packed {
87 void *data; /* Data for callback. */
88 struct vring_packed_desc *indir_desc; /* Indirect descriptor, if any. */
89 u16 num; /* Descriptor list length. */
90 u16 next; /* The next desc state in a list. */
91 u16 last; /* The last desc state in a list. */
94 struct vring_desc_extra_packed {
95 dma_addr_t addr; /* Buffer DMA addr. */
96 u32 len; /* Buffer length. */
97 u16 flags; /* Descriptor flags. */
100 struct vring_virtqueue {
103 /* Is this a packed ring? */
106 /* Is DMA API used? */
109 /* Can we use weak barriers? */
112 /* Other side has made a mess, don't try any more. */
115 /* Host supports indirect buffers */
118 /* Host publishes avail event idx */
121 /* Head of free buffer list. */
122 unsigned int free_head;
123 /* Number we've added since last sync. */
124 unsigned int num_added;
126 /* Last used index we've seen. */
130 /* Available for split ring */
132 /* Actual memory layout for this queue. */
135 /* Last written value to avail->flags */
136 u16 avail_flags_shadow;
139 * Last written value to avail->idx in
142 u16 avail_idx_shadow;
144 /* Per-descriptor state. */
145 struct vring_desc_state_split *desc_state;
147 /* DMA address and size information */
148 dma_addr_t queue_dma_addr;
149 size_t queue_size_in_bytes;
152 /* Available for packed ring */
154 /* Actual memory layout for this queue. */
155 struct vring_packed vring;
157 /* Driver ring wrap counter. */
158 bool avail_wrap_counter;
160 /* Device ring wrap counter. */
161 bool used_wrap_counter;
163 /* Avail used flags. */
164 u16 avail_used_flags;
166 /* Index of the next avail descriptor. */
170 * Last written value to driver->flags in
173 u16 event_flags_shadow;
175 /* Per-descriptor state. */
176 struct vring_desc_state_packed *desc_state;
177 struct vring_desc_extra_packed *desc_extra;
179 /* DMA address and size information */
180 dma_addr_t ring_dma_addr;
181 dma_addr_t driver_event_dma_addr;
182 dma_addr_t device_event_dma_addr;
183 size_t ring_size_in_bytes;
184 size_t event_size_in_bytes;
188 /* How to notify other side. FIXME: commonalize hcalls! */
189 bool (*notify)(struct virtqueue *vq);
191 /* DMA, allocation, and size information */
195 /* They're supposed to lock for us. */
198 /* Figure out if their kicks are too delayed. */
199 bool last_add_time_valid;
200 ktime_t last_add_time;
209 #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
211 static inline bool virtqueue_use_indirect(struct virtqueue *_vq,
212 unsigned int total_sg)
214 struct vring_virtqueue *vq = to_vvq(_vq);
217 * If the host supports indirect descriptor tables, and we have multiple
218 * buffers, then go indirect. FIXME: tune this threshold
220 return (vq->indirect && total_sg > 1 && vq->vq.num_free);
224 * Modern virtio devices have feature bits to specify whether they need a
225 * quirk and bypass the IOMMU. If not there, just use the DMA API.
227 * If there, the interaction between virtio and DMA API is messy.
229 * On most systems with virtio, physical addresses match bus addresses,
230 * and it doesn't particularly matter whether we use the DMA API.
232 * On some systems, including Xen and any system with a physical device
233 * that speaks virtio behind a physical IOMMU, we must use the DMA API
234 * for virtio DMA to work at all.
236 * On other systems, including SPARC and PPC64, virtio-pci devices are
237 * enumerated as though they are behind an IOMMU, but the virtio host
238 * ignores the IOMMU, so we must either pretend that the IOMMU isn't
239 * there or somehow map everything as the identity.
241 * For the time being, we preserve historic behavior and bypass the DMA
244 * TODO: install a per-device DMA ops structure that does the right thing
245 * taking into account all the above quirks, and use the DMA API
246 * unconditionally on data path.
249 static bool vring_use_dma_api(struct virtio_device *vdev)
251 if (!virtio_has_iommu_quirk(vdev))
254 /* Otherwise, we are left to guess. */
256 * In theory, it's possible to have a buggy QEMU-supposed
257 * emulated Q35 IOMMU and Xen enabled at the same time. On
258 * such a configuration, virtio has never worked and will
259 * not work without an even larger kludge. Instead, enable
260 * the DMA API if we're a Xen guest, which at least allows
261 * all of the sensible Xen configurations to work correctly.
269 static void *vring_alloc_queue(struct virtio_device *vdev, size_t size,
270 dma_addr_t *dma_handle, gfp_t flag)
272 if (vring_use_dma_api(vdev)) {
273 return dma_alloc_coherent(vdev->dev.parent, size,
276 void *queue = alloc_pages_exact(PAGE_ALIGN(size), flag);
279 phys_addr_t phys_addr = virt_to_phys(queue);
280 *dma_handle = (dma_addr_t)phys_addr;
283 * Sanity check: make sure we dind't truncate
284 * the address. The only arches I can find that
285 * have 64-bit phys_addr_t but 32-bit dma_addr_t
286 * are certain non-highmem MIPS and x86
287 * configurations, but these configurations
288 * should never allocate physical pages above 32
289 * bits, so this is fine. Just in case, throw a
290 * warning and abort if we end up with an
291 * unrepresentable address.
293 if (WARN_ON_ONCE(*dma_handle != phys_addr)) {
294 free_pages_exact(queue, PAGE_ALIGN(size));
302 static void vring_free_queue(struct virtio_device *vdev, size_t size,
303 void *queue, dma_addr_t dma_handle)
305 if (vring_use_dma_api(vdev))
306 dma_free_coherent(vdev->dev.parent, size, queue, dma_handle);
308 free_pages_exact(queue, PAGE_ALIGN(size));
312 * The DMA ops on various arches are rather gnarly right now, and
313 * making all of the arch DMA ops work on the vring device itself
314 * is a mess. For now, we use the parent device for DMA ops.
316 static inline struct device *vring_dma_dev(const struct vring_virtqueue *vq)
318 return vq->vq.vdev->dev.parent;
321 /* Map one sg entry. */
322 static dma_addr_t vring_map_one_sg(const struct vring_virtqueue *vq,
323 struct scatterlist *sg,
324 enum dma_data_direction direction)
326 if (!vq->use_dma_api)
327 return (dma_addr_t)sg_phys(sg);
330 * We can't use dma_map_sg, because we don't use scatterlists in
331 * the way it expects (we don't guarantee that the scatterlist
332 * will exist for the lifetime of the mapping).
334 return dma_map_page(vring_dma_dev(vq),
335 sg_page(sg), sg->offset, sg->length,
339 static dma_addr_t vring_map_single(const struct vring_virtqueue *vq,
340 void *cpu_addr, size_t size,
341 enum dma_data_direction direction)
343 if (!vq->use_dma_api)
344 return (dma_addr_t)virt_to_phys(cpu_addr);
346 return dma_map_single(vring_dma_dev(vq),
347 cpu_addr, size, direction);
350 static int vring_mapping_error(const struct vring_virtqueue *vq,
353 if (!vq->use_dma_api)
356 return dma_mapping_error(vring_dma_dev(vq), addr);
361 * Split ring specific functions - *_split().
364 static void vring_unmap_one_split(const struct vring_virtqueue *vq,
365 struct vring_desc *desc)
369 if (!vq->use_dma_api)
372 flags = virtio16_to_cpu(vq->vq.vdev, desc->flags);
374 if (flags & VRING_DESC_F_INDIRECT) {
375 dma_unmap_single(vring_dma_dev(vq),
376 virtio64_to_cpu(vq->vq.vdev, desc->addr),
377 virtio32_to_cpu(vq->vq.vdev, desc->len),
378 (flags & VRING_DESC_F_WRITE) ?
379 DMA_FROM_DEVICE : DMA_TO_DEVICE);
381 dma_unmap_page(vring_dma_dev(vq),
382 virtio64_to_cpu(vq->vq.vdev, desc->addr),
383 virtio32_to_cpu(vq->vq.vdev, desc->len),
384 (flags & VRING_DESC_F_WRITE) ?
385 DMA_FROM_DEVICE : DMA_TO_DEVICE);
389 static struct vring_desc *alloc_indirect_split(struct virtqueue *_vq,
390 unsigned int total_sg,
393 struct vring_desc *desc;
397 * We require lowmem mappings for the descriptors because
398 * otherwise virt_to_phys will give us bogus addresses in the
401 gfp &= ~__GFP_HIGHMEM;
403 desc = kmalloc_array(total_sg, sizeof(struct vring_desc), gfp);
407 for (i = 0; i < total_sg; i++)
408 desc[i].next = cpu_to_virtio16(_vq->vdev, i + 1);
412 static inline int virtqueue_add_split(struct virtqueue *_vq,
413 struct scatterlist *sgs[],
414 unsigned int total_sg,
415 unsigned int out_sgs,
421 struct vring_virtqueue *vq = to_vvq(_vq);
422 struct scatterlist *sg;
423 struct vring_desc *desc;
424 unsigned int i, n, avail, descs_used, uninitialized_var(prev), err_idx;
430 BUG_ON(data == NULL);
431 BUG_ON(ctx && vq->indirect);
433 if (unlikely(vq->broken)) {
438 LAST_ADD_TIME_UPDATE(vq);
440 BUG_ON(total_sg == 0);
442 head = vq->free_head;
444 if (virtqueue_use_indirect(_vq, total_sg))
445 desc = alloc_indirect_split(_vq, total_sg, gfp);
448 WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect);
452 /* Use a single buffer which doesn't continue */
454 /* Set up rest to use this indirect table. */
459 desc = vq->split.vring.desc;
461 descs_used = total_sg;
464 if (vq->vq.num_free < descs_used) {
465 pr_debug("Can't add buf len %i - avail = %i\n",
466 descs_used, vq->vq.num_free);
467 /* FIXME: for historical reasons, we force a notify here if
468 * there are outgoing parts to the buffer. Presumably the
469 * host should service the ring ASAP. */
478 for (n = 0; n < out_sgs; n++) {
479 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
480 dma_addr_t addr = vring_map_one_sg(vq, sg, DMA_TO_DEVICE);
481 if (vring_mapping_error(vq, addr))
484 desc[i].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_NEXT);
485 desc[i].addr = cpu_to_virtio64(_vq->vdev, addr);
486 desc[i].len = cpu_to_virtio32(_vq->vdev, sg->length);
488 i = virtio16_to_cpu(_vq->vdev, desc[i].next);
491 for (; n < (out_sgs + in_sgs); n++) {
492 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
493 dma_addr_t addr = vring_map_one_sg(vq, sg, DMA_FROM_DEVICE);
494 if (vring_mapping_error(vq, addr))
497 desc[i].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_NEXT | VRING_DESC_F_WRITE);
498 desc[i].addr = cpu_to_virtio64(_vq->vdev, addr);
499 desc[i].len = cpu_to_virtio32(_vq->vdev, sg->length);
501 i = virtio16_to_cpu(_vq->vdev, desc[i].next);
504 /* Last one doesn't continue. */
505 desc[prev].flags &= cpu_to_virtio16(_vq->vdev, ~VRING_DESC_F_NEXT);
508 /* Now that the indirect table is filled in, map it. */
509 dma_addr_t addr = vring_map_single(
510 vq, desc, total_sg * sizeof(struct vring_desc),
512 if (vring_mapping_error(vq, addr))
515 vq->split.vring.desc[head].flags = cpu_to_virtio16(_vq->vdev,
516 VRING_DESC_F_INDIRECT);
517 vq->split.vring.desc[head].addr = cpu_to_virtio64(_vq->vdev,
520 vq->split.vring.desc[head].len = cpu_to_virtio32(_vq->vdev,
521 total_sg * sizeof(struct vring_desc));
524 /* We're using some buffers from the free list. */
525 vq->vq.num_free -= descs_used;
527 /* Update free pointer */
529 vq->free_head = virtio16_to_cpu(_vq->vdev,
530 vq->split.vring.desc[head].next);
534 /* Store token and indirect buffer state. */
535 vq->split.desc_state[head].data = data;
537 vq->split.desc_state[head].indir_desc = desc;
539 vq->split.desc_state[head].indir_desc = ctx;
541 /* Put entry in available array (but don't update avail->idx until they
543 avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1);
544 vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);
546 /* Descriptors and available array need to be set before we expose the
547 * new available array entries. */
548 virtio_wmb(vq->weak_barriers);
549 vq->split.avail_idx_shadow++;
550 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
551 vq->split.avail_idx_shadow);
554 pr_debug("Added buffer head %i to %p\n", head, vq);
557 /* This is very unlikely, but theoretically possible. Kick
559 if (unlikely(vq->num_added == (1 << 16) - 1))
568 for (n = 0; n < total_sg; n++) {
571 vring_unmap_one_split(vq, &desc[i]);
572 i = virtio16_to_cpu(_vq->vdev, vq->split.vring.desc[i].next);
582 static bool virtqueue_kick_prepare_split(struct virtqueue *_vq)
584 struct vring_virtqueue *vq = to_vvq(_vq);
589 /* We need to expose available array entries before checking avail
591 virtio_mb(vq->weak_barriers);
593 old = vq->split.avail_idx_shadow - vq->num_added;
594 new = vq->split.avail_idx_shadow;
597 LAST_ADD_TIME_CHECK(vq);
598 LAST_ADD_TIME_INVALID(vq);
601 needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev,
602 vring_avail_event(&vq->split.vring)),
605 needs_kick = !(vq->split.vring.used->flags &
606 cpu_to_virtio16(_vq->vdev,
607 VRING_USED_F_NO_NOTIFY));
613 static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
617 __virtio16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT);
619 /* Clear data ptr. */
620 vq->split.desc_state[head].data = NULL;
622 /* Put back on free list: unmap first-level descriptors and find end */
625 while (vq->split.vring.desc[i].flags & nextflag) {
626 vring_unmap_one_split(vq, &vq->split.vring.desc[i]);
627 i = virtio16_to_cpu(vq->vq.vdev, vq->split.vring.desc[i].next);
631 vring_unmap_one_split(vq, &vq->split.vring.desc[i]);
632 vq->split.vring.desc[i].next = cpu_to_virtio16(vq->vq.vdev,
634 vq->free_head = head;
636 /* Plus final descriptor */
640 struct vring_desc *indir_desc =
641 vq->split.desc_state[head].indir_desc;
644 /* Free the indirect table, if any, now that it's unmapped. */
648 len = virtio32_to_cpu(vq->vq.vdev,
649 vq->split.vring.desc[head].len);
651 BUG_ON(!(vq->split.vring.desc[head].flags &
652 cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_INDIRECT)));
653 BUG_ON(len == 0 || len % sizeof(struct vring_desc));
655 for (j = 0; j < len / sizeof(struct vring_desc); j++)
656 vring_unmap_one_split(vq, &indir_desc[j]);
659 vq->split.desc_state[head].indir_desc = NULL;
661 *ctx = vq->split.desc_state[head].indir_desc;
665 static inline bool more_used_split(const struct vring_virtqueue *vq)
667 return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev,
668 vq->split.vring.used->idx);
671 static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq,
675 struct vring_virtqueue *vq = to_vvq(_vq);
682 if (unlikely(vq->broken)) {
687 if (!more_used_split(vq)) {
688 pr_debug("No more buffers in queue\n");
693 /* Only get used array entries after they have been exposed by host. */
694 virtio_rmb(vq->weak_barriers);
696 last_used = (vq->last_used_idx & (vq->split.vring.num - 1));
697 i = virtio32_to_cpu(_vq->vdev,
698 vq->split.vring.used->ring[last_used].id);
699 *len = virtio32_to_cpu(_vq->vdev,
700 vq->split.vring.used->ring[last_used].len);
702 if (unlikely(i >= vq->split.vring.num)) {
703 BAD_RING(vq, "id %u out of range\n", i);
706 if (unlikely(!vq->split.desc_state[i].data)) {
707 BAD_RING(vq, "id %u is not a head!\n", i);
711 /* detach_buf_split clears data, so grab it now. */
712 ret = vq->split.desc_state[i].data;
713 detach_buf_split(vq, i, ctx);
715 /* If we expect an interrupt for the next entry, tell host
716 * by writing event index and flush out the write before
717 * the read in the next get_buf call. */
718 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
719 virtio_store_mb(vq->weak_barriers,
720 &vring_used_event(&vq->split.vring),
721 cpu_to_virtio16(_vq->vdev, vq->last_used_idx));
723 LAST_ADD_TIME_INVALID(vq);
729 static void virtqueue_disable_cb_split(struct virtqueue *_vq)
731 struct vring_virtqueue *vq = to_vvq(_vq);
733 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
734 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
736 vq->split.vring.avail->flags =
737 cpu_to_virtio16(_vq->vdev,
738 vq->split.avail_flags_shadow);
742 static unsigned virtqueue_enable_cb_prepare_split(struct virtqueue *_vq)
744 struct vring_virtqueue *vq = to_vvq(_vq);
749 /* We optimistically turn back on interrupts, then check if there was
751 /* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to
752 * either clear the flags bit or point the event index at the next
753 * entry. Always do both to keep code simple. */
754 if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
755 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
757 vq->split.vring.avail->flags =
758 cpu_to_virtio16(_vq->vdev,
759 vq->split.avail_flags_shadow);
761 vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev,
762 last_used_idx = vq->last_used_idx);
764 return last_used_idx;
767 static bool virtqueue_poll_split(struct virtqueue *_vq, unsigned last_used_idx)
769 struct vring_virtqueue *vq = to_vvq(_vq);
771 return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev,
772 vq->split.vring.used->idx);
775 static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq)
777 struct vring_virtqueue *vq = to_vvq(_vq);
782 /* We optimistically turn back on interrupts, then check if there was
784 /* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to
785 * either clear the flags bit or point the event index at the next
786 * entry. Always update the event index to keep code simple. */
787 if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
788 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
790 vq->split.vring.avail->flags =
791 cpu_to_virtio16(_vq->vdev,
792 vq->split.avail_flags_shadow);
794 /* TODO: tune this threshold */
795 bufs = (u16)(vq->split.avail_idx_shadow - vq->last_used_idx) * 3 / 4;
797 virtio_store_mb(vq->weak_barriers,
798 &vring_used_event(&vq->split.vring),
799 cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs));
801 if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx)
802 - vq->last_used_idx) > bufs)) {
811 static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq)
813 struct vring_virtqueue *vq = to_vvq(_vq);
819 for (i = 0; i < vq->split.vring.num; i++) {
820 if (!vq->split.desc_state[i].data)
822 /* detach_buf_split clears data, so grab it now. */
823 buf = vq->split.desc_state[i].data;
824 detach_buf_split(vq, i, NULL);
825 vq->split.avail_idx_shadow--;
826 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
827 vq->split.avail_idx_shadow);
831 /* That should have freed everything. */
832 BUG_ON(vq->vq.num_free != vq->split.vring.num);
838 static struct virtqueue *vring_create_virtqueue_split(
841 unsigned int vring_align,
842 struct virtio_device *vdev,
846 bool (*notify)(struct virtqueue *),
847 void (*callback)(struct virtqueue *),
850 struct virtqueue *vq;
853 size_t queue_size_in_bytes;
856 /* We assume num is a power of 2. */
857 if (num & (num - 1)) {
858 dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num);
862 /* TODO: allocate each queue chunk individually */
863 for (; num && vring_size(num, vring_align) > PAGE_SIZE; num /= 2) {
864 queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
866 GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
875 /* Try to get a single page. You are my only hope! */
876 queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
877 &dma_addr, GFP_KERNEL|__GFP_ZERO);
882 queue_size_in_bytes = vring_size(num, vring_align);
883 vring_init(&vring, num, queue, vring_align);
885 vq = __vring_new_virtqueue(index, vring, vdev, weak_barriers, context,
886 notify, callback, name);
888 vring_free_queue(vdev, queue_size_in_bytes, queue,
893 to_vvq(vq)->split.queue_dma_addr = dma_addr;
894 to_vvq(vq)->split.queue_size_in_bytes = queue_size_in_bytes;
895 to_vvq(vq)->we_own_ring = true;
902 * Packed ring specific functions - *_packed().
905 static void vring_unmap_state_packed(const struct vring_virtqueue *vq,
906 struct vring_desc_extra_packed *state)
910 if (!vq->use_dma_api)
913 flags = state->flags;
915 if (flags & VRING_DESC_F_INDIRECT) {
916 dma_unmap_single(vring_dma_dev(vq),
917 state->addr, state->len,
918 (flags & VRING_DESC_F_WRITE) ?
919 DMA_FROM_DEVICE : DMA_TO_DEVICE);
921 dma_unmap_page(vring_dma_dev(vq),
922 state->addr, state->len,
923 (flags & VRING_DESC_F_WRITE) ?
924 DMA_FROM_DEVICE : DMA_TO_DEVICE);
928 static void vring_unmap_desc_packed(const struct vring_virtqueue *vq,
929 struct vring_packed_desc *desc)
933 if (!vq->use_dma_api)
936 flags = le16_to_cpu(desc->flags);
938 if (flags & VRING_DESC_F_INDIRECT) {
939 dma_unmap_single(vring_dma_dev(vq),
940 le64_to_cpu(desc->addr),
941 le32_to_cpu(desc->len),
942 (flags & VRING_DESC_F_WRITE) ?
943 DMA_FROM_DEVICE : DMA_TO_DEVICE);
945 dma_unmap_page(vring_dma_dev(vq),
946 le64_to_cpu(desc->addr),
947 le32_to_cpu(desc->len),
948 (flags & VRING_DESC_F_WRITE) ?
949 DMA_FROM_DEVICE : DMA_TO_DEVICE);
953 static struct vring_packed_desc *alloc_indirect_packed(unsigned int total_sg,
956 struct vring_packed_desc *desc;
959 * We require lowmem mappings for the descriptors because
960 * otherwise virt_to_phys will give us bogus addresses in the
963 gfp &= ~__GFP_HIGHMEM;
965 desc = kmalloc_array(total_sg, sizeof(struct vring_packed_desc), gfp);
970 static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
971 struct scatterlist *sgs[],
972 unsigned int total_sg,
973 unsigned int out_sgs,
978 struct vring_packed_desc *desc;
979 struct scatterlist *sg;
980 unsigned int i, n, err_idx;
984 head = vq->packed.next_avail_idx;
985 desc = alloc_indirect_packed(total_sg, gfp);
987 if (unlikely(vq->vq.num_free < 1)) {
988 pr_debug("Can't add buf len 1 - avail = 0\n");
995 BUG_ON(id == vq->packed.vring.num);
997 for (n = 0; n < out_sgs + in_sgs; n++) {
998 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
999 addr = vring_map_one_sg(vq, sg, n < out_sgs ?
1000 DMA_TO_DEVICE : DMA_FROM_DEVICE);
1001 if (vring_mapping_error(vq, addr))
1004 desc[i].flags = cpu_to_le16(n < out_sgs ?
1005 0 : VRING_DESC_F_WRITE);
1006 desc[i].addr = cpu_to_le64(addr);
1007 desc[i].len = cpu_to_le32(sg->length);
1012 /* Now that the indirect table is filled in, map it. */
1013 addr = vring_map_single(vq, desc,
1014 total_sg * sizeof(struct vring_packed_desc),
1016 if (vring_mapping_error(vq, addr))
1019 vq->packed.vring.desc[head].addr = cpu_to_le64(addr);
1020 vq->packed.vring.desc[head].len = cpu_to_le32(total_sg *
1021 sizeof(struct vring_packed_desc));
1022 vq->packed.vring.desc[head].id = cpu_to_le16(id);
1024 if (vq->use_dma_api) {
1025 vq->packed.desc_extra[id].addr = addr;
1026 vq->packed.desc_extra[id].len = total_sg *
1027 sizeof(struct vring_packed_desc);
1028 vq->packed.desc_extra[id].flags = VRING_DESC_F_INDIRECT |
1029 vq->packed.avail_used_flags;
1033 * A driver MUST NOT make the first descriptor in the list
1034 * available before all subsequent descriptors comprising
1035 * the list are made available.
1037 virtio_wmb(vq->weak_barriers);
1038 vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT |
1039 vq->packed.avail_used_flags);
1041 /* We're using some buffers from the free list. */
1042 vq->vq.num_free -= 1;
1044 /* Update free pointer */
1046 if (n >= vq->packed.vring.num) {
1048 vq->packed.avail_wrap_counter ^= 1;
1049 vq->packed.avail_used_flags ^=
1050 1 << VRING_PACKED_DESC_F_AVAIL |
1051 1 << VRING_PACKED_DESC_F_USED;
1053 vq->packed.next_avail_idx = n;
1054 vq->free_head = vq->packed.desc_state[id].next;
1056 /* Store token and indirect buffer state. */
1057 vq->packed.desc_state[id].num = 1;
1058 vq->packed.desc_state[id].data = data;
1059 vq->packed.desc_state[id].indir_desc = desc;
1060 vq->packed.desc_state[id].last = id;
1064 pr_debug("Added buffer head %i to %p\n", head, vq);
1072 for (i = 0; i < err_idx; i++)
1073 vring_unmap_desc_packed(vq, &desc[i]);
1081 static inline int virtqueue_add_packed(struct virtqueue *_vq,
1082 struct scatterlist *sgs[],
1083 unsigned int total_sg,
1084 unsigned int out_sgs,
1085 unsigned int in_sgs,
1090 struct vring_virtqueue *vq = to_vvq(_vq);
1091 struct vring_packed_desc *desc;
1092 struct scatterlist *sg;
1093 unsigned int i, n, c, descs_used, err_idx;
1094 __le16 uninitialized_var(head_flags), flags;
1095 u16 head, id, uninitialized_var(prev), curr, avail_used_flags;
1099 BUG_ON(data == NULL);
1100 BUG_ON(ctx && vq->indirect);
1102 if (unlikely(vq->broken)) {
1107 LAST_ADD_TIME_UPDATE(vq);
1109 BUG_ON(total_sg == 0);
1111 if (virtqueue_use_indirect(_vq, total_sg))
1112 return virtqueue_add_indirect_packed(vq, sgs, total_sg,
1113 out_sgs, in_sgs, data, gfp);
1115 head = vq->packed.next_avail_idx;
1116 avail_used_flags = vq->packed.avail_used_flags;
1118 WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);
1120 desc = vq->packed.vring.desc;
1122 descs_used = total_sg;
1124 if (unlikely(vq->vq.num_free < descs_used)) {
1125 pr_debug("Can't add buf len %i - avail = %i\n",
1126 descs_used, vq->vq.num_free);
1132 BUG_ON(id == vq->packed.vring.num);
1136 for (n = 0; n < out_sgs + in_sgs; n++) {
1137 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
1138 dma_addr_t addr = vring_map_one_sg(vq, sg, n < out_sgs ?
1139 DMA_TO_DEVICE : DMA_FROM_DEVICE);
1140 if (vring_mapping_error(vq, addr))
1143 flags = cpu_to_le16(vq->packed.avail_used_flags |
1144 (++c == total_sg ? 0 : VRING_DESC_F_NEXT) |
1145 (n < out_sgs ? 0 : VRING_DESC_F_WRITE));
1149 desc[i].flags = flags;
1151 desc[i].addr = cpu_to_le64(addr);
1152 desc[i].len = cpu_to_le32(sg->length);
1153 desc[i].id = cpu_to_le16(id);
1155 if (unlikely(vq->use_dma_api)) {
1156 vq->packed.desc_extra[curr].addr = addr;
1157 vq->packed.desc_extra[curr].len = sg->length;
1158 vq->packed.desc_extra[curr].flags =
1162 curr = vq->packed.desc_state[curr].next;
1164 if ((unlikely(++i >= vq->packed.vring.num))) {
1166 vq->packed.avail_used_flags ^=
1167 1 << VRING_PACKED_DESC_F_AVAIL |
1168 1 << VRING_PACKED_DESC_F_USED;
1174 vq->packed.avail_wrap_counter ^= 1;
1176 /* We're using some buffers from the free list. */
1177 vq->vq.num_free -= descs_used;
1179 /* Update free pointer */
1180 vq->packed.next_avail_idx = i;
1181 vq->free_head = curr;
1184 vq->packed.desc_state[id].num = descs_used;
1185 vq->packed.desc_state[id].data = data;
1186 vq->packed.desc_state[id].indir_desc = ctx;
1187 vq->packed.desc_state[id].last = prev;
1190 * A driver MUST NOT make the first descriptor in the list
1191 * available before all subsequent descriptors comprising
1192 * the list are made available.
1194 virtio_wmb(vq->weak_barriers);
1195 vq->packed.vring.desc[head].flags = head_flags;
1196 vq->num_added += descs_used;
1198 pr_debug("Added buffer head %i to %p\n", head, vq);
1207 vq->packed.avail_used_flags = avail_used_flags;
1209 for (n = 0; n < total_sg; n++) {
1212 vring_unmap_desc_packed(vq, &desc[i]);
1214 if (i >= vq->packed.vring.num)
1222 static bool virtqueue_kick_prepare_packed(struct virtqueue *_vq)
1224 struct vring_virtqueue *vq = to_vvq(_vq);
1225 u16 new, old, off_wrap, flags, wrap_counter, event_idx;
1238 * We need to expose the new flags value before checking notification
1241 virtio_mb(vq->weak_barriers);
1243 old = vq->packed.next_avail_idx - vq->num_added;
1244 new = vq->packed.next_avail_idx;
1247 snapshot.u32 = *(u32 *)vq->packed.vring.device;
1248 flags = le16_to_cpu(snapshot.flags);
1250 LAST_ADD_TIME_CHECK(vq);
1251 LAST_ADD_TIME_INVALID(vq);
1253 if (flags != VRING_PACKED_EVENT_FLAG_DESC) {
1254 needs_kick = (flags != VRING_PACKED_EVENT_FLAG_DISABLE);
1258 off_wrap = le16_to_cpu(snapshot.off_wrap);
1260 wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR;
1261 event_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR);
1262 if (wrap_counter != vq->packed.avail_wrap_counter)
1263 event_idx -= vq->packed.vring.num;
1265 needs_kick = vring_need_event(event_idx, new, old);
1271 static void detach_buf_packed(struct vring_virtqueue *vq,
1272 unsigned int id, void **ctx)
1274 struct vring_desc_state_packed *state = NULL;
1275 struct vring_packed_desc *desc;
1276 unsigned int i, curr;
1278 state = &vq->packed.desc_state[id];
1280 /* Clear data ptr. */
1283 vq->packed.desc_state[state->last].next = vq->free_head;
1285 vq->vq.num_free += state->num;
1287 if (unlikely(vq->use_dma_api)) {
1289 for (i = 0; i < state->num; i++) {
1290 vring_unmap_state_packed(vq,
1291 &vq->packed.desc_extra[curr]);
1292 curr = vq->packed.desc_state[curr].next;
1299 /* Free the indirect table, if any, now that it's unmapped. */
1300 desc = state->indir_desc;
1304 if (vq->use_dma_api) {
1305 len = vq->packed.desc_extra[id].len;
1306 for (i = 0; i < len / sizeof(struct vring_packed_desc);
1308 vring_unmap_desc_packed(vq, &desc[i]);
1311 state->indir_desc = NULL;
1313 *ctx = state->indir_desc;
1317 static inline bool is_used_desc_packed(const struct vring_virtqueue *vq,
1318 u16 idx, bool used_wrap_counter)
1323 flags = le16_to_cpu(vq->packed.vring.desc[idx].flags);
1324 avail = !!(flags & (1 << VRING_PACKED_DESC_F_AVAIL));
1325 used = !!(flags & (1 << VRING_PACKED_DESC_F_USED));
1327 return avail == used && used == used_wrap_counter;
1330 static inline bool more_used_packed(const struct vring_virtqueue *vq)
1332 return is_used_desc_packed(vq, vq->last_used_idx,
1333 vq->packed.used_wrap_counter);
1336 static void *virtqueue_get_buf_ctx_packed(struct virtqueue *_vq,
1340 struct vring_virtqueue *vq = to_vvq(_vq);
1346 if (unlikely(vq->broken)) {
1351 if (!more_used_packed(vq)) {
1352 pr_debug("No more buffers in queue\n");
1357 /* Only get used elements after they have been exposed by host. */
1358 virtio_rmb(vq->weak_barriers);
1360 last_used = vq->last_used_idx;
1361 id = le16_to_cpu(vq->packed.vring.desc[last_used].id);
1362 *len = le32_to_cpu(vq->packed.vring.desc[last_used].len);
1364 if (unlikely(id >= vq->packed.vring.num)) {
1365 BAD_RING(vq, "id %u out of range\n", id);
1368 if (unlikely(!vq->packed.desc_state[id].data)) {
1369 BAD_RING(vq, "id %u is not a head!\n", id);
1373 /* detach_buf_packed clears data, so grab it now. */
1374 ret = vq->packed.desc_state[id].data;
1375 detach_buf_packed(vq, id, ctx);
1377 vq->last_used_idx += vq->packed.desc_state[id].num;
1378 if (unlikely(vq->last_used_idx >= vq->packed.vring.num)) {
1379 vq->last_used_idx -= vq->packed.vring.num;
1380 vq->packed.used_wrap_counter ^= 1;
1384 * If we expect an interrupt for the next entry, tell host
1385 * by writing event index and flush out the write before
1386 * the read in the next get_buf call.
1388 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC)
1389 virtio_store_mb(vq->weak_barriers,
1390 &vq->packed.vring.driver->off_wrap,
1391 cpu_to_le16(vq->last_used_idx |
1392 (vq->packed.used_wrap_counter <<
1393 VRING_PACKED_EVENT_F_WRAP_CTR)));
1395 LAST_ADD_TIME_INVALID(vq);
1401 static void virtqueue_disable_cb_packed(struct virtqueue *_vq)
1403 struct vring_virtqueue *vq = to_vvq(_vq);
1405 if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) {
1406 vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
1407 vq->packed.vring.driver->flags =
1408 cpu_to_le16(vq->packed.event_flags_shadow);
1412 static unsigned virtqueue_enable_cb_prepare_packed(struct virtqueue *_vq)
1414 struct vring_virtqueue *vq = to_vvq(_vq);
1419 * We optimistically turn back on interrupts, then check if there was
1424 vq->packed.vring.driver->off_wrap =
1425 cpu_to_le16(vq->last_used_idx |
1426 (vq->packed.used_wrap_counter <<
1427 VRING_PACKED_EVENT_F_WRAP_CTR));
1429 * We need to update event offset and event wrap
1430 * counter first before updating event flags.
1432 virtio_wmb(vq->weak_barriers);
1435 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
1436 vq->packed.event_flags_shadow = vq->event ?
1437 VRING_PACKED_EVENT_FLAG_DESC :
1438 VRING_PACKED_EVENT_FLAG_ENABLE;
1439 vq->packed.vring.driver->flags =
1440 cpu_to_le16(vq->packed.event_flags_shadow);
1444 return vq->last_used_idx | ((u16)vq->packed.used_wrap_counter <<
1445 VRING_PACKED_EVENT_F_WRAP_CTR);
1448 static bool virtqueue_poll_packed(struct virtqueue *_vq, u16 off_wrap)
1450 struct vring_virtqueue *vq = to_vvq(_vq);
1454 wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR;
1455 used_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR);
1457 return is_used_desc_packed(vq, used_idx, wrap_counter);
1460 static bool virtqueue_enable_cb_delayed_packed(struct virtqueue *_vq)
1462 struct vring_virtqueue *vq = to_vvq(_vq);
1463 u16 used_idx, wrap_counter;
1469 * We optimistically turn back on interrupts, then check if there was
1474 /* TODO: tune this threshold */
1475 bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4;
1476 wrap_counter = vq->packed.used_wrap_counter;
1478 used_idx = vq->last_used_idx + bufs;
1479 if (used_idx >= vq->packed.vring.num) {
1480 used_idx -= vq->packed.vring.num;
1484 vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx |
1485 (wrap_counter << VRING_PACKED_EVENT_F_WRAP_CTR));
1488 * We need to update event offset and event wrap
1489 * counter first before updating event flags.
1491 virtio_wmb(vq->weak_barriers);
1493 used_idx = vq->last_used_idx;
1494 wrap_counter = vq->packed.used_wrap_counter;
1497 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
1498 vq->packed.event_flags_shadow = vq->event ?
1499 VRING_PACKED_EVENT_FLAG_DESC :
1500 VRING_PACKED_EVENT_FLAG_ENABLE;
1501 vq->packed.vring.driver->flags =
1502 cpu_to_le16(vq->packed.event_flags_shadow);
1506 * We need to update event suppression structure first
1507 * before re-checking for more used buffers.
1509 virtio_mb(vq->weak_barriers);
1511 if (is_used_desc_packed(vq, used_idx, wrap_counter)) {
1520 static void *virtqueue_detach_unused_buf_packed(struct virtqueue *_vq)
1522 struct vring_virtqueue *vq = to_vvq(_vq);
1528 for (i = 0; i < vq->packed.vring.num; i++) {
1529 if (!vq->packed.desc_state[i].data)
1531 /* detach_buf clears data, so grab it now. */
1532 buf = vq->packed.desc_state[i].data;
1533 detach_buf_packed(vq, i, NULL);
1537 /* That should have freed everything. */
1538 BUG_ON(vq->vq.num_free != vq->packed.vring.num);
1544 static struct virtqueue *vring_create_virtqueue_packed(
1547 unsigned int vring_align,
1548 struct virtio_device *vdev,
1550 bool may_reduce_num,
1552 bool (*notify)(struct virtqueue *),
1553 void (*callback)(struct virtqueue *),
1556 struct vring_virtqueue *vq;
1557 struct vring_packed_desc *ring;
1558 struct vring_packed_desc_event *driver, *device;
1559 dma_addr_t ring_dma_addr, driver_event_dma_addr, device_event_dma_addr;
1560 size_t ring_size_in_bytes, event_size_in_bytes;
1563 ring_size_in_bytes = num * sizeof(struct vring_packed_desc);
1565 ring = vring_alloc_queue(vdev, ring_size_in_bytes,
1567 GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
1571 event_size_in_bytes = sizeof(struct vring_packed_desc_event);
1573 driver = vring_alloc_queue(vdev, event_size_in_bytes,
1574 &driver_event_dma_addr,
1575 GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
1579 device = vring_alloc_queue(vdev, event_size_in_bytes,
1580 &device_event_dma_addr,
1581 GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
1585 vq = kmalloc(sizeof(*vq), GFP_KERNEL);
1589 vq->vq.callback = callback;
1592 vq->vq.num_free = num;
1593 vq->vq.index = index;
1594 vq->we_own_ring = true;
1595 vq->notify = notify;
1596 vq->weak_barriers = weak_barriers;
1598 vq->last_used_idx = 0;
1600 vq->packed_ring = true;
1601 vq->use_dma_api = vring_use_dma_api(vdev);
1602 list_add_tail(&vq->vq.list, &vdev->vqs);
1605 vq->last_add_time_valid = false;
1608 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
1610 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
1612 vq->packed.ring_dma_addr = ring_dma_addr;
1613 vq->packed.driver_event_dma_addr = driver_event_dma_addr;
1614 vq->packed.device_event_dma_addr = device_event_dma_addr;
1616 vq->packed.ring_size_in_bytes = ring_size_in_bytes;
1617 vq->packed.event_size_in_bytes = event_size_in_bytes;
1619 vq->packed.vring.num = num;
1620 vq->packed.vring.desc = ring;
1621 vq->packed.vring.driver = driver;
1622 vq->packed.vring.device = device;
1624 vq->packed.next_avail_idx = 0;
1625 vq->packed.avail_wrap_counter = 1;
1626 vq->packed.used_wrap_counter = 1;
1627 vq->packed.event_flags_shadow = 0;
1628 vq->packed.avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL;
1630 vq->packed.desc_state = kmalloc_array(num,
1631 sizeof(struct vring_desc_state_packed),
1633 if (!vq->packed.desc_state)
1634 goto err_desc_state;
1636 memset(vq->packed.desc_state, 0,
1637 num * sizeof(struct vring_desc_state_packed));
1639 /* Put everything in free lists. */
1641 for (i = 0; i < num-1; i++)
1642 vq->packed.desc_state[i].next = i + 1;
1644 vq->packed.desc_extra = kmalloc_array(num,
1645 sizeof(struct vring_desc_extra_packed),
1647 if (!vq->packed.desc_extra)
1648 goto err_desc_extra;
1650 memset(vq->packed.desc_extra, 0,
1651 num * sizeof(struct vring_desc_extra_packed));
1653 /* No callback? Tell other side not to bother us. */
1655 vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
1656 vq->packed.vring.driver->flags =
1657 cpu_to_le16(vq->packed.event_flags_shadow);
1663 kfree(vq->packed.desc_state);
1667 vring_free_queue(vdev, event_size_in_bytes, device, ring_dma_addr);
1669 vring_free_queue(vdev, event_size_in_bytes, driver, ring_dma_addr);
1671 vring_free_queue(vdev, ring_size_in_bytes, ring, ring_dma_addr);
1678 * Generic functions and exported symbols.
1681 static inline int virtqueue_add(struct virtqueue *_vq,
1682 struct scatterlist *sgs[],
1683 unsigned int total_sg,
1684 unsigned int out_sgs,
1685 unsigned int in_sgs,
1690 struct vring_virtqueue *vq = to_vvq(_vq);
1692 return vq->packed_ring ? virtqueue_add_packed(_vq, sgs, total_sg,
1693 out_sgs, in_sgs, data, ctx, gfp) :
1694 virtqueue_add_split(_vq, sgs, total_sg,
1695 out_sgs, in_sgs, data, ctx, gfp);
1699 * virtqueue_add_sgs - expose buffers to other end
1700 * @vq: the struct virtqueue we're talking about.
1701 * @sgs: array of terminated scatterlists.
1702 * @out_num: the number of scatterlists readable by other side
1703 * @in_num: the number of scatterlists which are writable (after readable ones)
1704 * @data: the token identifying the buffer.
1705 * @gfp: how to do memory allocations (if necessary).
1707 * Caller must ensure we don't call this with other virtqueue operations
1708 * at the same time (except where noted).
1710 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1712 int virtqueue_add_sgs(struct virtqueue *_vq,
1713 struct scatterlist *sgs[],
1714 unsigned int out_sgs,
1715 unsigned int in_sgs,
1719 unsigned int i, total_sg = 0;
1721 /* Count them first. */
1722 for (i = 0; i < out_sgs + in_sgs; i++) {
1723 struct scatterlist *sg;
1725 for (sg = sgs[i]; sg; sg = sg_next(sg))
1728 return virtqueue_add(_vq, sgs, total_sg, out_sgs, in_sgs,
1731 EXPORT_SYMBOL_GPL(virtqueue_add_sgs);
1734 * virtqueue_add_outbuf - expose output buffers to other end
1735 * @vq: the struct virtqueue we're talking about.
1736 * @sg: scatterlist (must be well-formed and terminated!)
1737 * @num: the number of entries in @sg readable by other side
1738 * @data: the token identifying the buffer.
1739 * @gfp: how to do memory allocations (if necessary).
1741 * Caller must ensure we don't call this with other virtqueue operations
1742 * at the same time (except where noted).
1744 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1746 int virtqueue_add_outbuf(struct virtqueue *vq,
1747 struct scatterlist *sg, unsigned int num,
1751 return virtqueue_add(vq, &sg, num, 1, 0, data, NULL, gfp);
1753 EXPORT_SYMBOL_GPL(virtqueue_add_outbuf);
1756 * virtqueue_add_inbuf - expose input buffers to other end
1757 * @vq: the struct virtqueue we're talking about.
1758 * @sg: scatterlist (must be well-formed and terminated!)
1759 * @num: the number of entries in @sg writable by other side
1760 * @data: the token identifying the buffer.
1761 * @gfp: how to do memory allocations (if necessary).
1763 * Caller must ensure we don't call this with other virtqueue operations
1764 * at the same time (except where noted).
1766 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1768 int virtqueue_add_inbuf(struct virtqueue *vq,
1769 struct scatterlist *sg, unsigned int num,
1773 return virtqueue_add(vq, &sg, num, 0, 1, data, NULL, gfp);
1775 EXPORT_SYMBOL_GPL(virtqueue_add_inbuf);
1778 * virtqueue_add_inbuf_ctx - expose input buffers to other end
1779 * @vq: the struct virtqueue we're talking about.
1780 * @sg: scatterlist (must be well-formed and terminated!)
1781 * @num: the number of entries in @sg writable by other side
1782 * @data: the token identifying the buffer.
1783 * @ctx: extra context for the token
1784 * @gfp: how to do memory allocations (if necessary).
1786 * Caller must ensure we don't call this with other virtqueue operations
1787 * at the same time (except where noted).
1789 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1791 int virtqueue_add_inbuf_ctx(struct virtqueue *vq,
1792 struct scatterlist *sg, unsigned int num,
1797 return virtqueue_add(vq, &sg, num, 0, 1, data, ctx, gfp);
1799 EXPORT_SYMBOL_GPL(virtqueue_add_inbuf_ctx);
1802 * virtqueue_kick_prepare - first half of split virtqueue_kick call.
1803 * @vq: the struct virtqueue
1805 * Instead of virtqueue_kick(), you can do:
1806 * if (virtqueue_kick_prepare(vq))
1807 * virtqueue_notify(vq);
1809 * This is sometimes useful because the virtqueue_kick_prepare() needs
1810 * to be serialized, but the actual virtqueue_notify() call does not.
1812 bool virtqueue_kick_prepare(struct virtqueue *_vq)
1814 struct vring_virtqueue *vq = to_vvq(_vq);
1816 return vq->packed_ring ? virtqueue_kick_prepare_packed(_vq) :
1817 virtqueue_kick_prepare_split(_vq);
1819 EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
1822 * virtqueue_notify - second half of split virtqueue_kick call.
1823 * @vq: the struct virtqueue
1825 * This does not need to be serialized.
1827 * Returns false if host notify failed or queue is broken, otherwise true.
1829 bool virtqueue_notify(struct virtqueue *_vq)
1831 struct vring_virtqueue *vq = to_vvq(_vq);
1833 if (unlikely(vq->broken))
1836 /* Prod other side to tell it about changes. */
1837 if (!vq->notify(_vq)) {
1843 EXPORT_SYMBOL_GPL(virtqueue_notify);
1846 * virtqueue_kick - update after add_buf
1847 * @vq: the struct virtqueue
1849 * After one or more virtqueue_add_* calls, invoke this to kick
1852 * Caller must ensure we don't call this with other virtqueue
1853 * operations at the same time (except where noted).
1855 * Returns false if kick failed, otherwise true.
1857 bool virtqueue_kick(struct virtqueue *vq)
1859 if (virtqueue_kick_prepare(vq))
1860 return virtqueue_notify(vq);
1863 EXPORT_SYMBOL_GPL(virtqueue_kick);
1866 * virtqueue_get_buf - get the next used buffer
1867 * @vq: the struct virtqueue we're talking about.
1868 * @len: the length written into the buffer
1870 * If the device wrote data into the buffer, @len will be set to the
1871 * amount written. This means you don't need to clear the buffer
1872 * beforehand to ensure there's no data leakage in the case of short
1875 * Caller must ensure we don't call this with other virtqueue
1876 * operations at the same time (except where noted).
1878 * Returns NULL if there are no used buffers, or the "data" token
1879 * handed to virtqueue_add_*().
1881 void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,
1884 struct vring_virtqueue *vq = to_vvq(_vq);
1886 return vq->packed_ring ? virtqueue_get_buf_ctx_packed(_vq, len, ctx) :
1887 virtqueue_get_buf_ctx_split(_vq, len, ctx);
1889 EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx);
1891 void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
1893 return virtqueue_get_buf_ctx(_vq, len, NULL);
1895 EXPORT_SYMBOL_GPL(virtqueue_get_buf);
1897 * virtqueue_disable_cb - disable callbacks
1898 * @vq: the struct virtqueue we're talking about.
1900 * Note that this is not necessarily synchronous, hence unreliable and only
1901 * useful as an optimization.
1903 * Unlike other operations, this need not be serialized.
1905 void virtqueue_disable_cb(struct virtqueue *_vq)
1907 struct vring_virtqueue *vq = to_vvq(_vq);
1909 if (vq->packed_ring)
1910 virtqueue_disable_cb_packed(_vq);
1912 virtqueue_disable_cb_split(_vq);
1914 EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
1917 * virtqueue_enable_cb_prepare - restart callbacks after disable_cb
1918 * @vq: the struct virtqueue we're talking about.
1920 * This re-enables callbacks; it returns current queue state
1921 * in an opaque unsigned value. This value should be later tested by
1922 * virtqueue_poll, to detect a possible race between the driver checking for
1923 * more work, and enabling callbacks.
1925 * Caller must ensure we don't call this with other virtqueue
1926 * operations at the same time (except where noted).
1928 unsigned virtqueue_enable_cb_prepare(struct virtqueue *_vq)
1930 struct vring_virtqueue *vq = to_vvq(_vq);
1932 return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(_vq) :
1933 virtqueue_enable_cb_prepare_split(_vq);
1935 EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
1938 * virtqueue_poll - query pending used buffers
1939 * @vq: the struct virtqueue we're talking about.
1940 * @last_used_idx: virtqueue state (from call to virtqueue_enable_cb_prepare).
1942 * Returns "true" if there are pending used buffers in the queue.
1944 * This does not need to be serialized.
1946 bool virtqueue_poll(struct virtqueue *_vq, unsigned last_used_idx)
1948 struct vring_virtqueue *vq = to_vvq(_vq);
1950 virtio_mb(vq->weak_barriers);
1951 return vq->packed_ring ? virtqueue_poll_packed(_vq, last_used_idx) :
1952 virtqueue_poll_split(_vq, last_used_idx);
1954 EXPORT_SYMBOL_GPL(virtqueue_poll);
1957 * virtqueue_enable_cb - restart callbacks after disable_cb.
1958 * @vq: the struct virtqueue we're talking about.
1960 * This re-enables callbacks; it returns "false" if there are pending
1961 * buffers in the queue, to detect a possible race between the driver
1962 * checking for more work, and enabling callbacks.
1964 * Caller must ensure we don't call this with other virtqueue
1965 * operations at the same time (except where noted).
1967 bool virtqueue_enable_cb(struct virtqueue *_vq)
1969 unsigned last_used_idx = virtqueue_enable_cb_prepare(_vq);
1971 return !virtqueue_poll(_vq, last_used_idx);
1973 EXPORT_SYMBOL_GPL(virtqueue_enable_cb);
1976 * virtqueue_enable_cb_delayed - restart callbacks after disable_cb.
1977 * @vq: the struct virtqueue we're talking about.
1979 * This re-enables callbacks but hints to the other side to delay
1980 * interrupts until most of the available buffers have been processed;
1981 * it returns "false" if there are many pending buffers in the queue,
1982 * to detect a possible race between the driver checking for more work,
1983 * and enabling callbacks.
1985 * Caller must ensure we don't call this with other virtqueue
1986 * operations at the same time (except where noted).
1988 bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
1990 struct vring_virtqueue *vq = to_vvq(_vq);
1992 return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(_vq) :
1993 virtqueue_enable_cb_delayed_split(_vq);
1995 EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
1998 * virtqueue_detach_unused_buf - detach first unused buffer
1999 * @vq: the struct virtqueue we're talking about.
2001 * Returns NULL or the "data" token handed to virtqueue_add_*().
2002 * This is not valid on an active queue; it is useful only for device
2005 void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
2007 struct vring_virtqueue *vq = to_vvq(_vq);
2009 return vq->packed_ring ? virtqueue_detach_unused_buf_packed(_vq) :
2010 virtqueue_detach_unused_buf_split(_vq);
2012 EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
2014 static inline bool more_used(const struct vring_virtqueue *vq)
2016 return vq->packed_ring ? more_used_packed(vq) : more_used_split(vq);
2019 irqreturn_t vring_interrupt(int irq, void *_vq)
2021 struct vring_virtqueue *vq = to_vvq(_vq);
2023 if (!more_used(vq)) {
2024 pr_debug("virtqueue interrupt with no work for %p\n", vq);
2028 if (unlikely(vq->broken))
2031 pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback);
2032 if (vq->vq.callback)
2033 vq->vq.callback(&vq->vq);
2037 EXPORT_SYMBOL_GPL(vring_interrupt);
2039 /* Only available for split ring */
2040 struct virtqueue *__vring_new_virtqueue(unsigned int index,
2042 struct virtio_device *vdev,
2045 bool (*notify)(struct virtqueue *),
2046 void (*callback)(struct virtqueue *),
2050 struct vring_virtqueue *vq;
2052 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2055 vq = kmalloc(sizeof(*vq), GFP_KERNEL);
2059 vq->packed_ring = false;
2060 vq->vq.callback = callback;
2063 vq->vq.num_free = vring.num;
2064 vq->vq.index = index;
2065 vq->we_own_ring = false;
2066 vq->notify = notify;
2067 vq->weak_barriers = weak_barriers;
2069 vq->last_used_idx = 0;
2071 vq->use_dma_api = vring_use_dma_api(vdev);
2072 list_add_tail(&vq->vq.list, &vdev->vqs);
2075 vq->last_add_time_valid = false;
2078 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
2080 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
2082 vq->split.queue_dma_addr = 0;
2083 vq->split.queue_size_in_bytes = 0;
2085 vq->split.vring = vring;
2086 vq->split.avail_flags_shadow = 0;
2087 vq->split.avail_idx_shadow = 0;
2089 /* No callback? Tell other side not to bother us. */
2091 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
2093 vq->split.vring.avail->flags = cpu_to_virtio16(vdev,
2094 vq->split.avail_flags_shadow);
2097 vq->split.desc_state = kmalloc_array(vring.num,
2098 sizeof(struct vring_desc_state_split), GFP_KERNEL);
2099 if (!vq->split.desc_state) {
2104 /* Put everything in free lists. */
2106 for (i = 0; i < vring.num-1; i++)
2107 vq->split.vring.desc[i].next = cpu_to_virtio16(vdev, i + 1);
2108 memset(vq->split.desc_state, 0, vring.num *
2109 sizeof(struct vring_desc_state_split));
2113 EXPORT_SYMBOL_GPL(__vring_new_virtqueue);
2115 struct virtqueue *vring_create_virtqueue(
2118 unsigned int vring_align,
2119 struct virtio_device *vdev,
2121 bool may_reduce_num,
2123 bool (*notify)(struct virtqueue *),
2124 void (*callback)(struct virtqueue *),
2128 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2129 return vring_create_virtqueue_packed(index, num, vring_align,
2130 vdev, weak_barriers, may_reduce_num,
2131 context, notify, callback, name);
2133 return vring_create_virtqueue_split(index, num, vring_align,
2134 vdev, weak_barriers, may_reduce_num,
2135 context, notify, callback, name);
2137 EXPORT_SYMBOL_GPL(vring_create_virtqueue);
2139 /* Only available for split ring */
2140 struct virtqueue *vring_new_virtqueue(unsigned int index,
2142 unsigned int vring_align,
2143 struct virtio_device *vdev,
2147 bool (*notify)(struct virtqueue *vq),
2148 void (*callback)(struct virtqueue *vq),
2153 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2156 vring_init(&vring, num, pages, vring_align);
2157 return __vring_new_virtqueue(index, vring, vdev, weak_barriers, context,
2158 notify, callback, name);
2160 EXPORT_SYMBOL_GPL(vring_new_virtqueue);
2162 void vring_del_virtqueue(struct virtqueue *_vq)
2164 struct vring_virtqueue *vq = to_vvq(_vq);
2166 if (vq->we_own_ring) {
2167 if (vq->packed_ring) {
2168 vring_free_queue(vq->vq.vdev,
2169 vq->packed.ring_size_in_bytes,
2170 vq->packed.vring.desc,
2171 vq->packed.ring_dma_addr);
2173 vring_free_queue(vq->vq.vdev,
2174 vq->packed.event_size_in_bytes,
2175 vq->packed.vring.driver,
2176 vq->packed.driver_event_dma_addr);
2178 vring_free_queue(vq->vq.vdev,
2179 vq->packed.event_size_in_bytes,
2180 vq->packed.vring.device,
2181 vq->packed.device_event_dma_addr);
2183 kfree(vq->packed.desc_state);
2184 kfree(vq->packed.desc_extra);
2186 vring_free_queue(vq->vq.vdev,
2187 vq->split.queue_size_in_bytes,
2188 vq->split.vring.desc,
2189 vq->split.queue_dma_addr);
2191 kfree(vq->split.desc_state);
2194 list_del(&_vq->list);
2197 EXPORT_SYMBOL_GPL(vring_del_virtqueue);
2199 /* Manipulates transport-specific feature bits. */
2200 void vring_transport_features(struct virtio_device *vdev)
2204 for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) {
2206 case VIRTIO_RING_F_INDIRECT_DESC:
2208 case VIRTIO_RING_F_EVENT_IDX:
2210 case VIRTIO_F_VERSION_1:
2212 case VIRTIO_F_IOMMU_PLATFORM:
2214 case VIRTIO_F_RING_PACKED:
2217 /* We don't understand this bit. */
2218 __virtio_clear_bit(vdev, i);
2222 EXPORT_SYMBOL_GPL(vring_transport_features);
2225 * virtqueue_get_vring_size - return the size of the virtqueue's vring
2226 * @vq: the struct virtqueue containing the vring of interest.
2228 * Returns the size of the vring. This is mainly used for boasting to
2229 * userspace. Unlike other operations, this need not be serialized.
2231 unsigned int virtqueue_get_vring_size(struct virtqueue *_vq)
2234 struct vring_virtqueue *vq = to_vvq(_vq);
2236 return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;
2238 EXPORT_SYMBOL_GPL(virtqueue_get_vring_size);
2240 bool virtqueue_is_broken(struct virtqueue *_vq)
2242 struct vring_virtqueue *vq = to_vvq(_vq);
2246 EXPORT_SYMBOL_GPL(virtqueue_is_broken);
2249 * This should prevent the device from being used, allowing drivers to
2250 * recover. You may need to grab appropriate locks to flush.
2252 void virtio_break_device(struct virtio_device *dev)
2254 struct virtqueue *_vq;
2256 list_for_each_entry(_vq, &dev->vqs, list) {
2257 struct vring_virtqueue *vq = to_vvq(_vq);
2261 EXPORT_SYMBOL_GPL(virtio_break_device);
2263 dma_addr_t virtqueue_get_desc_addr(struct virtqueue *_vq)
2265 struct vring_virtqueue *vq = to_vvq(_vq);
2267 BUG_ON(!vq->we_own_ring);
2269 if (vq->packed_ring)
2270 return vq->packed.ring_dma_addr;
2272 return vq->split.queue_dma_addr;
2274 EXPORT_SYMBOL_GPL(virtqueue_get_desc_addr);
2276 dma_addr_t virtqueue_get_avail_addr(struct virtqueue *_vq)
2278 struct vring_virtqueue *vq = to_vvq(_vq);
2280 BUG_ON(!vq->we_own_ring);
2282 if (vq->packed_ring)
2283 return vq->packed.driver_event_dma_addr;
2285 return vq->split.queue_dma_addr +
2286 ((char *)vq->split.vring.avail - (char *)vq->split.vring.desc);
2288 EXPORT_SYMBOL_GPL(virtqueue_get_avail_addr);
2290 dma_addr_t virtqueue_get_used_addr(struct virtqueue *_vq)
2292 struct vring_virtqueue *vq = to_vvq(_vq);
2294 BUG_ON(!vq->we_own_ring);
2296 if (vq->packed_ring)
2297 return vq->packed.device_event_dma_addr;
2299 return vq->split.queue_dma_addr +
2300 ((char *)vq->split.vring.used - (char *)vq->split.vring.desc);
2302 EXPORT_SYMBOL_GPL(virtqueue_get_used_addr);
2304 /* Only available for split ring */
2305 const struct vring *virtqueue_get_vring(struct virtqueue *vq)
2307 return &to_vvq(vq)->split.vring;
2309 EXPORT_SYMBOL_GPL(virtqueue_get_vring);
2311 MODULE_LICENSE("GPL");