drivers/virtio/virtio_ring.c

   1 /* Virtio ring implementation.
   2  *
   3  *  Copyright 2007 Rusty Russell IBM Corporation
   4  *
   5  *  This program is free software; you can redistribute it and/or modify
   6  *  it under the terms of the GNU General Public License as published by
   7  *  the Free Software Foundation; either version 2 of the License, or
   8  *  (at your option) any later version.
   9  *
  10  *  This program is distributed in the hope that it will be useful,
  11  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  *  GNU General Public License for more details.
  14  *
  15  *  You should have received a copy of the GNU General Public License
  16  *  along with this program; if not, write to the Free Software
  17  *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  18  */
  19 #include <linux/virtio.h>
  20 #include <linux/virtio_ring.h>
  21 #include <linux/virtio_config.h>
  22 #include <linux/device.h>
  23 #include <linux/slab.h>
  24 #include <linux/module.h>
  25 #include <linux/hrtimer.h>
  26 #include <linux/dma-mapping.h>
  27 #include <xen/xen.h>
  28
  29 #ifdef DEBUG
  30 /* For development, we want to crash whenever the ring is screwed. */
  31 #define BAD_RING(_vq, fmt, args...)                             \
  32         do {                                                    \
  33                 dev_err(&(_vq)->vq.vdev->dev,                   \
  34                         "%s:"fmt, (_vq)->vq.name, ##args);      \
  35                 BUG();                                          \
  36         } while (0)
  37 /* Caller is supposed to guarantee no reentry. */
  38 #define START_USE(_vq)                                          \
  39         do {                                                    \
  40                 if ((_vq)->in_use)                              \
  41                         panic("%s:in_use = %i\n",               \
  42                               (_vq)->vq.name, (_vq)->in_use);   \
  43                 (_vq)->in_use = __LINE__;                       \
  44         } while (0)
  45 #define END_USE(_vq) \
  46         do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0)
  47 #define LAST_ADD_TIME_UPDATE(_vq)                               \
  48         do {                                                    \
  49                 ktime_t now = ktime_get();                      \
  50                                                                 \
  51                 /* No kick or get, with .1 second between?  Warn. */ \
  52                 if ((_vq)->last_add_time_valid)                 \
  53                         WARN_ON(ktime_to_ms(ktime_sub(now,      \
  54                                 (_vq)->last_add_time)) > 100);  \
  55                 (_vq)->last_add_time = now;                     \
  56                 (_vq)->last_add_time_valid = true;              \
  57         } while (0)
  58 #define LAST_ADD_TIME_CHECK(_vq)                                \
  59         do {                                                    \
  60                 if ((_vq)->last_add_time_valid) {               \
  61                         WARN_ON(ktime_to_ms(ktime_sub(ktime_get(), \
  62                                       (_vq)->last_add_time)) > 100); \
  63                 }                                               \
  64         } while (0)
  65 #define LAST_ADD_TIME_INVALID(_vq)                              \
  66         ((_vq)->last_add_time_valid = false)
  67 #else
  68 #define BAD_RING(_vq, fmt, args...)                             \
  69         do {                                                    \
  70                 dev_err(&_vq->vq.vdev->dev,                     \
  71                         "%s:"fmt, (_vq)->vq.name, ##args);      \
  72                 (_vq)->broken = true;                           \
  73         } while (0)
  74 #define START_USE(vq)
  75 #define END_USE(vq)
  76 #define LAST_ADD_TIME_UPDATE(vq)
  77 #define LAST_ADD_TIME_CHECK(vq)
  78 #define LAST_ADD_TIME_INVALID(vq)
  79 #endif
  80
  81 struct vring_desc_state_split {
  82         void *data;                     /* Data for callback. */
  83         struct vring_desc *indir_desc;  /* Indirect descriptor, if any. */
  84 };
  85
  86 struct vring_desc_state_packed {
  87         void *data;                     /* Data for callback. */
  88         struct vring_packed_desc *indir_desc; /* Indirect descriptor, if any. */
  89         u16 num;                        /* Descriptor list length. */
  90         u16 next;                       /* The next desc state in a list. */
  91         u16 last;                       /* The last desc state in a list. */
  92 };
  93
  94 struct vring_desc_extra_packed {
  95         dma_addr_t addr;                /* Buffer DMA addr. */
  96         u32 len;                        /* Buffer length. */
  97         u16 flags;                      /* Descriptor flags. */
  98 };
  99
 100 struct vring_virtqueue {
 101         struct virtqueue vq;
 102
 103         /* Is this a packed ring? */
 104         bool packed_ring;
 105
 106         /* Is DMA API used? */
 107         bool use_dma_api;
 108
 109         /* Can we use weak barriers? */
 110         bool weak_barriers;
 111
 112         /* Other side has made a mess, don't try any more. */
 113         bool broken;
 114
 115         /* Host supports indirect buffers */
 116         bool indirect;
 117
 118         /* Host publishes avail event idx */
 119         bool event;
 120
 121         /* Head of free buffer list. */
 122         unsigned int free_head;
 123         /* Number we've added since last sync. */
 124         unsigned int num_added;
 125
 126         /* Last used index we've seen. */
 127         u16 last_used_idx;
 128
 129         union {
 130                 /* Available for split ring */
 131                 struct {
 132                         /* Actual memory layout for this queue. */
 133                         struct vring vring;
 134
 135                         /* Last written value to avail->flags */
 136                         u16 avail_flags_shadow;
 137
 138                         /*
 139                          * Last written value to avail->idx in
 140                          * guest byte order.
 141                          */
 142                         u16 avail_idx_shadow;
 143
 144                         /* Per-descriptor state. */
 145                         struct vring_desc_state_split *desc_state;
 146
 147                         /* DMA address and size information */
 148                         dma_addr_t queue_dma_addr;
 149                         size_t queue_size_in_bytes;
 150                 } split;
 151
 152                 /* Available for packed ring */
 153                 struct {
 154                         /* Actual memory layout for this queue. */
 155                         struct {
 156                                 unsigned int num;
 157                                 struct vring_packed_desc *desc;
 158                                 struct vring_packed_desc_event *driver;
 159                                 struct vring_packed_desc_event *device;
 160                         } vring;
 161
 162                         /* Driver ring wrap counter. */
 163                         bool avail_wrap_counter;
 164
 165                         /* Device ring wrap counter. */
 166                         bool used_wrap_counter;
 167
 168                         /* Avail used flags. */
 169                         u16 avail_used_flags;
 170
 171                         /* Index of the next avail descriptor. */
 172                         u16 next_avail_idx;
 173
 174                         /*
 175                          * Last written value to driver->flags in
 176                          * guest byte order.
 177                          */
 178                         u16 event_flags_shadow;
 179
 180                         /* Per-descriptor state. */
 181                         struct vring_desc_state_packed *desc_state;
 182                         struct vring_desc_extra_packed *desc_extra;
 183
 184                         /* DMA address and size information */
 185                         dma_addr_t ring_dma_addr;
 186                         dma_addr_t driver_event_dma_addr;
 187                         dma_addr_t device_event_dma_addr;
 188                         size_t ring_size_in_bytes;
 189                         size_t event_size_in_bytes;
 190                 } packed;
 191         };
 192
 193         /* How to notify other side. FIXME: commonalize hcalls! */
 194         bool (*notify)(struct virtqueue *vq);
 195
 196         /* DMA, allocation, and size information */
 197         bool we_own_ring;
 198
 199 #ifdef DEBUG
 200         /* They're supposed to lock for us. */
 201         unsigned int in_use;
 202
 203         /* Figure out if their kicks are too delayed. */
 204         bool last_add_time_valid;
 205         ktime_t last_add_time;
 206 #endif
 207 };
 208
 209
 210 /*
 211  * Helpers.
 212  */
 213
 214 #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
 215
 216 static inline bool virtqueue_use_indirect(struct virtqueue *_vq,
 217                                           unsigned int total_sg)
 218 {
 219         struct vring_virtqueue *vq = to_vvq(_vq);
 220
 221         /*
 222          * If the host supports indirect descriptor tables, and we have multiple
 223          * buffers, then go indirect. FIXME: tune this threshold
 224          */
 225         return (vq->indirect && total_sg > 1 && vq->vq.num_free);
 226 }
 227
 228 /*
 229  * Modern virtio devices have feature bits to specify whether they need a
 230  * quirk and bypass the IOMMU. If not there, just use the DMA API.
 231  *
 232  * If there, the interaction between virtio and DMA API is messy.
 233  *
 234  * On most systems with virtio, physical addresses match bus addresses,
 235  * and it doesn't particularly matter whether we use the DMA API.
 236  *
 237  * On some systems, including Xen and any system with a physical device
 238  * that speaks virtio behind a physical IOMMU, we must use the DMA API
 239  * for virtio DMA to work at all.
 240  *
 241  * On other systems, including SPARC and PPC64, virtio-pci devices are
 242  * enumerated as though they are behind an IOMMU, but the virtio host
 243  * ignores the IOMMU, so we must either pretend that the IOMMU isn't
 244  * there or somehow map everything as the identity.
 245  *
 246  * For the time being, we preserve historic behavior and bypass the DMA
 247  * API.
 248  *
 249  * TODO: install a per-device DMA ops structure that does the right thing
 250  * taking into account all the above quirks, and use the DMA API
 251  * unconditionally on data path.
 252  */
 253
 254 static bool vring_use_dma_api(struct virtio_device *vdev)
 255 {
 256         if (!virtio_has_iommu_quirk(vdev))
 257                 return true;
 258
 259         /* Otherwise, we are left to guess. */
 260         /*
 261          * In theory, it's possible to have a buggy QEMU-supposed
 262          * emulated Q35 IOMMU and Xen enabled at the same time.  On
 263          * such a configuration, virtio has never worked and will
 264          * not work without an even larger kludge.  Instead, enable
 265          * the DMA API if we're a Xen guest, which at least allows
 266          * all of the sensible Xen configurations to work correctly.
 267          */
 268         if (xen_domain())
 269                 return true;
 270
 271         return false;
 272 }
 273
 274 size_t virtio_max_dma_size(struct virtio_device *vdev)
 275 {
 276         size_t max_segment_size = SIZE_MAX;
 277
 278         if (vring_use_dma_api(vdev))
 279                 max_segment_size = dma_max_mapping_size(&vdev->dev);
 280
 281         return max_segment_size;
 282 }
 283 EXPORT_SYMBOL_GPL(virtio_max_dma_size);
 284
 285 static void *vring_alloc_queue(struct virtio_device *vdev, size_t size,
 286                               dma_addr_t *dma_handle, gfp_t flag)
 287 {
 288         if (vring_use_dma_api(vdev)) {
 289                 return dma_alloc_coherent(vdev->dev.parent, size,
 290                                           dma_handle, flag);
 291         } else {
 292                 void *queue = alloc_pages_exact(PAGE_ALIGN(size), flag);
 293
 294                 if (queue) {
 295                         phys_addr_t phys_addr = virt_to_phys(queue);
 296                         *dma_handle = (dma_addr_t)phys_addr;
 297
 298                         /*
 299                          * Sanity check: make sure we dind't truncate
 300                          * the address.  The only arches I can find that
 301                          * have 64-bit phys_addr_t but 32-bit dma_addr_t
 302                          * are certain non-highmem MIPS and x86
 303                          * configurations, but these configurations
 304                          * should never allocate physical pages above 32
 305                          * bits, so this is fine.  Just in case, throw a
 306                          * warning and abort if we end up with an
 307                          * unrepresentable address.
 308                          */
 309                         if (WARN_ON_ONCE(*dma_handle != phys_addr)) {
 310                                 free_pages_exact(queue, PAGE_ALIGN(size));
 311                                 return NULL;
 312                         }
 313                 }
 314                 return queue;
 315         }
 316 }
 317
 318 static void vring_free_queue(struct virtio_device *vdev, size_t size,
 319                              void *queue, dma_addr_t dma_handle)
 320 {
 321         if (vring_use_dma_api(vdev))
 322                 dma_free_coherent(vdev->dev.parent, size, queue, dma_handle);
 323         else
 324                 free_pages_exact(queue, PAGE_ALIGN(size));
 325 }
 326
 327 /*
 328  * The DMA ops on various arches are rather gnarly right now, and
 329  * making all of the arch DMA ops work on the vring device itself
 330  * is a mess.  For now, we use the parent device for DMA ops.
 331  */
 332 static inline struct device *vring_dma_dev(const struct vring_virtqueue *vq)
 333 {
 334         return vq->vq.vdev->dev.parent;
 335 }
 336
 337 /* Map one sg entry. */
 338 static dma_addr_t vring_map_one_sg(const struct vring_virtqueue *vq,
 339                                    struct scatterlist *sg,
 340                                    enum dma_data_direction direction)
 341 {
 342         if (!vq->use_dma_api)
 343                 return (dma_addr_t)sg_phys(sg);
 344
 345         /*
 346          * We can't use dma_map_sg, because we don't use scatterlists in
 347          * the way it expects (we don't guarantee that the scatterlist
 348          * will exist for the lifetime of the mapping).
 349          */
 350         return dma_map_page(vring_dma_dev(vq),
 351                             sg_page(sg), sg->offset, sg->length,
 352                             direction);
 353 }
 354
 355 static dma_addr_t vring_map_single(const struct vring_virtqueue *vq,
 356                                    void *cpu_addr, size_t size,
 357                                    enum dma_data_direction direction)
 358 {
 359         if (!vq->use_dma_api)
 360                 return (dma_addr_t)virt_to_phys(cpu_addr);
 361
 362         return dma_map_single(vring_dma_dev(vq),
 363                               cpu_addr, size, direction);
 364 }
 365
 366 static int vring_mapping_error(const struct vring_virtqueue *vq,
 367                                dma_addr_t addr)
 368 {
 369         if (!vq->use_dma_api)
 370                 return 0;
 371
 372         return dma_mapping_error(vring_dma_dev(vq), addr);
 373 }
 374
 375
 376 /*
 377  * Split ring specific functions - *_split().
 378  */
 379
 380 static void vring_unmap_one_split(const struct vring_virtqueue *vq,
 381                                   struct vring_desc *desc)
 382 {
 383         u16 flags;
 384
 385         if (!vq->use_dma_api)
 386                 return;
 387
 388         flags = virtio16_to_cpu(vq->vq.vdev, desc->flags);
 389
 390         if (flags & VRING_DESC_F_INDIRECT) {
 391                 dma_unmap_single(vring_dma_dev(vq),
 392                                  virtio64_to_cpu(vq->vq.vdev, desc->addr),
 393                                  virtio32_to_cpu(vq->vq.vdev, desc->len),
 394                                  (flags & VRING_DESC_F_WRITE) ?
 395                                  DMA_FROM_DEVICE : DMA_TO_DEVICE);
 396         } else {
 397                 dma_unmap_page(vring_dma_dev(vq),
 398                                virtio64_to_cpu(vq->vq.vdev, desc->addr),
 399                                virtio32_to_cpu(vq->vq.vdev, desc->len),
 400                                (flags & VRING_DESC_F_WRITE) ?
 401                                DMA_FROM_DEVICE : DMA_TO_DEVICE);
 402         }
 403 }
 404
 405 static struct vring_desc *alloc_indirect_split(struct virtqueue *_vq,
 406                                                unsigned int total_sg,
 407                                                gfp_t gfp)
 408 {
 409         struct vring_desc *desc;
 410         unsigned int i;
 411
 412         /*
 413          * We require lowmem mappings for the descriptors because
 414          * otherwise virt_to_phys will give us bogus addresses in the
 415          * virtqueue.
 416          */
 417         gfp &= ~__GFP_HIGHMEM;
 418
 419         desc = kmalloc_array(total_sg, sizeof(struct vring_desc), gfp);
 420         if (!desc)
 421                 return NULL;
 422
 423         for (i = 0; i < total_sg; i++)
 424                 desc[i].next = cpu_to_virtio16(_vq->vdev, i + 1);
 425         return desc;
 426 }
 427
 428 static inline int virtqueue_add_split(struct virtqueue *_vq,
 429                                       struct scatterlist *sgs[],
 430                                       unsigned int total_sg,
 431                                       unsigned int out_sgs,
 432                                       unsigned int in_sgs,
 433                                       void *data,
 434                                       void *ctx,
 435                                       gfp_t gfp)
 436 {
 437         struct vring_virtqueue *vq = to_vvq(_vq);
 438         struct scatterlist *sg;
 439         struct vring_desc *desc;
 440         unsigned int i, n, avail, descs_used, uninitialized_var(prev), err_idx;
 441         int head;
 442         bool indirect;
 443
 444         START_USE(vq);
 445
 446         BUG_ON(data == NULL);
 447         BUG_ON(ctx && vq->indirect);
 448
 449         if (unlikely(vq->broken)) {
 450                 END_USE(vq);
 451                 return -EIO;
 452         }
 453
 454         LAST_ADD_TIME_UPDATE(vq);
 455
 456         BUG_ON(total_sg == 0);
 457
 458         head = vq->free_head;
 459
 460         if (virtqueue_use_indirect(_vq, total_sg))
 461                 desc = alloc_indirect_split(_vq, total_sg, gfp);
 462         else {
 463                 desc = NULL;
 464                 WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect);
 465         }
 466
 467         if (desc) {
 468                 /* Use a single buffer which doesn't continue */
 469                 indirect = true;
 470                 /* Set up rest to use this indirect table. */
 471                 i = 0;
 472                 descs_used = 1;
 473         } else {
 474                 indirect = false;
 475                 desc = vq->split.vring.desc;
 476                 i = head;
 477                 descs_used = total_sg;
 478         }
 479
 480         if (vq->vq.num_free < descs_used) {
 481                 pr_debug("Can't add buf len %i - avail = %i\n",
 482                          descs_used, vq->vq.num_free);
 483                 /* FIXME: for historical reasons, we force a notify here if
 484                  * there are outgoing parts to the buffer.  Presumably the
 485                  * host should service the ring ASAP. */
 486                 if (out_sgs)
 487                         vq->notify(&vq->vq);
 488                 if (indirect)
 489                         kfree(desc);
 490                 END_USE(vq);
 491                 return -ENOSPC;
 492         }
 493
 494         for (n = 0; n < out_sgs; n++) {
 495                 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
 496                         dma_addr_t addr = vring_map_one_sg(vq, sg, DMA_TO_DEVICE);
 497                         if (vring_mapping_error(vq, addr))
 498                                 goto unmap_release;
 499
 500                         desc[i].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_NEXT);
 501                         desc[i].addr = cpu_to_virtio64(_vq->vdev, addr);
 502                         desc[i].len = cpu_to_virtio32(_vq->vdev, sg->length);
 503                         prev = i;
 504                         i = virtio16_to_cpu(_vq->vdev, desc[i].next);
 505                 }
 506         }
 507         for (; n < (out_sgs + in_sgs); n++) {
 508                 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
 509                         dma_addr_t addr = vring_map_one_sg(vq, sg, DMA_FROM_DEVICE);
 510                         if (vring_mapping_error(vq, addr))
 511                                 goto unmap_release;
 512
 513                         desc[i].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_NEXT | VRING_DESC_F_WRITE);
 514                         desc[i].addr = cpu_to_virtio64(_vq->vdev, addr);
 515                         desc[i].len = cpu_to_virtio32(_vq->vdev, sg->length);
 516                         prev = i;
 517                         i = virtio16_to_cpu(_vq->vdev, desc[i].next);
 518                 }
 519         }
 520         /* Last one doesn't continue. */
 521         desc[prev].flags &= cpu_to_virtio16(_vq->vdev, ~VRING_DESC_F_NEXT);
 522
 523         if (indirect) {
 524                 /* Now that the indirect table is filled in, map it. */
 525                 dma_addr_t addr = vring_map_single(
 526                         vq, desc, total_sg * sizeof(struct vring_desc),
 527                         DMA_TO_DEVICE);
 528                 if (vring_mapping_error(vq, addr))
 529                         goto unmap_release;
 530
 531                 vq->split.vring.desc[head].flags = cpu_to_virtio16(_vq->vdev,
 532                                 VRING_DESC_F_INDIRECT);
 533                 vq->split.vring.desc[head].addr = cpu_to_virtio64(_vq->vdev,
 534                                 addr);
 535
 536                 vq->split.vring.desc[head].len = cpu_to_virtio32(_vq->vdev,
 537                                 total_sg * sizeof(struct vring_desc));
 538         }
 539
 540         /* We're using some buffers from the free list. */
 541         vq->vq.num_free -= descs_used;
 542
 543         /* Update free pointer */
 544         if (indirect)
 545                 vq->free_head = virtio16_to_cpu(_vq->vdev,
 546                                         vq->split.vring.desc[head].next);
 547         else
 548                 vq->free_head = i;
 549
 550         /* Store token and indirect buffer state. */
 551         vq->split.desc_state[head].data = data;
 552         if (indirect)
 553                 vq->split.desc_state[head].indir_desc = desc;
 554         else
 555                 vq->split.desc_state[head].indir_desc = ctx;
 556
 557         /* Put entry in available array (but don't update avail->idx until they
 558          * do sync). */
 559         avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1);
 560         vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);
 561
 562         /* Descriptors and available array need to be set before we expose the
 563          * new available array entries. */
 564         virtio_wmb(vq->weak_barriers);
 565         vq->split.avail_idx_shadow++;
 566         vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
 567                                                 vq->split.avail_idx_shadow);
 568         vq->num_added++;
 569
 570         pr_debug("Added buffer head %i to %p\n", head, vq);
 571         END_USE(vq);
 572
 573         /* This is very unlikely, but theoretically possible.  Kick
 574          * just in case. */
 575         if (unlikely(vq->num_added == (1 << 16) - 1))
 576                 virtqueue_kick(_vq);
 577
 578         return 0;
 579
 580 unmap_release:
 581         err_idx = i;
 582         i = head;
 583
 584         for (n = 0; n < total_sg; n++) {
 585                 if (i == err_idx)
 586                         break;
 587                 vring_unmap_one_split(vq, &desc[i]);
 588                 i = virtio16_to_cpu(_vq->vdev, vq->split.vring.desc[i].next);
 589         }
 590
 591         if (indirect)
 592                 kfree(desc);
 593
 594         END_USE(vq);
 595         return -EIO;
 596 }
 597
 598 static bool virtqueue_kick_prepare_split(struct virtqueue *_vq)
 599 {
 600         struct vring_virtqueue *vq = to_vvq(_vq);
 601         u16 new, old;
 602         bool needs_kick;
 603
 604         START_USE(vq);
 605         /* We need to expose available array entries before checking avail
 606          * event. */
 607         virtio_mb(vq->weak_barriers);
 608
 609         old = vq->split.avail_idx_shadow - vq->num_added;
 610         new = vq->split.avail_idx_shadow;
 611         vq->num_added = 0;
 612
 613         LAST_ADD_TIME_CHECK(vq);
 614         LAST_ADD_TIME_INVALID(vq);
 615
 616         if (vq->event) {
 617                 needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev,
 618                                         vring_avail_event(&vq->split.vring)),
 619                                               new, old);
 620         } else {
 621                 needs_kick = !(vq->split.vring.used->flags &
 622                                         cpu_to_virtio16(_vq->vdev,
 623                                                 VRING_USED_F_NO_NOTIFY));
 624         }
 625         END_USE(vq);
 626         return needs_kick;
 627 }
 628
 629 static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
 630                              void **ctx)
 631 {
 632         unsigned int i, j;
 633         __virtio16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT);
 634
 635         /* Clear data ptr. */
 636         vq->split.desc_state[head].data = NULL;
 637
 638         /* Put back on free list: unmap first-level descriptors and find end */
 639         i = head;
 640
 641         while (vq->split.vring.desc[i].flags & nextflag) {
 642                 vring_unmap_one_split(vq, &vq->split.vring.desc[i]);
 643                 i = virtio16_to_cpu(vq->vq.vdev, vq->split.vring.desc[i].next);
 644                 vq->vq.num_free++;
 645         }
 646
 647         vring_unmap_one_split(vq, &vq->split.vring.desc[i]);
 648         vq->split.vring.desc[i].next = cpu_to_virtio16(vq->vq.vdev,
 649                                                 vq->free_head);
 650         vq->free_head = head;
 651
 652         /* Plus final descriptor */
 653         vq->vq.num_free++;
 654
 655         if (vq->indirect) {
 656                 struct vring_desc *indir_desc =
 657                                 vq->split.desc_state[head].indir_desc;
 658                 u32 len;
 659
 660                 /* Free the indirect table, if any, now that it's unmapped. */
 661                 if (!indir_desc)
 662                         return;
 663
 664                 len = virtio32_to_cpu(vq->vq.vdev,
 665                                 vq->split.vring.desc[head].len);
 666
 667                 BUG_ON(!(vq->split.vring.desc[head].flags &
 668                          cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_INDIRECT)));
 669                 BUG_ON(len == 0 || len % sizeof(struct vring_desc));
 670
 671                 for (j = 0; j < len / sizeof(struct vring_desc); j++)
 672                         vring_unmap_one_split(vq, &indir_desc[j]);
 673
 674                 kfree(indir_desc);
 675                 vq->split.desc_state[head].indir_desc = NULL;
 676         } else if (ctx) {
 677                 *ctx = vq->split.desc_state[head].indir_desc;
 678         }
 679 }
 680
 681 static inline bool more_used_split(const struct vring_virtqueue *vq)
 682 {
 683         return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev,
 684                         vq->split.vring.used->idx);
 685 }
 686
 687 static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq,
 688                                          unsigned int *len,
 689                                          void **ctx)
 690 {
 691         struct vring_virtqueue *vq = to_vvq(_vq);
 692         void *ret;
 693         unsigned int i;
 694         u16 last_used;
 695
 696         START_USE(vq);
 697
 698         if (unlikely(vq->broken)) {
 699                 END_USE(vq);
 700                 return NULL;
 701         }
 702
 703         if (!more_used_split(vq)) {
 704                 pr_debug("No more buffers in queue\n");
 705                 END_USE(vq);
 706                 return NULL;
 707         }
 708
 709         /* Only get used array entries after they have been exposed by host. */
 710         virtio_rmb(vq->weak_barriers);
 711
 712         last_used = (vq->last_used_idx & (vq->split.vring.num - 1));
 713         i = virtio32_to_cpu(_vq->vdev,
 714                         vq->split.vring.used->ring[last_used].id);
 715         *len = virtio32_to_cpu(_vq->vdev,
 716                         vq->split.vring.used->ring[last_used].len);
 717
 718         if (unlikely(i >= vq->split.vring.num)) {
 719                 BAD_RING(vq, "id %u out of range\n", i);
 720                 return NULL;
 721         }
 722         if (unlikely(!vq->split.desc_state[i].data)) {
 723                 BAD_RING(vq, "id %u is not a head!\n", i);
 724                 return NULL;
 725         }
 726
 727         /* detach_buf_split clears data, so grab it now. */
 728         ret = vq->split.desc_state[i].data;
 729         detach_buf_split(vq, i, ctx);
 730         vq->last_used_idx++;
 731         /* If we expect an interrupt for the next entry, tell host
 732          * by writing event index and flush out the write before
 733          * the read in the next get_buf call. */
 734         if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
 735                 virtio_store_mb(vq->weak_barriers,
 736                                 &vring_used_event(&vq->split.vring),
 737                                 cpu_to_virtio16(_vq->vdev, vq->last_used_idx));
 738
 739         LAST_ADD_TIME_INVALID(vq);
 740
 741         END_USE(vq);
 742         return ret;
 743 }
 744
 745 static void virtqueue_disable_cb_split(struct virtqueue *_vq)
 746 {
 747         struct vring_virtqueue *vq = to_vvq(_vq);
 748
 749         if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
 750                 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
 751                 if (!vq->event)
 752                         vq->split.vring.avail->flags =
 753                                 cpu_to_virtio16(_vq->vdev,
 754                                                 vq->split.avail_flags_shadow);
 755         }
 756 }
 757
 758 static unsigned virtqueue_enable_cb_prepare_split(struct virtqueue *_vq)
 759 {
 760         struct vring_virtqueue *vq = to_vvq(_vq);
 761         u16 last_used_idx;
 762
 763         START_USE(vq);
 764
 765         /* We optimistically turn back on interrupts, then check if there was
 766          * more to do. */
 767         /* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to
 768          * either clear the flags bit or point the event index at the next
 769          * entry. Always do both to keep code simple. */
 770         if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
 771                 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
 772                 if (!vq->event)
 773                         vq->split.vring.avail->flags =
 774                                 cpu_to_virtio16(_vq->vdev,
 775                                                 vq->split.avail_flags_shadow);
 776         }
 777         vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev,
 778                         last_used_idx = vq->last_used_idx);
 779         END_USE(vq);
 780         return last_used_idx;
 781 }
 782
 783 static bool virtqueue_poll_split(struct virtqueue *_vq, unsigned last_used_idx)
 784 {
 785         struct vring_virtqueue *vq = to_vvq(_vq);
 786
 787         return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev,
 788                         vq->split.vring.used->idx);
 789 }
 790
 791 static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq)
 792 {
 793         struct vring_virtqueue *vq = to_vvq(_vq);
 794         u16 bufs;
 795
 796         START_USE(vq);
 797
 798         /* We optimistically turn back on interrupts, then check if there was
 799          * more to do. */
 800         /* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to
 801          * either clear the flags bit or point the event index at the next
 802          * entry. Always update the event index to keep code simple. */
 803         if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
 804                 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
 805                 if (!vq->event)
 806                         vq->split.vring.avail->flags =
 807                                 cpu_to_virtio16(_vq->vdev,
 808                                                 vq->split.avail_flags_shadow);
 809         }
 810         /* TODO: tune this threshold */
 811         bufs = (u16)(vq->split.avail_idx_shadow - vq->last_used_idx) * 3 / 4;
 812
 813         virtio_store_mb(vq->weak_barriers,
 814                         &vring_used_event(&vq->split.vring),
 815                         cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs));
 816
 817         if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx)
 818                                         - vq->last_used_idx) > bufs)) {
 819                 END_USE(vq);
 820                 return false;
 821         }
 822
 823         END_USE(vq);
 824         return true;
 825 }
 826
 827 static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq)
 828 {
 829         struct vring_virtqueue *vq = to_vvq(_vq);
 830         unsigned int i;
 831         void *buf;
 832
 833         START_USE(vq);
 834
 835         for (i = 0; i < vq->split.vring.num; i++) {
 836                 if (!vq->split.desc_state[i].data)
 837                         continue;
 838                 /* detach_buf_split clears data, so grab it now. */
 839                 buf = vq->split.desc_state[i].data;
 840                 detach_buf_split(vq, i, NULL);
 841                 vq->split.avail_idx_shadow--;
 842                 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
 843                                 vq->split.avail_idx_shadow);
 844                 END_USE(vq);
 845                 return buf;
 846         }
 847         /* That should have freed everything. */
 848         BUG_ON(vq->vq.num_free != vq->split.vring.num);
 849
 850         END_USE(vq);
 851         return NULL;
 852 }
 853
 854 static struct virtqueue *vring_create_virtqueue_split(
 855         unsigned int index,
 856         unsigned int num,
 857         unsigned int vring_align,
 858         struct virtio_device *vdev,
 859         bool weak_barriers,
 860         bool may_reduce_num,
 861         bool context,
 862         bool (*notify)(struct virtqueue *),
 863         void (*callback)(struct virtqueue *),
 864         const char *name)
 865 {
 866         struct virtqueue *vq;
 867         void *queue = NULL;
 868         dma_addr_t dma_addr;
 869         size_t queue_size_in_bytes;
 870         struct vring vring;
 871
 872         /* We assume num is a power of 2. */
 873         if (num & (num - 1)) {
 874                 dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num);
 875                 return NULL;
 876         }
 877
 878         /* TODO: allocate each queue chunk individually */
 879         for (; num && vring_size(num, vring_align) > PAGE_SIZE; num /= 2) {
 880                 queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
 881                                           &dma_addr,
 882                                           GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
 883                 if (queue)
 884                         break;
 885         }
 886
 887         if (!num)
 888                 return NULL;
 889
 890         if (!queue) {
 891                 /* Try to get a single page. You are my only hope! */
 892                 queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
 893                                           &dma_addr, GFP_KERNEL|__GFP_ZERO);
 894         }
 895         if (!queue)
 896                 return NULL;
 897
 898         queue_size_in_bytes = vring_size(num, vring_align);
 899         vring_init(&vring, num, queue, vring_align);
 900
 901         vq = __vring_new_virtqueue(index, vring, vdev, weak_barriers, context,
 902                                    notify, callback, name);
 903         if (!vq) {
 904                 vring_free_queue(vdev, queue_size_in_bytes, queue,
 905                                  dma_addr);
 906                 return NULL;
 907         }
 908
 909         to_vvq(vq)->split.queue_dma_addr = dma_addr;
 910         to_vvq(vq)->split.queue_size_in_bytes = queue_size_in_bytes;
 911         to_vvq(vq)->we_own_ring = true;
 912
 913         return vq;
 914 }
 915
 916
 917 /*
 918  * Packed ring specific functions - *_packed().
 919  */
 920
 921 static void vring_unmap_state_packed(const struct vring_virtqueue *vq,
 922                                      struct vring_desc_extra_packed *state)
 923 {
 924         u16 flags;
 925
 926         if (!vq->use_dma_api)
 927                 return;
 928
 929         flags = state->flags;
 930
 931         if (flags & VRING_DESC_F_INDIRECT) {
 932                 dma_unmap_single(vring_dma_dev(vq),
 933                                  state->addr, state->len,
 934                                  (flags & VRING_DESC_F_WRITE) ?
 935                                  DMA_FROM_DEVICE : DMA_TO_DEVICE);
 936         } else {
 937                 dma_unmap_page(vring_dma_dev(vq),
 938                                state->addr, state->len,
 939                                (flags & VRING_DESC_F_WRITE) ?
 940                                DMA_FROM_DEVICE : DMA_TO_DEVICE);
 941         }
 942 }
 943
 944 static void vring_unmap_desc_packed(const struct vring_virtqueue *vq,
 945                                    struct vring_packed_desc *desc)
 946 {
 947         u16 flags;
 948
 949         if (!vq->use_dma_api)
 950                 return;
 951
 952         flags = le16_to_cpu(desc->flags);
 953
 954         if (flags & VRING_DESC_F_INDIRECT) {
 955                 dma_unmap_single(vring_dma_dev(vq),
 956                                  le64_to_cpu(desc->addr),
 957                                  le32_to_cpu(desc->len),
 958                                  (flags & VRING_DESC_F_WRITE) ?
 959                                  DMA_FROM_DEVICE : DMA_TO_DEVICE);
 960         } else {
 961                 dma_unmap_page(vring_dma_dev(vq),
 962                                le64_to_cpu(desc->addr),
 963                                le32_to_cpu(desc->len),
 964                                (flags & VRING_DESC_F_WRITE) ?
 965                                DMA_FROM_DEVICE : DMA_TO_DEVICE);
 966         }
 967 }
 968
 969 static struct vring_packed_desc *alloc_indirect_packed(unsigned int total_sg,
 970                                                        gfp_t gfp)
 971 {
 972         struct vring_packed_desc *desc;
 973
 974         /*
 975          * We require lowmem mappings for the descriptors because
 976          * otherwise virt_to_phys will give us bogus addresses in the
 977          * virtqueue.
 978          */
 979         gfp &= ~__GFP_HIGHMEM;
 980
 981         desc = kmalloc_array(total_sg, sizeof(struct vring_packed_desc), gfp);
 982
 983         return desc;
 984 }
 985
 986 static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
 987                                        struct scatterlist *sgs[],
 988                                        unsigned int total_sg,
 989                                        unsigned int out_sgs,
 990                                        unsigned int in_sgs,
 991                                        void *data,
 992                                        gfp_t gfp)
 993 {
 994         struct vring_packed_desc *desc;
 995         struct scatterlist *sg;
 996         unsigned int i, n, err_idx;
 997         u16 head, id;
 998         dma_addr_t addr;
 999
1000         head = vq->packed.next_avail_idx;
1001         desc = alloc_indirect_packed(total_sg, gfp);
1002
1003         if (unlikely(vq->vq.num_free < 1)) {
1004                 pr_debug("Can't add buf len 1 - avail = 0\n");
1005                 END_USE(vq);
1006                 return -ENOSPC;
1007         }
1008
1009         i = 0;
1010         id = vq->free_head;
1011         BUG_ON(id == vq->packed.vring.num);
1012
1013         for (n = 0; n < out_sgs + in_sgs; n++) {
1014                 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
1015                         addr = vring_map_one_sg(vq, sg, n < out_sgs ?
1016                                         DMA_TO_DEVICE : DMA_FROM_DEVICE);
1017                         if (vring_mapping_error(vq, addr))
1018                                 goto unmap_release;
1019
1020                         desc[i].flags = cpu_to_le16(n < out_sgs ?
1021                                                 0 : VRING_DESC_F_WRITE);
1022                         desc[i].addr = cpu_to_le64(addr);
1023                         desc[i].len = cpu_to_le32(sg->length);
1024                         i++;
1025                 }
1026         }
1027
1028         /* Now that the indirect table is filled in, map it. */
1029         addr = vring_map_single(vq, desc,
1030                         total_sg * sizeof(struct vring_packed_desc),
1031                         DMA_TO_DEVICE);
1032         if (vring_mapping_error(vq, addr))
1033                 goto unmap_release;
1034
1035         vq->packed.vring.desc[head].addr = cpu_to_le64(addr);
1036         vq->packed.vring.desc[head].len = cpu_to_le32(total_sg *
1037                                 sizeof(struct vring_packed_desc));
1038         vq->packed.vring.desc[head].id = cpu_to_le16(id);
1039
1040         if (vq->use_dma_api) {
1041                 vq->packed.desc_extra[id].addr = addr;
1042                 vq->packed.desc_extra[id].len = total_sg *
1043                                 sizeof(struct vring_packed_desc);
1044                 vq->packed.desc_extra[id].flags = VRING_DESC_F_INDIRECT |
1045                                                   vq->packed.avail_used_flags;
1046         }
1047
1048         /*
1049          * A driver MUST NOT make the first descriptor in the list
1050          * available before all subsequent descriptors comprising
1051          * the list are made available.
1052          */
1053         virtio_wmb(vq->weak_barriers);
1054         vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT |
1055                                                 vq->packed.avail_used_flags);
1056
1057         /* We're using some buffers from the free list. */
1058         vq->vq.num_free -= 1;
1059
1060         /* Update free pointer */
1061         n = head + 1;
1062         if (n >= vq->packed.vring.num) {
1063                 n = 0;
1064                 vq->packed.avail_wrap_counter ^= 1;
1065                 vq->packed.avail_used_flags ^=
1066                                 1 << VRING_PACKED_DESC_F_AVAIL |
1067                                 1 << VRING_PACKED_DESC_F_USED;
1068         }
1069         vq->packed.next_avail_idx = n;
1070         vq->free_head = vq->packed.desc_state[id].next;
1071
1072         /* Store token and indirect buffer state. */
1073         vq->packed.desc_state[id].num = 1;
1074         vq->packed.desc_state[id].data = data;
1075         vq->packed.desc_state[id].indir_desc = desc;
1076         vq->packed.desc_state[id].last = id;
1077
1078         vq->num_added += 1;
1079
1080         pr_debug("Added buffer head %i to %p\n", head, vq);
1081         END_USE(vq);
1082
1083         return 0;
1084
1085 unmap_release:
1086         err_idx = i;
1087
1088         for (i = 0; i < err_idx; i++)
1089                 vring_unmap_desc_packed(vq, &desc[i]);
1090
1091         kfree(desc);
1092
1093         END_USE(vq);
1094         return -EIO;
1095 }
1096
1097 static inline int virtqueue_add_packed(struct virtqueue *_vq,
1098                                        struct scatterlist *sgs[],
1099                                        unsigned int total_sg,
1100                                        unsigned int out_sgs,
1101                                        unsigned int in_sgs,
1102                                        void *data,
1103                                        void *ctx,
1104                                        gfp_t gfp)
1105 {
1106         struct vring_virtqueue *vq = to_vvq(_vq);
1107         struct vring_packed_desc *desc;
1108         struct scatterlist *sg;
1109         unsigned int i, n, c, descs_used, err_idx;
1110         __le16 uninitialized_var(head_flags), flags;
1111         u16 head, id, uninitialized_var(prev), curr, avail_used_flags;
1112
1113         START_USE(vq);
1114
1115         BUG_ON(data == NULL);
1116         BUG_ON(ctx && vq->indirect);
1117
1118         if (unlikely(vq->broken)) {
1119                 END_USE(vq);
1120                 return -EIO;
1121         }
1122
1123         LAST_ADD_TIME_UPDATE(vq);
1124
1125         BUG_ON(total_sg == 0);
1126
1127         if (virtqueue_use_indirect(_vq, total_sg))
1128                 return virtqueue_add_indirect_packed(vq, sgs, total_sg,
1129                                 out_sgs, in_sgs, data, gfp);
1130
1131         head = vq->packed.next_avail_idx;
1132         avail_used_flags = vq->packed.avail_used_flags;
1133
1134         WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);
1135
1136         desc = vq->packed.vring.desc;
1137         i = head;
1138         descs_used = total_sg;
1139
1140         if (unlikely(vq->vq.num_free < descs_used)) {
1141                 pr_debug("Can't add buf len %i - avail = %i\n",
1142                          descs_used, vq->vq.num_free);
1143                 END_USE(vq);
1144                 return -ENOSPC;
1145         }
1146
1147         id = vq->free_head;
1148         BUG_ON(id == vq->packed.vring.num);
1149
1150         curr = id;
1151         c = 0;
1152         for (n = 0; n < out_sgs + in_sgs; n++) {
1153                 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
1154                         dma_addr_t addr = vring_map_one_sg(vq, sg, n < out_sgs ?
1155                                         DMA_TO_DEVICE : DMA_FROM_DEVICE);
1156                         if (vring_mapping_error(vq, addr))
1157                                 goto unmap_release;
1158
1159                         flags = cpu_to_le16(vq->packed.avail_used_flags |
1160                                     (++c == total_sg ? 0 : VRING_DESC_F_NEXT) |
1161                                     (n < out_sgs ? 0 : VRING_DESC_F_WRITE));
1162                         if (i == head)
1163                                 head_flags = flags;
1164                         else
1165                                 desc[i].flags = flags;
1166
1167                         desc[i].addr = cpu_to_le64(addr);
1168                         desc[i].len = cpu_to_le32(sg->length);
1169                         desc[i].id = cpu_to_le16(id);
1170
1171                         if (unlikely(vq->use_dma_api)) {
1172                                 vq->packed.desc_extra[curr].addr = addr;
1173                                 vq->packed.desc_extra[curr].len = sg->length;
1174                                 vq->packed.desc_extra[curr].flags =
1175                                         le16_to_cpu(flags);
1176                         }
1177                         prev = curr;
1178                         curr = vq->packed.desc_state[curr].next;
1179
1180                         if ((unlikely(++i >= vq->packed.vring.num))) {
1181                                 i = 0;
1182                                 vq->packed.avail_used_flags ^=
1183                                         1 << VRING_PACKED_DESC_F_AVAIL |
1184                                         1 << VRING_PACKED_DESC_F_USED;
1185                         }
1186                 }
1187         }
1188
1189         if (i < head)
1190                 vq->packed.avail_wrap_counter ^= 1;
1191
1192         /* We're using some buffers from the free list. */
1193         vq->vq.num_free -= descs_used;
1194
1195         /* Update free pointer */
1196         vq->packed.next_avail_idx = i;
1197         vq->free_head = curr;
1198
1199         /* Store token. */
1200         vq->packed.desc_state[id].num = descs_used;
1201         vq->packed.desc_state[id].data = data;
1202         vq->packed.desc_state[id].indir_desc = ctx;
1203         vq->packed.desc_state[id].last = prev;
1204
1205         /*
1206          * A driver MUST NOT make the first descriptor in the list
1207          * available before all subsequent descriptors comprising
1208          * the list are made available.
1209          */
1210         virtio_wmb(vq->weak_barriers);
1211         vq->packed.vring.desc[head].flags = head_flags;
1212         vq->num_added += descs_used;
1213
1214         pr_debug("Added buffer head %i to %p\n", head, vq);
1215         END_USE(vq);
1216
1217         return 0;
1218
1219 unmap_release:
1220         err_idx = i;
1221         i = head;
1222
1223         vq->packed.avail_used_flags = avail_used_flags;
1224
1225         for (n = 0; n < total_sg; n++) {
1226                 if (i == err_idx)
1227                         break;
1228                 vring_unmap_desc_packed(vq, &desc[i]);
1229                 i++;
1230                 if (i >= vq->packed.vring.num)
1231                         i = 0;
1232         }
1233
1234         END_USE(vq);
1235         return -EIO;
1236 }
1237
1238 static bool virtqueue_kick_prepare_packed(struct virtqueue *_vq)
1239 {
1240         struct vring_virtqueue *vq = to_vvq(_vq);
1241         u16 new, old, off_wrap, flags, wrap_counter, event_idx;
1242         bool needs_kick;
1243         union {
1244                 struct {
1245                         __le16 off_wrap;
1246                         __le16 flags;
1247                 };
1248                 u32 u32;
1249         } snapshot;
1250
1251         START_USE(vq);
1252
1253         /*
1254          * We need to expose the new flags value before checking notification
1255          * suppressions.
1256          */
1257         virtio_mb(vq->weak_barriers);
1258
1259         old = vq->packed.next_avail_idx - vq->num_added;
1260         new = vq->packed.next_avail_idx;
1261         vq->num_added = 0;
1262
1263         snapshot.u32 = *(u32 *)vq->packed.vring.device;
1264         flags = le16_to_cpu(snapshot.flags);
1265
1266         LAST_ADD_TIME_CHECK(vq);
1267         LAST_ADD_TIME_INVALID(vq);
1268
1269         if (flags != VRING_PACKED_EVENT_FLAG_DESC) {
1270                 needs_kick = (flags != VRING_PACKED_EVENT_FLAG_DISABLE);
1271                 goto out;
1272         }
1273
1274         off_wrap = le16_to_cpu(snapshot.off_wrap);
1275
1276         wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR;
1277         event_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR);
1278         if (wrap_counter != vq->packed.avail_wrap_counter)
1279                 event_idx -= vq->packed.vring.num;
1280
1281         needs_kick = vring_need_event(event_idx, new, old);
1282 out:
1283         END_USE(vq);
1284         return needs_kick;
1285 }
1286
1287 static void detach_buf_packed(struct vring_virtqueue *vq,
1288                               unsigned int id, void **ctx)
1289 {
1290         struct vring_desc_state_packed *state = NULL;
1291         struct vring_packed_desc *desc;
1292         unsigned int i, curr;
1293
1294         state = &vq->packed.desc_state[id];
1295
1296         /* Clear data ptr. */
1297         state->data = NULL;
1298
1299         vq->packed.desc_state[state->last].next = vq->free_head;
1300         vq->free_head = id;
1301         vq->vq.num_free += state->num;
1302
1303         if (unlikely(vq->use_dma_api)) {
1304                 curr = id;
1305                 for (i = 0; i < state->num; i++) {
1306                         vring_unmap_state_packed(vq,
1307                                 &vq->packed.desc_extra[curr]);
1308                         curr = vq->packed.desc_state[curr].next;
1309                 }
1310         }
1311
1312         if (vq->indirect) {
1313                 u32 len;
1314
1315                 /* Free the indirect table, if any, now that it's unmapped. */
1316                 desc = state->indir_desc;
1317                 if (!desc)
1318                         return;
1319
1320                 if (vq->use_dma_api) {
1321                         len = vq->packed.desc_extra[id].len;
1322                         for (i = 0; i < len / sizeof(struct vring_packed_desc);
1323                                         i++)
1324                                 vring_unmap_desc_packed(vq, &desc[i]);
1325                 }
1326                 kfree(desc);
1327                 state->indir_desc = NULL;
1328         } else if (ctx) {
1329                 *ctx = state->indir_desc;
1330         }
1331 }
1332
1333 static inline bool is_used_desc_packed(const struct vring_virtqueue *vq,
1334                                        u16 idx, bool used_wrap_counter)
1335 {
1336         bool avail, used;
1337         u16 flags;
1338
1339         flags = le16_to_cpu(vq->packed.vring.desc[idx].flags);
1340         avail = !!(flags & (1 << VRING_PACKED_DESC_F_AVAIL));
1341         used = !!(flags & (1 << VRING_PACKED_DESC_F_USED));
1342
1343         return avail == used && used == used_wrap_counter;
1344 }
1345
1346 static inline bool more_used_packed(const struct vring_virtqueue *vq)
1347 {
1348         return is_used_desc_packed(vq, vq->last_used_idx,
1349                         vq->packed.used_wrap_counter);
1350 }
1351
1352 static void *virtqueue_get_buf_ctx_packed(struct virtqueue *_vq,
1353                                           unsigned int *len,
1354                                           void **ctx)
1355 {
1356         struct vring_virtqueue *vq = to_vvq(_vq);
1357         u16 last_used, id;
1358         void *ret;
1359
1360         START_USE(vq);
1361
1362         if (unlikely(vq->broken)) {
1363                 END_USE(vq);
1364                 return NULL;
1365         }
1366
1367         if (!more_used_packed(vq)) {
1368                 pr_debug("No more buffers in queue\n");
1369                 END_USE(vq);
1370                 return NULL;
1371         }
1372
1373         /* Only get used elements after they have been exposed by host. */
1374         virtio_rmb(vq->weak_barriers);
1375
1376         last_used = vq->last_used_idx;
1377         id = le16_to_cpu(vq->packed.vring.desc[last_used].id);
1378         *len = le32_to_cpu(vq->packed.vring.desc[last_used].len);
1379
1380         if (unlikely(id >= vq->packed.vring.num)) {
1381                 BAD_RING(vq, "id %u out of range\n", id);
1382                 return NULL;
1383         }
1384         if (unlikely(!vq->packed.desc_state[id].data)) {
1385                 BAD_RING(vq, "id %u is not a head!\n", id);
1386                 return NULL;
1387         }
1388
1389         /* detach_buf_packed clears data, so grab it now. */
1390         ret = vq->packed.desc_state[id].data;
1391         detach_buf_packed(vq, id, ctx);
1392
1393         vq->last_used_idx += vq->packed.desc_state[id].num;
1394         if (unlikely(vq->last_used_idx >= vq->packed.vring.num)) {
1395                 vq->last_used_idx -= vq->packed.vring.num;
1396                 vq->packed.used_wrap_counter ^= 1;
1397         }
1398
1399         /*
1400          * If we expect an interrupt for the next entry, tell host
1401          * by writing event index and flush out the write before
1402          * the read in the next get_buf call.
1403          */
1404         if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC)
1405                 virtio_store_mb(vq->weak_barriers,
1406                                 &vq->packed.vring.driver->off_wrap,
1407                                 cpu_to_le16(vq->last_used_idx |
1408                                         (vq->packed.used_wrap_counter <<
1409                                          VRING_PACKED_EVENT_F_WRAP_CTR)));
1410
1411         LAST_ADD_TIME_INVALID(vq);
1412
1413         END_USE(vq);
1414         return ret;
1415 }
1416
1417 static void virtqueue_disable_cb_packed(struct virtqueue *_vq)
1418 {
1419         struct vring_virtqueue *vq = to_vvq(_vq);
1420
1421         if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) {
1422                 vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
1423                 vq->packed.vring.driver->flags =
1424                         cpu_to_le16(vq->packed.event_flags_shadow);
1425         }
1426 }
1427
1428 static unsigned virtqueue_enable_cb_prepare_packed(struct virtqueue *_vq)
1429 {
1430         struct vring_virtqueue *vq = to_vvq(_vq);
1431
1432         START_USE(vq);
1433
1434         /*
1435          * We optimistically turn back on interrupts, then check if there was
1436          * more to do.
1437          */
1438
1439         if (vq->event) {
1440                 vq->packed.vring.driver->off_wrap =
1441                         cpu_to_le16(vq->last_used_idx |
1442                                 (vq->packed.used_wrap_counter <<
1443                                  VRING_PACKED_EVENT_F_WRAP_CTR));
1444                 /*
1445                  * We need to update event offset and event wrap
1446                  * counter first before updating event flags.
1447                  */
1448                 virtio_wmb(vq->weak_barriers);
1449         }
1450
1451         if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
1452                 vq->packed.event_flags_shadow = vq->event ?
1453                                 VRING_PACKED_EVENT_FLAG_DESC :
1454                                 VRING_PACKED_EVENT_FLAG_ENABLE;
1455                 vq->packed.vring.driver->flags =
1456                                 cpu_to_le16(vq->packed.event_flags_shadow);
1457         }
1458
1459         END_USE(vq);
1460         return vq->last_used_idx | ((u16)vq->packed.used_wrap_counter <<
1461                         VRING_PACKED_EVENT_F_WRAP_CTR);
1462 }
1463
1464 static bool virtqueue_poll_packed(struct virtqueue *_vq, u16 off_wrap)
1465 {
1466         struct vring_virtqueue *vq = to_vvq(_vq);
1467         bool wrap_counter;
1468         u16 used_idx;
1469
1470         wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR;
1471         used_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR);
1472
1473         return is_used_desc_packed(vq, used_idx, wrap_counter);
1474 }
1475
1476 static bool virtqueue_enable_cb_delayed_packed(struct virtqueue *_vq)
1477 {
1478         struct vring_virtqueue *vq = to_vvq(_vq);
1479         u16 used_idx, wrap_counter;
1480         u16 bufs;
1481
1482         START_USE(vq);
1483
1484         /*
1485          * We optimistically turn back on interrupts, then check if there was
1486          * more to do.
1487          */
1488
1489         if (vq->event) {
1490                 /* TODO: tune this threshold */
1491                 bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4;
1492                 wrap_counter = vq->packed.used_wrap_counter;
1493
1494                 used_idx = vq->last_used_idx + bufs;
1495                 if (used_idx >= vq->packed.vring.num) {
1496                         used_idx -= vq->packed.vring.num;
1497                         wrap_counter ^= 1;
1498                 }
1499
1500                 vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx |
1501                         (wrap_counter << VRING_PACKED_EVENT_F_WRAP_CTR));
1502
1503                 /*
1504                  * We need to update event offset and event wrap
1505                  * counter first before updating event flags.
1506                  */
1507                 virtio_wmb(vq->weak_barriers);
1508         } else {
1509                 used_idx = vq->last_used_idx;
1510                 wrap_counter = vq->packed.used_wrap_counter;
1511         }
1512
1513         if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
1514                 vq->packed.event_flags_shadow = vq->event ?
1515                                 VRING_PACKED_EVENT_FLAG_DESC :
1516                                 VRING_PACKED_EVENT_FLAG_ENABLE;
1517                 vq->packed.vring.driver->flags =
1518                                 cpu_to_le16(vq->packed.event_flags_shadow);
1519         }
1520
1521         /*
1522          * We need to update event suppression structure first
1523          * before re-checking for more used buffers.
1524          */
1525         virtio_mb(vq->weak_barriers);
1526
1527         if (is_used_desc_packed(vq, used_idx, wrap_counter)) {
1528                 END_USE(vq);
1529                 return false;
1530         }
1531
1532         END_USE(vq);
1533         return true;
1534 }
1535
1536 static void *virtqueue_detach_unused_buf_packed(struct virtqueue *_vq)
1537 {
1538         struct vring_virtqueue *vq = to_vvq(_vq);
1539         unsigned int i;
1540         void *buf;
1541
1542         START_USE(vq);
1543
1544         for (i = 0; i < vq->packed.vring.num; i++) {
1545                 if (!vq->packed.desc_state[i].data)
1546                         continue;
1547                 /* detach_buf clears data, so grab it now. */
1548                 buf = vq->packed.desc_state[i].data;
1549                 detach_buf_packed(vq, i, NULL);
1550                 END_USE(vq);
1551                 return buf;
1552         }
1553         /* That should have freed everything. */
1554         BUG_ON(vq->vq.num_free != vq->packed.vring.num);
1555
1556         END_USE(vq);
1557         return NULL;
1558 }
1559
1560 static struct virtqueue *vring_create_virtqueue_packed(
1561         unsigned int index,
1562         unsigned int num,
1563         unsigned int vring_align,
1564         struct virtio_device *vdev,
1565         bool weak_barriers,
1566         bool may_reduce_num,
1567         bool context,
1568         bool (*notify)(struct virtqueue *),
1569         void (*callback)(struct virtqueue *),
1570         const char *name)
1571 {
1572         struct vring_virtqueue *vq;
1573         struct vring_packed_desc *ring;
1574         struct vring_packed_desc_event *driver, *device;
1575         dma_addr_t ring_dma_addr, driver_event_dma_addr, device_event_dma_addr;
1576         size_t ring_size_in_bytes, event_size_in_bytes;
1577         unsigned int i;
1578
1579         ring_size_in_bytes = num * sizeof(struct vring_packed_desc);
1580
1581         ring = vring_alloc_queue(vdev, ring_size_in_bytes,
1582                                  &ring_dma_addr,
1583                                  GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
1584         if (!ring)
1585                 goto err_ring;
1586
1587         event_size_in_bytes = sizeof(struct vring_packed_desc_event);
1588
1589         driver = vring_alloc_queue(vdev, event_size_in_bytes,
1590                                    &driver_event_dma_addr,
1591                                    GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
1592         if (!driver)
1593                 goto err_driver;
1594
1595         device = vring_alloc_queue(vdev, event_size_in_bytes,
1596                                    &device_event_dma_addr,
1597                                    GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
1598         if (!device)
1599                 goto err_device;
1600
1601         vq = kmalloc(sizeof(*vq), GFP_KERNEL);
1602         if (!vq)
1603                 goto err_vq;
1604
1605         vq->vq.callback = callback;
1606         vq->vq.vdev = vdev;
1607         vq->vq.name = name;
1608         vq->vq.num_free = num;
1609         vq->vq.index = index;
1610         vq->we_own_ring = true;
1611         vq->notify = notify;
1612         vq->weak_barriers = weak_barriers;
1613         vq->broken = false;
1614         vq->last_used_idx = 0;
1615         vq->num_added = 0;
1616         vq->packed_ring = true;
1617         vq->use_dma_api = vring_use_dma_api(vdev);
1618         list_add_tail(&vq->vq.list, &vdev->vqs);
1619 #ifdef DEBUG
1620         vq->in_use = false;
1621         vq->last_add_time_valid = false;
1622 #endif
1623
1624         vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
1625                 !context;
1626         vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
1627
1628         if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
1629                 vq->weak_barriers = false;
1630
1631         vq->packed.ring_dma_addr = ring_dma_addr;
1632         vq->packed.driver_event_dma_addr = driver_event_dma_addr;
1633         vq->packed.device_event_dma_addr = device_event_dma_addr;
1634
1635         vq->packed.ring_size_in_bytes = ring_size_in_bytes;
1636         vq->packed.event_size_in_bytes = event_size_in_bytes;
1637
1638         vq->packed.vring.num = num;
1639         vq->packed.vring.desc = ring;
1640         vq->packed.vring.driver = driver;
1641         vq->packed.vring.device = device;
1642
1643         vq->packed.next_avail_idx = 0;
1644         vq->packed.avail_wrap_counter = 1;
1645         vq->packed.used_wrap_counter = 1;
1646         vq->packed.event_flags_shadow = 0;
1647         vq->packed.avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL;
1648
1649         vq->packed.desc_state = kmalloc_array(num,
1650                         sizeof(struct vring_desc_state_packed),
1651                         GFP_KERNEL);
1652         if (!vq->packed.desc_state)
1653                 goto err_desc_state;
1654
1655         memset(vq->packed.desc_state, 0,
1656                 num * sizeof(struct vring_desc_state_packed));
1657
1658         /* Put everything in free lists. */
1659         vq->free_head = 0;
1660         for (i = 0; i < num-1; i++)
1661                 vq->packed.desc_state[i].next = i + 1;
1662
1663         vq->packed.desc_extra = kmalloc_array(num,
1664                         sizeof(struct vring_desc_extra_packed),
1665                         GFP_KERNEL);
1666         if (!vq->packed.desc_extra)
1667                 goto err_desc_extra;
1668
1669         memset(vq->packed.desc_extra, 0,
1670                 num * sizeof(struct vring_desc_extra_packed));
1671
1672         /* No callback?  Tell other side not to bother us. */
1673         if (!callback) {
1674                 vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
1675                 vq->packed.vring.driver->flags =
1676                         cpu_to_le16(vq->packed.event_flags_shadow);
1677         }
1678
1679         return &vq->vq;
1680
1681 err_desc_extra:
1682         kfree(vq->packed.desc_state);
1683 err_desc_state:
1684         kfree(vq);
1685 err_vq:
1686         vring_free_queue(vdev, event_size_in_bytes, device, ring_dma_addr);
1687 err_device:
1688         vring_free_queue(vdev, event_size_in_bytes, driver, ring_dma_addr);
1689 err_driver:
1690         vring_free_queue(vdev, ring_size_in_bytes, ring, ring_dma_addr);
1691 err_ring:
1692         return NULL;
1693 }
1694
1695
1696 /*
1697  * Generic functions and exported symbols.
1698  */
1699
1700 static inline int virtqueue_add(struct virtqueue *_vq,
1701                                 struct scatterlist *sgs[],
1702                                 unsigned int total_sg,
1703                                 unsigned int out_sgs,
1704                                 unsigned int in_sgs,
1705                                 void *data,
1706                                 void *ctx,
1707                                 gfp_t gfp)
1708 {
1709         struct vring_virtqueue *vq = to_vvq(_vq);
1710
1711         return vq->packed_ring ? virtqueue_add_packed(_vq, sgs, total_sg,
1712                                         out_sgs, in_sgs, data, ctx, gfp) :
1713                                  virtqueue_add_split(_vq, sgs, total_sg,
1714                                         out_sgs, in_sgs, data, ctx, gfp);
1715 }
1716
1717 /**
1718  * virtqueue_add_sgs - expose buffers to other end
1719  * @vq: the struct virtqueue we're talking about.
1720  * @sgs: array of terminated scatterlists.
1721  * @out_num: the number of scatterlists readable by other side
1722  * @in_num: the number of scatterlists which are writable (after readable ones)
1723  * @data: the token identifying the buffer.
1724  * @gfp: how to do memory allocations (if necessary).
1725  *
1726  * Caller must ensure we don't call this with other virtqueue operations
1727  * at the same time (except where noted).
1728  *
1729  * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1730  */
1731 int virtqueue_add_sgs(struct virtqueue *_vq,
1732                       struct scatterlist *sgs[],
1733                       unsigned int out_sgs,
1734                       unsigned int in_sgs,
1735                       void *data,
1736                       gfp_t gfp)
1737 {
1738         unsigned int i, total_sg = 0;
1739
1740         /* Count them first. */
1741         for (i = 0; i < out_sgs + in_sgs; i++) {
1742                 struct scatterlist *sg;
1743
1744                 for (sg = sgs[i]; sg; sg = sg_next(sg))
1745                         total_sg++;
1746         }
1747         return virtqueue_add(_vq, sgs, total_sg, out_sgs, in_sgs,
1748                              data, NULL, gfp);
1749 }
1750 EXPORT_SYMBOL_GPL(virtqueue_add_sgs);
1751
1752 /**
1753  * virtqueue_add_outbuf - expose output buffers to other end
1754  * @vq: the struct virtqueue we're talking about.
1755  * @sg: scatterlist (must be well-formed and terminated!)
1756  * @num: the number of entries in @sg readable by other side
1757  * @data: the token identifying the buffer.
1758  * @gfp: how to do memory allocations (if necessary).
1759  *
1760  * Caller must ensure we don't call this with other virtqueue operations
1761  * at the same time (except where noted).
1762  *
1763  * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1764  */
1765 int virtqueue_add_outbuf(struct virtqueue *vq,
1766                          struct scatterlist *sg, unsigned int num,
1767                          void *data,
1768                          gfp_t gfp)
1769 {
1770         return virtqueue_add(vq, &sg, num, 1, 0, data, NULL, gfp);
1771 }
1772 EXPORT_SYMBOL_GPL(virtqueue_add_outbuf);
1773
1774 /**
1775  * virtqueue_add_inbuf - expose input buffers to other end
1776  * @vq: the struct virtqueue we're talking about.
1777  * @sg: scatterlist (must be well-formed and terminated!)
1778  * @num: the number of entries in @sg writable by other side
1779  * @data: the token identifying the buffer.
1780  * @gfp: how to do memory allocations (if necessary).
1781  *
1782  * Caller must ensure we don't call this with other virtqueue operations
1783  * at the same time (except where noted).
1784  *
1785  * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1786  */
1787 int virtqueue_add_inbuf(struct virtqueue *vq,
1788                         struct scatterlist *sg, unsigned int num,
1789                         void *data,
1790                         gfp_t gfp)
1791 {
1792         return virtqueue_add(vq, &sg, num, 0, 1, data, NULL, gfp);
1793 }
1794 EXPORT_SYMBOL_GPL(virtqueue_add_inbuf);
1795
1796 /**
1797  * virtqueue_add_inbuf_ctx - expose input buffers to other end
1798  * @vq: the struct virtqueue we're talking about.
1799  * @sg: scatterlist (must be well-formed and terminated!)
1800  * @num: the number of entries in @sg writable by other side
1801  * @data: the token identifying the buffer.
1802  * @ctx: extra context for the token
1803  * @gfp: how to do memory allocations (if necessary).
1804  *
1805  * Caller must ensure we don't call this with other virtqueue operations
1806  * at the same time (except where noted).
1807  *
1808  * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1809  */
1810 int virtqueue_add_inbuf_ctx(struct virtqueue *vq,
1811                         struct scatterlist *sg, unsigned int num,
1812                         void *data,
1813                         void *ctx,
1814                         gfp_t gfp)
1815 {
1816         return virtqueue_add(vq, &sg, num, 0, 1, data, ctx, gfp);
1817 }
1818 EXPORT_SYMBOL_GPL(virtqueue_add_inbuf_ctx);
1819
1820 /**
1821  * virtqueue_kick_prepare - first half of split virtqueue_kick call.
1822  * @vq: the struct virtqueue
1823  *
1824  * Instead of virtqueue_kick(), you can do:
1825  *      if (virtqueue_kick_prepare(vq))
1826  *              virtqueue_notify(vq);
1827  *
1828  * This is sometimes useful because the virtqueue_kick_prepare() needs
1829  * to be serialized, but the actual virtqueue_notify() call does not.
1830  */
1831 bool virtqueue_kick_prepare(struct virtqueue *_vq)
1832 {
1833         struct vring_virtqueue *vq = to_vvq(_vq);
1834
1835         return vq->packed_ring ? virtqueue_kick_prepare_packed(_vq) :
1836                                  virtqueue_kick_prepare_split(_vq);
1837 }
1838 EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
1839
1840 /**
1841  * virtqueue_notify - second half of split virtqueue_kick call.
1842  * @vq: the struct virtqueue
1843  *
1844  * This does not need to be serialized.
1845  *
1846  * Returns false if host notify failed or queue is broken, otherwise true.
1847  */
1848 bool virtqueue_notify(struct virtqueue *_vq)
1849 {
1850         struct vring_virtqueue *vq = to_vvq(_vq);
1851
1852         if (unlikely(vq->broken))
1853                 return false;
1854
1855         /* Prod other side to tell it about changes. */
1856         if (!vq->notify(_vq)) {
1857                 vq->broken = true;
1858                 return false;
1859         }
1860         return true;
1861 }
1862 EXPORT_SYMBOL_GPL(virtqueue_notify);
1863
1864 /**
1865  * virtqueue_kick - update after add_buf
1866  * @vq: the struct virtqueue
1867  *
1868  * After one or more virtqueue_add_* calls, invoke this to kick
1869  * the other side.
1870  *
1871  * Caller must ensure we don't call this with other virtqueue
1872  * operations at the same time (except where noted).
1873  *
1874  * Returns false if kick failed, otherwise true.
1875  */
1876 bool virtqueue_kick(struct virtqueue *vq)
1877 {
1878         if (virtqueue_kick_prepare(vq))
1879                 return virtqueue_notify(vq);
1880         return true;
1881 }
1882 EXPORT_SYMBOL_GPL(virtqueue_kick);
1883
1884 /**
1885  * virtqueue_get_buf - get the next used buffer
1886  * @vq: the struct virtqueue we're talking about.
1887  * @len: the length written into the buffer
1888  *
1889  * If the device wrote data into the buffer, @len will be set to the
1890  * amount written.  This means you don't need to clear the buffer
1891  * beforehand to ensure there's no data leakage in the case of short
1892  * writes.
1893  *
1894  * Caller must ensure we don't call this with other virtqueue
1895  * operations at the same time (except where noted).
1896  *
1897  * Returns NULL if there are no used buffers, or the "data" token
1898  * handed to virtqueue_add_*().
1899  */
1900 void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,
1901                             void **ctx)
1902 {
1903         struct vring_virtqueue *vq = to_vvq(_vq);
1904
1905         return vq->packed_ring ? virtqueue_get_buf_ctx_packed(_vq, len, ctx) :
1906                                  virtqueue_get_buf_ctx_split(_vq, len, ctx);
1907 }
1908 EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx);
1909
1910 void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
1911 {
1912         return virtqueue_get_buf_ctx(_vq, len, NULL);
1913 }
1914 EXPORT_SYMBOL_GPL(virtqueue_get_buf);
1915 /**
1916  * virtqueue_disable_cb - disable callbacks
1917  * @vq: the struct virtqueue we're talking about.
1918  *
1919  * Note that this is not necessarily synchronous, hence unreliable and only
1920  * useful as an optimization.
1921  *
1922  * Unlike other operations, this need not be serialized.
1923  */
1924 void virtqueue_disable_cb(struct virtqueue *_vq)
1925 {
1926         struct vring_virtqueue *vq = to_vvq(_vq);
1927
1928         if (vq->packed_ring)
1929                 virtqueue_disable_cb_packed(_vq);
1930         else
1931                 virtqueue_disable_cb_split(_vq);
1932 }
1933 EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
1934
1935 /**
1936  * virtqueue_enable_cb_prepare - restart callbacks after disable_cb
1937  * @vq: the struct virtqueue we're talking about.
1938  *
1939  * This re-enables callbacks; it returns current queue state
1940  * in an opaque unsigned value. This value should be later tested by
1941  * virtqueue_poll, to detect a possible race between the driver checking for
1942  * more work, and enabling callbacks.
1943  *
1944  * Caller must ensure we don't call this with other virtqueue
1945  * operations at the same time (except where noted).
1946  */
1947 unsigned virtqueue_enable_cb_prepare(struct virtqueue *_vq)
1948 {
1949         struct vring_virtqueue *vq = to_vvq(_vq);
1950
1951         return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(_vq) :
1952                                  virtqueue_enable_cb_prepare_split(_vq);
1953 }
1954 EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
1955
1956 /**
1957  * virtqueue_poll - query pending used buffers
1958  * @vq: the struct virtqueue we're talking about.
1959  * @last_used_idx: virtqueue state (from call to virtqueue_enable_cb_prepare).
1960  *
1961  * Returns "true" if there are pending used buffers in the queue.
1962  *
1963  * This does not need to be serialized.
1964  */
1965 bool virtqueue_poll(struct virtqueue *_vq, unsigned last_used_idx)
1966 {
1967         struct vring_virtqueue *vq = to_vvq(_vq);
1968
1969         virtio_mb(vq->weak_barriers);
1970         return vq->packed_ring ? virtqueue_poll_packed(_vq, last_used_idx) :
1971                                  virtqueue_poll_split(_vq, last_used_idx);
1972 }
1973 EXPORT_SYMBOL_GPL(virtqueue_poll);
1974
1975 /**
1976  * virtqueue_enable_cb - restart callbacks after disable_cb.
1977  * @vq: the struct virtqueue we're talking about.
1978  *
1979  * This re-enables callbacks; it returns "false" if there are pending
1980  * buffers in the queue, to detect a possible race between the driver
1981  * checking for more work, and enabling callbacks.
1982  *
1983  * Caller must ensure we don't call this with other virtqueue
1984  * operations at the same time (except where noted).
1985  */
1986 bool virtqueue_enable_cb(struct virtqueue *_vq)
1987 {
1988         unsigned last_used_idx = virtqueue_enable_cb_prepare(_vq);
1989
1990         return !virtqueue_poll(_vq, last_used_idx);
1991 }
1992 EXPORT_SYMBOL_GPL(virtqueue_enable_cb);
1993
1994 /**
1995  * virtqueue_enable_cb_delayed - restart callbacks after disable_cb.
1996  * @vq: the struct virtqueue we're talking about.
1997  *
1998  * This re-enables callbacks but hints to the other side to delay
1999  * interrupts until most of the available buffers have been processed;
2000  * it returns "false" if there are many pending buffers in the queue,
2001  * to detect a possible race between the driver checking for more work,
2002  * and enabling callbacks.
2003  *
2004  * Caller must ensure we don't call this with other virtqueue
2005  * operations at the same time (except where noted).
2006  */
2007 bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
2008 {
2009         struct vring_virtqueue *vq = to_vvq(_vq);
2010
2011         return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(_vq) :
2012                                  virtqueue_enable_cb_delayed_split(_vq);
2013 }
2014 EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
2015
2016 /**
2017  * virtqueue_detach_unused_buf - detach first unused buffer
2018  * @vq: the struct virtqueue we're talking about.
2019  *
2020  * Returns NULL or the "data" token handed to virtqueue_add_*().
2021  * This is not valid on an active queue; it is useful only for device
2022  * shutdown.
2023  */
2024 void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
2025 {
2026         struct vring_virtqueue *vq = to_vvq(_vq);
2027
2028         return vq->packed_ring ? virtqueue_detach_unused_buf_packed(_vq) :
2029                                  virtqueue_detach_unused_buf_split(_vq);
2030 }
2031 EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
2032
2033 static inline bool more_used(const struct vring_virtqueue *vq)
2034 {
2035         return vq->packed_ring ? more_used_packed(vq) : more_used_split(vq);
2036 }
2037
2038 irqreturn_t vring_interrupt(int irq, void *_vq)
2039 {
2040         struct vring_virtqueue *vq = to_vvq(_vq);
2041
2042         if (!more_used(vq)) {
2043                 pr_debug("virtqueue interrupt with no work for %p\n", vq);
2044                 return IRQ_NONE;
2045         }
2046
2047         if (unlikely(vq->broken))
2048                 return IRQ_HANDLED;
2049
2050         pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback);
2051         if (vq->vq.callback)
2052                 vq->vq.callback(&vq->vq);
2053
2054         return IRQ_HANDLED;
2055 }
2056 EXPORT_SYMBOL_GPL(vring_interrupt);
2057
2058 /* Only available for split ring */
2059 struct virtqueue *__vring_new_virtqueue(unsigned int index,
2060                                         struct vring vring,
2061                                         struct virtio_device *vdev,
2062                                         bool weak_barriers,
2063                                         bool context,
2064                                         bool (*notify)(struct virtqueue *),
2065                                         void (*callback)(struct virtqueue *),
2066                                         const char *name)
2067 {
2068         unsigned int i;
2069         struct vring_virtqueue *vq;
2070
2071         if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2072                 return NULL;
2073
2074         vq = kmalloc(sizeof(*vq), GFP_KERNEL);
2075         if (!vq)
2076                 return NULL;
2077
2078         vq->packed_ring = false;
2079         vq->vq.callback = callback;
2080         vq->vq.vdev = vdev;
2081         vq->vq.name = name;
2082         vq->vq.num_free = vring.num;
2083         vq->vq.index = index;
2084         vq->we_own_ring = false;
2085         vq->notify = notify;
2086         vq->weak_barriers = weak_barriers;
2087         vq->broken = false;
2088         vq->last_used_idx = 0;
2089         vq->num_added = 0;
2090         vq->use_dma_api = vring_use_dma_api(vdev);
2091         list_add_tail(&vq->vq.list, &vdev->vqs);
2092 #ifdef DEBUG
2093         vq->in_use = false;
2094         vq->last_add_time_valid = false;
2095 #endif
2096
2097         vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
2098                 !context;
2099         vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
2100
2101         if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
2102                 vq->weak_barriers = false;
2103
2104         vq->split.queue_dma_addr = 0;
2105         vq->split.queue_size_in_bytes = 0;
2106
2107         vq->split.vring = vring;
2108         vq->split.avail_flags_shadow = 0;
2109         vq->split.avail_idx_shadow = 0;
2110
2111         /* No callback?  Tell other side not to bother us. */
2112         if (!callback) {
2113                 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
2114                 if (!vq->event)
2115                         vq->split.vring.avail->flags = cpu_to_virtio16(vdev,
2116                                         vq->split.avail_flags_shadow);
2117         }
2118
2119         vq->split.desc_state = kmalloc_array(vring.num,
2120                         sizeof(struct vring_desc_state_split), GFP_KERNEL);
2121         if (!vq->split.desc_state) {
2122                 kfree(vq);
2123                 return NULL;
2124         }
2125
2126         /* Put everything in free lists. */
2127         vq->free_head = 0;
2128         for (i = 0; i < vring.num-1; i++)
2129                 vq->split.vring.desc[i].next = cpu_to_virtio16(vdev, i + 1);
2130         memset(vq->split.desc_state, 0, vring.num *
2131                         sizeof(struct vring_desc_state_split));
2132
2133         return &vq->vq;
2134 }
2135 EXPORT_SYMBOL_GPL(__vring_new_virtqueue);
2136
2137 struct virtqueue *vring_create_virtqueue(
2138         unsigned int index,
2139         unsigned int num,
2140         unsigned int vring_align,
2141         struct virtio_device *vdev,
2142         bool weak_barriers,
2143         bool may_reduce_num,
2144         bool context,
2145         bool (*notify)(struct virtqueue *),
2146         void (*callback)(struct virtqueue *),
2147         const char *name)
2148 {
2149
2150         if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2151                 return vring_create_virtqueue_packed(index, num, vring_align,
2152                                 vdev, weak_barriers, may_reduce_num,
2153                                 context, notify, callback, name);
2154
2155         return vring_create_virtqueue_split(index, num, vring_align,
2156                         vdev, weak_barriers, may_reduce_num,
2157                         context, notify, callback, name);
2158 }
2159 EXPORT_SYMBOL_GPL(vring_create_virtqueue);
2160
2161 /* Only available for split ring */
2162 struct virtqueue *vring_new_virtqueue(unsigned int index,
2163                                       unsigned int num,
2164                                       unsigned int vring_align,
2165                                       struct virtio_device *vdev,
2166                                       bool weak_barriers,
2167                                       bool context,
2168                                       void *pages,
2169                                       bool (*notify)(struct virtqueue *vq),
2170                                       void (*callback)(struct virtqueue *vq),
2171                                       const char *name)
2172 {
2173         struct vring vring;
2174
2175         if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2176                 return NULL;
2177
2178         vring_init(&vring, num, pages, vring_align);
2179         return __vring_new_virtqueue(index, vring, vdev, weak_barriers, context,
2180                                      notify, callback, name);
2181 }
2182 EXPORT_SYMBOL_GPL(vring_new_virtqueue);
2183
2184 void vring_del_virtqueue(struct virtqueue *_vq)
2185 {
2186         struct vring_virtqueue *vq = to_vvq(_vq);
2187
2188         if (vq->we_own_ring) {
2189                 if (vq->packed_ring) {
2190                         vring_free_queue(vq->vq.vdev,
2191                                          vq->packed.ring_size_in_bytes,
2192                                          vq->packed.vring.desc,
2193                                          vq->packed.ring_dma_addr);
2194
2195                         vring_free_queue(vq->vq.vdev,
2196                                          vq->packed.event_size_in_bytes,
2197                                          vq->packed.vring.driver,
2198                                          vq->packed.driver_event_dma_addr);
2199
2200                         vring_free_queue(vq->vq.vdev,
2201                                          vq->packed.event_size_in_bytes,
2202                                          vq->packed.vring.device,
2203                                          vq->packed.device_event_dma_addr);
2204
2205                         kfree(vq->packed.desc_state);
2206                         kfree(vq->packed.desc_extra);
2207                 } else {
2208                         vring_free_queue(vq->vq.vdev,
2209                                          vq->split.queue_size_in_bytes,
2210                                          vq->split.vring.desc,
2211                                          vq->split.queue_dma_addr);
2212
2213                         kfree(vq->split.desc_state);
2214                 }
2215         }
2216         list_del(&_vq->list);
2217         kfree(vq);
2218 }
2219 EXPORT_SYMBOL_GPL(vring_del_virtqueue);
2220
2221 /* Manipulates transport-specific feature bits. */
2222 void vring_transport_features(struct virtio_device *vdev)
2223 {
2224         unsigned int i;
2225
2226         for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) {
2227                 switch (i) {
2228                 case VIRTIO_RING_F_INDIRECT_DESC:
2229                         break;
2230                 case VIRTIO_RING_F_EVENT_IDX:
2231                         break;
2232                 case VIRTIO_F_VERSION_1:
2233                         break;
2234                 case VIRTIO_F_IOMMU_PLATFORM:
2235                         break;
2236                 case VIRTIO_F_RING_PACKED:
2237                         break;
2238                 case VIRTIO_F_ORDER_PLATFORM:
2239                         break;
2240                 default:
2241                         /* We don't understand this bit. */
2242                         __virtio_clear_bit(vdev, i);
2243                 }
2244         }
2245 }
2246 EXPORT_SYMBOL_GPL(vring_transport_features);
2247
2248 /**
2249  * virtqueue_get_vring_size - return the size of the virtqueue's vring
2250  * @vq: the struct virtqueue containing the vring of interest.
2251  *
2252  * Returns the size of the vring.  This is mainly used for boasting to
2253  * userspace.  Unlike other operations, this need not be serialized.
2254  */
2255 unsigned int virtqueue_get_vring_size(struct virtqueue *_vq)
2256 {
2257
2258         struct vring_virtqueue *vq = to_vvq(_vq);
2259
2260         return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;
2261 }
2262 EXPORT_SYMBOL_GPL(virtqueue_get_vring_size);
2263
2264 bool virtqueue_is_broken(struct virtqueue *_vq)
2265 {
2266         struct vring_virtqueue *vq = to_vvq(_vq);
2267
2268         return vq->broken;
2269 }
2270 EXPORT_SYMBOL_GPL(virtqueue_is_broken);
2271
2272 /*
2273  * This should prevent the device from being used, allowing drivers to
2274  * recover.  You may need to grab appropriate locks to flush.
2275  */
2276 void virtio_break_device(struct virtio_device *dev)
2277 {
2278         struct virtqueue *_vq;
2279
2280         list_for_each_entry(_vq, &dev->vqs, list) {
2281                 struct vring_virtqueue *vq = to_vvq(_vq);
2282                 vq->broken = true;
2283         }
2284 }
2285 EXPORT_SYMBOL_GPL(virtio_break_device);
2286
2287 dma_addr_t virtqueue_get_desc_addr(struct virtqueue *_vq)
2288 {
2289         struct vring_virtqueue *vq = to_vvq(_vq);
2290
2291         BUG_ON(!vq->we_own_ring);
2292
2293         if (vq->packed_ring)
2294                 return vq->packed.ring_dma_addr;
2295
2296         return vq->split.queue_dma_addr;
2297 }
2298 EXPORT_SYMBOL_GPL(virtqueue_get_desc_addr);
2299
2300 dma_addr_t virtqueue_get_avail_addr(struct virtqueue *_vq)
2301 {
2302         struct vring_virtqueue *vq = to_vvq(_vq);
2303
2304         BUG_ON(!vq->we_own_ring);
2305
2306         if (vq->packed_ring)
2307                 return vq->packed.driver_event_dma_addr;
2308
2309         return vq->split.queue_dma_addr +
2310                 ((char *)vq->split.vring.avail - (char *)vq->split.vring.desc);
2311 }
2312 EXPORT_SYMBOL_GPL(virtqueue_get_avail_addr);
2313
2314 dma_addr_t virtqueue_get_used_addr(struct virtqueue *_vq)
2315 {
2316         struct vring_virtqueue *vq = to_vvq(_vq);
2317
2318         BUG_ON(!vq->we_own_ring);
2319
2320         if (vq->packed_ring)
2321                 return vq->packed.device_event_dma_addr;
2322
2323         return vq->split.queue_dma_addr +
2324                 ((char *)vq->split.vring.used - (char *)vq->split.vring.desc);
2325 }
2326 EXPORT_SYMBOL_GPL(virtqueue_get_used_addr);
2327
2328 /* Only available for split ring */
2329 const struct vring *virtqueue_get_vring(struct virtqueue *vq)
2330 {
2331         return &to_vvq(vq)->split.vring;
2332 }
2333 EXPORT_SYMBOL_GPL(virtqueue_get_vring);
2334
2335 MODULE_LICENSE("GPL");