2 * Intel MIC Platform Software Stack (MPSS)
4 * Copyright(c) 2016 Intel Corporation.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License, version 2, as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
15 * The full GNU General Public License is included in this distribution in
16 * the file called "COPYING".
20 * virtio for kvm on s390
22 * Copyright IBM Corp. 2008
24 * This program is free software; you can redistribute it and/or modify
25 * it under the terms of the GNU General Public License (version 2 only)
26 * as published by the Free Software Foundation.
28 * Author(s): Christian Borntraeger <borntraeger@de.ibm.com>
30 * Intel Virtio Over PCIe (VOP) driver.
33 #include <linux/delay.h>
34 #include <linux/module.h>
35 #include <linux/sched.h>
36 #include <linux/dma-mapping.h>
40 #define VOP_MAX_VRINGS 4
43 * _vop_vdev - Allocated per virtio device instance injected by the peer.
45 * @vdev: Virtio device
46 * @desc: Virtio device page descriptor
47 * @dc: Virtio device control
48 * @vpdev: VOP device which is the parent for this virtio device
49 * @vr: Buffer for accessing the VRING
50 * @used_virt: Virtual address of used ring
51 * @used: DMA address of used ring
52 * @used_size: Size of the used buffer
53 * @reset_done: Track whether VOP reset is complete
54 * @virtio_cookie: Cookie returned upon requesting a interrupt
55 * @c2h_vdev_db: The doorbell used by the guest to interrupt the host
56 * @h2c_vdev_db: The doorbell used by the host to interrupt the guest
57 * @dnode: The destination node
60 struct virtio_device vdev;
61 struct mic_device_desc __iomem *desc;
62 struct mic_device_ctrl __iomem *dc;
63 struct vop_device *vpdev;
64 void __iomem *vr[VOP_MAX_VRINGS];
65 void *used_virt[VOP_MAX_VRINGS];
66 dma_addr_t used[VOP_MAX_VRINGS];
67 int used_size[VOP_MAX_VRINGS];
68 struct completion reset_done;
69 struct mic_irq *virtio_cookie;
75 #define to_vopvdev(vd) container_of(vd, struct _vop_vdev, vdev)
77 #define _vop_aligned_desc_size(d) __mic_align(_vop_desc_size(d), 8)
79 /* Helper API to obtain the parent of the virtio device */
80 static inline struct device *_vop_dev(struct _vop_vdev *vdev)
82 return vdev->vdev.dev.parent;
85 static inline unsigned _vop_desc_size(struct mic_device_desc __iomem *desc)
88 + ioread8(&desc->num_vq) * sizeof(struct mic_vqconfig)
89 + ioread8(&desc->feature_len) * 2
90 + ioread8(&desc->config_len);
93 static inline struct mic_vqconfig __iomem *
94 _vop_vq_config(struct mic_device_desc __iomem *desc)
96 return (struct mic_vqconfig __iomem *)(desc + 1);
99 static inline u8 __iomem *
100 _vop_vq_features(struct mic_device_desc __iomem *desc)
102 return (u8 __iomem *)(_vop_vq_config(desc) + ioread8(&desc->num_vq));
105 static inline u8 __iomem *
106 _vop_vq_configspace(struct mic_device_desc __iomem *desc)
108 return _vop_vq_features(desc) + ioread8(&desc->feature_len) * 2;
111 static inline unsigned
112 _vop_total_desc_size(struct mic_device_desc __iomem *desc)
114 return _vop_aligned_desc_size(desc) + sizeof(struct mic_device_ctrl);
117 /* This gets the device's feature bits. */
118 static u64 vop_get_features(struct virtio_device *vdev)
120 unsigned int i, bits;
122 struct mic_device_desc __iomem *desc = to_vopvdev(vdev)->desc;
123 u8 __iomem *in_features = _vop_vq_features(desc);
124 int feature_len = ioread8(&desc->feature_len);
126 bits = min_t(unsigned, feature_len, sizeof(vdev->features)) * 8;
127 for (i = 0; i < bits; i++)
128 if (ioread8(&in_features[i / 8]) & (BIT(i % 8)))
134 static void vop_transport_features(struct virtio_device *vdev)
137 * Packed ring isn't enabled on virtio_vop for now,
138 * because virtio_vop uses vring_new_virtqueue() which
139 * creates virtio rings on preallocated memory.
141 __virtio_clear_bit(vdev, VIRTIO_F_RING_PACKED);
144 static int vop_finalize_features(struct virtio_device *vdev)
146 unsigned int i, bits;
147 struct mic_device_desc __iomem *desc = to_vopvdev(vdev)->desc;
148 u8 feature_len = ioread8(&desc->feature_len);
149 /* Second half of bitmap is features we accept. */
150 u8 __iomem *out_features =
151 _vop_vq_features(desc) + feature_len;
153 /* Give virtio_ring a chance to accept features. */
154 vring_transport_features(vdev);
156 /* Give virtio_vop a chance to accept features. */
157 vop_transport_features(vdev);
159 memset_io(out_features, 0, feature_len);
160 bits = min_t(unsigned, feature_len,
161 sizeof(vdev->features)) * 8;
162 for (i = 0; i < bits; i++) {
163 if (__virtio_test_bit(vdev, i))
164 iowrite8(ioread8(&out_features[i / 8]) | (1 << (i % 8)),
165 &out_features[i / 8]);
171 * Reading and writing elements in config space
173 static void vop_get(struct virtio_device *vdev, unsigned int offset,
174 void *buf, unsigned len)
176 struct mic_device_desc __iomem *desc = to_vopvdev(vdev)->desc;
178 if (offset + len > ioread8(&desc->config_len))
180 memcpy_fromio(buf, _vop_vq_configspace(desc) + offset, len);
183 static void vop_set(struct virtio_device *vdev, unsigned int offset,
184 const void *buf, unsigned len)
186 struct mic_device_desc __iomem *desc = to_vopvdev(vdev)->desc;
188 if (offset + len > ioread8(&desc->config_len))
190 memcpy_toio(_vop_vq_configspace(desc) + offset, buf, len);
194 * The operations to get and set the status word just access the status
195 * field of the device descriptor. set_status also interrupts the host
196 * to tell about status changes.
198 static u8 vop_get_status(struct virtio_device *vdev)
200 return ioread8(&to_vopvdev(vdev)->desc->status);
203 static void vop_set_status(struct virtio_device *dev, u8 status)
205 struct _vop_vdev *vdev = to_vopvdev(dev);
206 struct vop_device *vpdev = vdev->vpdev;
210 iowrite8(status, &vdev->desc->status);
211 vpdev->hw_ops->send_intr(vpdev, vdev->c2h_vdev_db);
214 /* Inform host on a virtio device reset and wait for ack from host */
215 static void vop_reset_inform_host(struct virtio_device *dev)
217 struct _vop_vdev *vdev = to_vopvdev(dev);
218 struct mic_device_ctrl __iomem *dc = vdev->dc;
219 struct vop_device *vpdev = vdev->vpdev;
222 iowrite8(0, &dc->host_ack);
223 iowrite8(1, &dc->vdev_reset);
224 vpdev->hw_ops->send_intr(vpdev, vdev->c2h_vdev_db);
226 /* Wait till host completes all card accesses and acks the reset */
227 for (retry = 100; retry--;) {
228 if (ioread8(&dc->host_ack))
233 dev_dbg(_vop_dev(vdev), "%s: retry: %d\n", __func__, retry);
235 /* Reset status to 0 in case we timed out */
236 iowrite8(0, &vdev->desc->status);
239 static void vop_reset(struct virtio_device *dev)
241 struct _vop_vdev *vdev = to_vopvdev(dev);
243 dev_dbg(_vop_dev(vdev), "%s: virtio id %d\n",
244 __func__, dev->id.device);
246 vop_reset_inform_host(dev);
247 complete_all(&vdev->reset_done);
251 * The virtio_ring code calls this API when it wants to notify the Host.
253 static bool vop_notify(struct virtqueue *vq)
255 struct _vop_vdev *vdev = vq->priv;
256 struct vop_device *vpdev = vdev->vpdev;
258 vpdev->hw_ops->send_intr(vpdev, vdev->c2h_vdev_db);
262 static void vop_del_vq(struct virtqueue *vq, int n)
264 struct _vop_vdev *vdev = to_vopvdev(vq->vdev);
265 struct vop_device *vpdev = vdev->vpdev;
267 dma_unmap_single(&vpdev->dev, vdev->used[n],
268 vdev->used_size[n], DMA_BIDIRECTIONAL);
269 free_pages((unsigned long)vdev->used_virt[n],
270 get_order(vdev->used_size[n]));
271 vring_del_virtqueue(vq);
272 vpdev->hw_ops->iounmap(vpdev, vdev->vr[n]);
276 static void vop_del_vqs(struct virtio_device *dev)
278 struct _vop_vdev *vdev = to_vopvdev(dev);
279 struct virtqueue *vq, *n;
282 dev_dbg(_vop_dev(vdev), "%s\n", __func__);
284 list_for_each_entry_safe(vq, n, &dev->vqs, list)
285 vop_del_vq(vq, idx++);
288 static struct virtqueue *vop_new_virtqueue(unsigned int index,
290 struct virtio_device *vdev,
293 bool (*notify)(struct virtqueue *vq),
294 void (*callback)(struct virtqueue *vq),
298 bool weak_barriers = false;
301 vring_init(&vring, num, pages, MIC_VIRTIO_RING_ALIGN);
304 return __vring_new_virtqueue(index, vring, vdev, weak_barriers, context,
305 notify, callback, name);
309 * This routine will assign vring's allocated in host/io memory. Code in
310 * virtio_ring.c however continues to access this io memory as if it were local
311 * memory without io accessors.
313 static struct virtqueue *vop_find_vq(struct virtio_device *dev,
315 void (*callback)(struct virtqueue *vq),
316 const char *name, bool ctx)
318 struct _vop_vdev *vdev = to_vopvdev(dev);
319 struct vop_device *vpdev = vdev->vpdev;
320 struct mic_vqconfig __iomem *vqconfig;
321 struct mic_vqconfig config;
322 struct virtqueue *vq;
324 struct _mic_vring_info __iomem *info;
326 int vr_size, _vr_size, err, magic;
327 u8 type = ioread8(&vdev->desc->type);
329 if (index >= ioread8(&vdev->desc->num_vq))
330 return ERR_PTR(-ENOENT);
333 return ERR_PTR(-ENOENT);
335 /* First assign the vring's allocated in host memory */
336 vqconfig = _vop_vq_config(vdev->desc) + index;
337 memcpy_fromio(&config, vqconfig, sizeof(config));
338 _vr_size = vring_size(le16_to_cpu(config.num), MIC_VIRTIO_RING_ALIGN);
339 vr_size = PAGE_ALIGN(_vr_size + sizeof(struct _mic_vring_info));
340 va = vpdev->hw_ops->ioremap(vpdev, le64_to_cpu(config.address),
343 return ERR_PTR(-ENOMEM);
344 vdev->vr[index] = va;
345 memset_io(va, 0x0, _vr_size);
347 info = va + _vr_size;
348 magic = ioread32(&info->magic);
350 if (WARN(magic != MIC_MAGIC + type + index, "magic mismatch")) {
355 vdev->used_size[index] = PAGE_ALIGN(sizeof(__u16) * 3 +
356 sizeof(struct vring_used_elem) *
357 le16_to_cpu(config.num));
358 used = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
359 get_order(vdev->used_size[index]));
360 vdev->used_virt[index] = used;
363 dev_err(_vop_dev(vdev), "%s %d err %d\n",
364 __func__, __LINE__, err);
368 vq = vop_new_virtqueue(index, le16_to_cpu(config.num), dev, ctx,
369 (void __force *)va, vop_notify, callback,
376 vdev->used[index] = dma_map_single(&vpdev->dev, used,
377 vdev->used_size[index],
379 if (dma_mapping_error(&vpdev->dev, vdev->used[index])) {
381 dev_err(_vop_dev(vdev), "%s %d err %d\n",
382 __func__, __LINE__, err);
385 writeq(vdev->used[index], &vqconfig->used_address);
390 vring_del_virtqueue(vq);
392 free_pages((unsigned long)used,
393 get_order(vdev->used_size[index]));
395 vpdev->hw_ops->iounmap(vpdev, vdev->vr[index]);
399 static int vop_find_vqs(struct virtio_device *dev, unsigned nvqs,
400 struct virtqueue *vqs[],
401 vq_callback_t *callbacks[],
402 const char * const names[], const bool *ctx,
403 struct irq_affinity *desc)
405 struct _vop_vdev *vdev = to_vopvdev(dev);
406 struct vop_device *vpdev = vdev->vpdev;
407 struct mic_device_ctrl __iomem *dc = vdev->dc;
408 int i, err, retry, queue_idx = 0;
410 /* We must have this many virtqueues. */
411 if (nvqs > ioread8(&vdev->desc->num_vq))
414 for (i = 0; i < nvqs; ++i) {
420 dev_dbg(_vop_dev(vdev), "%s: %d: %s\n",
421 __func__, i, names[i]);
422 vqs[i] = vop_find_vq(dev, queue_idx++, callbacks[i], names[i],
423 ctx ? ctx[i] : false);
424 if (IS_ERR(vqs[i])) {
425 err = PTR_ERR(vqs[i]);
430 iowrite8(1, &dc->used_address_updated);
432 * Send an interrupt to the host to inform it that used
433 * rings have been re-assigned.
435 vpdev->hw_ops->send_intr(vpdev, vdev->c2h_vdev_db);
436 for (retry = 100; --retry;) {
437 if (!ioread8(&dc->used_address_updated))
442 dev_dbg(_vop_dev(vdev), "%s: retry: %d\n", __func__, retry);
455 * The config ops structure as defined by virtio config
457 static struct virtio_config_ops vop_vq_config_ops = {
458 .get_features = vop_get_features,
459 .finalize_features = vop_finalize_features,
462 .get_status = vop_get_status,
463 .set_status = vop_set_status,
465 .find_vqs = vop_find_vqs,
466 .del_vqs = vop_del_vqs,
469 static irqreturn_t vop_virtio_intr_handler(int irq, void *data)
471 struct _vop_vdev *vdev = data;
472 struct vop_device *vpdev = vdev->vpdev;
473 struct virtqueue *vq;
475 vpdev->hw_ops->ack_interrupt(vpdev, vdev->h2c_vdev_db);
476 list_for_each_entry(vq, &vdev->vdev.vqs, list)
477 vring_interrupt(0, vq);
482 static void vop_virtio_release_dev(struct device *_d)
484 struct virtio_device *vdev =
485 container_of(_d, struct virtio_device, dev);
486 struct _vop_vdev *vop_vdev =
487 container_of(vdev, struct _vop_vdev, vdev);
493 * adds a new device and register it with virtio
494 * appropriate drivers are loaded by the device model
496 static int _vop_add_device(struct mic_device_desc __iomem *d,
497 unsigned int offset, struct vop_device *vpdev,
500 struct _vop_vdev *vdev, *reg_dev = NULL;
502 u8 type = ioread8(&d->type);
504 vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
509 vdev->vdev.dev.parent = &vpdev->dev;
510 vdev->vdev.dev.release = vop_virtio_release_dev;
511 vdev->vdev.id.device = type;
512 vdev->vdev.config = &vop_vq_config_ops;
514 vdev->dc = (void __iomem *)d + _vop_aligned_desc_size(d);
516 vdev->vdev.priv = (void *)(u64)dnode;
517 init_completion(&vdev->reset_done);
519 vdev->h2c_vdev_db = vpdev->hw_ops->next_db(vpdev);
520 vdev->virtio_cookie = vpdev->hw_ops->request_irq(vpdev,
521 vop_virtio_intr_handler, "virtio intr",
522 vdev, vdev->h2c_vdev_db);
523 if (IS_ERR(vdev->virtio_cookie)) {
524 ret = PTR_ERR(vdev->virtio_cookie);
527 iowrite8((u8)vdev->h2c_vdev_db, &vdev->dc->h2c_vdev_db);
528 vdev->c2h_vdev_db = ioread8(&vdev->dc->c2h_vdev_db);
530 ret = register_virtio_device(&vdev->vdev);
533 dev_err(_vop_dev(vdev),
534 "Failed to register vop device %u type %u\n",
538 writeq((u64)vdev, &vdev->dc->vdev);
539 dev_dbg(_vop_dev(vdev), "%s: registered vop device %u type %u vdev %p\n",
540 __func__, offset, type, vdev);
545 vpdev->hw_ops->free_irq(vpdev, vdev->virtio_cookie, vdev);
548 put_device(&vdev->vdev.dev);
555 * match for a vop device with a specific desc pointer
557 static int vop_match_desc(struct device *dev, void *data)
559 struct virtio_device *_dev = dev_to_virtio(dev);
560 struct _vop_vdev *vdev = to_vopvdev(_dev);
562 return vdev->desc == (void __iomem *)data;
565 static void _vop_handle_config_change(struct mic_device_desc __iomem *d,
567 struct vop_device *vpdev)
569 struct mic_device_ctrl __iomem *dc
570 = (void __iomem *)d + _vop_aligned_desc_size(d);
571 struct _vop_vdev *vdev = (struct _vop_vdev *)readq(&dc->vdev);
573 if (ioread8(&dc->config_change) != MIC_VIRTIO_PARAM_CONFIG_CHANGED)
576 dev_dbg(&vpdev->dev, "%s %d\n", __func__, __LINE__);
577 virtio_config_changed(&vdev->vdev);
578 iowrite8(1, &dc->guest_ack);
582 * removes a virtio device if a hot remove event has been
583 * requested by the host.
585 static int _vop_remove_device(struct mic_device_desc __iomem *d,
586 unsigned int offset, struct vop_device *vpdev)
588 struct mic_device_ctrl __iomem *dc
589 = (void __iomem *)d + _vop_aligned_desc_size(d);
590 struct _vop_vdev *vdev = (struct _vop_vdev *)readq(&dc->vdev);
594 if (ioread8(&dc->config_change) == MIC_VIRTIO_PARAM_DEV_REMOVE) {
595 struct device *dev = get_device(&vdev->vdev.dev);
598 "%s %d config_change %d type %d vdev %p\n",
600 ioread8(&dc->config_change), ioread8(&d->type), vdev);
601 status = ioread8(&d->status);
602 reinit_completion(&vdev->reset_done);
603 unregister_virtio_device(&vdev->vdev);
604 vpdev->hw_ops->free_irq(vpdev, vdev->virtio_cookie, vdev);
605 iowrite8(-1, &dc->h2c_vdev_db);
606 if (status & VIRTIO_CONFIG_S_DRIVER_OK)
607 wait_for_completion(&vdev->reset_done);
609 iowrite8(1, &dc->guest_ack);
610 dev_dbg(&vpdev->dev, "%s %d guest_ack %d\n",
611 __func__, __LINE__, ioread8(&dc->guest_ack));
612 iowrite8(-1, &d->type);
618 #define REMOVE_DEVICES true
620 static void _vop_scan_devices(void __iomem *dp, struct vop_device *vpdev,
621 bool remove, int dnode)
625 struct mic_device_desc __iomem *d;
626 struct mic_device_ctrl __iomem *dc;
630 for (i = sizeof(struct mic_bootparam);
631 i < MIC_DP_SIZE; i += _vop_total_desc_size(d)) {
633 dc = (void __iomem *)d + _vop_aligned_desc_size(d);
635 * This read barrier is paired with the corresponding write
636 * barrier on the host which is inserted before adding or
637 * removing a virtio device descriptor, by updating the type.
640 type = ioread8(&d->type);
649 /* device already exists */
650 dev = device_find_child(&vpdev->dev, (void __force *)d,
654 iowrite8(MIC_VIRTIO_PARAM_DEV_REMOVE,
657 _vop_handle_config_change(d, i, vpdev);
658 ret = _vop_remove_device(d, i, vpdev);
660 iowrite8(0, &dc->config_change);
661 iowrite8(0, &dc->guest_ack);
667 dev_dbg(&vpdev->dev, "%s %d Adding new virtio device %p\n",
668 __func__, __LINE__, d);
670 _vop_add_device(d, i, vpdev, dnode);
674 static void vop_scan_devices(struct vop_info *vi,
675 struct vop_device *vpdev, bool remove)
677 void __iomem *dp = vpdev->hw_ops->get_remote_dp(vpdev);
681 mutex_lock(&vi->vop_mutex);
682 _vop_scan_devices(dp, vpdev, remove, vpdev->dnode);
683 mutex_unlock(&vi->vop_mutex);
687 * vop_hotplug_device tries to find changes in the device page.
689 static void vop_hotplug_devices(struct work_struct *work)
691 struct vop_info *vi = container_of(work, struct vop_info,
694 vop_scan_devices(vi, vi->vpdev, !REMOVE_DEVICES);
698 * Interrupt handler for hot plug/config changes etc.
700 static irqreturn_t vop_extint_handler(int irq, void *data)
702 struct vop_info *vi = data;
703 struct mic_bootparam __iomem *bp;
704 struct vop_device *vpdev = vi->vpdev;
706 bp = vpdev->hw_ops->get_remote_dp(vpdev);
707 dev_dbg(&vpdev->dev, "%s %d hotplug work\n",
709 vpdev->hw_ops->ack_interrupt(vpdev, ioread8(&bp->h2c_config_db));
710 schedule_work(&vi->hotplug_work);
714 static int vop_driver_probe(struct vop_device *vpdev)
719 vi = kzalloc(sizeof(*vi), GFP_KERNEL);
724 dev_set_drvdata(&vpdev->dev, vi);
727 mutex_init(&vi->vop_mutex);
728 INIT_WORK(&vi->hotplug_work, vop_hotplug_devices);
730 rc = vop_host_init(vi);
734 struct mic_bootparam __iomem *bootparam;
736 vop_scan_devices(vi, vpdev, !REMOVE_DEVICES);
738 vi->h2c_config_db = vpdev->hw_ops->next_db(vpdev);
739 vi->cookie = vpdev->hw_ops->request_irq(vpdev,
741 "virtio_config_intr",
742 vi, vi->h2c_config_db);
743 if (IS_ERR(vi->cookie)) {
744 rc = PTR_ERR(vi->cookie);
747 bootparam = vpdev->hw_ops->get_remote_dp(vpdev);
748 iowrite8(vi->h2c_config_db, &bootparam->h2c_config_db);
750 vop_init_debugfs(vi);
758 static void vop_driver_remove(struct vop_device *vpdev)
760 struct vop_info *vi = dev_get_drvdata(&vpdev->dev);
765 struct mic_bootparam __iomem *bootparam =
766 vpdev->hw_ops->get_remote_dp(vpdev);
768 iowrite8(-1, &bootparam->h2c_config_db);
769 vpdev->hw_ops->free_irq(vpdev, vi->cookie, vi);
770 flush_work(&vi->hotplug_work);
771 vop_scan_devices(vi, vpdev, REMOVE_DEVICES);
773 vop_exit_debugfs(vi);
777 static struct vop_device_id id_table[] = {
778 { VOP_DEV_TRNSP, VOP_DEV_ANY_ID },
782 static struct vop_driver vop_driver = {
783 .driver.name = KBUILD_MODNAME,
784 .driver.owner = THIS_MODULE,
785 .id_table = id_table,
786 .probe = vop_driver_probe,
787 .remove = vop_driver_remove,
790 module_vop_driver(vop_driver);
792 MODULE_DEVICE_TABLE(mbus, id_table);
793 MODULE_AUTHOR("Intel Corporation");
794 MODULE_DESCRIPTION("Intel(R) Virtio Over PCIe (VOP) driver");
795 MODULE_LICENSE("GPL v2");