drivers/gpu/drm/i915/gvt/kvmgt.c

   1 /*
   2  * KVMGT - the implementation of Intel mediated pass-through framework for KVM
   3  *
   4  * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  23  * SOFTWARE.
  24  *
  25  * Authors:
  26  *    Kevin Tian <kevin.tian@intel.com>
  27  *    Jike Song <jike.song@intel.com>
  28  *    Xiaoguang Chen <xiaoguang.chen@intel.com>
  29  *    Eddie Dong <eddie.dong@intel.com>
  30  *
  31  * Contributors:
  32  *    Niu Bing <bing.niu@intel.com>
  33  *    Zhi Wang <zhi.a.wang@intel.com>
  34  */
  35
  36 #include <linux/init.h>
  37 #include <linux/mm.h>
  38 #include <linux/kthread.h>
  39 #include <linux/sched/mm.h>
  40 #include <linux/types.h>
  41 #include <linux/list.h>
  42 #include <linux/rbtree.h>
  43 #include <linux/spinlock.h>
  44 #include <linux/eventfd.h>
  45 #include <linux/mdev.h>
  46 #include <linux/debugfs.h>
  47
  48 #include <linux/nospec.h>
  49
  50 #include <drm/drm_edid.h>
  51
  52 #include "i915_drv.h"
  53 #include "intel_gvt.h"
  54 #include "gvt.h"
  55
  56 MODULE_IMPORT_NS(DMA_BUF);
  57 MODULE_IMPORT_NS(I915_GVT);
  58
  59 /* helper macros copied from vfio-pci */
  60 #define VFIO_PCI_OFFSET_SHIFT   40
  61 #define VFIO_PCI_OFFSET_TO_INDEX(off)   (off >> VFIO_PCI_OFFSET_SHIFT)
  62 #define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT)
  63 #define VFIO_PCI_OFFSET_MASK    (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1)
  64
  65 #define EDID_BLOB_OFFSET (PAGE_SIZE/2)
  66
  67 #define OPREGION_SIGNATURE "IntelGraphicsMem"
  68
  69 struct vfio_region;
  70 struct intel_vgpu_regops {
  71         size_t (*rw)(struct intel_vgpu *vgpu, char *buf,
  72                         size_t count, loff_t *ppos, bool iswrite);
  73         void (*release)(struct intel_vgpu *vgpu,
  74                         struct vfio_region *region);
  75 };
  76
  77 struct vfio_region {
  78         u32                             type;
  79         u32                             subtype;
  80         size_t                          size;
  81         u32                             flags;
  82         const struct intel_vgpu_regops  *ops;
  83         void                            *data;
  84 };
  85
  86 struct vfio_edid_region {
  87         struct vfio_region_gfx_edid vfio_edid_regs;
  88         void *edid_blob;
  89 };
  90
  91 struct kvmgt_pgfn {
  92         gfn_t gfn;
  93         struct hlist_node hnode;
  94 };
  95
  96 struct gvt_dma {
  97         struct intel_vgpu *vgpu;
  98         struct rb_node gfn_node;
  99         struct rb_node dma_addr_node;
 100         gfn_t gfn;
 101         dma_addr_t dma_addr;
 102         unsigned long size;
 103         struct kref ref;
 104 };
 105
 106 #define vfio_dev_to_vgpu(vfio_dev) \
 107         container_of((vfio_dev), struct intel_vgpu, vfio_device)
 108
 109 static void kvmgt_page_track_write(gpa_t gpa, const u8 *val, int len,
 110                                    struct kvm_page_track_notifier_node *node);
 111 static void kvmgt_page_track_remove_region(gfn_t gfn, unsigned long nr_pages,
 112                                            struct kvm_page_track_notifier_node *node);
 113
 114 static ssize_t intel_vgpu_show_description(struct mdev_type *mtype, char *buf)
 115 {
 116         struct intel_vgpu_type *type =
 117                 container_of(mtype, struct intel_vgpu_type, type);
 118
 119         return sprintf(buf, "low_gm_size: %dMB\nhigh_gm_size: %dMB\n"
 120                        "fence: %d\nresolution: %s\n"
 121                        "weight: %d\n",
 122                        BYTES_TO_MB(type->conf->low_mm),
 123                        BYTES_TO_MB(type->conf->high_mm),
 124                        type->conf->fence, vgpu_edid_str(type->conf->edid),
 125                        type->conf->weight);
 126 }
 127
 128 static void gvt_unpin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn,
 129                 unsigned long size)
 130 {
 131         vfio_unpin_pages(&vgpu->vfio_device, gfn << PAGE_SHIFT,
 132                          DIV_ROUND_UP(size, PAGE_SIZE));
 133 }
 134
 135 /* Pin a normal or compound guest page for dma. */
 136 static int gvt_pin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn,
 137                 unsigned long size, struct page **page)
 138 {
 139         int total_pages = DIV_ROUND_UP(size, PAGE_SIZE);
 140         struct page *base_page = NULL;
 141         int npage;
 142         int ret;
 143
 144         /*
 145          * We pin the pages one-by-one to avoid allocating a big arrary
 146          * on stack to hold pfns.
 147          */
 148         for (npage = 0; npage < total_pages; npage++) {
 149                 dma_addr_t cur_iova = (gfn + npage) << PAGE_SHIFT;
 150                 struct page *cur_page;
 151
 152                 ret = vfio_pin_pages(&vgpu->vfio_device, cur_iova, 1,
 153                                      IOMMU_READ | IOMMU_WRITE, &cur_page);
 154                 if (ret != 1) {
 155                         gvt_vgpu_err("vfio_pin_pages failed for iova %pad, ret %d\n",
 156                                      &cur_iova, ret);
 157                         goto err;
 158                 }
 159
 160                 if (npage == 0)
 161                         base_page = cur_page;
 162                 else if (page_to_pfn(base_page) + npage != page_to_pfn(cur_page)) {
 163                         ret = -EINVAL;
 164                         npage++;
 165                         goto err;
 166                 }
 167         }
 168
 169         *page = base_page;
 170         return 0;
 171 err:
 172         if (npage)
 173                 gvt_unpin_guest_page(vgpu, gfn, npage * PAGE_SIZE);
 174         return ret;
 175 }
 176
 177 static int gvt_dma_map_page(struct intel_vgpu *vgpu, unsigned long gfn,
 178                 dma_addr_t *dma_addr, unsigned long size)
 179 {
 180         struct device *dev = vgpu->gvt->gt->i915->drm.dev;
 181         struct page *page = NULL;
 182         int ret;
 183
 184         ret = gvt_pin_guest_page(vgpu, gfn, size, &page);
 185         if (ret)
 186                 return ret;
 187
 188         /* Setup DMA mapping. */
 189         *dma_addr = dma_map_page(dev, page, 0, size, DMA_BIDIRECTIONAL);
 190         if (dma_mapping_error(dev, *dma_addr)) {
 191                 gvt_vgpu_err("DMA mapping failed for pfn 0x%lx, ret %d\n",
 192                              page_to_pfn(page), ret);
 193                 gvt_unpin_guest_page(vgpu, gfn, size);
 194                 return -ENOMEM;
 195         }
 196
 197         return 0;
 198 }
 199
 200 static void gvt_dma_unmap_page(struct intel_vgpu *vgpu, unsigned long gfn,
 201                 dma_addr_t dma_addr, unsigned long size)
 202 {
 203         struct device *dev = vgpu->gvt->gt->i915->drm.dev;
 204
 205         dma_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL);
 206         gvt_unpin_guest_page(vgpu, gfn, size);
 207 }
 208
 209 static struct gvt_dma *__gvt_cache_find_dma_addr(struct intel_vgpu *vgpu,
 210                 dma_addr_t dma_addr)
 211 {
 212         struct rb_node *node = vgpu->dma_addr_cache.rb_node;
 213         struct gvt_dma *itr;
 214
 215         while (node) {
 216                 itr = rb_entry(node, struct gvt_dma, dma_addr_node);
 217
 218                 if (dma_addr < itr->dma_addr)
 219                         node = node->rb_left;
 220                 else if (dma_addr > itr->dma_addr)
 221                         node = node->rb_right;
 222                 else
 223                         return itr;
 224         }
 225         return NULL;
 226 }
 227
 228 static struct gvt_dma *__gvt_cache_find_gfn(struct intel_vgpu *vgpu, gfn_t gfn)
 229 {
 230         struct rb_node *node = vgpu->gfn_cache.rb_node;
 231         struct gvt_dma *itr;
 232
 233         while (node) {
 234                 itr = rb_entry(node, struct gvt_dma, gfn_node);
 235
 236                 if (gfn < itr->gfn)
 237                         node = node->rb_left;
 238                 else if (gfn > itr->gfn)
 239                         node = node->rb_right;
 240                 else
 241                         return itr;
 242         }
 243         return NULL;
 244 }
 245
 246 static int __gvt_cache_add(struct intel_vgpu *vgpu, gfn_t gfn,
 247                 dma_addr_t dma_addr, unsigned long size)
 248 {
 249         struct gvt_dma *new, *itr;
 250         struct rb_node **link, *parent = NULL;
 251
 252         new = kzalloc(sizeof(struct gvt_dma), GFP_KERNEL);
 253         if (!new)
 254                 return -ENOMEM;
 255
 256         new->vgpu = vgpu;
 257         new->gfn = gfn;
 258         new->dma_addr = dma_addr;
 259         new->size = size;
 260         kref_init(&new->ref);
 261
 262         /* gfn_cache maps gfn to struct gvt_dma. */
 263         link = &vgpu->gfn_cache.rb_node;
 264         while (*link) {
 265                 parent = *link;
 266                 itr = rb_entry(parent, struct gvt_dma, gfn_node);
 267
 268                 if (gfn < itr->gfn)
 269                         link = &parent->rb_left;
 270                 else
 271                         link = &parent->rb_right;
 272         }
 273         rb_link_node(&new->gfn_node, parent, link);
 274         rb_insert_color(&new->gfn_node, &vgpu->gfn_cache);
 275
 276         /* dma_addr_cache maps dma addr to struct gvt_dma. */
 277         parent = NULL;
 278         link = &vgpu->dma_addr_cache.rb_node;
 279         while (*link) {
 280                 parent = *link;
 281                 itr = rb_entry(parent, struct gvt_dma, dma_addr_node);
 282
 283                 if (dma_addr < itr->dma_addr)
 284                         link = &parent->rb_left;
 285                 else
 286                         link = &parent->rb_right;
 287         }
 288         rb_link_node(&new->dma_addr_node, parent, link);
 289         rb_insert_color(&new->dma_addr_node, &vgpu->dma_addr_cache);
 290
 291         vgpu->nr_cache_entries++;
 292         return 0;
 293 }
 294
 295 static void __gvt_cache_remove_entry(struct intel_vgpu *vgpu,
 296                                 struct gvt_dma *entry)
 297 {
 298         rb_erase(&entry->gfn_node, &vgpu->gfn_cache);
 299         rb_erase(&entry->dma_addr_node, &vgpu->dma_addr_cache);
 300         kfree(entry);
 301         vgpu->nr_cache_entries--;
 302 }
 303
 304 static void gvt_cache_destroy(struct intel_vgpu *vgpu)
 305 {
 306         struct gvt_dma *dma;
 307         struct rb_node *node = NULL;
 308
 309         for (;;) {
 310                 mutex_lock(&vgpu->cache_lock);
 311                 node = rb_first(&vgpu->gfn_cache);
 312                 if (!node) {
 313                         mutex_unlock(&vgpu->cache_lock);
 314                         break;
 315                 }
 316                 dma = rb_entry(node, struct gvt_dma, gfn_node);
 317                 gvt_dma_unmap_page(vgpu, dma->gfn, dma->dma_addr, dma->size);
 318                 __gvt_cache_remove_entry(vgpu, dma);
 319                 mutex_unlock(&vgpu->cache_lock);
 320         }
 321 }
 322
 323 static void gvt_cache_init(struct intel_vgpu *vgpu)
 324 {
 325         vgpu->gfn_cache = RB_ROOT;
 326         vgpu->dma_addr_cache = RB_ROOT;
 327         vgpu->nr_cache_entries = 0;
 328         mutex_init(&vgpu->cache_lock);
 329 }
 330
 331 static void kvmgt_protect_table_init(struct intel_vgpu *info)
 332 {
 333         hash_init(info->ptable);
 334 }
 335
 336 static void kvmgt_protect_table_destroy(struct intel_vgpu *info)
 337 {
 338         struct kvmgt_pgfn *p;
 339         struct hlist_node *tmp;
 340         int i;
 341
 342         hash_for_each_safe(info->ptable, i, tmp, p, hnode) {
 343                 hash_del(&p->hnode);
 344                 kfree(p);
 345         }
 346 }
 347
 348 static struct kvmgt_pgfn *
 349 __kvmgt_protect_table_find(struct intel_vgpu *info, gfn_t gfn)
 350 {
 351         struct kvmgt_pgfn *p, *res = NULL;
 352
 353         lockdep_assert_held(&info->vgpu_lock);
 354
 355         hash_for_each_possible(info->ptable, p, hnode, gfn) {
 356                 if (gfn == p->gfn) {
 357                         res = p;
 358                         break;
 359                 }
 360         }
 361
 362         return res;
 363 }
 364
 365 static bool kvmgt_gfn_is_write_protected(struct intel_vgpu *info, gfn_t gfn)
 366 {
 367         struct kvmgt_pgfn *p;
 368
 369         p = __kvmgt_protect_table_find(info, gfn);
 370         return !!p;
 371 }
 372
 373 static void kvmgt_protect_table_add(struct intel_vgpu *info, gfn_t gfn)
 374 {
 375         struct kvmgt_pgfn *p;
 376
 377         if (kvmgt_gfn_is_write_protected(info, gfn))
 378                 return;
 379
 380         p = kzalloc(sizeof(struct kvmgt_pgfn), GFP_ATOMIC);
 381         if (WARN(!p, "gfn: 0x%llx\n", gfn))
 382                 return;
 383
 384         p->gfn = gfn;
 385         hash_add(info->ptable, &p->hnode, gfn);
 386 }
 387
 388 static void kvmgt_protect_table_del(struct intel_vgpu *info, gfn_t gfn)
 389 {
 390         struct kvmgt_pgfn *p;
 391
 392         p = __kvmgt_protect_table_find(info, gfn);
 393         if (p) {
 394                 hash_del(&p->hnode);
 395                 kfree(p);
 396         }
 397 }
 398
 399 static size_t intel_vgpu_reg_rw_opregion(struct intel_vgpu *vgpu, char *buf,
 400                 size_t count, loff_t *ppos, bool iswrite)
 401 {
 402         unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) -
 403                         VFIO_PCI_NUM_REGIONS;
 404         void *base = vgpu->region[i].data;
 405         loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
 406
 407
 408         if (pos >= vgpu->region[i].size || iswrite) {
 409                 gvt_vgpu_err("invalid op or offset for Intel vgpu OpRegion\n");
 410                 return -EINVAL;
 411         }
 412         count = min(count, (size_t)(vgpu->region[i].size - pos));
 413         memcpy(buf, base + pos, count);
 414
 415         return count;
 416 }
 417
 418 static void intel_vgpu_reg_release_opregion(struct intel_vgpu *vgpu,
 419                 struct vfio_region *region)
 420 {
 421 }
 422
 423 static const struct intel_vgpu_regops intel_vgpu_regops_opregion = {
 424         .rw = intel_vgpu_reg_rw_opregion,
 425         .release = intel_vgpu_reg_release_opregion,
 426 };
 427
 428 static int handle_edid_regs(struct intel_vgpu *vgpu,
 429                         struct vfio_edid_region *region, char *buf,
 430                         size_t count, u16 offset, bool is_write)
 431 {
 432         struct vfio_region_gfx_edid *regs = &region->vfio_edid_regs;
 433         unsigned int data;
 434
 435         if (offset + count > sizeof(*regs))
 436                 return -EINVAL;
 437
 438         if (count != 4)
 439                 return -EINVAL;
 440
 441         if (is_write) {
 442                 data = *((unsigned int *)buf);
 443                 switch (offset) {
 444                 case offsetof(struct vfio_region_gfx_edid, link_state):
 445                         if (data == VFIO_DEVICE_GFX_LINK_STATE_UP) {
 446                                 if (!drm_edid_block_valid(
 447                                         (u8 *)region->edid_blob,
 448                                         0,
 449                                         true,
 450                                         NULL)) {
 451                                         gvt_vgpu_err("invalid EDID blob\n");
 452                                         return -EINVAL;
 453                                 }
 454                                 intel_vgpu_emulate_hotplug(vgpu, true);
 455                         } else if (data == VFIO_DEVICE_GFX_LINK_STATE_DOWN)
 456                                 intel_vgpu_emulate_hotplug(vgpu, false);
 457                         else {
 458                                 gvt_vgpu_err("invalid EDID link state %d\n",
 459                                         regs->link_state);
 460                                 return -EINVAL;
 461                         }
 462                         regs->link_state = data;
 463                         break;
 464                 case offsetof(struct vfio_region_gfx_edid, edid_size):
 465                         if (data > regs->edid_max_size) {
 466                                 gvt_vgpu_err("EDID size is bigger than %d!\n",
 467                                         regs->edid_max_size);
 468                                 return -EINVAL;
 469                         }
 470                         regs->edid_size = data;
 471                         break;
 472                 default:
 473                         /* read-only regs */
 474                         gvt_vgpu_err("write read-only EDID region at offset %d\n",
 475                                 offset);
 476                         return -EPERM;
 477                 }
 478         } else {
 479                 memcpy(buf, (char *)regs + offset, count);
 480         }
 481
 482         return count;
 483 }
 484
 485 static int handle_edid_blob(struct vfio_edid_region *region, char *buf,
 486                         size_t count, u16 offset, bool is_write)
 487 {
 488         if (offset + count > region->vfio_edid_regs.edid_size)
 489                 return -EINVAL;
 490
 491         if (is_write)
 492                 memcpy(region->edid_blob + offset, buf, count);
 493         else
 494                 memcpy(buf, region->edid_blob + offset, count);
 495
 496         return count;
 497 }
 498
 499 static size_t intel_vgpu_reg_rw_edid(struct intel_vgpu *vgpu, char *buf,
 500                 size_t count, loff_t *ppos, bool iswrite)
 501 {
 502         int ret;
 503         unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) -
 504                         VFIO_PCI_NUM_REGIONS;
 505         struct vfio_edid_region *region = vgpu->region[i].data;
 506         loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
 507
 508         if (pos < region->vfio_edid_regs.edid_offset) {
 509                 ret = handle_edid_regs(vgpu, region, buf, count, pos, iswrite);
 510         } else {
 511                 pos -= EDID_BLOB_OFFSET;
 512                 ret = handle_edid_blob(region, buf, count, pos, iswrite);
 513         }
 514
 515         if (ret < 0)
 516                 gvt_vgpu_err("failed to access EDID region\n");
 517
 518         return ret;
 519 }
 520
 521 static void intel_vgpu_reg_release_edid(struct intel_vgpu *vgpu,
 522                                         struct vfio_region *region)
 523 {
 524         kfree(region->data);
 525 }
 526
 527 static const struct intel_vgpu_regops intel_vgpu_regops_edid = {
 528         .rw = intel_vgpu_reg_rw_edid,
 529         .release = intel_vgpu_reg_release_edid,
 530 };
 531
 532 static int intel_vgpu_register_reg(struct intel_vgpu *vgpu,
 533                 unsigned int type, unsigned int subtype,
 534                 const struct intel_vgpu_regops *ops,
 535                 size_t size, u32 flags, void *data)
 536 {
 537         struct vfio_region *region;
 538
 539         region = krealloc(vgpu->region,
 540                         (vgpu->num_regions + 1) * sizeof(*region),
 541                         GFP_KERNEL);
 542         if (!region)
 543                 return -ENOMEM;
 544
 545         vgpu->region = region;
 546         vgpu->region[vgpu->num_regions].type = type;
 547         vgpu->region[vgpu->num_regions].subtype = subtype;
 548         vgpu->region[vgpu->num_regions].ops = ops;
 549         vgpu->region[vgpu->num_regions].size = size;
 550         vgpu->region[vgpu->num_regions].flags = flags;
 551         vgpu->region[vgpu->num_regions].data = data;
 552         vgpu->num_regions++;
 553         return 0;
 554 }
 555
 556 int intel_gvt_set_opregion(struct intel_vgpu *vgpu)
 557 {
 558         void *base;
 559         int ret;
 560
 561         /* Each vgpu has its own opregion, although VFIO would create another
 562          * one later. This one is used to expose opregion to VFIO. And the
 563          * other one created by VFIO later, is used by guest actually.
 564          */
 565         base = vgpu_opregion(vgpu)->va;
 566         if (!base)
 567                 return -ENOMEM;
 568
 569         if (memcmp(base, OPREGION_SIGNATURE, 16)) {
 570                 memunmap(base);
 571                 return -EINVAL;
 572         }
 573
 574         ret = intel_vgpu_register_reg(vgpu,
 575                         PCI_VENDOR_ID_INTEL | VFIO_REGION_TYPE_PCI_VENDOR_TYPE,
 576                         VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION,
 577                         &intel_vgpu_regops_opregion, OPREGION_SIZE,
 578                         VFIO_REGION_INFO_FLAG_READ, base);
 579
 580         return ret;
 581 }
 582
 583 int intel_gvt_set_edid(struct intel_vgpu *vgpu, int port_num)
 584 {
 585         struct intel_vgpu_port *port = intel_vgpu_port(vgpu, port_num);
 586         struct vfio_edid_region *base;
 587         int ret;
 588
 589         base = kzalloc(sizeof(*base), GFP_KERNEL);
 590         if (!base)
 591                 return -ENOMEM;
 592
 593         /* TODO: Add multi-port and EDID extension block support */
 594         base->vfio_edid_regs.edid_offset = EDID_BLOB_OFFSET;
 595         base->vfio_edid_regs.edid_max_size = EDID_SIZE;
 596         base->vfio_edid_regs.edid_size = EDID_SIZE;
 597         base->vfio_edid_regs.max_xres = vgpu_edid_xres(port->id);
 598         base->vfio_edid_regs.max_yres = vgpu_edid_yres(port->id);
 599         base->edid_blob = port->edid->edid_block;
 600
 601         ret = intel_vgpu_register_reg(vgpu,
 602                         VFIO_REGION_TYPE_GFX,
 603                         VFIO_REGION_SUBTYPE_GFX_EDID,
 604                         &intel_vgpu_regops_edid, EDID_SIZE,
 605                         VFIO_REGION_INFO_FLAG_READ |
 606                         VFIO_REGION_INFO_FLAG_WRITE |
 607                         VFIO_REGION_INFO_FLAG_CAPS, base);
 608
 609         return ret;
 610 }
 611
 612 static void intel_vgpu_dma_unmap(struct vfio_device *vfio_dev, u64 iova,
 613                                  u64 length)
 614 {
 615         struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
 616         struct gvt_dma *entry;
 617         u64 iov_pfn = iova >> PAGE_SHIFT;
 618         u64 end_iov_pfn = iov_pfn + length / PAGE_SIZE;
 619
 620         mutex_lock(&vgpu->cache_lock);
 621         for (; iov_pfn < end_iov_pfn; iov_pfn++) {
 622                 entry = __gvt_cache_find_gfn(vgpu, iov_pfn);
 623                 if (!entry)
 624                         continue;
 625
 626                 gvt_dma_unmap_page(vgpu, entry->gfn, entry->dma_addr,
 627                                    entry->size);
 628                 __gvt_cache_remove_entry(vgpu, entry);
 629         }
 630         mutex_unlock(&vgpu->cache_lock);
 631 }
 632
 633 static bool __kvmgt_vgpu_exist(struct intel_vgpu *vgpu)
 634 {
 635         struct intel_vgpu *itr;
 636         int id;
 637         bool ret = false;
 638
 639         mutex_lock(&vgpu->gvt->lock);
 640         for_each_active_vgpu(vgpu->gvt, itr, id) {
 641                 if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, itr->status))
 642                         continue;
 643
 644                 if (vgpu->vfio_device.kvm == itr->vfio_device.kvm) {
 645                         ret = true;
 646                         goto out;
 647                 }
 648         }
 649 out:
 650         mutex_unlock(&vgpu->gvt->lock);
 651         return ret;
 652 }
 653
 654 static int intel_vgpu_open_device(struct vfio_device *vfio_dev)
 655 {
 656         struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
 657         int ret;
 658
 659         if (__kvmgt_vgpu_exist(vgpu))
 660                 return -EEXIST;
 661
 662         vgpu->track_node.track_write = kvmgt_page_track_write;
 663         vgpu->track_node.track_remove_region = kvmgt_page_track_remove_region;
 664         ret = kvm_page_track_register_notifier(vgpu->vfio_device.kvm,
 665                                                &vgpu->track_node);
 666         if (ret) {
 667                 gvt_vgpu_err("KVM is required to use Intel vGPU\n");
 668                 return ret;
 669         }
 670
 671         set_bit(INTEL_VGPU_STATUS_ATTACHED, vgpu->status);
 672
 673         debugfs_create_ulong(KVMGT_DEBUGFS_FILENAME, 0444, vgpu->debugfs,
 674                              &vgpu->nr_cache_entries);
 675
 676         intel_gvt_activate_vgpu(vgpu);
 677
 678         return 0;
 679 }
 680
 681 static void intel_vgpu_release_msi_eventfd_ctx(struct intel_vgpu *vgpu)
 682 {
 683         struct eventfd_ctx *trigger;
 684
 685         trigger = vgpu->msi_trigger;
 686         if (trigger) {
 687                 eventfd_ctx_put(trigger);
 688                 vgpu->msi_trigger = NULL;
 689         }
 690 }
 691
 692 static void intel_vgpu_close_device(struct vfio_device *vfio_dev)
 693 {
 694         struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
 695
 696         intel_gvt_release_vgpu(vgpu);
 697
 698         clear_bit(INTEL_VGPU_STATUS_ATTACHED, vgpu->status);
 699
 700         debugfs_lookup_and_remove(KVMGT_DEBUGFS_FILENAME, vgpu->debugfs);
 701
 702         kvm_page_track_unregister_notifier(vgpu->vfio_device.kvm,
 703                                            &vgpu->track_node);
 704
 705         kvmgt_protect_table_destroy(vgpu);
 706         gvt_cache_destroy(vgpu);
 707
 708         WARN_ON(vgpu->nr_cache_entries);
 709
 710         vgpu->gfn_cache = RB_ROOT;
 711         vgpu->dma_addr_cache = RB_ROOT;
 712
 713         intel_vgpu_release_msi_eventfd_ctx(vgpu);
 714 }
 715
 716 static u64 intel_vgpu_get_bar_addr(struct intel_vgpu *vgpu, int bar)
 717 {
 718         u32 start_lo, start_hi;
 719         u32 mem_type;
 720
 721         start_lo = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &
 722                         PCI_BASE_ADDRESS_MEM_MASK;
 723         mem_type = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &
 724                         PCI_BASE_ADDRESS_MEM_TYPE_MASK;
 725
 726         switch (mem_type) {
 727         case PCI_BASE_ADDRESS_MEM_TYPE_64:
 728                 start_hi = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space
 729                                                 + bar + 4));
 730                 break;
 731         case PCI_BASE_ADDRESS_MEM_TYPE_32:
 732         case PCI_BASE_ADDRESS_MEM_TYPE_1M:
 733                 /* 1M mem BAR treated as 32-bit BAR */
 734         default:
 735                 /* mem unknown type treated as 32-bit BAR */
 736                 start_hi = 0;
 737                 break;
 738         }
 739
 740         return ((u64)start_hi << 32) | start_lo;
 741 }
 742
 743 static int intel_vgpu_bar_rw(struct intel_vgpu *vgpu, int bar, u64 off,
 744                              void *buf, unsigned int count, bool is_write)
 745 {
 746         u64 bar_start = intel_vgpu_get_bar_addr(vgpu, bar);
 747         int ret;
 748
 749         if (is_write)
 750                 ret = intel_vgpu_emulate_mmio_write(vgpu,
 751                                         bar_start + off, buf, count);
 752         else
 753                 ret = intel_vgpu_emulate_mmio_read(vgpu,
 754                                         bar_start + off, buf, count);
 755         return ret;
 756 }
 757
 758 static inline bool intel_vgpu_in_aperture(struct intel_vgpu *vgpu, u64 off)
 759 {
 760         return off >= vgpu_aperture_offset(vgpu) &&
 761                off < vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu);
 762 }
 763
 764 static int intel_vgpu_aperture_rw(struct intel_vgpu *vgpu, u64 off,
 765                 void *buf, unsigned long count, bool is_write)
 766 {
 767         void __iomem *aperture_va;
 768
 769         if (!intel_vgpu_in_aperture(vgpu, off) ||
 770             !intel_vgpu_in_aperture(vgpu, off + count)) {
 771                 gvt_vgpu_err("Invalid aperture offset %llu\n", off);
 772                 return -EINVAL;
 773         }
 774
 775         aperture_va = io_mapping_map_wc(&vgpu->gvt->gt->ggtt->iomap,
 776                                         ALIGN_DOWN(off, PAGE_SIZE),
 777                                         count + offset_in_page(off));
 778         if (!aperture_va)
 779                 return -EIO;
 780
 781         if (is_write)
 782                 memcpy_toio(aperture_va + offset_in_page(off), buf, count);
 783         else
 784                 memcpy_fromio(buf, aperture_va + offset_in_page(off), count);
 785
 786         io_mapping_unmap(aperture_va);
 787
 788         return 0;
 789 }
 790
 791 static ssize_t intel_vgpu_rw(struct intel_vgpu *vgpu, char *buf,
 792                         size_t count, loff_t *ppos, bool is_write)
 793 {
 794         unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
 795         u64 pos = *ppos & VFIO_PCI_OFFSET_MASK;
 796         int ret = -EINVAL;
 797
 798
 799         if (index >= VFIO_PCI_NUM_REGIONS + vgpu->num_regions) {
 800                 gvt_vgpu_err("invalid index: %u\n", index);
 801                 return -EINVAL;
 802         }
 803
 804         switch (index) {
 805         case VFIO_PCI_CONFIG_REGION_INDEX:
 806                 if (is_write)
 807                         ret = intel_vgpu_emulate_cfg_write(vgpu, pos,
 808                                                 buf, count);
 809                 else
 810                         ret = intel_vgpu_emulate_cfg_read(vgpu, pos,
 811                                                 buf, count);
 812                 break;
 813         case VFIO_PCI_BAR0_REGION_INDEX:
 814                 ret = intel_vgpu_bar_rw(vgpu, PCI_BASE_ADDRESS_0, pos,
 815                                         buf, count, is_write);
 816                 break;
 817         case VFIO_PCI_BAR2_REGION_INDEX:
 818                 ret = intel_vgpu_aperture_rw(vgpu, pos, buf, count, is_write);
 819                 break;
 820         case VFIO_PCI_BAR1_REGION_INDEX:
 821         case VFIO_PCI_BAR3_REGION_INDEX:
 822         case VFIO_PCI_BAR4_REGION_INDEX:
 823         case VFIO_PCI_BAR5_REGION_INDEX:
 824         case VFIO_PCI_VGA_REGION_INDEX:
 825         case VFIO_PCI_ROM_REGION_INDEX:
 826                 break;
 827         default:
 828                 if (index >= VFIO_PCI_NUM_REGIONS + vgpu->num_regions)
 829                         return -EINVAL;
 830
 831                 index -= VFIO_PCI_NUM_REGIONS;
 832                 return vgpu->region[index].ops->rw(vgpu, buf, count,
 833                                 ppos, is_write);
 834         }
 835
 836         return ret == 0 ? count : ret;
 837 }
 838
 839 static bool gtt_entry(struct intel_vgpu *vgpu, loff_t *ppos)
 840 {
 841         unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
 842         struct intel_gvt *gvt = vgpu->gvt;
 843         int offset;
 844
 845         /* Only allow MMIO GGTT entry access */
 846         if (index != PCI_BASE_ADDRESS_0)
 847                 return false;
 848
 849         offset = (u64)(*ppos & VFIO_PCI_OFFSET_MASK) -
 850                 intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_0);
 851
 852         return (offset >= gvt->device_info.gtt_start_offset &&
 853                 offset < gvt->device_info.gtt_start_offset + gvt_ggtt_sz(gvt)) ?
 854                         true : false;
 855 }
 856
 857 static ssize_t intel_vgpu_read(struct vfio_device *vfio_dev, char __user *buf,
 858                         size_t count, loff_t *ppos)
 859 {
 860         struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
 861         unsigned int done = 0;
 862         int ret;
 863
 864         while (count) {
 865                 size_t filled;
 866
 867                 /* Only support GGTT entry 8 bytes read */
 868                 if (count >= 8 && !(*ppos % 8) &&
 869                         gtt_entry(vgpu, ppos)) {
 870                         u64 val;
 871
 872                         ret = intel_vgpu_rw(vgpu, (char *)&val, sizeof(val),
 873                                         ppos, false);
 874                         if (ret <= 0)
 875                                 goto read_err;
 876
 877                         if (copy_to_user(buf, &val, sizeof(val)))
 878                                 goto read_err;
 879
 880                         filled = 8;
 881                 } else if (count >= 4 && !(*ppos % 4)) {
 882                         u32 val;
 883
 884                         ret = intel_vgpu_rw(vgpu, (char *)&val, sizeof(val),
 885                                         ppos, false);
 886                         if (ret <= 0)
 887                                 goto read_err;
 888
 889                         if (copy_to_user(buf, &val, sizeof(val)))
 890                                 goto read_err;
 891
 892                         filled = 4;
 893                 } else if (count >= 2 && !(*ppos % 2)) {
 894                         u16 val;
 895
 896                         ret = intel_vgpu_rw(vgpu, (char *)&val, sizeof(val),
 897                                         ppos, false);
 898                         if (ret <= 0)
 899                                 goto read_err;
 900
 901                         if (copy_to_user(buf, &val, sizeof(val)))
 902                                 goto read_err;
 903
 904                         filled = 2;
 905                 } else {
 906                         u8 val;
 907
 908                         ret = intel_vgpu_rw(vgpu, &val, sizeof(val), ppos,
 909                                         false);
 910                         if (ret <= 0)
 911                                 goto read_err;
 912
 913                         if (copy_to_user(buf, &val, sizeof(val)))
 914                                 goto read_err;
 915
 916                         filled = 1;
 917                 }
 918
 919                 count -= filled;
 920                 done += filled;
 921                 *ppos += filled;
 922                 buf += filled;
 923         }
 924
 925         return done;
 926
 927 read_err:
 928         return -EFAULT;
 929 }
 930
 931 static ssize_t intel_vgpu_write(struct vfio_device *vfio_dev,
 932                                 const char __user *buf,
 933                                 size_t count, loff_t *ppos)
 934 {
 935         struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
 936         unsigned int done = 0;
 937         int ret;
 938
 939         while (count) {
 940                 size_t filled;
 941
 942                 /* Only support GGTT entry 8 bytes write */
 943                 if (count >= 8 && !(*ppos % 8) &&
 944                         gtt_entry(vgpu, ppos)) {
 945                         u64 val;
 946
 947                         if (copy_from_user(&val, buf, sizeof(val)))
 948                                 goto write_err;
 949
 950                         ret = intel_vgpu_rw(vgpu, (char *)&val, sizeof(val),
 951                                         ppos, true);
 952                         if (ret <= 0)
 953                                 goto write_err;
 954
 955                         filled = 8;
 956                 } else if (count >= 4 && !(*ppos % 4)) {
 957                         u32 val;
 958
 959                         if (copy_from_user(&val, buf, sizeof(val)))
 960                                 goto write_err;
 961
 962                         ret = intel_vgpu_rw(vgpu, (char *)&val, sizeof(val),
 963                                         ppos, true);
 964                         if (ret <= 0)
 965                                 goto write_err;
 966
 967                         filled = 4;
 968                 } else if (count >= 2 && !(*ppos % 2)) {
 969                         u16 val;
 970
 971                         if (copy_from_user(&val, buf, sizeof(val)))
 972                                 goto write_err;
 973
 974                         ret = intel_vgpu_rw(vgpu, (char *)&val,
 975                                         sizeof(val), ppos, true);
 976                         if (ret <= 0)
 977                                 goto write_err;
 978
 979                         filled = 2;
 980                 } else {
 981                         u8 val;
 982
 983                         if (copy_from_user(&val, buf, sizeof(val)))
 984                                 goto write_err;
 985
 986                         ret = intel_vgpu_rw(vgpu, &val, sizeof(val),
 987                                         ppos, true);
 988                         if (ret <= 0)
 989                                 goto write_err;
 990
 991                         filled = 1;
 992                 }
 993
 994                 count -= filled;
 995                 done += filled;
 996                 *ppos += filled;
 997                 buf += filled;
 998         }
 999
1000         return done;
1001 write_err:
1002         return -EFAULT;
1003 }
1004
1005 static int intel_vgpu_mmap(struct vfio_device *vfio_dev,
1006                 struct vm_area_struct *vma)
1007 {
1008         struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
1009         unsigned int index;
1010         u64 virtaddr;
1011         unsigned long req_size, pgoff, req_start;
1012         pgprot_t pg_prot;
1013
1014         index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
1015         if (index >= VFIO_PCI_ROM_REGION_INDEX)
1016                 return -EINVAL;
1017
1018         if (vma->vm_end < vma->vm_start)
1019                 return -EINVAL;
1020         if ((vma->vm_flags & VM_SHARED) == 0)
1021                 return -EINVAL;
1022         if (index != VFIO_PCI_BAR2_REGION_INDEX)
1023                 return -EINVAL;
1024
1025         pg_prot = vma->vm_page_prot;
1026         virtaddr = vma->vm_start;
1027         req_size = vma->vm_end - vma->vm_start;
1028         pgoff = vma->vm_pgoff &
1029                 ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
1030         req_start = pgoff << PAGE_SHIFT;
1031
1032         if (!intel_vgpu_in_aperture(vgpu, req_start))
1033                 return -EINVAL;
1034         if (req_start + req_size >
1035             vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu))
1036                 return -EINVAL;
1037
1038         pgoff = (gvt_aperture_pa_base(vgpu->gvt) >> PAGE_SHIFT) + pgoff;
1039
1040         return remap_pfn_range(vma, virtaddr, pgoff, req_size, pg_prot);
1041 }
1042
1043 static int intel_vgpu_get_irq_count(struct intel_vgpu *vgpu, int type)
1044 {
1045         if (type == VFIO_PCI_INTX_IRQ_INDEX || type == VFIO_PCI_MSI_IRQ_INDEX)
1046                 return 1;
1047
1048         return 0;
1049 }
1050
1051 static int intel_vgpu_set_intx_mask(struct intel_vgpu *vgpu,
1052                         unsigned int index, unsigned int start,
1053                         unsigned int count, u32 flags,
1054                         void *data)
1055 {
1056         return 0;
1057 }
1058
1059 static int intel_vgpu_set_intx_unmask(struct intel_vgpu *vgpu,
1060                         unsigned int index, unsigned int start,
1061                         unsigned int count, u32 flags, void *data)
1062 {
1063         return 0;
1064 }
1065
1066 static int intel_vgpu_set_intx_trigger(struct intel_vgpu *vgpu,
1067                 unsigned int index, unsigned int start, unsigned int count,
1068                 u32 flags, void *data)
1069 {
1070         return 0;
1071 }
1072
1073 static int intel_vgpu_set_msi_trigger(struct intel_vgpu *vgpu,
1074                 unsigned int index, unsigned int start, unsigned int count,
1075                 u32 flags, void *data)
1076 {
1077         struct eventfd_ctx *trigger;
1078
1079         if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
1080                 int fd = *(int *)data;
1081
1082                 trigger = eventfd_ctx_fdget(fd);
1083                 if (IS_ERR(trigger)) {
1084                         gvt_vgpu_err("eventfd_ctx_fdget failed\n");
1085                         return PTR_ERR(trigger);
1086                 }
1087                 vgpu->msi_trigger = trigger;
1088         } else if ((flags & VFIO_IRQ_SET_DATA_NONE) && !count)
1089                 intel_vgpu_release_msi_eventfd_ctx(vgpu);
1090
1091         return 0;
1092 }
1093
1094 static int intel_vgpu_set_irqs(struct intel_vgpu *vgpu, u32 flags,
1095                 unsigned int index, unsigned int start, unsigned int count,
1096                 void *data)
1097 {
1098         int (*func)(struct intel_vgpu *vgpu, unsigned int index,
1099                         unsigned int start, unsigned int count, u32 flags,
1100                         void *data) = NULL;
1101
1102         switch (index) {
1103         case VFIO_PCI_INTX_IRQ_INDEX:
1104                 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
1105                 case VFIO_IRQ_SET_ACTION_MASK:
1106                         func = intel_vgpu_set_intx_mask;
1107                         break;
1108                 case VFIO_IRQ_SET_ACTION_UNMASK:
1109                         func = intel_vgpu_set_intx_unmask;
1110                         break;
1111                 case VFIO_IRQ_SET_ACTION_TRIGGER:
1112                         func = intel_vgpu_set_intx_trigger;
1113                         break;
1114                 }
1115                 break;
1116         case VFIO_PCI_MSI_IRQ_INDEX:
1117                 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
1118                 case VFIO_IRQ_SET_ACTION_MASK:
1119                 case VFIO_IRQ_SET_ACTION_UNMASK:
1120                         /* XXX Need masking support exported */
1121                         break;
1122                 case VFIO_IRQ_SET_ACTION_TRIGGER:
1123                         func = intel_vgpu_set_msi_trigger;
1124                         break;
1125                 }
1126                 break;
1127         }
1128
1129         if (!func)
1130                 return -ENOTTY;
1131
1132         return func(vgpu, index, start, count, flags, data);
1133 }
1134
1135 static long intel_vgpu_ioctl(struct vfio_device *vfio_dev, unsigned int cmd,
1136                              unsigned long arg)
1137 {
1138         struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
1139         unsigned long minsz;
1140
1141         gvt_dbg_core("vgpu%d ioctl, cmd: %d\n", vgpu->id, cmd);
1142
1143         if (cmd == VFIO_DEVICE_GET_INFO) {
1144                 struct vfio_device_info info;
1145
1146                 minsz = offsetofend(struct vfio_device_info, num_irqs);
1147
1148                 if (copy_from_user(&info, (void __user *)arg, minsz))
1149                         return -EFAULT;
1150
1151                 if (info.argsz < minsz)
1152                         return -EINVAL;
1153
1154                 info.flags = VFIO_DEVICE_FLAGS_PCI;
1155                 info.flags |= VFIO_DEVICE_FLAGS_RESET;
1156                 info.num_regions = VFIO_PCI_NUM_REGIONS +
1157                                 vgpu->num_regions;
1158                 info.num_irqs = VFIO_PCI_NUM_IRQS;
1159
1160                 return copy_to_user((void __user *)arg, &info, minsz) ?
1161                         -EFAULT : 0;
1162
1163         } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
1164                 struct vfio_region_info info;
1165                 struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
1166                 unsigned int i;
1167                 int ret;
1168                 struct vfio_region_info_cap_sparse_mmap *sparse = NULL;
1169                 int nr_areas = 1;
1170                 int cap_type_id;
1171
1172                 minsz = offsetofend(struct vfio_region_info, offset);
1173
1174                 if (copy_from_user(&info, (void __user *)arg, minsz))
1175                         return -EFAULT;
1176
1177                 if (info.argsz < minsz)
1178                         return -EINVAL;
1179
1180                 switch (info.index) {
1181                 case VFIO_PCI_CONFIG_REGION_INDEX:
1182                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1183                         info.size = vgpu->gvt->device_info.cfg_space_size;
1184                         info.flags = VFIO_REGION_INFO_FLAG_READ |
1185                                      VFIO_REGION_INFO_FLAG_WRITE;
1186                         break;
1187                 case VFIO_PCI_BAR0_REGION_INDEX:
1188                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1189                         info.size = vgpu->cfg_space.bar[info.index].size;
1190                         if (!info.size) {
1191                                 info.flags = 0;
1192                                 break;
1193                         }
1194
1195                         info.flags = VFIO_REGION_INFO_FLAG_READ |
1196                                      VFIO_REGION_INFO_FLAG_WRITE;
1197                         break;
1198                 case VFIO_PCI_BAR1_REGION_INDEX:
1199                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1200                         info.size = 0;
1201                         info.flags = 0;
1202                         break;
1203                 case VFIO_PCI_BAR2_REGION_INDEX:
1204                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1205                         info.flags = VFIO_REGION_INFO_FLAG_CAPS |
1206                                         VFIO_REGION_INFO_FLAG_MMAP |
1207                                         VFIO_REGION_INFO_FLAG_READ |
1208                                         VFIO_REGION_INFO_FLAG_WRITE;
1209                         info.size = gvt_aperture_sz(vgpu->gvt);
1210
1211                         sparse = kzalloc(struct_size(sparse, areas, nr_areas),
1212                                          GFP_KERNEL);
1213                         if (!sparse)
1214                                 return -ENOMEM;
1215
1216                         sparse->header.id = VFIO_REGION_INFO_CAP_SPARSE_MMAP;
1217                         sparse->header.version = 1;
1218                         sparse->nr_areas = nr_areas;
1219                         cap_type_id = VFIO_REGION_INFO_CAP_SPARSE_MMAP;
1220                         sparse->areas[0].offset =
1221                                         PAGE_ALIGN(vgpu_aperture_offset(vgpu));
1222                         sparse->areas[0].size = vgpu_aperture_sz(vgpu);
1223                         break;
1224
1225                 case VFIO_PCI_BAR3_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
1226                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1227                         info.size = 0;
1228                         info.flags = 0;
1229
1230                         gvt_dbg_core("get region info bar:%d\n", info.index);
1231                         break;
1232
1233                 case VFIO_PCI_ROM_REGION_INDEX:
1234                 case VFIO_PCI_VGA_REGION_INDEX:
1235                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1236                         info.size = 0;
1237                         info.flags = 0;
1238
1239                         gvt_dbg_core("get region info index:%d\n", info.index);
1240                         break;
1241                 default:
1242                         {
1243                                 struct vfio_region_info_cap_type cap_type = {
1244                                         .header.id = VFIO_REGION_INFO_CAP_TYPE,
1245                                         .header.version = 1 };
1246
1247                                 if (info.index >= VFIO_PCI_NUM_REGIONS +
1248                                                 vgpu->num_regions)
1249                                         return -EINVAL;
1250                                 info.index =
1251                                         array_index_nospec(info.index,
1252                                                         VFIO_PCI_NUM_REGIONS +
1253                                                         vgpu->num_regions);
1254
1255                                 i = info.index - VFIO_PCI_NUM_REGIONS;
1256
1257                                 info.offset =
1258                                         VFIO_PCI_INDEX_TO_OFFSET(info.index);
1259                                 info.size = vgpu->region[i].size;
1260                                 info.flags = vgpu->region[i].flags;
1261
1262                                 cap_type.type = vgpu->region[i].type;
1263                                 cap_type.subtype = vgpu->region[i].subtype;
1264
1265                                 ret = vfio_info_add_capability(&caps,
1266                                                         &cap_type.header,
1267                                                         sizeof(cap_type));
1268                                 if (ret)
1269                                         return ret;
1270                         }
1271                 }
1272
1273                 if ((info.flags & VFIO_REGION_INFO_FLAG_CAPS) && sparse) {
1274                         switch (cap_type_id) {
1275                         case VFIO_REGION_INFO_CAP_SPARSE_MMAP:
1276                                 ret = vfio_info_add_capability(&caps,
1277                                         &sparse->header,
1278                                         struct_size(sparse, areas,
1279                                                     sparse->nr_areas));
1280                                 if (ret) {
1281                                         kfree(sparse);
1282                                         return ret;
1283                                 }
1284                                 break;
1285                         default:
1286                                 kfree(sparse);
1287                                 return -EINVAL;
1288                         }
1289                 }
1290
1291                 if (caps.size) {
1292                         info.flags |= VFIO_REGION_INFO_FLAG_CAPS;
1293                         if (info.argsz < sizeof(info) + caps.size) {
1294                                 info.argsz = sizeof(info) + caps.size;
1295                                 info.cap_offset = 0;
1296                         } else {
1297                                 vfio_info_cap_shift(&caps, sizeof(info));
1298                                 if (copy_to_user((void __user *)arg +
1299                                                   sizeof(info), caps.buf,
1300                                                   caps.size)) {
1301                                         kfree(caps.buf);
1302                                         kfree(sparse);
1303                                         return -EFAULT;
1304                                 }
1305                                 info.cap_offset = sizeof(info);
1306                         }
1307
1308                         kfree(caps.buf);
1309                 }
1310
1311                 kfree(sparse);
1312                 return copy_to_user((void __user *)arg, &info, minsz) ?
1313                         -EFAULT : 0;
1314         } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
1315                 struct vfio_irq_info info;
1316
1317                 minsz = offsetofend(struct vfio_irq_info, count);
1318
1319                 if (copy_from_user(&info, (void __user *)arg, minsz))
1320                         return -EFAULT;
1321
1322                 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS)
1323                         return -EINVAL;
1324
1325                 switch (info.index) {
1326                 case VFIO_PCI_INTX_IRQ_INDEX:
1327                 case VFIO_PCI_MSI_IRQ_INDEX:
1328                         break;
1329                 default:
1330                         return -EINVAL;
1331                 }
1332
1333                 info.flags = VFIO_IRQ_INFO_EVENTFD;
1334
1335                 info.count = intel_vgpu_get_irq_count(vgpu, info.index);
1336
1337                 if (info.index == VFIO_PCI_INTX_IRQ_INDEX)
1338                         info.flags |= (VFIO_IRQ_INFO_MASKABLE |
1339                                        VFIO_IRQ_INFO_AUTOMASKED);
1340                 else
1341                         info.flags |= VFIO_IRQ_INFO_NORESIZE;
1342
1343                 return copy_to_user((void __user *)arg, &info, minsz) ?
1344                         -EFAULT : 0;
1345         } else if (cmd == VFIO_DEVICE_SET_IRQS) {
1346                 struct vfio_irq_set hdr;
1347                 u8 *data = NULL;
1348                 int ret = 0;
1349                 size_t data_size = 0;
1350
1351                 minsz = offsetofend(struct vfio_irq_set, count);
1352
1353                 if (copy_from_user(&hdr, (void __user *)arg, minsz))
1354                         return -EFAULT;
1355
1356                 if (!(hdr.flags & VFIO_IRQ_SET_DATA_NONE)) {
1357                         int max = intel_vgpu_get_irq_count(vgpu, hdr.index);
1358
1359                         ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
1360                                                 VFIO_PCI_NUM_IRQS, &data_size);
1361                         if (ret) {
1362                                 gvt_vgpu_err("intel:vfio_set_irqs_validate_and_prepare failed\n");
1363                                 return -EINVAL;
1364                         }
1365                         if (data_size) {
1366                                 data = memdup_user((void __user *)(arg + minsz),
1367                                                    data_size);
1368                                 if (IS_ERR(data))
1369                                         return PTR_ERR(data);
1370                         }
1371                 }
1372
1373                 ret = intel_vgpu_set_irqs(vgpu, hdr.flags, hdr.index,
1374                                         hdr.start, hdr.count, data);
1375                 kfree(data);
1376
1377                 return ret;
1378         } else if (cmd == VFIO_DEVICE_RESET) {
1379                 intel_gvt_reset_vgpu(vgpu);
1380                 return 0;
1381         } else if (cmd == VFIO_DEVICE_QUERY_GFX_PLANE) {
1382                 struct vfio_device_gfx_plane_info dmabuf;
1383                 int ret = 0;
1384
1385                 minsz = offsetofend(struct vfio_device_gfx_plane_info,
1386                                     dmabuf_id);
1387                 if (copy_from_user(&dmabuf, (void __user *)arg, minsz))
1388                         return -EFAULT;
1389                 if (dmabuf.argsz < minsz)
1390                         return -EINVAL;
1391
1392                 ret = intel_vgpu_query_plane(vgpu, &dmabuf);
1393                 if (ret != 0)
1394                         return ret;
1395
1396                 return copy_to_user((void __user *)arg, &dmabuf, minsz) ?
1397                                                                 -EFAULT : 0;
1398         } else if (cmd == VFIO_DEVICE_GET_GFX_DMABUF) {
1399                 __u32 dmabuf_id;
1400
1401                 if (get_user(dmabuf_id, (__u32 __user *)arg))
1402                         return -EFAULT;
1403                 return intel_vgpu_get_dmabuf(vgpu, dmabuf_id);
1404         }
1405
1406         return -ENOTTY;
1407 }
1408
1409 static ssize_t
1410 vgpu_id_show(struct device *dev, struct device_attribute *attr,
1411              char *buf)
1412 {
1413         struct intel_vgpu *vgpu = dev_get_drvdata(dev);
1414
1415         return sprintf(buf, "%d\n", vgpu->id);
1416 }
1417
1418 static DEVICE_ATTR_RO(vgpu_id);
1419
1420 static struct attribute *intel_vgpu_attrs[] = {
1421         &dev_attr_vgpu_id.attr,
1422         NULL
1423 };
1424
1425 static const struct attribute_group intel_vgpu_group = {
1426         .name = "intel_vgpu",
1427         .attrs = intel_vgpu_attrs,
1428 };
1429
1430 static const struct attribute_group *intel_vgpu_groups[] = {
1431         &intel_vgpu_group,
1432         NULL,
1433 };
1434
1435 static int intel_vgpu_init_dev(struct vfio_device *vfio_dev)
1436 {
1437         struct mdev_device *mdev = to_mdev_device(vfio_dev->dev);
1438         struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
1439         struct intel_vgpu_type *type =
1440                 container_of(mdev->type, struct intel_vgpu_type, type);
1441         int ret;
1442
1443         vgpu->gvt = kdev_to_i915(mdev->type->parent->dev)->gvt;
1444         ret = intel_gvt_create_vgpu(vgpu, type->conf);
1445         if (ret)
1446                 return ret;
1447
1448         kvmgt_protect_table_init(vgpu);
1449         gvt_cache_init(vgpu);
1450
1451         return 0;
1452 }
1453
1454 static void intel_vgpu_release_dev(struct vfio_device *vfio_dev)
1455 {
1456         struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
1457
1458         intel_gvt_destroy_vgpu(vgpu);
1459 }
1460
1461 static const struct vfio_device_ops intel_vgpu_dev_ops = {
1462         .init           = intel_vgpu_init_dev,
1463         .release        = intel_vgpu_release_dev,
1464         .open_device    = intel_vgpu_open_device,
1465         .close_device   = intel_vgpu_close_device,
1466         .read           = intel_vgpu_read,
1467         .write          = intel_vgpu_write,
1468         .mmap           = intel_vgpu_mmap,
1469         .ioctl          = intel_vgpu_ioctl,
1470         .dma_unmap      = intel_vgpu_dma_unmap,
1471         .bind_iommufd   = vfio_iommufd_emulated_bind,
1472         .unbind_iommufd = vfio_iommufd_emulated_unbind,
1473         .attach_ioas    = vfio_iommufd_emulated_attach_ioas,
1474         .detach_ioas    = vfio_iommufd_emulated_detach_ioas,
1475 };
1476
1477 static int intel_vgpu_probe(struct mdev_device *mdev)
1478 {
1479         struct intel_vgpu *vgpu;
1480         int ret;
1481
1482         vgpu = vfio_alloc_device(intel_vgpu, vfio_device, &mdev->dev,
1483                                  &intel_vgpu_dev_ops);
1484         if (IS_ERR(vgpu)) {
1485                 gvt_err("failed to create intel vgpu: %ld\n", PTR_ERR(vgpu));
1486                 return PTR_ERR(vgpu);
1487         }
1488
1489         dev_set_drvdata(&mdev->dev, vgpu);
1490         ret = vfio_register_emulated_iommu_dev(&vgpu->vfio_device);
1491         if (ret)
1492                 goto out_put_vdev;
1493
1494         gvt_dbg_core("intel_vgpu_create succeeded for mdev: %s\n",
1495                      dev_name(mdev_dev(mdev)));
1496         return 0;
1497
1498 out_put_vdev:
1499         vfio_put_device(&vgpu->vfio_device);
1500         return ret;
1501 }
1502
1503 static void intel_vgpu_remove(struct mdev_device *mdev)
1504 {
1505         struct intel_vgpu *vgpu = dev_get_drvdata(&mdev->dev);
1506
1507         vfio_unregister_group_dev(&vgpu->vfio_device);
1508         vfio_put_device(&vgpu->vfio_device);
1509 }
1510
1511 static unsigned int intel_vgpu_get_available(struct mdev_type *mtype)
1512 {
1513         struct intel_vgpu_type *type =
1514                 container_of(mtype, struct intel_vgpu_type, type);
1515         struct intel_gvt *gvt = kdev_to_i915(mtype->parent->dev)->gvt;
1516         unsigned int low_gm_avail, high_gm_avail, fence_avail;
1517
1518         mutex_lock(&gvt->lock);
1519         low_gm_avail = gvt_aperture_sz(gvt) - HOST_LOW_GM_SIZE -
1520                 gvt->gm.vgpu_allocated_low_gm_size;
1521         high_gm_avail = gvt_hidden_sz(gvt) - HOST_HIGH_GM_SIZE -
1522                 gvt->gm.vgpu_allocated_high_gm_size;
1523         fence_avail = gvt_fence_sz(gvt) - HOST_FENCE -
1524                 gvt->fence.vgpu_allocated_fence_num;
1525         mutex_unlock(&gvt->lock);
1526
1527         return min3(low_gm_avail / type->conf->low_mm,
1528                     high_gm_avail / type->conf->high_mm,
1529                     fence_avail / type->conf->fence);
1530 }
1531
1532 static struct mdev_driver intel_vgpu_mdev_driver = {
1533         .device_api     = VFIO_DEVICE_API_PCI_STRING,
1534         .driver = {
1535                 .name           = "intel_vgpu_mdev",
1536                 .owner          = THIS_MODULE,
1537                 .dev_groups     = intel_vgpu_groups,
1538         },
1539         .probe                  = intel_vgpu_probe,
1540         .remove                 = intel_vgpu_remove,
1541         .get_available          = intel_vgpu_get_available,
1542         .show_description       = intel_vgpu_show_description,
1543 };
1544
1545 int intel_gvt_page_track_add(struct intel_vgpu *info, u64 gfn)
1546 {
1547         int r;
1548
1549         if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, info->status))
1550                 return -ESRCH;
1551
1552         if (kvmgt_gfn_is_write_protected(info, gfn))
1553                 return 0;
1554
1555         r = kvm_write_track_add_gfn(info->vfio_device.kvm, gfn);
1556         if (r)
1557                 return r;
1558
1559         kvmgt_protect_table_add(info, gfn);
1560         return 0;
1561 }
1562
1563 int intel_gvt_page_track_remove(struct intel_vgpu *info, u64 gfn)
1564 {
1565         int r;
1566
1567         if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, info->status))
1568                 return -ESRCH;
1569
1570         if (!kvmgt_gfn_is_write_protected(info, gfn))
1571                 return 0;
1572
1573         r = kvm_write_track_remove_gfn(info->vfio_device.kvm, gfn);
1574         if (r)
1575                 return r;
1576
1577         kvmgt_protect_table_del(info, gfn);
1578         return 0;
1579 }
1580
1581 static void kvmgt_page_track_write(gpa_t gpa, const u8 *val, int len,
1582                                    struct kvm_page_track_notifier_node *node)
1583 {
1584         struct intel_vgpu *info =
1585                 container_of(node, struct intel_vgpu, track_node);
1586
1587         mutex_lock(&info->vgpu_lock);
1588
1589         if (kvmgt_gfn_is_write_protected(info, gpa >> PAGE_SHIFT))
1590                 intel_vgpu_page_track_handler(info, gpa,
1591                                                      (void *)val, len);
1592
1593         mutex_unlock(&info->vgpu_lock);
1594 }
1595
1596 static void kvmgt_page_track_remove_region(gfn_t gfn, unsigned long nr_pages,
1597                                            struct kvm_page_track_notifier_node *node)
1598 {
1599         unsigned long i;
1600         struct intel_vgpu *info =
1601                 container_of(node, struct intel_vgpu, track_node);
1602
1603         mutex_lock(&info->vgpu_lock);
1604
1605         for (i = 0; i < nr_pages; i++) {
1606                 if (kvmgt_gfn_is_write_protected(info, gfn + i))
1607                         kvmgt_protect_table_del(info, gfn + i);
1608         }
1609
1610         mutex_unlock(&info->vgpu_lock);
1611 }
1612
1613 void intel_vgpu_detach_regions(struct intel_vgpu *vgpu)
1614 {
1615         int i;
1616
1617         if (!vgpu->region)
1618                 return;
1619
1620         for (i = 0; i < vgpu->num_regions; i++)
1621                 if (vgpu->region[i].ops->release)
1622                         vgpu->region[i].ops->release(vgpu,
1623                                         &vgpu->region[i]);
1624         vgpu->num_regions = 0;
1625         kfree(vgpu->region);
1626         vgpu->region = NULL;
1627 }
1628
1629 int intel_gvt_dma_map_guest_page(struct intel_vgpu *vgpu, unsigned long gfn,
1630                 unsigned long size, dma_addr_t *dma_addr)
1631 {
1632         struct gvt_dma *entry;
1633         int ret;
1634
1635         if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, vgpu->status))
1636                 return -EINVAL;
1637
1638         mutex_lock(&vgpu->cache_lock);
1639
1640         entry = __gvt_cache_find_gfn(vgpu, gfn);
1641         if (!entry) {
1642                 ret = gvt_dma_map_page(vgpu, gfn, dma_addr, size);
1643                 if (ret)
1644                         goto err_unlock;
1645
1646                 ret = __gvt_cache_add(vgpu, gfn, *dma_addr, size);
1647                 if (ret)
1648                         goto err_unmap;
1649         } else if (entry->size != size) {
1650                 /* the same gfn with different size: unmap and re-map */
1651                 gvt_dma_unmap_page(vgpu, gfn, entry->dma_addr, entry->size);
1652                 __gvt_cache_remove_entry(vgpu, entry);
1653
1654                 ret = gvt_dma_map_page(vgpu, gfn, dma_addr, size);
1655                 if (ret)
1656                         goto err_unlock;
1657
1658                 ret = __gvt_cache_add(vgpu, gfn, *dma_addr, size);
1659                 if (ret)
1660                         goto err_unmap;
1661         } else {
1662                 kref_get(&entry->ref);
1663                 *dma_addr = entry->dma_addr;
1664         }
1665
1666         mutex_unlock(&vgpu->cache_lock);
1667         return 0;
1668
1669 err_unmap:
1670         gvt_dma_unmap_page(vgpu, gfn, *dma_addr, size);
1671 err_unlock:
1672         mutex_unlock(&vgpu->cache_lock);
1673         return ret;
1674 }
1675
1676 int intel_gvt_dma_pin_guest_page(struct intel_vgpu *vgpu, dma_addr_t dma_addr)
1677 {
1678         struct gvt_dma *entry;
1679         int ret = 0;
1680
1681         if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, vgpu->status))
1682                 return -EINVAL;
1683
1684         mutex_lock(&vgpu->cache_lock);
1685         entry = __gvt_cache_find_dma_addr(vgpu, dma_addr);
1686         if (entry)
1687                 kref_get(&entry->ref);
1688         else
1689                 ret = -ENOMEM;
1690         mutex_unlock(&vgpu->cache_lock);
1691
1692         return ret;
1693 }
1694
1695 static void __gvt_dma_release(struct kref *ref)
1696 {
1697         struct gvt_dma *entry = container_of(ref, typeof(*entry), ref);
1698
1699         gvt_dma_unmap_page(entry->vgpu, entry->gfn, entry->dma_addr,
1700                            entry->size);
1701         __gvt_cache_remove_entry(entry->vgpu, entry);
1702 }
1703
1704 void intel_gvt_dma_unmap_guest_page(struct intel_vgpu *vgpu,
1705                 dma_addr_t dma_addr)
1706 {
1707         struct gvt_dma *entry;
1708
1709         if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, vgpu->status))
1710                 return;
1711
1712         mutex_lock(&vgpu->cache_lock);
1713         entry = __gvt_cache_find_dma_addr(vgpu, dma_addr);
1714         if (entry)
1715                 kref_put(&entry->ref, __gvt_dma_release);
1716         mutex_unlock(&vgpu->cache_lock);
1717 }
1718
1719 static void init_device_info(struct intel_gvt *gvt)
1720 {
1721         struct intel_gvt_device_info *info = &gvt->device_info;
1722         struct pci_dev *pdev = to_pci_dev(gvt->gt->i915->drm.dev);
1723
1724         info->max_support_vgpus = 8;
1725         info->cfg_space_size = PCI_CFG_SPACE_EXP_SIZE;
1726         info->mmio_size = 2 * 1024 * 1024;
1727         info->mmio_bar = 0;
1728         info->gtt_start_offset = 8 * 1024 * 1024;
1729         info->gtt_entry_size = 8;
1730         info->gtt_entry_size_shift = 3;
1731         info->gmadr_bytes_in_cmd = 8;
1732         info->max_surface_size = 36 * 1024 * 1024;
1733         info->msi_cap_offset = pdev->msi_cap;
1734 }
1735
1736 static void intel_gvt_test_and_emulate_vblank(struct intel_gvt *gvt)
1737 {
1738         struct intel_vgpu *vgpu;
1739         int id;
1740
1741         mutex_lock(&gvt->lock);
1742         idr_for_each_entry((&(gvt)->vgpu_idr), (vgpu), (id)) {
1743                 if (test_and_clear_bit(INTEL_GVT_REQUEST_EMULATE_VBLANK + id,
1744                                        (void *)&gvt->service_request)) {
1745                         if (test_bit(INTEL_VGPU_STATUS_ACTIVE, vgpu->status))
1746                                 intel_vgpu_emulate_vblank(vgpu);
1747                 }
1748         }
1749         mutex_unlock(&gvt->lock);
1750 }
1751
1752 static int gvt_service_thread(void *data)
1753 {
1754         struct intel_gvt *gvt = (struct intel_gvt *)data;
1755         int ret;
1756
1757         gvt_dbg_core("service thread start\n");
1758
1759         while (!kthread_should_stop()) {
1760                 ret = wait_event_interruptible(gvt->service_thread_wq,
1761                                 kthread_should_stop() || gvt->service_request);
1762
1763                 if (kthread_should_stop())
1764                         break;
1765
1766                 if (WARN_ONCE(ret, "service thread is waken up by signal.\n"))
1767                         continue;
1768
1769                 intel_gvt_test_and_emulate_vblank(gvt);
1770
1771                 if (test_bit(INTEL_GVT_REQUEST_SCHED,
1772                                 (void *)&gvt->service_request) ||
1773                         test_bit(INTEL_GVT_REQUEST_EVENT_SCHED,
1774                                         (void *)&gvt->service_request)) {
1775                         intel_gvt_schedule(gvt);
1776                 }
1777         }
1778
1779         return 0;
1780 }
1781
1782 static void clean_service_thread(struct intel_gvt *gvt)
1783 {
1784         kthread_stop(gvt->service_thread);
1785 }
1786
1787 static int init_service_thread(struct intel_gvt *gvt)
1788 {
1789         init_waitqueue_head(&gvt->service_thread_wq);
1790
1791         gvt->service_thread = kthread_run(gvt_service_thread,
1792                         gvt, "gvt_service_thread");
1793         if (IS_ERR(gvt->service_thread)) {
1794                 gvt_err("fail to start service thread.\n");
1795                 return PTR_ERR(gvt->service_thread);
1796         }
1797         return 0;
1798 }
1799
1800 /**
1801  * intel_gvt_clean_device - clean a GVT device
1802  * @i915: i915 private
1803  *
1804  * This function is called at the driver unloading stage, to free the
1805  * resources owned by a GVT device.
1806  *
1807  */
1808 static void intel_gvt_clean_device(struct drm_i915_private *i915)
1809 {
1810         struct intel_gvt *gvt = fetch_and_zero(&i915->gvt);
1811
1812         if (drm_WARN_ON(&i915->drm, !gvt))
1813                 return;
1814
1815         mdev_unregister_parent(&gvt->parent);
1816         intel_gvt_destroy_idle_vgpu(gvt->idle_vgpu);
1817         intel_gvt_clean_vgpu_types(gvt);
1818
1819         intel_gvt_debugfs_clean(gvt);
1820         clean_service_thread(gvt);
1821         intel_gvt_clean_cmd_parser(gvt);
1822         intel_gvt_clean_sched_policy(gvt);
1823         intel_gvt_clean_workload_scheduler(gvt);
1824         intel_gvt_clean_gtt(gvt);
1825         intel_gvt_free_firmware(gvt);
1826         intel_gvt_clean_mmio_info(gvt);
1827         idr_destroy(&gvt->vgpu_idr);
1828
1829         kfree(i915->gvt);
1830 }
1831
1832 /**
1833  * intel_gvt_init_device - initialize a GVT device
1834  * @i915: drm i915 private data
1835  *
1836  * This function is called at the initialization stage, to initialize
1837  * necessary GVT components.
1838  *
1839  * Returns:
1840  * Zero on success, negative error code if failed.
1841  *
1842  */
1843 static int intel_gvt_init_device(struct drm_i915_private *i915)
1844 {
1845         struct intel_gvt *gvt;
1846         struct intel_vgpu *vgpu;
1847         int ret;
1848
1849         if (drm_WARN_ON(&i915->drm, i915->gvt))
1850                 return -EEXIST;
1851
1852         gvt = kzalloc(sizeof(struct intel_gvt), GFP_KERNEL);
1853         if (!gvt)
1854                 return -ENOMEM;
1855
1856         gvt_dbg_core("init gvt device\n");
1857
1858         idr_init_base(&gvt->vgpu_idr, 1);
1859         spin_lock_init(&gvt->scheduler.mmio_context_lock);
1860         mutex_init(&gvt->lock);
1861         mutex_init(&gvt->sched_lock);
1862         gvt->gt = to_gt(i915);
1863         i915->gvt = gvt;
1864
1865         init_device_info(gvt);
1866
1867         ret = intel_gvt_setup_mmio_info(gvt);
1868         if (ret)
1869                 goto out_clean_idr;
1870
1871         intel_gvt_init_engine_mmio_context(gvt);
1872
1873         ret = intel_gvt_load_firmware(gvt);
1874         if (ret)
1875                 goto out_clean_mmio_info;
1876
1877         ret = intel_gvt_init_irq(gvt);
1878         if (ret)
1879                 goto out_free_firmware;
1880
1881         ret = intel_gvt_init_gtt(gvt);
1882         if (ret)
1883                 goto out_free_firmware;
1884
1885         ret = intel_gvt_init_workload_scheduler(gvt);
1886         if (ret)
1887                 goto out_clean_gtt;
1888
1889         ret = intel_gvt_init_sched_policy(gvt);
1890         if (ret)
1891                 goto out_clean_workload_scheduler;
1892
1893         ret = intel_gvt_init_cmd_parser(gvt);
1894         if (ret)
1895                 goto out_clean_sched_policy;
1896
1897         ret = init_service_thread(gvt);
1898         if (ret)
1899                 goto out_clean_cmd_parser;
1900
1901         ret = intel_gvt_init_vgpu_types(gvt);
1902         if (ret)
1903                 goto out_clean_thread;
1904
1905         vgpu = intel_gvt_create_idle_vgpu(gvt);
1906         if (IS_ERR(vgpu)) {
1907                 ret = PTR_ERR(vgpu);
1908                 gvt_err("failed to create idle vgpu\n");
1909                 goto out_clean_types;
1910         }
1911         gvt->idle_vgpu = vgpu;
1912
1913         intel_gvt_debugfs_init(gvt);
1914
1915         ret = mdev_register_parent(&gvt->parent, i915->drm.dev,
1916                                    &intel_vgpu_mdev_driver,
1917                                    gvt->mdev_types, gvt->num_types);
1918         if (ret)
1919                 goto out_destroy_idle_vgpu;
1920
1921         gvt_dbg_core("gvt device initialization is done\n");
1922         return 0;
1923
1924 out_destroy_idle_vgpu:
1925         intel_gvt_destroy_idle_vgpu(gvt->idle_vgpu);
1926         intel_gvt_debugfs_clean(gvt);
1927 out_clean_types:
1928         intel_gvt_clean_vgpu_types(gvt);
1929 out_clean_thread:
1930         clean_service_thread(gvt);
1931 out_clean_cmd_parser:
1932         intel_gvt_clean_cmd_parser(gvt);
1933 out_clean_sched_policy:
1934         intel_gvt_clean_sched_policy(gvt);
1935 out_clean_workload_scheduler:
1936         intel_gvt_clean_workload_scheduler(gvt);
1937 out_clean_gtt:
1938         intel_gvt_clean_gtt(gvt);
1939 out_free_firmware:
1940         intel_gvt_free_firmware(gvt);
1941 out_clean_mmio_info:
1942         intel_gvt_clean_mmio_info(gvt);
1943 out_clean_idr:
1944         idr_destroy(&gvt->vgpu_idr);
1945         kfree(gvt);
1946         i915->gvt = NULL;
1947         return ret;
1948 }
1949
1950 static void intel_gvt_pm_resume(struct drm_i915_private *i915)
1951 {
1952         struct intel_gvt *gvt = i915->gvt;
1953
1954         intel_gvt_restore_fence(gvt);
1955         intel_gvt_restore_mmio(gvt);
1956         intel_gvt_restore_ggtt(gvt);
1957 }
1958
1959 static const struct intel_vgpu_ops intel_gvt_vgpu_ops = {
1960         .init_device    = intel_gvt_init_device,
1961         .clean_device   = intel_gvt_clean_device,
1962         .pm_resume      = intel_gvt_pm_resume,
1963 };
1964
1965 static int __init kvmgt_init(void)
1966 {
1967         int ret;
1968
1969         ret = intel_gvt_set_ops(&intel_gvt_vgpu_ops);
1970         if (ret)
1971                 return ret;
1972
1973         ret = mdev_register_driver(&intel_vgpu_mdev_driver);
1974         if (ret)
1975                 intel_gvt_clear_ops(&intel_gvt_vgpu_ops);
1976         return ret;
1977 }
1978
1979 static void __exit kvmgt_exit(void)
1980 {
1981         mdev_unregister_driver(&intel_vgpu_mdev_driver);
1982         intel_gvt_clear_ops(&intel_gvt_vgpu_ops);
1983 }
1984
1985 module_init(kvmgt_init);
1986 module_exit(kvmgt_exit);
1987
1988 MODULE_LICENSE("GPL and additional rights");
1989 MODULE_AUTHOR("Intel Corporation");