drivers/gpu/drm/i915/i915_gem_gtt.c

   1 /*
   2  * Copyright © 2010 Daniel Vetter
   3  * Copyright © 2011-2014 Intel Corporation
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <linux/slab.h> /* fault-inject.h is not standalone! */
  27
  28 #include <linux/fault-inject.h>
  29 #include <linux/log2.h>
  30 #include <linux/random.h>
  31 #include <linux/seq_file.h>
  32 #include <linux/stop_machine.h>
  33
  34 #include <drm/drmP.h>
  35 #include <drm/i915_drm.h>
  36
  37 #include "i915_drv.h"
  38 #include "i915_vgpu.h"
  39 #include "i915_trace.h"
  40 #include "intel_drv.h"
  41 #include "intel_frontbuffer.h"
  42
  43 #define I915_GFP_DMA (GFP_KERNEL | __GFP_HIGHMEM)
  44
  45 /**
  46  * DOC: Global GTT views
  47  *
  48  * Background and previous state
  49  *
  50  * Historically objects could exists (be bound) in global GTT space only as
  51  * singular instances with a view representing all of the object's backing pages
  52  * in a linear fashion. This view will be called a normal view.
  53  *
  54  * To support multiple views of the same object, where the number of mapped
  55  * pages is not equal to the backing store, or where the layout of the pages
  56  * is not linear, concept of a GGTT view was added.
  57  *
  58  * One example of an alternative view is a stereo display driven by a single
  59  * image. In this case we would have a framebuffer looking like this
  60  * (2x2 pages):
  61  *
  62  *    12
  63  *    34
  64  *
  65  * Above would represent a normal GGTT view as normally mapped for GPU or CPU
  66  * rendering. In contrast, fed to the display engine would be an alternative
  67  * view which could look something like this:
  68  *
  69  *   1212
  70  *   3434
  71  *
  72  * In this example both the size and layout of pages in the alternative view is
  73  * different from the normal view.
  74  *
  75  * Implementation and usage
  76  *
  77  * GGTT views are implemented using VMAs and are distinguished via enum
  78  * i915_ggtt_view_type and struct i915_ggtt_view.
  79  *
  80  * A new flavour of core GEM functions which work with GGTT bound objects were
  81  * added with the _ggtt_ infix, and sometimes with _view postfix to avoid
  82  * renaming  in large amounts of code. They take the struct i915_ggtt_view
  83  * parameter encapsulating all metadata required to implement a view.
  84  *
  85  * As a helper for callers which are only interested in the normal view,
  86  * globally const i915_ggtt_view_normal singleton instance exists. All old core
  87  * GEM API functions, the ones not taking the view parameter, are operating on,
  88  * or with the normal GGTT view.
  89  *
  90  * Code wanting to add or use a new GGTT view needs to:
  91  *
  92  * 1. Add a new enum with a suitable name.
  93  * 2. Extend the metadata in the i915_ggtt_view structure if required.
  94  * 3. Add support to i915_get_vma_pages().
  95  *
  96  * New views are required to build a scatter-gather table from within the
  97  * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and
  98  * exists for the lifetime of an VMA.
  99  *
 100  * Core API is designed to have copy semantics which means that passed in
 101  * struct i915_ggtt_view does not need to be persistent (left around after
 102  * calling the core API functions).
 103  *
 104  */
 105
 106 static int
 107 i915_get_ggtt_vma_pages(struct i915_vma *vma);
 108
 109 static void gen6_ggtt_invalidate(struct drm_i915_private *dev_priv)
 110 {
 111         /* Note that as an uncached mmio write, this should flush the
 112          * WCB of the writes into the GGTT before it triggers the invalidate.
 113          */
 114         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
 115 }
 116
 117 static void guc_ggtt_invalidate(struct drm_i915_private *dev_priv)
 118 {
 119         gen6_ggtt_invalidate(dev_priv);
 120         I915_WRITE(GEN8_GTCR, GEN8_GTCR_INVALIDATE);
 121 }
 122
 123 static void gmch_ggtt_invalidate(struct drm_i915_private *dev_priv)
 124 {
 125         intel_gtt_chipset_flush();
 126 }
 127
 128 static inline void i915_ggtt_invalidate(struct drm_i915_private *i915)
 129 {
 130         i915->ggtt.invalidate(i915);
 131 }
 132
 133 int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv,
 134                                 int enable_ppgtt)
 135 {
 136         bool has_aliasing_ppgtt;
 137         bool has_full_ppgtt;
 138         bool has_full_48bit_ppgtt;
 139
 140         has_aliasing_ppgtt = dev_priv->info.has_aliasing_ppgtt;
 141         has_full_ppgtt = dev_priv->info.has_full_ppgtt;
 142         has_full_48bit_ppgtt = dev_priv->info.has_full_48bit_ppgtt;
 143
 144         if (intel_vgpu_active(dev_priv)) {
 145                 /* emulation is too hard */
 146                 has_full_ppgtt = false;
 147                 has_full_48bit_ppgtt = false;
 148         }
 149
 150         if (!has_aliasing_ppgtt)
 151                 return 0;
 152
 153         /*
 154          * We don't allow disabling PPGTT for gen9+ as it's a requirement for
 155          * execlists, the sole mechanism available to submit work.
 156          */
 157         if (enable_ppgtt == 0 && INTEL_GEN(dev_priv) < 9)
 158                 return 0;
 159
 160         if (enable_ppgtt == 1)
 161                 return 1;
 162
 163         if (enable_ppgtt == 2 && has_full_ppgtt)
 164                 return 2;
 165
 166         if (enable_ppgtt == 3 && has_full_48bit_ppgtt)
 167                 return 3;
 168
 169 #ifdef CONFIG_INTEL_IOMMU
 170         /* Disable ppgtt on SNB if VT-d is on. */
 171         if (IS_GEN6(dev_priv) && intel_iommu_gfx_mapped) {
 172                 DRM_INFO("Disabling PPGTT because VT-d is on\n");
 173                 return 0;
 174         }
 175 #endif
 176
 177         /* Early VLV doesn't have this */
 178         if (IS_VALLEYVIEW(dev_priv) && dev_priv->drm.pdev->revision < 0xb) {
 179                 DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n");
 180                 return 0;
 181         }
 182
 183         if (INTEL_GEN(dev_priv) >= 8 && i915.enable_execlists && has_full_ppgtt)
 184                 return has_full_48bit_ppgtt ? 3 : 2;
 185         else
 186                 return has_aliasing_ppgtt ? 1 : 0;
 187 }
 188
 189 static int ppgtt_bind_vma(struct i915_vma *vma,
 190                           enum i915_cache_level cache_level,
 191                           u32 unused)
 192 {
 193         u32 pte_flags = 0;
 194
 195         vma->pages = vma->obj->mm.pages;
 196
 197         /* Currently applicable only to VLV */
 198         if (vma->obj->gt_ro)
 199                 pte_flags |= PTE_READ_ONLY;
 200
 201         vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start,
 202                                 cache_level, pte_flags);
 203
 204         return 0;
 205 }
 206
 207 static void ppgtt_unbind_vma(struct i915_vma *vma)
 208 {
 209         vma->vm->clear_range(vma->vm,
 210                              vma->node.start,
 211                              vma->size);
 212 }
 213
 214 static gen8_pte_t gen8_pte_encode(dma_addr_t addr,
 215                                   enum i915_cache_level level)
 216 {
 217         gen8_pte_t pte = _PAGE_PRESENT | _PAGE_RW;
 218         pte |= addr;
 219
 220         switch (level) {
 221         case I915_CACHE_NONE:
 222                 pte |= PPAT_UNCACHED_INDEX;
 223                 break;
 224         case I915_CACHE_WT:
 225                 pte |= PPAT_DISPLAY_ELLC_INDEX;
 226                 break;
 227         default:
 228                 pte |= PPAT_CACHED_INDEX;
 229                 break;
 230         }
 231
 232         return pte;
 233 }
 234
 235 static gen8_pde_t gen8_pde_encode(const dma_addr_t addr,
 236                                   const enum i915_cache_level level)
 237 {
 238         gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW;
 239         pde |= addr;
 240         if (level != I915_CACHE_NONE)
 241                 pde |= PPAT_CACHED_PDE_INDEX;
 242         else
 243                 pde |= PPAT_UNCACHED_INDEX;
 244         return pde;
 245 }
 246
 247 #define gen8_pdpe_encode gen8_pde_encode
 248 #define gen8_pml4e_encode gen8_pde_encode
 249
 250 static gen6_pte_t snb_pte_encode(dma_addr_t addr,
 251                                  enum i915_cache_level level,
 252                                  u32 unused)
 253 {
 254         gen6_pte_t pte = GEN6_PTE_VALID;
 255         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 256
 257         switch (level) {
 258         case I915_CACHE_L3_LLC:
 259         case I915_CACHE_LLC:
 260                 pte |= GEN6_PTE_CACHE_LLC;
 261                 break;
 262         case I915_CACHE_NONE:
 263                 pte |= GEN6_PTE_UNCACHED;
 264                 break;
 265         default:
 266                 MISSING_CASE(level);
 267         }
 268
 269         return pte;
 270 }
 271
 272 static gen6_pte_t ivb_pte_encode(dma_addr_t addr,
 273                                  enum i915_cache_level level,
 274                                  u32 unused)
 275 {
 276         gen6_pte_t pte = GEN6_PTE_VALID;
 277         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 278
 279         switch (level) {
 280         case I915_CACHE_L3_LLC:
 281                 pte |= GEN7_PTE_CACHE_L3_LLC;
 282                 break;
 283         case I915_CACHE_LLC:
 284                 pte |= GEN6_PTE_CACHE_LLC;
 285                 break;
 286         case I915_CACHE_NONE:
 287                 pte |= GEN6_PTE_UNCACHED;
 288                 break;
 289         default:
 290                 MISSING_CASE(level);
 291         }
 292
 293         return pte;
 294 }
 295
 296 static gen6_pte_t byt_pte_encode(dma_addr_t addr,
 297                                  enum i915_cache_level level,
 298                                  u32 flags)
 299 {
 300         gen6_pte_t pte = GEN6_PTE_VALID;
 301         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 302
 303         if (!(flags & PTE_READ_ONLY))
 304                 pte |= BYT_PTE_WRITEABLE;
 305
 306         if (level != I915_CACHE_NONE)
 307                 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
 308
 309         return pte;
 310 }
 311
 312 static gen6_pte_t hsw_pte_encode(dma_addr_t addr,
 313                                  enum i915_cache_level level,
 314                                  u32 unused)
 315 {
 316         gen6_pte_t pte = GEN6_PTE_VALID;
 317         pte |= HSW_PTE_ADDR_ENCODE(addr);
 318
 319         if (level != I915_CACHE_NONE)
 320                 pte |= HSW_WB_LLC_AGE3;
 321
 322         return pte;
 323 }
 324
 325 static gen6_pte_t iris_pte_encode(dma_addr_t addr,
 326                                   enum i915_cache_level level,
 327                                   u32 unused)
 328 {
 329         gen6_pte_t pte = GEN6_PTE_VALID;
 330         pte |= HSW_PTE_ADDR_ENCODE(addr);
 331
 332         switch (level) {
 333         case I915_CACHE_NONE:
 334                 break;
 335         case I915_CACHE_WT:
 336                 pte |= HSW_WT_ELLC_LLC_AGE3;
 337                 break;
 338         default:
 339                 pte |= HSW_WB_ELLC_LLC_AGE3;
 340                 break;
 341         }
 342
 343         return pte;
 344 }
 345
 346 static struct page *vm_alloc_page(struct i915_address_space *vm, gfp_t gfp)
 347 {
 348         struct page *page;
 349
 350         if (I915_SELFTEST_ONLY(should_fail(&vm->fault_attr, 1)))
 351                 i915_gem_shrink_all(vm->i915);
 352
 353         if (vm->free_pages.nr)
 354                 return vm->free_pages.pages[--vm->free_pages.nr];
 355
 356         page = alloc_page(gfp);
 357         if (!page)
 358                 return NULL;
 359
 360         if (vm->pt_kmap_wc)
 361                 set_pages_array_wc(&page, 1);
 362
 363         return page;
 364 }
 365
 366 static void vm_free_pages_release(struct i915_address_space *vm)
 367 {
 368         GEM_BUG_ON(!pagevec_count(&vm->free_pages));
 369
 370         if (vm->pt_kmap_wc)
 371                 set_pages_array_wb(vm->free_pages.pages,
 372                                    pagevec_count(&vm->free_pages));
 373
 374         __pagevec_release(&vm->free_pages);
 375 }
 376
 377 static void vm_free_page(struct i915_address_space *vm, struct page *page)
 378 {
 379         if (!pagevec_add(&vm->free_pages, page))
 380                 vm_free_pages_release(vm);
 381 }
 382
 383 static int __setup_page_dma(struct i915_address_space *vm,
 384                             struct i915_page_dma *p,
 385                             gfp_t gfp)
 386 {
 387         p->page = vm_alloc_page(vm, gfp | __GFP_NOWARN | __GFP_NORETRY);
 388         if (unlikely(!p->page))
 389                 return -ENOMEM;
 390
 391         p->daddr = dma_map_page(vm->dma, p->page, 0, PAGE_SIZE,
 392                                 PCI_DMA_BIDIRECTIONAL);
 393         if (unlikely(dma_mapping_error(vm->dma, p->daddr))) {
 394                 vm_free_page(vm, p->page);
 395                 return -ENOMEM;
 396         }
 397
 398         return 0;
 399 }
 400
 401 static int setup_page_dma(struct i915_address_space *vm,
 402                           struct i915_page_dma *p)
 403 {
 404         return __setup_page_dma(vm, p, I915_GFP_DMA);
 405 }
 406
 407 static void cleanup_page_dma(struct i915_address_space *vm,
 408                              struct i915_page_dma *p)
 409 {
 410         dma_unmap_page(vm->dma, p->daddr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 411         vm_free_page(vm, p->page);
 412 }
 413
 414 static void *kmap_page_dma(struct i915_page_dma *p)
 415 {
 416         return kmap_atomic(p->page);
 417 }
 418
 419 /* We use the flushing unmap only with ppgtt structures:
 420  * page directories, page tables and scratch pages.
 421  */
 422 static void kunmap_page_dma(void *vaddr)
 423 {
 424         kunmap_atomic(vaddr);
 425 }
 426
 427 #define kmap_px(px) kmap_page_dma(px_base(px))
 428 #define kunmap_px(vaddr) kunmap_page_dma((vaddr))
 429
 430 #define setup_px(vm, px) setup_page_dma((vm), px_base(px))
 431 #define cleanup_px(vm, px) cleanup_page_dma((vm), px_base(px))
 432 #define fill_px(ppgtt, px, v) fill_page_dma((vm), px_base(px), (v))
 433 #define fill32_px(ppgtt, px, v) fill_page_dma_32((vm), px_base(px), (v))
 434
 435 static void fill_page_dma(struct i915_address_space *vm,
 436                           struct i915_page_dma *p,
 437                           const u64 val)
 438 {
 439         u64 * const vaddr = kmap_page_dma(p);
 440         int i;
 441
 442         for (i = 0; i < 512; i++)
 443                 vaddr[i] = val;
 444
 445         kunmap_page_dma(vaddr);
 446 }
 447
 448 static void fill_page_dma_32(struct i915_address_space *vm,
 449                              struct i915_page_dma *p,
 450                              const u32 v)
 451 {
 452         fill_page_dma(vm, p, (u64)v << 32 | v);
 453 }
 454
 455 static int
 456 setup_scratch_page(struct i915_address_space *vm, gfp_t gfp)
 457 {
 458         return __setup_page_dma(vm, &vm->scratch_page, gfp | __GFP_ZERO);
 459 }
 460
 461 static void cleanup_scratch_page(struct i915_address_space *vm)
 462 {
 463         cleanup_page_dma(vm, &vm->scratch_page);
 464 }
 465
 466 static struct i915_page_table *alloc_pt(struct i915_address_space *vm)
 467 {
 468         struct i915_page_table *pt;
 469         const size_t count = INTEL_GEN(vm->i915) >= 8 ? GEN8_PTES : GEN6_PTES;
 470         int ret = -ENOMEM;
 471
 472         pt = kzalloc(sizeof(*pt), GFP_KERNEL);
 473         if (!pt)
 474                 return ERR_PTR(-ENOMEM);
 475
 476         pt->used_ptes = kcalloc(BITS_TO_LONGS(count), sizeof(*pt->used_ptes),
 477                                 GFP_KERNEL);
 478
 479         if (!pt->used_ptes)
 480                 goto fail_bitmap;
 481
 482         ret = setup_px(vm, pt);
 483         if (ret)
 484                 goto fail_page_m;
 485
 486         return pt;
 487
 488 fail_page_m:
 489         kfree(pt->used_ptes);
 490 fail_bitmap:
 491         kfree(pt);
 492
 493         return ERR_PTR(ret);
 494 }
 495
 496 static void free_pt(struct i915_address_space *vm, struct i915_page_table *pt)
 497 {
 498         cleanup_px(vm, pt);
 499         kfree(pt->used_ptes);
 500         kfree(pt);
 501 }
 502
 503 static void gen8_initialize_pt(struct i915_address_space *vm,
 504                                struct i915_page_table *pt)
 505 {
 506         gen8_pte_t scratch_pte;
 507
 508         scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
 509                                       I915_CACHE_LLC);
 510
 511         fill_px(vm, pt, scratch_pte);
 512 }
 513
 514 static void gen6_initialize_pt(struct i915_address_space *vm,
 515                                struct i915_page_table *pt)
 516 {
 517         gen6_pte_t scratch_pte;
 518
 519         WARN_ON(vm->scratch_page.daddr == 0);
 520
 521         scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
 522                                      I915_CACHE_LLC, 0);
 523
 524         fill32_px(vm, pt, scratch_pte);
 525 }
 526
 527 static struct i915_page_directory *alloc_pd(struct i915_address_space *vm)
 528 {
 529         struct i915_page_directory *pd;
 530         int ret = -ENOMEM;
 531
 532         pd = kzalloc(sizeof(*pd), GFP_KERNEL);
 533         if (!pd)
 534                 return ERR_PTR(-ENOMEM);
 535
 536         pd->used_pdes = kcalloc(BITS_TO_LONGS(I915_PDES),
 537                                 sizeof(*pd->used_pdes), GFP_KERNEL);
 538         if (!pd->used_pdes)
 539                 goto fail_bitmap;
 540
 541         ret = setup_px(vm, pd);
 542         if (ret)
 543                 goto fail_page_m;
 544
 545         return pd;
 546
 547 fail_page_m:
 548         kfree(pd->used_pdes);
 549 fail_bitmap:
 550         kfree(pd);
 551
 552         return ERR_PTR(ret);
 553 }
 554
 555 static void free_pd(struct i915_address_space *vm,
 556                     struct i915_page_directory *pd)
 557 {
 558         if (px_page(pd)) {
 559                 cleanup_px(vm, pd);
 560                 kfree(pd->used_pdes);
 561                 kfree(pd);
 562         }
 563 }
 564
 565 static void gen8_initialize_pd(struct i915_address_space *vm,
 566                                struct i915_page_directory *pd)
 567 {
 568         gen8_pde_t scratch_pde;
 569
 570         scratch_pde = gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC);
 571
 572         fill_px(vm, pd, scratch_pde);
 573 }
 574
 575 static int __pdp_init(struct drm_i915_private *dev_priv,
 576                       struct i915_page_directory_pointer *pdp)
 577 {
 578         size_t pdpes = I915_PDPES_PER_PDP(dev_priv);
 579
 580         pdp->used_pdpes = kcalloc(BITS_TO_LONGS(pdpes),
 581                                   sizeof(unsigned long),
 582                                   GFP_KERNEL);
 583         if (!pdp->used_pdpes)
 584                 return -ENOMEM;
 585
 586         pdp->page_directory = kcalloc(pdpes, sizeof(*pdp->page_directory),
 587                                       GFP_KERNEL);
 588         if (!pdp->page_directory) {
 589                 kfree(pdp->used_pdpes);
 590                 /* the PDP might be the statically allocated top level. Keep it
 591                  * as clean as possible */
 592                 pdp->used_pdpes = NULL;
 593                 return -ENOMEM;
 594         }
 595
 596         return 0;
 597 }
 598
 599 static void __pdp_fini(struct i915_page_directory_pointer *pdp)
 600 {
 601         kfree(pdp->used_pdpes);
 602         kfree(pdp->page_directory);
 603         pdp->page_directory = NULL;
 604 }
 605
 606 static struct i915_page_directory_pointer *
 607 alloc_pdp(struct i915_address_space *vm)
 608 {
 609         struct i915_page_directory_pointer *pdp;
 610         int ret = -ENOMEM;
 611
 612         WARN_ON(!USES_FULL_48BIT_PPGTT(vm->i915));
 613
 614         pdp = kzalloc(sizeof(*pdp), GFP_KERNEL);
 615         if (!pdp)
 616                 return ERR_PTR(-ENOMEM);
 617
 618         ret = __pdp_init(vm->i915, pdp);
 619         if (ret)
 620                 goto fail_bitmap;
 621
 622         ret = setup_px(vm, pdp);
 623         if (ret)
 624                 goto fail_page_m;
 625
 626         return pdp;
 627
 628 fail_page_m:
 629         __pdp_fini(pdp);
 630 fail_bitmap:
 631         kfree(pdp);
 632
 633         return ERR_PTR(ret);
 634 }
 635
 636 static void free_pdp(struct i915_address_space *vm,
 637                      struct i915_page_directory_pointer *pdp)
 638 {
 639         __pdp_fini(pdp);
 640         if (USES_FULL_48BIT_PPGTT(vm->i915)) {
 641                 cleanup_px(vm, pdp);
 642                 kfree(pdp);
 643         }
 644 }
 645
 646 static void gen8_initialize_pdp(struct i915_address_space *vm,
 647                                 struct i915_page_directory_pointer *pdp)
 648 {
 649         gen8_ppgtt_pdpe_t scratch_pdpe;
 650
 651         scratch_pdpe = gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC);
 652
 653         fill_px(vm, pdp, scratch_pdpe);
 654 }
 655
 656 static void gen8_initialize_pml4(struct i915_address_space *vm,
 657                                  struct i915_pml4 *pml4)
 658 {
 659         gen8_ppgtt_pml4e_t scratch_pml4e;
 660
 661         scratch_pml4e = gen8_pml4e_encode(px_dma(vm->scratch_pdp),
 662                                           I915_CACHE_LLC);
 663
 664         fill_px(vm, pml4, scratch_pml4e);
 665 }
 666
 667 static void
 668 gen8_setup_pdpe(struct i915_hw_ppgtt *ppgtt,
 669                 struct i915_page_directory_pointer *pdp,
 670                 struct i915_page_directory *pd,
 671                 int index)
 672 {
 673         gen8_ppgtt_pdpe_t *page_directorypo;
 674
 675         if (!USES_FULL_48BIT_PPGTT(to_i915(ppgtt->base.dev)))
 676                 return;
 677
 678         page_directorypo = kmap_px(pdp);
 679         page_directorypo[index] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC);
 680         kunmap_px(page_directorypo);
 681 }
 682
 683 static void
 684 gen8_setup_pml4e(struct i915_pml4 *pml4,
 685                  struct i915_page_directory_pointer *pdp,
 686                  int index)
 687 {
 688         gen8_ppgtt_pml4e_t *pagemap = kmap_px(pml4);
 689
 690         pagemap[index] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC);
 691         kunmap_px(pagemap);
 692 }
 693
 694 /* Broadwell Page Directory Pointer Descriptors */
 695 static int gen8_write_pdp(struct drm_i915_gem_request *req,
 696                           unsigned entry,
 697                           dma_addr_t addr)
 698 {
 699         struct intel_engine_cs *engine = req->engine;
 700         u32 *cs;
 701
 702         BUG_ON(entry >= 4);
 703
 704         cs = intel_ring_begin(req, 6);
 705         if (IS_ERR(cs))
 706                 return PTR_ERR(cs);
 707
 708         *cs++ = MI_LOAD_REGISTER_IMM(1);
 709         *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(engine, entry));
 710         *cs++ = upper_32_bits(addr);
 711         *cs++ = MI_LOAD_REGISTER_IMM(1);
 712         *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(engine, entry));
 713         *cs++ = lower_32_bits(addr);
 714         intel_ring_advance(req, cs);
 715
 716         return 0;
 717 }
 718
 719 static int gen8_legacy_mm_switch(struct i915_hw_ppgtt *ppgtt,
 720                                  struct drm_i915_gem_request *req)
 721 {
 722         int i, ret;
 723
 724         for (i = GEN8_LEGACY_PDPES - 1; i >= 0; i--) {
 725                 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
 726
 727                 ret = gen8_write_pdp(req, i, pd_daddr);
 728                 if (ret)
 729                         return ret;
 730         }
 731
 732         return 0;
 733 }
 734
 735 static int gen8_48b_mm_switch(struct i915_hw_ppgtt *ppgtt,
 736                               struct drm_i915_gem_request *req)
 737 {
 738         return gen8_write_pdp(req, 0, px_dma(&ppgtt->pml4));
 739 }
 740
 741 /* PDE TLBs are a pain to invalidate on GEN8+. When we modify
 742  * the page table structures, we mark them dirty so that
 743  * context switching/execlist queuing code takes extra steps
 744  * to ensure that tlbs are flushed.
 745  */
 746 static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt)
 747 {
 748         ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.i915)->ring_mask;
 749 }
 750
 751 /* Removes entries from a single page table, releasing it if it's empty.
 752  * Caller can use the return value to update higher-level entries.
 753  */
 754 static bool gen8_ppgtt_clear_pt(struct i915_address_space *vm,
 755                                 struct i915_page_table *pt,
 756                                 uint64_t start,
 757                                 uint64_t length)
 758 {
 759         unsigned int num_entries = gen8_pte_count(start, length);
 760         unsigned int pte = gen8_pte_index(start);
 761         unsigned int pte_end = pte + num_entries;
 762         const gen8_pte_t scratch_pte =
 763                 gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC);
 764         gen8_pte_t *vaddr;
 765
 766         if (WARN_ON(!px_page(pt)))
 767                 return false;
 768
 769         GEM_BUG_ON(pte_end > GEN8_PTES);
 770
 771         bitmap_clear(pt->used_ptes, pte, num_entries);
 772         if (USES_FULL_PPGTT(vm->i915)) {
 773                 if (bitmap_empty(pt->used_ptes, GEN8_PTES))
 774                         return true;
 775         }
 776
 777         vaddr = kmap_px(pt);
 778         while (pte < pte_end)
 779                 vaddr[pte++] = scratch_pte;
 780         kunmap_px(vaddr);
 781
 782         return false;
 783 }
 784
 785 /* Removes entries from a single page dir, releasing it if it's empty.
 786  * Caller can use the return value to update higher-level entries
 787  */
 788 static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm,
 789                                 struct i915_page_directory *pd,
 790                                 uint64_t start,
 791                                 uint64_t length)
 792 {
 793         struct i915_page_table *pt;
 794         uint64_t pde;
 795         gen8_pde_t *pde_vaddr;
 796         gen8_pde_t scratch_pde = gen8_pde_encode(px_dma(vm->scratch_pt),
 797                                                  I915_CACHE_LLC);
 798
 799         gen8_for_each_pde(pt, pd, start, length, pde) {
 800                 if (WARN_ON(!pd->page_table[pde]))
 801                         break;
 802
 803                 if (gen8_ppgtt_clear_pt(vm, pt, start, length)) {
 804                         __clear_bit(pde, pd->used_pdes);
 805                         pde_vaddr = kmap_px(pd);
 806                         pde_vaddr[pde] = scratch_pde;
 807                         kunmap_px(pde_vaddr);
 808                         free_pt(vm, pt);
 809                 }
 810         }
 811
 812         if (bitmap_empty(pd->used_pdes, I915_PDES))
 813                 return true;
 814
 815         return false;
 816 }
 817
 818 /* Removes entries from a single page dir pointer, releasing it if it's empty.
 819  * Caller can use the return value to update higher-level entries
 820  */
 821 static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
 822                                  struct i915_page_directory_pointer *pdp,
 823                                  uint64_t start,
 824                                  uint64_t length)
 825 {
 826         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 827         struct i915_page_directory *pd;
 828         uint64_t pdpe;
 829
 830         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
 831                 if (WARN_ON(!pdp->page_directory[pdpe]))
 832                         break;
 833
 834                 if (gen8_ppgtt_clear_pd(vm, pd, start, length)) {
 835                         __clear_bit(pdpe, pdp->used_pdpes);
 836                         gen8_setup_pdpe(ppgtt, pdp, vm->scratch_pd, pdpe);
 837                         free_pd(vm, pd);
 838                 }
 839         }
 840
 841         mark_tlbs_dirty(ppgtt);
 842
 843         if (bitmap_empty(pdp->used_pdpes, I915_PDPES_PER_PDP(dev_priv)))
 844                 return true;
 845
 846         return false;
 847 }
 848
 849 /* Removes entries from a single pml4.
 850  * This is the top-level structure in 4-level page tables used on gen8+.
 851  * Empty entries are always scratch pml4e.
 852  */
 853 static void gen8_ppgtt_clear_pml4(struct i915_address_space *vm,
 854                                   struct i915_pml4 *pml4,
 855                                   uint64_t start,
 856                                   uint64_t length)
 857 {
 858         struct i915_page_directory_pointer *pdp;
 859         uint64_t pml4e;
 860
 861         GEM_BUG_ON(!USES_FULL_48BIT_PPGTT(vm->i915));
 862
 863         gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
 864                 if (WARN_ON(!pml4->pdps[pml4e]))
 865                         break;
 866
 867                 if (gen8_ppgtt_clear_pdp(vm, pdp, start, length)) {
 868                         __clear_bit(pml4e, pml4->used_pml4es);
 869                         gen8_setup_pml4e(pml4, vm->scratch_pdp, pml4e);
 870                         free_pdp(vm, pdp);
 871                 }
 872         }
 873 }
 874
 875 static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
 876                                    uint64_t start, uint64_t length)
 877 {
 878         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 879
 880         if (USES_FULL_48BIT_PPGTT(vm->i915))
 881                 gen8_ppgtt_clear_pml4(vm, &ppgtt->pml4, start, length);
 882         else
 883                 gen8_ppgtt_clear_pdp(vm, &ppgtt->pdp, start, length);
 884 }
 885
 886 struct sgt_dma {
 887         struct scatterlist *sg;
 888         dma_addr_t dma, max;
 889 };
 890
 891 static __always_inline bool
 892 gen8_ppgtt_insert_pte_entries(struct i915_hw_ppgtt *ppgtt,
 893                               struct i915_page_directory_pointer *pdp,
 894                               struct sgt_dma *iter,
 895                               u64 start,
 896                               enum i915_cache_level cache_level)
 897 {
 898         unsigned int pdpe = gen8_pdpe_index(start);
 899         unsigned int pde = gen8_pde_index(start);
 900         unsigned int pte = gen8_pte_index(start);
 901         struct i915_page_directory *pd;
 902         const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level);
 903         gen8_pte_t *vaddr;
 904         bool ret;
 905
 906         pd = pdp->page_directory[pdpe];
 907         vaddr = kmap_px(pd->page_table[pde]);
 908         do {
 909                 vaddr[pte] = pte_encode | iter->dma;
 910                 iter->dma += PAGE_SIZE;
 911                 if (iter->dma >= iter->max) {
 912                         iter->sg = __sg_next(iter->sg);
 913                         if (!iter->sg) {
 914                                 ret = false;
 915                                 break;
 916                         }
 917
 918                         iter->dma = sg_dma_address(iter->sg);
 919                         iter->max = iter->dma + iter->sg->length;
 920                 }
 921
 922                 if (++pte == GEN8_PTES) {
 923                         if (++pde == I915_PDES) {
 924                                 /* Limited by sg length for 3lvl */
 925                                 if (++pdpe == GEN8_PML4ES_PER_PML4) {
 926                                         ret = true;
 927                                         break;
 928                                 }
 929
 930                                 GEM_BUG_ON(pdpe > GEN8_LEGACY_PDPES);
 931                                 pd = pdp->page_directory[pdpe];
 932                                 pde = 0;
 933                         }
 934
 935                         kunmap_px(vaddr);
 936                         vaddr = kmap_px(pd->page_table[pde]);
 937                         pte = 0;
 938                 }
 939         } while (1);
 940         kunmap_px(vaddr);
 941
 942         return ret;
 943 }
 944
 945 static void gen8_ppgtt_insert_3lvl(struct i915_address_space *vm,
 946                                    struct sg_table *pages,
 947                                    u64 start,
 948                                    enum i915_cache_level cache_level,
 949                                    u32 unused)
 950 {
 951         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 952         struct sgt_dma iter = {
 953                 .sg = pages->sgl,
 954                 .dma = sg_dma_address(iter.sg),
 955                 .max = iter.dma + iter.sg->length,
 956         };
 957
 958         gen8_ppgtt_insert_pte_entries(ppgtt, &ppgtt->pdp, &iter,
 959                                       start, cache_level);
 960 }
 961
 962 static void gen8_ppgtt_insert_4lvl(struct i915_address_space *vm,
 963                                    struct sg_table *pages,
 964                                    uint64_t start,
 965                                    enum i915_cache_level cache_level,
 966                                    u32 unused)
 967 {
 968         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 969         struct sgt_dma iter = {
 970                 .sg = pages->sgl,
 971                 .dma = sg_dma_address(iter.sg),
 972                 .max = iter.dma + iter.sg->length,
 973         };
 974         struct i915_page_directory_pointer **pdps = ppgtt->pml4.pdps;
 975         unsigned int pml4e = gen8_pml4e_index(start);
 976
 977         while (gen8_ppgtt_insert_pte_entries(ppgtt, pdps[pml4e++], &iter,
 978                                              start, cache_level))
 979                 ;
 980 }
 981
 982 static void gen8_free_page_tables(struct i915_address_space *vm,
 983                                   struct i915_page_directory *pd)
 984 {
 985         int i;
 986
 987         if (!px_page(pd))
 988                 return;
 989
 990         for_each_set_bit(i, pd->used_pdes, I915_PDES) {
 991                 if (WARN_ON(!pd->page_table[i]))
 992                         continue;
 993
 994                 free_pt(vm, pd->page_table[i]);
 995                 pd->page_table[i] = NULL;
 996         }
 997 }
 998
 999 static int gen8_init_scratch(struct i915_address_space *vm)
1000 {
1001         int ret;
1002
1003         ret = setup_scratch_page(vm, I915_GFP_DMA);
1004         if (ret)
1005                 return ret;
1006
1007         vm->scratch_pt = alloc_pt(vm);
1008         if (IS_ERR(vm->scratch_pt)) {
1009                 ret = PTR_ERR(vm->scratch_pt);
1010                 goto free_scratch_page;
1011         }
1012
1013         vm->scratch_pd = alloc_pd(vm);
1014         if (IS_ERR(vm->scratch_pd)) {
1015                 ret = PTR_ERR(vm->scratch_pd);
1016                 goto free_pt;
1017         }
1018
1019         if (USES_FULL_48BIT_PPGTT(dev)) {
1020                 vm->scratch_pdp = alloc_pdp(vm);
1021                 if (IS_ERR(vm->scratch_pdp)) {
1022                         ret = PTR_ERR(vm->scratch_pdp);
1023                         goto free_pd;
1024                 }
1025         }
1026
1027         gen8_initialize_pt(vm, vm->scratch_pt);
1028         gen8_initialize_pd(vm, vm->scratch_pd);
1029         if (USES_FULL_48BIT_PPGTT(dev_priv))
1030                 gen8_initialize_pdp(vm, vm->scratch_pdp);
1031
1032         return 0;
1033
1034 free_pd:
1035         free_pd(vm, vm->scratch_pd);
1036 free_pt:
1037         free_pt(vm, vm->scratch_pt);
1038 free_scratch_page:
1039         cleanup_scratch_page(vm);
1040
1041         return ret;
1042 }
1043
1044 static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create)
1045 {
1046         enum vgt_g2v_type msg;
1047         struct drm_i915_private *dev_priv = ppgtt->base.i915;
1048         int i;
1049
1050         if (USES_FULL_48BIT_PPGTT(dev_priv)) {
1051                 u64 daddr = px_dma(&ppgtt->pml4);
1052
1053                 I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr));
1054                 I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr));
1055
1056                 msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE :
1057                                 VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY);
1058         } else {
1059                 for (i = 0; i < GEN8_LEGACY_PDPES; i++) {
1060                         u64 daddr = i915_page_dir_dma_addr(ppgtt, i);
1061
1062                         I915_WRITE(vgtif_reg(pdp[i].lo), lower_32_bits(daddr));
1063                         I915_WRITE(vgtif_reg(pdp[i].hi), upper_32_bits(daddr));
1064                 }
1065
1066                 msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE :
1067                                 VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY);
1068         }
1069
1070         I915_WRITE(vgtif_reg(g2v_notify), msg);
1071
1072         return 0;
1073 }
1074
1075 static void gen8_free_scratch(struct i915_address_space *vm)
1076 {
1077         if (USES_FULL_48BIT_PPGTT(vm->i915))
1078                 free_pdp(vm, vm->scratch_pdp);
1079         free_pd(vm, vm->scratch_pd);
1080         free_pt(vm, vm->scratch_pt);
1081         cleanup_scratch_page(vm);
1082 }
1083
1084 static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm,
1085                                     struct i915_page_directory_pointer *pdp)
1086 {
1087         int i;
1088
1089         for_each_set_bit(i, pdp->used_pdpes, I915_PDPES_PER_PDP(vm->i915)) {
1090                 if (WARN_ON(!pdp->page_directory[i]))
1091                         continue;
1092
1093                 gen8_free_page_tables(vm, pdp->page_directory[i]);
1094                 free_pd(vm, pdp->page_directory[i]);
1095         }
1096
1097         free_pdp(vm, pdp);
1098 }
1099
1100 static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt)
1101 {
1102         int i;
1103
1104         for_each_set_bit(i, ppgtt->pml4.used_pml4es, GEN8_PML4ES_PER_PML4) {
1105                 if (WARN_ON(!ppgtt->pml4.pdps[i]))
1106                         continue;
1107
1108                 gen8_ppgtt_cleanup_3lvl(&ppgtt->base, ppgtt->pml4.pdps[i]);
1109         }
1110
1111         cleanup_px(&ppgtt->base, &ppgtt->pml4);
1112 }
1113
1114 static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
1115 {
1116         struct drm_i915_private *dev_priv = vm->i915;
1117         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1118
1119         if (intel_vgpu_active(dev_priv))
1120                 gen8_ppgtt_notify_vgt(ppgtt, false);
1121
1122         if (!USES_FULL_48BIT_PPGTT(vm->i915))
1123                 gen8_ppgtt_cleanup_3lvl(&ppgtt->base, &ppgtt->pdp);
1124         else
1125                 gen8_ppgtt_cleanup_4lvl(ppgtt);
1126
1127         gen8_free_scratch(vm);
1128 }
1129
1130 /**
1131  * gen8_ppgtt_alloc_pagetabs() - Allocate page tables for VA range.
1132  * @vm: Master vm structure.
1133  * @pd: Page directory for this address range.
1134  * @start:      Starting virtual address to begin allocations.
1135  * @length:     Size of the allocations.
1136  * @new_pts:    Bitmap set by function with new allocations. Likely used by the
1137  *              caller to free on error.
1138  *
1139  * Allocate the required number of page tables. Extremely similar to
1140  * gen8_ppgtt_alloc_page_directories(). The main difference is here we are limited by
1141  * the page directory boundary (instead of the page directory pointer). That
1142  * boundary is 1GB virtual. Therefore, unlike gen8_ppgtt_alloc_page_directories(), it is
1143  * possible, and likely that the caller will need to use multiple calls of this
1144  * function to achieve the appropriate allocation.
1145  *
1146  * Return: 0 if success; negative error code otherwise.
1147  */
1148 static int gen8_ppgtt_alloc_pagetabs(struct i915_address_space *vm,
1149                                      struct i915_page_directory *pd,
1150                                      uint64_t start,
1151                                      uint64_t length,
1152                                      unsigned long *new_pts)
1153 {
1154         struct i915_page_table *pt;
1155         uint32_t pde;
1156
1157         gen8_for_each_pde(pt, pd, start, length, pde) {
1158                 /* Don't reallocate page tables */
1159                 if (test_bit(pde, pd->used_pdes)) {
1160                         /* Scratch is never allocated this way */
1161                         WARN_ON(pt == vm->scratch_pt);
1162                         continue;
1163                 }
1164
1165                 pt = alloc_pt(vm);
1166                 if (IS_ERR(pt))
1167                         goto unwind_out;
1168
1169                 gen8_initialize_pt(vm, pt);
1170                 pd->page_table[pde] = pt;
1171                 __set_bit(pde, new_pts);
1172                 trace_i915_page_table_entry_alloc(vm, pde, start, GEN8_PDE_SHIFT);
1173         }
1174
1175         return 0;
1176
1177 unwind_out:
1178         for_each_set_bit(pde, new_pts, I915_PDES)
1179                 free_pt(vm, pd->page_table[pde]);
1180
1181         return -ENOMEM;
1182 }
1183
1184 /**
1185  * gen8_ppgtt_alloc_page_directories() - Allocate page directories for VA range.
1186  * @vm: Master vm structure.
1187  * @pdp:        Page directory pointer for this address range.
1188  * @start:      Starting virtual address to begin allocations.
1189  * @length:     Size of the allocations.
1190  * @new_pds:    Bitmap set by function with new allocations. Likely used by the
1191  *              caller to free on error.
1192  *
1193  * Allocate the required number of page directories starting at the pde index of
1194  * @start, and ending at the pde index @start + @length. This function will skip
1195  * over already allocated page directories within the range, and only allocate
1196  * new ones, setting the appropriate pointer within the pdp as well as the
1197  * correct position in the bitmap @new_pds.
1198  *
1199  * The function will only allocate the pages within the range for a give page
1200  * directory pointer. In other words, if @start + @length straddles a virtually
1201  * addressed PDP boundary (512GB for 4k pages), there will be more allocations
1202  * required by the caller, This is not currently possible, and the BUG in the
1203  * code will prevent it.
1204  *
1205  * Return: 0 if success; negative error code otherwise.
1206  */
1207 static int
1208 gen8_ppgtt_alloc_page_directories(struct i915_address_space *vm,
1209                                   struct i915_page_directory_pointer *pdp,
1210                                   uint64_t start,
1211                                   uint64_t length,
1212                                   unsigned long *new_pds)
1213 {
1214         struct i915_page_directory *pd;
1215         uint32_t pdpe;
1216         uint32_t pdpes = I915_PDPES_PER_PDP(dev_priv);
1217
1218         WARN_ON(!bitmap_empty(new_pds, pdpes));
1219
1220         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1221                 if (test_bit(pdpe, pdp->used_pdpes))
1222                         continue;
1223
1224                 pd = alloc_pd(vm);
1225                 if (IS_ERR(pd))
1226                         goto unwind_out;
1227
1228                 gen8_initialize_pd(vm, pd);
1229                 pdp->page_directory[pdpe] = pd;
1230                 __set_bit(pdpe, new_pds);
1231                 trace_i915_page_directory_entry_alloc(vm, pdpe, start, GEN8_PDPE_SHIFT);
1232         }
1233
1234         return 0;
1235
1236 unwind_out:
1237         for_each_set_bit(pdpe, new_pds, pdpes)
1238                 free_pd(vm, pdp->page_directory[pdpe]);
1239
1240         return -ENOMEM;
1241 }
1242
1243 /**
1244  * gen8_ppgtt_alloc_page_dirpointers() - Allocate pdps for VA range.
1245  * @vm: Master vm structure.
1246  * @pml4:       Page map level 4 for this address range.
1247  * @start:      Starting virtual address to begin allocations.
1248  * @length:     Size of the allocations.
1249  * @new_pdps:   Bitmap set by function with new allocations. Likely used by the
1250  *              caller to free on error.
1251  *
1252  * Allocate the required number of page directory pointers. Extremely similar to
1253  * gen8_ppgtt_alloc_page_directories() and gen8_ppgtt_alloc_pagetabs().
1254  * The main difference is here we are limited by the pml4 boundary (instead of
1255  * the page directory pointer).
1256  *
1257  * Return: 0 if success; negative error code otherwise.
1258  */
1259 static int
1260 gen8_ppgtt_alloc_page_dirpointers(struct i915_address_space *vm,
1261                                   struct i915_pml4 *pml4,
1262                                   uint64_t start,
1263                                   uint64_t length,
1264                                   unsigned long *new_pdps)
1265 {
1266         struct i915_page_directory_pointer *pdp;
1267         uint32_t pml4e;
1268
1269         WARN_ON(!bitmap_empty(new_pdps, GEN8_PML4ES_PER_PML4));
1270
1271         gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
1272                 if (!test_bit(pml4e, pml4->used_pml4es)) {
1273                         pdp = alloc_pdp(vm);
1274                         if (IS_ERR(pdp))
1275                                 goto unwind_out;
1276
1277                         gen8_initialize_pdp(vm, pdp);
1278                         pml4->pdps[pml4e] = pdp;
1279                         __set_bit(pml4e, new_pdps);
1280                         trace_i915_page_directory_pointer_entry_alloc(vm,
1281                                                                       pml4e,
1282                                                                       start,
1283                                                                       GEN8_PML4E_SHIFT);
1284                 }
1285         }
1286
1287         return 0;
1288
1289 unwind_out:
1290         for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4)
1291                 free_pdp(vm, pml4->pdps[pml4e]);
1292
1293         return -ENOMEM;
1294 }
1295
1296 static void
1297 free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long *new_pts)
1298 {
1299         kfree(new_pts);
1300         kfree(new_pds);
1301 }
1302
1303 /* Fills in the page directory bitmap, and the array of page tables bitmap. Both
1304  * of these are based on the number of PDPEs in the system.
1305  */
1306 static
1307 int __must_check alloc_gen8_temp_bitmaps(unsigned long **new_pds,
1308                                          unsigned long **new_pts,
1309                                          uint32_t pdpes)
1310 {
1311         unsigned long *pds;
1312         unsigned long *pts;
1313
1314         pds = kcalloc(BITS_TO_LONGS(pdpes), sizeof(unsigned long), GFP_TEMPORARY);
1315         if (!pds)
1316                 return -ENOMEM;
1317
1318         pts = kcalloc(pdpes, BITS_TO_LONGS(I915_PDES) * sizeof(unsigned long),
1319                       GFP_TEMPORARY);
1320         if (!pts)
1321                 goto err_out;
1322
1323         *new_pds = pds;
1324         *new_pts = pts;
1325
1326         return 0;
1327
1328 err_out:
1329         free_gen8_temp_bitmaps(pds, pts);
1330         return -ENOMEM;
1331 }
1332
1333 static int gen8_alloc_va_range_3lvl(struct i915_address_space *vm,
1334                                     struct i915_page_directory_pointer *pdp,
1335                                     uint64_t start,
1336                                     uint64_t length)
1337 {
1338         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1339         unsigned long *new_page_dirs, *new_page_tables;
1340         struct i915_page_directory *pd;
1341         const uint64_t orig_start = start;
1342         const uint64_t orig_length = length;
1343         uint32_t pdpe;
1344         uint32_t pdpes = I915_PDPES_PER_PDP(dev_priv);
1345         int ret;
1346
1347         ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes);
1348         if (ret)
1349                 return ret;
1350
1351         /* Do the allocations first so we can easily bail out */
1352         ret = gen8_ppgtt_alloc_page_directories(vm, pdp, start, length,
1353                                                 new_page_dirs);
1354         if (ret) {
1355                 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
1356                 return ret;
1357         }
1358
1359         /* For every page directory referenced, allocate page tables */
1360         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1361                 ret = gen8_ppgtt_alloc_pagetabs(vm, pd, start, length,
1362                                                 new_page_tables + pdpe * BITS_TO_LONGS(I915_PDES));
1363                 if (ret)
1364                         goto err_out;
1365         }
1366
1367         start = orig_start;
1368         length = orig_length;
1369
1370         /* Allocations have completed successfully, so set the bitmaps, and do
1371          * the mappings. */
1372         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1373                 gen8_pde_t *const page_directory = kmap_px(pd);
1374                 struct i915_page_table *pt;
1375                 uint64_t pd_len = length;
1376                 uint64_t pd_start = start;
1377                 uint32_t pde;
1378
1379                 /* Every pd should be allocated, we just did that above. */
1380                 WARN_ON(!pd);
1381
1382                 gen8_for_each_pde(pt, pd, pd_start, pd_len, pde) {
1383                         /* Same reasoning as pd */
1384                         WARN_ON(!pt);
1385                         WARN_ON(!pd_len);
1386                         WARN_ON(!gen8_pte_count(pd_start, pd_len));
1387
1388                         /* Set our used ptes within the page table */
1389                         bitmap_set(pt->used_ptes,
1390                                    gen8_pte_index(pd_start),
1391                                    gen8_pte_count(pd_start, pd_len));
1392
1393                         /* Our pde is now pointing to the pagetable, pt */
1394                         __set_bit(pde, pd->used_pdes);
1395
1396                         /* Map the PDE to the page table */
1397                         page_directory[pde] = gen8_pde_encode(px_dma(pt),
1398                                                               I915_CACHE_LLC);
1399                         trace_i915_page_table_entry_map(&ppgtt->base, pde, pt,
1400                                                         gen8_pte_index(start),
1401                                                         gen8_pte_count(start, length),
1402                                                         GEN8_PTES);
1403
1404                         /* NB: We haven't yet mapped ptes to pages. At this
1405                          * point we're still relying on insert_entries() */
1406                 }
1407
1408                 kunmap_px(page_directory);
1409                 __set_bit(pdpe, pdp->used_pdpes);
1410                 gen8_setup_pdpe(ppgtt, pdp, pd, pdpe);
1411         }
1412
1413         free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
1414         mark_tlbs_dirty(ppgtt);
1415         return 0;
1416
1417 err_out:
1418         while (pdpe--) {
1419                 unsigned long temp;
1420
1421                 for_each_set_bit(temp, new_page_tables + pdpe *
1422                                 BITS_TO_LONGS(I915_PDES), I915_PDES)
1423                         free_pt(vm, pdp->page_directory[pdpe]->page_table[temp]);
1424         }
1425
1426         for_each_set_bit(pdpe, new_page_dirs, pdpes)
1427                 free_pd(vm, pdp->page_directory[pdpe]);
1428
1429         free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
1430         mark_tlbs_dirty(ppgtt);
1431         return ret;
1432 }
1433
1434 static int gen8_alloc_va_range_4lvl(struct i915_address_space *vm,
1435                                     struct i915_pml4 *pml4,
1436                                     uint64_t start,
1437                                     uint64_t length)
1438 {
1439         DECLARE_BITMAP(new_pdps, GEN8_PML4ES_PER_PML4);
1440         struct i915_page_directory_pointer *pdp;
1441         uint64_t pml4e;
1442         int ret = 0;
1443
1444         /* Do the pml4 allocations first, so we don't need to track the newly
1445          * allocated tables below the pdp */
1446         bitmap_zero(new_pdps, GEN8_PML4ES_PER_PML4);
1447
1448         /* The pagedirectory and pagetable allocations are done in the shared 3
1449          * and 4 level code. Just allocate the pdps.
1450          */
1451         ret = gen8_ppgtt_alloc_page_dirpointers(vm, pml4, start, length,
1452                                                 new_pdps);
1453         if (ret)
1454                 return ret;
1455
1456         gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
1457                 WARN_ON(!pdp);
1458
1459                 ret = gen8_alloc_va_range_3lvl(vm, pdp, start, length);
1460                 if (ret)
1461                         goto err_out;
1462
1463                 gen8_setup_pml4e(pml4, pdp, pml4e);
1464         }
1465
1466         bitmap_or(pml4->used_pml4es, new_pdps, pml4->used_pml4es,
1467                   GEN8_PML4ES_PER_PML4);
1468
1469         return 0;
1470
1471 err_out:
1472         for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4)
1473                 gen8_ppgtt_cleanup_3lvl(vm, pml4->pdps[pml4e]);
1474
1475         return ret;
1476 }
1477
1478 static int gen8_alloc_va_range(struct i915_address_space *vm,
1479                                uint64_t start, uint64_t length)
1480 {
1481         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1482
1483         if (USES_FULL_48BIT_PPGTT(vm->i915))
1484                 return gen8_alloc_va_range_4lvl(vm, &ppgtt->pml4, start, length);
1485         else
1486                 return gen8_alloc_va_range_3lvl(vm, &ppgtt->pdp, start, length);
1487 }
1488
1489 static void gen8_dump_pdp(struct i915_hw_ppgtt *ppgtt,
1490                           struct i915_page_directory_pointer *pdp,
1491                           uint64_t start, uint64_t length,
1492                           gen8_pte_t scratch_pte,
1493                           struct seq_file *m)
1494 {
1495         struct i915_page_directory *pd;
1496         uint32_t pdpe;
1497
1498         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1499                 struct i915_page_table *pt;
1500                 uint64_t pd_len = length;
1501                 uint64_t pd_start = start;
1502                 uint32_t pde;
1503
1504                 if (!test_bit(pdpe, pdp->used_pdpes))
1505                         continue;
1506
1507                 seq_printf(m, "\tPDPE #%d\n", pdpe);
1508                 gen8_for_each_pde(pt, pd, pd_start, pd_len, pde) {
1509                         uint32_t  pte;
1510                         gen8_pte_t *pt_vaddr;
1511
1512                         if (!test_bit(pde, pd->used_pdes))
1513                                 continue;
1514
1515                         pt_vaddr = kmap_px(pt);
1516                         for (pte = 0; pte < GEN8_PTES; pte += 4) {
1517                                 uint64_t va =
1518                                         (pdpe << GEN8_PDPE_SHIFT) |
1519                                         (pde << GEN8_PDE_SHIFT) |
1520                                         (pte << GEN8_PTE_SHIFT);
1521                                 int i;
1522                                 bool found = false;
1523
1524                                 for (i = 0; i < 4; i++)
1525                                         if (pt_vaddr[pte + i] != scratch_pte)
1526                                                 found = true;
1527                                 if (!found)
1528                                         continue;
1529
1530                                 seq_printf(m, "\t\t0x%llx [%03d,%03d,%04d]: =", va, pdpe, pde, pte);
1531                                 for (i = 0; i < 4; i++) {
1532                                         if (pt_vaddr[pte + i] != scratch_pte)
1533                                                 seq_printf(m, " %llx", pt_vaddr[pte + i]);
1534                                         else
1535                                                 seq_puts(m, "  SCRATCH ");
1536                                 }
1537                                 seq_puts(m, "\n");
1538                         }
1539                         /* don't use kunmap_px, it could trigger
1540                          * an unnecessary flush.
1541                          */
1542                         kunmap_atomic(pt_vaddr);
1543                 }
1544         }
1545 }
1546
1547 static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
1548 {
1549         struct i915_address_space *vm = &ppgtt->base;
1550         uint64_t start = ppgtt->base.start;
1551         uint64_t length = ppgtt->base.total;
1552         const gen8_pte_t scratch_pte =
1553                 gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC);
1554
1555         if (!USES_FULL_48BIT_PPGTT(vm->i915)) {
1556                 gen8_dump_pdp(ppgtt, &ppgtt->pdp, start, length, scratch_pte, m);
1557         } else {
1558                 uint64_t pml4e;
1559                 struct i915_pml4 *pml4 = &ppgtt->pml4;
1560                 struct i915_page_directory_pointer *pdp;
1561
1562                 gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
1563                         if (!test_bit(pml4e, pml4->used_pml4es))
1564                                 continue;
1565
1566                         seq_printf(m, "    PML4E #%llu\n", pml4e);
1567                         gen8_dump_pdp(ppgtt, pdp, start, length, scratch_pte, m);
1568                 }
1569         }
1570 }
1571
1572 static int gen8_preallocate_top_level_pdps(struct i915_hw_ppgtt *ppgtt)
1573 {
1574         unsigned long *new_page_dirs, *new_page_tables;
1575         uint32_t pdpes = I915_PDPES_PER_PDP(to_i915(ppgtt->base.dev));
1576         int ret;
1577
1578         /* We allocate temp bitmap for page tables for no gain
1579          * but as this is for init only, lets keep the things simple
1580          */
1581         ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes);
1582         if (ret)
1583                 return ret;
1584
1585         /* Allocate for all pdps regardless of how the ppgtt
1586          * was defined.
1587          */
1588         ret = gen8_ppgtt_alloc_page_directories(&ppgtt->base, &ppgtt->pdp,
1589                                                 0, 1ULL << 32,
1590                                                 new_page_dirs);
1591         if (!ret)
1592                 *ppgtt->pdp.used_pdpes = *new_page_dirs;
1593
1594         free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
1595
1596         return ret;
1597 }
1598
1599 /*
1600  * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
1601  * with a net effect resembling a 2-level page table in normal x86 terms. Each
1602  * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address
1603  * space.
1604  *
1605  */
1606 static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
1607 {
1608         struct drm_i915_private *dev_priv = ppgtt->base.i915;
1609         int ret;
1610
1611         ret = gen8_init_scratch(&ppgtt->base);
1612         if (ret)
1613                 return ret;
1614
1615         ppgtt->base.start = 0;
1616         ppgtt->base.cleanup = gen8_ppgtt_cleanup;
1617         ppgtt->base.allocate_va_range = gen8_alloc_va_range;
1618         ppgtt->base.clear_range = gen8_ppgtt_clear_range;
1619         ppgtt->base.unbind_vma = ppgtt_unbind_vma;
1620         ppgtt->base.bind_vma = ppgtt_bind_vma;
1621         ppgtt->debug_dump = gen8_dump_ppgtt;
1622
1623         /* There are only few exceptions for gen >=6. chv and bxt.
1624          * And we are not sure about the latter so play safe for now.
1625          */
1626         if (IS_CHERRYVIEW(dev_priv) || IS_BROXTON(dev_priv))
1627                 ppgtt->base.pt_kmap_wc = true;
1628
1629         if (USES_FULL_48BIT_PPGTT(dev_priv)) {
1630                 ret = setup_px(&ppgtt->base, &ppgtt->pml4);
1631                 if (ret)
1632                         goto free_scratch;
1633
1634                 gen8_initialize_pml4(&ppgtt->base, &ppgtt->pml4);
1635
1636                 ppgtt->base.total = 1ULL << 48;
1637                 ppgtt->switch_mm = gen8_48b_mm_switch;
1638
1639                 ppgtt->base.insert_entries = gen8_ppgtt_insert_4lvl;
1640         } else {
1641                 ret = __pdp_init(dev_priv, &ppgtt->pdp);
1642                 if (ret)
1643                         goto free_scratch;
1644
1645                 ppgtt->base.total = 1ULL << 32;
1646                 ppgtt->switch_mm = gen8_legacy_mm_switch;
1647                 trace_i915_page_directory_pointer_entry_alloc(&ppgtt->base,
1648                                                               0, 0,
1649                                                               GEN8_PML4E_SHIFT);
1650
1651                 if (intel_vgpu_active(dev_priv)) {
1652                         ret = gen8_preallocate_top_level_pdps(ppgtt);
1653                         if (ret)
1654                                 goto free_scratch;
1655                 }
1656
1657                 ppgtt->base.insert_entries = gen8_ppgtt_insert_3lvl;
1658         }
1659
1660         if (intel_vgpu_active(dev_priv))
1661                 gen8_ppgtt_notify_vgt(ppgtt, true);
1662
1663         return 0;
1664
1665 free_scratch:
1666         gen8_free_scratch(&ppgtt->base);
1667         return ret;
1668 }
1669
1670 static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
1671 {
1672         struct i915_address_space *vm = &ppgtt->base;
1673         struct i915_page_table *unused;
1674         gen6_pte_t scratch_pte;
1675         uint32_t pd_entry;
1676         uint32_t  pte, pde;
1677         uint32_t start = ppgtt->base.start, length = ppgtt->base.total;
1678
1679         scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
1680                                      I915_CACHE_LLC, 0);
1681
1682         gen6_for_each_pde(unused, &ppgtt->pd, start, length, pde) {
1683                 u32 expected;
1684                 gen6_pte_t *pt_vaddr;
1685                 const dma_addr_t pt_addr = px_dma(ppgtt->pd.page_table[pde]);
1686                 pd_entry = readl(ppgtt->pd_addr + pde);
1687                 expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID);
1688
1689                 if (pd_entry != expected)
1690                         seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n",
1691                                    pde,
1692                                    pd_entry,
1693                                    expected);
1694                 seq_printf(m, "\tPDE: %x\n", pd_entry);
1695
1696                 pt_vaddr = kmap_px(ppgtt->pd.page_table[pde]);
1697
1698                 for (pte = 0; pte < GEN6_PTES; pte+=4) {
1699                         unsigned long va =
1700                                 (pde * PAGE_SIZE * GEN6_PTES) +
1701                                 (pte * PAGE_SIZE);
1702                         int i;
1703                         bool found = false;
1704                         for (i = 0; i < 4; i++)
1705                                 if (pt_vaddr[pte + i] != scratch_pte)
1706                                         found = true;
1707                         if (!found)
1708                                 continue;
1709
1710                         seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte);
1711                         for (i = 0; i < 4; i++) {
1712                                 if (pt_vaddr[pte + i] != scratch_pte)
1713                                         seq_printf(m, " %08x", pt_vaddr[pte + i]);
1714                                 else
1715                                         seq_puts(m, "  SCRATCH ");
1716                         }
1717                         seq_puts(m, "\n");
1718                 }
1719                 kunmap_px(pt_vaddr);
1720         }
1721 }
1722
1723 /* Write pde (index) from the page directory @pd to the page table @pt */
1724 static void gen6_write_pde(struct i915_page_directory *pd,
1725                             const int pde, struct i915_page_table *pt)
1726 {
1727         /* Caller needs to make sure the write completes if necessary */
1728         struct i915_hw_ppgtt *ppgtt =
1729                 container_of(pd, struct i915_hw_ppgtt, pd);
1730         u32 pd_entry;
1731
1732         pd_entry = GEN6_PDE_ADDR_ENCODE(px_dma(pt));
1733         pd_entry |= GEN6_PDE_VALID;
1734
1735         writel(pd_entry, ppgtt->pd_addr + pde);
1736 }
1737
1738 /* Write all the page tables found in the ppgtt structure to incrementing page
1739  * directories. */
1740 static void gen6_write_page_range(struct drm_i915_private *dev_priv,
1741                                   struct i915_page_directory *pd,
1742                                   uint32_t start, uint32_t length)
1743 {
1744         struct i915_ggtt *ggtt = &dev_priv->ggtt;
1745         struct i915_page_table *pt;
1746         uint32_t pde;
1747
1748         gen6_for_each_pde(pt, pd, start, length, pde)
1749                 gen6_write_pde(pd, pde, pt);
1750
1751         /* Make sure write is complete before other code can use this page
1752          * table. Also require for WC mapped PTEs */
1753         readl(ggtt->gsm);
1754 }
1755
1756 static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt)
1757 {
1758         BUG_ON(ppgtt->pd.base.ggtt_offset & 0x3f);
1759
1760         return (ppgtt->pd.base.ggtt_offset / 64) << 16;
1761 }
1762
1763 static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
1764                          struct drm_i915_gem_request *req)
1765 {
1766         struct intel_engine_cs *engine = req->engine;
1767         u32 *cs;
1768         int ret;
1769
1770         /* NB: TLBs must be flushed and invalidated before a switch */
1771         ret = engine->emit_flush(req, EMIT_INVALIDATE | EMIT_FLUSH);
1772         if (ret)
1773                 return ret;
1774
1775         cs = intel_ring_begin(req, 6);
1776         if (IS_ERR(cs))
1777                 return PTR_ERR(cs);
1778
1779         *cs++ = MI_LOAD_REGISTER_IMM(2);
1780         *cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine));
1781         *cs++ = PP_DIR_DCLV_2G;
1782         *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine));
1783         *cs++ = get_pd_offset(ppgtt);
1784         *cs++ = MI_NOOP;
1785         intel_ring_advance(req, cs);
1786
1787         return 0;
1788 }
1789
1790 static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
1791                           struct drm_i915_gem_request *req)
1792 {
1793         struct intel_engine_cs *engine = req->engine;
1794         u32 *cs;
1795         int ret;
1796
1797         /* NB: TLBs must be flushed and invalidated before a switch */
1798         ret = engine->emit_flush(req, EMIT_INVALIDATE | EMIT_FLUSH);
1799         if (ret)
1800                 return ret;
1801
1802         cs = intel_ring_begin(req, 6);
1803         if (IS_ERR(cs))
1804                 return PTR_ERR(cs);
1805
1806         *cs++ = MI_LOAD_REGISTER_IMM(2);
1807         *cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine));
1808         *cs++ = PP_DIR_DCLV_2G;
1809         *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine));
1810         *cs++ = get_pd_offset(ppgtt);
1811         *cs++ = MI_NOOP;
1812         intel_ring_advance(req, cs);
1813
1814         /* XXX: RCS is the only one to auto invalidate the TLBs? */
1815         if (engine->id != RCS) {
1816                 ret = engine->emit_flush(req, EMIT_INVALIDATE | EMIT_FLUSH);
1817                 if (ret)
1818                         return ret;
1819         }
1820
1821         return 0;
1822 }
1823
1824 static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
1825                           struct drm_i915_gem_request *req)
1826 {
1827         struct intel_engine_cs *engine = req->engine;
1828         struct drm_i915_private *dev_priv = req->i915;
1829
1830         I915_WRITE(RING_PP_DIR_DCLV(engine), PP_DIR_DCLV_2G);
1831         I915_WRITE(RING_PP_DIR_BASE(engine), get_pd_offset(ppgtt));
1832         return 0;
1833 }
1834
1835 static void gen8_ppgtt_enable(struct drm_i915_private *dev_priv)
1836 {
1837         struct intel_engine_cs *engine;
1838         enum intel_engine_id id;
1839
1840         for_each_engine(engine, dev_priv, id) {
1841                 u32 four_level = USES_FULL_48BIT_PPGTT(dev_priv) ?
1842                                  GEN8_GFX_PPGTT_48B : 0;
1843                 I915_WRITE(RING_MODE_GEN7(engine),
1844                            _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE | four_level));
1845         }
1846 }
1847
1848 static void gen7_ppgtt_enable(struct drm_i915_private *dev_priv)
1849 {
1850         struct intel_engine_cs *engine;
1851         uint32_t ecochk, ecobits;
1852         enum intel_engine_id id;
1853
1854         ecobits = I915_READ(GAC_ECO_BITS);
1855         I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
1856
1857         ecochk = I915_READ(GAM_ECOCHK);
1858         if (IS_HASWELL(dev_priv)) {
1859                 ecochk |= ECOCHK_PPGTT_WB_HSW;
1860         } else {
1861                 ecochk |= ECOCHK_PPGTT_LLC_IVB;
1862                 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
1863         }
1864         I915_WRITE(GAM_ECOCHK, ecochk);
1865
1866         for_each_engine(engine, dev_priv, id) {
1867                 /* GFX_MODE is per-ring on gen7+ */
1868                 I915_WRITE(RING_MODE_GEN7(engine),
1869                            _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
1870         }
1871 }
1872
1873 static void gen6_ppgtt_enable(struct drm_i915_private *dev_priv)
1874 {
1875         uint32_t ecochk, gab_ctl, ecobits;
1876
1877         ecobits = I915_READ(GAC_ECO_BITS);
1878         I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
1879                    ECOBITS_PPGTT_CACHE64B);
1880
1881         gab_ctl = I915_READ(GAB_CTL);
1882         I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
1883
1884         ecochk = I915_READ(GAM_ECOCHK);
1885         I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);
1886
1887         I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
1888 }
1889
1890 /* PPGTT support for Sandybdrige/Gen6 and later */
1891 static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
1892                                    uint64_t start,
1893                                    uint64_t length)
1894 {
1895         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1896         gen6_pte_t *pt_vaddr, scratch_pte;
1897         unsigned first_entry = start >> PAGE_SHIFT;
1898         unsigned num_entries = length >> PAGE_SHIFT;
1899         unsigned act_pt = first_entry / GEN6_PTES;
1900         unsigned first_pte = first_entry % GEN6_PTES;
1901         unsigned last_pte, i;
1902
1903         scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
1904                                      I915_CACHE_LLC, 0);
1905
1906         while (num_entries) {
1907                 last_pte = first_pte + num_entries;
1908                 if (last_pte > GEN6_PTES)
1909                         last_pte = GEN6_PTES;
1910
1911                 pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]);
1912
1913                 for (i = first_pte; i < last_pte; i++)
1914                         pt_vaddr[i] = scratch_pte;
1915
1916                 kunmap_px(pt_vaddr);
1917
1918                 num_entries -= last_pte - first_pte;
1919                 first_pte = 0;
1920                 act_pt++;
1921         }
1922 }
1923
1924 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
1925                                       struct sg_table *pages,
1926                                       uint64_t start,
1927                                       enum i915_cache_level cache_level, u32 flags)
1928 {
1929         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1930         unsigned first_entry = start >> PAGE_SHIFT;
1931         unsigned act_pt = first_entry / GEN6_PTES;
1932         unsigned act_pte = first_entry % GEN6_PTES;
1933         const u32 pte_encode = vm->pte_encode(0, cache_level, flags);
1934         struct sgt_dma iter;
1935         gen6_pte_t *vaddr;
1936
1937         vaddr = kmap_px(ppgtt->pd.page_table[act_pt]);
1938         iter.sg = pages->sgl;
1939         iter.dma = sg_dma_address(iter.sg);
1940         iter.max = iter.dma + iter.sg->length;
1941         do {
1942                 vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma);
1943
1944                 iter.dma += PAGE_SIZE;
1945                 if (iter.dma == iter.max) {
1946                         iter.sg = __sg_next(iter.sg);
1947                         if (!iter.sg)
1948                                 break;
1949
1950                         iter.dma = sg_dma_address(iter.sg);
1951                         iter.max = iter.dma + iter.sg->length;
1952                 }
1953
1954                 if (++act_pte == GEN6_PTES) {
1955                         kunmap_px(vaddr);
1956                         vaddr = kmap_px(ppgtt->pd.page_table[++act_pt]);
1957                         act_pte = 0;
1958                 }
1959         } while (1);
1960         kunmap_px(vaddr);
1961 }
1962
1963 static int gen6_alloc_va_range(struct i915_address_space *vm,
1964                                uint64_t start_in, uint64_t length_in)
1965 {
1966         DECLARE_BITMAP(new_page_tables, I915_PDES);
1967         struct drm_i915_private *dev_priv = vm->i915;
1968         struct i915_ggtt *ggtt = &dev_priv->ggtt;
1969         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1970         struct i915_page_table *pt;
1971         uint32_t start, length, start_save, length_save;
1972         uint32_t pde;
1973         int ret;
1974
1975         start = start_save = start_in;
1976         length = length_save = length_in;
1977
1978         bitmap_zero(new_page_tables, I915_PDES);
1979
1980         /* The allocation is done in two stages so that we can bail out with
1981          * minimal amount of pain. The first stage finds new page tables that
1982          * need allocation. The second stage marks use ptes within the page
1983          * tables.
1984          */
1985         gen6_for_each_pde(pt, &ppgtt->pd, start, length, pde) {
1986                 if (pt != vm->scratch_pt) {
1987                         WARN_ON(bitmap_empty(pt->used_ptes, GEN6_PTES));
1988                         continue;
1989                 }
1990
1991                 /* We've already allocated a page table */
1992                 WARN_ON(!bitmap_empty(pt->used_ptes, GEN6_PTES));
1993
1994                 pt = alloc_pt(vm);
1995                 if (IS_ERR(pt)) {
1996                         ret = PTR_ERR(pt);
1997                         goto unwind_out;
1998                 }
1999
2000                 gen6_initialize_pt(vm, pt);
2001
2002                 ppgtt->pd.page_table[pde] = pt;
2003                 __set_bit(pde, new_page_tables);
2004                 trace_i915_page_table_entry_alloc(vm, pde, start, GEN6_PDE_SHIFT);
2005         }
2006
2007         start = start_save;
2008         length = length_save;
2009
2010         gen6_for_each_pde(pt, &ppgtt->pd, start, length, pde) {
2011                 DECLARE_BITMAP(tmp_bitmap, GEN6_PTES);
2012
2013                 bitmap_zero(tmp_bitmap, GEN6_PTES);
2014                 bitmap_set(tmp_bitmap, gen6_pte_index(start),
2015                            gen6_pte_count(start, length));
2016
2017                 if (__test_and_clear_bit(pde, new_page_tables))
2018                         gen6_write_pde(&ppgtt->pd, pde, pt);
2019
2020                 trace_i915_page_table_entry_map(vm, pde, pt,
2021                                          gen6_pte_index(start),
2022                                          gen6_pte_count(start, length),
2023                                          GEN6_PTES);
2024                 bitmap_or(pt->used_ptes, tmp_bitmap, pt->used_ptes,
2025                                 GEN6_PTES);
2026         }
2027
2028         WARN_ON(!bitmap_empty(new_page_tables, I915_PDES));
2029
2030         /* Make sure write is complete before other code can use this page
2031          * table. Also require for WC mapped PTEs */
2032         readl(ggtt->gsm);
2033
2034         mark_tlbs_dirty(ppgtt);
2035         return 0;
2036
2037 unwind_out:
2038         for_each_set_bit(pde, new_page_tables, I915_PDES) {
2039                 struct i915_page_table *pt = ppgtt->pd.page_table[pde];
2040
2041                 ppgtt->pd.page_table[pde] = vm->scratch_pt;
2042                 free_pt(vm, pt);
2043         }
2044
2045         mark_tlbs_dirty(ppgtt);
2046         return ret;
2047 }
2048
2049 static int gen6_init_scratch(struct i915_address_space *vm)
2050 {
2051         int ret;
2052
2053         ret = setup_scratch_page(vm, I915_GFP_DMA);
2054         if (ret)
2055                 return ret;
2056
2057         vm->scratch_pt = alloc_pt(vm);
2058         if (IS_ERR(vm->scratch_pt)) {
2059                 cleanup_scratch_page(vm);
2060                 return PTR_ERR(vm->scratch_pt);
2061         }
2062
2063         gen6_initialize_pt(vm, vm->scratch_pt);
2064
2065         return 0;
2066 }
2067
2068 static void gen6_free_scratch(struct i915_address_space *vm)
2069 {
2070         free_pt(vm, vm->scratch_pt);
2071         cleanup_scratch_page(vm);
2072 }
2073
2074 static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
2075 {
2076         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
2077         struct i915_page_directory *pd = &ppgtt->pd;
2078         struct i915_page_table *pt;
2079         uint32_t pde;
2080
2081         drm_mm_remove_node(&ppgtt->node);
2082
2083         gen6_for_all_pdes(pt, pd, pde)
2084                 if (pt != vm->scratch_pt)
2085                         free_pt(vm, pt);
2086
2087         gen6_free_scratch(vm);
2088 }
2089
2090 static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)
2091 {
2092         struct i915_address_space *vm = &ppgtt->base;
2093         struct drm_i915_private *dev_priv = ppgtt->base.i915;
2094         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2095         int ret;
2096
2097         /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The
2098          * allocator works in address space sizes, so it's multiplied by page
2099          * size. We allocate at the top of the GTT to avoid fragmentation.
2100          */
2101         BUG_ON(!drm_mm_initialized(&ggtt->base.mm));
2102
2103         ret = gen6_init_scratch(vm);
2104         if (ret)
2105                 return ret;
2106
2107         ret = i915_gem_gtt_insert(&ggtt->base, &ppgtt->node,
2108                                   GEN6_PD_SIZE, GEN6_PD_ALIGN,
2109                                   I915_COLOR_UNEVICTABLE,
2110                                   0, ggtt->base.total,
2111                                   PIN_HIGH);
2112         if (ret)
2113                 goto err_out;
2114
2115         if (ppgtt->node.start < ggtt->mappable_end)
2116                 DRM_DEBUG("Forced to use aperture for PDEs\n");
2117
2118         return 0;
2119
2120 err_out:
2121         gen6_free_scratch(vm);
2122         return ret;
2123 }
2124
2125 static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)
2126 {
2127         return gen6_ppgtt_allocate_page_directories(ppgtt);
2128 }
2129
2130 static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt,
2131                                   uint64_t start, uint64_t length)
2132 {
2133         struct i915_page_table *unused;
2134         uint32_t pde;
2135
2136         gen6_for_each_pde(unused, &ppgtt->pd, start, length, pde)
2137                 ppgtt->pd.page_table[pde] = ppgtt->base.scratch_pt;
2138 }
2139
2140 static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
2141 {
2142         struct drm_i915_private *dev_priv = ppgtt->base.i915;
2143         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2144         int ret;
2145
2146         ppgtt->base.pte_encode = ggtt->base.pte_encode;
2147         if (intel_vgpu_active(dev_priv) || IS_GEN6(dev_priv))
2148                 ppgtt->switch_mm = gen6_mm_switch;
2149         else if (IS_HASWELL(dev_priv))
2150                 ppgtt->switch_mm = hsw_mm_switch;
2151         else if (IS_GEN7(dev_priv))
2152                 ppgtt->switch_mm = gen7_mm_switch;
2153         else
2154                 BUG();
2155
2156         ret = gen6_ppgtt_alloc(ppgtt);
2157         if (ret)
2158                 return ret;
2159
2160         ppgtt->base.allocate_va_range = gen6_alloc_va_range;
2161         ppgtt->base.clear_range = gen6_ppgtt_clear_range;
2162         ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
2163         ppgtt->base.unbind_vma = ppgtt_unbind_vma;
2164         ppgtt->base.bind_vma = ppgtt_bind_vma;
2165         ppgtt->base.cleanup = gen6_ppgtt_cleanup;
2166         ppgtt->base.start = 0;
2167         ppgtt->base.total = I915_PDES * GEN6_PTES * PAGE_SIZE;
2168         ppgtt->debug_dump = gen6_dump_ppgtt;
2169
2170         ppgtt->pd.base.ggtt_offset =
2171                 ppgtt->node.start / PAGE_SIZE * sizeof(gen6_pte_t);
2172
2173         ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm +
2174                 ppgtt->pd.base.ggtt_offset / sizeof(gen6_pte_t);
2175
2176         gen6_scratch_va_range(ppgtt, 0, ppgtt->base.total);
2177
2178         gen6_write_page_range(dev_priv, &ppgtt->pd, 0, ppgtt->base.total);
2179
2180         DRM_DEBUG_DRIVER("Allocated pde space (%lldM) at GTT entry: %llx\n",
2181                          ppgtt->node.size >> 20,
2182                          ppgtt->node.start / PAGE_SIZE);
2183
2184         DRM_DEBUG("Adding PPGTT at offset %x\n",
2185                   ppgtt->pd.base.ggtt_offset << 10);
2186
2187         return 0;
2188 }
2189
2190 static int __hw_ppgtt_init(struct i915_hw_ppgtt *ppgtt,
2191                            struct drm_i915_private *dev_priv)
2192 {
2193         ppgtt->base.i915 = dev_priv;
2194         ppgtt->base.dma = &dev_priv->drm.pdev->dev;
2195
2196         if (INTEL_INFO(dev_priv)->gen < 8)
2197                 return gen6_ppgtt_init(ppgtt);
2198         else
2199                 return gen8_ppgtt_init(ppgtt);
2200 }
2201
2202 static void i915_address_space_init(struct i915_address_space *vm,
2203                                     struct drm_i915_private *dev_priv,
2204                                     const char *name)
2205 {
2206         i915_gem_timeline_init(dev_priv, &vm->timeline, name);
2207
2208         drm_mm_init(&vm->mm, vm->start, vm->total);
2209         vm->mm.head_node.color = I915_COLOR_UNEVICTABLE;
2210
2211         INIT_LIST_HEAD(&vm->active_list);
2212         INIT_LIST_HEAD(&vm->inactive_list);
2213         INIT_LIST_HEAD(&vm->unbound_list);
2214
2215         list_add_tail(&vm->global_link, &dev_priv->vm_list);
2216         pagevec_init(&vm->free_pages, false);
2217 }
2218
2219 static void i915_address_space_fini(struct i915_address_space *vm)
2220 {
2221         if (pagevec_count(&vm->free_pages))
2222                 vm_free_pages_release(vm);
2223
2224         i915_gem_timeline_fini(&vm->timeline);
2225         drm_mm_takedown(&vm->mm);
2226         list_del(&vm->global_link);
2227 }
2228
2229 static void gtt_write_workarounds(struct drm_i915_private *dev_priv)
2230 {
2231         /* This function is for gtt related workarounds. This function is
2232          * called on driver load and after a GPU reset, so you can place
2233          * workarounds here even if they get overwritten by GPU reset.
2234          */
2235         /* WaIncreaseDefaultTLBEntries:chv,bdw,skl,bxt,kbl,glk */
2236         if (IS_BROADWELL(dev_priv))
2237                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_BDW);
2238         else if (IS_CHERRYVIEW(dev_priv))
2239                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_CHV);
2240         else if (IS_GEN9_BC(dev_priv))
2241                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_SKL);
2242         else if (IS_GEN9_LP(dev_priv))
2243                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT);
2244 }
2245
2246 int i915_ppgtt_init_hw(struct drm_i915_private *dev_priv)
2247 {
2248         gtt_write_workarounds(dev_priv);
2249
2250         /* In the case of execlists, PPGTT is enabled by the context descriptor
2251          * and the PDPs are contained within the context itself.  We don't
2252          * need to do anything here. */
2253         if (i915.enable_execlists)
2254                 return 0;
2255
2256         if (!USES_PPGTT(dev_priv))
2257                 return 0;
2258
2259         if (IS_GEN6(dev_priv))
2260                 gen6_ppgtt_enable(dev_priv);
2261         else if (IS_GEN7(dev_priv))
2262                 gen7_ppgtt_enable(dev_priv);
2263         else if (INTEL_GEN(dev_priv) >= 8)
2264                 gen8_ppgtt_enable(dev_priv);
2265         else
2266                 MISSING_CASE(INTEL_GEN(dev_priv));
2267
2268         return 0;
2269 }
2270
2271 struct i915_hw_ppgtt *
2272 i915_ppgtt_create(struct drm_i915_private *dev_priv,
2273                   struct drm_i915_file_private *fpriv,
2274                   const char *name)
2275 {
2276         struct i915_hw_ppgtt *ppgtt;
2277         int ret;
2278
2279         ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
2280         if (!ppgtt)
2281                 return ERR_PTR(-ENOMEM);
2282
2283         ret = __hw_ppgtt_init(ppgtt, dev_priv);
2284         if (ret) {
2285                 kfree(ppgtt);
2286                 return ERR_PTR(ret);
2287         }
2288
2289         kref_init(&ppgtt->ref);
2290         i915_address_space_init(&ppgtt->base, dev_priv, name);
2291         ppgtt->base.file = fpriv;
2292
2293         trace_i915_ppgtt_create(&ppgtt->base);
2294
2295         return ppgtt;
2296 }
2297
2298 void i915_ppgtt_close(struct i915_address_space *vm)
2299 {
2300         struct list_head *phases[] = {
2301                 &vm->active_list,
2302                 &vm->inactive_list,
2303                 &vm->unbound_list,
2304                 NULL,
2305         }, **phase;
2306
2307         GEM_BUG_ON(vm->closed);
2308         vm->closed = true;
2309
2310         for (phase = phases; *phase; phase++) {
2311                 struct i915_vma *vma, *vn;
2312
2313                 list_for_each_entry_safe(vma, vn, *phase, vm_link)
2314                         if (!i915_vma_is_closed(vma))
2315                                 i915_vma_close(vma);
2316         }
2317 }
2318
2319 void i915_ppgtt_release(struct kref *kref)
2320 {
2321         struct i915_hw_ppgtt *ppgtt =
2322                 container_of(kref, struct i915_hw_ppgtt, ref);
2323
2324         trace_i915_ppgtt_release(&ppgtt->base);
2325
2326         /* vmas should already be unbound and destroyed */
2327         WARN_ON(!list_empty(&ppgtt->base.active_list));
2328         WARN_ON(!list_empty(&ppgtt->base.inactive_list));
2329         WARN_ON(!list_empty(&ppgtt->base.unbound_list));
2330
2331         ppgtt->base.cleanup(&ppgtt->base);
2332         i915_address_space_fini(&ppgtt->base);
2333         kfree(ppgtt);
2334 }
2335
2336 /* Certain Gen5 chipsets require require idling the GPU before
2337  * unmapping anything from the GTT when VT-d is enabled.
2338  */
2339 static bool needs_idle_maps(struct drm_i915_private *dev_priv)
2340 {
2341 #ifdef CONFIG_INTEL_IOMMU
2342         /* Query intel_iommu to see if we need the workaround. Presumably that
2343          * was loaded first.
2344          */
2345         if (IS_GEN5(dev_priv) && IS_MOBILE(dev_priv) && intel_iommu_gfx_mapped)
2346                 return true;
2347 #endif
2348         return false;
2349 }
2350
2351 void i915_check_and_clear_faults(struct drm_i915_private *dev_priv)
2352 {
2353         struct intel_engine_cs *engine;
2354         enum intel_engine_id id;
2355
2356         if (INTEL_INFO(dev_priv)->gen < 6)
2357                 return;
2358
2359         for_each_engine(engine, dev_priv, id) {
2360                 u32 fault_reg;
2361                 fault_reg = I915_READ(RING_FAULT_REG(engine));
2362                 if (fault_reg & RING_FAULT_VALID) {
2363                         DRM_DEBUG_DRIVER("Unexpected fault\n"
2364                                          "\tAddr: 0x%08lx\n"
2365                                          "\tAddress space: %s\n"
2366                                          "\tSource ID: %d\n"
2367                                          "\tType: %d\n",
2368                                          fault_reg & PAGE_MASK,
2369                                          fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
2370                                          RING_FAULT_SRCID(fault_reg),
2371                                          RING_FAULT_FAULT_TYPE(fault_reg));
2372                         I915_WRITE(RING_FAULT_REG(engine),
2373                                    fault_reg & ~RING_FAULT_VALID);
2374                 }
2375         }
2376
2377         /* Engine specific init may not have been done till this point. */
2378         if (dev_priv->engine[RCS])
2379                 POSTING_READ(RING_FAULT_REG(dev_priv->engine[RCS]));
2380 }
2381
2382 void i915_gem_suspend_gtt_mappings(struct drm_i915_private *dev_priv)
2383 {
2384         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2385
2386         /* Don't bother messing with faults pre GEN6 as we have little
2387          * documentation supporting that it's a good idea.
2388          */
2389         if (INTEL_GEN(dev_priv) < 6)
2390                 return;
2391
2392         i915_check_and_clear_faults(dev_priv);
2393
2394         ggtt->base.clear_range(&ggtt->base, ggtt->base.start, ggtt->base.total);
2395
2396         i915_ggtt_invalidate(dev_priv);
2397 }
2398
2399 int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj,
2400                                struct sg_table *pages)
2401 {
2402         do {
2403                 if (dma_map_sg(&obj->base.dev->pdev->dev,
2404                                pages->sgl, pages->nents,
2405                                PCI_DMA_BIDIRECTIONAL))
2406                         return 0;
2407
2408                 /* If the DMA remap fails, one cause can be that we have
2409                  * too many objects pinned in a small remapping table,
2410                  * such as swiotlb. Incrementally purge all other objects and
2411                  * try again - if there are no more pages to remove from
2412                  * the DMA remapper, i915_gem_shrink will return 0.
2413                  */
2414                 GEM_BUG_ON(obj->mm.pages == pages);
2415         } while (i915_gem_shrink(to_i915(obj->base.dev),
2416                                  obj->base.size >> PAGE_SHIFT,
2417                                  I915_SHRINK_BOUND |
2418                                  I915_SHRINK_UNBOUND |
2419                                  I915_SHRINK_ACTIVE));
2420
2421         return -ENOSPC;
2422 }
2423
2424 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
2425 {
2426         writeq(pte, addr);
2427 }
2428
2429 static void gen8_ggtt_insert_page(struct i915_address_space *vm,
2430                                   dma_addr_t addr,
2431                                   uint64_t offset,
2432                                   enum i915_cache_level level,
2433                                   u32 unused)
2434 {
2435         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2436         gen8_pte_t __iomem *pte =
2437                 (gen8_pte_t __iomem *)ggtt->gsm + (offset >> PAGE_SHIFT);
2438
2439         gen8_set_pte(pte, gen8_pte_encode(addr, level));
2440
2441         ggtt->invalidate(vm->i915);
2442 }
2443
2444 static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
2445                                      struct sg_table *st,
2446                                      uint64_t start,
2447                                      enum i915_cache_level level, u32 unused)
2448 {
2449         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2450         struct sgt_iter sgt_iter;
2451         gen8_pte_t __iomem *gtt_entries;
2452         const gen8_pte_t pte_encode = gen8_pte_encode(0, level);
2453         dma_addr_t addr;
2454
2455         gtt_entries = (gen8_pte_t __iomem *)ggtt->gsm;
2456         gtt_entries += start >> PAGE_SHIFT;
2457         for_each_sgt_dma(addr, sgt_iter, st)
2458                 gen8_set_pte(gtt_entries++, pte_encode | addr);
2459
2460         wmb();
2461
2462         /* This next bit makes the above posting read even more important. We
2463          * want to flush the TLBs only after we're certain all the PTE updates
2464          * have finished.
2465          */
2466         ggtt->invalidate(vm->i915);
2467 }
2468
2469 struct insert_entries {
2470         struct i915_address_space *vm;
2471         struct sg_table *st;
2472         uint64_t start;
2473         enum i915_cache_level level;
2474         u32 flags;
2475 };
2476
2477 static int gen8_ggtt_insert_entries__cb(void *_arg)
2478 {
2479         struct insert_entries *arg = _arg;
2480         gen8_ggtt_insert_entries(arg->vm, arg->st,
2481                                  arg->start, arg->level, arg->flags);
2482         return 0;
2483 }
2484
2485 static void gen8_ggtt_insert_entries__BKL(struct i915_address_space *vm,
2486                                           struct sg_table *st,
2487                                           uint64_t start,
2488                                           enum i915_cache_level level,
2489                                           u32 flags)
2490 {
2491         struct insert_entries arg = { vm, st, start, level, flags };
2492         stop_machine(gen8_ggtt_insert_entries__cb, &arg, NULL);
2493 }
2494
2495 static void gen6_ggtt_insert_page(struct i915_address_space *vm,
2496                                   dma_addr_t addr,
2497                                   uint64_t offset,
2498                                   enum i915_cache_level level,
2499                                   u32 flags)
2500 {
2501         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2502         gen6_pte_t __iomem *pte =
2503                 (gen6_pte_t __iomem *)ggtt->gsm + (offset >> PAGE_SHIFT);
2504
2505         iowrite32(vm->pte_encode(addr, level, flags), pte);
2506
2507         ggtt->invalidate(vm->i915);
2508 }
2509
2510 /*
2511  * Binds an object into the global gtt with the specified cache level. The object
2512  * will be accessible to the GPU via commands whose operands reference offsets
2513  * within the global GTT as well as accessible by the GPU through the GMADR
2514  * mapped BAR (dev_priv->mm.gtt->gtt).
2515  */
2516 static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
2517                                      struct sg_table *st,
2518                                      uint64_t start,
2519                                      enum i915_cache_level level, u32 flags)
2520 {
2521         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2522         gen6_pte_t __iomem *entries = (gen6_pte_t __iomem *)ggtt->gsm;
2523         unsigned int i = start >> PAGE_SHIFT;
2524         struct sgt_iter iter;
2525         dma_addr_t addr;
2526         for_each_sgt_dma(addr, iter, st)
2527                 iowrite32(vm->pte_encode(addr, level, flags), &entries[i++]);
2528         wmb();
2529
2530         /* This next bit makes the above posting read even more important. We
2531          * want to flush the TLBs only after we're certain all the PTE updates
2532          * have finished.
2533          */
2534         ggtt->invalidate(vm->i915);
2535 }
2536
2537 static void nop_clear_range(struct i915_address_space *vm,
2538                             uint64_t start, uint64_t length)
2539 {
2540 }
2541
2542 static void gen8_ggtt_clear_range(struct i915_address_space *vm,
2543                                   uint64_t start, uint64_t length)
2544 {
2545         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2546         unsigned first_entry = start >> PAGE_SHIFT;
2547         unsigned num_entries = length >> PAGE_SHIFT;
2548         const gen8_pte_t scratch_pte =
2549                 gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC);
2550         gen8_pte_t __iomem *gtt_base =
2551                 (gen8_pte_t __iomem *)ggtt->gsm + first_entry;
2552         const int max_entries = ggtt_total_entries(ggtt) - first_entry;
2553         int i;
2554
2555         if (WARN(num_entries > max_entries,
2556                  "First entry = %d; Num entries = %d (max=%d)\n",
2557                  first_entry, num_entries, max_entries))
2558                 num_entries = max_entries;
2559
2560         for (i = 0; i < num_entries; i++)
2561                 gen8_set_pte(&gtt_base[i], scratch_pte);
2562         readl(gtt_base);
2563 }
2564
2565 static void gen6_ggtt_clear_range(struct i915_address_space *vm,
2566                                   uint64_t start,
2567                                   uint64_t length)
2568 {
2569         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2570         unsigned first_entry = start >> PAGE_SHIFT;
2571         unsigned num_entries = length >> PAGE_SHIFT;
2572         gen6_pte_t scratch_pte, __iomem *gtt_base =
2573                 (gen6_pte_t __iomem *)ggtt->gsm + first_entry;
2574         const int max_entries = ggtt_total_entries(ggtt) - first_entry;
2575         int i;
2576
2577         if (WARN(num_entries > max_entries,
2578                  "First entry = %d; Num entries = %d (max=%d)\n",
2579                  first_entry, num_entries, max_entries))
2580                 num_entries = max_entries;
2581
2582         scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
2583                                      I915_CACHE_LLC, 0);
2584
2585         for (i = 0; i < num_entries; i++)
2586                 iowrite32(scratch_pte, &gtt_base[i]);
2587         readl(gtt_base);
2588 }
2589
2590 static void i915_ggtt_insert_page(struct i915_address_space *vm,
2591                                   dma_addr_t addr,
2592                                   uint64_t offset,
2593                                   enum i915_cache_level cache_level,
2594                                   u32 unused)
2595 {
2596         unsigned int flags = (cache_level == I915_CACHE_NONE) ?
2597                 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
2598
2599         intel_gtt_insert_page(addr, offset >> PAGE_SHIFT, flags);
2600 }
2601
2602 static void i915_ggtt_insert_entries(struct i915_address_space *vm,
2603                                      struct sg_table *pages,
2604                                      uint64_t start,
2605                                      enum i915_cache_level cache_level, u32 unused)
2606 {
2607         unsigned int flags = (cache_level == I915_CACHE_NONE) ?
2608                 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
2609
2610         intel_gtt_insert_sg_entries(pages, start >> PAGE_SHIFT, flags);
2611
2612 }
2613
2614 static void i915_ggtt_clear_range(struct i915_address_space *vm,
2615                                   uint64_t start,
2616                                   uint64_t length)
2617 {
2618         intel_gtt_clear_range(start >> PAGE_SHIFT, length >> PAGE_SHIFT);
2619 }
2620
2621 static int ggtt_bind_vma(struct i915_vma *vma,
2622                          enum i915_cache_level cache_level,
2623                          u32 flags)
2624 {
2625         struct drm_i915_private *i915 = vma->vm->i915;
2626         struct drm_i915_gem_object *obj = vma->obj;
2627         u32 pte_flags;
2628
2629         if (unlikely(!vma->pages)) {
2630                 int ret = i915_get_ggtt_vma_pages(vma);
2631                 if (ret)
2632                         return ret;
2633         }
2634
2635         /* Currently applicable only to VLV */
2636         pte_flags = 0;
2637         if (obj->gt_ro)
2638                 pte_flags |= PTE_READ_ONLY;
2639
2640         intel_runtime_pm_get(i915);
2641         vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start,
2642                                 cache_level, pte_flags);
2643         intel_runtime_pm_put(i915);
2644
2645         /*
2646          * Without aliasing PPGTT there's no difference between
2647          * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally
2648          * upgrade to both bound if we bind either to avoid double-binding.
2649          */
2650         vma->flags |= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
2651
2652         return 0;
2653 }
2654
2655 static void ggtt_unbind_vma(struct i915_vma *vma)
2656 {
2657         struct drm_i915_private *i915 = vma->vm->i915;
2658
2659         intel_runtime_pm_get(i915);
2660         vma->vm->clear_range(vma->vm, vma->node.start, vma->size);
2661         intel_runtime_pm_put(i915);
2662 }
2663
2664 static int aliasing_gtt_bind_vma(struct i915_vma *vma,
2665                                  enum i915_cache_level cache_level,
2666                                  u32 flags)
2667 {
2668         struct drm_i915_private *i915 = vma->vm->i915;
2669         u32 pte_flags;
2670
2671         if (unlikely(!vma->pages)) {
2672                 int ret = i915_get_ggtt_vma_pages(vma);
2673                 if (ret)
2674                         return ret;
2675         }
2676
2677         /* Currently applicable only to VLV */
2678         pte_flags = 0;
2679         if (vma->obj->gt_ro)
2680                 pte_flags |= PTE_READ_ONLY;
2681
2682         if (flags & I915_VMA_GLOBAL_BIND) {
2683                 intel_runtime_pm_get(i915);
2684                 vma->vm->insert_entries(vma->vm,
2685                                         vma->pages, vma->node.start,
2686                                         cache_level, pte_flags);
2687                 intel_runtime_pm_put(i915);
2688         }
2689
2690         if (flags & I915_VMA_LOCAL_BIND) {
2691                 struct i915_hw_ppgtt *appgtt = i915->mm.aliasing_ppgtt;
2692                 appgtt->base.insert_entries(&appgtt->base,
2693                                             vma->pages, vma->node.start,
2694                                             cache_level, pte_flags);
2695         }
2696
2697         return 0;
2698 }
2699
2700 static void aliasing_gtt_unbind_vma(struct i915_vma *vma)
2701 {
2702         struct drm_i915_private *i915 = vma->vm->i915;
2703
2704         if (vma->flags & I915_VMA_GLOBAL_BIND) {
2705                 intel_runtime_pm_get(i915);
2706                 vma->vm->clear_range(vma->vm, vma->node.start, vma->size);
2707                 intel_runtime_pm_put(i915);
2708         }
2709
2710         if (vma->flags & I915_VMA_LOCAL_BIND) {
2711                 struct i915_address_space *vm = &i915->mm.aliasing_ppgtt->base;
2712
2713                 vm->clear_range(vm, vma->node.start, vma->size);
2714         }
2715 }
2716
2717 void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj,
2718                                struct sg_table *pages)
2719 {
2720         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2721         struct device *kdev = &dev_priv->drm.pdev->dev;
2722         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2723
2724         if (unlikely(ggtt->do_idle_maps)) {
2725                 if (i915_gem_wait_for_idle(dev_priv, I915_WAIT_LOCKED)) {
2726                         DRM_ERROR("Failed to wait for idle; VT'd may hang.\n");
2727                         /* Wait a bit, in hopes it avoids the hang */
2728                         udelay(10);
2729                 }
2730         }
2731
2732         dma_unmap_sg(kdev, pages->sgl, pages->nents, PCI_DMA_BIDIRECTIONAL);
2733 }
2734
2735 static void i915_gtt_color_adjust(const struct drm_mm_node *node,
2736                                   unsigned long color,
2737                                   u64 *start,
2738                                   u64 *end)
2739 {
2740         if (node->allocated && node->color != color)
2741                 *start += I915_GTT_PAGE_SIZE;
2742
2743         /* Also leave a space between the unallocated reserved node after the
2744          * GTT and any objects within the GTT, i.e. we use the color adjustment
2745          * to insert a guard page to prevent prefetches crossing over the
2746          * GTT boundary.
2747          */
2748         node = list_next_entry(node, node_list);
2749         if (node->color != color)
2750                 *end -= I915_GTT_PAGE_SIZE;
2751 }
2752
2753 int i915_gem_init_aliasing_ppgtt(struct drm_i915_private *i915)
2754 {
2755         struct i915_ggtt *ggtt = &i915->ggtt;
2756         struct i915_hw_ppgtt *ppgtt;
2757         int err;
2758
2759         ppgtt = i915_ppgtt_create(i915, NULL, "[alias]");
2760         if (IS_ERR(ppgtt))
2761                 return PTR_ERR(ppgtt);
2762
2763         if (ppgtt->base.allocate_va_range) {
2764                 err = ppgtt->base.allocate_va_range(&ppgtt->base,
2765                                                     0, ppgtt->base.total);
2766                 if (err)
2767                         goto err_ppgtt;
2768         }
2769
2770         ppgtt->base.clear_range(&ppgtt->base,
2771                                 ppgtt->base.start,
2772                                 ppgtt->base.total);
2773
2774         i915->mm.aliasing_ppgtt = ppgtt;
2775
2776         WARN_ON(ggtt->base.bind_vma != ggtt_bind_vma);
2777         ggtt->base.bind_vma = aliasing_gtt_bind_vma;
2778
2779         WARN_ON(ggtt->base.unbind_vma != ggtt_unbind_vma);
2780         ggtt->base.unbind_vma = aliasing_gtt_unbind_vma;
2781
2782         return 0;
2783
2784 err_ppgtt:
2785         i915_ppgtt_put(ppgtt);
2786         return err;
2787 }
2788
2789 void i915_gem_fini_aliasing_ppgtt(struct drm_i915_private *i915)
2790 {
2791         struct i915_ggtt *ggtt = &i915->ggtt;
2792         struct i915_hw_ppgtt *ppgtt;
2793
2794         ppgtt = fetch_and_zero(&i915->mm.aliasing_ppgtt);
2795         if (!ppgtt)
2796                 return;
2797
2798         i915_ppgtt_put(ppgtt);
2799
2800         ggtt->base.bind_vma = ggtt_bind_vma;
2801         ggtt->base.unbind_vma = ggtt_unbind_vma;
2802 }
2803
2804 int i915_gem_init_ggtt(struct drm_i915_private *dev_priv)
2805 {
2806         /* Let GEM Manage all of the aperture.
2807          *
2808          * However, leave one page at the end still bound to the scratch page.
2809          * There are a number of places where the hardware apparently prefetches
2810          * past the end of the object, and we've seen multiple hangs with the
2811          * GPU head pointer stuck in a batchbuffer bound at the last page of the
2812          * aperture.  One page should be enough to keep any prefetching inside
2813          * of the aperture.
2814          */
2815         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2816         unsigned long hole_start, hole_end;
2817         struct drm_mm_node *entry;
2818         int ret;
2819
2820         ret = intel_vgt_balloon(dev_priv);
2821         if (ret)
2822                 return ret;
2823
2824         /* Reserve a mappable slot for our lockless error capture */
2825         ret = drm_mm_insert_node_in_range(&ggtt->base.mm, &ggtt->error_capture,
2826                                           PAGE_SIZE, 0, I915_COLOR_UNEVICTABLE,
2827                                           0, ggtt->mappable_end,
2828                                           DRM_MM_INSERT_LOW);
2829         if (ret)
2830                 return ret;
2831
2832         /* Clear any non-preallocated blocks */
2833         drm_mm_for_each_hole(entry, &ggtt->base.mm, hole_start, hole_end) {
2834                 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
2835                               hole_start, hole_end);
2836                 ggtt->base.clear_range(&ggtt->base, hole_start,
2837                                        hole_end - hole_start);
2838         }
2839
2840         /* And finally clear the reserved guard page */
2841         ggtt->base.clear_range(&ggtt->base,
2842                                ggtt->base.total - PAGE_SIZE, PAGE_SIZE);
2843
2844         if (USES_PPGTT(dev_priv) && !USES_FULL_PPGTT(dev_priv)) {
2845                 ret = i915_gem_init_aliasing_ppgtt(dev_priv);
2846                 if (ret)
2847                         goto err;
2848         }
2849
2850         return 0;
2851
2852 err:
2853         drm_mm_remove_node(&ggtt->error_capture);
2854         return ret;
2855 }
2856
2857 /**
2858  * i915_ggtt_cleanup_hw - Clean up GGTT hardware initialization
2859  * @dev_priv: i915 device
2860  */
2861 void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv)
2862 {
2863         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2864         struct i915_vma *vma, *vn;
2865
2866         ggtt->base.closed = true;
2867
2868         mutex_lock(&dev_priv->drm.struct_mutex);
2869         WARN_ON(!list_empty(&ggtt->base.active_list));
2870         list_for_each_entry_safe(vma, vn, &ggtt->base.inactive_list, vm_link)
2871                 WARN_ON(i915_vma_unbind(vma));
2872         mutex_unlock(&dev_priv->drm.struct_mutex);
2873
2874         i915_gem_cleanup_stolen(&dev_priv->drm);
2875
2876         mutex_lock(&dev_priv->drm.struct_mutex);
2877         i915_gem_fini_aliasing_ppgtt(dev_priv);
2878
2879         if (drm_mm_node_allocated(&ggtt->error_capture))
2880                 drm_mm_remove_node(&ggtt->error_capture);
2881
2882         if (drm_mm_initialized(&ggtt->base.mm)) {
2883                 intel_vgt_deballoon(dev_priv);
2884                 i915_address_space_fini(&ggtt->base);
2885         }
2886
2887         ggtt->base.cleanup(&ggtt->base);
2888         mutex_unlock(&dev_priv->drm.struct_mutex);
2889
2890         arch_phys_wc_del(ggtt->mtrr);
2891         io_mapping_fini(&ggtt->mappable);
2892 }
2893
2894 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
2895 {
2896         snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
2897         snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
2898         return snb_gmch_ctl << 20;
2899 }
2900
2901 static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
2902 {
2903         bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
2904         bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
2905         if (bdw_gmch_ctl)
2906                 bdw_gmch_ctl = 1 << bdw_gmch_ctl;
2907
2908 #ifdef CONFIG_X86_32
2909         /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */
2910         if (bdw_gmch_ctl > 4)
2911                 bdw_gmch_ctl = 4;
2912 #endif
2913
2914         return bdw_gmch_ctl << 20;
2915 }
2916
2917 static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
2918 {
2919         gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
2920         gmch_ctrl &= SNB_GMCH_GGMS_MASK;
2921
2922         if (gmch_ctrl)
2923                 return 1 << (20 + gmch_ctrl);
2924
2925         return 0;
2926 }
2927
2928 static size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
2929 {
2930         snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
2931         snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
2932         return snb_gmch_ctl << 25; /* 32 MB units */
2933 }
2934
2935 static size_t gen8_get_stolen_size(u16 bdw_gmch_ctl)
2936 {
2937         bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
2938         bdw_gmch_ctl &= BDW_GMCH_GMS_MASK;
2939         return bdw_gmch_ctl << 25; /* 32 MB units */
2940 }
2941
2942 static size_t chv_get_stolen_size(u16 gmch_ctrl)
2943 {
2944         gmch_ctrl >>= SNB_GMCH_GMS_SHIFT;
2945         gmch_ctrl &= SNB_GMCH_GMS_MASK;
2946
2947         /*
2948          * 0x0  to 0x10: 32MB increments starting at 0MB
2949          * 0x11 to 0x16: 4MB increments starting at 8MB
2950          * 0x17 to 0x1d: 4MB increments start at 36MB
2951          */
2952         if (gmch_ctrl < 0x11)
2953                 return gmch_ctrl << 25;
2954         else if (gmch_ctrl < 0x17)
2955                 return (gmch_ctrl - 0x11 + 2) << 22;
2956         else
2957                 return (gmch_ctrl - 0x17 + 9) << 22;
2958 }
2959
2960 static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl)
2961 {
2962         gen9_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
2963         gen9_gmch_ctl &= BDW_GMCH_GMS_MASK;
2964
2965         if (gen9_gmch_ctl < 0xf0)
2966                 return gen9_gmch_ctl << 25; /* 32 MB units */
2967         else
2968                 /* 4MB increments starting at 0xf0 for 4MB */
2969                 return (gen9_gmch_ctl - 0xf0 + 1) << 22;
2970 }
2971
2972 static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
2973 {
2974         struct drm_i915_private *dev_priv = ggtt->base.i915;
2975         struct pci_dev *pdev = dev_priv->drm.pdev;
2976         phys_addr_t phys_addr;
2977         int ret;
2978
2979         /* For Modern GENs the PTEs and register space are split in the BAR */
2980         phys_addr = pci_resource_start(pdev, 0) + pci_resource_len(pdev, 0) / 2;
2981
2982         /*
2983          * On BXT writes larger than 64 bit to the GTT pagetable range will be
2984          * dropped. For WC mappings in general we have 64 byte burst writes
2985          * when the WC buffer is flushed, so we can't use it, but have to
2986          * resort to an uncached mapping. The WC issue is easily caught by the
2987          * readback check when writing GTT PTE entries.
2988          */
2989         if (IS_GEN9_LP(dev_priv))
2990                 ggtt->gsm = ioremap_nocache(phys_addr, size);
2991         else
2992                 ggtt->gsm = ioremap_wc(phys_addr, size);
2993         if (!ggtt->gsm) {
2994                 DRM_ERROR("Failed to map the ggtt page table\n");
2995                 return -ENOMEM;
2996         }
2997
2998         ret = setup_scratch_page(&ggtt->base, GFP_DMA32);
2999         if (ret) {
3000                 DRM_ERROR("Scratch setup failed\n");
3001                 /* iounmap will also get called at remove, but meh */
3002                 iounmap(ggtt->gsm);
3003                 return ret;
3004         }
3005
3006         return 0;
3007 }
3008
3009 /* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
3010  * bits. When using advanced contexts each context stores its own PAT, but
3011  * writing this data shouldn't be harmful even in those cases. */
3012 static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv)
3013 {
3014         uint64_t pat;
3015
3016         pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC)     | /* for normal objects, no eLLC */
3017               GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */
3018               GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */
3019               GEN8_PPAT(3, GEN8_PPAT_UC)                     | /* Uncached objects, mostly for scanout */
3020               GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) |
3021               GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) |
3022               GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) |
3023               GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
3024
3025         if (!USES_PPGTT(dev_priv))
3026                 /* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry,
3027                  * so RTL will always use the value corresponding to
3028                  * pat_sel = 000".
3029                  * So let's disable cache for GGTT to avoid screen corruptions.
3030                  * MOCS still can be used though.
3031                  * - System agent ggtt writes (i.e. cpu gtt mmaps) already work
3032                  * before this patch, i.e. the same uncached + snooping access
3033                  * like on gen6/7 seems to be in effect.
3034                  * - So this just fixes blitter/render access. Again it looks
3035                  * like it's not just uncached access, but uncached + snooping.
3036                  * So we can still hold onto all our assumptions wrt cpu
3037                  * clflushing on LLC machines.
3038                  */
3039                 pat = GEN8_PPAT(0, GEN8_PPAT_UC);
3040
3041         /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b
3042          * write would work. */
3043         I915_WRITE(GEN8_PRIVATE_PAT_LO, pat);
3044         I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32);
3045 }
3046
3047 static void chv_setup_private_ppat(struct drm_i915_private *dev_priv)
3048 {
3049         uint64_t pat;
3050
3051         /*
3052          * Map WB on BDW to snooped on CHV.
3053          *
3054          * Only the snoop bit has meaning for CHV, the rest is
3055          * ignored.
3056          *
3057          * The hardware will never snoop for certain types of accesses:
3058          * - CPU GTT (GMADR->GGTT->no snoop->memory)
3059          * - PPGTT page tables
3060          * - some other special cycles
3061          *
3062          * As with BDW, we also need to consider the following for GT accesses:
3063          * "For GGTT, there is NO pat_sel[2:0] from the entry,
3064          * so RTL will always use the value corresponding to
3065          * pat_sel = 000".
3066          * Which means we must set the snoop bit in PAT entry 0
3067          * in order to keep the global status page working.
3068          */
3069         pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) |
3070               GEN8_PPAT(1, 0) |
3071               GEN8_PPAT(2, 0) |
3072               GEN8_PPAT(3, 0) |
3073               GEN8_PPAT(4, CHV_PPAT_SNOOP) |
3074               GEN8_PPAT(5, CHV_PPAT_SNOOP) |
3075               GEN8_PPAT(6, CHV_PPAT_SNOOP) |
3076               GEN8_PPAT(7, CHV_PPAT_SNOOP);
3077
3078         I915_WRITE(GEN8_PRIVATE_PAT_LO, pat);
3079         I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32);
3080 }
3081
3082 static void gen6_gmch_remove(struct i915_address_space *vm)
3083 {
3084         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
3085
3086         iounmap(ggtt->gsm);
3087         cleanup_scratch_page(vm);
3088 }
3089
3090 static int gen8_gmch_probe(struct i915_ggtt *ggtt)
3091 {
3092         struct drm_i915_private *dev_priv = ggtt->base.i915;
3093         struct pci_dev *pdev = dev_priv->drm.pdev;
3094         unsigned int size;
3095         u16 snb_gmch_ctl;
3096
3097         /* TODO: We're not aware of mappable constraints on gen8 yet */
3098         ggtt->mappable_base = pci_resource_start(pdev, 2);
3099         ggtt->mappable_end = pci_resource_len(pdev, 2);
3100
3101         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(39)))
3102                 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(39));
3103
3104         pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
3105
3106         if (INTEL_GEN(dev_priv) >= 9) {
3107                 ggtt->stolen_size = gen9_get_stolen_size(snb_gmch_ctl);
3108                 size = gen8_get_total_gtt_size(snb_gmch_ctl);
3109         } else if (IS_CHERRYVIEW(dev_priv)) {
3110                 ggtt->stolen_size = chv_get_stolen_size(snb_gmch_ctl);
3111                 size = chv_get_total_gtt_size(snb_gmch_ctl);
3112         } else {
3113                 ggtt->stolen_size = gen8_get_stolen_size(snb_gmch_ctl);
3114                 size = gen8_get_total_gtt_size(snb_gmch_ctl);
3115         }
3116
3117         ggtt->base.total = (size / sizeof(gen8_pte_t)) << PAGE_SHIFT;
3118
3119         if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv))
3120                 chv_setup_private_ppat(dev_priv);
3121         else
3122                 bdw_setup_private_ppat(dev_priv);
3123
3124         ggtt->base.cleanup = gen6_gmch_remove;
3125         ggtt->base.bind_vma = ggtt_bind_vma;
3126         ggtt->base.unbind_vma = ggtt_unbind_vma;
3127         ggtt->base.insert_page = gen8_ggtt_insert_page;
3128         ggtt->base.clear_range = nop_clear_range;
3129         if (!USES_FULL_PPGTT(dev_priv) || intel_scanout_needs_vtd_wa(dev_priv))
3130                 ggtt->base.clear_range = gen8_ggtt_clear_range;
3131
3132         ggtt->base.insert_entries = gen8_ggtt_insert_entries;
3133         if (IS_CHERRYVIEW(dev_priv))
3134                 ggtt->base.insert_entries = gen8_ggtt_insert_entries__BKL;
3135
3136         ggtt->invalidate = gen6_ggtt_invalidate;
3137
3138         return ggtt_probe_common(ggtt, size);
3139 }
3140
3141 static int gen6_gmch_probe(struct i915_ggtt *ggtt)
3142 {
3143         struct drm_i915_private *dev_priv = ggtt->base.i915;
3144         struct pci_dev *pdev = dev_priv->drm.pdev;
3145         unsigned int size;
3146         u16 snb_gmch_ctl;
3147
3148         ggtt->mappable_base = pci_resource_start(pdev, 2);
3149         ggtt->mappable_end = pci_resource_len(pdev, 2);
3150
3151         /* 64/512MB is the current min/max we actually know of, but this is just
3152          * a coarse sanity check.
3153          */
3154         if (ggtt->mappable_end < (64<<20) || ggtt->mappable_end > (512<<20)) {
3155                 DRM_ERROR("Unknown GMADR size (%llx)\n", ggtt->mappable_end);
3156                 return -ENXIO;
3157         }
3158
3159         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(40)))
3160                 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(40));
3161         pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
3162
3163         ggtt->stolen_size = gen6_get_stolen_size(snb_gmch_ctl);
3164
3165         size = gen6_get_total_gtt_size(snb_gmch_ctl);
3166         ggtt->base.total = (size / sizeof(gen6_pte_t)) << PAGE_SHIFT;
3167
3168         ggtt->base.clear_range = gen6_ggtt_clear_range;
3169         ggtt->base.insert_page = gen6_ggtt_insert_page;
3170         ggtt->base.insert_entries = gen6_ggtt_insert_entries;
3171         ggtt->base.bind_vma = ggtt_bind_vma;
3172         ggtt->base.unbind_vma = ggtt_unbind_vma;
3173         ggtt->base.cleanup = gen6_gmch_remove;
3174
3175         ggtt->invalidate = gen6_ggtt_invalidate;
3176
3177         if (HAS_EDRAM(dev_priv))
3178                 ggtt->base.pte_encode = iris_pte_encode;
3179         else if (IS_HASWELL(dev_priv))
3180                 ggtt->base.pte_encode = hsw_pte_encode;
3181         else if (IS_VALLEYVIEW(dev_priv))
3182                 ggtt->base.pte_encode = byt_pte_encode;
3183         else if (INTEL_GEN(dev_priv) >= 7)
3184                 ggtt->base.pte_encode = ivb_pte_encode;
3185         else
3186                 ggtt->base.pte_encode = snb_pte_encode;
3187
3188         return ggtt_probe_common(ggtt, size);
3189 }
3190
3191 static void i915_gmch_remove(struct i915_address_space *vm)
3192 {
3193         intel_gmch_remove();
3194 }
3195
3196 static int i915_gmch_probe(struct i915_ggtt *ggtt)
3197 {
3198         struct drm_i915_private *dev_priv = ggtt->base.i915;
3199         int ret;
3200
3201         ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->drm.pdev, NULL);
3202         if (!ret) {
3203                 DRM_ERROR("failed to set up gmch\n");
3204                 return -EIO;
3205         }
3206
3207         intel_gtt_get(&ggtt->base.total,
3208                       &ggtt->stolen_size,
3209                       &ggtt->mappable_base,
3210                       &ggtt->mappable_end);
3211
3212         ggtt->do_idle_maps = needs_idle_maps(dev_priv);
3213         ggtt->base.insert_page = i915_ggtt_insert_page;
3214         ggtt->base.insert_entries = i915_ggtt_insert_entries;
3215         ggtt->base.clear_range = i915_ggtt_clear_range;
3216         ggtt->base.bind_vma = ggtt_bind_vma;
3217         ggtt->base.unbind_vma = ggtt_unbind_vma;
3218         ggtt->base.cleanup = i915_gmch_remove;
3219
3220         ggtt->invalidate = gmch_ggtt_invalidate;
3221
3222         if (unlikely(ggtt->do_idle_maps))
3223                 DRM_INFO("applying Ironlake quirks for intel_iommu\n");
3224
3225         return 0;
3226 }
3227
3228 /**
3229  * i915_ggtt_probe_hw - Probe GGTT hardware location
3230  * @dev_priv: i915 device
3231  */
3232 int i915_ggtt_probe_hw(struct drm_i915_private *dev_priv)
3233 {
3234         struct i915_ggtt *ggtt = &dev_priv->ggtt;
3235         int ret;
3236
3237         ggtt->base.i915 = dev_priv;
3238         ggtt->base.dma = &dev_priv->drm.pdev->dev;
3239
3240         if (INTEL_GEN(dev_priv) <= 5)
3241                 ret = i915_gmch_probe(ggtt);
3242         else if (INTEL_GEN(dev_priv) < 8)
3243                 ret = gen6_gmch_probe(ggtt);
3244         else
3245                 ret = gen8_gmch_probe(ggtt);
3246         if (ret)
3247                 return ret;
3248
3249         /* Trim the GGTT to fit the GuC mappable upper range (when enabled).
3250          * This is easier than doing range restriction on the fly, as we
3251          * currently don't have any bits spare to pass in this upper
3252          * restriction!
3253          */
3254         if (HAS_GUC(dev_priv) && i915.enable_guc_loading) {
3255                 ggtt->base.total = min_t(u64, ggtt->base.total, GUC_GGTT_TOP);
3256                 ggtt->mappable_end = min(ggtt->mappable_end, ggtt->base.total);
3257         }
3258
3259         if ((ggtt->base.total - 1) >> 32) {
3260                 DRM_ERROR("We never expected a Global GTT with more than 32bits"
3261                           " of address space! Found %lldM!\n",
3262                           ggtt->base.total >> 20);
3263                 ggtt->base.total = 1ULL << 32;
3264                 ggtt->mappable_end = min(ggtt->mappable_end, ggtt->base.total);
3265         }
3266
3267         if (ggtt->mappable_end > ggtt->base.total) {
3268                 DRM_ERROR("mappable aperture extends past end of GGTT,"
3269                           " aperture=%llx, total=%llx\n",
3270                           ggtt->mappable_end, ggtt->base.total);
3271                 ggtt->mappable_end = ggtt->base.total;
3272         }
3273
3274         /* GMADR is the PCI mmio aperture into the global GTT. */
3275         DRM_INFO("Memory usable by graphics device = %lluM\n",
3276                  ggtt->base.total >> 20);
3277         DRM_DEBUG_DRIVER("GMADR size = %lldM\n", ggtt->mappable_end >> 20);
3278         DRM_DEBUG_DRIVER("GTT stolen size = %uM\n", ggtt->stolen_size >> 20);
3279 #ifdef CONFIG_INTEL_IOMMU
3280         if (intel_iommu_gfx_mapped)
3281                 DRM_INFO("VT-d active for gfx access\n");
3282 #endif
3283
3284         return 0;
3285 }
3286
3287 /**
3288  * i915_ggtt_init_hw - Initialize GGTT hardware
3289  * @dev_priv: i915 device
3290  */
3291 int i915_ggtt_init_hw(struct drm_i915_private *dev_priv)
3292 {
3293         struct i915_ggtt *ggtt = &dev_priv->ggtt;
3294         int ret;
3295
3296         INIT_LIST_HEAD(&dev_priv->vm_list);
3297
3298         /* Note that we use page colouring to enforce a guard page at the
3299          * end of the address space. This is required as the CS may prefetch
3300          * beyond the end of the batch buffer, across the page boundary,
3301          * and beyond the end of the GTT if we do not provide a guard.
3302          */
3303         mutex_lock(&dev_priv->drm.struct_mutex);
3304         i915_address_space_init(&ggtt->base, dev_priv, "[global]");
3305         if (!HAS_LLC(dev_priv) && !USES_PPGTT(dev_priv))
3306                 ggtt->base.mm.color_adjust = i915_gtt_color_adjust;
3307         mutex_unlock(&dev_priv->drm.struct_mutex);
3308
3309         if (!io_mapping_init_wc(&dev_priv->ggtt.mappable,
3310                                 dev_priv->ggtt.mappable_base,
3311                                 dev_priv->ggtt.mappable_end)) {
3312                 ret = -EIO;
3313                 goto out_gtt_cleanup;
3314         }
3315
3316         ggtt->mtrr = arch_phys_wc_add(ggtt->mappable_base, ggtt->mappable_end);
3317
3318         /*
3319          * Initialise stolen early so that we may reserve preallocated
3320          * objects for the BIOS to KMS transition.
3321          */
3322         ret = i915_gem_init_stolen(dev_priv);
3323         if (ret)
3324                 goto out_gtt_cleanup;
3325
3326         return 0;
3327
3328 out_gtt_cleanup:
3329         ggtt->base.cleanup(&ggtt->base);
3330         return ret;
3331 }
3332
3333 int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv)
3334 {
3335         if (INTEL_GEN(dev_priv) < 6 && !intel_enable_gtt())
3336                 return -EIO;
3337
3338         return 0;
3339 }
3340
3341 void i915_ggtt_enable_guc(struct drm_i915_private *i915)
3342 {
3343         i915->ggtt.invalidate = guc_ggtt_invalidate;
3344 }
3345
3346 void i915_ggtt_disable_guc(struct drm_i915_private *i915)
3347 {
3348         i915->ggtt.invalidate = gen6_ggtt_invalidate;
3349 }
3350
3351 void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv)
3352 {
3353         struct i915_ggtt *ggtt = &dev_priv->ggtt;
3354         struct drm_i915_gem_object *obj, *on;
3355
3356         i915_check_and_clear_faults(dev_priv);
3357
3358         /* First fill our portion of the GTT with scratch pages */
3359         ggtt->base.clear_range(&ggtt->base, ggtt->base.start, ggtt->base.total);
3360
3361         ggtt->base.closed = true; /* skip rewriting PTE on VMA unbind */
3362
3363         /* clflush objects bound into the GGTT and rebind them. */
3364         list_for_each_entry_safe(obj, on,
3365                                  &dev_priv->mm.bound_list, global_link) {
3366                 bool ggtt_bound = false;
3367                 struct i915_vma *vma;
3368
3369                 list_for_each_entry(vma, &obj->vma_list, obj_link) {
3370                         if (vma->vm != &ggtt->base)
3371                                 continue;
3372
3373                         if (!i915_vma_unbind(vma))
3374                                 continue;
3375
3376                         WARN_ON(i915_vma_bind(vma, obj->cache_level,
3377                                               PIN_UPDATE));
3378                         ggtt_bound = true;
3379                 }
3380
3381                 if (ggtt_bound)
3382                         WARN_ON(i915_gem_object_set_to_gtt_domain(obj, false));
3383         }
3384
3385         ggtt->base.closed = false;
3386
3387         if (INTEL_GEN(dev_priv) >= 8) {
3388                 if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv))
3389                         chv_setup_private_ppat(dev_priv);
3390                 else
3391                         bdw_setup_private_ppat(dev_priv);
3392
3393                 return;
3394         }
3395
3396         if (USES_PPGTT(dev_priv)) {
3397                 struct i915_address_space *vm;
3398
3399                 list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
3400                         /* TODO: Perhaps it shouldn't be gen6 specific */
3401
3402                         struct i915_hw_ppgtt *ppgtt;
3403
3404                         if (i915_is_ggtt(vm))
3405                                 ppgtt = dev_priv->mm.aliasing_ppgtt;
3406                         else
3407                                 ppgtt = i915_vm_to_ppgtt(vm);
3408
3409                         gen6_write_page_range(dev_priv, &ppgtt->pd,
3410                                               0, ppgtt->base.total);
3411                 }
3412         }
3413
3414         i915_ggtt_invalidate(dev_priv);
3415 }
3416
3417 static struct scatterlist *
3418 rotate_pages(const dma_addr_t *in, unsigned int offset,
3419              unsigned int width, unsigned int height,
3420              unsigned int stride,
3421              struct sg_table *st, struct scatterlist *sg)
3422 {
3423         unsigned int column, row;
3424         unsigned int src_idx;
3425
3426         for (column = 0; column < width; column++) {
3427                 src_idx = stride * (height - 1) + column;
3428                 for (row = 0; row < height; row++) {
3429                         st->nents++;
3430                         /* We don't need the pages, but need to initialize
3431                          * the entries so the sg list can be happily traversed.
3432                          * The only thing we need are DMA addresses.
3433                          */
3434                         sg_set_page(sg, NULL, PAGE_SIZE, 0);
3435                         sg_dma_address(sg) = in[offset + src_idx];
3436                         sg_dma_len(sg) = PAGE_SIZE;
3437                         sg = sg_next(sg);
3438                         src_idx -= stride;
3439                 }
3440         }
3441
3442         return sg;
3443 }
3444
3445 static noinline struct sg_table *
3446 intel_rotate_pages(struct intel_rotation_info *rot_info,
3447                    struct drm_i915_gem_object *obj)
3448 {
3449         const size_t n_pages = obj->base.size / PAGE_SIZE;
3450         unsigned int size = intel_rotation_info_size(rot_info);
3451         struct sgt_iter sgt_iter;
3452         dma_addr_t dma_addr;
3453         unsigned long i;
3454         dma_addr_t *page_addr_list;
3455         struct sg_table *st;
3456         struct scatterlist *sg;
3457         int ret = -ENOMEM;
3458
3459         /* Allocate a temporary list of source pages for random access. */
3460         page_addr_list = drm_malloc_gfp(n_pages,
3461                                         sizeof(dma_addr_t),
3462                                         GFP_TEMPORARY);
3463         if (!page_addr_list)
3464                 return ERR_PTR(ret);
3465
3466         /* Allocate target SG list. */
3467         st = kmalloc(sizeof(*st), GFP_KERNEL);
3468         if (!st)
3469                 goto err_st_alloc;
3470
3471         ret = sg_alloc_table(st, size, GFP_KERNEL);
3472         if (ret)
3473                 goto err_sg_alloc;
3474
3475         /* Populate source page list from the object. */
3476         i = 0;
3477         for_each_sgt_dma(dma_addr, sgt_iter, obj->mm.pages)
3478                 page_addr_list[i++] = dma_addr;
3479
3480         GEM_BUG_ON(i != n_pages);
3481         st->nents = 0;
3482         sg = st->sgl;
3483
3484         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++) {
3485                 sg = rotate_pages(page_addr_list, rot_info->plane[i].offset,
3486                                   rot_info->plane[i].width, rot_info->plane[i].height,
3487                                   rot_info->plane[i].stride, st, sg);
3488         }
3489
3490         DRM_DEBUG_KMS("Created rotated page mapping for object size %zu (%ux%u tiles, %u pages)\n",
3491                       obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size);
3492
3493         drm_free_large(page_addr_list);
3494
3495         return st;
3496
3497 err_sg_alloc:
3498         kfree(st);
3499 err_st_alloc:
3500         drm_free_large(page_addr_list);
3501
3502         DRM_DEBUG_KMS("Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n",
3503                       obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size);
3504
3505         return ERR_PTR(ret);
3506 }
3507
3508 static noinline struct sg_table *
3509 intel_partial_pages(const struct i915_ggtt_view *view,
3510                     struct drm_i915_gem_object *obj)
3511 {
3512         struct sg_table *st;
3513         struct scatterlist *sg, *iter;
3514         unsigned int count = view->partial.size;
3515         unsigned int offset;
3516         int ret = -ENOMEM;
3517
3518         st = kmalloc(sizeof(*st), GFP_KERNEL);
3519         if (!st)
3520                 goto err_st_alloc;
3521
3522         ret = sg_alloc_table(st, count, GFP_KERNEL);
3523         if (ret)
3524                 goto err_sg_alloc;
3525
3526         iter = i915_gem_object_get_sg(obj, view->partial.offset, &offset);
3527         GEM_BUG_ON(!iter);
3528
3529         sg = st->sgl;
3530         st->nents = 0;
3531         do {
3532                 unsigned int len;
3533
3534                 len = min(iter->length - (offset << PAGE_SHIFT),
3535                           count << PAGE_SHIFT);
3536                 sg_set_page(sg, NULL, len, 0);
3537                 sg_dma_address(sg) =
3538                         sg_dma_address(iter) + (offset << PAGE_SHIFT);
3539                 sg_dma_len(sg) = len;
3540
3541                 st->nents++;
3542                 count -= len >> PAGE_SHIFT;
3543                 if (count == 0) {
3544                         sg_mark_end(sg);
3545                         return st;
3546                 }
3547
3548                 sg = __sg_next(sg);
3549                 iter = __sg_next(iter);
3550                 offset = 0;
3551         } while (1);
3552
3553 err_sg_alloc:
3554         kfree(st);
3555 err_st_alloc:
3556         return ERR_PTR(ret);
3557 }
3558
3559 static int
3560 i915_get_ggtt_vma_pages(struct i915_vma *vma)
3561 {
3562         int ret;
3563
3564         /* The vma->pages are only valid within the lifespan of the borrowed
3565          * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so
3566          * must be the vma->pages. A simple rule is that vma->pages must only
3567          * be accessed when the obj->mm.pages are pinned.
3568          */
3569         GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj));
3570
3571         switch (vma->ggtt_view.type) {
3572         case I915_GGTT_VIEW_NORMAL:
3573                 vma->pages = vma->obj->mm.pages;
3574                 return 0;
3575
3576         case I915_GGTT_VIEW_ROTATED:
3577                 vma->pages =
3578                         intel_rotate_pages(&vma->ggtt_view.rotated, vma->obj);
3579                 break;
3580
3581         case I915_GGTT_VIEW_PARTIAL:
3582                 vma->pages = intel_partial_pages(&vma->ggtt_view, vma->obj);
3583                 break;
3584
3585         default:
3586                 WARN_ONCE(1, "GGTT view %u not implemented!\n",
3587                           vma->ggtt_view.type);
3588                 return -EINVAL;
3589         }
3590
3591         ret = 0;
3592         if (unlikely(IS_ERR(vma->pages))) {
3593                 ret = PTR_ERR(vma->pages);
3594                 vma->pages = NULL;
3595                 DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n",
3596                           vma->ggtt_view.type, ret);
3597         }
3598         return ret;
3599 }
3600
3601 /**
3602  * i915_gem_gtt_reserve - reserve a node in an address_space (GTT)
3603  * @vm: the &struct i915_address_space
3604  * @node: the &struct drm_mm_node (typically i915_vma.mode)
3605  * @size: how much space to allocate inside the GTT,
3606  *        must be #I915_GTT_PAGE_SIZE aligned
3607  * @offset: where to insert inside the GTT,
3608  *          must be #I915_GTT_MIN_ALIGNMENT aligned, and the node
3609  *          (@offset + @size) must fit within the address space
3610  * @color: color to apply to node, if this node is not from a VMA,
3611  *         color must be #I915_COLOR_UNEVICTABLE
3612  * @flags: control search and eviction behaviour
3613  *
3614  * i915_gem_gtt_reserve() tries to insert the @node at the exact @offset inside
3615  * the address space (using @size and @color). If the @node does not fit, it
3616  * tries to evict any overlapping nodes from the GTT, including any
3617  * neighbouring nodes if the colors do not match (to ensure guard pages between
3618  * differing domains). See i915_gem_evict_for_node() for the gory details
3619  * on the eviction algorithm. #PIN_NONBLOCK may used to prevent waiting on
3620  * evicting active overlapping objects, and any overlapping node that is pinned
3621  * or marked as unevictable will also result in failure.
3622  *
3623  * Returns: 0 on success, -ENOSPC if no suitable hole is found, -EINTR if
3624  * asked to wait for eviction and interrupted.
3625  */
3626 int i915_gem_gtt_reserve(struct i915_address_space *vm,
3627                          struct drm_mm_node *node,
3628                          u64 size, u64 offset, unsigned long color,
3629                          unsigned int flags)
3630 {
3631         int err;
3632
3633         GEM_BUG_ON(!size);
3634         GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
3635         GEM_BUG_ON(!IS_ALIGNED(offset, I915_GTT_MIN_ALIGNMENT));
3636         GEM_BUG_ON(range_overflows(offset, size, vm->total));
3637         GEM_BUG_ON(vm == &vm->i915->mm.aliasing_ppgtt->base);
3638         GEM_BUG_ON(drm_mm_node_allocated(node));
3639
3640         node->size = size;
3641         node->start = offset;
3642         node->color = color;
3643
3644         err = drm_mm_reserve_node(&vm->mm, node);
3645         if (err != -ENOSPC)
3646                 return err;
3647
3648         err = i915_gem_evict_for_node(vm, node, flags);
3649         if (err == 0)
3650                 err = drm_mm_reserve_node(&vm->mm, node);
3651
3652         return err;
3653 }
3654
3655 static u64 random_offset(u64 start, u64 end, u64 len, u64 align)
3656 {
3657         u64 range, addr;
3658
3659         GEM_BUG_ON(range_overflows(start, len, end));
3660         GEM_BUG_ON(round_up(start, align) > round_down(end - len, align));
3661
3662         range = round_down(end - len, align) - round_up(start, align);
3663         if (range) {
3664                 if (sizeof(unsigned long) == sizeof(u64)) {
3665                         addr = get_random_long();
3666                 } else {
3667                         addr = get_random_int();
3668                         if (range > U32_MAX) {
3669                                 addr <<= 32;
3670                                 addr |= get_random_int();
3671                         }
3672                 }
3673                 div64_u64_rem(addr, range, &addr);
3674                 start += addr;
3675         }
3676
3677         return round_up(start, align);
3678 }
3679
3680 /**
3681  * i915_gem_gtt_insert - insert a node into an address_space (GTT)
3682  * @vm: the &struct i915_address_space
3683  * @node: the &struct drm_mm_node (typically i915_vma.node)
3684  * @size: how much space to allocate inside the GTT,
3685  *        must be #I915_GTT_PAGE_SIZE aligned
3686  * @alignment: required alignment of starting offset, may be 0 but
3687  *             if specified, this must be a power-of-two and at least
3688  *             #I915_GTT_MIN_ALIGNMENT
3689  * @color: color to apply to node
3690  * @start: start of any range restriction inside GTT (0 for all),
3691  *         must be #I915_GTT_PAGE_SIZE aligned
3692  * @end: end of any range restriction inside GTT (U64_MAX for all),
3693  *       must be #I915_GTT_PAGE_SIZE aligned if not U64_MAX
3694  * @flags: control search and eviction behaviour
3695  *
3696  * i915_gem_gtt_insert() first searches for an available hole into which
3697  * is can insert the node. The hole address is aligned to @alignment and
3698  * its @size must then fit entirely within the [@start, @end] bounds. The
3699  * nodes on either side of the hole must match @color, or else a guard page
3700  * will be inserted between the two nodes (or the node evicted). If no
3701  * suitable hole is found, first a victim is randomly selected and tested
3702  * for eviction, otherwise then the LRU list of objects within the GTT
3703  * is scanned to find the first set of replacement nodes to create the hole.
3704  * Those old overlapping nodes are evicted from the GTT (and so must be
3705  * rebound before any future use). Any node that is currently pinned cannot
3706  * be evicted (see i915_vma_pin()). Similar if the node's VMA is currently
3707  * active and #PIN_NONBLOCK is specified, that node is also skipped when
3708  * searching for an eviction candidate. See i915_gem_evict_something() for
3709  * the gory details on the eviction algorithm.
3710  *
3711  * Returns: 0 on success, -ENOSPC if no suitable hole is found, -EINTR if
3712  * asked to wait for eviction and interrupted.
3713  */
3714 int i915_gem_gtt_insert(struct i915_address_space *vm,
3715                         struct drm_mm_node *node,
3716                         u64 size, u64 alignment, unsigned long color,
3717                         u64 start, u64 end, unsigned int flags)
3718 {
3719         enum drm_mm_insert_mode mode;
3720         u64 offset;
3721         int err;
3722
3723         lockdep_assert_held(&vm->i915->drm.struct_mutex);
3724         GEM_BUG_ON(!size);
3725         GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
3726         GEM_BUG_ON(alignment && !is_power_of_2(alignment));
3727         GEM_BUG_ON(alignment && !IS_ALIGNED(alignment, I915_GTT_MIN_ALIGNMENT));
3728         GEM_BUG_ON(start >= end);
3729         GEM_BUG_ON(start > 0  && !IS_ALIGNED(start, I915_GTT_PAGE_SIZE));
3730         GEM_BUG_ON(end < U64_MAX && !IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
3731         GEM_BUG_ON(vm == &vm->i915->mm.aliasing_ppgtt->base);
3732         GEM_BUG_ON(drm_mm_node_allocated(node));
3733
3734         if (unlikely(range_overflows(start, size, end)))
3735                 return -ENOSPC;
3736
3737         if (unlikely(round_up(start, alignment) > round_down(end - size, alignment)))
3738                 return -ENOSPC;
3739
3740         mode = DRM_MM_INSERT_BEST;
3741         if (flags & PIN_HIGH)
3742                 mode = DRM_MM_INSERT_HIGH;
3743         if (flags & PIN_MAPPABLE)
3744                 mode = DRM_MM_INSERT_LOW;
3745
3746         /* We only allocate in PAGE_SIZE/GTT_PAGE_SIZE (4096) chunks,
3747          * so we know that we always have a minimum alignment of 4096.
3748          * The drm_mm range manager is optimised to return results
3749          * with zero alignment, so where possible use the optimal
3750          * path.
3751          */
3752         BUILD_BUG_ON(I915_GTT_MIN_ALIGNMENT > I915_GTT_PAGE_SIZE);
3753         if (alignment <= I915_GTT_MIN_ALIGNMENT)
3754                 alignment = 0;
3755
3756         err = drm_mm_insert_node_in_range(&vm->mm, node,
3757                                           size, alignment, color,
3758                                           start, end, mode);
3759         if (err != -ENOSPC)
3760                 return err;
3761
3762         /* No free space, pick a slot at random.
3763          *
3764          * There is a pathological case here using a GTT shared between
3765          * mmap and GPU (i.e. ggtt/aliasing_ppgtt but not full-ppgtt):
3766          *
3767          *    |<-- 256 MiB aperture -->||<-- 1792 MiB unmappable -->|
3768          *         (64k objects)             (448k objects)
3769          *
3770          * Now imagine that the eviction LRU is ordered top-down (just because
3771          * pathology meets real life), and that we need to evict an object to
3772          * make room inside the aperture. The eviction scan then has to walk
3773          * the 448k list before it finds one within range. And now imagine that
3774          * it has to search for a new hole between every byte inside the memcpy,
3775          * for several simultaneous clients.
3776          *
3777          * On a full-ppgtt system, if we have run out of available space, there
3778          * will be lots and lots of objects in the eviction list! Again,
3779          * searching that LRU list may be slow if we are also applying any
3780          * range restrictions (e.g. restriction to low 4GiB) and so, for
3781          * simplicity and similarilty between different GTT, try the single
3782          * random replacement first.
3783          */
3784         offset = random_offset(start, end,
3785                                size, alignment ?: I915_GTT_MIN_ALIGNMENT);
3786         err = i915_gem_gtt_reserve(vm, node, size, offset, color, flags);
3787         if (err != -ENOSPC)
3788                 return err;
3789
3790         /* Randomly selected placement is pinned, do a search */
3791         err = i915_gem_evict_something(vm, size, alignment, color,
3792                                        start, end, flags);
3793         if (err)
3794                 return err;
3795
3796         return drm_mm_insert_node_in_range(&vm->mm, node,
3797                                            size, alignment, color,
3798                                            start, end, DRM_MM_INSERT_EVICT);
3799 }
3800
3801 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
3802 #include "selftests/mock_gtt.c"
3803 #include "selftests/i915_gem_gtt.c"
3804 #endif