drivers/iommu/intel-iommu.c

   1 /*
   2  * Copyright © 2006-2014 Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * Authors: David Woodhouse <dwmw2@infradead.org>,
  14  *          Ashok Raj <ashok.raj@intel.com>,
  15  *          Shaohua Li <shaohua.li@intel.com>,
  16  *          Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
  17  *          Fenghua Yu <fenghua.yu@intel.com>
  18  *          Joerg Roedel <jroedel@suse.de>
  19  */
  20
  21 #define pr_fmt(fmt)     "DMAR: " fmt
  22
  23 #include <linux/init.h>
  24 #include <linux/bitmap.h>
  25 #include <linux/debugfs.h>
  26 #include <linux/export.h>
  27 #include <linux/slab.h>
  28 #include <linux/irq.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/spinlock.h>
  31 #include <linux/pci.h>
  32 #include <linux/dmar.h>
  33 #include <linux/dma-mapping.h>
  34 #include <linux/mempool.h>
  35 #include <linux/memory.h>
  36 #include <linux/cpu.h>
  37 #include <linux/timer.h>
  38 #include <linux/io.h>
  39 #include <linux/iova.h>
  40 #include <linux/iommu.h>
  41 #include <linux/intel-iommu.h>
  42 #include <linux/syscore_ops.h>
  43 #include <linux/tboot.h>
  44 #include <linux/dmi.h>
  45 #include <linux/pci-ats.h>
  46 #include <linux/memblock.h>
  47 #include <linux/dma-contiguous.h>
  48 #include <linux/crash_dump.h>
  49 #include <asm/irq_remapping.h>
  50 #include <asm/cacheflush.h>
  51 #include <asm/iommu.h>
  52
  53 #include "irq_remapping.h"
  54
  55 #define ROOT_SIZE               VTD_PAGE_SIZE
  56 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  57
  58 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  59 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
  60 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  61 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
  62
  63 #define IOAPIC_RANGE_START      (0xfee00000)
  64 #define IOAPIC_RANGE_END        (0xfeefffff)
  65 #define IOVA_START_ADDR         (0x1000)
  66
  67 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
  68
  69 #define MAX_AGAW_WIDTH 64
  70 #define MAX_AGAW_PFN_WIDTH      (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
  71
  72 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
  73 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
  74
  75 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
  76    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
  77 #define DOMAIN_MAX_PFN(gaw)     ((unsigned long) min_t(uint64_t, \
  78                                 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
  79 #define DOMAIN_MAX_ADDR(gaw)    (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
  80
  81 /* IO virtual address start page frame number */
  82 #define IOVA_START_PFN          (1)
  83
  84 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  85 #define DMA_32BIT_PFN           IOVA_PFN(DMA_BIT_MASK(32))
  86 #define DMA_64BIT_PFN           IOVA_PFN(DMA_BIT_MASK(64))
  87
  88 /* page table handling */
  89 #define LEVEL_STRIDE            (9)
  90 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
  91
  92 /*
  93  * This bitmap is used to advertise the page sizes our hardware support
  94  * to the IOMMU core, which will then use this information to split
  95  * physically contiguous memory regions it is mapping into page sizes
  96  * that we support.
  97  *
  98  * Traditionally the IOMMU core just handed us the mappings directly,
  99  * after making sure the size is an order of a 4KiB page and that the
 100  * mapping has natural alignment.
 101  *
 102  * To retain this behavior, we currently advertise that we support
 103  * all page sizes that are an order of 4KiB.
 104  *
 105  * If at some point we'd like to utilize the IOMMU core's new behavior,
 106  * we could change this to advertise the real page sizes we support.
 107  */
 108 #define INTEL_IOMMU_PGSIZES     (~0xFFFUL)
 109
 110 static inline int agaw_to_level(int agaw)
 111 {
 112         return agaw + 2;
 113 }
 114
 115 static inline int agaw_to_width(int agaw)
 116 {
 117         return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
 118 }
 119
 120 static inline int width_to_agaw(int width)
 121 {
 122         return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
 123 }
 124
 125 static inline unsigned int level_to_offset_bits(int level)
 126 {
 127         return (level - 1) * LEVEL_STRIDE;
 128 }
 129
 130 static inline int pfn_level_offset(unsigned long pfn, int level)
 131 {
 132         return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
 133 }
 134
 135 static inline unsigned long level_mask(int level)
 136 {
 137         return -1UL << level_to_offset_bits(level);
 138 }
 139
 140 static inline unsigned long level_size(int level)
 141 {
 142         return 1UL << level_to_offset_bits(level);
 143 }
 144
 145 static inline unsigned long align_to_level(unsigned long pfn, int level)
 146 {
 147         return (pfn + level_size(level) - 1) & level_mask(level);
 148 }
 149
 150 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
 151 {
 152         return  1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
 153 }
 154
 155 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
 156    are never going to work. */
 157 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
 158 {
 159         return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
 160 }
 161
 162 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
 163 {
 164         return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
 165 }
 166 static inline unsigned long page_to_dma_pfn(struct page *pg)
 167 {
 168         return mm_to_dma_pfn(page_to_pfn(pg));
 169 }
 170 static inline unsigned long virt_to_dma_pfn(void *p)
 171 {
 172         return page_to_dma_pfn(virt_to_page(p));
 173 }
 174
 175 /* global iommu list, set NULL for ignored DMAR units */
 176 static struct intel_iommu **g_iommus;
 177
 178 static void __init check_tylersburg_isoch(void);
 179 static int rwbf_quirk;
 180
 181 /*
 182  * set to 1 to panic kernel if can't successfully enable VT-d
 183  * (used when kernel is launched w/ TXT)
 184  */
 185 static int force_on = 0;
 186
 187 /*
 188  * 0: Present
 189  * 1-11: Reserved
 190  * 12-63: Context Ptr (12 - (haw-1))
 191  * 64-127: Reserved
 192  */
 193 struct root_entry {
 194         u64     lo;
 195         u64     hi;
 196 };
 197 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
 198
 199 /*
 200  * Take a root_entry and return the Lower Context Table Pointer (LCTP)
 201  * if marked present.
 202  */
 203 static phys_addr_t root_entry_lctp(struct root_entry *re)
 204 {
 205         if (!(re->lo & 1))
 206                 return 0;
 207
 208         return re->lo & VTD_PAGE_MASK;
 209 }
 210
 211 /*
 212  * Take a root_entry and return the Upper Context Table Pointer (UCTP)
 213  * if marked present.
 214  */
 215 static phys_addr_t root_entry_uctp(struct root_entry *re)
 216 {
 217         if (!(re->hi & 1))
 218                 return 0;
 219
 220         return re->hi & VTD_PAGE_MASK;
 221 }
 222 /*
 223  * low 64 bits:
 224  * 0: present
 225  * 1: fault processing disable
 226  * 2-3: translation type
 227  * 12-63: address space root
 228  * high 64 bits:
 229  * 0-2: address width
 230  * 3-6: aval
 231  * 8-23: domain id
 232  */
 233 struct context_entry {
 234         u64 lo;
 235         u64 hi;
 236 };
 237
 238 static inline void context_clear_pasid_enable(struct context_entry *context)
 239 {
 240         context->lo &= ~(1ULL << 11);
 241 }
 242
 243 static inline bool context_pasid_enabled(struct context_entry *context)
 244 {
 245         return !!(context->lo & (1ULL << 11));
 246 }
 247
 248 static inline void context_set_copied(struct context_entry *context)
 249 {
 250         context->hi |= (1ull << 3);
 251 }
 252
 253 static inline bool context_copied(struct context_entry *context)
 254 {
 255         return !!(context->hi & (1ULL << 3));
 256 }
 257
 258 static inline bool __context_present(struct context_entry *context)
 259 {
 260         return (context->lo & 1);
 261 }
 262
 263 static inline bool context_present(struct context_entry *context)
 264 {
 265         return context_pasid_enabled(context) ?
 266              __context_present(context) :
 267              __context_present(context) && !context_copied(context);
 268 }
 269
 270 static inline void context_set_present(struct context_entry *context)
 271 {
 272         context->lo |= 1;
 273 }
 274
 275 static inline void context_set_fault_enable(struct context_entry *context)
 276 {
 277         context->lo &= (((u64)-1) << 2) | 1;
 278 }
 279
 280 static inline void context_set_translation_type(struct context_entry *context,
 281                                                 unsigned long value)
 282 {
 283         context->lo &= (((u64)-1) << 4) | 3;
 284         context->lo |= (value & 3) << 2;
 285 }
 286
 287 static inline void context_set_address_root(struct context_entry *context,
 288                                             unsigned long value)
 289 {
 290         context->lo &= ~VTD_PAGE_MASK;
 291         context->lo |= value & VTD_PAGE_MASK;
 292 }
 293
 294 static inline void context_set_address_width(struct context_entry *context,
 295                                              unsigned long value)
 296 {
 297         context->hi |= value & 7;
 298 }
 299
 300 static inline void context_set_domain_id(struct context_entry *context,
 301                                          unsigned long value)
 302 {
 303         context->hi |= (value & ((1 << 16) - 1)) << 8;
 304 }
 305
 306 static inline int context_domain_id(struct context_entry *c)
 307 {
 308         return((c->hi >> 8) & 0xffff);
 309 }
 310
 311 static inline void context_clear_entry(struct context_entry *context)
 312 {
 313         context->lo = 0;
 314         context->hi = 0;
 315 }
 316
 317 /*
 318  * 0: readable
 319  * 1: writable
 320  * 2-6: reserved
 321  * 7: super page
 322  * 8-10: available
 323  * 11: snoop behavior
 324  * 12-63: Host physcial address
 325  */
 326 struct dma_pte {
 327         u64 val;
 328 };
 329
 330 static inline void dma_clear_pte(struct dma_pte *pte)
 331 {
 332         pte->val = 0;
 333 }
 334
 335 static inline u64 dma_pte_addr(struct dma_pte *pte)
 336 {
 337 #ifdef CONFIG_64BIT
 338         return pte->val & VTD_PAGE_MASK;
 339 #else
 340         /* Must have a full atomic 64-bit read */
 341         return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
 342 #endif
 343 }
 344
 345 static inline bool dma_pte_present(struct dma_pte *pte)
 346 {
 347         return (pte->val & 3) != 0;
 348 }
 349
 350 static inline bool dma_pte_superpage(struct dma_pte *pte)
 351 {
 352         return (pte->val & DMA_PTE_LARGE_PAGE);
 353 }
 354
 355 static inline int first_pte_in_page(struct dma_pte *pte)
 356 {
 357         return !((unsigned long)pte & ~VTD_PAGE_MASK);
 358 }
 359
 360 /*
 361  * This domain is a statically identity mapping domain.
 362  *      1. This domain creats a static 1:1 mapping to all usable memory.
 363  *      2. It maps to each iommu if successful.
 364  *      3. Each iommu mapps to this domain if successful.
 365  */
 366 static struct dmar_domain *si_domain;
 367 static int hw_pass_through = 1;
 368
 369 /*
 370  * Domain represents a virtual machine, more than one devices
 371  * across iommus may be owned in one domain, e.g. kvm guest.
 372  */
 373 #define DOMAIN_FLAG_VIRTUAL_MACHINE     (1 << 0)
 374
 375 /* si_domain contains mulitple devices */
 376 #define DOMAIN_FLAG_STATIC_IDENTITY     (1 << 1)
 377
 378 #define for_each_domain_iommu(idx, domain)                      \
 379         for (idx = 0; idx < g_num_of_iommus; idx++)             \
 380                 if (domain->iommu_refcnt[idx])
 381
 382 struct dmar_domain {
 383         int     nid;                    /* node id */
 384
 385         unsigned        iommu_refcnt[DMAR_UNITS_SUPPORTED];
 386                                         /* Refcount of devices per iommu */
 387
 388
 389         u16             iommu_did[DMAR_UNITS_SUPPORTED];
 390                                         /* Domain ids per IOMMU. Use u16 since
 391                                          * domain ids are 16 bit wide according
 392                                          * to VT-d spec, section 9.3 */
 393
 394         bool has_iotlb_device;
 395         struct list_head devices;       /* all devices' list */
 396         struct iova_domain iovad;       /* iova's that belong to this domain */
 397
 398         struct dma_pte  *pgd;           /* virtual address */
 399         int             gaw;            /* max guest address width */
 400
 401         /* adjusted guest address width, 0 is level 2 30-bit */
 402         int             agaw;
 403
 404         int             flags;          /* flags to find out type of domain */
 405
 406         int             iommu_coherency;/* indicate coherency of iommu access */
 407         int             iommu_snooping; /* indicate snooping control feature*/
 408         int             iommu_count;    /* reference count of iommu */
 409         int             iommu_superpage;/* Level of superpages supported:
 410                                            0 == 4KiB (no superpages), 1 == 2MiB,
 411                                            2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
 412         u64             max_addr;       /* maximum mapped address */
 413
 414         struct iommu_domain domain;     /* generic domain data structure for
 415                                            iommu core */
 416 };
 417
 418 /* PCI domain-device relationship */
 419 struct device_domain_info {
 420         struct list_head link;  /* link to domain siblings */
 421         struct list_head global; /* link to global list */
 422         u8 bus;                 /* PCI bus number */
 423         u8 devfn;               /* PCI devfn number */
 424         u16 pfsid;              /* SRIOV physical function source ID */
 425         u8 pasid_supported:3;
 426         u8 pasid_enabled:1;
 427         u8 pri_supported:1;
 428         u8 pri_enabled:1;
 429         u8 ats_supported:1;
 430         u8 ats_enabled:1;
 431         u8 ats_qdep;
 432         struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
 433         struct intel_iommu *iommu; /* IOMMU used by this device */
 434         struct dmar_domain *domain; /* pointer to domain */
 435 };
 436
 437 struct dmar_rmrr_unit {
 438         struct list_head list;          /* list of rmrr units   */
 439         struct acpi_dmar_header *hdr;   /* ACPI header          */
 440         u64     base_address;           /* reserved base address*/
 441         u64     end_address;            /* reserved end address */
 442         struct dmar_dev_scope *devices; /* target devices */
 443         int     devices_cnt;            /* target device count */
 444 };
 445
 446 struct dmar_atsr_unit {
 447         struct list_head list;          /* list of ATSR units */
 448         struct acpi_dmar_header *hdr;   /* ACPI header */
 449         struct dmar_dev_scope *devices; /* target devices */
 450         int devices_cnt;                /* target device count */
 451         u8 include_all:1;               /* include all ports */
 452 };
 453
 454 static LIST_HEAD(dmar_atsr_units);
 455 static LIST_HEAD(dmar_rmrr_units);
 456
 457 #define for_each_rmrr_units(rmrr) \
 458         list_for_each_entry(rmrr, &dmar_rmrr_units, list)
 459
 460 static void flush_unmaps_timeout(unsigned long data);
 461
 462 struct deferred_flush_entry {
 463         unsigned long iova_pfn;
 464         unsigned long nrpages;
 465         struct dmar_domain *domain;
 466         struct page *freelist;
 467 };
 468
 469 #define HIGH_WATER_MARK 250
 470 struct deferred_flush_table {
 471         int next;
 472         struct deferred_flush_entry entries[HIGH_WATER_MARK];
 473 };
 474
 475 struct deferred_flush_data {
 476         spinlock_t lock;
 477         int timer_on;
 478         struct timer_list timer;
 479         long size;
 480         struct deferred_flush_table *tables;
 481 };
 482
 483 DEFINE_PER_CPU(struct deferred_flush_data, deferred_flush);
 484
 485 /* bitmap for indexing intel_iommus */
 486 static int g_num_of_iommus;
 487
 488 static void domain_exit(struct dmar_domain *domain);
 489 static void domain_remove_dev_info(struct dmar_domain *domain);
 490 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
 491                                      struct device *dev);
 492 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
 493 static void domain_context_clear(struct intel_iommu *iommu,
 494                                  struct device *dev);
 495 static int domain_detach_iommu(struct dmar_domain *domain,
 496                                struct intel_iommu *iommu);
 497
 498 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
 499 int dmar_disabled = 0;
 500 #else
 501 int dmar_disabled = 1;
 502 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
 503
 504 int intel_iommu_enabled = 0;
 505 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
 506
 507 static int dmar_map_gfx = 1;
 508 static int dmar_forcedac;
 509 static int intel_iommu_strict;
 510 static int intel_iommu_superpage = 1;
 511 static int intel_iommu_ecs = 1;
 512 static int intel_iommu_pasid28;
 513 static int iommu_identity_mapping;
 514
 515 #define IDENTMAP_ALL            1
 516 #define IDENTMAP_GFX            2
 517 #define IDENTMAP_AZALIA         4
 518
 519 /* Broadwell and Skylake have broken ECS support — normal so-called "second
 520  * level" translation of DMA requests-without-PASID doesn't actually happen
 521  * unless you also set the NESTE bit in an extended context-entry. Which of
 522  * course means that SVM doesn't work because it's trying to do nested
 523  * translation of the physical addresses it finds in the process page tables,
 524  * through the IOVA->phys mapping found in the "second level" page tables.
 525  *
 526  * The VT-d specification was retroactively changed to change the definition
 527  * of the capability bits and pretend that Broadwell/Skylake never happened...
 528  * but unfortunately the wrong bit was changed. It's ECS which is broken, but
 529  * for some reason it was the PASID capability bit which was redefined (from
 530  * bit 28 on BDW/SKL to bit 40 in future).
 531  *
 532  * So our test for ECS needs to eschew those implementations which set the old
 533  * PASID capabiity bit 28, since those are the ones on which ECS is broken.
 534  * Unless we are working around the 'pasid28' limitations, that is, by putting
 535  * the device into passthrough mode for normal DMA and thus masking the bug.
 536  */
 537 #define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
 538                             (intel_iommu_pasid28 || !ecap_broken_pasid(iommu->ecap)))
 539 /* PASID support is thus enabled if ECS is enabled and *either* of the old
 540  * or new capability bits are set. */
 541 #define pasid_enabled(iommu) (ecs_enabled(iommu) &&                     \
 542                               (ecap_pasid(iommu->ecap) || ecap_broken_pasid(iommu->ecap)))
 543
 544 int intel_iommu_gfx_mapped;
 545 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
 546
 547 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 548 static DEFINE_SPINLOCK(device_domain_lock);
 549 static LIST_HEAD(device_domain_list);
 550
 551 static const struct iommu_ops intel_iommu_ops;
 552
 553 static bool translation_pre_enabled(struct intel_iommu *iommu)
 554 {
 555         return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
 556 }
 557
 558 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
 559 {
 560         iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
 561 }
 562
 563 static void init_translation_status(struct intel_iommu *iommu)
 564 {
 565         u32 gsts;
 566
 567         gsts = readl(iommu->reg + DMAR_GSTS_REG);
 568         if (gsts & DMA_GSTS_TES)
 569                 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
 570 }
 571
 572 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
 573 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
 574 {
 575         return container_of(dom, struct dmar_domain, domain);
 576 }
 577
 578 static int __init intel_iommu_setup(char *str)
 579 {
 580         if (!str)
 581                 return -EINVAL;
 582         while (*str) {
 583                 if (!strncmp(str, "on", 2)) {
 584                         dmar_disabled = 0;
 585                         pr_info("IOMMU enabled\n");
 586                 } else if (!strncmp(str, "off", 3)) {
 587                         dmar_disabled = 1;
 588                         pr_info("IOMMU disabled\n");
 589                 } else if (!strncmp(str, "igfx_off", 8)) {
 590                         dmar_map_gfx = 0;
 591                         pr_info("Disable GFX device mapping\n");
 592                 } else if (!strncmp(str, "forcedac", 8)) {
 593                         pr_info("Forcing DAC for PCI devices\n");
 594                         dmar_forcedac = 1;
 595                 } else if (!strncmp(str, "strict", 6)) {
 596                         pr_info("Disable batched IOTLB flush\n");
 597                         intel_iommu_strict = 1;
 598                 } else if (!strncmp(str, "sp_off", 6)) {
 599                         pr_info("Disable supported super page\n");
 600                         intel_iommu_superpage = 0;
 601                 } else if (!strncmp(str, "ecs_off", 7)) {
 602                         printk(KERN_INFO
 603                                 "Intel-IOMMU: disable extended context table support\n");
 604                         intel_iommu_ecs = 0;
 605                 } else if (!strncmp(str, "pasid28", 7)) {
 606                         printk(KERN_INFO
 607                                 "Intel-IOMMU: enable pre-production PASID support\n");
 608                         intel_iommu_pasid28 = 1;
 609                         iommu_identity_mapping |= IDENTMAP_GFX;
 610                 }
 611
 612                 str += strcspn(str, ",");
 613                 while (*str == ',')
 614                         str++;
 615         }
 616         return 0;
 617 }
 618 __setup("intel_iommu=", intel_iommu_setup);
 619
 620 static struct kmem_cache *iommu_domain_cache;
 621 static struct kmem_cache *iommu_devinfo_cache;
 622
 623 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
 624 {
 625         struct dmar_domain **domains;
 626         int idx = did >> 8;
 627
 628         domains = iommu->domains[idx];
 629         if (!domains)
 630                 return NULL;
 631
 632         return domains[did & 0xff];
 633 }
 634
 635 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
 636                              struct dmar_domain *domain)
 637 {
 638         struct dmar_domain **domains;
 639         int idx = did >> 8;
 640
 641         if (!iommu->domains[idx]) {
 642                 size_t size = 256 * sizeof(struct dmar_domain *);
 643                 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
 644         }
 645
 646         domains = iommu->domains[idx];
 647         if (WARN_ON(!domains))
 648                 return;
 649         else
 650                 domains[did & 0xff] = domain;
 651 }
 652
 653 static inline void *alloc_pgtable_page(int node)
 654 {
 655         struct page *page;
 656         void *vaddr = NULL;
 657
 658         page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
 659         if (page)
 660                 vaddr = page_address(page);
 661         return vaddr;
 662 }
 663
 664 static inline void free_pgtable_page(void *vaddr)
 665 {
 666         free_page((unsigned long)vaddr);
 667 }
 668
 669 static inline void *alloc_domain_mem(void)
 670 {
 671         return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
 672 }
 673
 674 static void free_domain_mem(void *vaddr)
 675 {
 676         kmem_cache_free(iommu_domain_cache, vaddr);
 677 }
 678
 679 static inline void * alloc_devinfo_mem(void)
 680 {
 681         return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
 682 }
 683
 684 static inline void free_devinfo_mem(void *vaddr)
 685 {
 686         kmem_cache_free(iommu_devinfo_cache, vaddr);
 687 }
 688
 689 static inline int domain_type_is_vm(struct dmar_domain *domain)
 690 {
 691         return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
 692 }
 693
 694 static inline int domain_type_is_si(struct dmar_domain *domain)
 695 {
 696         return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
 697 }
 698
 699 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
 700 {
 701         return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
 702                                 DOMAIN_FLAG_STATIC_IDENTITY);
 703 }
 704
 705 static inline int domain_pfn_supported(struct dmar_domain *domain,
 706                                        unsigned long pfn)
 707 {
 708         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 709
 710         return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
 711 }
 712
 713 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
 714 {
 715         unsigned long sagaw;
 716         int agaw = -1;
 717
 718         sagaw = cap_sagaw(iommu->cap);
 719         for (agaw = width_to_agaw(max_gaw);
 720              agaw >= 0; agaw--) {
 721                 if (test_bit(agaw, &sagaw))
 722                         break;
 723         }
 724
 725         return agaw;
 726 }
 727
 728 /*
 729  * Calculate max SAGAW for each iommu.
 730  */
 731 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
 732 {
 733         return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
 734 }
 735
 736 /*
 737  * calculate agaw for each iommu.
 738  * "SAGAW" may be different across iommus, use a default agaw, and
 739  * get a supported less agaw for iommus that don't support the default agaw.
 740  */
 741 int iommu_calculate_agaw(struct intel_iommu *iommu)
 742 {
 743         return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 744 }
 745
 746 /* This functionin only returns single iommu in a domain */
 747 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
 748 {
 749         int iommu_id;
 750
 751         /* si_domain and vm domain should not get here. */
 752         BUG_ON(domain_type_is_vm_or_si(domain));
 753         for_each_domain_iommu(iommu_id, domain)
 754                 break;
 755
 756         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
 757                 return NULL;
 758
 759         return g_iommus[iommu_id];
 760 }
 761
 762 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 763 {
 764         struct dmar_drhd_unit *drhd;
 765         struct intel_iommu *iommu;
 766         bool found = false;
 767         int i;
 768
 769         domain->iommu_coherency = 1;
 770
 771         for_each_domain_iommu(i, domain) {
 772                 found = true;
 773                 if (!ecap_coherent(g_iommus[i]->ecap)) {
 774                         domain->iommu_coherency = 0;
 775                         break;
 776                 }
 777         }
 778         if (found)
 779                 return;
 780
 781         /* No hardware attached; use lowest common denominator */
 782         rcu_read_lock();
 783         for_each_active_iommu(iommu, drhd) {
 784                 if (!ecap_coherent(iommu->ecap)) {
 785                         domain->iommu_coherency = 0;
 786                         break;
 787                 }
 788         }
 789         rcu_read_unlock();
 790 }
 791
 792 static int domain_update_iommu_snooping(struct intel_iommu *skip)
 793 {
 794         struct dmar_drhd_unit *drhd;
 795         struct intel_iommu *iommu;
 796         int ret = 1;
 797
 798         rcu_read_lock();
 799         for_each_active_iommu(iommu, drhd) {
 800                 if (iommu != skip) {
 801                         if (!ecap_sc_support(iommu->ecap)) {
 802                                 ret = 0;
 803                                 break;
 804                         }
 805                 }
 806         }
 807         rcu_read_unlock();
 808
 809         return ret;
 810 }
 811
 812 static int domain_update_iommu_superpage(struct intel_iommu *skip)
 813 {
 814         struct dmar_drhd_unit *drhd;
 815         struct intel_iommu *iommu;
 816         int mask = 0xf;
 817
 818         if (!intel_iommu_superpage) {
 819                 return 0;
 820         }
 821
 822         /* set iommu_superpage to the smallest common denominator */
 823         rcu_read_lock();
 824         for_each_active_iommu(iommu, drhd) {
 825                 if (iommu != skip) {
 826                         mask &= cap_super_page_val(iommu->cap);
 827                         if (!mask)
 828                                 break;
 829                 }
 830         }
 831         rcu_read_unlock();
 832
 833         return fls(mask);
 834 }
 835
 836 /* Some capabilities may be different across iommus */
 837 static void domain_update_iommu_cap(struct dmar_domain *domain)
 838 {
 839         domain_update_iommu_coherency(domain);
 840         domain->iommu_snooping = domain_update_iommu_snooping(NULL);
 841         domain->iommu_superpage = domain_update_iommu_superpage(NULL);
 842 }
 843
 844 static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
 845                                                        u8 bus, u8 devfn, int alloc)
 846 {
 847         struct root_entry *root = &iommu->root_entry[bus];
 848         struct context_entry *context;
 849         u64 *entry;
 850
 851         entry = &root->lo;
 852         if (ecs_enabled(iommu)) {
 853                 if (devfn >= 0x80) {
 854                         devfn -= 0x80;
 855                         entry = &root->hi;
 856                 }
 857                 devfn *= 2;
 858         }
 859         if (*entry & 1)
 860                 context = phys_to_virt(*entry & VTD_PAGE_MASK);
 861         else {
 862                 unsigned long phy_addr;
 863                 if (!alloc)
 864                         return NULL;
 865
 866                 context = alloc_pgtable_page(iommu->node);
 867                 if (!context)
 868                         return NULL;
 869
 870                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 871                 phy_addr = virt_to_phys((void *)context);
 872                 *entry = phy_addr | 1;
 873                 __iommu_flush_cache(iommu, entry, sizeof(*entry));
 874         }
 875         return &context[devfn];
 876 }
 877
 878 static int iommu_dummy(struct device *dev)
 879 {
 880         return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
 881 }
 882
 883 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
 884 {
 885         struct dmar_drhd_unit *drhd = NULL;
 886         struct intel_iommu *iommu;
 887         struct device *tmp;
 888         struct pci_dev *ptmp, *pdev = NULL;
 889         u16 segment = 0;
 890         int i;
 891
 892         if (iommu_dummy(dev))
 893                 return NULL;
 894
 895         if (dev_is_pci(dev)) {
 896                 struct pci_dev *pf_pdev;
 897
 898                 pdev = to_pci_dev(dev);
 899                 /* VFs aren't listed in scope tables; we need to look up
 900                  * the PF instead to find the IOMMU. */
 901                 pf_pdev = pci_physfn(pdev);
 902                 dev = &pf_pdev->dev;
 903                 segment = pci_domain_nr(pdev->bus);
 904         } else if (has_acpi_companion(dev))
 905                 dev = &ACPI_COMPANION(dev)->dev;
 906
 907         rcu_read_lock();
 908         for_each_active_iommu(iommu, drhd) {
 909                 if (pdev && segment != drhd->segment)
 910                         continue;
 911
 912                 for_each_active_dev_scope(drhd->devices,
 913                                           drhd->devices_cnt, i, tmp) {
 914                         if (tmp == dev) {
 915                                 /* For a VF use its original BDF# not that of the PF
 916                                  * which we used for the IOMMU lookup. Strictly speaking
 917                                  * we could do this for all PCI devices; we only need to
 918                                  * get the BDF# from the scope table for ACPI matches. */
 919                                 if (pdev && pdev->is_virtfn)
 920                                         goto got_pdev;
 921
 922                                 *bus = drhd->devices[i].bus;
 923                                 *devfn = drhd->devices[i].devfn;
 924                                 goto out;
 925                         }
 926
 927                         if (!pdev || !dev_is_pci(tmp))
 928                                 continue;
 929
 930                         ptmp = to_pci_dev(tmp);
 931                         if (ptmp->subordinate &&
 932                             ptmp->subordinate->number <= pdev->bus->number &&
 933                             ptmp->subordinate->busn_res.end >= pdev->bus->number)
 934                                 goto got_pdev;
 935                 }
 936
 937                 if (pdev && drhd->include_all) {
 938                 got_pdev:
 939                         *bus = pdev->bus->number;
 940                         *devfn = pdev->devfn;
 941                         goto out;
 942                 }
 943         }
 944         iommu = NULL;
 945  out:
 946         rcu_read_unlock();
 947
 948         return iommu;
 949 }
 950
 951 static void domain_flush_cache(struct dmar_domain *domain,
 952                                void *addr, int size)
 953 {
 954         if (!domain->iommu_coherency)
 955                 clflush_cache_range(addr, size);
 956 }
 957
 958 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 959 {
 960         struct context_entry *context;
 961         int ret = 0;
 962         unsigned long flags;
 963
 964         spin_lock_irqsave(&iommu->lock, flags);
 965         context = iommu_context_addr(iommu, bus, devfn, 0);
 966         if (context)
 967                 ret = context_present(context);
 968         spin_unlock_irqrestore(&iommu->lock, flags);
 969         return ret;
 970 }
 971
 972 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
 973 {
 974         struct context_entry *context;
 975         unsigned long flags;
 976
 977         spin_lock_irqsave(&iommu->lock, flags);
 978         context = iommu_context_addr(iommu, bus, devfn, 0);
 979         if (context) {
 980                 context_clear_entry(context);
 981                 __iommu_flush_cache(iommu, context, sizeof(*context));
 982         }
 983         spin_unlock_irqrestore(&iommu->lock, flags);
 984 }
 985
 986 static void free_context_table(struct intel_iommu *iommu)
 987 {
 988         int i;
 989         unsigned long flags;
 990         struct context_entry *context;
 991
 992         spin_lock_irqsave(&iommu->lock, flags);
 993         if (!iommu->root_entry) {
 994                 goto out;
 995         }
 996         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 997                 context = iommu_context_addr(iommu, i, 0, 0);
 998                 if (context)
 999                         free_pgtable_page(context);
1000
1001                 if (!ecs_enabled(iommu))
1002                         continue;
1003
1004                 context = iommu_context_addr(iommu, i, 0x80, 0);
1005                 if (context)
1006                         free_pgtable_page(context);
1007
1008         }
1009         free_pgtable_page(iommu->root_entry);
1010         iommu->root_entry = NULL;
1011 out:
1012         spin_unlock_irqrestore(&iommu->lock, flags);
1013 }
1014
1015 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
1016                                       unsigned long pfn, int *target_level)
1017 {
1018         struct dma_pte *parent, *pte = NULL;
1019         int level = agaw_to_level(domain->agaw);
1020         int offset;
1021
1022         BUG_ON(!domain->pgd);
1023
1024         if (!domain_pfn_supported(domain, pfn))
1025                 /* Address beyond IOMMU's addressing capabilities. */
1026                 return NULL;
1027
1028         parent = domain->pgd;
1029
1030         while (1) {
1031                 void *tmp_page;
1032
1033                 offset = pfn_level_offset(pfn, level);
1034                 pte = &parent[offset];
1035                 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
1036                         break;
1037                 if (level == *target_level)
1038                         break;
1039
1040                 if (!dma_pte_present(pte)) {
1041                         uint64_t pteval;
1042
1043                         tmp_page = alloc_pgtable_page(domain->nid);
1044
1045                         if (!tmp_page)
1046                                 return NULL;
1047
1048                         domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
1049                         pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
1050                         if (cmpxchg64(&pte->val, 0ULL, pteval))
1051                                 /* Someone else set it while we were thinking; use theirs. */
1052                                 free_pgtable_page(tmp_page);
1053                         else
1054                                 domain_flush_cache(domain, pte, sizeof(*pte));
1055                 }
1056                 if (level == 1)
1057                         break;
1058
1059                 parent = phys_to_virt(dma_pte_addr(pte));
1060                 level--;
1061         }
1062
1063         if (!*target_level)
1064                 *target_level = level;
1065
1066         return pte;
1067 }
1068
1069
1070 /* return address's pte at specific level */
1071 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1072                                          unsigned long pfn,
1073                                          int level, int *large_page)
1074 {
1075         struct dma_pte *parent, *pte = NULL;
1076         int total = agaw_to_level(domain->agaw);
1077         int offset;
1078
1079         parent = domain->pgd;
1080         while (level <= total) {
1081                 offset = pfn_level_offset(pfn, total);
1082                 pte = &parent[offset];
1083                 if (level == total)
1084                         return pte;
1085
1086                 if (!dma_pte_present(pte)) {
1087                         *large_page = total;
1088                         break;
1089                 }
1090
1091                 if (dma_pte_superpage(pte)) {
1092                         *large_page = total;
1093                         return pte;
1094                 }
1095
1096                 parent = phys_to_virt(dma_pte_addr(pte));
1097                 total--;
1098         }
1099         return NULL;
1100 }
1101
1102 /* clear last level pte, a tlb flush should be followed */
1103 static void dma_pte_clear_range(struct dmar_domain *domain,
1104                                 unsigned long start_pfn,
1105                                 unsigned long last_pfn)
1106 {
1107         unsigned int large_page = 1;
1108         struct dma_pte *first_pte, *pte;
1109
1110         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1111         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1112         BUG_ON(start_pfn > last_pfn);
1113
1114         /* we don't need lock here; nobody else touches the iova range */
1115         do {
1116                 large_page = 1;
1117                 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
1118                 if (!pte) {
1119                         start_pfn = align_to_level(start_pfn + 1, large_page + 1);
1120                         continue;
1121                 }
1122                 do {
1123                         dma_clear_pte(pte);
1124                         start_pfn += lvl_to_nr_pages(large_page);
1125                         pte++;
1126                 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1127
1128                 domain_flush_cache(domain, first_pte,
1129                                    (void *)pte - (void *)first_pte);
1130
1131         } while (start_pfn && start_pfn <= last_pfn);
1132 }
1133
1134 static void dma_pte_free_level(struct dmar_domain *domain, int level,
1135                                struct dma_pte *pte, unsigned long pfn,
1136                                unsigned long start_pfn, unsigned long last_pfn)
1137 {
1138         pfn = max(start_pfn, pfn);
1139         pte = &pte[pfn_level_offset(pfn, level)];
1140
1141         do {
1142                 unsigned long level_pfn;
1143                 struct dma_pte *level_pte;
1144
1145                 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1146                         goto next;
1147
1148                 level_pfn = pfn & level_mask(level);
1149                 level_pte = phys_to_virt(dma_pte_addr(pte));
1150
1151                 if (level > 2)
1152                         dma_pte_free_level(domain, level - 1, level_pte,
1153                                            level_pfn, start_pfn, last_pfn);
1154
1155                 /* If range covers entire pagetable, free it */
1156                 if (!(start_pfn > level_pfn ||
1157                       last_pfn < level_pfn + level_size(level) - 1)) {
1158                         dma_clear_pte(pte);
1159                         domain_flush_cache(domain, pte, sizeof(*pte));
1160                         free_pgtable_page(level_pte);
1161                 }
1162 next:
1163                 pfn += level_size(level);
1164         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1165 }
1166
1167 /* clear last level (leaf) ptes and free page table pages. */
1168 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1169                                    unsigned long start_pfn,
1170                                    unsigned long last_pfn)
1171 {
1172         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1173         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1174         BUG_ON(start_pfn > last_pfn);
1175
1176         dma_pte_clear_range(domain, start_pfn, last_pfn);
1177
1178         /* We don't need lock here; nobody else touches the iova range */
1179         dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1180                            domain->pgd, 0, start_pfn, last_pfn);
1181
1182         /* free pgd */
1183         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1184                 free_pgtable_page(domain->pgd);
1185                 domain->pgd = NULL;
1186         }
1187 }
1188
1189 /* When a page at a given level is being unlinked from its parent, we don't
1190    need to *modify* it at all. All we need to do is make a list of all the
1191    pages which can be freed just as soon as we've flushed the IOTLB and we
1192    know the hardware page-walk will no longer touch them.
1193    The 'pte' argument is the *parent* PTE, pointing to the page that is to
1194    be freed. */
1195 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1196                                             int level, struct dma_pte *pte,
1197                                             struct page *freelist)
1198 {
1199         struct page *pg;
1200
1201         pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1202         pg->freelist = freelist;
1203         freelist = pg;
1204
1205         if (level == 1)
1206                 return freelist;
1207
1208         pte = page_address(pg);
1209         do {
1210                 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1211                         freelist = dma_pte_list_pagetables(domain, level - 1,
1212                                                            pte, freelist);
1213                 pte++;
1214         } while (!first_pte_in_page(pte));
1215
1216         return freelist;
1217 }
1218
1219 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1220                                         struct dma_pte *pte, unsigned long pfn,
1221                                         unsigned long start_pfn,
1222                                         unsigned long last_pfn,
1223                                         struct page *freelist)
1224 {
1225         struct dma_pte *first_pte = NULL, *last_pte = NULL;
1226
1227         pfn = max(start_pfn, pfn);
1228         pte = &pte[pfn_level_offset(pfn, level)];
1229
1230         do {
1231                 unsigned long level_pfn;
1232
1233                 if (!dma_pte_present(pte))
1234                         goto next;
1235
1236                 level_pfn = pfn & level_mask(level);
1237
1238                 /* If range covers entire pagetable, free it */
1239                 if (start_pfn <= level_pfn &&
1240                     last_pfn >= level_pfn + level_size(level) - 1) {
1241                         /* These suborbinate page tables are going away entirely. Don't
1242                            bother to clear them; we're just going to *free* them. */
1243                         if (level > 1 && !dma_pte_superpage(pte))
1244                                 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1245
1246                         dma_clear_pte(pte);
1247                         if (!first_pte)
1248                                 first_pte = pte;
1249                         last_pte = pte;
1250                 } else if (level > 1) {
1251                         /* Recurse down into a level that isn't *entirely* obsolete */
1252                         freelist = dma_pte_clear_level(domain, level - 1,
1253                                                        phys_to_virt(dma_pte_addr(pte)),
1254                                                        level_pfn, start_pfn, last_pfn,
1255                                                        freelist);
1256                 }
1257 next:
1258                 pfn += level_size(level);
1259         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1260
1261         if (first_pte)
1262                 domain_flush_cache(domain, first_pte,
1263                                    (void *)++last_pte - (void *)first_pte);
1264
1265         return freelist;
1266 }
1267
1268 /* We can't just free the pages because the IOMMU may still be walking
1269    the page tables, and may have cached the intermediate levels. The
1270    pages can only be freed after the IOTLB flush has been done. */
1271 static struct page *domain_unmap(struct dmar_domain *domain,
1272                                  unsigned long start_pfn,
1273                                  unsigned long last_pfn)
1274 {
1275         struct page *freelist = NULL;
1276
1277         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1278         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1279         BUG_ON(start_pfn > last_pfn);
1280
1281         /* we don't need lock here; nobody else touches the iova range */
1282         freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1283                                        domain->pgd, 0, start_pfn, last_pfn, NULL);
1284
1285         /* free pgd */
1286         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1287                 struct page *pgd_page = virt_to_page(domain->pgd);
1288                 pgd_page->freelist = freelist;
1289                 freelist = pgd_page;
1290
1291                 domain->pgd = NULL;
1292         }
1293
1294         return freelist;
1295 }
1296
1297 static void dma_free_pagelist(struct page *freelist)
1298 {
1299         struct page *pg;
1300
1301         while ((pg = freelist)) {
1302                 freelist = pg->freelist;
1303                 free_pgtable_page(page_address(pg));
1304         }
1305 }
1306
1307 /* iommu handling */
1308 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1309 {
1310         struct root_entry *root;
1311         unsigned long flags;
1312
1313         root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1314         if (!root) {
1315                 pr_err("Allocating root entry for %s failed\n",
1316                         iommu->name);
1317                 return -ENOMEM;
1318         }
1319
1320         __iommu_flush_cache(iommu, root, ROOT_SIZE);
1321
1322         spin_lock_irqsave(&iommu->lock, flags);
1323         iommu->root_entry = root;
1324         spin_unlock_irqrestore(&iommu->lock, flags);
1325
1326         return 0;
1327 }
1328
1329 static void iommu_set_root_entry(struct intel_iommu *iommu)
1330 {
1331         u64 addr;
1332         u32 sts;
1333         unsigned long flag;
1334
1335         addr = virt_to_phys(iommu->root_entry);
1336         if (ecs_enabled(iommu))
1337                 addr |= DMA_RTADDR_RTT;
1338
1339         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1340         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1341
1342         writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1343
1344         /* Make sure hardware complete it */
1345         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1346                       readl, (sts & DMA_GSTS_RTPS), sts);
1347
1348         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1349 }
1350
1351 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1352 {
1353         u32 val;
1354         unsigned long flag;
1355
1356         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1357                 return;
1358
1359         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1360         writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1361
1362         /* Make sure hardware complete it */
1363         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1364                       readl, (!(val & DMA_GSTS_WBFS)), val);
1365
1366         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1367 }
1368
1369 /* return value determine if we need a write buffer flush */
1370 static void __iommu_flush_context(struct intel_iommu *iommu,
1371                                   u16 did, u16 source_id, u8 function_mask,
1372                                   u64 type)
1373 {
1374         u64 val = 0;
1375         unsigned long flag;
1376
1377         switch (type) {
1378         case DMA_CCMD_GLOBAL_INVL:
1379                 val = DMA_CCMD_GLOBAL_INVL;
1380                 break;
1381         case DMA_CCMD_DOMAIN_INVL:
1382                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1383                 break;
1384         case DMA_CCMD_DEVICE_INVL:
1385                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1386                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1387                 break;
1388         default:
1389                 BUG();
1390         }
1391         val |= DMA_CCMD_ICC;
1392
1393         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1394         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1395
1396         /* Make sure hardware complete it */
1397         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1398                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1399
1400         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1401 }
1402
1403 /* return value determine if we need a write buffer flush */
1404 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1405                                 u64 addr, unsigned int size_order, u64 type)
1406 {
1407         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1408         u64 val = 0, val_iva = 0;
1409         unsigned long flag;
1410
1411         switch (type) {
1412         case DMA_TLB_GLOBAL_FLUSH:
1413                 /* global flush doesn't need set IVA_REG */
1414                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1415                 break;
1416         case DMA_TLB_DSI_FLUSH:
1417                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1418                 break;
1419         case DMA_TLB_PSI_FLUSH:
1420                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1421                 /* IH bit is passed in as part of address */
1422                 val_iva = size_order | addr;
1423                 break;
1424         default:
1425                 BUG();
1426         }
1427         /* Note: set drain read/write */
1428 #if 0
1429         /*
1430          * This is probably to be super secure.. Looks like we can
1431          * ignore it without any impact.
1432          */
1433         if (cap_read_drain(iommu->cap))
1434                 val |= DMA_TLB_READ_DRAIN;
1435 #endif
1436         if (cap_write_drain(iommu->cap))
1437                 val |= DMA_TLB_WRITE_DRAIN;
1438
1439         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1440         /* Note: Only uses first TLB reg currently */
1441         if (val_iva)
1442                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1443         dmar_writeq(iommu->reg + tlb_offset + 8, val);
1444
1445         /* Make sure hardware complete it */
1446         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1447                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1448
1449         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1450
1451         /* check IOTLB invalidation granularity */
1452         if (DMA_TLB_IAIG(val) == 0)
1453                 pr_err("Flush IOTLB failed\n");
1454         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1455                 pr_debug("TLB flush request %Lx, actual %Lx\n",
1456                         (unsigned long long)DMA_TLB_IIRG(type),
1457                         (unsigned long long)DMA_TLB_IAIG(val));
1458 }
1459
1460 static struct device_domain_info *
1461 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1462                          u8 bus, u8 devfn)
1463 {
1464         struct device_domain_info *info;
1465
1466         assert_spin_locked(&device_domain_lock);
1467
1468         if (!iommu->qi)
1469                 return NULL;
1470
1471         list_for_each_entry(info, &domain->devices, link)
1472                 if (info->iommu == iommu && info->bus == bus &&
1473                     info->devfn == devfn) {
1474                         if (info->ats_supported && info->dev)
1475                                 return info;
1476                         break;
1477                 }
1478
1479         return NULL;
1480 }
1481
1482 static void domain_update_iotlb(struct dmar_domain *domain)
1483 {
1484         struct device_domain_info *info;
1485         bool has_iotlb_device = false;
1486
1487         assert_spin_locked(&device_domain_lock);
1488
1489         list_for_each_entry(info, &domain->devices, link) {
1490                 struct pci_dev *pdev;
1491
1492                 if (!info->dev || !dev_is_pci(info->dev))
1493                         continue;
1494
1495                 pdev = to_pci_dev(info->dev);
1496                 if (pdev->ats_enabled) {
1497                         has_iotlb_device = true;
1498                         break;
1499                 }
1500         }
1501
1502         domain->has_iotlb_device = has_iotlb_device;
1503 }
1504
1505 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1506 {
1507         struct pci_dev *pdev;
1508
1509         assert_spin_locked(&device_domain_lock);
1510
1511         if (!info || !dev_is_pci(info->dev))
1512                 return;
1513
1514         pdev = to_pci_dev(info->dev);
1515         /* For IOMMU that supports device IOTLB throttling (DIT), we assign
1516          * PFSID to the invalidation desc of a VF such that IOMMU HW can gauge
1517          * queue depth at PF level. If DIT is not set, PFSID will be treated as
1518          * reserved, which should be set to 0.
1519          */
1520         if (!ecap_dit(info->iommu->ecap))
1521                 info->pfsid = 0;
1522         else {
1523                 struct pci_dev *pf_pdev;
1524
1525                 /* pdev will be returned if device is not a vf */
1526                 pf_pdev = pci_physfn(pdev);
1527                 info->pfsid = PCI_DEVID(pf_pdev->bus->number, pf_pdev->devfn);
1528         }
1529
1530 #ifdef CONFIG_INTEL_IOMMU_SVM
1531         /* The PCIe spec, in its wisdom, declares that the behaviour of
1532            the device if you enable PASID support after ATS support is
1533            undefined. So always enable PASID support on devices which
1534            have it, even if we can't yet know if we're ever going to
1535            use it. */
1536         if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1537                 info->pasid_enabled = 1;
1538
1539         if (info->pri_supported && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1540                 info->pri_enabled = 1;
1541 #endif
1542         if (info->ats_supported && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1543                 info->ats_enabled = 1;
1544                 domain_update_iotlb(info->domain);
1545                 info->ats_qdep = pci_ats_queue_depth(pdev);
1546         }
1547 }
1548
1549 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1550 {
1551         struct pci_dev *pdev;
1552
1553         assert_spin_locked(&device_domain_lock);
1554
1555         if (!dev_is_pci(info->dev))
1556                 return;
1557
1558         pdev = to_pci_dev(info->dev);
1559
1560         if (info->ats_enabled) {
1561                 pci_disable_ats(pdev);
1562                 info->ats_enabled = 0;
1563                 domain_update_iotlb(info->domain);
1564         }
1565 #ifdef CONFIG_INTEL_IOMMU_SVM
1566         if (info->pri_enabled) {
1567                 pci_disable_pri(pdev);
1568                 info->pri_enabled = 0;
1569         }
1570         if (info->pasid_enabled) {
1571                 pci_disable_pasid(pdev);
1572                 info->pasid_enabled = 0;
1573         }
1574 #endif
1575 }
1576
1577 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1578                                   u64 addr, unsigned mask)
1579 {
1580         u16 sid, qdep;
1581         unsigned long flags;
1582         struct device_domain_info *info;
1583
1584         if (!domain->has_iotlb_device)
1585                 return;
1586
1587         spin_lock_irqsave(&device_domain_lock, flags);
1588         list_for_each_entry(info, &domain->devices, link) {
1589                 if (!info->ats_enabled)
1590                         continue;
1591
1592                 sid = info->bus << 8 | info->devfn;
1593                 qdep = info->ats_qdep;
1594                 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
1595                                 qdep, addr, mask);
1596         }
1597         spin_unlock_irqrestore(&device_domain_lock, flags);
1598 }
1599
1600 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1601                                   struct dmar_domain *domain,
1602                                   unsigned long pfn, unsigned int pages,
1603                                   int ih, int map)
1604 {
1605         unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1606         uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1607         u16 did = domain->iommu_did[iommu->seq_id];
1608
1609         BUG_ON(pages == 0);
1610
1611         if (ih)
1612                 ih = 1 << 6;
1613         /*
1614          * Fallback to domain selective flush if no PSI support or the size is
1615          * too big.
1616          * PSI requires page size to be 2 ^ x, and the base address is naturally
1617          * aligned to the size
1618          */
1619         if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1620                 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1621                                                 DMA_TLB_DSI_FLUSH);
1622         else
1623                 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1624                                                 DMA_TLB_PSI_FLUSH);
1625
1626         /*
1627          * In caching mode, changes of pages from non-present to present require
1628          * flush. However, device IOTLB doesn't need to be flushed in this case.
1629          */
1630         if (!cap_caching_mode(iommu->cap) || !map)
1631                 iommu_flush_dev_iotlb(domain, addr, mask);
1632 }
1633
1634 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1635 {
1636         u32 pmen;
1637         unsigned long flags;
1638
1639         if (!cap_plmr(iommu->cap) && !cap_phmr(iommu->cap))
1640                 return;
1641
1642         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1643         pmen = readl(iommu->reg + DMAR_PMEN_REG);
1644         pmen &= ~DMA_PMEN_EPM;
1645         writel(pmen, iommu->reg + DMAR_PMEN_REG);
1646
1647         /* wait for the protected region status bit to clear */
1648         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1649                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1650
1651         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1652 }
1653
1654 static void iommu_enable_translation(struct intel_iommu *iommu)
1655 {
1656         u32 sts;
1657         unsigned long flags;
1658
1659         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1660         iommu->gcmd |= DMA_GCMD_TE;
1661         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1662
1663         /* Make sure hardware complete it */
1664         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1665                       readl, (sts & DMA_GSTS_TES), sts);
1666
1667         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1668 }
1669
1670 static void iommu_disable_translation(struct intel_iommu *iommu)
1671 {
1672         u32 sts;
1673         unsigned long flag;
1674
1675         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1676         iommu->gcmd &= ~DMA_GCMD_TE;
1677         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1678
1679         /* Make sure hardware complete it */
1680         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1681                       readl, (!(sts & DMA_GSTS_TES)), sts);
1682
1683         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1684 }
1685
1686
1687 static int iommu_init_domains(struct intel_iommu *iommu)
1688 {
1689         u32 ndomains, nlongs;
1690         size_t size;
1691
1692         ndomains = cap_ndoms(iommu->cap);
1693         pr_debug("%s: Number of Domains supported <%d>\n",
1694                  iommu->name, ndomains);
1695         nlongs = BITS_TO_LONGS(ndomains);
1696
1697         spin_lock_init(&iommu->lock);
1698
1699         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1700         if (!iommu->domain_ids) {
1701                 pr_err("%s: Allocating domain id array failed\n",
1702                        iommu->name);
1703                 return -ENOMEM;
1704         }
1705
1706         size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
1707         iommu->domains = kzalloc(size, GFP_KERNEL);
1708
1709         if (iommu->domains) {
1710                 size = 256 * sizeof(struct dmar_domain *);
1711                 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1712         }
1713
1714         if (!iommu->domains || !iommu->domains[0]) {
1715                 pr_err("%s: Allocating domain array failed\n",
1716                        iommu->name);
1717                 kfree(iommu->domain_ids);
1718                 kfree(iommu->domains);
1719                 iommu->domain_ids = NULL;
1720                 iommu->domains    = NULL;
1721                 return -ENOMEM;
1722         }
1723
1724
1725
1726         /*
1727          * If Caching mode is set, then invalid translations are tagged
1728          * with domain-id 0, hence we need to pre-allocate it. We also
1729          * use domain-id 0 as a marker for non-allocated domain-id, so
1730          * make sure it is not used for a real domain.
1731          */
1732         set_bit(0, iommu->domain_ids);
1733
1734         return 0;
1735 }
1736
1737 static void disable_dmar_iommu(struct intel_iommu *iommu)
1738 {
1739         struct device_domain_info *info, *tmp;
1740         unsigned long flags;
1741
1742         if (!iommu->domains || !iommu->domain_ids)
1743                 return;
1744
1745 again:
1746         spin_lock_irqsave(&device_domain_lock, flags);
1747         list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1748                 struct dmar_domain *domain;
1749
1750                 if (info->iommu != iommu)
1751                         continue;
1752
1753                 if (!info->dev || !info->domain)
1754                         continue;
1755
1756                 domain = info->domain;
1757
1758                 __dmar_remove_one_dev_info(info);
1759
1760                 if (!domain_type_is_vm_or_si(domain)) {
1761                         /*
1762                          * The domain_exit() function  can't be called under
1763                          * device_domain_lock, as it takes this lock itself.
1764                          * So release the lock here and re-run the loop
1765                          * afterwards.
1766                          */
1767                         spin_unlock_irqrestore(&device_domain_lock, flags);
1768                         domain_exit(domain);
1769                         goto again;
1770                 }
1771         }
1772         spin_unlock_irqrestore(&device_domain_lock, flags);
1773
1774         if (iommu->gcmd & DMA_GCMD_TE)
1775                 iommu_disable_translation(iommu);
1776 }
1777
1778 static void free_dmar_iommu(struct intel_iommu *iommu)
1779 {
1780         if ((iommu->domains) && (iommu->domain_ids)) {
1781                 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
1782                 int i;
1783
1784                 for (i = 0; i < elems; i++)
1785                         kfree(iommu->domains[i]);
1786                 kfree(iommu->domains);
1787                 kfree(iommu->domain_ids);
1788                 iommu->domains = NULL;
1789                 iommu->domain_ids = NULL;
1790         }
1791
1792         g_iommus[iommu->seq_id] = NULL;
1793
1794         /* free context mapping */
1795         free_context_table(iommu);
1796
1797 #ifdef CONFIG_INTEL_IOMMU_SVM
1798         if (pasid_enabled(iommu)) {
1799                 if (ecap_prs(iommu->ecap))
1800                         intel_svm_finish_prq(iommu);
1801                 intel_svm_free_pasid_tables(iommu);
1802         }
1803 #endif
1804 }
1805
1806 static struct dmar_domain *alloc_domain(int flags)
1807 {
1808         struct dmar_domain *domain;
1809
1810         domain = alloc_domain_mem();
1811         if (!domain)
1812                 return NULL;
1813
1814         memset(domain, 0, sizeof(*domain));
1815         domain->nid = -1;
1816         domain->flags = flags;
1817         domain->has_iotlb_device = false;
1818         INIT_LIST_HEAD(&domain->devices);
1819
1820         return domain;
1821 }
1822
1823 /* Must be called with iommu->lock */
1824 static int domain_attach_iommu(struct dmar_domain *domain,
1825                                struct intel_iommu *iommu)
1826 {
1827         unsigned long ndomains;
1828         int num;
1829
1830         assert_spin_locked(&device_domain_lock);
1831         assert_spin_locked(&iommu->lock);
1832
1833         domain->iommu_refcnt[iommu->seq_id] += 1;
1834         domain->iommu_count += 1;
1835         if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1836                 ndomains = cap_ndoms(iommu->cap);
1837                 num      = find_first_zero_bit(iommu->domain_ids, ndomains);
1838
1839                 if (num >= ndomains) {
1840                         pr_err("%s: No free domain ids\n", iommu->name);
1841                         domain->iommu_refcnt[iommu->seq_id] -= 1;
1842                         domain->iommu_count -= 1;
1843                         return -ENOSPC;
1844                 }
1845
1846                 set_bit(num, iommu->domain_ids);
1847                 set_iommu_domain(iommu, num, domain);
1848
1849                 domain->iommu_did[iommu->seq_id] = num;
1850                 domain->nid                      = iommu->node;
1851
1852                 domain_update_iommu_cap(domain);
1853         }
1854
1855         return 0;
1856 }
1857
1858 static int domain_detach_iommu(struct dmar_domain *domain,
1859                                struct intel_iommu *iommu)
1860 {
1861         int num, count = INT_MAX;
1862
1863         assert_spin_locked(&device_domain_lock);
1864         assert_spin_locked(&iommu->lock);
1865
1866         domain->iommu_refcnt[iommu->seq_id] -= 1;
1867         count = --domain->iommu_count;
1868         if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1869                 num = domain->iommu_did[iommu->seq_id];
1870                 clear_bit(num, iommu->domain_ids);
1871                 set_iommu_domain(iommu, num, NULL);
1872
1873                 domain_update_iommu_cap(domain);
1874                 domain->iommu_did[iommu->seq_id] = 0;
1875         }
1876
1877         return count;
1878 }
1879
1880 static struct iova_domain reserved_iova_list;
1881 static struct lock_class_key reserved_rbtree_key;
1882
1883 static int dmar_init_reserved_ranges(void)
1884 {
1885         struct pci_dev *pdev = NULL;
1886         struct iova *iova;
1887         int i;
1888
1889         init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1890                         DMA_32BIT_PFN);
1891
1892         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1893                 &reserved_rbtree_key);
1894
1895         /* IOAPIC ranges shouldn't be accessed by DMA */
1896         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1897                 IOVA_PFN(IOAPIC_RANGE_END));
1898         if (!iova) {
1899                 pr_err("Reserve IOAPIC range failed\n");
1900                 return -ENODEV;
1901         }
1902
1903         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1904         for_each_pci_dev(pdev) {
1905                 struct resource *r;
1906
1907                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1908                         r = &pdev->resource[i];
1909                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1910                                 continue;
1911                         iova = reserve_iova(&reserved_iova_list,
1912                                             IOVA_PFN(r->start),
1913                                             IOVA_PFN(r->end));
1914                         if (!iova) {
1915                                 pr_err("Reserve iova failed\n");
1916                                 return -ENODEV;
1917                         }
1918                 }
1919         }
1920         return 0;
1921 }
1922
1923 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1924 {
1925         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1926 }
1927
1928 static inline int guestwidth_to_adjustwidth(int gaw)
1929 {
1930         int agaw;
1931         int r = (gaw - 12) % 9;
1932
1933         if (r == 0)
1934                 agaw = gaw;
1935         else
1936                 agaw = gaw + 9 - r;
1937         if (agaw > 64)
1938                 agaw = 64;
1939         return agaw;
1940 }
1941
1942 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1943                        int guest_width)
1944 {
1945         int adjust_width, agaw;
1946         unsigned long sagaw;
1947
1948         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1949                         DMA_32BIT_PFN);
1950         domain_reserve_special_ranges(domain);
1951
1952         /* calculate AGAW */
1953         if (guest_width > cap_mgaw(iommu->cap))
1954                 guest_width = cap_mgaw(iommu->cap);
1955         domain->gaw = guest_width;
1956         adjust_width = guestwidth_to_adjustwidth(guest_width);
1957         agaw = width_to_agaw(adjust_width);
1958         sagaw = cap_sagaw(iommu->cap);
1959         if (!test_bit(agaw, &sagaw)) {
1960                 /* hardware doesn't support it, choose a bigger one */
1961                 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1962                 agaw = find_next_bit(&sagaw, 5, agaw);
1963                 if (agaw >= 5)
1964                         return -ENODEV;
1965         }
1966         domain->agaw = agaw;
1967
1968         if (ecap_coherent(iommu->ecap))
1969                 domain->iommu_coherency = 1;
1970         else
1971                 domain->iommu_coherency = 0;
1972
1973         if (ecap_sc_support(iommu->ecap))
1974                 domain->iommu_snooping = 1;
1975         else
1976                 domain->iommu_snooping = 0;
1977
1978         if (intel_iommu_superpage)
1979                 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1980         else
1981                 domain->iommu_superpage = 0;
1982
1983         domain->nid = iommu->node;
1984
1985         /* always allocate the top pgd */
1986         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1987         if (!domain->pgd)
1988                 return -ENOMEM;
1989         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1990         return 0;
1991 }
1992
1993 static void domain_exit(struct dmar_domain *domain)
1994 {
1995         struct page *freelist = NULL;
1996
1997         /* Domain 0 is reserved, so dont process it */
1998         if (!domain)
1999                 return;
2000
2001         /* Flush any lazy unmaps that may reference this domain */
2002         if (!intel_iommu_strict) {
2003                 int cpu;
2004
2005                 for_each_possible_cpu(cpu)
2006                         flush_unmaps_timeout(cpu);
2007         }
2008
2009         /* Remove associated devices and clear attached or cached domains */
2010         rcu_read_lock();
2011         domain_remove_dev_info(domain);
2012         rcu_read_unlock();
2013
2014         /* destroy iovas */
2015         put_iova_domain(&domain->iovad);
2016
2017         freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
2018
2019         dma_free_pagelist(freelist);
2020
2021         free_domain_mem(domain);
2022 }
2023
2024 static int domain_context_mapping_one(struct dmar_domain *domain,
2025                                       struct intel_iommu *iommu,
2026                                       u8 bus, u8 devfn)
2027 {
2028         u16 did = domain->iommu_did[iommu->seq_id];
2029         int translation = CONTEXT_TT_MULTI_LEVEL;
2030         struct device_domain_info *info = NULL;
2031         struct context_entry *context;
2032         unsigned long flags;
2033         struct dma_pte *pgd;
2034         int ret, agaw;
2035
2036         WARN_ON(did == 0);
2037
2038         if (hw_pass_through && domain_type_is_si(domain))
2039                 translation = CONTEXT_TT_PASS_THROUGH;
2040
2041         pr_debug("Set context mapping for %02x:%02x.%d\n",
2042                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
2043
2044         BUG_ON(!domain->pgd);
2045
2046         spin_lock_irqsave(&device_domain_lock, flags);
2047         spin_lock(&iommu->lock);
2048
2049         ret = -ENOMEM;
2050         context = iommu_context_addr(iommu, bus, devfn, 1);
2051         if (!context)
2052                 goto out_unlock;
2053
2054         ret = 0;
2055         if (context_present(context))
2056                 goto out_unlock;
2057
2058         /*
2059          * For kdump cases, old valid entries may be cached due to the
2060          * in-flight DMA and copied pgtable, but there is no unmapping
2061          * behaviour for them, thus we need an explicit cache flush for
2062          * the newly-mapped device. For kdump, at this point, the device
2063          * is supposed to finish reset at its driver probe stage, so no
2064          * in-flight DMA will exist, and we don't need to worry anymore
2065          * hereafter.
2066          */
2067         if (context_copied(context)) {
2068                 u16 did_old = context_domain_id(context);
2069
2070                 if (did_old >= 0 && did_old < cap_ndoms(iommu->cap)) {
2071                         iommu->flush.flush_context(iommu, did_old,
2072                                                    (((u16)bus) << 8) | devfn,
2073                                                    DMA_CCMD_MASK_NOBIT,
2074                                                    DMA_CCMD_DEVICE_INVL);
2075                         iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
2076                                                  DMA_TLB_DSI_FLUSH);
2077                 }
2078         }
2079
2080         pgd = domain->pgd;
2081
2082         context_clear_entry(context);
2083         context_set_domain_id(context, did);
2084
2085         /*
2086          * Skip top levels of page tables for iommu which has less agaw
2087          * than default.  Unnecessary for PT mode.
2088          */
2089         if (translation != CONTEXT_TT_PASS_THROUGH) {
2090                 for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
2091                         ret = -ENOMEM;
2092                         pgd = phys_to_virt(dma_pte_addr(pgd));
2093                         if (!dma_pte_present(pgd))
2094                                 goto out_unlock;
2095                 }
2096
2097                 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2098                 if (info && info->ats_supported)
2099                         translation = CONTEXT_TT_DEV_IOTLB;
2100                 else
2101                         translation = CONTEXT_TT_MULTI_LEVEL;
2102
2103                 context_set_address_root(context, virt_to_phys(pgd));
2104                 context_set_address_width(context, agaw);
2105         } else {
2106                 /*
2107                  * In pass through mode, AW must be programmed to
2108                  * indicate the largest AGAW value supported by
2109                  * hardware. And ASR is ignored by hardware.
2110                  */
2111                 context_set_address_width(context, iommu->msagaw);
2112         }
2113
2114         context_set_translation_type(context, translation);
2115         context_set_fault_enable(context);
2116         context_set_present(context);
2117         domain_flush_cache(domain, context, sizeof(*context));
2118
2119         /*
2120          * It's a non-present to present mapping. If hardware doesn't cache
2121          * non-present entry we only need to flush the write-buffer. If the
2122          * _does_ cache non-present entries, then it does so in the special
2123          * domain #0, which we have to flush:
2124          */
2125         if (cap_caching_mode(iommu->cap)) {
2126                 iommu->flush.flush_context(iommu, 0,
2127                                            (((u16)bus) << 8) | devfn,
2128                                            DMA_CCMD_MASK_NOBIT,
2129                                            DMA_CCMD_DEVICE_INVL);
2130                 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2131         } else {
2132                 iommu_flush_write_buffer(iommu);
2133         }
2134         iommu_enable_dev_iotlb(info);
2135
2136         ret = 0;
2137
2138 out_unlock:
2139         spin_unlock(&iommu->lock);
2140         spin_unlock_irqrestore(&device_domain_lock, flags);
2141
2142         return ret;
2143 }
2144
2145 struct domain_context_mapping_data {
2146         struct dmar_domain *domain;
2147         struct intel_iommu *iommu;
2148 };
2149
2150 static int domain_context_mapping_cb(struct pci_dev *pdev,
2151                                      u16 alias, void *opaque)
2152 {
2153         struct domain_context_mapping_data *data = opaque;
2154
2155         return domain_context_mapping_one(data->domain, data->iommu,
2156                                           PCI_BUS_NUM(alias), alias & 0xff);
2157 }
2158
2159 static int
2160 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2161 {
2162         struct intel_iommu *iommu;
2163         u8 bus, devfn;
2164         struct domain_context_mapping_data data;
2165
2166         iommu = device_to_iommu(dev, &bus, &devfn);
2167         if (!iommu)
2168                 return -ENODEV;
2169
2170         if (!dev_is_pci(dev))
2171                 return domain_context_mapping_one(domain, iommu, bus, devfn);
2172
2173         data.domain = domain;
2174         data.iommu = iommu;
2175
2176         return pci_for_each_dma_alias(to_pci_dev(dev),
2177                                       &domain_context_mapping_cb, &data);
2178 }
2179
2180 static int domain_context_mapped_cb(struct pci_dev *pdev,
2181                                     u16 alias, void *opaque)
2182 {
2183         struct intel_iommu *iommu = opaque;
2184
2185         return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2186 }
2187
2188 static int domain_context_mapped(struct device *dev)
2189 {
2190         struct intel_iommu *iommu;
2191         u8 bus, devfn;
2192
2193         iommu = device_to_iommu(dev, &bus, &devfn);
2194         if (!iommu)
2195                 return -ENODEV;
2196
2197         if (!dev_is_pci(dev))
2198                 return device_context_mapped(iommu, bus, devfn);
2199
2200         return !pci_for_each_dma_alias(to_pci_dev(dev),
2201                                        domain_context_mapped_cb, iommu);
2202 }
2203
2204 /* Returns a number of VTD pages, but aligned to MM page size */
2205 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2206                                             size_t size)
2207 {
2208         host_addr &= ~PAGE_MASK;
2209         return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2210 }
2211
2212 /* Return largest possible superpage level for a given mapping */
2213 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2214                                           unsigned long iov_pfn,
2215                                           unsigned long phy_pfn,
2216                                           unsigned long pages)
2217 {
2218         int support, level = 1;
2219         unsigned long pfnmerge;
2220
2221         support = domain->iommu_superpage;
2222
2223         /* To use a large page, the virtual *and* physical addresses
2224            must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2225            of them will mean we have to use smaller pages. So just
2226            merge them and check both at once. */
2227         pfnmerge = iov_pfn | phy_pfn;
2228
2229         while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2230                 pages >>= VTD_STRIDE_SHIFT;
2231                 if (!pages)
2232                         break;
2233                 pfnmerge >>= VTD_STRIDE_SHIFT;
2234                 level++;
2235                 support--;
2236         }
2237         return level;
2238 }
2239
2240 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2241                             struct scatterlist *sg, unsigned long phys_pfn,
2242                             unsigned long nr_pages, int prot)
2243 {
2244         struct dma_pte *first_pte = NULL, *pte = NULL;
2245         phys_addr_t uninitialized_var(pteval);
2246         unsigned long sg_res = 0;
2247         unsigned int largepage_lvl = 0;
2248         unsigned long lvl_pages = 0;
2249
2250         BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2251
2252         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2253                 return -EINVAL;
2254
2255         prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2256
2257         if (!sg) {
2258                 sg_res = nr_pages;
2259                 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2260         }
2261
2262         while (nr_pages > 0) {
2263                 uint64_t tmp;
2264
2265                 if (!sg_res) {
2266                         unsigned int pgoff = sg->offset & ~PAGE_MASK;
2267
2268                         sg_res = aligned_nrpages(sg->offset, sg->length);
2269                         sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + pgoff;
2270                         sg->dma_length = sg->length;
2271                         pteval = (sg_phys(sg) - pgoff) | prot;
2272                         phys_pfn = pteval >> VTD_PAGE_SHIFT;
2273                 }
2274
2275                 if (!pte) {
2276                         largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2277
2278                         first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2279                         if (!pte)
2280                                 return -ENOMEM;
2281                         /* It is large page*/
2282                         if (largepage_lvl > 1) {
2283                                 unsigned long nr_superpages, end_pfn;
2284
2285                                 pteval |= DMA_PTE_LARGE_PAGE;
2286                                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2287
2288                                 nr_superpages = sg_res / lvl_pages;
2289                                 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2290
2291                                 /*
2292                                  * Ensure that old small page tables are
2293                                  * removed to make room for superpage(s).
2294                                  */
2295                                 dma_pte_free_pagetable(domain, iov_pfn, end_pfn);
2296                         } else {
2297                                 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2298                         }
2299
2300                 }
2301                 /* We don't need lock here, nobody else
2302                  * touches the iova range
2303                  */
2304                 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2305                 if (tmp) {
2306                         static int dumps = 5;
2307                         pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2308                                 iov_pfn, tmp, (unsigned long long)pteval);
2309                         if (dumps) {
2310                                 dumps--;
2311                                 debug_dma_dump_mappings(NULL);
2312                         }
2313                         WARN_ON(1);
2314                 }
2315
2316                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2317
2318                 BUG_ON(nr_pages < lvl_pages);
2319                 BUG_ON(sg_res < lvl_pages);
2320
2321                 nr_pages -= lvl_pages;
2322                 iov_pfn += lvl_pages;
2323                 phys_pfn += lvl_pages;
2324                 pteval += lvl_pages * VTD_PAGE_SIZE;
2325                 sg_res -= lvl_pages;
2326
2327                 /* If the next PTE would be the first in a new page, then we
2328                    need to flush the cache on the entries we've just written.
2329                    And then we'll need to recalculate 'pte', so clear it and
2330                    let it get set again in the if (!pte) block above.
2331
2332                    If we're done (!nr_pages) we need to flush the cache too.
2333
2334                    Also if we've been setting superpages, we may need to
2335                    recalculate 'pte' and switch back to smaller pages for the
2336                    end of the mapping, if the trailing size is not enough to
2337                    use another superpage (i.e. sg_res < lvl_pages). */
2338                 pte++;
2339                 if (!nr_pages || first_pte_in_page(pte) ||
2340                     (largepage_lvl > 1 && sg_res < lvl_pages)) {
2341                         domain_flush_cache(domain, first_pte,
2342                                            (void *)pte - (void *)first_pte);
2343                         pte = NULL;
2344                 }
2345
2346                 if (!sg_res && nr_pages)
2347                         sg = sg_next(sg);
2348         }
2349         return 0;
2350 }
2351
2352 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2353                                     struct scatterlist *sg, unsigned long nr_pages,
2354                                     int prot)
2355 {
2356         return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2357 }
2358
2359 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2360                                      unsigned long phys_pfn, unsigned long nr_pages,
2361                                      int prot)
2362 {
2363         return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2364 }
2365
2366 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2367 {
2368         if (!iommu)
2369                 return;
2370
2371         clear_context_table(iommu, bus, devfn);
2372         iommu->flush.flush_context(iommu, 0, 0, 0,
2373                                            DMA_CCMD_GLOBAL_INVL);
2374         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2375 }
2376
2377 static inline void unlink_domain_info(struct device_domain_info *info)
2378 {
2379         assert_spin_locked(&device_domain_lock);
2380         list_del(&info->link);
2381         list_del(&info->global);
2382         if (info->dev)
2383                 info->dev->archdata.iommu = NULL;
2384 }
2385
2386 static void domain_remove_dev_info(struct dmar_domain *domain)
2387 {
2388         struct device_domain_info *info, *tmp;
2389         unsigned long flags;
2390
2391         spin_lock_irqsave(&device_domain_lock, flags);
2392         list_for_each_entry_safe(info, tmp, &domain->devices, link)
2393                 __dmar_remove_one_dev_info(info);
2394         spin_unlock_irqrestore(&device_domain_lock, flags);
2395 }
2396
2397 /*
2398  * find_domain
2399  * Note: we use struct device->archdata.iommu stores the info
2400  */
2401 static struct dmar_domain *find_domain(struct device *dev)
2402 {
2403         struct device_domain_info *info;
2404
2405         /* No lock here, assumes no domain exit in normal case */
2406         info = dev->archdata.iommu;
2407         if (info)
2408                 return info->domain;
2409         return NULL;
2410 }
2411
2412 static inline struct device_domain_info *
2413 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2414 {
2415         struct device_domain_info *info;
2416
2417         list_for_each_entry(info, &device_domain_list, global)
2418                 if (info->iommu->segment == segment && info->bus == bus &&
2419                     info->devfn == devfn)
2420                         return info;
2421
2422         return NULL;
2423 }
2424
2425 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2426                                                     int bus, int devfn,
2427                                                     struct device *dev,
2428                                                     struct dmar_domain *domain)
2429 {
2430         struct dmar_domain *found = NULL;
2431         struct device_domain_info *info;
2432         unsigned long flags;
2433         int ret;
2434
2435         info = alloc_devinfo_mem();
2436         if (!info)
2437                 return NULL;
2438
2439         info->bus = bus;
2440         info->devfn = devfn;
2441         info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2442         info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2443         info->ats_qdep = 0;
2444         info->dev = dev;
2445         info->domain = domain;
2446         info->iommu = iommu;
2447
2448         if (dev && dev_is_pci(dev)) {
2449                 struct pci_dev *pdev = to_pci_dev(info->dev);
2450
2451                 if (ecap_dev_iotlb_support(iommu->ecap) &&
2452                     pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2453                     dmar_find_matched_atsr_unit(pdev))
2454                         info->ats_supported = 1;
2455
2456                 if (ecs_enabled(iommu)) {
2457                         if (pasid_enabled(iommu)) {
2458                                 int features = pci_pasid_features(pdev);
2459                                 if (features >= 0)
2460                                         info->pasid_supported = features | 1;
2461                         }
2462
2463                         if (info->ats_supported && ecap_prs(iommu->ecap) &&
2464                             pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2465                                 info->pri_supported = 1;
2466                 }
2467         }
2468
2469         spin_lock_irqsave(&device_domain_lock, flags);
2470         if (dev)
2471                 found = find_domain(dev);
2472
2473         if (!found) {
2474                 struct device_domain_info *info2;
2475                 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2476                 if (info2) {
2477                         found      = info2->domain;
2478                         info2->dev = dev;
2479                 }
2480         }
2481
2482         if (found) {
2483                 spin_unlock_irqrestore(&device_domain_lock, flags);
2484                 free_devinfo_mem(info);
2485                 /* Caller must free the original domain */
2486                 return found;
2487         }
2488
2489         spin_lock(&iommu->lock);
2490         ret = domain_attach_iommu(domain, iommu);
2491         spin_unlock(&iommu->lock);
2492
2493         if (ret) {
2494                 spin_unlock_irqrestore(&device_domain_lock, flags);
2495                 free_devinfo_mem(info);
2496                 return NULL;
2497         }
2498
2499         list_add(&info->link, &domain->devices);
2500         list_add(&info->global, &device_domain_list);
2501         if (dev)
2502                 dev->archdata.iommu = info;
2503         spin_unlock_irqrestore(&device_domain_lock, flags);
2504
2505         if (dev && domain_context_mapping(domain, dev)) {
2506                 pr_err("Domain context map for %s failed\n", dev_name(dev));
2507                 dmar_remove_one_dev_info(domain, dev);
2508                 return NULL;
2509         }
2510
2511         return domain;
2512 }
2513
2514 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2515 {
2516         *(u16 *)opaque = alias;
2517         return 0;
2518 }
2519
2520 static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
2521 {
2522         struct device_domain_info *info = NULL;
2523         struct dmar_domain *domain = NULL;
2524         struct intel_iommu *iommu;
2525         u16 req_id, dma_alias;
2526         unsigned long flags;
2527         u8 bus, devfn;
2528
2529         iommu = device_to_iommu(dev, &bus, &devfn);
2530         if (!iommu)
2531                 return NULL;
2532
2533         req_id = ((u16)bus << 8) | devfn;
2534
2535         if (dev_is_pci(dev)) {
2536                 struct pci_dev *pdev = to_pci_dev(dev);
2537
2538                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2539
2540                 spin_lock_irqsave(&device_domain_lock, flags);
2541                 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2542                                                       PCI_BUS_NUM(dma_alias),
2543                                                       dma_alias & 0xff);
2544                 if (info) {
2545                         iommu = info->iommu;
2546                         domain = info->domain;
2547                 }
2548                 spin_unlock_irqrestore(&device_domain_lock, flags);
2549
2550                 /* DMA alias already has a domain, use it */
2551                 if (info)
2552                         goto out;
2553         }
2554
2555         /* Allocate and initialize new domain for the device */
2556         domain = alloc_domain(0);
2557         if (!domain)
2558                 return NULL;
2559         if (domain_init(domain, iommu, gaw)) {
2560                 domain_exit(domain);
2561                 return NULL;
2562         }
2563
2564 out:
2565
2566         return domain;
2567 }
2568
2569 static struct dmar_domain *set_domain_for_dev(struct device *dev,
2570                                               struct dmar_domain *domain)
2571 {
2572         struct intel_iommu *iommu;
2573         struct dmar_domain *tmp;
2574         u16 req_id, dma_alias;
2575         u8 bus, devfn;
2576
2577         iommu = device_to_iommu(dev, &bus, &devfn);
2578         if (!iommu)
2579                 return NULL;
2580
2581         req_id = ((u16)bus << 8) | devfn;
2582
2583         if (dev_is_pci(dev)) {
2584                 struct pci_dev *pdev = to_pci_dev(dev);
2585
2586                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2587
2588                 /* register PCI DMA alias device */
2589                 if (req_id != dma_alias) {
2590                         tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2591                                         dma_alias & 0xff, NULL, domain);
2592
2593                         if (!tmp || tmp != domain)
2594                                 return tmp;
2595                 }
2596         }
2597
2598         tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2599         if (!tmp || tmp != domain)
2600                 return tmp;
2601
2602         return domain;
2603 }
2604
2605 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2606 {
2607         struct dmar_domain *domain, *tmp;
2608
2609         domain = find_domain(dev);
2610         if (domain)
2611                 goto out;
2612
2613         domain = find_or_alloc_domain(dev, gaw);
2614         if (!domain)
2615                 goto out;
2616
2617         tmp = set_domain_for_dev(dev, domain);
2618         if (!tmp || domain != tmp) {
2619                 domain_exit(domain);
2620                 domain = tmp;
2621         }
2622
2623 out:
2624
2625         return domain;
2626 }
2627
2628 static int iommu_domain_identity_map(struct dmar_domain *domain,
2629                                      unsigned long long start,
2630                                      unsigned long long end)
2631 {
2632         unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2633         unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2634
2635         if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2636                           dma_to_mm_pfn(last_vpfn))) {
2637                 pr_err("Reserving iova failed\n");
2638                 return -ENOMEM;
2639         }
2640
2641         pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2642         /*
2643          * RMRR range might have overlap with physical memory range,
2644          * clear it first
2645          */
2646         dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2647
2648         return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2649                                   last_vpfn - first_vpfn + 1,
2650                                   DMA_PTE_READ|DMA_PTE_WRITE);
2651 }
2652
2653 static int domain_prepare_identity_map(struct device *dev,
2654                                        struct dmar_domain *domain,
2655                                        unsigned long long start,
2656                                        unsigned long long end)
2657 {
2658         /* For _hardware_ passthrough, don't bother. But for software
2659            passthrough, we do it anyway -- it may indicate a memory
2660            range which is reserved in E820, so which didn't get set
2661            up to start with in si_domain */
2662         if (domain == si_domain && hw_pass_through) {
2663                 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2664                         dev_name(dev), start, end);
2665                 return 0;
2666         }
2667
2668         pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2669                 dev_name(dev), start, end);
2670
2671         if (end < start) {
2672                 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2673                         "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2674                         dmi_get_system_info(DMI_BIOS_VENDOR),
2675                         dmi_get_system_info(DMI_BIOS_VERSION),
2676                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2677                 return -EIO;
2678         }
2679
2680         if (end >> agaw_to_width(domain->agaw)) {
2681                 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2682                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2683                      agaw_to_width(domain->agaw),
2684                      dmi_get_system_info(DMI_BIOS_VENDOR),
2685                      dmi_get_system_info(DMI_BIOS_VERSION),
2686                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2687                 return -EIO;
2688         }
2689
2690         return iommu_domain_identity_map(domain, start, end);
2691 }
2692
2693 static int iommu_prepare_identity_map(struct device *dev,
2694                                       unsigned long long start,
2695                                       unsigned long long end)
2696 {
2697         struct dmar_domain *domain;
2698         int ret;
2699
2700         domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2701         if (!domain)
2702                 return -ENOMEM;
2703
2704         ret = domain_prepare_identity_map(dev, domain, start, end);
2705         if (ret)
2706                 domain_exit(domain);
2707
2708         return ret;
2709 }
2710
2711 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2712                                          struct device *dev)
2713 {
2714         if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2715                 return 0;
2716         return iommu_prepare_identity_map(dev, rmrr->base_address,
2717                                           rmrr->end_address);
2718 }
2719
2720 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2721 static inline void iommu_prepare_isa(void)
2722 {
2723         struct pci_dev *pdev;
2724         int ret;
2725
2726         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2727         if (!pdev)
2728                 return;
2729
2730         pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2731         ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2732
2733         if (ret)
2734                 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2735
2736         pci_dev_put(pdev);
2737 }
2738 #else
2739 static inline void iommu_prepare_isa(void)
2740 {
2741         return;
2742 }
2743 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2744
2745 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2746
2747 static int __init si_domain_init(int hw)
2748 {
2749         int nid, ret = 0;
2750
2751         si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2752         if (!si_domain)
2753                 return -EFAULT;
2754
2755         if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2756                 domain_exit(si_domain);
2757                 return -EFAULT;
2758         }
2759
2760         pr_debug("Identity mapping domain allocated\n");
2761
2762         if (hw)
2763                 return 0;
2764
2765         for_each_online_node(nid) {
2766                 unsigned long start_pfn, end_pfn;
2767                 int i;
2768
2769                 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2770                         ret = iommu_domain_identity_map(si_domain,
2771                                         PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2772                         if (ret)
2773                                 return ret;
2774                 }
2775         }
2776
2777         return 0;
2778 }
2779
2780 static int identity_mapping(struct device *dev)
2781 {
2782         struct device_domain_info *info;
2783
2784         if (likely(!iommu_identity_mapping))
2785                 return 0;
2786
2787         info = dev->archdata.iommu;
2788         if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2789                 return (info->domain == si_domain);
2790
2791         return 0;
2792 }
2793
2794 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2795 {
2796         struct dmar_domain *ndomain;
2797         struct intel_iommu *iommu;
2798         u8 bus, devfn;
2799
2800         iommu = device_to_iommu(dev, &bus, &devfn);
2801         if (!iommu)
2802                 return -ENODEV;
2803
2804         ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2805         if (ndomain != domain)
2806                 return -EBUSY;
2807
2808         return 0;
2809 }
2810
2811 static bool device_has_rmrr(struct device *dev)
2812 {
2813         struct dmar_rmrr_unit *rmrr;
2814         struct device *tmp;
2815         int i;
2816
2817         rcu_read_lock();
2818         for_each_rmrr_units(rmrr) {
2819                 /*
2820                  * Return TRUE if this RMRR contains the device that
2821                  * is passed in.
2822                  */
2823                 for_each_active_dev_scope(rmrr->devices,
2824                                           rmrr->devices_cnt, i, tmp)
2825                         if (tmp == dev) {
2826                                 rcu_read_unlock();
2827                                 return true;
2828                         }
2829         }
2830         rcu_read_unlock();
2831         return false;
2832 }
2833
2834 /*
2835  * There are a couple cases where we need to restrict the functionality of
2836  * devices associated with RMRRs.  The first is when evaluating a device for
2837  * identity mapping because problems exist when devices are moved in and out
2838  * of domains and their respective RMRR information is lost.  This means that
2839  * a device with associated RMRRs will never be in a "passthrough" domain.
2840  * The second is use of the device through the IOMMU API.  This interface
2841  * expects to have full control of the IOVA space for the device.  We cannot
2842  * satisfy both the requirement that RMRR access is maintained and have an
2843  * unencumbered IOVA space.  We also have no ability to quiesce the device's
2844  * use of the RMRR space or even inform the IOMMU API user of the restriction.
2845  * We therefore prevent devices associated with an RMRR from participating in
2846  * the IOMMU API, which eliminates them from device assignment.
2847  *
2848  * In both cases we assume that PCI USB devices with RMRRs have them largely
2849  * for historical reasons and that the RMRR space is not actively used post
2850  * boot.  This exclusion may change if vendors begin to abuse it.
2851  *
2852  * The same exception is made for graphics devices, with the requirement that
2853  * any use of the RMRR regions will be torn down before assigning the device
2854  * to a guest.
2855  */
2856 static bool device_is_rmrr_locked(struct device *dev)
2857 {
2858         if (!device_has_rmrr(dev))
2859                 return false;
2860
2861         if (dev_is_pci(dev)) {
2862                 struct pci_dev *pdev = to_pci_dev(dev);
2863
2864                 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2865                         return false;
2866         }
2867
2868         return true;
2869 }
2870
2871 static int iommu_should_identity_map(struct device *dev, int startup)
2872 {
2873
2874         if (dev_is_pci(dev)) {
2875                 struct pci_dev *pdev = to_pci_dev(dev);
2876
2877                 if (device_is_rmrr_locked(dev))
2878                         return 0;
2879
2880                 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2881                         return 1;
2882
2883                 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2884                         return 1;
2885
2886                 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2887                         return 0;
2888
2889                 /*
2890                  * We want to start off with all devices in the 1:1 domain, and
2891                  * take them out later if we find they can't access all of memory.
2892                  *
2893                  * However, we can't do this for PCI devices behind bridges,
2894                  * because all PCI devices behind the same bridge will end up
2895                  * with the same source-id on their transactions.
2896                  *
2897                  * Practically speaking, we can't change things around for these
2898                  * devices at run-time, because we can't be sure there'll be no
2899                  * DMA transactions in flight for any of their siblings.
2900                  *
2901                  * So PCI devices (unless they're on the root bus) as well as
2902                  * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2903                  * the 1:1 domain, just in _case_ one of their siblings turns out
2904                  * not to be able to map all of memory.
2905                  */
2906                 if (!pci_is_pcie(pdev)) {
2907                         if (!pci_is_root_bus(pdev->bus))
2908                                 return 0;
2909                         if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2910                                 return 0;
2911                 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2912                         return 0;
2913         } else {
2914                 if (device_has_rmrr(dev))
2915                         return 0;
2916         }
2917
2918         /*
2919          * At boot time, we don't yet know if devices will be 64-bit capable.
2920          * Assume that they will — if they turn out not to be, then we can
2921          * take them out of the 1:1 domain later.
2922          */
2923         if (!startup) {
2924                 /*
2925                  * If the device's dma_mask is less than the system's memory
2926                  * size then this is not a candidate for identity mapping.
2927                  */
2928                 u64 dma_mask = *dev->dma_mask;
2929
2930                 if (dev->coherent_dma_mask &&
2931                     dev->coherent_dma_mask < dma_mask)
2932                         dma_mask = dev->coherent_dma_mask;
2933
2934                 return dma_mask >= dma_get_required_mask(dev);
2935         }
2936
2937         return 1;
2938 }
2939
2940 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2941 {
2942         int ret;
2943
2944         if (!iommu_should_identity_map(dev, 1))
2945                 return 0;
2946
2947         ret = domain_add_dev_info(si_domain, dev);
2948         if (!ret)
2949                 pr_info("%s identity mapping for device %s\n",
2950                         hw ? "Hardware" : "Software", dev_name(dev));
2951         else if (ret == -ENODEV)
2952                 /* device not associated with an iommu */
2953                 ret = 0;
2954
2955         return ret;
2956 }
2957
2958
2959 static int __init iommu_prepare_static_identity_mapping(int hw)
2960 {
2961         struct pci_dev *pdev = NULL;
2962         struct dmar_drhd_unit *drhd;
2963         struct intel_iommu *iommu;
2964         struct device *dev;
2965         int i;
2966         int ret = 0;
2967
2968         for_each_pci_dev(pdev) {
2969                 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2970                 if (ret)
2971                         return ret;
2972         }
2973
2974         for_each_active_iommu(iommu, drhd)
2975                 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2976                         struct acpi_device_physical_node *pn;
2977                         struct acpi_device *adev;
2978
2979                         if (dev->bus != &acpi_bus_type)
2980                                 continue;
2981
2982                         adev= to_acpi_device(dev);
2983                         mutex_lock(&adev->physical_node_lock);
2984                         list_for_each_entry(pn, &adev->physical_node_list, node) {
2985                                 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2986                                 if (ret)
2987                                         break;
2988                         }
2989                         mutex_unlock(&adev->physical_node_lock);
2990                         if (ret)
2991                                 return ret;
2992                 }
2993
2994         return 0;
2995 }
2996
2997 static void intel_iommu_init_qi(struct intel_iommu *iommu)
2998 {
2999         /*
3000          * Start from the sane iommu hardware state.
3001          * If the queued invalidation is already initialized by us
3002          * (for example, while enabling interrupt-remapping) then
3003          * we got the things already rolling from a sane state.
3004          */
3005         if (!iommu->qi) {
3006                 /*
3007                  * Clear any previous faults.
3008                  */
3009                 dmar_fault(-1, iommu);
3010                 /*
3011                  * Disable queued invalidation if supported and already enabled
3012                  * before OS handover.
3013                  */
3014                 dmar_disable_qi(iommu);
3015         }
3016
3017         if (dmar_enable_qi(iommu)) {
3018                 /*
3019                  * Queued Invalidate not enabled, use Register Based Invalidate
3020                  */
3021                 iommu->flush.flush_context = __iommu_flush_context;
3022                 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
3023                 pr_info("%s: Using Register based invalidation\n",
3024                         iommu->name);
3025         } else {
3026                 iommu->flush.flush_context = qi_flush_context;
3027                 iommu->flush.flush_iotlb = qi_flush_iotlb;
3028                 pr_info("%s: Using Queued invalidation\n", iommu->name);
3029         }
3030 }
3031
3032 static int copy_context_table(struct intel_iommu *iommu,
3033                               struct root_entry *old_re,
3034                               struct context_entry **tbl,
3035                               int bus, bool ext)
3036 {
3037         int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
3038         struct context_entry *new_ce = NULL, ce;
3039         struct context_entry *old_ce = NULL;
3040         struct root_entry re;
3041         phys_addr_t old_ce_phys;
3042
3043         tbl_idx = ext ? bus * 2 : bus;
3044         memcpy(&re, old_re, sizeof(re));
3045
3046         for (devfn = 0; devfn < 256; devfn++) {
3047                 /* First calculate the correct index */
3048                 idx = (ext ? devfn * 2 : devfn) % 256;
3049
3050                 if (idx == 0) {
3051                         /* First save what we may have and clean up */
3052                         if (new_ce) {
3053                                 tbl[tbl_idx] = new_ce;
3054                                 __iommu_flush_cache(iommu, new_ce,
3055                                                     VTD_PAGE_SIZE);
3056                                 pos = 1;
3057                         }
3058
3059                         if (old_ce)
3060                                 memunmap(old_ce);
3061
3062                         ret = 0;
3063                         if (devfn < 0x80)
3064                                 old_ce_phys = root_entry_lctp(&re);
3065                         else
3066                                 old_ce_phys = root_entry_uctp(&re);
3067
3068                         if (!old_ce_phys) {
3069                                 if (ext && devfn == 0) {
3070                                         /* No LCTP, try UCTP */
3071                                         devfn = 0x7f;
3072                                         continue;
3073                                 } else {
3074                                         goto out;
3075                                 }
3076                         }
3077
3078                         ret = -ENOMEM;
3079                         old_ce = memremap(old_ce_phys, PAGE_SIZE,
3080                                         MEMREMAP_WB);
3081                         if (!old_ce)
3082                                 goto out;
3083
3084                         new_ce = alloc_pgtable_page(iommu->node);
3085                         if (!new_ce)
3086                                 goto out_unmap;
3087
3088                         ret = 0;
3089                 }
3090
3091                 /* Now copy the context entry */
3092                 memcpy(&ce, old_ce + idx, sizeof(ce));
3093
3094                 if (!__context_present(&ce))
3095                         continue;
3096
3097                 did = context_domain_id(&ce);
3098                 if (did >= 0 && did < cap_ndoms(iommu->cap))
3099                         set_bit(did, iommu->domain_ids);
3100
3101                 /*
3102                  * We need a marker for copied context entries. This
3103                  * marker needs to work for the old format as well as
3104                  * for extended context entries.
3105                  *
3106                  * Bit 67 of the context entry is used. In the old
3107                  * format this bit is available to software, in the
3108                  * extended format it is the PGE bit, but PGE is ignored
3109                  * by HW if PASIDs are disabled (and thus still
3110                  * available).
3111                  *
3112                  * So disable PASIDs first and then mark the entry
3113                  * copied. This means that we don't copy PASID
3114                  * translations from the old kernel, but this is fine as
3115                  * faults there are not fatal.
3116                  */
3117                 context_clear_pasid_enable(&ce);
3118                 context_set_copied(&ce);
3119
3120                 new_ce[idx] = ce;
3121         }
3122
3123         tbl[tbl_idx + pos] = new_ce;
3124
3125         __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3126
3127 out_unmap:
3128         memunmap(old_ce);
3129
3130 out:
3131         return ret;
3132 }
3133
3134 static int copy_translation_tables(struct intel_iommu *iommu)
3135 {
3136         struct context_entry **ctxt_tbls;
3137         struct root_entry *old_rt;
3138         phys_addr_t old_rt_phys;
3139         int ctxt_table_entries;
3140         unsigned long flags;
3141         u64 rtaddr_reg;
3142         int bus, ret;
3143         bool new_ext, ext;
3144
3145         rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3146         ext        = !!(rtaddr_reg & DMA_RTADDR_RTT);
3147         new_ext    = !!ecap_ecs(iommu->ecap);
3148
3149         /*
3150          * The RTT bit can only be changed when translation is disabled,
3151          * but disabling translation means to open a window for data
3152          * corruption. So bail out and don't copy anything if we would
3153          * have to change the bit.
3154          */
3155         if (new_ext != ext)
3156                 return -EINVAL;
3157
3158         old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3159         if (!old_rt_phys)
3160                 return -EINVAL;
3161
3162         old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
3163         if (!old_rt)
3164                 return -ENOMEM;
3165
3166         /* This is too big for the stack - allocate it from slab */
3167         ctxt_table_entries = ext ? 512 : 256;
3168         ret = -ENOMEM;
3169         ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
3170         if (!ctxt_tbls)
3171                 goto out_unmap;
3172
3173         for (bus = 0; bus < 256; bus++) {
3174                 ret = copy_context_table(iommu, &old_rt[bus],
3175                                          ctxt_tbls, bus, ext);
3176                 if (ret) {
3177                         pr_err("%s: Failed to copy context table for bus %d\n",
3178                                 iommu->name, bus);
3179                         continue;
3180                 }
3181         }
3182
3183         spin_lock_irqsave(&iommu->lock, flags);
3184
3185         /* Context tables are copied, now write them to the root_entry table */
3186         for (bus = 0; bus < 256; bus++) {
3187                 int idx = ext ? bus * 2 : bus;
3188                 u64 val;
3189
3190                 if (ctxt_tbls[idx]) {
3191                         val = virt_to_phys(ctxt_tbls[idx]) | 1;
3192                         iommu->root_entry[bus].lo = val;
3193                 }
3194
3195                 if (!ext || !ctxt_tbls[idx + 1])
3196                         continue;
3197
3198                 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3199                 iommu->root_entry[bus].hi = val;
3200         }
3201
3202         spin_unlock_irqrestore(&iommu->lock, flags);
3203
3204         kfree(ctxt_tbls);
3205
3206         __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3207
3208         ret = 0;
3209
3210 out_unmap:
3211         memunmap(old_rt);
3212
3213         return ret;
3214 }
3215
3216 static int __init init_dmars(void)
3217 {
3218         struct dmar_drhd_unit *drhd;
3219         struct dmar_rmrr_unit *rmrr;
3220         bool copied_tables = false;
3221         struct device *dev;
3222         struct intel_iommu *iommu;
3223         int i, ret, cpu;
3224
3225         /*
3226          * for each drhd
3227          *    allocate root
3228          *    initialize and program root entry to not present
3229          * endfor
3230          */
3231         for_each_drhd_unit(drhd) {
3232                 /*
3233                  * lock not needed as this is only incremented in the single
3234                  * threaded kernel __init code path all other access are read
3235                  * only
3236                  */
3237                 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3238                         g_num_of_iommus++;
3239                         continue;
3240                 }
3241                 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3242         }
3243
3244         /* Preallocate enough resources for IOMMU hot-addition */
3245         if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3246                 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3247
3248         g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3249                         GFP_KERNEL);
3250         if (!g_iommus) {
3251                 pr_err("Allocating global iommu array failed\n");
3252                 ret = -ENOMEM;
3253                 goto error;
3254         }
3255
3256         for_each_possible_cpu(cpu) {
3257                 struct deferred_flush_data *dfd = per_cpu_ptr(&deferred_flush,
3258                                                               cpu);
3259
3260                 dfd->tables = kzalloc(g_num_of_iommus *
3261                                       sizeof(struct deferred_flush_table),
3262                                       GFP_KERNEL);
3263                 if (!dfd->tables) {
3264                         ret = -ENOMEM;
3265                         goto free_g_iommus;
3266                 }
3267
3268                 spin_lock_init(&dfd->lock);
3269                 setup_timer(&dfd->timer, flush_unmaps_timeout, cpu);
3270         }
3271
3272         for_each_active_iommu(iommu, drhd) {
3273                 g_iommus[iommu->seq_id] = iommu;
3274
3275                 intel_iommu_init_qi(iommu);
3276
3277                 ret = iommu_init_domains(iommu);
3278                 if (ret)
3279                         goto free_iommu;
3280
3281                 init_translation_status(iommu);
3282
3283                 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3284                         iommu_disable_translation(iommu);
3285                         clear_translation_pre_enabled(iommu);
3286                         pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3287                                 iommu->name);
3288                 }
3289
3290                 /*
3291                  * TBD:
3292                  * we could share the same root & context tables
3293                  * among all IOMMU's. Need to Split it later.
3294                  */
3295                 ret = iommu_alloc_root_entry(iommu);
3296                 if (ret)
3297                         goto free_iommu;
3298
3299                 if (translation_pre_enabled(iommu)) {
3300                         pr_info("Translation already enabled - trying to copy translation structures\n");
3301
3302                         ret = copy_translation_tables(iommu);
3303                         if (ret) {
3304                                 /*
3305                                  * We found the IOMMU with translation
3306                                  * enabled - but failed to copy over the
3307                                  * old root-entry table. Try to proceed
3308                                  * by disabling translation now and
3309                                  * allocating a clean root-entry table.
3310                                  * This might cause DMAR faults, but
3311                                  * probably the dump will still succeed.
3312                                  */
3313                                 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3314                                        iommu->name);
3315                                 iommu_disable_translation(iommu);
3316                                 clear_translation_pre_enabled(iommu);
3317                         } else {
3318                                 pr_info("Copied translation tables from previous kernel for %s\n",
3319                                         iommu->name);
3320                                 copied_tables = true;
3321                         }
3322                 }
3323
3324                 if (!ecap_pass_through(iommu->ecap))
3325                         hw_pass_through = 0;
3326 #ifdef CONFIG_INTEL_IOMMU_SVM
3327                 if (pasid_enabled(iommu))
3328                         intel_svm_alloc_pasid_tables(iommu);
3329 #endif
3330         }
3331
3332         /*
3333          * Now that qi is enabled on all iommus, set the root entry and flush
3334          * caches. This is required on some Intel X58 chipsets, otherwise the
3335          * flush_context function will loop forever and the boot hangs.
3336          */
3337         for_each_active_iommu(iommu, drhd) {
3338                 iommu_flush_write_buffer(iommu);
3339                 iommu_set_root_entry(iommu);
3340                 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3341                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3342         }
3343
3344         if (iommu_pass_through)
3345                 iommu_identity_mapping |= IDENTMAP_ALL;
3346
3347 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3348         iommu_identity_mapping |= IDENTMAP_GFX;
3349 #endif
3350
3351         check_tylersburg_isoch();
3352
3353         if (iommu_identity_mapping) {
3354                 ret = si_domain_init(hw_pass_through);
3355                 if (ret)
3356                         goto free_iommu;
3357         }
3358
3359
3360         /*
3361          * If we copied translations from a previous kernel in the kdump
3362          * case, we can not assign the devices to domains now, as that
3363          * would eliminate the old mappings. So skip this part and defer
3364          * the assignment to device driver initialization time.
3365          */
3366         if (copied_tables)
3367                 goto domains_done;
3368
3369         /*
3370          * If pass through is not set or not enabled, setup context entries for
3371          * identity mappings for rmrr, gfx, and isa and may fall back to static
3372          * identity mapping if iommu_identity_mapping is set.
3373          */
3374         if (iommu_identity_mapping) {
3375                 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3376                 if (ret) {
3377                         pr_crit("Failed to setup IOMMU pass-through\n");
3378                         goto free_iommu;
3379                 }
3380         }
3381         /*
3382          * For each rmrr
3383          *   for each dev attached to rmrr
3384          *   do
3385          *     locate drhd for dev, alloc domain for dev
3386          *     allocate free domain
3387          *     allocate page table entries for rmrr
3388          *     if context not allocated for bus
3389          *           allocate and init context
3390          *           set present in root table for this bus
3391          *     init context with domain, translation etc
3392          *    endfor
3393          * endfor
3394          */
3395         pr_info("Setting RMRR:\n");
3396         for_each_rmrr_units(rmrr) {
3397                 /* some BIOS lists non-exist devices in DMAR table. */
3398                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3399                                           i, dev) {
3400                         ret = iommu_prepare_rmrr_dev(rmrr, dev);
3401                         if (ret)
3402                                 pr_err("Mapping reserved region failed\n");
3403                 }
3404         }
3405
3406         iommu_prepare_isa();
3407
3408 domains_done:
3409
3410         /*
3411          * for each drhd
3412          *   enable fault log
3413          *   global invalidate context cache
3414          *   global invalidate iotlb
3415          *   enable translation
3416          */
3417         for_each_iommu(iommu, drhd) {
3418                 if (drhd->ignored) {
3419                         /*
3420                          * we always have to disable PMRs or DMA may fail on
3421                          * this device
3422                          */
3423                         if (force_on)
3424                                 iommu_disable_protect_mem_regions(iommu);
3425                         continue;
3426                 }
3427
3428                 iommu_flush_write_buffer(iommu);
3429
3430 #ifdef CONFIG_INTEL_IOMMU_SVM
3431                 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
3432                         ret = intel_svm_enable_prq(iommu);
3433                         if (ret)
3434                                 goto free_iommu;
3435                 }
3436 #endif
3437                 ret = dmar_set_interrupt(iommu);
3438                 if (ret)
3439                         goto free_iommu;
3440
3441                 if (!translation_pre_enabled(iommu))
3442                         iommu_enable_translation(iommu);
3443
3444                 iommu_disable_protect_mem_regions(iommu);
3445         }
3446
3447         return 0;
3448
3449 free_iommu:
3450         for_each_active_iommu(iommu, drhd) {
3451                 disable_dmar_iommu(iommu);
3452                 free_dmar_iommu(iommu);
3453         }
3454 free_g_iommus:
3455         for_each_possible_cpu(cpu)
3456                 kfree(per_cpu_ptr(&deferred_flush, cpu)->tables);
3457         kfree(g_iommus);
3458 error:
3459         return ret;
3460 }
3461
3462 /* This takes a number of _MM_ pages, not VTD pages */
3463 static unsigned long intel_alloc_iova(struct device *dev,
3464                                      struct dmar_domain *domain,
3465                                      unsigned long nrpages, uint64_t dma_mask)
3466 {
3467         unsigned long iova_pfn = 0;
3468
3469         /* Restrict dma_mask to the width that the iommu can handle */
3470         dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3471         /* Ensure we reserve the whole size-aligned region */
3472         nrpages = __roundup_pow_of_two(nrpages);
3473
3474         if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3475                 /*
3476                  * First try to allocate an io virtual address in
3477                  * DMA_BIT_MASK(32) and if that fails then try allocating
3478                  * from higher range
3479                  */
3480                 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3481                                            IOVA_PFN(DMA_BIT_MASK(32)));
3482                 if (iova_pfn)
3483                         return iova_pfn;
3484         }
3485         iova_pfn = alloc_iova_fast(&domain->iovad, nrpages, IOVA_PFN(dma_mask));
3486         if (unlikely(!iova_pfn)) {
3487                 pr_err("Allocating %ld-page iova for %s failed",
3488                        nrpages, dev_name(dev));
3489                 return 0;
3490         }
3491
3492         return iova_pfn;
3493 }
3494
3495 static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
3496 {
3497         struct dmar_domain *domain, *tmp;
3498         struct dmar_rmrr_unit *rmrr;
3499         struct device *i_dev;
3500         int i, ret;
3501
3502         domain = find_domain(dev);
3503         if (domain)
3504                 goto out;
3505
3506         domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3507         if (!domain)
3508                 goto out;
3509
3510         /* We have a new domain - setup possible RMRRs for the device */
3511         rcu_read_lock();
3512         for_each_rmrr_units(rmrr) {
3513                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3514                                           i, i_dev) {
3515                         if (i_dev != dev)
3516                                 continue;
3517
3518                         ret = domain_prepare_identity_map(dev, domain,
3519                                                           rmrr->base_address,
3520                                                           rmrr->end_address);
3521                         if (ret)
3522                                 dev_err(dev, "Mapping reserved region failed\n");
3523                 }
3524         }
3525         rcu_read_unlock();
3526
3527         tmp = set_domain_for_dev(dev, domain);
3528         if (!tmp || domain != tmp) {
3529                 domain_exit(domain);
3530                 domain = tmp;
3531         }
3532
3533 out:
3534
3535         if (!domain)
3536                 pr_err("Allocating domain for %s failed\n", dev_name(dev));
3537
3538
3539         return domain;
3540 }
3541
3542 static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3543 {
3544         struct device_domain_info *info;
3545
3546         /* No lock here, assumes no domain exit in normal case */
3547         info = dev->archdata.iommu;
3548         if (likely(info))
3549                 return info->domain;
3550
3551         return __get_valid_domain_for_dev(dev);
3552 }
3553
3554 /* Check if the dev needs to go through non-identity map and unmap process.*/
3555 static int iommu_no_mapping(struct device *dev)
3556 {
3557         int found;
3558
3559         if (iommu_dummy(dev))
3560                 return 1;
3561
3562         if (!iommu_identity_mapping)
3563                 return 0;
3564
3565         found = identity_mapping(dev);
3566         if (found) {
3567                 if (iommu_should_identity_map(dev, 0))
3568                         return 1;
3569                 else {
3570                         /*
3571                          * 32 bit DMA is removed from si_domain and fall back
3572                          * to non-identity mapping.
3573                          */
3574                         dmar_remove_one_dev_info(si_domain, dev);
3575                         pr_info("32bit %s uses non-identity mapping\n",
3576                                 dev_name(dev));
3577                         return 0;
3578                 }
3579         } else {
3580                 /*
3581                  * In case of a detached 64 bit DMA device from vm, the device
3582                  * is put into si_domain for identity mapping.
3583                  */
3584                 if (iommu_should_identity_map(dev, 0)) {
3585                         int ret;
3586                         ret = domain_add_dev_info(si_domain, dev);
3587                         if (!ret) {
3588                                 pr_info("64bit %s uses identity mapping\n",
3589                                         dev_name(dev));
3590                                 return 1;
3591                         }
3592                 }
3593         }
3594
3595         return 0;
3596 }
3597
3598 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3599                                      size_t size, int dir, u64 dma_mask)
3600 {
3601         struct dmar_domain *domain;
3602         phys_addr_t start_paddr;
3603         unsigned long iova_pfn;
3604         int prot = 0;
3605         int ret;
3606         struct intel_iommu *iommu;
3607         unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3608
3609         BUG_ON(dir == DMA_NONE);
3610
3611         if (iommu_no_mapping(dev))
3612                 return paddr;
3613
3614         domain = get_valid_domain_for_dev(dev);
3615         if (!domain)
3616                 return 0;
3617
3618         iommu = domain_get_iommu(domain);
3619         size = aligned_nrpages(paddr, size);
3620
3621         iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3622         if (!iova_pfn)
3623                 goto error;
3624
3625         /*
3626          * Check if DMAR supports zero-length reads on write only
3627          * mappings..
3628          */
3629         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3630                         !cap_zlr(iommu->cap))
3631                 prot |= DMA_PTE_READ;
3632         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3633                 prot |= DMA_PTE_WRITE;
3634         /*
3635          * paddr - (paddr + size) might be partial page, we should map the whole
3636          * page.  Note: if two part of one page are separately mapped, we
3637          * might have two guest_addr mapping to the same host paddr, but this
3638          * is not a big problem
3639          */
3640         ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
3641                                  mm_to_dma_pfn(paddr_pfn), size, prot);
3642         if (ret)
3643                 goto error;
3644
3645         /* it's a non-present to present mapping. Only flush if caching mode */
3646         if (cap_caching_mode(iommu->cap))
3647                 iommu_flush_iotlb_psi(iommu, domain,
3648                                       mm_to_dma_pfn(iova_pfn),
3649                                       size, 0, 1);
3650         else
3651                 iommu_flush_write_buffer(iommu);
3652
3653         start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
3654         start_paddr += paddr & ~PAGE_MASK;
3655         return start_paddr;
3656
3657 error:
3658         if (iova_pfn)
3659                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3660         pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
3661                 dev_name(dev), size, (unsigned long long)paddr, dir);
3662         return 0;
3663 }
3664
3665 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3666                                  unsigned long offset, size_t size,
3667                                  enum dma_data_direction dir,
3668                                  unsigned long attrs)
3669 {
3670         return __intel_map_single(dev, page_to_phys(page) + offset, size,
3671                                   dir, *dev->dma_mask);
3672 }
3673
3674 static void flush_unmaps(struct deferred_flush_data *flush_data)
3675 {
3676         int i, j;
3677
3678         flush_data->timer_on = 0;
3679
3680         /* just flush them all */
3681         for (i = 0; i < g_num_of_iommus; i++) {
3682                 struct intel_iommu *iommu = g_iommus[i];
3683                 struct deferred_flush_table *flush_table =
3684                                 &flush_data->tables[i];
3685                 if (!iommu)
3686                         continue;
3687
3688                 if (!flush_table->next)
3689                         continue;
3690
3691                 /* In caching mode, global flushes turn emulation expensive */
3692                 if (!cap_caching_mode(iommu->cap))
3693                         iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3694                                          DMA_TLB_GLOBAL_FLUSH);
3695                 for (j = 0; j < flush_table->next; j++) {
3696                         unsigned long mask;
3697                         struct deferred_flush_entry *entry =
3698                                                 &flush_table->entries[j];
3699                         unsigned long iova_pfn = entry->iova_pfn;
3700                         unsigned long nrpages = entry->nrpages;
3701                         struct dmar_domain *domain = entry->domain;
3702                         struct page *freelist = entry->freelist;
3703
3704                         /* On real hardware multiple invalidations are expensive */
3705                         if (cap_caching_mode(iommu->cap))
3706                                 iommu_flush_iotlb_psi(iommu, domain,
3707                                         mm_to_dma_pfn(iova_pfn),
3708                                         nrpages, !freelist, 0);
3709                         else {
3710                                 mask = ilog2(nrpages);
3711                                 iommu_flush_dev_iotlb(domain,
3712                                                 (uint64_t)iova_pfn << PAGE_SHIFT, mask);
3713                         }
3714                         free_iova_fast(&domain->iovad, iova_pfn, nrpages);
3715                         if (freelist)
3716                                 dma_free_pagelist(freelist);
3717                 }
3718                 flush_table->next = 0;
3719         }
3720
3721         flush_data->size = 0;
3722 }
3723
3724 static void flush_unmaps_timeout(unsigned long cpuid)
3725 {
3726         struct deferred_flush_data *flush_data = per_cpu_ptr(&deferred_flush, cpuid);
3727         unsigned long flags;
3728
3729         spin_lock_irqsave(&flush_data->lock, flags);
3730         flush_unmaps(flush_data);
3731         spin_unlock_irqrestore(&flush_data->lock, flags);
3732 }
3733
3734 static void add_unmap(struct dmar_domain *dom, unsigned long iova_pfn,
3735                       unsigned long nrpages, struct page *freelist)
3736 {
3737         unsigned long flags;
3738         int entry_id, iommu_id;
3739         struct intel_iommu *iommu;
3740         struct deferred_flush_entry *entry;
3741         struct deferred_flush_data *flush_data;
3742         unsigned int cpuid;
3743
3744         cpuid = get_cpu();
3745         flush_data = per_cpu_ptr(&deferred_flush, cpuid);
3746
3747         /* Flush all CPUs' entries to avoid deferring too much.  If
3748          * this becomes a bottleneck, can just flush us, and rely on
3749          * flush timer for the rest.
3750          */
3751         if (flush_data->size == HIGH_WATER_MARK) {
3752                 int cpu;
3753
3754                 for_each_online_cpu(cpu)
3755                         flush_unmaps_timeout(cpu);
3756         }
3757
3758         spin_lock_irqsave(&flush_data->lock, flags);
3759
3760         iommu = domain_get_iommu(dom);
3761         iommu_id = iommu->seq_id;
3762
3763         entry_id = flush_data->tables[iommu_id].next;
3764         ++(flush_data->tables[iommu_id].next);
3765
3766         entry = &flush_data->tables[iommu_id].entries[entry_id];
3767         entry->domain = dom;
3768         entry->iova_pfn = iova_pfn;
3769         entry->nrpages = nrpages;
3770         entry->freelist = freelist;
3771
3772         if (!flush_data->timer_on) {
3773                 mod_timer(&flush_data->timer, jiffies + msecs_to_jiffies(10));
3774                 flush_data->timer_on = 1;
3775         }
3776         flush_data->size++;
3777         spin_unlock_irqrestore(&flush_data->lock, flags);
3778
3779         put_cpu();
3780 }
3781
3782 static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
3783 {
3784         struct dmar_domain *domain;
3785         unsigned long start_pfn, last_pfn;
3786         unsigned long nrpages;
3787         unsigned long iova_pfn;
3788         struct intel_iommu *iommu;
3789         struct page *freelist;
3790
3791         if (iommu_no_mapping(dev))
3792                 return;
3793
3794         domain = find_domain(dev);
3795         BUG_ON(!domain);
3796
3797         iommu = domain_get_iommu(domain);
3798
3799         iova_pfn = IOVA_PFN(dev_addr);
3800
3801         nrpages = aligned_nrpages(dev_addr, size);
3802         start_pfn = mm_to_dma_pfn(iova_pfn);
3803         last_pfn = start_pfn + nrpages - 1;
3804
3805         pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3806                  dev_name(dev), start_pfn, last_pfn);
3807
3808         freelist = domain_unmap(domain, start_pfn, last_pfn);
3809
3810         if (intel_iommu_strict) {
3811                 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3812                                       nrpages, !freelist, 0);
3813                 /* free iova */
3814                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
3815                 dma_free_pagelist(freelist);
3816         } else {
3817                 add_unmap(domain, iova_pfn, nrpages, freelist);
3818                 /*
3819                  * queue up the release of the unmap to save the 1/6th of the
3820                  * cpu used up by the iotlb flush operation...
3821                  */
3822         }
3823 }
3824
3825 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3826                              size_t size, enum dma_data_direction dir,
3827                              unsigned long attrs)
3828 {
3829         intel_unmap(dev, dev_addr, size);
3830 }
3831
3832 static void *intel_alloc_coherent(struct device *dev, size_t size,
3833                                   dma_addr_t *dma_handle, gfp_t flags,
3834                                   unsigned long attrs)
3835 {
3836         struct page *page = NULL;
3837         int order;
3838
3839         size = PAGE_ALIGN(size);
3840         order = get_order(size);
3841
3842         if (!iommu_no_mapping(dev))
3843                 flags &= ~(GFP_DMA | GFP_DMA32);
3844         else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3845                 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3846                         flags |= GFP_DMA;
3847                 else
3848                         flags |= GFP_DMA32;
3849         }
3850
3851         if (gfpflags_allow_blocking(flags)) {
3852                 unsigned int count = size >> PAGE_SHIFT;
3853
3854                 page = dma_alloc_from_contiguous(dev, count, order);
3855                 if (page && iommu_no_mapping(dev) &&
3856                     page_to_phys(page) + size > dev->coherent_dma_mask) {
3857                         dma_release_from_contiguous(dev, page, count);
3858                         page = NULL;
3859                 }
3860         }
3861
3862         if (!page)
3863                 page = alloc_pages(flags, order);
3864         if (!page)
3865                 return NULL;
3866         memset(page_address(page), 0, size);
3867
3868         *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3869                                          DMA_BIDIRECTIONAL,
3870                                          dev->coherent_dma_mask);
3871         if (*dma_handle)
3872                 return page_address(page);
3873         if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3874                 __free_pages(page, order);
3875
3876         return NULL;
3877 }
3878
3879 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3880                                 dma_addr_t dma_handle, unsigned long attrs)
3881 {
3882         int order;
3883         struct page *page = virt_to_page(vaddr);
3884
3885         size = PAGE_ALIGN(size);
3886         order = get_order(size);
3887
3888         intel_unmap(dev, dma_handle, size);
3889         if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3890                 __free_pages(page, order);
3891 }
3892
3893 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3894                            int nelems, enum dma_data_direction dir,
3895                            unsigned long attrs)
3896 {
3897         dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
3898         unsigned long nrpages = 0;
3899         struct scatterlist *sg;
3900         int i;
3901
3902         for_each_sg(sglist, sg, nelems, i) {
3903                 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3904         }
3905
3906         intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
3907 }
3908
3909 static int intel_nontranslate_map_sg(struct device *hddev,
3910         struct scatterlist *sglist, int nelems, int dir)
3911 {
3912         int i;
3913         struct scatterlist *sg;
3914
3915         for_each_sg(sglist, sg, nelems, i) {
3916                 BUG_ON(!sg_page(sg));
3917                 sg->dma_address = sg_phys(sg);
3918                 sg->dma_length = sg->length;
3919         }
3920         return nelems;
3921 }
3922
3923 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3924                         enum dma_data_direction dir, unsigned long attrs)
3925 {
3926         int i;
3927         struct dmar_domain *domain;
3928         size_t size = 0;
3929         int prot = 0;
3930         unsigned long iova_pfn;
3931         int ret;
3932         struct scatterlist *sg;
3933         unsigned long start_vpfn;
3934         struct intel_iommu *iommu;
3935
3936         BUG_ON(dir == DMA_NONE);
3937         if (iommu_no_mapping(dev))
3938                 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3939
3940         domain = get_valid_domain_for_dev(dev);
3941         if (!domain)
3942                 return 0;
3943
3944         iommu = domain_get_iommu(domain);
3945
3946         for_each_sg(sglist, sg, nelems, i)
3947                 size += aligned_nrpages(sg->offset, sg->length);
3948
3949         iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3950                                 *dev->dma_mask);
3951         if (!iova_pfn) {
3952                 sglist->dma_length = 0;
3953                 return 0;
3954         }
3955
3956         /*
3957          * Check if DMAR supports zero-length reads on write only
3958          * mappings..
3959          */
3960         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3961                         !cap_zlr(iommu->cap))
3962                 prot |= DMA_PTE_READ;
3963         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3964                 prot |= DMA_PTE_WRITE;
3965
3966         start_vpfn = mm_to_dma_pfn(iova_pfn);
3967
3968         ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3969         if (unlikely(ret)) {
3970                 dma_pte_free_pagetable(domain, start_vpfn,
3971                                        start_vpfn + size - 1);
3972                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3973                 return 0;
3974         }
3975
3976         /* it's a non-present to present mapping. Only flush if caching mode */
3977         if (cap_caching_mode(iommu->cap))
3978                 iommu_flush_iotlb_psi(iommu, domain, start_vpfn, size, 0, 1);
3979         else
3980                 iommu_flush_write_buffer(iommu);
3981
3982         return nelems;
3983 }
3984
3985 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3986 {
3987         return !dma_addr;
3988 }
3989
3990 struct dma_map_ops intel_dma_ops = {
3991         .alloc = intel_alloc_coherent,
3992         .free = intel_free_coherent,
3993         .map_sg = intel_map_sg,
3994         .unmap_sg = intel_unmap_sg,
3995         .map_page = intel_map_page,
3996         .unmap_page = intel_unmap_page,
3997         .mapping_error = intel_mapping_error,
3998 };
3999
4000 static inline int iommu_domain_cache_init(void)
4001 {
4002         int ret = 0;
4003
4004         iommu_domain_cache = kmem_cache_create("iommu_domain",
4005                                          sizeof(struct dmar_domain),
4006                                          0,
4007                                          SLAB_HWCACHE_ALIGN,
4008
4009                                          NULL);
4010         if (!iommu_domain_cache) {
4011                 pr_err("Couldn't create iommu_domain cache\n");
4012                 ret = -ENOMEM;
4013         }
4014
4015         return ret;
4016 }
4017
4018 static inline int iommu_devinfo_cache_init(void)
4019 {
4020         int ret = 0;
4021
4022         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
4023                                          sizeof(struct device_domain_info),
4024                                          0,
4025                                          SLAB_HWCACHE_ALIGN,
4026                                          NULL);
4027         if (!iommu_devinfo_cache) {
4028                 pr_err("Couldn't create devinfo cache\n");
4029                 ret = -ENOMEM;
4030         }
4031
4032         return ret;
4033 }
4034
4035 static int __init iommu_init_mempool(void)
4036 {
4037         int ret;
4038         ret = iova_cache_get();
4039         if (ret)
4040                 return ret;
4041
4042         ret = iommu_domain_cache_init();
4043         if (ret)
4044                 goto domain_error;
4045
4046         ret = iommu_devinfo_cache_init();
4047         if (!ret)
4048                 return ret;
4049
4050         kmem_cache_destroy(iommu_domain_cache);
4051 domain_error:
4052         iova_cache_put();
4053
4054         return -ENOMEM;
4055 }
4056
4057 static void __init iommu_exit_mempool(void)
4058 {
4059         kmem_cache_destroy(iommu_devinfo_cache);
4060         kmem_cache_destroy(iommu_domain_cache);
4061         iova_cache_put();
4062 }
4063
4064 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
4065 {
4066         struct dmar_drhd_unit *drhd;
4067         u32 vtbar;
4068         int rc;
4069
4070         /* We know that this device on this chipset has its own IOMMU.
4071          * If we find it under a different IOMMU, then the BIOS is lying
4072          * to us. Hope that the IOMMU for this device is actually
4073          * disabled, and it needs no translation...
4074          */
4075         rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
4076         if (rc) {
4077                 /* "can't" happen */
4078                 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
4079                 return;
4080         }
4081         vtbar &= 0xffff0000;
4082
4083         /* we know that the this iommu should be at offset 0xa000 from vtbar */
4084         drhd = dmar_find_matched_drhd_unit(pdev);
4085         if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
4086                             TAINT_FIRMWARE_WORKAROUND,
4087                             "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
4088                 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4089 }
4090 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
4091
4092 static void __init init_no_remapping_devices(void)
4093 {
4094         struct dmar_drhd_unit *drhd;
4095         struct device *dev;
4096         int i;
4097
4098         for_each_drhd_unit(drhd) {
4099                 if (!drhd->include_all) {
4100                         for_each_active_dev_scope(drhd->devices,
4101                                                   drhd->devices_cnt, i, dev)
4102                                 break;
4103                         /* ignore DMAR unit if no devices exist */
4104                         if (i == drhd->devices_cnt)
4105                                 drhd->ignored = 1;
4106                 }
4107         }
4108
4109         for_each_active_drhd_unit(drhd) {
4110                 if (drhd->include_all)
4111                         continue;
4112
4113                 for_each_active_dev_scope(drhd->devices,
4114                                           drhd->devices_cnt, i, dev)
4115                         if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
4116                                 break;
4117                 if (i < drhd->devices_cnt)
4118                         continue;
4119
4120                 /* This IOMMU has *only* gfx devices. Either bypass it or
4121                    set the gfx_mapped flag, as appropriate */
4122                 if (!dmar_map_gfx) {
4123                         drhd->ignored = 1;
4124                         for_each_active_dev_scope(drhd->devices,
4125                                                   drhd->devices_cnt, i, dev)
4126                                 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4127                 }
4128         }
4129 }
4130
4131 #ifdef CONFIG_SUSPEND
4132 static int init_iommu_hw(void)
4133 {
4134         struct dmar_drhd_unit *drhd;
4135         struct intel_iommu *iommu = NULL;
4136
4137         for_each_active_iommu(iommu, drhd)
4138                 if (iommu->qi)
4139                         dmar_reenable_qi(iommu);
4140
4141         for_each_iommu(iommu, drhd) {
4142                 if (drhd->ignored) {
4143                         /*
4144                          * we always have to disable PMRs or DMA may fail on
4145                          * this device
4146                          */
4147                         if (force_on)
4148                                 iommu_disable_protect_mem_regions(iommu);
4149                         continue;
4150                 }
4151
4152                 iommu_flush_write_buffer(iommu);
4153
4154                 iommu_set_root_entry(iommu);
4155
4156                 iommu->flush.flush_context(iommu, 0, 0, 0,
4157                                            DMA_CCMD_GLOBAL_INVL);
4158                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4159                 iommu_enable_translation(iommu);
4160                 iommu_disable_protect_mem_regions(iommu);
4161         }
4162
4163         return 0;
4164 }
4165
4166 static void iommu_flush_all(void)
4167 {
4168         struct dmar_drhd_unit *drhd;
4169         struct intel_iommu *iommu;
4170
4171         for_each_active_iommu(iommu, drhd) {
4172                 iommu->flush.flush_context(iommu, 0, 0, 0,
4173                                            DMA_CCMD_GLOBAL_INVL);
4174                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
4175                                          DMA_TLB_GLOBAL_FLUSH);
4176         }
4177 }
4178
4179 static int iommu_suspend(void)
4180 {
4181         struct dmar_drhd_unit *drhd;
4182         struct intel_iommu *iommu = NULL;
4183         unsigned long flag;
4184
4185         for_each_active_iommu(iommu, drhd) {
4186                 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
4187                                                  GFP_ATOMIC);
4188                 if (!iommu->iommu_state)
4189                         goto nomem;
4190         }
4191
4192         iommu_flush_all();
4193
4194         for_each_active_iommu(iommu, drhd) {
4195                 iommu_disable_translation(iommu);
4196
4197                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4198
4199                 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4200                         readl(iommu->reg + DMAR_FECTL_REG);
4201                 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4202                         readl(iommu->reg + DMAR_FEDATA_REG);
4203                 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4204                         readl(iommu->reg + DMAR_FEADDR_REG);
4205                 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4206                         readl(iommu->reg + DMAR_FEUADDR_REG);
4207
4208                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4209         }
4210         return 0;
4211
4212 nomem:
4213         for_each_active_iommu(iommu, drhd)
4214                 kfree(iommu->iommu_state);
4215
4216         return -ENOMEM;
4217 }
4218
4219 static void iommu_resume(void)
4220 {
4221         struct dmar_drhd_unit *drhd;
4222         struct intel_iommu *iommu = NULL;
4223         unsigned long flag;
4224
4225         if (init_iommu_hw()) {
4226                 if (force_on)
4227                         panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4228                 else
4229                         WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
4230                 return;
4231         }
4232
4233         for_each_active_iommu(iommu, drhd) {
4234
4235                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4236
4237                 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4238                         iommu->reg + DMAR_FECTL_REG);
4239                 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4240                         iommu->reg + DMAR_FEDATA_REG);
4241                 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4242                         iommu->reg + DMAR_FEADDR_REG);
4243                 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4244                         iommu->reg + DMAR_FEUADDR_REG);
4245
4246                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4247         }
4248
4249         for_each_active_iommu(iommu, drhd)
4250                 kfree(iommu->iommu_state);
4251 }
4252
4253 static struct syscore_ops iommu_syscore_ops = {
4254         .resume         = iommu_resume,
4255         .suspend        = iommu_suspend,
4256 };
4257
4258 static void __init init_iommu_pm_ops(void)
4259 {
4260         register_syscore_ops(&iommu_syscore_ops);
4261 }
4262
4263 #else
4264 static inline void init_iommu_pm_ops(void) {}
4265 #endif  /* CONFIG_PM */
4266
4267
4268 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
4269 {
4270         struct acpi_dmar_reserved_memory *rmrr;
4271         struct dmar_rmrr_unit *rmrru;
4272
4273         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4274         if (!rmrru)
4275                 return -ENOMEM;
4276
4277         rmrru->hdr = header;
4278         rmrr = (struct acpi_dmar_reserved_memory *)header;
4279         rmrru->base_address = rmrr->base_address;
4280         rmrru->end_address = rmrr->end_address;
4281         rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4282                                 ((void *)rmrr) + rmrr->header.length,
4283                                 &rmrru->devices_cnt);
4284         if (rmrru->devices_cnt && rmrru->devices == NULL) {
4285                 kfree(rmrru);
4286                 return -ENOMEM;
4287         }
4288
4289         list_add(&rmrru->list, &dmar_rmrr_units);
4290
4291         return 0;
4292 }
4293
4294 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4295 {
4296         struct dmar_atsr_unit *atsru;
4297         struct acpi_dmar_atsr *tmp;
4298
4299         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4300                 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4301                 if (atsr->segment != tmp->segment)
4302                         continue;
4303                 if (atsr->header.length != tmp->header.length)
4304                         continue;
4305                 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4306                         return atsru;
4307         }
4308
4309         return NULL;
4310 }
4311
4312 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4313 {
4314         struct acpi_dmar_atsr *atsr;
4315         struct dmar_atsr_unit *atsru;
4316
4317         if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
4318                 return 0;
4319
4320         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4321         atsru = dmar_find_atsr(atsr);
4322         if (atsru)
4323                 return 0;
4324
4325         atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4326         if (!atsru)
4327                 return -ENOMEM;
4328
4329         /*
4330          * If memory is allocated from slab by ACPI _DSM method, we need to
4331          * copy the memory content because the memory buffer will be freed
4332          * on return.
4333          */
4334         atsru->hdr = (void *)(atsru + 1);
4335         memcpy(atsru->hdr, hdr, hdr->length);
4336         atsru->include_all = atsr->flags & 0x1;
4337         if (!atsru->include_all) {
4338                 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4339                                 (void *)atsr + atsr->header.length,
4340                                 &atsru->devices_cnt);
4341                 if (atsru->devices_cnt && atsru->devices == NULL) {
4342                         kfree(atsru);
4343                         return -ENOMEM;
4344                 }
4345         }
4346
4347         list_add_rcu(&atsru->list, &dmar_atsr_units);
4348
4349         return 0;
4350 }
4351
4352 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4353 {
4354         dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4355         kfree(atsru);
4356 }
4357
4358 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4359 {
4360         struct acpi_dmar_atsr *atsr;
4361         struct dmar_atsr_unit *atsru;
4362
4363         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4364         atsru = dmar_find_atsr(atsr);
4365         if (atsru) {
4366                 list_del_rcu(&atsru->list);
4367                 synchronize_rcu();
4368                 intel_iommu_free_atsr(atsru);
4369         }
4370
4371         return 0;
4372 }
4373
4374 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4375 {
4376         int i;
4377         struct device *dev;
4378         struct acpi_dmar_atsr *atsr;
4379         struct dmar_atsr_unit *atsru;
4380
4381         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4382         atsru = dmar_find_atsr(atsr);
4383         if (!atsru)
4384                 return 0;
4385
4386         if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
4387                 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4388                                           i, dev)
4389                         return -EBUSY;
4390         }
4391
4392         return 0;
4393 }
4394
4395 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4396 {
4397         int sp, ret = 0;
4398         struct intel_iommu *iommu = dmaru->iommu;
4399
4400         if (g_iommus[iommu->seq_id])
4401                 return 0;
4402
4403         if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4404                 pr_warn("%s: Doesn't support hardware pass through.\n",
4405                         iommu->name);
4406                 return -ENXIO;
4407         }
4408         if (!ecap_sc_support(iommu->ecap) &&
4409             domain_update_iommu_snooping(iommu)) {
4410                 pr_warn("%s: Doesn't support snooping.\n",
4411                         iommu->name);
4412                 return -ENXIO;
4413         }
4414         sp = domain_update_iommu_superpage(iommu) - 1;
4415         if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4416                 pr_warn("%s: Doesn't support large page.\n",
4417                         iommu->name);
4418                 return -ENXIO;
4419         }
4420
4421         /*
4422          * Disable translation if already enabled prior to OS handover.
4423          */
4424         if (iommu->gcmd & DMA_GCMD_TE)
4425                 iommu_disable_translation(iommu);
4426
4427         g_iommus[iommu->seq_id] = iommu;
4428         ret = iommu_init_domains(iommu);
4429         if (ret == 0)
4430                 ret = iommu_alloc_root_entry(iommu);
4431         if (ret)
4432                 goto out;
4433
4434 #ifdef CONFIG_INTEL_IOMMU_SVM
4435         if (pasid_enabled(iommu))
4436                 intel_svm_alloc_pasid_tables(iommu);
4437 #endif
4438
4439         if (dmaru->ignored) {
4440                 /*
4441                  * we always have to disable PMRs or DMA may fail on this device
4442                  */
4443                 if (force_on)
4444                         iommu_disable_protect_mem_regions(iommu);
4445                 return 0;
4446         }
4447
4448         intel_iommu_init_qi(iommu);
4449         iommu_flush_write_buffer(iommu);
4450
4451 #ifdef CONFIG_INTEL_IOMMU_SVM
4452         if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
4453                 ret = intel_svm_enable_prq(iommu);
4454                 if (ret)
4455                         goto disable_iommu;
4456         }
4457 #endif
4458         ret = dmar_set_interrupt(iommu);
4459         if (ret)
4460                 goto disable_iommu;
4461
4462         iommu_set_root_entry(iommu);
4463         iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4464         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4465         iommu_enable_translation(iommu);
4466
4467         iommu_disable_protect_mem_regions(iommu);
4468         return 0;
4469
4470 disable_iommu:
4471         disable_dmar_iommu(iommu);
4472 out:
4473         free_dmar_iommu(iommu);
4474         return ret;
4475 }
4476
4477 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4478 {
4479         int ret = 0;
4480         struct intel_iommu *iommu = dmaru->iommu;
4481
4482         if (!intel_iommu_enabled)
4483                 return 0;
4484         if (iommu == NULL)
4485                 return -EINVAL;
4486
4487         if (insert) {
4488                 ret = intel_iommu_add(dmaru);
4489         } else {
4490                 disable_dmar_iommu(iommu);
4491                 free_dmar_iommu(iommu);
4492         }
4493
4494         return ret;
4495 }
4496
4497 static void intel_iommu_free_dmars(void)
4498 {
4499         struct dmar_rmrr_unit *rmrru, *rmrr_n;
4500         struct dmar_atsr_unit *atsru, *atsr_n;
4501
4502         list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4503                 list_del(&rmrru->list);
4504                 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4505                 kfree(rmrru);
4506         }
4507
4508         list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4509                 list_del(&atsru->list);
4510                 intel_iommu_free_atsr(atsru);
4511         }
4512 }
4513
4514 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4515 {
4516         int i, ret = 1;
4517         struct pci_bus *bus;
4518         struct pci_dev *bridge = NULL;
4519         struct device *tmp;
4520         struct acpi_dmar_atsr *atsr;
4521         struct dmar_atsr_unit *atsru;
4522
4523         dev = pci_physfn(dev);
4524         for (bus = dev->bus; bus; bus = bus->parent) {
4525                 bridge = bus->self;
4526                 /* If it's an integrated device, allow ATS */
4527                 if (!bridge)
4528                         return 1;
4529                 /* Connected via non-PCIe: no ATS */
4530                 if (!pci_is_pcie(bridge) ||
4531                     pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4532                         return 0;
4533                 /* If we found the root port, look it up in the ATSR */
4534                 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4535                         break;
4536         }
4537
4538         rcu_read_lock();
4539         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4540                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4541                 if (atsr->segment != pci_domain_nr(dev->bus))
4542                         continue;
4543
4544                 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4545                         if (tmp == &bridge->dev)
4546                                 goto out;
4547
4548                 if (atsru->include_all)
4549                         goto out;
4550         }
4551         ret = 0;
4552 out:
4553         rcu_read_unlock();
4554
4555         return ret;
4556 }
4557
4558 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4559 {
4560         int ret = 0;
4561         struct dmar_rmrr_unit *rmrru;
4562         struct dmar_atsr_unit *atsru;
4563         struct acpi_dmar_atsr *atsr;
4564         struct acpi_dmar_reserved_memory *rmrr;
4565
4566         if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
4567                 return 0;
4568
4569         list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4570                 rmrr = container_of(rmrru->hdr,
4571                                     struct acpi_dmar_reserved_memory, header);
4572                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4573                         ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4574                                 ((void *)rmrr) + rmrr->header.length,
4575                                 rmrr->segment, rmrru->devices,
4576                                 rmrru->devices_cnt);
4577                         if(ret < 0)
4578                                 return ret;
4579                 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4580                         dmar_remove_dev_scope(info, rmrr->segment,
4581                                 rmrru->devices, rmrru->devices_cnt);
4582                 }
4583         }
4584
4585         list_for_each_entry(atsru, &dmar_atsr_units, list) {
4586                 if (atsru->include_all)
4587                         continue;
4588
4589                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4590                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4591                         ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4592                                         (void *)atsr + atsr->header.length,
4593                                         atsr->segment, atsru->devices,
4594                                         atsru->devices_cnt);
4595                         if (ret > 0)
4596                                 break;
4597                         else if(ret < 0)
4598                                 return ret;
4599                 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4600                         if (dmar_remove_dev_scope(info, atsr->segment,
4601                                         atsru->devices, atsru->devices_cnt))
4602                                 break;
4603                 }
4604         }
4605
4606         return 0;
4607 }
4608
4609 /*
4610  * Here we only respond to action of unbound device from driver.
4611  *
4612  * Added device is not attached to its DMAR domain here yet. That will happen
4613  * when mapping the device to iova.
4614  */
4615 static int device_notifier(struct notifier_block *nb,
4616                                   unsigned long action, void *data)
4617 {
4618         struct device *dev = data;
4619         struct dmar_domain *domain;
4620
4621         if (iommu_dummy(dev))
4622                 return 0;
4623
4624         if (action != BUS_NOTIFY_REMOVED_DEVICE)
4625                 return 0;
4626
4627         domain = find_domain(dev);
4628         if (!domain)
4629                 return 0;
4630
4631         dmar_remove_one_dev_info(domain, dev);
4632         if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4633                 domain_exit(domain);
4634
4635         return 0;
4636 }
4637
4638 static struct notifier_block device_nb = {
4639         .notifier_call = device_notifier,
4640 };
4641
4642 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4643                                        unsigned long val, void *v)
4644 {
4645         struct memory_notify *mhp = v;
4646         unsigned long long start, end;
4647         unsigned long start_vpfn, last_vpfn;
4648
4649         switch (val) {
4650         case MEM_GOING_ONLINE:
4651                 start = mhp->start_pfn << PAGE_SHIFT;
4652                 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4653                 if (iommu_domain_identity_map(si_domain, start, end)) {
4654                         pr_warn("Failed to build identity map for [%llx-%llx]\n",
4655                                 start, end);
4656                         return NOTIFY_BAD;
4657                 }
4658                 break;
4659
4660         case MEM_OFFLINE:
4661         case MEM_CANCEL_ONLINE:
4662                 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4663                 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4664                 while (start_vpfn <= last_vpfn) {
4665                         struct iova *iova;
4666                         struct dmar_drhd_unit *drhd;
4667                         struct intel_iommu *iommu;
4668                         struct page *freelist;
4669
4670                         iova = find_iova(&si_domain->iovad, start_vpfn);
4671                         if (iova == NULL) {
4672                                 pr_debug("Failed get IOVA for PFN %lx\n",
4673                                          start_vpfn);
4674                                 break;
4675                         }
4676
4677                         iova = split_and_remove_iova(&si_domain->iovad, iova,
4678                                                      start_vpfn, last_vpfn);
4679                         if (iova == NULL) {
4680                                 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4681                                         start_vpfn, last_vpfn);
4682                                 return NOTIFY_BAD;
4683                         }
4684
4685                         freelist = domain_unmap(si_domain, iova->pfn_lo,
4686                                                iova->pfn_hi);
4687
4688                         rcu_read_lock();
4689                         for_each_active_iommu(iommu, drhd)
4690                                 iommu_flush_iotlb_psi(iommu, si_domain,
4691                                         iova->pfn_lo, iova_size(iova),
4692                                         !freelist, 0);
4693                         rcu_read_unlock();
4694                         dma_free_pagelist(freelist);
4695
4696                         start_vpfn = iova->pfn_hi + 1;
4697                         free_iova_mem(iova);
4698                 }
4699                 break;
4700         }
4701
4702         return NOTIFY_OK;
4703 }
4704
4705 static struct notifier_block intel_iommu_memory_nb = {
4706         .notifier_call = intel_iommu_memory_notifier,
4707         .priority = 0
4708 };
4709
4710 static void free_all_cpu_cached_iovas(unsigned int cpu)
4711 {
4712         int i;
4713
4714         for (i = 0; i < g_num_of_iommus; i++) {
4715                 struct intel_iommu *iommu = g_iommus[i];
4716                 struct dmar_domain *domain;
4717                 int did;
4718
4719                 if (!iommu)
4720                         continue;
4721
4722                 for (did = 0; did < cap_ndoms(iommu->cap); did++) {
4723                         domain = get_iommu_domain(iommu, (u16)did);
4724
4725                         if (!domain)
4726                                 continue;
4727                         free_cpu_cached_iovas(cpu, &domain->iovad);
4728                 }
4729         }
4730 }
4731
4732 static int intel_iommu_cpu_notifier(struct notifier_block *nfb,
4733                                     unsigned long action, void *v)
4734 {
4735         unsigned int cpu = (unsigned long)v;
4736
4737         switch (action) {
4738         case CPU_DEAD:
4739         case CPU_DEAD_FROZEN:
4740                 free_all_cpu_cached_iovas(cpu);
4741                 flush_unmaps_timeout(cpu);
4742                 break;
4743         }
4744         return NOTIFY_OK;
4745 }
4746
4747 static struct notifier_block intel_iommu_cpu_nb = {
4748         .notifier_call = intel_iommu_cpu_notifier,
4749 };
4750
4751 static ssize_t intel_iommu_show_version(struct device *dev,
4752                                         struct device_attribute *attr,
4753                                         char *buf)
4754 {
4755         struct intel_iommu *iommu = dev_get_drvdata(dev);
4756         u32 ver = readl(iommu->reg + DMAR_VER_REG);
4757         return sprintf(buf, "%d:%d\n",
4758                        DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4759 }
4760 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4761
4762 static ssize_t intel_iommu_show_address(struct device *dev,
4763                                         struct device_attribute *attr,
4764                                         char *buf)
4765 {
4766         struct intel_iommu *iommu = dev_get_drvdata(dev);
4767         return sprintf(buf, "%llx\n", iommu->reg_phys);
4768 }
4769 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4770
4771 static ssize_t intel_iommu_show_cap(struct device *dev,
4772                                     struct device_attribute *attr,
4773                                     char *buf)
4774 {
4775         struct intel_iommu *iommu = dev_get_drvdata(dev);
4776         return sprintf(buf, "%llx\n", iommu->cap);
4777 }
4778 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4779
4780 static ssize_t intel_iommu_show_ecap(struct device *dev,
4781                                     struct device_attribute *attr,
4782                                     char *buf)
4783 {
4784         struct intel_iommu *iommu = dev_get_drvdata(dev);
4785         return sprintf(buf, "%llx\n", iommu->ecap);
4786 }
4787 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4788
4789 static ssize_t intel_iommu_show_ndoms(struct device *dev,
4790                                       struct device_attribute *attr,
4791                                       char *buf)
4792 {
4793         struct intel_iommu *iommu = dev_get_drvdata(dev);
4794         return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4795 }
4796 static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4797
4798 static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4799                                            struct device_attribute *attr,
4800                                            char *buf)
4801 {
4802         struct intel_iommu *iommu = dev_get_drvdata(dev);
4803         return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4804                                                   cap_ndoms(iommu->cap)));
4805 }
4806 static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4807
4808 static struct attribute *intel_iommu_attrs[] = {
4809         &dev_attr_version.attr,
4810         &dev_attr_address.attr,
4811         &dev_attr_cap.attr,
4812         &dev_attr_ecap.attr,
4813         &dev_attr_domains_supported.attr,
4814         &dev_attr_domains_used.attr,
4815         NULL,
4816 };
4817
4818 static struct attribute_group intel_iommu_group = {
4819         .name = "intel-iommu",
4820         .attrs = intel_iommu_attrs,
4821 };
4822
4823 const struct attribute_group *intel_iommu_groups[] = {
4824         &intel_iommu_group,
4825         NULL,
4826 };
4827
4828 int __init intel_iommu_init(void)
4829 {
4830         int ret = -ENODEV;
4831         struct dmar_drhd_unit *drhd;
4832         struct intel_iommu *iommu;
4833
4834         /* VT-d is required for a TXT/tboot launch, so enforce that */
4835         force_on = tboot_force_iommu();
4836
4837         if (iommu_init_mempool()) {
4838                 if (force_on)
4839                         panic("tboot: Failed to initialize iommu memory\n");
4840                 return -ENOMEM;
4841         }
4842
4843         down_write(&dmar_global_lock);
4844         if (dmar_table_init()) {
4845                 if (force_on)
4846                         panic("tboot: Failed to initialize DMAR table\n");
4847                 goto out_free_dmar;
4848         }
4849
4850         if (dmar_dev_scope_init() < 0) {
4851                 if (force_on)
4852                         panic("tboot: Failed to initialize DMAR device scope\n");
4853                 goto out_free_dmar;
4854         }
4855
4856         if (no_iommu || dmar_disabled)
4857                 goto out_free_dmar;
4858
4859         if (list_empty(&dmar_rmrr_units))
4860                 pr_info("No RMRR found\n");
4861
4862         if (list_empty(&dmar_atsr_units))
4863                 pr_info("No ATSR found\n");
4864
4865         if (dmar_init_reserved_ranges()) {
4866                 if (force_on)
4867                         panic("tboot: Failed to reserve iommu ranges\n");
4868                 goto out_free_reserved_range;
4869         }
4870
4871         if (dmar_map_gfx)
4872                 intel_iommu_gfx_mapped = 1;
4873
4874         init_no_remapping_devices();
4875
4876         ret = init_dmars();
4877         if (ret) {
4878                 if (force_on)
4879                         panic("tboot: Failed to initialize DMARs\n");
4880                 pr_err("Initialization failed\n");
4881                 goto out_free_reserved_range;
4882         }
4883         up_write(&dmar_global_lock);
4884         pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4885
4886 #ifdef CONFIG_SWIOTLB
4887         swiotlb = 0;
4888 #endif
4889         dma_ops = &intel_dma_ops;
4890
4891         init_iommu_pm_ops();
4892
4893         for_each_active_iommu(iommu, drhd)
4894                 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4895                                                        intel_iommu_groups,
4896                                                        "%s", iommu->name);
4897
4898         bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4899         bus_register_notifier(&pci_bus_type, &device_nb);
4900         if (si_domain && !hw_pass_through)
4901                 register_memory_notifier(&intel_iommu_memory_nb);
4902         register_hotcpu_notifier(&intel_iommu_cpu_nb);
4903
4904         intel_iommu_enabled = 1;
4905
4906         return 0;
4907
4908 out_free_reserved_range:
4909         put_iova_domain(&reserved_iova_list);
4910 out_free_dmar:
4911         intel_iommu_free_dmars();
4912         up_write(&dmar_global_lock);
4913         iommu_exit_mempool();
4914         return ret;
4915 }
4916
4917 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4918 {
4919         struct intel_iommu *iommu = opaque;
4920
4921         domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4922         return 0;
4923 }
4924
4925 /*
4926  * NB - intel-iommu lacks any sort of reference counting for the users of
4927  * dependent devices.  If multiple endpoints have intersecting dependent
4928  * devices, unbinding the driver from any one of them will possibly leave
4929  * the others unable to operate.
4930  */
4931 static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
4932 {
4933         if (!iommu || !dev || !dev_is_pci(dev))
4934                 return;
4935
4936         pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
4937 }
4938
4939 static void __dmar_remove_one_dev_info(struct device_domain_info *info)
4940 {
4941         struct intel_iommu *iommu;
4942         unsigned long flags;
4943
4944         assert_spin_locked(&device_domain_lock);
4945
4946         if (WARN_ON(!info))
4947                 return;
4948
4949         iommu = info->iommu;
4950
4951         if (info->dev) {
4952                 iommu_disable_dev_iotlb(info);
4953                 domain_context_clear(iommu, info->dev);
4954         }
4955
4956         unlink_domain_info(info);
4957
4958         spin_lock_irqsave(&iommu->lock, flags);
4959         domain_detach_iommu(info->domain, iommu);
4960         spin_unlock_irqrestore(&iommu->lock, flags);
4961
4962         free_devinfo_mem(info);
4963 }
4964
4965 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4966                                      struct device *dev)
4967 {
4968         struct device_domain_info *info;
4969         unsigned long flags;
4970
4971         spin_lock_irqsave(&device_domain_lock, flags);
4972         info = dev->archdata.iommu;
4973         __dmar_remove_one_dev_info(info);
4974         spin_unlock_irqrestore(&device_domain_lock, flags);
4975 }
4976
4977 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4978 {
4979         int adjust_width;
4980
4981         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4982                         DMA_32BIT_PFN);
4983         domain_reserve_special_ranges(domain);
4984
4985         /* calculate AGAW */
4986         domain->gaw = guest_width;
4987         adjust_width = guestwidth_to_adjustwidth(guest_width);
4988         domain->agaw = width_to_agaw(adjust_width);
4989
4990         domain->iommu_coherency = 0;
4991         domain->iommu_snooping = 0;
4992         domain->iommu_superpage = 0;
4993         domain->max_addr = 0;
4994
4995         /* always allocate the top pgd */
4996         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
4997         if (!domain->pgd)
4998                 return -ENOMEM;
4999         domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
5000         return 0;
5001 }
5002
5003 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
5004 {
5005         struct dmar_domain *dmar_domain;
5006         struct iommu_domain *domain;
5007
5008         if (type != IOMMU_DOMAIN_UNMANAGED)
5009                 return NULL;
5010
5011         dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
5012         if (!dmar_domain) {
5013                 pr_err("Can't allocate dmar_domain\n");
5014                 return NULL;
5015         }
5016         if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
5017                 pr_err("Domain initialization failed\n");
5018                 domain_exit(dmar_domain);
5019                 return NULL;
5020         }
5021         domain_update_iommu_cap(dmar_domain);
5022
5023         domain = &dmar_domain->domain;
5024         domain->geometry.aperture_start = 0;
5025         domain->geometry.aperture_end   = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
5026         domain->geometry.force_aperture = true;
5027
5028         return domain;
5029 }
5030
5031 static void intel_iommu_domain_free(struct iommu_domain *domain)
5032 {
5033         domain_exit(to_dmar_domain(domain));
5034 }
5035
5036 static int intel_iommu_attach_device(struct iommu_domain *domain,
5037                                      struct device *dev)
5038 {
5039         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5040         struct intel_iommu *iommu;
5041         int addr_width;
5042         u8 bus, devfn;
5043
5044         if (device_is_rmrr_locked(dev)) {
5045                 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement.  Contact your platform vendor.\n");
5046                 return -EPERM;
5047         }
5048
5049         /* normally dev is not mapped */
5050         if (unlikely(domain_context_mapped(dev))) {
5051                 struct dmar_domain *old_domain;
5052
5053                 old_domain = find_domain(dev);
5054                 if (old_domain) {
5055                         rcu_read_lock();
5056                         dmar_remove_one_dev_info(old_domain, dev);
5057                         rcu_read_unlock();
5058
5059                         if (!domain_type_is_vm_or_si(old_domain) &&
5060                              list_empty(&old_domain->devices))
5061                                 domain_exit(old_domain);
5062                 }
5063         }
5064
5065         iommu = device_to_iommu(dev, &bus, &devfn);
5066         if (!iommu)
5067                 return -ENODEV;
5068
5069         /* check if this iommu agaw is sufficient for max mapped address */
5070         addr_width = agaw_to_width(iommu->agaw);
5071         if (addr_width > cap_mgaw(iommu->cap))
5072                 addr_width = cap_mgaw(iommu->cap);
5073
5074         if (dmar_domain->max_addr > (1LL << addr_width)) {
5075                 pr_err("%s: iommu width (%d) is not "
5076                        "sufficient for the mapped address (%llx)\n",
5077                        __func__, addr_width, dmar_domain->max_addr);
5078                 return -EFAULT;
5079         }
5080         dmar_domain->gaw = addr_width;
5081
5082         /*
5083          * Knock out extra levels of page tables if necessary
5084          */
5085         while (iommu->agaw < dmar_domain->agaw) {
5086                 struct dma_pte *pte;
5087
5088                 pte = dmar_domain->pgd;
5089                 if (dma_pte_present(pte)) {
5090                         dmar_domain->pgd = (struct dma_pte *)
5091                                 phys_to_virt(dma_pte_addr(pte));
5092                         free_pgtable_page(pte);
5093                 }
5094                 dmar_domain->agaw--;
5095         }
5096
5097         return domain_add_dev_info(dmar_domain, dev);
5098 }
5099
5100 static void intel_iommu_detach_device(struct iommu_domain *domain,
5101                                       struct device *dev)
5102 {
5103         dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
5104 }
5105
5106 static int intel_iommu_map(struct iommu_domain *domain,
5107                            unsigned long iova, phys_addr_t hpa,
5108                            size_t size, int iommu_prot)
5109 {
5110         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5111         u64 max_addr;
5112         int prot = 0;
5113         int ret;
5114
5115         if (iommu_prot & IOMMU_READ)
5116                 prot |= DMA_PTE_READ;
5117         if (iommu_prot & IOMMU_WRITE)
5118                 prot |= DMA_PTE_WRITE;
5119         if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
5120                 prot |= DMA_PTE_SNP;
5121
5122         max_addr = iova + size;
5123         if (dmar_domain->max_addr < max_addr) {
5124                 u64 end;
5125
5126                 /* check if minimum agaw is sufficient for mapped address */
5127                 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
5128                 if (end < max_addr) {
5129                         pr_err("%s: iommu width (%d) is not "
5130                                "sufficient for the mapped address (%llx)\n",
5131                                __func__, dmar_domain->gaw, max_addr);
5132                         return -EFAULT;
5133                 }
5134                 dmar_domain->max_addr = max_addr;
5135         }
5136         /* Round up size to next multiple of PAGE_SIZE, if it and
5137            the low bits of hpa would take us onto the next page */
5138         size = aligned_nrpages(hpa, size);
5139         ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
5140                                  hpa >> VTD_PAGE_SHIFT, size, prot);
5141         return ret;
5142 }
5143
5144 static size_t intel_iommu_unmap(struct iommu_domain *domain,
5145                                 unsigned long iova, size_t size)
5146 {
5147         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5148         struct page *freelist = NULL;
5149         struct intel_iommu *iommu;
5150         unsigned long start_pfn, last_pfn;
5151         unsigned int npages;
5152         int iommu_id, level = 0;
5153
5154         /* Cope with horrid API which requires us to unmap more than the
5155            size argument if it happens to be a large-page mapping. */
5156         BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
5157
5158         if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
5159                 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
5160
5161         start_pfn = iova >> VTD_PAGE_SHIFT;
5162         last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5163
5164         freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
5165
5166         npages = last_pfn - start_pfn + 1;
5167
5168         for_each_domain_iommu(iommu_id, dmar_domain) {
5169                 iommu = g_iommus[iommu_id];
5170
5171                 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5172                                       start_pfn, npages, !freelist, 0);
5173         }
5174
5175         dma_free_pagelist(freelist);
5176
5177         if (dmar_domain->max_addr == iova + size)
5178                 dmar_domain->max_addr = iova;
5179
5180         return size;
5181 }
5182
5183 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
5184                                             dma_addr_t iova)
5185 {
5186         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5187         struct dma_pte *pte;
5188         int level = 0;
5189         u64 phys = 0;
5190
5191         pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
5192         if (pte)
5193                 phys = dma_pte_addr(pte);
5194
5195         return phys;
5196 }
5197
5198 static bool intel_iommu_capable(enum iommu_cap cap)
5199 {
5200         if (cap == IOMMU_CAP_CACHE_COHERENCY)
5201                 return domain_update_iommu_snooping(NULL) == 1;
5202         if (cap == IOMMU_CAP_INTR_REMAP)
5203                 return irq_remapping_enabled == 1;
5204
5205         return false;
5206 }
5207
5208 static int intel_iommu_add_device(struct device *dev)
5209 {
5210         struct intel_iommu *iommu;
5211         struct iommu_group *group;
5212         u8 bus, devfn;
5213
5214         iommu = device_to_iommu(dev, &bus, &devfn);
5215         if (!iommu)
5216                 return -ENODEV;
5217
5218         iommu_device_link(iommu->iommu_dev, dev);
5219
5220         group = iommu_group_get_for_dev(dev);
5221
5222         if (IS_ERR(group))
5223                 return PTR_ERR(group);
5224
5225         iommu_group_put(group);
5226         return 0;
5227 }
5228
5229 static void intel_iommu_remove_device(struct device *dev)
5230 {
5231         struct intel_iommu *iommu;
5232         u8 bus, devfn;
5233
5234         iommu = device_to_iommu(dev, &bus, &devfn);
5235         if (!iommu)
5236                 return;
5237
5238         iommu_group_remove_device(dev);
5239
5240         iommu_device_unlink(iommu->iommu_dev, dev);
5241 }
5242
5243 #ifdef CONFIG_INTEL_IOMMU_SVM
5244 #define MAX_NR_PASID_BITS (20)
5245 static inline unsigned long intel_iommu_get_pts(struct intel_iommu *iommu)
5246 {
5247         /*
5248          * Convert ecap_pss to extend context entry pts encoding, also
5249          * respect the soft pasid_max value set by the iommu.
5250          * - number of PASID bits = ecap_pss + 1
5251          * - number of PASID table entries = 2^(pts + 5)
5252          * Therefore, pts = ecap_pss - 4
5253          * e.g. KBL ecap_pss = 0x13, PASID has 20 bits, pts = 15
5254          */
5255         if (ecap_pss(iommu->ecap) < 5)
5256                 return 0;
5257
5258         /* pasid_max is encoded as actual number of entries not the bits */
5259         return find_first_bit((unsigned long *)&iommu->pasid_max,
5260                         MAX_NR_PASID_BITS) - 5;
5261 }
5262
5263 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5264 {
5265         struct device_domain_info *info;
5266         struct context_entry *context;
5267         struct dmar_domain *domain;
5268         unsigned long flags;
5269         u64 ctx_lo;
5270         int ret;
5271
5272         domain = get_valid_domain_for_dev(sdev->dev);
5273         if (!domain)
5274                 return -EINVAL;
5275
5276         spin_lock_irqsave(&device_domain_lock, flags);
5277         spin_lock(&iommu->lock);
5278
5279         ret = -EINVAL;
5280         info = sdev->dev->archdata.iommu;
5281         if (!info || !info->pasid_supported)
5282                 goto out;
5283
5284         context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5285         if (WARN_ON(!context))
5286                 goto out;
5287
5288         ctx_lo = context[0].lo;
5289
5290         sdev->did = domain->iommu_did[iommu->seq_id];
5291         sdev->sid = PCI_DEVID(info->bus, info->devfn);
5292
5293         if (!(ctx_lo & CONTEXT_PASIDE)) {
5294                 context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
5295                 context[1].lo = (u64)virt_to_phys(iommu->pasid_table) |
5296                         intel_iommu_get_pts(iommu);
5297
5298                 wmb();
5299                 /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both
5300                  * extended to permit requests-with-PASID if the PASIDE bit
5301                  * is set. which makes sense. For CONTEXT_TT_PASS_THROUGH,
5302                  * however, the PASIDE bit is ignored and requests-with-PASID
5303                  * are unconditionally blocked. Which makes less sense.
5304                  * So convert from CONTEXT_TT_PASS_THROUGH to one of the new
5305                  * "guest mode" translation types depending on whether ATS
5306                  * is available or not. Annoyingly, we can't use the new
5307                  * modes *unless* PASIDE is set. */
5308                 if ((ctx_lo & CONTEXT_TT_MASK) == (CONTEXT_TT_PASS_THROUGH << 2)) {
5309                         ctx_lo &= ~CONTEXT_TT_MASK;
5310                         if (info->ats_supported)
5311                                 ctx_lo |= CONTEXT_TT_PT_PASID_DEV_IOTLB << 2;
5312                         else
5313                                 ctx_lo |= CONTEXT_TT_PT_PASID << 2;
5314                 }
5315                 ctx_lo |= CONTEXT_PASIDE;
5316                 if (iommu->pasid_state_table)
5317                         ctx_lo |= CONTEXT_DINVE;
5318                 if (info->pri_supported)
5319                         ctx_lo |= CONTEXT_PRS;
5320                 context[0].lo = ctx_lo;
5321                 wmb();
5322                 iommu->flush.flush_context(iommu, sdev->did, sdev->sid,
5323                                            DMA_CCMD_MASK_NOBIT,
5324                                            DMA_CCMD_DEVICE_INVL);
5325         }
5326
5327         /* Enable PASID support in the device, if it wasn't already */
5328         if (!info->pasid_enabled)
5329                 iommu_enable_dev_iotlb(info);
5330
5331         if (info->ats_enabled) {
5332                 sdev->dev_iotlb = 1;
5333                 sdev->qdep = info->ats_qdep;
5334                 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5335                         sdev->qdep = 0;
5336         }
5337         ret = 0;
5338
5339  out:
5340         spin_unlock(&iommu->lock);
5341         spin_unlock_irqrestore(&device_domain_lock, flags);
5342
5343         return ret;
5344 }
5345
5346 struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5347 {
5348         struct intel_iommu *iommu;
5349         u8 bus, devfn;
5350
5351         if (iommu_dummy(dev)) {
5352                 dev_warn(dev,
5353                          "No IOMMU translation for device; cannot enable SVM\n");
5354                 return NULL;
5355         }
5356
5357         iommu = device_to_iommu(dev, &bus, &devfn);
5358         if ((!iommu)) {
5359                 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
5360                 return NULL;
5361         }
5362
5363         if (!iommu->pasid_table) {
5364                 dev_err(dev, "PASID not enabled on IOMMU; cannot enable SVM\n");
5365                 return NULL;
5366         }
5367
5368         return iommu;
5369 }
5370 #endif /* CONFIG_INTEL_IOMMU_SVM */
5371
5372 static const struct iommu_ops intel_iommu_ops = {
5373         .capable        = intel_iommu_capable,
5374         .domain_alloc   = intel_iommu_domain_alloc,
5375         .domain_free    = intel_iommu_domain_free,
5376         .attach_dev     = intel_iommu_attach_device,
5377         .detach_dev     = intel_iommu_detach_device,
5378         .map            = intel_iommu_map,
5379         .unmap          = intel_iommu_unmap,
5380         .map_sg         = default_iommu_map_sg,
5381         .iova_to_phys   = intel_iommu_iova_to_phys,
5382         .add_device     = intel_iommu_add_device,
5383         .remove_device  = intel_iommu_remove_device,
5384         .device_group   = pci_device_group,
5385         .pgsize_bitmap  = INTEL_IOMMU_PGSIZES,
5386 };
5387
5388 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5389 {
5390         /* G4x/GM45 integrated gfx dmar support is totally busted. */
5391         pr_info("Disabling IOMMU for graphics on this chipset\n");
5392         dmar_map_gfx = 0;
5393 }
5394
5395 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5396 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5397 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5398 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5399 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5400 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5401 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5402
5403 static void quirk_iommu_rwbf(struct pci_dev *dev)
5404 {
5405         /*
5406          * Mobile 4 Series Chipset neglects to set RWBF capability,
5407          * but needs it. Same seems to hold for the desktop versions.
5408          */
5409         pr_info("Forcing write-buffer flush capability\n");
5410         rwbf_quirk = 1;
5411 }
5412
5413 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
5414 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5415 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5416 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5417 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5418 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5419 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
5420
5421 #define GGC 0x52
5422 #define GGC_MEMORY_SIZE_MASK    (0xf << 8)
5423 #define GGC_MEMORY_SIZE_NONE    (0x0 << 8)
5424 #define GGC_MEMORY_SIZE_1M      (0x1 << 8)
5425 #define GGC_MEMORY_SIZE_2M      (0x3 << 8)
5426 #define GGC_MEMORY_VT_ENABLED   (0x8 << 8)
5427 #define GGC_MEMORY_SIZE_2M_VT   (0x9 << 8)
5428 #define GGC_MEMORY_SIZE_3M_VT   (0xa << 8)
5429 #define GGC_MEMORY_SIZE_4M_VT   (0xb << 8)
5430
5431 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
5432 {
5433         unsigned short ggc;
5434
5435         if (pci_read_config_word(dev, GGC, &ggc))
5436                 return;
5437
5438         if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
5439                 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
5440                 dmar_map_gfx = 0;
5441         } else if (dmar_map_gfx) {
5442                 /* we have to ensure the gfx device is idle before we flush */
5443                 pr_info("Disabling batched IOTLB flush on Ironlake\n");
5444                 intel_iommu_strict = 1;
5445        }
5446 }
5447 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5448 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5449 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5450 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5451
5452 /* On Tylersburg chipsets, some BIOSes have been known to enable the
5453    ISOCH DMAR unit for the Azalia sound device, but not give it any
5454    TLB entries, which causes it to deadlock. Check for that.  We do
5455    this in a function called from init_dmars(), instead of in a PCI
5456    quirk, because we don't want to print the obnoxious "BIOS broken"
5457    message if VT-d is actually disabled.
5458 */
5459 static void __init check_tylersburg_isoch(void)
5460 {
5461         struct pci_dev *pdev;
5462         uint32_t vtisochctrl;
5463
5464         /* If there's no Azalia in the system anyway, forget it. */
5465         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5466         if (!pdev)
5467                 return;
5468         pci_dev_put(pdev);
5469
5470         /* System Management Registers. Might be hidden, in which case
5471            we can't do the sanity check. But that's OK, because the
5472            known-broken BIOSes _don't_ actually hide it, so far. */
5473         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5474         if (!pdev)
5475                 return;
5476
5477         if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5478                 pci_dev_put(pdev);
5479                 return;
5480         }
5481
5482         pci_dev_put(pdev);
5483
5484         /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5485         if (vtisochctrl & 1)
5486                 return;
5487
5488         /* Drop all bits other than the number of TLB entries */
5489         vtisochctrl &= 0x1c;
5490
5491         /* If we have the recommended number of TLB entries (16), fine. */
5492         if (vtisochctrl == 0x10)
5493                 return;
5494
5495         /* Zero TLB entries? You get to ride the short bus to school. */
5496         if (!vtisochctrl) {
5497                 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5498                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5499                      dmi_get_system_info(DMI_BIOS_VENDOR),
5500                      dmi_get_system_info(DMI_BIOS_VERSION),
5501                      dmi_get_system_info(DMI_PRODUCT_VERSION));
5502                 iommu_identity_mapping |= IDENTMAP_AZALIA;
5503                 return;
5504         }
5505
5506         pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
5507                vtisochctrl);
5508 }