drivers/iommu/arm-smmu.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * IOMMU API for ARM architected SMMU implementations.
   4  *
   5  * Copyright (C) 2013 ARM Limited
   6  *
   7  * Author: Will Deacon <will.deacon@arm.com>
   8  *
   9  * This driver currently supports:
  10  *      - SMMUv1 and v2 implementations
  11  *      - Stream-matching and stream-indexing
  12  *      - v7/v8 long-descriptor format
  13  *      - Non-secure access to the SMMU
  14  *      - Context fault reporting
  15  *      - Extended Stream ID (16 bit)
  16  */
  17
  18 #define pr_fmt(fmt) "arm-smmu: " fmt
  19
  20 #include <linux/acpi.h>
  21 #include <linux/acpi_iort.h>
  22 #include <linux/bitfield.h>
  23 #include <linux/delay.h>
  24 #include <linux/dma-iommu.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/err.h>
  27 #include <linux/interrupt.h>
  28 #include <linux/io.h>
  29 #include <linux/iopoll.h>
  30 #include <linux/module.h>
  31 #include <linux/of.h>
  32 #include <linux/of_address.h>
  33 #include <linux/of_device.h>
  34 #include <linux/of_iommu.h>
  35 #include <linux/pci.h>
  36 #include <linux/platform_device.h>
  37 #include <linux/pm_runtime.h>
  38 #include <linux/ratelimit.h>
  39 #include <linux/slab.h>
  40
  41 #include <linux/amba/bus.h>
  42 #include <linux/fsl/mc.h>
  43
  44 #include "arm-smmu.h"
  45
  46 /*
  47  * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
  48  * global register space are still, in fact, using a hypervisor to mediate it
  49  * by trapping and emulating register accesses. Sadly, some deployed versions
  50  * of said trapping code have bugs wherein they go horribly wrong for stores
  51  * using r31 (i.e. XZR/WZR) as the source register.
  52  */
  53 #define QCOM_DUMMY_VAL -1
  54
  55 #define TLB_LOOP_TIMEOUT                1000000 /* 1s! */
  56 #define TLB_SPIN_COUNT                  10
  57
  58 #define MSI_IOVA_BASE                   0x8000000
  59 #define MSI_IOVA_LENGTH                 0x100000
  60
  61 static int force_stage;
  62 module_param(force_stage, int, S_IRUGO);
  63 MODULE_PARM_DESC(force_stage,
  64         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
  65 static bool disable_bypass =
  66         IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
  67 module_param(disable_bypass, bool, S_IRUGO);
  68 MODULE_PARM_DESC(disable_bypass,
  69         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
  70
  71 struct arm_smmu_s2cr {
  72         struct iommu_group              *group;
  73         int                             count;
  74         enum arm_smmu_s2cr_type         type;
  75         enum arm_smmu_s2cr_privcfg      privcfg;
  76         u8                              cbndx;
  77 };
  78
  79 #define s2cr_init_val (struct arm_smmu_s2cr){                           \
  80         .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,    \
  81 }
  82
  83 struct arm_smmu_smr {
  84         u16                             mask;
  85         u16                             id;
  86         bool                            valid;
  87 };
  88
  89 struct arm_smmu_cb {
  90         u64                             ttbr[2];
  91         u32                             tcr[2];
  92         u32                             mair[2];
  93         struct arm_smmu_cfg             *cfg;
  94 };
  95
  96 struct arm_smmu_master_cfg {
  97         struct arm_smmu_device          *smmu;
  98         s16                             smendx[];
  99 };
 100 #define INVALID_SMENDX                  -1
 101 #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
 102 #define fwspec_smmu(fw)  (__fwspec_cfg(fw)->smmu)
 103 #define fwspec_smendx(fw, i) \
 104         (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i])
 105 #define for_each_cfg_sme(fw, i, idx) \
 106         for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i)
 107
 108 static bool using_legacy_binding, using_generic_binding;
 109
 110 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
 111 {
 112         if (pm_runtime_enabled(smmu->dev))
 113                 return pm_runtime_get_sync(smmu->dev);
 114
 115         return 0;
 116 }
 117
 118 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
 119 {
 120         if (pm_runtime_enabled(smmu->dev))
 121                 pm_runtime_put_autosuspend(smmu->dev);
 122 }
 123
 124 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
 125 {
 126         return container_of(dom, struct arm_smmu_domain, domain);
 127 }
 128
 129 static struct platform_driver arm_smmu_driver;
 130 static struct iommu_ops arm_smmu_ops;
 131
 132 #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS
 133 static int arm_smmu_bus_init(struct iommu_ops *ops);
 134
 135 static struct device_node *dev_get_dev_node(struct device *dev)
 136 {
 137         if (dev_is_pci(dev)) {
 138                 struct pci_bus *bus = to_pci_dev(dev)->bus;
 139
 140                 while (!pci_is_root_bus(bus))
 141                         bus = bus->parent;
 142                 return of_node_get(bus->bridge->parent->of_node);
 143         }
 144
 145         return of_node_get(dev->of_node);
 146 }
 147
 148 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
 149 {
 150         *((__be32 *)data) = cpu_to_be32(alias);
 151         return 0; /* Continue walking */
 152 }
 153
 154 static int __find_legacy_master_phandle(struct device *dev, void *data)
 155 {
 156         struct of_phandle_iterator *it = *(void **)data;
 157         struct device_node *np = it->node;
 158         int err;
 159
 160         of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
 161                             "#stream-id-cells", -1)
 162                 if (it->node == np) {
 163                         *(void **)data = dev;
 164                         return 1;
 165                 }
 166         it->node = np;
 167         return err == -ENOENT ? 0 : err;
 168 }
 169
 170 static int arm_smmu_register_legacy_master(struct device *dev,
 171                                            struct arm_smmu_device **smmu)
 172 {
 173         struct device *smmu_dev;
 174         struct device_node *np;
 175         struct of_phandle_iterator it;
 176         void *data = &it;
 177         u32 *sids;
 178         __be32 pci_sid;
 179         int err;
 180
 181         np = dev_get_dev_node(dev);
 182         if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
 183                 of_node_put(np);
 184                 return -ENODEV;
 185         }
 186
 187         it.node = np;
 188         err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
 189                                      __find_legacy_master_phandle);
 190         smmu_dev = data;
 191         of_node_put(np);
 192         if (err == 0)
 193                 return -ENODEV;
 194         if (err < 0)
 195                 return err;
 196
 197         if (dev_is_pci(dev)) {
 198                 /* "mmu-masters" assumes Stream ID == Requester ID */
 199                 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
 200                                        &pci_sid);
 201                 it.cur = &pci_sid;
 202                 it.cur_count = 1;
 203         }
 204
 205         err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
 206                                 &arm_smmu_ops);
 207         if (err)
 208                 return err;
 209
 210         sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
 211         if (!sids)
 212                 return -ENOMEM;
 213
 214         *smmu = dev_get_drvdata(smmu_dev);
 215         of_phandle_iterator_args(&it, sids, it.cur_count);
 216         err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
 217         kfree(sids);
 218         return err;
 219 }
 220
 221 /*
 222  * With the legacy DT binding in play, we have no guarantees about
 223  * probe order, but then we're also not doing default domains, so we can
 224  * delay setting bus ops until we're sure every possible SMMU is ready,
 225  * and that way ensure that no add_device() calls get missed.
 226  */
 227 static int arm_smmu_legacy_bus_init(void)
 228 {
 229         if (using_legacy_binding)
 230                 return arm_smmu_bus_init(&arm_smmu_ops);
 231         return 0;
 232 }
 233 device_initcall_sync(arm_smmu_legacy_bus_init);
 234 #else
 235 static int arm_smmu_register_legacy_master(struct device *dev,
 236                                            struct arm_smmu_device **smmu)
 237 {
 238         return -ENODEV;
 239 }
 240 #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
 241
 242 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
 243 {
 244         int idx;
 245
 246         do {
 247                 idx = find_next_zero_bit(map, end, start);
 248                 if (idx == end)
 249                         return -ENOSPC;
 250         } while (test_and_set_bit(idx, map));
 251
 252         return idx;
 253 }
 254
 255 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
 256 {
 257         clear_bit(idx, map);
 258 }
 259
 260 /* Wait for any pending TLB invalidations to complete */
 261 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
 262                                 int sync, int status)
 263 {
 264         unsigned int spin_cnt, delay;
 265         u32 reg;
 266
 267         if (smmu->impl && unlikely(smmu->impl->tlb_sync))
 268                 return smmu->impl->tlb_sync(smmu, page, sync, status);
 269
 270         arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
 271         for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
 272                 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
 273                         reg = arm_smmu_readl(smmu, page, status);
 274                         if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE))
 275                                 return;
 276                         cpu_relax();
 277                 }
 278                 udelay(delay);
 279         }
 280         dev_err_ratelimited(smmu->dev,
 281                             "TLB sync timed out -- SMMU may be deadlocked\n");
 282 }
 283
 284 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
 285 {
 286         unsigned long flags;
 287
 288         spin_lock_irqsave(&smmu->global_sync_lock, flags);
 289         __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
 290                             ARM_SMMU_GR0_sTLBGSTATUS);
 291         spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
 292 }
 293
 294 static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain)
 295 {
 296         struct arm_smmu_device *smmu = smmu_domain->smmu;
 297         unsigned long flags;
 298
 299         spin_lock_irqsave(&smmu_domain->cb_lock, flags);
 300         __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
 301                             ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
 302         spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
 303 }
 304
 305 static void arm_smmu_tlb_inv_context_s1(void *cookie)
 306 {
 307         struct arm_smmu_domain *smmu_domain = cookie;
 308         /*
 309          * The TLBI write may be relaxed, so ensure that PTEs cleared by the
 310          * current CPU are visible beforehand.
 311          */
 312         wmb();
 313         arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
 314                           ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
 315         arm_smmu_tlb_sync_context(smmu_domain);
 316 }
 317
 318 static void arm_smmu_tlb_inv_context_s2(void *cookie)
 319 {
 320         struct arm_smmu_domain *smmu_domain = cookie;
 321         struct arm_smmu_device *smmu = smmu_domain->smmu;
 322
 323         /* See above */
 324         wmb();
 325         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
 326         arm_smmu_tlb_sync_global(smmu);
 327 }
 328
 329 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
 330                                       size_t granule, void *cookie, int reg)
 331 {
 332         struct arm_smmu_domain *smmu_domain = cookie;
 333         struct arm_smmu_device *smmu = smmu_domain->smmu;
 334         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 335         int idx = cfg->cbndx;
 336
 337         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 338                 wmb();
 339
 340         if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
 341                 iova = (iova >> 12) << 12;
 342                 iova |= cfg->asid;
 343                 do {
 344                         arm_smmu_cb_write(smmu, idx, reg, iova);
 345                         iova += granule;
 346                 } while (size -= granule);
 347         } else {
 348                 iova >>= 12;
 349                 iova |= (u64)cfg->asid << 48;
 350                 do {
 351                         arm_smmu_cb_writeq(smmu, idx, reg, iova);
 352                         iova += granule >> 12;
 353                 } while (size -= granule);
 354         }
 355 }
 356
 357 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
 358                                       size_t granule, void *cookie, int reg)
 359 {
 360         struct arm_smmu_domain *smmu_domain = cookie;
 361         struct arm_smmu_device *smmu = smmu_domain->smmu;
 362         int idx = smmu_domain->cfg.cbndx;
 363
 364         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 365                 wmb();
 366
 367         iova >>= 12;
 368         do {
 369                 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
 370                         arm_smmu_cb_writeq(smmu, idx, reg, iova);
 371                 else
 372                         arm_smmu_cb_write(smmu, idx, reg, iova);
 373                 iova += granule >> 12;
 374         } while (size -= granule);
 375 }
 376
 377 static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size,
 378                                      size_t granule, void *cookie)
 379 {
 380         arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
 381                                   ARM_SMMU_CB_S1_TLBIVA);
 382         arm_smmu_tlb_sync_context(cookie);
 383 }
 384
 385 static void arm_smmu_tlb_inv_leaf_s1(unsigned long iova, size_t size,
 386                                      size_t granule, void *cookie)
 387 {
 388         arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
 389                                   ARM_SMMU_CB_S1_TLBIVAL);
 390         arm_smmu_tlb_sync_context(cookie);
 391 }
 392
 393 static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather,
 394                                      unsigned long iova, size_t granule,
 395                                      void *cookie)
 396 {
 397         arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie,
 398                                   ARM_SMMU_CB_S1_TLBIVAL);
 399 }
 400
 401 static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size,
 402                                      size_t granule, void *cookie)
 403 {
 404         arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
 405                                   ARM_SMMU_CB_S2_TLBIIPAS2);
 406         arm_smmu_tlb_sync_context(cookie);
 407 }
 408
 409 static void arm_smmu_tlb_inv_leaf_s2(unsigned long iova, size_t size,
 410                                      size_t granule, void *cookie)
 411 {
 412         arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
 413                                   ARM_SMMU_CB_S2_TLBIIPAS2L);
 414         arm_smmu_tlb_sync_context(cookie);
 415 }
 416
 417 static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather,
 418                                      unsigned long iova, size_t granule,
 419                                      void *cookie)
 420 {
 421         arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie,
 422                                   ARM_SMMU_CB_S2_TLBIIPAS2L);
 423 }
 424
 425 static void arm_smmu_tlb_inv_any_s2_v1(unsigned long iova, size_t size,
 426                                        size_t granule, void *cookie)
 427 {
 428         arm_smmu_tlb_inv_context_s2(cookie);
 429 }
 430 /*
 431  * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
 432  * almost negligible, but the benefit of getting the first one in as far ahead
 433  * of the sync as possible is significant, hence we don't just make this a
 434  * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might
 435  * think.
 436  */
 437 static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather,
 438                                         unsigned long iova, size_t granule,
 439                                         void *cookie)
 440 {
 441         struct arm_smmu_domain *smmu_domain = cookie;
 442         struct arm_smmu_device *smmu = smmu_domain->smmu;
 443
 444         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 445                 wmb();
 446
 447         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
 448 }
 449
 450 static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = {
 451         .tlb_flush_all  = arm_smmu_tlb_inv_context_s1,
 452         .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1,
 453         .tlb_flush_leaf = arm_smmu_tlb_inv_leaf_s1,
 454         .tlb_add_page   = arm_smmu_tlb_add_page_s1,
 455 };
 456
 457 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
 458         .tlb_flush_all  = arm_smmu_tlb_inv_context_s2,
 459         .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2,
 460         .tlb_flush_leaf = arm_smmu_tlb_inv_leaf_s2,
 461         .tlb_add_page   = arm_smmu_tlb_add_page_s2,
 462 };
 463
 464 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
 465         .tlb_flush_all  = arm_smmu_tlb_inv_context_s2,
 466         .tlb_flush_walk = arm_smmu_tlb_inv_any_s2_v1,
 467         .tlb_flush_leaf = arm_smmu_tlb_inv_any_s2_v1,
 468         .tlb_add_page   = arm_smmu_tlb_add_page_s2_v1,
 469 };
 470
 471 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 472 {
 473         u32 fsr, fsynr, cbfrsynra;
 474         unsigned long iova;
 475         struct iommu_domain *domain = dev;
 476         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 477         struct arm_smmu_device *smmu = smmu_domain->smmu;
 478         int idx = smmu_domain->cfg.cbndx;
 479
 480         fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
 481         if (!(fsr & ARM_SMMU_FSR_FAULT))
 482                 return IRQ_NONE;
 483
 484         fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
 485         iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
 486         cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
 487
 488         dev_err_ratelimited(smmu->dev,
 489         "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
 490                             fsr, iova, fsynr, cbfrsynra, idx);
 491
 492         arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);
 493         return IRQ_HANDLED;
 494 }
 495
 496 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 497 {
 498         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
 499         struct arm_smmu_device *smmu = dev;
 500         static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
 501                                       DEFAULT_RATELIMIT_BURST);
 502
 503         gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
 504         gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
 505         gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
 506         gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
 507
 508         if (!gfsr)
 509                 return IRQ_NONE;
 510
 511         if (__ratelimit(&rs)) {
 512                 if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&
 513                     (gfsr & ARM_SMMU_sGFSR_USF))
 514                         dev_err(smmu->dev,
 515                                 "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",
 516                                 (u16)gfsynr1);
 517                 else
 518                         dev_err(smmu->dev,
 519                                 "Unexpected global fault, this could be serious\n");
 520                 dev_err(smmu->dev,
 521                         "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
 522                         gfsr, gfsynr0, gfsynr1, gfsynr2);
 523         }
 524
 525         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
 526         return IRQ_HANDLED;
 527 }
 528
 529 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 530                                        struct io_pgtable_cfg *pgtbl_cfg)
 531 {
 532         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 533         struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
 534         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 535
 536         cb->cfg = cfg;
 537
 538         /* TCR */
 539         if (stage1) {
 540                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 541                         cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
 542                 } else {
 543                         cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg);
 544                         cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg);
 545                         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 546                                 cb->tcr[1] |= ARM_SMMU_TCR2_AS;
 547                         else
 548                                 cb->tcr[0] |= ARM_SMMU_TCR_EAE;
 549                 }
 550         } else {
 551                 cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg);
 552         }
 553
 554         /* TTBRs */
 555         if (stage1) {
 556                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 557                         cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
 558                         cb->ttbr[1] = 0;
 559                 } else {
 560                         cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
 561                         cb->ttbr[0] |= FIELD_PREP(ARM_SMMU_TTBRn_ASID,
 562                                                   cfg->asid);
 563                         cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
 564                                                  cfg->asid);
 565                 }
 566         } else {
 567                 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
 568         }
 569
 570         /* MAIRs (stage-1 only) */
 571         if (stage1) {
 572                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 573                         cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
 574                         cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
 575                 } else {
 576                         cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair;
 577                         cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32;
 578                 }
 579         }
 580 }
 581
 582 static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
 583 {
 584         u32 reg;
 585         bool stage1;
 586         struct arm_smmu_cb *cb = &smmu->cbs[idx];
 587         struct arm_smmu_cfg *cfg = cb->cfg;
 588
 589         /* Unassigned context banks only need disabling */
 590         if (!cfg) {
 591                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
 592                 return;
 593         }
 594
 595         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 596
 597         /* CBA2R */
 598         if (smmu->version > ARM_SMMU_V1) {
 599                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 600                         reg = ARM_SMMU_CBA2R_VA64;
 601                 else
 602                         reg = 0;
 603                 /* 16-bit VMIDs live in CBA2R */
 604                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
 605                         reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid);
 606
 607                 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
 608         }
 609
 610         /* CBAR */
 611         reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar);
 612         if (smmu->version < ARM_SMMU_V2)
 613                 reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx);
 614
 615         /*
 616          * Use the weakest shareability/memory types, so they are
 617          * overridden by the ttbcr/pte.
 618          */
 619         if (stage1) {
 620                 reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG,
 621                                   ARM_SMMU_CBAR_S1_BPSHCFG_NSH) |
 622                        FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR,
 623                                   ARM_SMMU_CBAR_S1_MEMATTR_WB);
 624         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
 625                 /* 8-bit VMIDs live in CBAR */
 626                 reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid);
 627         }
 628         arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
 629
 630         /*
 631          * TCR
 632          * We must write this before the TTBRs, since it determines the
 633          * access behaviour of some fields (in particular, ASID[15:8]).
 634          */
 635         if (stage1 && smmu->version > ARM_SMMU_V1)
 636                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
 637         arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
 638
 639         /* TTBRs */
 640         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 641                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
 642                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
 643                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
 644         } else {
 645                 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
 646                 if (stage1)
 647                         arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
 648                                            cb->ttbr[1]);
 649         }
 650
 651         /* MAIRs (stage-1 only) */
 652         if (stage1) {
 653                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
 654                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
 655         }
 656
 657         /* SCTLR */
 658         reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE |
 659               ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M;
 660         if (stage1)
 661                 reg |= ARM_SMMU_SCTLR_S1_ASIDPNE;
 662         if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
 663                 reg |= ARM_SMMU_SCTLR_E;
 664
 665         arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
 666 }
 667
 668 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 669                                         struct arm_smmu_device *smmu)
 670 {
 671         int irq, start, ret = 0;
 672         unsigned long ias, oas;
 673         struct io_pgtable_ops *pgtbl_ops;
 674         struct io_pgtable_cfg pgtbl_cfg;
 675         enum io_pgtable_fmt fmt;
 676         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 677         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 678
 679         mutex_lock(&smmu_domain->init_mutex);
 680         if (smmu_domain->smmu)
 681                 goto out_unlock;
 682
 683         if (domain->type == IOMMU_DOMAIN_IDENTITY) {
 684                 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
 685                 smmu_domain->smmu = smmu;
 686                 goto out_unlock;
 687         }
 688
 689         /*
 690          * Mapping the requested stage onto what we support is surprisingly
 691          * complicated, mainly because the spec allows S1+S2 SMMUs without
 692          * support for nested translation. That means we end up with the
 693          * following table:
 694          *
 695          * Requested        Supported        Actual
 696          *     S1               N              S1
 697          *     S1             S1+S2            S1
 698          *     S1               S2             S2
 699          *     S1               S1             S1
 700          *     N                N              N
 701          *     N              S1+S2            S2
 702          *     N                S2             S2
 703          *     N                S1             S1
 704          *
 705          * Note that you can't actually request stage-2 mappings.
 706          */
 707         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
 708                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 709         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 710                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 711
 712         /*
 713          * Choosing a suitable context format is even more fiddly. Until we
 714          * grow some way for the caller to express a preference, and/or move
 715          * the decision into the io-pgtable code where it arguably belongs,
 716          * just aim for the closest thing to the rest of the system, and hope
 717          * that the hardware isn't esoteric enough that we can't assume AArch64
 718          * support to be a superset of AArch32 support...
 719          */
 720         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
 721                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
 722         if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
 723             !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
 724             (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
 725             (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
 726                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
 727         if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
 728             (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
 729                                ARM_SMMU_FEAT_FMT_AARCH64_16K |
 730                                ARM_SMMU_FEAT_FMT_AARCH64_4K)))
 731                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
 732
 733         if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
 734                 ret = -EINVAL;
 735                 goto out_unlock;
 736         }
 737
 738         switch (smmu_domain->stage) {
 739         case ARM_SMMU_DOMAIN_S1:
 740                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
 741                 start = smmu->num_s2_context_banks;
 742                 ias = smmu->va_size;
 743                 oas = smmu->ipa_size;
 744                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 745                         fmt = ARM_64_LPAE_S1;
 746                 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
 747                         fmt = ARM_32_LPAE_S1;
 748                         ias = min(ias, 32UL);
 749                         oas = min(oas, 40UL);
 750                 } else {
 751                         fmt = ARM_V7S;
 752                         ias = min(ias, 32UL);
 753                         oas = min(oas, 32UL);
 754                 }
 755                 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
 756                 break;
 757         case ARM_SMMU_DOMAIN_NESTED:
 758                 /*
 759                  * We will likely want to change this if/when KVM gets
 760                  * involved.
 761                  */
 762         case ARM_SMMU_DOMAIN_S2:
 763                 cfg->cbar = CBAR_TYPE_S2_TRANS;
 764                 start = 0;
 765                 ias = smmu->ipa_size;
 766                 oas = smmu->pa_size;
 767                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 768                         fmt = ARM_64_LPAE_S2;
 769                 } else {
 770                         fmt = ARM_32_LPAE_S2;
 771                         ias = min(ias, 40UL);
 772                         oas = min(oas, 40UL);
 773                 }
 774                 if (smmu->version == ARM_SMMU_V2)
 775                         smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
 776                 else
 777                         smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
 778                 break;
 779         default:
 780                 ret = -EINVAL;
 781                 goto out_unlock;
 782         }
 783         ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
 784                                       smmu->num_context_banks);
 785         if (ret < 0)
 786                 goto out_unlock;
 787
 788         cfg->cbndx = ret;
 789         if (smmu->version < ARM_SMMU_V2) {
 790                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
 791                 cfg->irptndx %= smmu->num_context_irqs;
 792         } else {
 793                 cfg->irptndx = cfg->cbndx;
 794         }
 795
 796         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
 797                 cfg->vmid = cfg->cbndx + 1;
 798         else
 799                 cfg->asid = cfg->cbndx;
 800
 801         smmu_domain->smmu = smmu;
 802         if (smmu->impl && smmu->impl->init_context) {
 803                 ret = smmu->impl->init_context(smmu_domain);
 804                 if (ret)
 805                         goto out_unlock;
 806         }
 807
 808         pgtbl_cfg = (struct io_pgtable_cfg) {
 809                 .pgsize_bitmap  = smmu->pgsize_bitmap,
 810                 .ias            = ias,
 811                 .oas            = oas,
 812                 .coherent_walk  = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
 813                 .tlb            = smmu_domain->flush_ops,
 814                 .iommu_dev      = smmu->dev,
 815         };
 816
 817         if (smmu_domain->non_strict)
 818                 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
 819
 820         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
 821         if (!pgtbl_ops) {
 822                 ret = -ENOMEM;
 823                 goto out_clear_smmu;
 824         }
 825
 826         /* Update the domain's page sizes to reflect the page table format */
 827         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 828         domain->geometry.aperture_end = (1UL << ias) - 1;
 829         domain->geometry.force_aperture = true;
 830
 831         /* Initialise the context bank with our page table cfg */
 832         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
 833         arm_smmu_write_context_bank(smmu, cfg->cbndx);
 834
 835         /*
 836          * Request context fault interrupt. Do this last to avoid the
 837          * handler seeing a half-initialised domain state.
 838          */
 839         irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 840         ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
 841                                IRQF_SHARED, "arm-smmu-context-fault", domain);
 842         if (ret < 0) {
 843                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
 844                         cfg->irptndx, irq);
 845                 cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
 846         }
 847
 848         mutex_unlock(&smmu_domain->init_mutex);
 849
 850         /* Publish page table ops for map/unmap */
 851         smmu_domain->pgtbl_ops = pgtbl_ops;
 852         return 0;
 853
 854 out_clear_smmu:
 855         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
 856         smmu_domain->smmu = NULL;
 857 out_unlock:
 858         mutex_unlock(&smmu_domain->init_mutex);
 859         return ret;
 860 }
 861
 862 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 863 {
 864         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 865         struct arm_smmu_device *smmu = smmu_domain->smmu;
 866         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 867         int ret, irq;
 868
 869         if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
 870                 return;
 871
 872         ret = arm_smmu_rpm_get(smmu);
 873         if (ret < 0)
 874                 return;
 875
 876         /*
 877          * Disable the context bank and free the page tables before freeing
 878          * it.
 879          */
 880         smmu->cbs[cfg->cbndx].cfg = NULL;
 881         arm_smmu_write_context_bank(smmu, cfg->cbndx);
 882
 883         if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
 884                 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 885                 devm_free_irq(smmu->dev, irq, domain);
 886         }
 887
 888         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
 889         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
 890
 891         arm_smmu_rpm_put(smmu);
 892 }
 893
 894 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
 895 {
 896         struct arm_smmu_domain *smmu_domain;
 897
 898         if (type != IOMMU_DOMAIN_UNMANAGED &&
 899             type != IOMMU_DOMAIN_DMA &&
 900             type != IOMMU_DOMAIN_IDENTITY)
 901                 return NULL;
 902         /*
 903          * Allocate the domain and initialise some of its data structures.
 904          * We can't really do anything meaningful until we've added a
 905          * master.
 906          */
 907         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
 908         if (!smmu_domain)
 909                 return NULL;
 910
 911         if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
 912             iommu_get_dma_cookie(&smmu_domain->domain))) {
 913                 kfree(smmu_domain);
 914                 return NULL;
 915         }
 916
 917         mutex_init(&smmu_domain->init_mutex);
 918         spin_lock_init(&smmu_domain->cb_lock);
 919
 920         return &smmu_domain->domain;
 921 }
 922
 923 static void arm_smmu_domain_free(struct iommu_domain *domain)
 924 {
 925         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 926
 927         /*
 928          * Free the domain resources. We assume that all devices have
 929          * already been detached.
 930          */
 931         iommu_put_dma_cookie(domain);
 932         arm_smmu_destroy_domain_context(domain);
 933         kfree(smmu_domain);
 934 }
 935
 936 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
 937 {
 938         struct arm_smmu_smr *smr = smmu->smrs + idx;
 939         u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) |
 940                   FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask);
 941
 942         if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
 943                 reg |= ARM_SMMU_SMR_VALID;
 944         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
 945 }
 946
 947 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
 948 {
 949         struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
 950         u32 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) |
 951                   FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) |
 952                   FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
 953
 954         if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
 955             smmu->smrs[idx].valid)
 956                 reg |= ARM_SMMU_S2CR_EXIDVALID;
 957         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
 958 }
 959
 960 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
 961 {
 962         arm_smmu_write_s2cr(smmu, idx);
 963         if (smmu->smrs)
 964                 arm_smmu_write_smr(smmu, idx);
 965 }
 966
 967 /*
 968  * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
 969  * should be called after sCR0 is written.
 970  */
 971 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
 972 {
 973         u32 smr;
 974         int i;
 975
 976         if (!smmu->smrs)
 977                 return;
 978         /*
 979          * If we've had to accommodate firmware memory regions, we may
 980          * have live SMRs by now; tread carefully...
 981          *
 982          * Somewhat perversely, not having a free SMR for this test implies we
 983          * can get away without it anyway, as we'll only be able to 'allocate'
 984          * these SMRs for the ID/mask values we're already trusting to be OK.
 985          */
 986         for (i = 0; i < smmu->num_mapping_groups; i++)
 987                 if (!smmu->smrs[i].valid)
 988                         goto smr_ok;
 989         return;
 990 smr_ok:
 991         /*
 992          * SMR.ID bits may not be preserved if the corresponding MASK
 993          * bits are set, so check each one separately. We can reject
 994          * masters later if they try to claim IDs outside these masks.
 995          */
 996         smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask);
 997         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
 998         smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
 999         smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr);
1000
1001         smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask);
1002         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
1003         smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
1004         smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
1005 }
1006
1007 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1008 {
1009         struct arm_smmu_smr *smrs = smmu->smrs;
1010         int i, free_idx = -ENOSPC;
1011
1012         /* Stream indexing is blissfully easy */
1013         if (!smrs)
1014                 return id;
1015
1016         /* Validating SMRs is... less so */
1017         for (i = 0; i < smmu->num_mapping_groups; ++i) {
1018                 if (!smrs[i].valid) {
1019                         /*
1020                          * Note the first free entry we come across, which
1021                          * we'll claim in the end if nothing else matches.
1022                          */
1023                         if (free_idx < 0)
1024                                 free_idx = i;
1025                         continue;
1026                 }
1027                 /*
1028                  * If the new entry is _entirely_ matched by an existing entry,
1029                  * then reuse that, with the guarantee that there also cannot
1030                  * be any subsequent conflicting entries. In normal use we'd
1031                  * expect simply identical entries for this case, but there's
1032                  * no harm in accommodating the generalisation.
1033                  */
1034                 if ((mask & smrs[i].mask) == mask &&
1035                     !((id ^ smrs[i].id) & ~smrs[i].mask))
1036                         return i;
1037                 /*
1038                  * If the new entry has any other overlap with an existing one,
1039                  * though, then there always exists at least one stream ID
1040                  * which would cause a conflict, and we can't allow that risk.
1041                  */
1042                 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1043                         return -EINVAL;
1044         }
1045
1046         return free_idx;
1047 }
1048
1049 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1050 {
1051         if (--smmu->s2crs[idx].count)
1052                 return false;
1053
1054         smmu->s2crs[idx] = s2cr_init_val;
1055         if (smmu->smrs)
1056                 smmu->smrs[idx].valid = false;
1057
1058         return true;
1059 }
1060
1061 static int arm_smmu_master_alloc_smes(struct device *dev)
1062 {
1063         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1064         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1065         struct arm_smmu_device *smmu = cfg->smmu;
1066         struct arm_smmu_smr *smrs = smmu->smrs;
1067         struct iommu_group *group;
1068         int i, idx, ret;
1069
1070         mutex_lock(&smmu->stream_map_mutex);
1071         /* Figure out a viable stream map entry allocation */
1072         for_each_cfg_sme(fwspec, i, idx) {
1073                 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1074                 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1075
1076                 if (idx != INVALID_SMENDX) {
1077                         ret = -EEXIST;
1078                         goto out_err;
1079                 }
1080
1081                 ret = arm_smmu_find_sme(smmu, sid, mask);
1082                 if (ret < 0)
1083                         goto out_err;
1084
1085                 idx = ret;
1086                 if (smrs && smmu->s2crs[idx].count == 0) {
1087                         smrs[idx].id = sid;
1088                         smrs[idx].mask = mask;
1089                         smrs[idx].valid = true;
1090                 }
1091                 smmu->s2crs[idx].count++;
1092                 cfg->smendx[i] = (s16)idx;
1093         }
1094
1095         group = iommu_group_get_for_dev(dev);
1096         if (IS_ERR(group)) {
1097                 ret = PTR_ERR(group);
1098                 goto out_err;
1099         }
1100         iommu_group_put(group);
1101
1102         /* It worked! Now, poke the actual hardware */
1103         for_each_cfg_sme(fwspec, i, idx) {
1104                 arm_smmu_write_sme(smmu, idx);
1105                 smmu->s2crs[idx].group = group;
1106         }
1107
1108         mutex_unlock(&smmu->stream_map_mutex);
1109         return 0;
1110
1111 out_err:
1112         while (i--) {
1113                 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1114                 cfg->smendx[i] = INVALID_SMENDX;
1115         }
1116         mutex_unlock(&smmu->stream_map_mutex);
1117         return ret;
1118 }
1119
1120 static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
1121 {
1122         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1123         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1124         int i, idx;
1125
1126         mutex_lock(&smmu->stream_map_mutex);
1127         for_each_cfg_sme(fwspec, i, idx) {
1128                 if (arm_smmu_free_sme(smmu, idx))
1129                         arm_smmu_write_sme(smmu, idx);
1130                 cfg->smendx[i] = INVALID_SMENDX;
1131         }
1132         mutex_unlock(&smmu->stream_map_mutex);
1133 }
1134
1135 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1136                                       struct iommu_fwspec *fwspec)
1137 {
1138         struct arm_smmu_device *smmu = smmu_domain->smmu;
1139         struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1140         u8 cbndx = smmu_domain->cfg.cbndx;
1141         enum arm_smmu_s2cr_type type;
1142         int i, idx;
1143
1144         if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
1145                 type = S2CR_TYPE_BYPASS;
1146         else
1147                 type = S2CR_TYPE_TRANS;
1148
1149         for_each_cfg_sme(fwspec, i, idx) {
1150                 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1151                         continue;
1152
1153                 s2cr[idx].type = type;
1154                 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1155                 s2cr[idx].cbndx = cbndx;
1156                 arm_smmu_write_s2cr(smmu, idx);
1157         }
1158         return 0;
1159 }
1160
1161 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1162 {
1163         int ret;
1164         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1165         struct arm_smmu_device *smmu;
1166         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1167
1168         if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1169                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1170                 return -ENXIO;
1171         }
1172
1173         /*
1174          * FIXME: The arch/arm DMA API code tries to attach devices to its own
1175          * domains between of_xlate() and add_device() - we have no way to cope
1176          * with that, so until ARM gets converted to rely on groups and default
1177          * domains, just say no (but more politely than by dereferencing NULL).
1178          * This should be at least a WARN_ON once that's sorted.
1179          */
1180         if (!fwspec->iommu_priv)
1181                 return -ENODEV;
1182
1183         smmu = fwspec_smmu(fwspec);
1184
1185         ret = arm_smmu_rpm_get(smmu);
1186         if (ret < 0)
1187                 return ret;
1188
1189         /* Ensure that the domain is finalised */
1190         ret = arm_smmu_init_domain_context(domain, smmu);
1191         if (ret < 0)
1192                 goto rpm_put;
1193
1194         /*
1195          * Sanity check the domain. We don't support domains across
1196          * different SMMUs.
1197          */
1198         if (smmu_domain->smmu != smmu) {
1199                 dev_err(dev,
1200                         "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1201                         dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1202                 ret = -EINVAL;
1203                 goto rpm_put;
1204         }
1205
1206         /* Looks ok, so add the device to the domain */
1207         ret = arm_smmu_domain_add_master(smmu_domain, fwspec);
1208
1209         /*
1210          * Setup an autosuspend delay to avoid bouncing runpm state.
1211          * Otherwise, if a driver for a suspended consumer device
1212          * unmaps buffers, it will runpm resume/suspend for each one.
1213          *
1214          * For example, when used by a GPU device, when an application
1215          * or game exits, it can trigger unmapping 100s or 1000s of
1216          * buffers.  With a runpm cycle for each buffer, that adds up
1217          * to 5-10sec worth of reprogramming the context bank, while
1218          * the system appears to be locked up to the user.
1219          */
1220         pm_runtime_set_autosuspend_delay(smmu->dev, 20);
1221         pm_runtime_use_autosuspend(smmu->dev);
1222
1223 rpm_put:
1224         arm_smmu_rpm_put(smmu);
1225         return ret;
1226 }
1227
1228 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1229                         phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
1230 {
1231         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1232         struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1233         int ret;
1234
1235         if (!ops)
1236                 return -ENODEV;
1237
1238         arm_smmu_rpm_get(smmu);
1239         ret = ops->map(ops, iova, paddr, size, prot);
1240         arm_smmu_rpm_put(smmu);
1241
1242         return ret;
1243 }
1244
1245 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1246                              size_t size, struct iommu_iotlb_gather *gather)
1247 {
1248         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1249         struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1250         size_t ret;
1251
1252         if (!ops)
1253                 return 0;
1254
1255         arm_smmu_rpm_get(smmu);
1256         ret = ops->unmap(ops, iova, size, gather);
1257         arm_smmu_rpm_put(smmu);
1258
1259         return ret;
1260 }
1261
1262 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1263 {
1264         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1265         struct arm_smmu_device *smmu = smmu_domain->smmu;
1266
1267         if (smmu_domain->flush_ops) {
1268                 arm_smmu_rpm_get(smmu);
1269                 smmu_domain->flush_ops->tlb_flush_all(smmu_domain);
1270                 arm_smmu_rpm_put(smmu);
1271         }
1272 }
1273
1274 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
1275                                 struct iommu_iotlb_gather *gather)
1276 {
1277         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1278         struct arm_smmu_device *smmu = smmu_domain->smmu;
1279
1280         if (!smmu)
1281                 return;
1282
1283         arm_smmu_rpm_get(smmu);
1284         if (smmu->version == ARM_SMMU_V2 ||
1285             smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1286                 arm_smmu_tlb_sync_context(smmu_domain);
1287         else
1288                 arm_smmu_tlb_sync_global(smmu);
1289         arm_smmu_rpm_put(smmu);
1290 }
1291
1292 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1293                                               dma_addr_t iova)
1294 {
1295         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1296         struct arm_smmu_device *smmu = smmu_domain->smmu;
1297         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1298         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1299         struct device *dev = smmu->dev;
1300         void __iomem *reg;
1301         u32 tmp;
1302         u64 phys;
1303         unsigned long va, flags;
1304         int ret, idx = cfg->cbndx;
1305
1306         ret = arm_smmu_rpm_get(smmu);
1307         if (ret < 0)
1308                 return 0;
1309
1310         spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1311         va = iova & ~0xfffUL;
1312         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1313                 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1314         else
1315                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1316
1317         reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1318         if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_ATSR_ACTIVE),
1319                                       5, 50)) {
1320                 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1321                 dev_err(dev,
1322                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1323                         &iova);
1324                 return ops->iova_to_phys(ops, iova);
1325         }
1326
1327         phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1328         spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1329         if (phys & ARM_SMMU_CB_PAR_F) {
1330                 dev_err(dev, "translation fault!\n");
1331                 dev_err(dev, "PAR = 0x%llx\n", phys);
1332                 return 0;
1333         }
1334
1335         arm_smmu_rpm_put(smmu);
1336
1337         return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1338 }
1339
1340 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1341                                         dma_addr_t iova)
1342 {
1343         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1344         struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1345
1346         if (domain->type == IOMMU_DOMAIN_IDENTITY)
1347                 return iova;
1348
1349         if (!ops)
1350                 return 0;
1351
1352         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1353                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1354                 return arm_smmu_iova_to_phys_hard(domain, iova);
1355
1356         return ops->iova_to_phys(ops, iova);
1357 }
1358
1359 static bool arm_smmu_capable(enum iommu_cap cap)
1360 {
1361         switch (cap) {
1362         case IOMMU_CAP_CACHE_COHERENCY:
1363                 /*
1364                  * Return true here as the SMMU can always send out coherent
1365                  * requests.
1366                  */
1367                 return true;
1368         case IOMMU_CAP_NOEXEC:
1369                 return true;
1370         default:
1371                 return false;
1372         }
1373 }
1374
1375 static
1376 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1377 {
1378         struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
1379                                                           fwnode);
1380         put_device(dev);
1381         return dev ? dev_get_drvdata(dev) : NULL;
1382 }
1383
1384 static int arm_smmu_add_device(struct device *dev)
1385 {
1386         struct arm_smmu_device *smmu;
1387         struct arm_smmu_master_cfg *cfg;
1388         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1389         int i, ret;
1390
1391         if (using_legacy_binding) {
1392                 ret = arm_smmu_register_legacy_master(dev, &smmu);
1393
1394                 /*
1395                  * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1396                  * will allocate/initialise a new one. Thus we need to update fwspec for
1397                  * later use.
1398                  */
1399                 fwspec = dev_iommu_fwspec_get(dev);
1400                 if (ret)
1401                         goto out_free;
1402         } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1403                 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1404         } else {
1405                 return -ENODEV;
1406         }
1407
1408         ret = -EINVAL;
1409         for (i = 0; i < fwspec->num_ids; i++) {
1410                 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1411                 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1412
1413                 if (sid & ~smmu->streamid_mask) {
1414                         dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1415                                 sid, smmu->streamid_mask);
1416                         goto out_free;
1417                 }
1418                 if (mask & ~smmu->smr_mask_mask) {
1419                         dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1420                                 mask, smmu->smr_mask_mask);
1421                         goto out_free;
1422                 }
1423         }
1424
1425         ret = -ENOMEM;
1426         cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1427                       GFP_KERNEL);
1428         if (!cfg)
1429                 goto out_free;
1430
1431         cfg->smmu = smmu;
1432         fwspec->iommu_priv = cfg;
1433         while (i--)
1434                 cfg->smendx[i] = INVALID_SMENDX;
1435
1436         ret = arm_smmu_rpm_get(smmu);
1437         if (ret < 0)
1438                 goto out_cfg_free;
1439
1440         ret = arm_smmu_master_alloc_smes(dev);
1441         arm_smmu_rpm_put(smmu);
1442
1443         if (ret)
1444                 goto out_cfg_free;
1445
1446         iommu_device_link(&smmu->iommu, dev);
1447
1448         device_link_add(dev, smmu->dev,
1449                         DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1450
1451         return 0;
1452
1453 out_cfg_free:
1454         kfree(cfg);
1455 out_free:
1456         iommu_fwspec_free(dev);
1457         return ret;
1458 }
1459
1460 static void arm_smmu_remove_device(struct device *dev)
1461 {
1462         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1463         struct arm_smmu_master_cfg *cfg;
1464         struct arm_smmu_device *smmu;
1465         int ret;
1466
1467         if (!fwspec || fwspec->ops != &arm_smmu_ops)
1468                 return;
1469
1470         cfg  = fwspec->iommu_priv;
1471         smmu = cfg->smmu;
1472
1473         ret = arm_smmu_rpm_get(smmu);
1474         if (ret < 0)
1475                 return;
1476
1477         iommu_device_unlink(&smmu->iommu, dev);
1478         arm_smmu_master_free_smes(fwspec);
1479
1480         arm_smmu_rpm_put(smmu);
1481
1482         iommu_group_remove_device(dev);
1483         kfree(fwspec->iommu_priv);
1484         iommu_fwspec_free(dev);
1485 }
1486
1487 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1488 {
1489         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1490         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1491         struct iommu_group *group = NULL;
1492         int i, idx;
1493
1494         for_each_cfg_sme(fwspec, i, idx) {
1495                 if (group && smmu->s2crs[idx].group &&
1496                     group != smmu->s2crs[idx].group)
1497                         return ERR_PTR(-EINVAL);
1498
1499                 group = smmu->s2crs[idx].group;
1500         }
1501
1502         if (group)
1503                 return iommu_group_ref_get(group);
1504
1505         if (dev_is_pci(dev))
1506                 group = pci_device_group(dev);
1507         else if (dev_is_fsl_mc(dev))
1508                 group = fsl_mc_device_group(dev);
1509         else
1510                 group = generic_device_group(dev);
1511
1512         return group;
1513 }
1514
1515 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1516                                     enum iommu_attr attr, void *data)
1517 {
1518         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1519
1520         switch(domain->type) {
1521         case IOMMU_DOMAIN_UNMANAGED:
1522                 switch (attr) {
1523                 case DOMAIN_ATTR_NESTING:
1524                         *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1525                         return 0;
1526                 default:
1527                         return -ENODEV;
1528                 }
1529                 break;
1530         case IOMMU_DOMAIN_DMA:
1531                 switch (attr) {
1532                 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1533                         *(int *)data = smmu_domain->non_strict;
1534                         return 0;
1535                 default:
1536                         return -ENODEV;
1537                 }
1538                 break;
1539         default:
1540                 return -EINVAL;
1541         }
1542 }
1543
1544 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1545                                     enum iommu_attr attr, void *data)
1546 {
1547         int ret = 0;
1548         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1549
1550         mutex_lock(&smmu_domain->init_mutex);
1551
1552         switch(domain->type) {
1553         case IOMMU_DOMAIN_UNMANAGED:
1554                 switch (attr) {
1555                 case DOMAIN_ATTR_NESTING:
1556                         if (smmu_domain->smmu) {
1557                                 ret = -EPERM;
1558                                 goto out_unlock;
1559                         }
1560
1561                         if (*(int *)data)
1562                                 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1563                         else
1564                                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1565                         break;
1566                 default:
1567                         ret = -ENODEV;
1568                 }
1569                 break;
1570         case IOMMU_DOMAIN_DMA:
1571                 switch (attr) {
1572                 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1573                         smmu_domain->non_strict = *(int *)data;
1574                         break;
1575                 default:
1576                         ret = -ENODEV;
1577                 }
1578                 break;
1579         default:
1580                 ret = -EINVAL;
1581         }
1582 out_unlock:
1583         mutex_unlock(&smmu_domain->init_mutex);
1584         return ret;
1585 }
1586
1587 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1588 {
1589         u32 mask, fwid = 0;
1590
1591         if (args->args_count > 0)
1592                 fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]);
1593
1594         if (args->args_count > 1)
1595                 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]);
1596         else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1597                 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask);
1598
1599         return iommu_fwspec_add_ids(dev, &fwid, 1);
1600 }
1601
1602 static void arm_smmu_get_resv_regions(struct device *dev,
1603                                       struct list_head *head)
1604 {
1605         struct iommu_resv_region *region;
1606         int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1607
1608         region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1609                                          prot, IOMMU_RESV_SW_MSI);
1610         if (!region)
1611                 return;
1612
1613         list_add_tail(&region->list, head);
1614
1615         iommu_dma_get_resv_regions(dev, head);
1616 }
1617
1618 static struct iommu_ops arm_smmu_ops = {
1619         .capable                = arm_smmu_capable,
1620         .domain_alloc           = arm_smmu_domain_alloc,
1621         .domain_free            = arm_smmu_domain_free,
1622         .attach_dev             = arm_smmu_attach_dev,
1623         .map                    = arm_smmu_map,
1624         .unmap                  = arm_smmu_unmap,
1625         .flush_iotlb_all        = arm_smmu_flush_iotlb_all,
1626         .iotlb_sync             = arm_smmu_iotlb_sync,
1627         .iova_to_phys           = arm_smmu_iova_to_phys,
1628         .add_device             = arm_smmu_add_device,
1629         .remove_device          = arm_smmu_remove_device,
1630         .device_group           = arm_smmu_device_group,
1631         .domain_get_attr        = arm_smmu_domain_get_attr,
1632         .domain_set_attr        = arm_smmu_domain_set_attr,
1633         .of_xlate               = arm_smmu_of_xlate,
1634         .get_resv_regions       = arm_smmu_get_resv_regions,
1635         .put_resv_regions       = generic_iommu_put_resv_regions,
1636         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1637 };
1638
1639 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1640 {
1641         int i;
1642         u32 reg;
1643
1644         /* clear global FSR */
1645         reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1646         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1647
1648         /*
1649          * Reset stream mapping groups: Initial values mark all SMRn as
1650          * invalid and all S2CRn as bypass unless overridden.
1651          */
1652         for (i = 0; i < smmu->num_mapping_groups; ++i)
1653                 arm_smmu_write_sme(smmu, i);
1654
1655         /* Make sure all context banks are disabled and clear CB_FSR  */
1656         for (i = 0; i < smmu->num_context_banks; ++i) {
1657                 arm_smmu_write_context_bank(smmu, i);
1658                 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_FSR_FAULT);
1659         }
1660
1661         /* Invalidate the TLB, just in case */
1662         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1663         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1664
1665         reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1666
1667         /* Enable fault reporting */
1668         reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE |
1669                 ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE);
1670
1671         /* Disable TLB broadcasting. */
1672         reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM);
1673
1674         /* Enable client access, handling unmatched streams as appropriate */
1675         reg &= ~ARM_SMMU_sCR0_CLIENTPD;
1676         if (disable_bypass)
1677                 reg |= ARM_SMMU_sCR0_USFCFG;
1678         else
1679                 reg &= ~ARM_SMMU_sCR0_USFCFG;
1680
1681         /* Disable forced broadcasting */
1682         reg &= ~ARM_SMMU_sCR0_FB;
1683
1684         /* Don't upgrade barriers */
1685         reg &= ~(ARM_SMMU_sCR0_BSU);
1686
1687         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1688                 reg |= ARM_SMMU_sCR0_VMID16EN;
1689
1690         if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1691                 reg |= ARM_SMMU_sCR0_EXIDENABLE;
1692
1693         if (smmu->impl && smmu->impl->reset)
1694                 smmu->impl->reset(smmu);
1695
1696         /* Push the button */
1697         arm_smmu_tlb_sync_global(smmu);
1698         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1699 }
1700
1701 static int arm_smmu_id_size_to_bits(int size)
1702 {
1703         switch (size) {
1704         case 0:
1705                 return 32;
1706         case 1:
1707                 return 36;
1708         case 2:
1709                 return 40;
1710         case 3:
1711                 return 42;
1712         case 4:
1713                 return 44;
1714         case 5:
1715         default:
1716                 return 48;
1717         }
1718 }
1719
1720 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1721 {
1722         unsigned int size;
1723         u32 id;
1724         bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1725         int i;
1726
1727         dev_notice(smmu->dev, "probing hardware configuration...\n");
1728         dev_notice(smmu->dev, "SMMUv%d with:\n",
1729                         smmu->version == ARM_SMMU_V2 ? 2 : 1);
1730
1731         /* ID0 */
1732         id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1733
1734         /* Restrict available stages based on module parameter */
1735         if (force_stage == 1)
1736                 id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS);
1737         else if (force_stage == 2)
1738                 id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS);
1739
1740         if (id & ARM_SMMU_ID0_S1TS) {
1741                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1742                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1743         }
1744
1745         if (id & ARM_SMMU_ID0_S2TS) {
1746                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1747                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1748         }
1749
1750         if (id & ARM_SMMU_ID0_NTS) {
1751                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1752                 dev_notice(smmu->dev, "\tnested translation\n");
1753         }
1754
1755         if (!(smmu->features &
1756                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1757                 dev_err(smmu->dev, "\tno translation support!\n");
1758                 return -ENODEV;
1759         }
1760
1761         if ((id & ARM_SMMU_ID0_S1TS) &&
1762             ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) {
1763                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1764                 dev_notice(smmu->dev, "\taddress translation ops\n");
1765         }
1766
1767         /*
1768          * In order for DMA API calls to work properly, we must defer to what
1769          * the FW says about coherency, regardless of what the hardware claims.
1770          * Fortunately, this also opens up a workaround for systems where the
1771          * ID register value has ended up configured incorrectly.
1772          */
1773         cttw_reg = !!(id & ARM_SMMU_ID0_CTTW);
1774         if (cttw_fw || cttw_reg)
1775                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1776                            cttw_fw ? "" : "non-");
1777         if (cttw_fw != cttw_reg)
1778                 dev_notice(smmu->dev,
1779                            "\t(IDR0.CTTW overridden by FW configuration)\n");
1780
1781         /* Max. number of entries we have for stream matching/indexing */
1782         if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) {
1783                 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1784                 size = 1 << 16;
1785         } else {
1786                 size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id);
1787         }
1788         smmu->streamid_mask = size - 1;
1789         if (id & ARM_SMMU_ID0_SMS) {
1790                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1791                 size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id);
1792                 if (size == 0) {
1793                         dev_err(smmu->dev,
1794                                 "stream-matching supported, but no SMRs present!\n");
1795                         return -ENODEV;
1796                 }
1797
1798                 /* Zero-initialised to mark as invalid */
1799                 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1800                                           GFP_KERNEL);
1801                 if (!smmu->smrs)
1802                         return -ENOMEM;
1803
1804                 dev_notice(smmu->dev,
1805                            "\tstream matching with %u register groups", size);
1806         }
1807         /* s2cr->type == 0 means translation, so initialise explicitly */
1808         smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1809                                          GFP_KERNEL);
1810         if (!smmu->s2crs)
1811                 return -ENOMEM;
1812         for (i = 0; i < size; i++)
1813                 smmu->s2crs[i] = s2cr_init_val;
1814
1815         smmu->num_mapping_groups = size;
1816         mutex_init(&smmu->stream_map_mutex);
1817         spin_lock_init(&smmu->global_sync_lock);
1818
1819         if (smmu->version < ARM_SMMU_V2 ||
1820             !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) {
1821                 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1822                 if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S))
1823                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1824         }
1825
1826         /* ID1 */
1827         id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1828         smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12;
1829
1830         /* Check for size mismatch of SMMU address space from mapped region */
1831         size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1);
1832         if (smmu->numpage != 2 * size << smmu->pgshift)
1833                 dev_warn(smmu->dev,
1834                         "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1835                         2 * size << smmu->pgshift, smmu->numpage);
1836         /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1837         smmu->numpage = size;
1838
1839         smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id);
1840         smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id);
1841         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1842                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1843                 return -ENODEV;
1844         }
1845         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1846                    smmu->num_context_banks, smmu->num_s2_context_banks);
1847         smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1848                                  sizeof(*smmu->cbs), GFP_KERNEL);
1849         if (!smmu->cbs)
1850                 return -ENOMEM;
1851
1852         /* ID2 */
1853         id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1854         size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id));
1855         smmu->ipa_size = size;
1856
1857         /* The output mask is also applied for bypass */
1858         size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id));
1859         smmu->pa_size = size;
1860
1861         if (id & ARM_SMMU_ID2_VMID16)
1862                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1863
1864         /*
1865          * What the page table walker can address actually depends on which
1866          * descriptor format is in use, but since a) we don't know that yet,
1867          * and b) it can vary per context bank, this will have to do...
1868          */
1869         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1870                 dev_warn(smmu->dev,
1871                          "failed to set DMA mask for table walker\n");
1872
1873         if (smmu->version < ARM_SMMU_V2) {
1874                 smmu->va_size = smmu->ipa_size;
1875                 if (smmu->version == ARM_SMMU_V1_64K)
1876                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1877         } else {
1878                 size = FIELD_GET(ARM_SMMU_ID2_UBS, id);
1879                 smmu->va_size = arm_smmu_id_size_to_bits(size);
1880                 if (id & ARM_SMMU_ID2_PTFS_4K)
1881                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1882                 if (id & ARM_SMMU_ID2_PTFS_16K)
1883                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1884                 if (id & ARM_SMMU_ID2_PTFS_64K)
1885                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1886         }
1887
1888         /* Now we've corralled the various formats, what'll it do? */
1889         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1890                 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1891         if (smmu->features &
1892             (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1893                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1894         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1895                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1896         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1897                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1898
1899         if (arm_smmu_ops.pgsize_bitmap == -1UL)
1900                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1901         else
1902                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1903         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1904                    smmu->pgsize_bitmap);
1905
1906
1907         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1908                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1909                            smmu->va_size, smmu->ipa_size);
1910
1911         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1912                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1913                            smmu->ipa_size, smmu->pa_size);
1914
1915         if (smmu->impl && smmu->impl->cfg_probe)
1916                 return smmu->impl->cfg_probe(smmu);
1917
1918         return 0;
1919 }
1920
1921 struct arm_smmu_match_data {
1922         enum arm_smmu_arch_version version;
1923         enum arm_smmu_implementation model;
1924 };
1925
1926 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
1927 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1928
1929 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1930 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1931 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1932 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1933 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1934 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1935
1936 static const struct of_device_id arm_smmu_of_match[] = {
1937         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1938         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1939         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1940         { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1941         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1942         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1943         { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1944         { },
1945 };
1946 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1947
1948 #ifdef CONFIG_ACPI
1949 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1950 {
1951         int ret = 0;
1952
1953         switch (model) {
1954         case ACPI_IORT_SMMU_V1:
1955         case ACPI_IORT_SMMU_CORELINK_MMU400:
1956                 smmu->version = ARM_SMMU_V1;
1957                 smmu->model = GENERIC_SMMU;
1958                 break;
1959         case ACPI_IORT_SMMU_CORELINK_MMU401:
1960                 smmu->version = ARM_SMMU_V1_64K;
1961                 smmu->model = GENERIC_SMMU;
1962                 break;
1963         case ACPI_IORT_SMMU_V2:
1964                 smmu->version = ARM_SMMU_V2;
1965                 smmu->model = GENERIC_SMMU;
1966                 break;
1967         case ACPI_IORT_SMMU_CORELINK_MMU500:
1968                 smmu->version = ARM_SMMU_V2;
1969                 smmu->model = ARM_MMU500;
1970                 break;
1971         case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
1972                 smmu->version = ARM_SMMU_V2;
1973                 smmu->model = CAVIUM_SMMUV2;
1974                 break;
1975         default:
1976                 ret = -ENODEV;
1977         }
1978
1979         return ret;
1980 }
1981
1982 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
1983                                       struct arm_smmu_device *smmu)
1984 {
1985         struct device *dev = smmu->dev;
1986         struct acpi_iort_node *node =
1987                 *(struct acpi_iort_node **)dev_get_platdata(dev);
1988         struct acpi_iort_smmu *iort_smmu;
1989         int ret;
1990
1991         /* Retrieve SMMU1/2 specific data */
1992         iort_smmu = (struct acpi_iort_smmu *)node->node_data;
1993
1994         ret = acpi_smmu_get_data(iort_smmu->model, smmu);
1995         if (ret < 0)
1996                 return ret;
1997
1998         /* Ignore the configuration access interrupt */
1999         smmu->num_global_irqs = 1;
2000
2001         if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
2002                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2003
2004         return 0;
2005 }
2006 #else
2007 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2008                                              struct arm_smmu_device *smmu)
2009 {
2010         return -ENODEV;
2011 }
2012 #endif
2013
2014 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
2015                                     struct arm_smmu_device *smmu)
2016 {
2017         const struct arm_smmu_match_data *data;
2018         struct device *dev = &pdev->dev;
2019         bool legacy_binding;
2020
2021         if (of_property_read_u32(dev->of_node, "#global-interrupts",
2022                                  &smmu->num_global_irqs)) {
2023                 dev_err(dev, "missing #global-interrupts property\n");
2024                 return -ENODEV;
2025         }
2026
2027         data = of_device_get_match_data(dev);
2028         smmu->version = data->version;
2029         smmu->model = data->model;
2030
2031         legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2032         if (legacy_binding && !using_generic_binding) {
2033                 if (!using_legacy_binding) {
2034                         pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n",
2035                                   IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");
2036                 }
2037                 using_legacy_binding = true;
2038         } else if (!legacy_binding && !using_legacy_binding) {
2039                 using_generic_binding = true;
2040         } else {
2041                 dev_err(dev, "not probing due to mismatched DT properties\n");
2042                 return -ENODEV;
2043         }
2044
2045         if (of_dma_is_coherent(dev->of_node))
2046                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2047
2048         return 0;
2049 }
2050
2051 static int arm_smmu_bus_init(struct iommu_ops *ops)
2052 {
2053         int err;
2054
2055         /* Oh, for a proper bus abstraction */
2056         if (!iommu_present(&platform_bus_type)) {
2057                 err = bus_set_iommu(&platform_bus_type, ops);
2058                 if (err)
2059                         return err;
2060         }
2061 #ifdef CONFIG_ARM_AMBA
2062         if (!iommu_present(&amba_bustype)) {
2063                 err = bus_set_iommu(&amba_bustype, ops);
2064                 if (err)
2065                         goto err_reset_platform_ops;
2066         }
2067 #endif
2068 #ifdef CONFIG_PCI
2069         if (!iommu_present(&pci_bus_type)) {
2070                 err = bus_set_iommu(&pci_bus_type, ops);
2071                 if (err)
2072                         goto err_reset_amba_ops;
2073         }
2074 #endif
2075 #ifdef CONFIG_FSL_MC_BUS
2076         if (!iommu_present(&fsl_mc_bus_type)) {
2077                 err = bus_set_iommu(&fsl_mc_bus_type, ops);
2078                 if (err)
2079                         goto err_reset_pci_ops;
2080         }
2081 #endif
2082         return 0;
2083
2084 err_reset_pci_ops: __maybe_unused;
2085 #ifdef CONFIG_PCI
2086         bus_set_iommu(&pci_bus_type, NULL);
2087 #endif
2088 err_reset_amba_ops: __maybe_unused;
2089 #ifdef CONFIG_ARM_AMBA
2090         bus_set_iommu(&amba_bustype, NULL);
2091 #endif
2092 err_reset_platform_ops: __maybe_unused;
2093         bus_set_iommu(&platform_bus_type, NULL);
2094         return err;
2095 }
2096
2097 static int arm_smmu_device_probe(struct platform_device *pdev)
2098 {
2099         struct resource *res;
2100         resource_size_t ioaddr;
2101         struct arm_smmu_device *smmu;
2102         struct device *dev = &pdev->dev;
2103         int num_irqs, i, err;
2104
2105         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2106         if (!smmu) {
2107                 dev_err(dev, "failed to allocate arm_smmu_device\n");
2108                 return -ENOMEM;
2109         }
2110         smmu->dev = dev;
2111
2112         if (dev->of_node)
2113                 err = arm_smmu_device_dt_probe(pdev, smmu);
2114         else
2115                 err = arm_smmu_device_acpi_probe(pdev, smmu);
2116
2117         if (err)
2118                 return err;
2119
2120         smmu = arm_smmu_impl_init(smmu);
2121         if (IS_ERR(smmu))
2122                 return PTR_ERR(smmu);
2123
2124         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2125         ioaddr = res->start;
2126         smmu->base = devm_ioremap_resource(dev, res);
2127         if (IS_ERR(smmu->base))
2128                 return PTR_ERR(smmu->base);
2129         /*
2130          * The resource size should effectively match the value of SMMU_TOP;
2131          * stash that temporarily until we know PAGESIZE to validate it with.
2132          */
2133         smmu->numpage = resource_size(res);
2134
2135         num_irqs = 0;
2136         while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2137                 num_irqs++;
2138                 if (num_irqs > smmu->num_global_irqs)
2139                         smmu->num_context_irqs++;
2140         }
2141
2142         if (!smmu->num_context_irqs) {
2143                 dev_err(dev, "found %d interrupts but expected at least %d\n",
2144                         num_irqs, smmu->num_global_irqs + 1);
2145                 return -ENODEV;
2146         }
2147
2148         smmu->irqs = devm_kcalloc(dev, num_irqs, sizeof(*smmu->irqs),
2149                                   GFP_KERNEL);
2150         if (!smmu->irqs) {
2151                 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2152                 return -ENOMEM;
2153         }
2154
2155         for (i = 0; i < num_irqs; ++i) {
2156                 int irq = platform_get_irq(pdev, i);
2157
2158                 if (irq < 0)
2159                         return -ENODEV;
2160                 smmu->irqs[i] = irq;
2161         }
2162
2163         err = devm_clk_bulk_get_all(dev, &smmu->clks);
2164         if (err < 0) {
2165                 dev_err(dev, "failed to get clocks %d\n", err);
2166                 return err;
2167         }
2168         smmu->num_clks = err;
2169
2170         err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2171         if (err)
2172                 return err;
2173
2174         err = arm_smmu_device_cfg_probe(smmu);
2175         if (err)
2176                 return err;
2177
2178         if (smmu->version == ARM_SMMU_V2) {
2179                 if (smmu->num_context_banks > smmu->num_context_irqs) {
2180                         dev_err(dev,
2181                               "found only %d context irq(s) but %d required\n",
2182                               smmu->num_context_irqs, smmu->num_context_banks);
2183                         return -ENODEV;
2184                 }
2185
2186                 /* Ignore superfluous interrupts */
2187                 smmu->num_context_irqs = smmu->num_context_banks;
2188         }
2189
2190         for (i = 0; i < smmu->num_global_irqs; ++i) {
2191                 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2192                                        arm_smmu_global_fault,
2193                                        IRQF_SHARED,
2194                                        "arm-smmu global fault",
2195                                        smmu);
2196                 if (err) {
2197                         dev_err(dev, "failed to request global IRQ %d (%u)\n",
2198                                 i, smmu->irqs[i]);
2199                         return err;
2200                 }
2201         }
2202
2203         err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2204                                      "smmu.%pa", &ioaddr);
2205         if (err) {
2206                 dev_err(dev, "Failed to register iommu in sysfs\n");
2207                 return err;
2208         }
2209
2210         iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
2211         iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
2212
2213         err = iommu_device_register(&smmu->iommu);
2214         if (err) {
2215                 dev_err(dev, "Failed to register iommu\n");
2216                 return err;
2217         }
2218
2219         platform_set_drvdata(pdev, smmu);
2220         arm_smmu_device_reset(smmu);
2221         arm_smmu_test_smr_masks(smmu);
2222
2223         /*
2224          * We want to avoid touching dev->power.lock in fastpaths unless
2225          * it's really going to do something useful - pm_runtime_enabled()
2226          * can serve as an ideal proxy for that decision. So, conditionally
2227          * enable pm_runtime.
2228          */
2229         if (dev->pm_domain) {
2230                 pm_runtime_set_active(dev);
2231                 pm_runtime_enable(dev);
2232         }
2233
2234         /*
2235          * For ACPI and generic DT bindings, an SMMU will be probed before
2236          * any device which might need it, so we want the bus ops in place
2237          * ready to handle default domain setup as soon as any SMMU exists.
2238          */
2239         if (!using_legacy_binding)
2240                 return arm_smmu_bus_init(&arm_smmu_ops);
2241
2242         return 0;
2243 }
2244
2245 static int arm_smmu_device_remove(struct platform_device *pdev)
2246 {
2247         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2248
2249         if (!smmu)
2250                 return -ENODEV;
2251
2252         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2253                 dev_err(&pdev->dev, "removing device with active domains!\n");
2254
2255         arm_smmu_bus_init(NULL);
2256         iommu_device_unregister(&smmu->iommu);
2257         iommu_device_sysfs_remove(&smmu->iommu);
2258
2259         arm_smmu_rpm_get(smmu);
2260         /* Turn the thing off */
2261         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD);
2262         arm_smmu_rpm_put(smmu);
2263
2264         if (pm_runtime_enabled(smmu->dev))
2265                 pm_runtime_force_suspend(smmu->dev);
2266         else
2267                 clk_bulk_disable(smmu->num_clks, smmu->clks);
2268
2269         clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2270         return 0;
2271 }
2272
2273 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2274 {
2275         arm_smmu_device_remove(pdev);
2276 }
2277
2278 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2279 {
2280         struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2281         int ret;
2282
2283         ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2284         if (ret)
2285                 return ret;
2286
2287         arm_smmu_device_reset(smmu);
2288
2289         return 0;
2290 }
2291
2292 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2293 {
2294         struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2295
2296         clk_bulk_disable(smmu->num_clks, smmu->clks);
2297
2298         return 0;
2299 }
2300
2301 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2302 {
2303         if (pm_runtime_suspended(dev))
2304                 return 0;
2305
2306         return arm_smmu_runtime_resume(dev);
2307 }
2308
2309 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2310 {
2311         if (pm_runtime_suspended(dev))
2312                 return 0;
2313
2314         return arm_smmu_runtime_suspend(dev);
2315 }
2316
2317 static const struct dev_pm_ops arm_smmu_pm_ops = {
2318         SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2319         SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2320                            arm_smmu_runtime_resume, NULL)
2321 };
2322
2323 static struct platform_driver arm_smmu_driver = {
2324         .driver = {
2325                 .name                   = "arm-smmu",
2326                 .of_match_table         = arm_smmu_of_match,
2327                 .pm                     = &arm_smmu_pm_ops,
2328                 .suppress_bind_attrs    = true,
2329         },
2330         .probe  = arm_smmu_device_probe,
2331         .remove = arm_smmu_device_remove,
2332         .shutdown = arm_smmu_device_shutdown,
2333 };
2334 module_platform_driver(arm_smmu_driver);
2335
2336 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2337 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
2338 MODULE_ALIAS("platform:arm-smmu");
2339 MODULE_LICENSE("GPL v2");