iommu/io-pgtable: Delegate page table memory management to drivers

Some IOMMU drivers (like arm-smmu) need to perform fixups to page table
memory when it's allocated and freed.  Some callbacks were added for
this purpose ({prepare,unprepare}_pgtable) that are called from the
io-pgtable code after each allocation and before each free.  However,
this approach is prone to bugs where new calls to allocate/free are
added without corresponding calls to the fixup callbacks.  Furthermore,
allocation and free is often done from atomic context, so if the driver
needs to do non-atomic fixups during free they are out of luck since the
memory will be freed back to the system by the time control is turned
back over to the driver.

Adding yet another callback for non-atomic fixups would start to get
tedious, and would only increase the chance of missing callbacks as new
allocations/frees are added.  Instead, fix this by allowing all page
table memory allocation and free to be delegated entirely to the driver.
Fall back to {alloc,free}_pages_exact by default if the driver doesn't
need any special handling of page table memory.

Change-Id: I0361bb81e25ff5ad4ef93a45330a35af47bc6013
Signed-off-by: Mitchel Humpherys <mitchelh@codeaurora.org>

diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index 38e95fe..ae9bfd6 100644 (file)
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -950,13 +950,30 @@ static void arm_smmu_flush_pgtable(void *addr, size_t size, void *cookie)
 static void arm_smmu_prepare_pgtable(void *addr, void *cookie);
 static void arm_smmu_unprepare_pgtable(void *cookie, void *addr);
 
+static void *arm_smmu_alloc_pages_exact(void *cookie,
+                                       size_t size, gfp_t gfp_mask)
+{
+       void *ret = alloc_pages_exact(size, gfp_mask);
+
+       if (likely(ret))
+               arm_smmu_prepare_pgtable(ret, cookie);
+
+       return ret;
+}
+
+static void arm_smmu_free_pages_exact(void *cookie, void *virt, size_t size)
+{
+       arm_smmu_unprepare_pgtable(cookie, virt);
+       free_pages_exact(virt, size);
+}
+
 static struct iommu_gather_ops arm_smmu_gather_ops = {
        .tlb_flush_all  = arm_smmu_tlb_inv_context,
        .tlb_add_flush  = arm_smmu_tlb_inv_range_nosync,
        .tlb_sync       = arm_smmu_tlb_sync,
        .flush_pgtable  = arm_smmu_flush_pgtable,
-       .prepare_pgtable = arm_smmu_prepare_pgtable,
-       .unprepare_pgtable = arm_smmu_unprepare_pgtable,
+       .alloc_pages_exact = arm_smmu_alloc_pages_exact,
+       .free_pages_exact = arm_smmu_free_pages_exact,
 };
 
 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,

diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c
index 1777fd9..b78d8ca 100644 (file)
--- a/drivers/iommu/io-pgtable-arm.c
+++ b/drivers/iommu/io-pgtable-arm.c
@@ -325,7 +325,8 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
        /* Grab a pointer to the next level */
        pte = *ptep;
        if (!pte) {
-               cptep = io_pgtable_alloc_pages_exact(1UL << data->pg_shift,
+               cptep = io_pgtable_alloc_pages_exact(&data->iop.cfg, cookie,
+                                                    1UL << data->pg_shift,
                                                     GFP_ATOMIC | __GFP_ZERO);
                if (!cptep)
                        return -ENOMEM;
@@ -337,7 +338,6 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
                        pte |= ARM_LPAE_PTE_NSTABLE;
                *ptep = pte;
                data->iop.cfg.tlb->flush_pgtable(ptep, sizeof(*ptep), cookie);
-               data->iop.cfg.tlb->prepare_pgtable(cptep, cookie);
        } else {
                cptep = iopte_deref(pte, data);
        }
@@ -490,8 +490,8 @@ static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
                __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));
        }
 
-       data->iop.cfg.tlb->unprepare_pgtable(data->iop.cookie, start);
-       io_pgtable_free_pages_exact(start, table_size);
+       io_pgtable_free_pages_exact(&data->iop.cfg, data->iop.cookie,
+                                   start, table_size);
 }
 
 static void arm_lpae_free_pgtable(struct io_pgtable *iop)
@@ -600,7 +600,8 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
                        *ptep = 0;
                        tlb->flush_pgtable(ptep, sizeof(*ptep), cookie);
                        io_pgtable_free_pages_exact(
-                               table_base, max_entries * sizeof(*table_base));
+                               &data->iop.cfg, cookie, table_base,
+                               max_entries * sizeof(*table_base));
                }
 
                return entries * entry_size;
@@ -825,13 +826,12 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
        cfg->arm_lpae_s1_cfg.mair[1] = 0;
 
        /* Looking good; allocate a pgd */
-       data->pgd = io_pgtable_alloc_pages_exact(data->pgd_size,
+       data->pgd = io_pgtable_alloc_pages_exact(cfg, cookie, data->pgd_size,
                                                 GFP_KERNEL | __GFP_ZERO);
        if (!data->pgd)
                goto out_free_data;
 
        cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie);
-       cfg->tlb->prepare_pgtable(data->pgd, cookie);
        /* TTBRs */
        cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd);
        cfg->arm_lpae_s1_cfg.ttbr[1] = 0;
@@ -914,13 +914,12 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)
        cfg->arm_lpae_s2_cfg.vtcr = reg;
 
        /* Allocate pgd pages */
-       data->pgd = io_pgtable_alloc_pages_exact(data->pgd_size,
+       data->pgd = io_pgtable_alloc_pages_exact(cfg, cookie, data->pgd_size,
                                                 GFP_KERNEL | __GFP_ZERO);
        if (!data->pgd)
                goto out_free_data;
 
        cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie);
-       cfg->tlb->prepare_pgtable(data->pgd, cookie);
        /* VTTBR */
        cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd);
        return &data->iop;

diff --git a/drivers/iommu/io-pgtable.c b/drivers/iommu/io-pgtable.c
index 881c8a0..3a74d44 100644 (file)
--- a/drivers/iommu/io-pgtable.c
+++ b/drivers/iommu/io-pgtable.c
@@ -90,18 +90,30 @@ void free_io_pgtable_ops(struct io_pgtable_ops *ops)
 
 static atomic_t pages_allocated;
 
-void *io_pgtable_alloc_pages_exact(size_t size, gfp_t gfp_mask)
+void *io_pgtable_alloc_pages_exact(struct io_pgtable_cfg *cfg, void *cookie,
+                                  size_t size, gfp_t gfp_mask)
 {
-       void *ret = alloc_pages_exact(size, gfp_mask);
+       void *ret;
+
+       if (cfg->tlb->alloc_pages_exact)
+               ret = cfg->tlb->alloc_pages_exact(cookie, size, gfp_mask);
+       else
+               ret = alloc_pages_exact(size, gfp_mask);
 
        if (likely(ret))
                atomic_add(1 << get_order(size), &pages_allocated);
+
        return ret;
 }
 
-void io_pgtable_free_pages_exact(void *virt, size_t size)
+void io_pgtable_free_pages_exact(struct io_pgtable_cfg *cfg, void *cookie,
+                                void *virt, size_t size)
 {
-       free_pages_exact(virt, size);
+       if (cfg->tlb->free_pages_exact)
+               cfg->tlb->free_pages_exact(cookie, virt, size);
+       else
+               free_pages_exact(virt, size);
+
        atomic_sub(1 << get_order(size), &pages_allocated);
 }
 

diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h
index 2ca3393..c1c656b 100644 (file)
--- a/drivers/iommu/io-pgtable.h
+++ b/drivers/iommu/io-pgtable.h
@@ -21,8 +21,10 @@ enum io_pgtable_fmt {
  * @tlb_add_flush: Queue up a TLB invalidation for a virtual address range.
  * @tlb_sync:      Ensure any queue TLB invalidation has taken effect.
  * @flush_pgtable: Ensure page table updates are visible to the IOMMU.
- * @prepare_pgtable: Do necessary fixup for newly allocated page table memory
- * @unprepare_pgtable: Undo fixups done during @prepare_pgtable
+ * @alloc_pages_exact: Allocate page table memory (optional, defaults to
+ *                     alloc_pages_exact)
+ * @free_pages_exact:  Free page table memory (optional, defaults to
+ *                     free_pages_exact)
  *
  * Note that these can all be called in atomic context and must therefore
  * not block.
@@ -33,8 +35,8 @@ struct iommu_gather_ops {
                              void *cookie);
        void (*tlb_sync)(void *cookie);
        void (*flush_pgtable)(void *ptr, size_t size, void *cookie);
-       void (*prepare_pgtable)(void *addr, void *cookie);
-       void (*unprepare_pgtable)(void *cookie, void *addr);
+       void *(*alloc_pages_exact)(void *cookie, size_t size, gfp_t gfp_mask);
+       void (*free_pages_exact)(void *cookie, void *virt, size_t size);
 };
 
 /**
@@ -156,7 +158,8 @@ struct io_pgtable_init_fns {
  * Like alloc_pages_exact(), but with some additional accounting for debug
  * purposes.
  */
-void *io_pgtable_alloc_pages_exact(size_t size, gfp_t gfp_mask);
+void *io_pgtable_alloc_pages_exact(struct io_pgtable_cfg *cfg, void *cookie,
+                                  size_t size, gfp_t gfp_mask);
 
 /**
  * io_pgtable_free_pages_exact - release memory allocated via io_pgtable_alloc_pages_exact()
@@ -166,6 +169,7 @@ void *io_pgtable_alloc_pages_exact(size_t size, gfp_t gfp_mask);
  * Like free_pages_exact(), but with some additional accounting for debug
  * purposes.
  */
-void io_pgtable_free_pages_exact(void *virt, size_t size);
+void io_pgtable_free_pages_exact(struct io_pgtable_cfg *cfg, void *cookie,
+                                void *virt, size_t size);
 
 #endif /* __IO_PGTABLE_H */