static void arm_smmu_prepare_pgtable(void *addr, void *cookie);
static void arm_smmu_unprepare_pgtable(void *cookie, void *addr);
+static void *arm_smmu_alloc_pages_exact(void *cookie,
+ size_t size, gfp_t gfp_mask)
+{
+ void *ret = alloc_pages_exact(size, gfp_mask);
+
+ if (likely(ret))
+ arm_smmu_prepare_pgtable(ret, cookie);
+
+ return ret;
+}
+
+static void arm_smmu_free_pages_exact(void *cookie, void *virt, size_t size)
+{
+ arm_smmu_unprepare_pgtable(cookie, virt);
+ free_pages_exact(virt, size);
+}
+
static struct iommu_gather_ops arm_smmu_gather_ops = {
.tlb_flush_all = arm_smmu_tlb_inv_context,
.tlb_add_flush = arm_smmu_tlb_inv_range_nosync,
.tlb_sync = arm_smmu_tlb_sync,
.flush_pgtable = arm_smmu_flush_pgtable,
- .prepare_pgtable = arm_smmu_prepare_pgtable,
- .unprepare_pgtable = arm_smmu_unprepare_pgtable,
+ .alloc_pages_exact = arm_smmu_alloc_pages_exact,
+ .free_pages_exact = arm_smmu_free_pages_exact,
};
static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
/* Grab a pointer to the next level */
pte = *ptep;
if (!pte) {
- cptep = io_pgtable_alloc_pages_exact(1UL << data->pg_shift,
+ cptep = io_pgtable_alloc_pages_exact(&data->iop.cfg, cookie,
+ 1UL << data->pg_shift,
GFP_ATOMIC | __GFP_ZERO);
if (!cptep)
return -ENOMEM;
pte |= ARM_LPAE_PTE_NSTABLE;
*ptep = pte;
data->iop.cfg.tlb->flush_pgtable(ptep, sizeof(*ptep), cookie);
- data->iop.cfg.tlb->prepare_pgtable(cptep, cookie);
} else {
cptep = iopte_deref(pte, data);
}
__arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));
}
- data->iop.cfg.tlb->unprepare_pgtable(data->iop.cookie, start);
- io_pgtable_free_pages_exact(start, table_size);
+ io_pgtable_free_pages_exact(&data->iop.cfg, data->iop.cookie,
+ start, table_size);
}
static void arm_lpae_free_pgtable(struct io_pgtable *iop)
*ptep = 0;
tlb->flush_pgtable(ptep, sizeof(*ptep), cookie);
io_pgtable_free_pages_exact(
- table_base, max_entries * sizeof(*table_base));
+ &data->iop.cfg, cookie, table_base,
+ max_entries * sizeof(*table_base));
}
return entries * entry_size;
cfg->arm_lpae_s1_cfg.mair[1] = 0;
/* Looking good; allocate a pgd */
- data->pgd = io_pgtable_alloc_pages_exact(data->pgd_size,
+ data->pgd = io_pgtable_alloc_pages_exact(cfg, cookie, data->pgd_size,
GFP_KERNEL | __GFP_ZERO);
if (!data->pgd)
goto out_free_data;
cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie);
- cfg->tlb->prepare_pgtable(data->pgd, cookie);
/* TTBRs */
cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd);
cfg->arm_lpae_s1_cfg.ttbr[1] = 0;
cfg->arm_lpae_s2_cfg.vtcr = reg;
/* Allocate pgd pages */
- data->pgd = io_pgtable_alloc_pages_exact(data->pgd_size,
+ data->pgd = io_pgtable_alloc_pages_exact(cfg, cookie, data->pgd_size,
GFP_KERNEL | __GFP_ZERO);
if (!data->pgd)
goto out_free_data;
cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie);
- cfg->tlb->prepare_pgtable(data->pgd, cookie);
/* VTTBR */
cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd);
return &data->iop;
static atomic_t pages_allocated;
-void *io_pgtable_alloc_pages_exact(size_t size, gfp_t gfp_mask)
+void *io_pgtable_alloc_pages_exact(struct io_pgtable_cfg *cfg, void *cookie,
+ size_t size, gfp_t gfp_mask)
{
- void *ret = alloc_pages_exact(size, gfp_mask);
+ void *ret;
+
+ if (cfg->tlb->alloc_pages_exact)
+ ret = cfg->tlb->alloc_pages_exact(cookie, size, gfp_mask);
+ else
+ ret = alloc_pages_exact(size, gfp_mask);
if (likely(ret))
atomic_add(1 << get_order(size), &pages_allocated);
+
return ret;
}
-void io_pgtable_free_pages_exact(void *virt, size_t size)
+void io_pgtable_free_pages_exact(struct io_pgtable_cfg *cfg, void *cookie,
+ void *virt, size_t size)
{
- free_pages_exact(virt, size);
+ if (cfg->tlb->free_pages_exact)
+ cfg->tlb->free_pages_exact(cookie, virt, size);
+ else
+ free_pages_exact(virt, size);
+
atomic_sub(1 << get_order(size), &pages_allocated);
}
* @tlb_add_flush: Queue up a TLB invalidation for a virtual address range.
* @tlb_sync: Ensure any queue TLB invalidation has taken effect.
* @flush_pgtable: Ensure page table updates are visible to the IOMMU.
- * @prepare_pgtable: Do necessary fixup for newly allocated page table memory
- * @unprepare_pgtable: Undo fixups done during @prepare_pgtable
+ * @alloc_pages_exact: Allocate page table memory (optional, defaults to
+ * alloc_pages_exact)
+ * @free_pages_exact: Free page table memory (optional, defaults to
+ * free_pages_exact)
*
* Note that these can all be called in atomic context and must therefore
* not block.
void *cookie);
void (*tlb_sync)(void *cookie);
void (*flush_pgtable)(void *ptr, size_t size, void *cookie);
- void (*prepare_pgtable)(void *addr, void *cookie);
- void (*unprepare_pgtable)(void *cookie, void *addr);
+ void *(*alloc_pages_exact)(void *cookie, size_t size, gfp_t gfp_mask);
+ void (*free_pages_exact)(void *cookie, void *virt, size_t size);
};
/**
* Like alloc_pages_exact(), but with some additional accounting for debug
* purposes.
*/
-void *io_pgtable_alloc_pages_exact(size_t size, gfp_t gfp_mask);
+void *io_pgtable_alloc_pages_exact(struct io_pgtable_cfg *cfg, void *cookie,
+ size_t size, gfp_t gfp_mask);
/**
* io_pgtable_free_pages_exact - release memory allocated via io_pgtable_alloc_pages_exact()
* Like free_pages_exact(), but with some additional accounting for debug
* purposes.
*/
-void io_pgtable_free_pages_exact(void *virt, size_t size);
+void io_pgtable_free_pages_exact(struct io_pgtable_cfg *cfg, void *cookie,
+ void *virt, size_t size);
#endif /* __IO_PGTABLE_H */