From 52c8720e5cf479aae5dc8d0ac7d2c4afd261e446 Mon Sep 17 00:00:00 2001 From: Mitchel Humpherys Date: Thu, 30 Jul 2015 13:41:12 -0700 Subject: [PATCH] iommu/io-pgtable: Delegate page table memory management to drivers Some IOMMU drivers (like arm-smmu) need to perform fixups to page table memory when it's allocated and freed. Some callbacks were added for this purpose ({prepare,unprepare}_pgtable) that are called from the io-pgtable code after each allocation and before each free. However, this approach is prone to bugs where new calls to allocate/free are added without corresponding calls to the fixup callbacks. Furthermore, allocation and free is often done from atomic context, so if the driver needs to do non-atomic fixups during free they are out of luck since the memory will be freed back to the system by the time control is turned back over to the driver. Adding yet another callback for non-atomic fixups would start to get tedious, and would only increase the chance of missing callbacks as new allocations/frees are added. Instead, fix this by allowing all page table memory allocation and free to be delegated entirely to the driver. Fall back to {alloc,free}_pages_exact by default if the driver doesn't need any special handling of page table memory. Change-Id: I0361bb81e25ff5ad4ef93a45330a35af47bc6013 Signed-off-by: Mitchel Humpherys --- drivers/iommu/arm-smmu.c | 21 +++++++++++++++++++-- drivers/iommu/io-pgtable-arm.c | 17 ++++++++--------- drivers/iommu/io-pgtable.c | 20 ++++++++++++++++---- drivers/iommu/io-pgtable.h | 16 ++++++++++------ 4 files changed, 53 insertions(+), 21 deletions(-) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 38e95fea3f28..ae9bfd67d6e2 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -950,13 +950,30 @@ static void arm_smmu_flush_pgtable(void *addr, size_t size, void *cookie) static void arm_smmu_prepare_pgtable(void *addr, void *cookie); static void arm_smmu_unprepare_pgtable(void *cookie, void *addr); +static void *arm_smmu_alloc_pages_exact(void *cookie, + size_t size, gfp_t gfp_mask) +{ + void *ret = alloc_pages_exact(size, gfp_mask); + + if (likely(ret)) + arm_smmu_prepare_pgtable(ret, cookie); + + return ret; +} + +static void arm_smmu_free_pages_exact(void *cookie, void *virt, size_t size) +{ + arm_smmu_unprepare_pgtable(cookie, virt); + free_pages_exact(virt, size); +} + static struct iommu_gather_ops arm_smmu_gather_ops = { .tlb_flush_all = arm_smmu_tlb_inv_context, .tlb_add_flush = arm_smmu_tlb_inv_range_nosync, .tlb_sync = arm_smmu_tlb_sync, .flush_pgtable = arm_smmu_flush_pgtable, - .prepare_pgtable = arm_smmu_prepare_pgtable, - .unprepare_pgtable = arm_smmu_unprepare_pgtable, + .alloc_pages_exact = arm_smmu_alloc_pages_exact, + .free_pages_exact = arm_smmu_free_pages_exact, }; static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 1777fd9e6b39..b78d8caad492 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -325,7 +325,8 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, /* Grab a pointer to the next level */ pte = *ptep; if (!pte) { - cptep = io_pgtable_alloc_pages_exact(1UL << data->pg_shift, + cptep = io_pgtable_alloc_pages_exact(&data->iop.cfg, cookie, + 1UL << data->pg_shift, GFP_ATOMIC | __GFP_ZERO); if (!cptep) return -ENOMEM; @@ -337,7 +338,6 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, pte |= ARM_LPAE_PTE_NSTABLE; *ptep = pte; data->iop.cfg.tlb->flush_pgtable(ptep, sizeof(*ptep), cookie); - data->iop.cfg.tlb->prepare_pgtable(cptep, cookie); } else { cptep = iopte_deref(pte, data); } @@ -490,8 +490,8 @@ static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl, __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data)); } - data->iop.cfg.tlb->unprepare_pgtable(data->iop.cookie, start); - io_pgtable_free_pages_exact(start, table_size); + io_pgtable_free_pages_exact(&data->iop.cfg, data->iop.cookie, + start, table_size); } static void arm_lpae_free_pgtable(struct io_pgtable *iop) @@ -600,7 +600,8 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, *ptep = 0; tlb->flush_pgtable(ptep, sizeof(*ptep), cookie); io_pgtable_free_pages_exact( - table_base, max_entries * sizeof(*table_base)); + &data->iop.cfg, cookie, table_base, + max_entries * sizeof(*table_base)); } return entries * entry_size; @@ -825,13 +826,12 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) cfg->arm_lpae_s1_cfg.mair[1] = 0; /* Looking good; allocate a pgd */ - data->pgd = io_pgtable_alloc_pages_exact(data->pgd_size, + data->pgd = io_pgtable_alloc_pages_exact(cfg, cookie, data->pgd_size, GFP_KERNEL | __GFP_ZERO); if (!data->pgd) goto out_free_data; cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie); - cfg->tlb->prepare_pgtable(data->pgd, cookie); /* TTBRs */ cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd); cfg->arm_lpae_s1_cfg.ttbr[1] = 0; @@ -914,13 +914,12 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) cfg->arm_lpae_s2_cfg.vtcr = reg; /* Allocate pgd pages */ - data->pgd = io_pgtable_alloc_pages_exact(data->pgd_size, + data->pgd = io_pgtable_alloc_pages_exact(cfg, cookie, data->pgd_size, GFP_KERNEL | __GFP_ZERO); if (!data->pgd) goto out_free_data; cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie); - cfg->tlb->prepare_pgtable(data->pgd, cookie); /* VTTBR */ cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd); return &data->iop; diff --git a/drivers/iommu/io-pgtable.c b/drivers/iommu/io-pgtable.c index 881c8a0b663f..3a74d4440018 100644 --- a/drivers/iommu/io-pgtable.c +++ b/drivers/iommu/io-pgtable.c @@ -90,18 +90,30 @@ void free_io_pgtable_ops(struct io_pgtable_ops *ops) static atomic_t pages_allocated; -void *io_pgtable_alloc_pages_exact(size_t size, gfp_t gfp_mask) +void *io_pgtable_alloc_pages_exact(struct io_pgtable_cfg *cfg, void *cookie, + size_t size, gfp_t gfp_mask) { - void *ret = alloc_pages_exact(size, gfp_mask); + void *ret; + + if (cfg->tlb->alloc_pages_exact) + ret = cfg->tlb->alloc_pages_exact(cookie, size, gfp_mask); + else + ret = alloc_pages_exact(size, gfp_mask); if (likely(ret)) atomic_add(1 << get_order(size), &pages_allocated); + return ret; } -void io_pgtable_free_pages_exact(void *virt, size_t size) +void io_pgtable_free_pages_exact(struct io_pgtable_cfg *cfg, void *cookie, + void *virt, size_t size) { - free_pages_exact(virt, size); + if (cfg->tlb->free_pages_exact) + cfg->tlb->free_pages_exact(cookie, virt, size); + else + free_pages_exact(virt, size); + atomic_sub(1 << get_order(size), &pages_allocated); } diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h index 2ca3393988fd..c1c656bd7a8e 100644 --- a/drivers/iommu/io-pgtable.h +++ b/drivers/iommu/io-pgtable.h @@ -21,8 +21,10 @@ enum io_pgtable_fmt { * @tlb_add_flush: Queue up a TLB invalidation for a virtual address range. * @tlb_sync: Ensure any queue TLB invalidation has taken effect. * @flush_pgtable: Ensure page table updates are visible to the IOMMU. - * @prepare_pgtable: Do necessary fixup for newly allocated page table memory - * @unprepare_pgtable: Undo fixups done during @prepare_pgtable + * @alloc_pages_exact: Allocate page table memory (optional, defaults to + * alloc_pages_exact) + * @free_pages_exact: Free page table memory (optional, defaults to + * free_pages_exact) * * Note that these can all be called in atomic context and must therefore * not block. @@ -33,8 +35,8 @@ struct iommu_gather_ops { void *cookie); void (*tlb_sync)(void *cookie); void (*flush_pgtable)(void *ptr, size_t size, void *cookie); - void (*prepare_pgtable)(void *addr, void *cookie); - void (*unprepare_pgtable)(void *cookie, void *addr); + void *(*alloc_pages_exact)(void *cookie, size_t size, gfp_t gfp_mask); + void (*free_pages_exact)(void *cookie, void *virt, size_t size); }; /** @@ -156,7 +158,8 @@ struct io_pgtable_init_fns { * Like alloc_pages_exact(), but with some additional accounting for debug * purposes. */ -void *io_pgtable_alloc_pages_exact(size_t size, gfp_t gfp_mask); +void *io_pgtable_alloc_pages_exact(struct io_pgtable_cfg *cfg, void *cookie, + size_t size, gfp_t gfp_mask); /** * io_pgtable_free_pages_exact - release memory allocated via io_pgtable_alloc_pages_exact() @@ -166,6 +169,7 @@ void *io_pgtable_alloc_pages_exact(size_t size, gfp_t gfp_mask); * Like free_pages_exact(), but with some additional accounting for debug * purposes. */ -void io_pgtable_free_pages_exact(void *virt, size_t size); +void io_pgtable_free_pages_exact(struct io_pgtable_cfg *cfg, void *cookie, + void *virt, size_t size); #endif /* __IO_PGTABLE_H */ -- 2.11.0