2 * Copyright(c) 2015 Intel Corporation. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of version 2 of the GNU General Public License as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
13 #include <linux/radix-tree.h>
14 #include <linux/memremap.h>
15 #include <linux/device.h>
16 #include <linux/types.h>
17 #include <linux/pfn_t.h>
20 #include <linux/memory_hotplug.h>
23 /* temporary while we convert existing ioremap_cache users to memremap */
24 __weak void __iomem *ioremap_cache(resource_size_t offset, unsigned long size)
26 return ioremap(offset, size);
30 static void *try_ram_remap(resource_size_t offset, size_t size)
32 struct page *page = pfn_to_page(offset >> PAGE_SHIFT);
34 /* In the simple case just return the existing linear address */
35 if (!PageHighMem(page))
37 return NULL; /* fallback to ioremap_cache */
41 * memremap() - remap an iomem_resource as cacheable memory
42 * @offset: iomem resource start address
43 * @size: size of remap
44 * @flags: either MEMREMAP_WB or MEMREMAP_WT
46 * memremap() is "ioremap" for cases where it is known that the resource
47 * being mapped does not have i/o side effects and the __iomem
48 * annotation is not applicable.
50 * MEMREMAP_WB - matches the default mapping for "System RAM" on
51 * the architecture. This is usually a read-allocate write-back cache.
52 * Morever, if MEMREMAP_WB is specified and the requested remap region is RAM
53 * memremap() will bypass establishing a new mapping and instead return
54 * a pointer into the direct map.
56 * MEMREMAP_WT - establish a mapping whereby writes either bypass the
57 * cache or are written through to memory and never exist in a
58 * cache-dirty state with respect to program visibility. Attempts to
59 * map "System RAM" with this mapping type will fail.
61 void *memremap(resource_size_t offset, size_t size, unsigned long flags)
63 int is_ram = region_intersects(offset, size, "System RAM");
66 if (is_ram == REGION_MIXED) {
67 WARN_ONCE(1, "memremap attempted on mixed range %pa size: %#lx\n",
68 &offset, (unsigned long) size);
72 /* Try all mapping types requested until one returns non-NULL */
73 if (flags & MEMREMAP_WB) {
74 flags &= ~MEMREMAP_WB;
76 * MEMREMAP_WB is special in that it can be satisifed
77 * from the direct map. Some archs depend on the
78 * capability of memremap() to autodetect cases where
79 * the requested range is potentially in "System RAM"
81 if (is_ram == REGION_INTERSECTS)
82 addr = try_ram_remap(offset, size);
84 addr = ioremap_cache(offset, size);
88 * If we don't have a mapping yet and more request flags are
89 * pending then we will be attempting to establish a new virtual
90 * address mapping. Enforce that this mapping is not aliasing
93 if (!addr && is_ram == REGION_INTERSECTS && flags) {
94 WARN_ONCE(1, "memremap attempted on ram %pa size: %#lx\n",
95 &offset, (unsigned long) size);
99 if (!addr && (flags & MEMREMAP_WT)) {
100 flags &= ~MEMREMAP_WT;
101 addr = ioremap_wt(offset, size);
106 EXPORT_SYMBOL(memremap);
108 void memunmap(void *addr)
110 if (is_vmalloc_addr(addr))
111 iounmap((void __iomem *) addr);
113 EXPORT_SYMBOL(memunmap);
115 static void devm_memremap_release(struct device *dev, void *res)
120 static int devm_memremap_match(struct device *dev, void *res, void *match_data)
122 return *(void **)res == match_data;
125 void *devm_memremap(struct device *dev, resource_size_t offset,
126 size_t size, unsigned long flags)
130 ptr = devres_alloc_node(devm_memremap_release, sizeof(*ptr), GFP_KERNEL,
133 return ERR_PTR(-ENOMEM);
135 addr = memremap(offset, size, flags);
138 devres_add(dev, ptr);
144 EXPORT_SYMBOL(devm_memremap);
146 void devm_memunmap(struct device *dev, void *addr)
148 WARN_ON(devres_release(dev, devm_memremap_release,
149 devm_memremap_match, addr));
151 EXPORT_SYMBOL(devm_memunmap);
153 pfn_t phys_to_pfn_t(dma_addr_t addr, unsigned long flags)
155 return __pfn_to_pfn_t(addr >> PAGE_SHIFT, flags);
157 EXPORT_SYMBOL(phys_to_pfn_t);
159 #ifdef CONFIG_ZONE_DEVICE
160 static DEFINE_MUTEX(pgmap_lock);
161 static RADIX_TREE(pgmap_radix, GFP_KERNEL);
162 #define SECTION_MASK ~((1UL << PA_SECTION_SHIFT) - 1)
163 #define SECTION_SIZE (1UL << PA_SECTION_SHIFT)
167 struct percpu_ref *ref;
168 struct dev_pagemap pgmap;
169 struct vmem_altmap altmap;
172 static void pgmap_radix_release(struct resource *res)
176 mutex_lock(&pgmap_lock);
177 for (key = res->start; key <= res->end; key += SECTION_SIZE)
178 radix_tree_delete(&pgmap_radix, key >> PA_SECTION_SHIFT);
179 mutex_unlock(&pgmap_lock);
182 static void devm_memremap_pages_release(struct device *dev, void *data)
184 struct page_map *page_map = data;
185 struct resource *res = &page_map->res;
186 resource_size_t align_start, align_size;
187 struct dev_pagemap *pgmap = &page_map->pgmap;
189 pgmap_radix_release(res);
191 /* pages are dead and unused, undo the arch mapping */
192 align_start = res->start & ~(SECTION_SIZE - 1);
193 align_size = ALIGN(resource_size(res), SECTION_SIZE);
194 arch_remove_memory(align_start, align_size);
195 dev_WARN_ONCE(dev, pgmap->altmap && pgmap->altmap->alloc,
196 "%s: failed to free all reserved pages\n", __func__);
199 /* assumes rcu_read_lock() held at entry */
200 struct dev_pagemap *find_dev_pagemap(resource_size_t phys)
202 struct page_map *page_map;
204 WARN_ON_ONCE(!rcu_read_lock_held());
206 page_map = radix_tree_lookup(&pgmap_radix, phys >> PA_SECTION_SHIFT);
207 return page_map ? &page_map->pgmap : NULL;
211 * devm_memremap_pages - remap and provide memmap backing for the given resource
212 * @dev: hosting device for @res
213 * @res: "host memory" address range
214 * @altmap: optional descriptor for allocating the memmap from @res
216 * Note, the expectation is that @res is a host memory range that could
217 * feasibly be treated as a "System RAM" range, i.e. not a device mmio
218 * range, but this is not enforced.
220 void *devm_memremap_pages(struct device *dev, struct resource *res,
221 struct vmem_altmap *altmap)
223 int is_ram = region_intersects(res->start, resource_size(res),
225 resource_size_t key, align_start, align_size;
226 struct dev_pagemap *pgmap;
227 struct page_map *page_map;
230 if (is_ram == REGION_MIXED) {
231 WARN_ONCE(1, "%s attempted on mixed region %pr\n",
233 return ERR_PTR(-ENXIO);
236 if (is_ram == REGION_INTERSECTS)
237 return __va(res->start);
239 if (altmap && !IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP)) {
240 dev_err(dev, "%s: altmap requires CONFIG_SPARSEMEM_VMEMMAP=y\n",
242 return ERR_PTR(-ENXIO);
245 page_map = devres_alloc_node(devm_memremap_pages_release,
246 sizeof(*page_map), GFP_KERNEL, dev_to_node(dev));
248 return ERR_PTR(-ENOMEM);
249 pgmap = &page_map->pgmap;
251 memcpy(&page_map->res, res, sizeof(*res));
255 memcpy(&page_map->altmap, altmap, sizeof(*altmap));
256 pgmap->altmap = &page_map->altmap;
258 pgmap->res = &page_map->res;
260 mutex_lock(&pgmap_lock);
262 for (key = res->start; key <= res->end; key += SECTION_SIZE) {
263 struct dev_pagemap *dup;
266 dup = find_dev_pagemap(key);
269 dev_err(dev, "%s: %pr collides with mapping for %s\n",
270 __func__, res, dev_name(dup->dev));
274 error = radix_tree_insert(&pgmap_radix, key >> PA_SECTION_SHIFT,
277 dev_err(dev, "%s: failed: %d\n", __func__, error);
281 mutex_unlock(&pgmap_lock);
285 nid = dev_to_node(dev);
289 align_start = res->start & ~(SECTION_SIZE - 1);
290 align_size = ALIGN(resource_size(res), SECTION_SIZE);
291 error = arch_add_memory(nid, align_start, align_size, true);
295 devres_add(dev, page_map);
296 return __va(res->start);
300 pgmap_radix_release(res);
301 devres_free(page_map);
302 return ERR_PTR(error);
304 EXPORT_SYMBOL(devm_memremap_pages);
306 unsigned long vmem_altmap_offset(struct vmem_altmap *altmap)
308 /* number of pfns from base where pfn_to_page() is valid */
309 return altmap->reserve + altmap->free;
312 void vmem_altmap_free(struct vmem_altmap *altmap, unsigned long nr_pfns)
314 altmap->alloc -= nr_pfns;
317 #ifdef CONFIG_SPARSEMEM_VMEMMAP
318 struct vmem_altmap *to_vmem_altmap(unsigned long memmap_start)
321 * 'memmap_start' is the virtual address for the first "struct
322 * page" in this range of the vmemmap array. In the case of
323 * CONFIG_SPARSE_VMEMMAP a page_to_pfn conversion is simple
324 * pointer arithmetic, so we can perform this to_vmem_altmap()
325 * conversion without concern for the initialization state of
326 * the struct page fields.
328 struct page *page = (struct page *) memmap_start;
329 struct dev_pagemap *pgmap;
332 * Uncoditionally retrieve a dev_pagemap associated with the
333 * given physical address, this is only for use in the
334 * arch_{add|remove}_memory() for setting up and tearing down
338 pgmap = find_dev_pagemap(__pfn_to_phys(page_to_pfn(page)));
341 return pgmap ? pgmap->altmap : NULL;
343 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
344 #endif /* CONFIG_ZONE_DEVICE */