1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/device.h>
3 #include <linux/dma-buf.h>
5 #include <linux/highmem.h>
7 #include <linux/list.h>
8 #include <linux/slab.h>
9 #include <linux/uaccess.h>
10 #include <linux/vmalloc.h>
11 #include <uapi/linux/dma-heap.h>
13 #include "heap-helpers.h"
15 void init_heap_helper_buffer(struct heap_helper_buffer *buffer,
16 void (*free)(struct heap_helper_buffer *))
18 buffer->priv_virt = NULL;
19 mutex_init(&buffer->lock);
22 buffer->pagecount = 0;
24 INIT_LIST_HEAD(&buffer->attachments);
28 struct dma_buf *heap_helper_export_dmabuf(struct heap_helper_buffer *buffer,
31 DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
33 exp_info.ops = &heap_helper_ops;
34 exp_info.size = buffer->size;
35 exp_info.flags = fd_flags;
36 exp_info.priv = buffer;
38 return dma_buf_export(&exp_info);
41 static void *dma_heap_map_kernel(struct heap_helper_buffer *buffer)
45 vaddr = vmap(buffer->pages, buffer->pagecount, VM_MAP, PAGE_KERNEL);
47 return ERR_PTR(-ENOMEM);
52 static void dma_heap_buffer_destroy(struct heap_helper_buffer *buffer)
54 if (buffer->vmap_cnt > 0) {
55 WARN(1, "%s: buffer still mapped in the kernel\n", __func__);
56 vunmap(buffer->vaddr);
62 static void *dma_heap_buffer_vmap_get(struct heap_helper_buffer *buffer)
66 if (buffer->vmap_cnt) {
70 vaddr = dma_heap_map_kernel(buffer);
73 buffer->vaddr = vaddr;
78 static void dma_heap_buffer_vmap_put(struct heap_helper_buffer *buffer)
80 if (!--buffer->vmap_cnt) {
81 vunmap(buffer->vaddr);
86 struct dma_heaps_attachment {
88 struct sg_table table;
89 struct list_head list;
92 static int dma_heap_attach(struct dma_buf *dmabuf,
93 struct dma_buf_attachment *attachment)
95 struct dma_heaps_attachment *a;
96 struct heap_helper_buffer *buffer = dmabuf->priv;
99 a = kzalloc(sizeof(*a), GFP_KERNEL);
103 ret = sg_alloc_table_from_pages(&a->table, buffer->pages,
104 buffer->pagecount, 0,
105 buffer->pagecount << PAGE_SHIFT,
112 a->dev = attachment->dev;
113 INIT_LIST_HEAD(&a->list);
115 attachment->priv = a;
117 mutex_lock(&buffer->lock);
118 list_add(&a->list, &buffer->attachments);
119 mutex_unlock(&buffer->lock);
124 static void dma_heap_detach(struct dma_buf *dmabuf,
125 struct dma_buf_attachment *attachment)
127 struct dma_heaps_attachment *a = attachment->priv;
128 struct heap_helper_buffer *buffer = dmabuf->priv;
130 mutex_lock(&buffer->lock);
132 mutex_unlock(&buffer->lock);
134 sg_free_table(&a->table);
139 struct sg_table *dma_heap_map_dma_buf(struct dma_buf_attachment *attachment,
140 enum dma_data_direction direction)
142 struct dma_heaps_attachment *a = attachment->priv;
143 struct sg_table *table = &a->table;
146 ret = dma_map_sgtable(attachment->dev, table, direction, 0);
148 table = ERR_PTR(ret);
152 static void dma_heap_unmap_dma_buf(struct dma_buf_attachment *attachment,
153 struct sg_table *table,
154 enum dma_data_direction direction)
156 dma_unmap_sgtable(attachment->dev, table, direction, 0);
159 static vm_fault_t dma_heap_vm_fault(struct vm_fault *vmf)
161 struct vm_area_struct *vma = vmf->vma;
162 struct heap_helper_buffer *buffer = vma->vm_private_data;
164 if (vmf->pgoff > buffer->pagecount)
165 return VM_FAULT_SIGBUS;
167 vmf->page = buffer->pages[vmf->pgoff];
173 static const struct vm_operations_struct dma_heap_vm_ops = {
174 .fault = dma_heap_vm_fault,
177 static int dma_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
179 struct heap_helper_buffer *buffer = dmabuf->priv;
181 if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) == 0)
184 vma->vm_ops = &dma_heap_vm_ops;
185 vma->vm_private_data = buffer;
190 static void dma_heap_dma_buf_release(struct dma_buf *dmabuf)
192 struct heap_helper_buffer *buffer = dmabuf->priv;
194 dma_heap_buffer_destroy(buffer);
197 static int dma_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
198 enum dma_data_direction direction)
200 struct heap_helper_buffer *buffer = dmabuf->priv;
201 struct dma_heaps_attachment *a;
204 mutex_lock(&buffer->lock);
206 if (buffer->vmap_cnt)
207 invalidate_kernel_vmap_range(buffer->vaddr, buffer->size);
209 list_for_each_entry(a, &buffer->attachments, list) {
210 dma_sync_sg_for_cpu(a->dev, a->table.sgl, a->table.nents,
213 mutex_unlock(&buffer->lock);
218 static int dma_heap_dma_buf_end_cpu_access(struct dma_buf *dmabuf,
219 enum dma_data_direction direction)
221 struct heap_helper_buffer *buffer = dmabuf->priv;
222 struct dma_heaps_attachment *a;
224 mutex_lock(&buffer->lock);
226 if (buffer->vmap_cnt)
227 flush_kernel_vmap_range(buffer->vaddr, buffer->size);
229 list_for_each_entry(a, &buffer->attachments, list) {
230 dma_sync_sg_for_device(a->dev, a->table.sgl, a->table.nents,
233 mutex_unlock(&buffer->lock);
238 static int dma_heap_dma_buf_vmap(struct dma_buf *dmabuf, struct dma_buf_map *map)
240 struct heap_helper_buffer *buffer = dmabuf->priv;
243 mutex_lock(&buffer->lock);
244 vaddr = dma_heap_buffer_vmap_get(buffer);
245 mutex_unlock(&buffer->lock);
249 dma_buf_map_set_vaddr(map, vaddr);
254 static void dma_heap_dma_buf_vunmap(struct dma_buf *dmabuf, struct dma_buf_map *map)
256 struct heap_helper_buffer *buffer = dmabuf->priv;
258 mutex_lock(&buffer->lock);
259 dma_heap_buffer_vmap_put(buffer);
260 mutex_unlock(&buffer->lock);
263 const struct dma_buf_ops heap_helper_ops = {
264 .map_dma_buf = dma_heap_map_dma_buf,
265 .unmap_dma_buf = dma_heap_unmap_dma_buf,
266 .mmap = dma_heap_mmap,
267 .release = dma_heap_dma_buf_release,
268 .attach = dma_heap_attach,
269 .detach = dma_heap_detach,
270 .begin_cpu_access = dma_heap_dma_buf_begin_cpu_access,
271 .end_cpu_access = dma_heap_dma_buf_end_cpu_access,
272 .vmap = dma_heap_dma_buf_vmap,
273 .vunmap = dma_heap_dma_buf_vunmap,