1 /* Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License version 2 and
5 * only version 2 as published by the Free Software Foundation.
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
14 #include <linux/fdtable.h>
15 #include <linux/dma-buf.h>
16 #include "hab_grantable.h"
20 struct list_head list;
23 uint64_t index; /* for mmap first call */
30 struct file *filp_owner;
31 struct file *filp_mapper;
32 struct dma_buf *dmabuf;
35 struct physical_channel *pchan;
38 struct importer_context {
39 int cnt; /* pages allocated for local file */
40 struct list_head imp_list;
42 rwlock_t implist_lock;
45 void *habmm_hyp_allocate_grantable(int page_count,
48 if (!sizebytes || !page_count)
51 *sizebytes = page_count * sizeof(struct grantable);
52 return vmalloc(*sizebytes);
55 static int match_file(const void *p, struct file *file, unsigned int fd)
58 * We must return fd + 1 because iterate_fd stops searching on
59 * non-zero return, but 0 is a valid fd.
61 return (p == file) ? (fd + 1) : 0;
65 static int habmem_get_dma_pages_from_va(unsigned long address,
69 struct vm_area_struct *vma;
70 struct dma_buf *dmabuf = NULL;
72 unsigned long page_offset;
73 struct scatterlist *s;
74 struct sg_table *sg_table = NULL;
75 struct dma_buf_attachment *attach = NULL;
80 vma = find_vma(current->mm, address);
81 if (!vma || !vma->vm_file) {
82 pr_err("cannot find vma\n");
87 /* Look for the fd that matches this the vma file */
88 fd = iterate_fd(current->files, 0, match_file, vma->vm_file);
90 pr_err("iterate_fd failed\n");
95 offset = address - vma->vm_start;
96 page_offset = offset/PAGE_SIZE;
98 dmabuf = dma_buf_get(fd - 1);
99 if (IS_ERR_OR_NULL(dmabuf)) {
100 pr_err("dma_buf_get failed fd %d ret %pK\n", fd, dmabuf);
105 attach = dma_buf_attach(dmabuf, hab_driver.dev);
106 if (IS_ERR_OR_NULL(attach)) {
107 pr_err("dma_buf_attach failed\n");
112 sg_table = dma_buf_map_attachment(attach, DMA_TO_DEVICE);
114 if (IS_ERR_OR_NULL(sg_table)) {
115 pr_err("dma_buf_map_attachment failed\n");
120 for_each_sg(sg_table->sgl, s, sg_table->nents, i) {
123 for (j = page_offset; j < (s->length >> PAGE_SHIFT); j++) {
124 pages[rc] = nth_page(page, j);
126 if (rc >= page_count)
129 if (rc >= page_count)
132 if (page_offset > (s->length >> PAGE_SHIFT)) {
133 /* carry-over the remaining offset to next s list */
134 page_offset = page_offset-(s->length >> PAGE_SHIFT);
137 * the page_offset is within this s list
138 * there is no more offset for the next s list
146 if (!IS_ERR_OR_NULL(sg_table))
147 dma_buf_unmap_attachment(attach, sg_table, DMA_TO_DEVICE);
148 if (!IS_ERR_OR_NULL(attach))
149 dma_buf_detach(dmabuf, attach);
150 if (!IS_ERR_OR_NULL(dmabuf))
155 static int habmem_get_dma_pages_from_fd(int32_t fd,
159 struct dma_buf *dmabuf = NULL;
160 struct scatterlist *s;
161 struct sg_table *sg_table = NULL;
162 struct dma_buf_attachment *attach = NULL;
166 dmabuf = dma_buf_get(fd);
167 if (IS_ERR_OR_NULL(dmabuf)) {
168 pr_err("dma_buf_get failed fd %d ret %pK\n", fd, dmabuf);
173 attach = dma_buf_attach(dmabuf, hab_driver.dev);
174 if (IS_ERR_OR_NULL(attach)) {
175 pr_err("dma_buf_attach failed\n");
180 sg_table = dma_buf_map_attachment(attach, DMA_TO_DEVICE);
182 if (IS_ERR_OR_NULL(sg_table)) {
183 pr_err("dma_buf_map_attachment failed\n");
188 for_each_sg(sg_table->sgl, s, sg_table->nents, i) {
190 pr_debug("sgl length %d\n", s->length);
192 for (j = 0; j < (s->length >> PAGE_SHIFT); j++) {
193 pages[rc] = nth_page(page, j);
195 if (WARN_ON(rc >= page_count))
201 if (!IS_ERR_OR_NULL(sg_table))
202 dma_buf_unmap_attachment(attach, sg_table, DMA_TO_DEVICE);
203 if (!IS_ERR_OR_NULL(attach))
204 dma_buf_detach(dmabuf, attach);
205 if (!IS_ERR_OR_NULL(dmabuf))
211 * exporter - grant & revoke
212 * degenerate sharabled page list based on CPU friendly virtual "address".
213 * The result as an array is stored in ppdata to return to caller
214 * page size 4KB is assumed
216 int habmem_hyp_grant_user(unsigned long address,
222 int *compressed_size)
225 struct grantable *item = (struct grantable *)ppdata;
228 pages = vmalloc(page_count * sizeof(struct page *));
232 down_read(¤t->mm->mmap_sem);
234 if (HABMM_EXP_MEM_TYPE_DMA & flags) {
235 ret = habmem_get_dma_pages_from_va(address,
238 } else if (HABMM_EXPIMP_FLAGS_FD & flags) {
239 ret = habmem_get_dma_pages_from_fd(address,
243 ret = get_user_pages(current, current->mm,
253 for (i = 0; i < page_count; i++)
254 item[i].pfn = page_to_pfn(pages[i]);
256 pr_err("get %d user pages failed %d flags %d\n",
257 page_count, ret, flags);
261 up_read(¤t->mm->mmap_sem);
265 * exporter - grant & revoke
266 * generate shareable page list based on CPU friendly virtual "address".
267 * The result as an array is stored in ppdata to return to caller
268 * page size 4KB is assumed
270 int habmem_hyp_grant(unsigned long address,
276 int *compressed_size)
279 struct grantable *item;
280 void *kva = (void *)(uintptr_t)address;
281 int is_vmalloc = is_vmalloc_addr(kva);
283 item = (struct grantable *)ppdata;
285 for (i = 0; i < page_count; i++) {
286 kva = (void *)(uintptr_t)(address + i*PAGE_SIZE);
288 item[i].pfn = page_to_pfn(vmalloc_to_page(kva));
290 item[i].pfn = page_to_pfn(virt_to_page(kva));
296 int habmem_hyp_revoke(void *expdata, uint32_t count)
301 void *habmem_imp_hyp_open(void)
303 struct importer_context *priv;
305 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
309 rwlock_init(&priv->implist_lock);
310 INIT_LIST_HEAD(&priv->imp_list);
315 void habmem_imp_hyp_close(void *imp_ctx, int kernel)
317 struct importer_context *priv = imp_ctx;
318 struct pages_list *pglist, *pglist_tmp;
323 list_for_each_entry_safe(pglist, pglist_tmp, &priv->imp_list, list) {
324 if (kernel && pglist->kva)
327 list_del(&pglist->list);
330 kfree(pglist->pages);
337 static struct sg_table *hab_mem_map_dma_buf(
338 struct dma_buf_attachment *attachment,
339 enum dma_data_direction direction)
341 struct dma_buf *dmabuf = attachment->dmabuf;
342 struct pages_list *pglist = dmabuf->priv;
343 struct sg_table *sgt;
344 struct scatterlist *sg;
347 struct page **pages = pglist->pages;
349 sgt = kzalloc(sizeof(struct sg_table), GFP_KERNEL);
351 return ERR_PTR(-ENOMEM);
353 ret = sg_alloc_table(sgt, pglist->npages, GFP_KERNEL);
356 return ERR_PTR(-ENOMEM);
359 for_each_sg(sgt->sgl, sg, pglist->npages, i) {
360 sg_set_page(sg, pages[i], PAGE_SIZE, 0);
367 static void hab_mem_unmap_dma_buf(struct dma_buf_attachment *attachment,
368 struct sg_table *sgt,
369 enum dma_data_direction direction)
375 static int hab_map_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
378 struct pages_list *pglist;
379 unsigned long offset, fault_offset, fault_index;
383 return VM_FAULT_SIGBUS;
385 offset = vma->vm_pgoff << PAGE_SHIFT;
389 (unsigned long)vmf->virtual_address - vma->vm_start + offset;
390 fault_index = fault_offset>>PAGE_SHIFT;
392 pglist = vma->vm_private_data;
394 page_idx = fault_index - pglist->index;
395 if (page_idx < 0 || page_idx >= pglist->npages) {
396 pr_err("Out of page array! page_idx %d, pg cnt %ld",
397 page_idx, pglist->npages);
398 return VM_FAULT_SIGBUS;
401 page = pglist->pages[page_idx];
407 static void hab_map_open(struct vm_area_struct *vma)
411 static void hab_map_close(struct vm_area_struct *vma)
415 static const struct vm_operations_struct habmem_vm_ops = {
416 .fault = hab_map_fault,
417 .open = hab_map_open,
418 .close = hab_map_close,
421 static int hab_mem_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
423 struct pages_list *pglist = dmabuf->priv;
424 uint32_t obj_size = pglist->npages << PAGE_SHIFT;
427 return VM_FAULT_SIGBUS;
429 /* Check for valid size. */
430 if (obj_size < vma->vm_end - vma->vm_start)
433 vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
434 vma->vm_ops = &habmem_vm_ops;
435 vma->vm_private_data = pglist;
436 vma->vm_flags |= VM_MIXEDMAP;
441 static void hab_mem_dma_buf_release(struct dma_buf *dmabuf)
445 static void *hab_mem_dma_buf_kmap(struct dma_buf *dmabuf,
446 unsigned long offset)
451 static void hab_mem_dma_buf_kunmap(struct dma_buf *dmabuf,
452 unsigned long offset,
457 static struct dma_buf_ops dma_buf_ops = {
458 .map_dma_buf = hab_mem_map_dma_buf,
459 .unmap_dma_buf = hab_mem_unmap_dma_buf,
460 .mmap = hab_mem_mmap,
461 .release = hab_mem_dma_buf_release,
462 .kmap_atomic = hab_mem_dma_buf_kmap,
463 .kunmap_atomic = hab_mem_dma_buf_kunmap,
464 .kmap = hab_mem_dma_buf_kmap,
465 .kunmap = hab_mem_dma_buf_kunmap,
468 static int habmem_imp_hyp_map_fd(void *imp_ctx,
469 struct export_desc *exp,
474 struct compressed_pfns *pfn_table =
475 (struct compressed_pfns *)exp->payload;
476 struct pages_list *pglist;
477 struct importer_context *priv = imp_ctx;
480 pgprot_t prot = PAGE_KERNEL;
483 DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
485 if (!pfn_table || !priv)
487 size = exp->payload_count * sizeof(struct page *);
488 pages = kmalloc(size, GFP_KERNEL);
492 pglist = kzalloc(sizeof(*pglist), GFP_KERNEL);
498 pfn = pfn_table->first_pfn;
499 for (i = 0; i < pfn_table->nregions; i++) {
500 for (j = 0; j < pfn_table->region[i].size; j++) {
501 pages[k] = pfn_to_page(pfn+j);
504 pfn += pfn_table->region[i].size + pfn_table->region[i].space;
507 pglist->pages = pages;
508 pglist->npages = exp->payload_count;
511 pglist->refcntk = pglist->refcntu = 0;
512 pglist->userflags = userflags;
513 pglist->export_id = exp->export_id;
514 pglist->vcid = exp->vcid_remote;
515 pglist->pchan = exp->pchan;
517 if (!(userflags & HABMM_IMPORT_FLAGS_CACHED))
518 prot = pgprot_writecombine(prot);
520 exp_info.ops = &dma_buf_ops;
521 exp_info.size = exp->payload_count << PAGE_SHIFT;
522 exp_info.flags = O_RDWR;
523 exp_info.priv = pglist;
524 pglist->dmabuf = dma_buf_export(&exp_info);
525 if (IS_ERR(pglist->dmabuf)) {
526 ret = PTR_ERR(pglist->dmabuf);
532 fd = dma_buf_fd(pglist->dmabuf, O_CLOEXEC);
534 dma_buf_put(pglist->dmabuf);
542 write_lock(&priv->implist_lock);
543 list_add_tail(&pglist->list, &priv->imp_list);
545 write_unlock(&priv->implist_lock);
552 static int habmem_imp_hyp_map_kva(void *imp_ctx,
553 struct export_desc *exp,
558 struct compressed_pfns *pfn_table =
559 (struct compressed_pfns *)exp->payload;
560 struct pages_list *pglist;
561 struct importer_context *priv = imp_ctx;
563 int i, j, k = 0, size;
564 pgprot_t prot = PAGE_KERNEL;
566 if (!pfn_table || !priv)
568 size = exp->payload_count * sizeof(struct page *);
569 pages = kmalloc(size, GFP_KERNEL);
572 pglist = kzalloc(sizeof(*pglist), GFP_KERNEL);
578 pfn = pfn_table->first_pfn;
579 for (i = 0; i < pfn_table->nregions; i++) {
580 for (j = 0; j < pfn_table->region[i].size; j++) {
581 pages[k] = pfn_to_page(pfn+j);
584 pfn += pfn_table->region[i].size + pfn_table->region[i].space;
587 pglist->pages = pages;
588 pglist->npages = exp->payload_count;
590 pglist->refcntk = pglist->refcntu = 0;
591 pglist->userflags = userflags;
592 pglist->export_id = exp->export_id;
593 pglist->vcid = exp->vcid_remote;
594 pglist->pchan = exp->pchan;
596 if (!(userflags & HABMM_IMPORT_FLAGS_CACHED))
597 prot = pgprot_writecombine(prot);
599 pglist->kva = vmap(pglist->pages, pglist->npages, VM_MAP, prot);
600 if (pglist->kva == NULL) {
602 pr_err("%ld pages vmap failed\n", pglist->npages);
607 pr_debug("%ld pages vmap pass, return %p\n",
608 pglist->npages, pglist->kva);
612 write_lock(&priv->implist_lock);
613 list_add_tail(&pglist->list, &priv->imp_list);
615 write_unlock(&priv->implist_lock);
622 static int habmem_imp_hyp_map_uva(void *imp_ctx,
623 struct export_desc *exp,
628 struct compressed_pfns *pfn_table =
629 (struct compressed_pfns *)exp->payload;
630 struct pages_list *pglist;
631 struct importer_context *priv = imp_ctx;
633 int i, j, k = 0, size;
635 if (!pfn_table || !priv)
637 size = exp->payload_count * sizeof(struct page *);
638 pages = kmalloc(size, GFP_KERNEL);
642 pglist = kzalloc(sizeof(*pglist), GFP_KERNEL);
648 pfn = pfn_table->first_pfn;
649 for (i = 0; i < pfn_table->nregions; i++) {
650 for (j = 0; j < pfn_table->region[i].size; j++) {
651 pages[k] = pfn_to_page(pfn+j);
654 pfn += pfn_table->region[i].size + pfn_table->region[i].space;
657 pglist->pages = pages;
658 pglist->npages = exp->payload_count;
659 pglist->index = page_to_phys(pages[0]) >> PAGE_SHIFT;
660 pglist->refcntk = pglist->refcntu = 0;
661 pglist->userflags = userflags;
662 pglist->export_id = exp->export_id;
663 pglist->vcid = exp->vcid_remote;
664 pglist->pchan = exp->pchan;
666 write_lock(&priv->implist_lock);
667 list_add_tail(&pglist->list, &priv->imp_list);
669 write_unlock(&priv->implist_lock);
671 *index = pglist->index << PAGE_SHIFT;
676 int habmem_imp_hyp_map(void *imp_ctx, struct hab_import *param,
677 struct export_desc *exp, int kernel)
682 ret = habmem_imp_hyp_map_kva(imp_ctx, exp,
684 (void **)¶m->kva);
685 else if (param->flags & HABMM_EXPIMP_FLAGS_FD)
686 ret = habmem_imp_hyp_map_fd(imp_ctx, exp,
688 (int32_t *)¶m->kva);
690 ret = habmem_imp_hyp_map_uva(imp_ctx, exp,
697 int habmm_imp_hyp_unmap(void *imp_ctx, struct export_desc *exp, int kernel)
699 struct importer_context *priv = imp_ctx;
700 struct pages_list *pglist, *tmp;
703 write_lock(&priv->implist_lock);
704 list_for_each_entry_safe(pglist, tmp, &priv->imp_list, list) {
705 if (pglist->export_id == exp->export_id &&
706 pglist->pchan == exp->pchan) {
708 list_del(&pglist->list);
713 write_unlock(&priv->implist_lock);
716 pr_err("failed to find export id %u\n", exp->export_id);
720 pr_debug("detach pglist %p, kernel %d, list cnt %d\n",
721 pglist, pglist->kernel, priv->cnt);
727 dma_buf_put(pglist->dmabuf);
729 kfree(pglist->pages);
735 int habmem_imp_hyp_mmap(struct file *filp, struct vm_area_struct *vma)
737 struct uhab_context *ctx = (struct uhab_context *) filp->private_data;
738 struct importer_context *imp_ctx = ctx->import_ctx;
739 long length = vma->vm_end - vma->vm_start;
740 struct pages_list *pglist;
743 read_lock(&imp_ctx->implist_lock);
744 list_for_each_entry(pglist, &imp_ctx->imp_list, list) {
745 if (pglist->index == vma->vm_pgoff) {
750 read_unlock(&imp_ctx->implist_lock);
753 pr_err("Failed to find pglist vm_pgoff: %ld\n", vma->vm_pgoff);
757 if (length > pglist->npages * PAGE_SIZE) {
758 pr_err("Error vma length %ld not matching page list %ld\n",
759 length, pglist->npages * PAGE_SIZE);
763 vma->vm_ops = &habmem_vm_ops;
765 vma->vm_private_data = pglist;
767 if (!(pglist->userflags & HABMM_IMPORT_FLAGS_CACHED))
768 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
773 int habmm_imp_hyp_map_check(void *imp_ctx, struct export_desc *exp)
775 struct importer_context *priv = imp_ctx;
776 struct pages_list *pglist;
779 read_lock(&priv->implist_lock);
780 list_for_each_entry(pglist, &priv->imp_list, list) {
781 if (pglist->export_id == exp->export_id &&
782 pglist->pchan == exp->pchan) {
787 read_unlock(&priv->implist_lock);