drm/amdgpu: add AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS flag v3

Add a flag noting that a BO must be created using linear VRAM
and set this flag on all in kernel users where appropriate.

Hopefully I haven't missed anything.

v2: add it in a few more places, fix CPU mapping.
v3: rename to VRAM_CONTIGUOUS, fix typo in CS code.

Signed-off-by: Christian König <christian.koenig@amd.com>
Reviewed-by: Edward O'Callaghan <funfunctor@folklore1984.net>
Tested-by: Mike Lothian <mike@fireburn.co.uk>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
index 7a8bfa3..a61f418 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
@@ -146,7 +146,8 @@ static int amdgpu_cgs_alloc_gpu_mem(struct cgs_device *cgs_device,
        switch(type) {
        case CGS_GPU_MEM_TYPE__VISIBLE_CONTIG_FB:
        case CGS_GPU_MEM_TYPE__VISIBLE_FB:
-               flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
+               flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+                       AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
                domain = AMDGPU_GEM_DOMAIN_VRAM;
                if (max_offset > adev->mc.real_vram_size)
                        return -EINVAL;
@@ -157,7 +158,8 @@ static int amdgpu_cgs_alloc_gpu_mem(struct cgs_device *cgs_device,
                break;
        case CGS_GPU_MEM_TYPE__INVISIBLE_CONTIG_FB:
        case CGS_GPU_MEM_TYPE__INVISIBLE_FB:
-               flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
+               flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
+                       AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
                domain = AMDGPU_GEM_DOMAIN_VRAM;
                if (adev->mc.visible_vram_size < adev->mc.real_vram_size) {
                        place.fpfn =

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index b0f6e69..187c366 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -1195,6 +1195,15 @@ int amdgpu_cs_sysvm_access_required(struct amdgpu_cs_parser *parser)
                r = amdgpu_ttm_bind(&bo->tbo, &bo->tbo.mem);
                if (unlikely(r))
                        return r;
+
+               if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
+                       continue;
+
+               bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
+               amdgpu_ttm_placement_from_domain(bo, bo->allowed_domains);
+               r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
+               if (unlikely(r))
+                       return r;
        }
 
        return 0;

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 22d3319..246bbd1 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -264,7 +264,8 @@ static int amdgpu_vram_scratch_init(struct amdgpu_device *adev)
        if (adev->vram_scratch.robj == NULL) {
                r = amdgpu_bo_create(adev, AMDGPU_GPU_PAGE_SIZE,
                                     PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM,
-                                    AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+                                    AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+                                    AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
                                     NULL, NULL, &adev->vram_scratch.robj);
                if (r) {
                        return r;

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
index 8d01aa2..38bdc2d 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
@@ -152,7 +152,8 @@ static int amdgpufb_create_pinned_object(struct amdgpu_fbdev *rfbdev,
        aligned_size = ALIGN(size, PAGE_SIZE);
        ret = amdgpu_gem_object_create(adev, aligned_size, 0,
                                       AMDGPU_GEM_DOMAIN_VRAM,
-                                      AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+                                      AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+                                      AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
                                       true, &gobj);
        if (ret) {
                printk(KERN_ERR "failed to allocate framebuffer (%d)\n",

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
index 21a1242..964d2a9 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
@@ -126,7 +126,8 @@ int amdgpu_gart_table_vram_alloc(struct amdgpu_device *adev)
        if (adev->gart.robj == NULL) {
                r = amdgpu_bo_create(adev, adev->gart.table_size,
                                     PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM,
-                                    AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+                                    AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+                                    AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
                                     NULL, NULL, &adev->gart.robj);
                if (r) {
                        return r;

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index aa074fa..ba41807 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -245,7 +245,8 @@ int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
        int r;
 
        r = amdgpu_bo_create(adev, size, align, true, domain,
-                            AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+                            AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+                            AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
                             NULL, NULL, bo_ptr);
        if (r) {
                dev_err(adev->dev, "(%d) failed to allocate kernel bo\n", r);
@@ -643,6 +644,8 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
 
                return 0;
        }
+
+       bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
        amdgpu_ttm_placement_from_domain(bo, domain);
        for (i = 0; i < bo->placement.num_placement; i++) {
                /* force to pin into visible video ram */
@@ -885,7 +888,9 @@ int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
 
        size = bo->mem.num_pages << PAGE_SHIFT;
        offset = bo->mem.start << PAGE_SHIFT;
-       if ((offset + size) <= adev->mc.visible_vram_size)
+       /* TODO: figure out how to map scattered VRAM to the CPU */
+       if ((offset + size) <= adev->mc.visible_vram_size &&
+           (abo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS))
                return 0;
 
        /* Can't move a pinned BO to visible VRAM */
@@ -893,6 +898,7 @@ int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
                return -EINVAL;
 
        /* hurrah the memory is not visible ! */
+       abo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
        amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM);
        lpfn =  adev->mc.visible_vram_size >> PAGE_SHIFT;
        for (i = 0; i < abo->placement.num_placement; i++) {
@@ -954,6 +960,8 @@ u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)
        WARN_ON_ONCE(!ww_mutex_is_locked(&bo->tbo.resv->lock) &&
                     !bo->pin_count);
        WARN_ON_ONCE(bo->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET);
+       WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_VRAM &&
+                    !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS));
 
        return bo->tbo.offset;
 }

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index dcaf691..a06cf99 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -1119,7 +1119,8 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
 
        r = amdgpu_bo_create(adev, 256 * 1024, PAGE_SIZE, true,
                             AMDGPU_GEM_DOMAIN_VRAM,
-                            AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+                            AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+                            AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
                             NULL, NULL, &adev->stollen_vga_memory);
        if (r) {
                return r;

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
index e3281ca..4cf3ca7 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
@@ -1002,7 +1002,8 @@ int amdgpu_uvd_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
 
        r = amdgpu_bo_create(adev, 1024, PAGE_SIZE, true,
                             AMDGPU_GEM_DOMAIN_VRAM,
-                            AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+                            AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+                            AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
                             NULL, NULL, &bo);
        if (r)
                return r;
@@ -1051,7 +1052,8 @@ int amdgpu_uvd_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
 
        r = amdgpu_bo_create(adev, 1024, PAGE_SIZE, true,
                             AMDGPU_GEM_DOMAIN_VRAM,
-                            AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+                            AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+                            AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
                             NULL, NULL, &bo);
        if (r)
                return r;

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
index 7fe8fd8..9a534e2 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
@@ -157,7 +157,8 @@ int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size)
 
        r = amdgpu_bo_create(adev, size, PAGE_SIZE, true,
                             AMDGPU_GEM_DOMAIN_VRAM,
-                            AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+                            AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+                            AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
                             NULL, NULL, &adev->vce.vcpu_bo);
        if (r) {
                dev_err(adev->dev, "(%d) failed to allocate VCE bo\n", r);

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 06f2432..7b9e94f 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -1416,7 +1416,8 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
                                     AMDGPU_GPU_PAGE_SIZE, true,
                                     AMDGPU_GEM_DOMAIN_VRAM,
                                     AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
-                                    AMDGPU_GEM_CREATE_SHADOW,
+                                    AMDGPU_GEM_CREATE_SHADOW |
+                                    AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
                                     NULL, resv, &pt);
                if (r)
                        goto error_free;
@@ -1626,7 +1627,8 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
        r = amdgpu_bo_create(adev, pd_size, align, true,
                             AMDGPU_GEM_DOMAIN_VRAM,
                             AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
-                            AMDGPU_GEM_CREATE_SHADOW,
+                            AMDGPU_GEM_CREATE_SHADOW |
+                            AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
                             NULL, NULL, &vm->page_directory);
        if (r)
                goto error_free_sched_entity;

diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
index 71116da..b2cf129 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
@@ -3391,7 +3391,8 @@ static int gfx_v7_0_rlc_init(struct amdgpu_device *adev)
                if (adev->gfx.rlc.save_restore_obj == NULL) {
                        r = amdgpu_bo_create(adev, dws * 4, PAGE_SIZE, true,
                                             AMDGPU_GEM_DOMAIN_VRAM,
-                                            AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+                                            AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+                                            AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
                                             NULL, NULL,
                                             &adev->gfx.rlc.save_restore_obj);
                        if (r) {
@@ -3435,7 +3436,8 @@ static int gfx_v7_0_rlc_init(struct amdgpu_device *adev)
                if (adev->gfx.rlc.clear_state_obj == NULL) {
                        r = amdgpu_bo_create(adev, dws * 4, PAGE_SIZE, true,
                                             AMDGPU_GEM_DOMAIN_VRAM,
-                                            AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+                                            AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+                                            AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
                                             NULL, NULL,
                                             &adev->gfx.rlc.clear_state_obj);
                        if (r) {
@@ -3475,7 +3477,8 @@ static int gfx_v7_0_rlc_init(struct amdgpu_device *adev)
                if (adev->gfx.rlc.cp_table_obj == NULL) {
                        r = amdgpu_bo_create(adev, adev->gfx.rlc.cp_table_size, PAGE_SIZE, true,
                                             AMDGPU_GEM_DOMAIN_VRAM,
-                                            AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+                                            AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+                                            AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
                                             NULL, NULL,
                                             &adev->gfx.rlc.cp_table_obj);
                        if (r) {

diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index ee6a48a..bd4d041 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -1273,7 +1273,8 @@ static int gfx_v8_0_rlc_init(struct amdgpu_device *adev)
                if (adev->gfx.rlc.clear_state_obj == NULL) {
                        r = amdgpu_bo_create(adev, dws * 4, PAGE_SIZE, true,
                                             AMDGPU_GEM_DOMAIN_VRAM,
-                                            AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+                                            AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+                                            AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
                                             NULL, NULL,
                                             &adev->gfx.rlc.clear_state_obj);
                        if (r) {
@@ -1315,7 +1316,8 @@ static int gfx_v8_0_rlc_init(struct amdgpu_device *adev)
                if (adev->gfx.rlc.cp_table_obj == NULL) {
                        r = amdgpu_bo_create(adev, adev->gfx.rlc.cp_table_size, PAGE_SIZE, true,
                                             AMDGPU_GEM_DOMAIN_VRAM,
-                                            AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+                                            AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+                                            AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
                                             NULL, NULL,
                                             &adev->gfx.rlc.cp_table_obj);
                        if (r) {

diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h
index 0db0d66..6491e8b 100644 (file)
--- a/include/uapi/drm/amdgpu_drm.h
+++ b/include/uapi/drm/amdgpu_drm.h
@@ -81,6 +81,8 @@ extern "C" {
 #define AMDGPU_GEM_CREATE_VRAM_CLEARED         (1 << 3)
 /* Flag that create shadow bo(GTT) while allocating vram bo */
 #define AMDGPU_GEM_CREATE_SHADOW               (1 << 4)
+/* Flag that allocating the BO should use linear VRAM */
+#define AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS      (1 << 5)
 
 struct drm_amdgpu_gem_create_in  {
        /** the requested memory size */