radv: emit the GLC bit for SSBO loads/stores when needed

This fixes some new memory model tests:
dEQP-VK.memory_model.message_passing.core11.u32.coherent.fence_fence.atomicwrite.device.*

Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=108112
Signed-off-by: Samuel Pitoiset <samuel.pitoiset@gmail.com>
Reviewed-by: Bas Nieuwenhuizen <bas@basnieuwenhuizen.nl>

diff --git a/src/amd/common/ac_llvm_build.c b/src/amd/common/ac_llvm_build.c
index d11c12f..d0bcfe2 100644 (file)
--- a/src/amd/common/ac_llvm_build.c
+++ b/src/amd/common/ac_llvm_build.c
@@ -1179,7 +1179,8 @@ ac_build_tbuffer_load_short(struct ac_llvm_context *ctx,
                            LLVMValueRef vindex,
                            LLVMValueRef voffset,
                                LLVMValueRef soffset,
-                               LLVMValueRef immoffset)
+                               LLVMValueRef immoffset,
+                               LLVMValueRef glc)
 {
        const char *name = "llvm.amdgcn.tbuffer.load.i32";
        LLVMTypeRef type = ctx->i32;
@@ -1191,7 +1192,7 @@ ac_build_tbuffer_load_short(struct ac_llvm_context *ctx,
                                immoffset,
                                LLVMConstInt(ctx->i32, V_008F0C_BUF_DATA_FORMAT_16, false),
                                LLVMConstInt(ctx->i32, V_008F0C_BUF_NUM_FORMAT_UINT, false),
-                               ctx->i1false,
+                               glc,
                                ctx->i1false,
        };
        LLVMValueRef res = ac_build_intrinsic(ctx, name, type, params, 9, 0);

diff --git a/src/amd/common/ac_llvm_build.h b/src/amd/common/ac_llvm_build.h
index 65235fb..5c3e221 100644 (file)
--- a/src/amd/common/ac_llvm_build.h
+++ b/src/amd/common/ac_llvm_build.h
@@ -274,7 +274,8 @@ ac_build_tbuffer_load_short(struct ac_llvm_context *ctx,
                            LLVMValueRef vindex,
                            LLVMValueRef voffset,
                                LLVMValueRef soffset,
-                               LLVMValueRef immoffset);
+                               LLVMValueRef immoffset,
+                               LLVMValueRef glc);
 
 LLVMValueRef
 ac_get_thread_id(struct ac_llvm_context *ctx);

diff --git a/src/amd/common/ac_nir_to_llvm.c b/src/amd/common/ac_nir_to_llvm.c
index ffc64a7..e0a8e04 100644 (file)
--- a/src/amd/common/ac_nir_to_llvm.c
+++ b/src/amd/common/ac_nir_to_llvm.c
@@ -1456,6 +1456,11 @@ static void visit_store_ssbo(struct ac_nir_context *ctx,
        LLVMValueRef src_data = get_src(ctx, instr->src[0]);
        int elem_size_bytes = ac_get_elem_bits(&ctx->ac, LLVMTypeOf(src_data)) / 8;
        unsigned writemask = nir_intrinsic_write_mask(instr);
+       enum gl_access_qualifier access = nir_intrinsic_access(instr);
+       LLVMValueRef glc = ctx->ac.i1false;
+
+       if (access & (ACCESS_VOLATILE | ACCESS_COHERENT))
+               glc = ctx->ac.i1true;
 
        LLVMValueRef rsrc = ctx->abi->load_ssbo(ctx->abi,
                                        get_src(ctx, instr->src[1]), true);
@@ -1512,7 +1517,7 @@ static void visit_store_ssbo(struct ac_nir_context *ctx,
                                ctx->ac.i32_0,
                                LLVMConstInt(ctx->ac.i32, 2, false), // dfmt (= 16bit)
                                LLVMConstInt(ctx->ac.i32, 4, false), // nfmt (= uint)
-                               ctx->ac.i1false,
+                               glc,
                                ctx->ac.i1false,
                        };
                        ac_build_intrinsic(&ctx->ac, store_name,
@@ -1540,7 +1545,7 @@ static void visit_store_ssbo(struct ac_nir_context *ctx,
                                rsrc,
                                ctx->ac.i32_0, /* vindex */
                                offset,
-                               ctx->ac.i1false,  /* glc */
+                               glc,
                                ctx->ac.i1false,  /* slc */
                        };
                        ac_build_intrinsic(&ctx->ac, store_name,
@@ -1613,6 +1618,11 @@ static LLVMValueRef visit_load_buffer(struct ac_nir_context *ctx,
        int elem_size_bytes = instr->dest.ssa.bit_size / 8;
        int num_components = instr->num_components;
        int num_bytes = num_components * elem_size_bytes;
+       enum gl_access_qualifier access = nir_intrinsic_access(instr);
+       LLVMValueRef glc = ctx->ac.i1false;
+
+       if (access & (ACCESS_VOLATILE | ACCESS_COHERENT))
+               glc = ctx->ac.i1true;
 
        for (int i = 0; i < num_bytes; i += load_bytes) {
                load_bytes = MIN2(num_bytes - i, 16);
@@ -1631,7 +1641,8 @@ static LLVMValueRef visit_load_buffer(struct ac_nir_context *ctx,
                                                                   vindex,
                                                                   offset,
                                                                   ctx->ac.i32_0,
-                                                                  immoffset);
+                                                                  immoffset,
+                                                                  glc);
                } else {
                        switch (load_bytes) {
                        case 16:
@@ -1655,7 +1666,7 @@ static LLVMValueRef visit_load_buffer(struct ac_nir_context *ctx,
                                rsrc,
                                vindex,
                                LLVMBuildAdd(ctx->ac.builder, offset, immoffset, ""),
-                               ctx->ac.i1false,
+                               glc,
                                ctx->ac.i1false,
                        };
                        results[idx] = ac_build_intrinsic(&ctx->ac, load_name, data_type, params, 5, 0);
@@ -1709,7 +1720,8 @@ static LLVMValueRef visit_load_ubo_buffer(struct ac_nir_context *ctx,
                                                                 ctx->ac.i32_0,
                                                                 offset,
                                                                 ctx->ac.i32_0,
-                                                                LLVMConstInt(ctx->ac.i32, 2 * i, 0));
+                                                                LLVMConstInt(ctx->ac.i32, 2 * i, 0),
+                                                                ctx->ac.i1false);
                }
                ret = ac_build_gather_values(&ctx->ac, results, num_components);
        } else {