ac/llvm: Implement AMD-specific buffer load/store intrinsics.

Signed-off-by: Timur Kristóf <timur.kristof@gmail.com>
Reviewed-by: Marek Olšák <marek.olsak@amd.com>
Reviewed-by: Rhys Perry <pendingchaos02@gmail.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/9201>

diff --git a/src/amd/llvm/ac_nir_to_llvm.c b/src/amd/llvm/ac_nir_to_llvm.c
index fffe766..7cbe8c9 100644 (file)
--- a/src/amd/llvm/ac_nir_to_llvm.c
+++ b/src/amd/llvm/ac_nir_to_llvm.c
@@ -3965,6 +3965,65 @@ static void visit_intrinsic(struct ac_nir_context *ctx, nir_intrinsic_instr *ins
    case nir_intrinsic_set_vertex_and_primitive_count:
       /* Currently ignored. */
       break;
+   case nir_intrinsic_load_buffer_amd: {
+      LLVMValueRef descriptor = get_src(ctx, instr->src[0]);
+      LLVMValueRef addr_voffset = get_src(ctx, instr->src[1]);
+      LLVMValueRef addr_soffset = get_src(ctx, instr->src[2]);
+      unsigned num_components = instr->dest.ssa.num_components;
+      unsigned const_offset = nir_intrinsic_base(instr);
+      bool swizzled = nir_intrinsic_is_swizzled(instr);
+      bool reorder = nir_intrinsic_can_reorder(instr);
+      bool slc = nir_intrinsic_slc_amd(instr);
+
+      enum ac_image_cache_policy cache_policy = ac_glc;
+      if (swizzled)
+         cache_policy |= ac_swizzled;
+      if (slc)
+         cache_policy |= ac_slc;
+      if (ctx->ac.chip_class >= GFX10)
+         cache_policy |= ac_dlc;
+
+      LLVMTypeRef channel_type;
+      if (instr->dest.ssa.bit_size == 8)
+         channel_type = ctx->ac.i8;
+      else if (instr->dest.ssa.bit_size == 16)
+         channel_type = ctx->ac.i16;
+      else if (instr->dest.ssa.bit_size == 32)
+         channel_type = ctx->ac.i32;
+      else if (instr->dest.ssa.bit_size == 64)
+         channel_type = ctx->ac.i64;
+      else if (instr->dest.ssa.bit_size == 128)
+         channel_type = ctx->ac.i128;
+      else
+         unreachable("Unsupported channel type for load_buffer_amd");
+
+      result = ac_build_buffer_load(&ctx->ac, descriptor, num_components, NULL,
+                                    addr_voffset, addr_soffset, const_offset,
+                                    channel_type, cache_policy, reorder, false);
+      result = ac_to_integer(&ctx->ac, ac_trim_vector(&ctx->ac, result, num_components));
+      break;
+   }
+   case nir_intrinsic_store_buffer_amd: {
+      LLVMValueRef store_data = get_src(ctx, instr->src[0]);
+      LLVMValueRef descriptor = get_src(ctx, instr->src[1]);
+      LLVMValueRef addr_voffset = get_src(ctx, instr->src[2]);
+      LLVMValueRef addr_soffset = get_src(ctx, instr->src[3]);
+      unsigned num_components = instr->src[0].ssa->num_components;
+      unsigned const_offset = nir_intrinsic_base(instr);
+      bool swizzled = nir_intrinsic_is_swizzled(instr);
+      bool slc = nir_intrinsic_slc_amd(instr);
+
+      enum ac_image_cache_policy cache_policy = ac_glc;
+      if (swizzled)
+         cache_policy |= ac_swizzled;
+      if (slc)
+         cache_policy |= ac_slc;
+
+      ac_build_buffer_store_dword(&ctx->ac, descriptor, store_data, num_components,
+                                  addr_voffset, addr_soffset, const_offset,
+                                  cache_policy);
+      break;
+   }
    default:
       fprintf(stderr, "Unknown intrinsic: ");
       nir_print_instr(&instr->instr, stderr);