radeonsi: add support for nir atomic_inc_wrap/atomic_dec_wrap

Reviewed-by: Marek Olšák <marek.olsak@amd.com>

diff --git a/src/amd/common/ac_nir_to_llvm.c b/src/amd/common/ac_nir_to_llvm.c
index 879d32f..42ca6a4 100644 (file)
--- a/src/amd/common/ac_nir_to_llvm.c
+++ b/src/amd/common/ac_nir_to_llvm.c
@@ -2677,6 +2677,27 @@ static LLVMValueRef visit_image_atomic(struct ac_nir_context *ctx,
                atomic_name = "cmpswap";
                atomic_subop = 0; /* not used */
                break;
+       case nir_intrinsic_bindless_image_atomic_inc_wrap:
+       case nir_intrinsic_image_deref_atomic_inc_wrap: {
+               atomic_name = "inc";
+               atomic_subop = ac_atomic_inc_wrap;
+               /* ATOMIC_INC instruction does:
+                *      value = (value + 1) % (data + 1)
+                * but we want:
+                *      value = (value + 1) % data
+                * So replace 'data' by 'data - 1'.
+                */
+               ctx->ssa_defs[instr->src[3].ssa->index] =
+                       LLVMBuildSub(ctx->ac.builder,
+                                    ctx->ssa_defs[instr->src[3].ssa->index],
+                                    ctx->ac.i32_1, "");
+               break;
+       }
+       case nir_intrinsic_bindless_image_atomic_dec_wrap:
+       case nir_intrinsic_image_deref_atomic_dec_wrap:
+               atomic_name = "dec";
+               atomic_subop = ac_atomic_dec_wrap;
+               break;
        default:
                abort();
        }
@@ -3384,6 +3405,8 @@ static void visit_intrinsic(struct ac_nir_context *ctx,
        case nir_intrinsic_bindless_image_atomic_xor:
        case nir_intrinsic_bindless_image_atomic_exchange:
        case nir_intrinsic_bindless_image_atomic_comp_swap:
+       case nir_intrinsic_bindless_image_atomic_inc_wrap:
+       case nir_intrinsic_bindless_image_atomic_dec_wrap:
                result = visit_image_atomic(ctx, instr, true);
                break;
        case nir_intrinsic_image_deref_atomic_add:
@@ -3394,6 +3417,8 @@ static void visit_intrinsic(struct ac_nir_context *ctx,
        case nir_intrinsic_image_deref_atomic_xor:
        case nir_intrinsic_image_deref_atomic_exchange:
        case nir_intrinsic_image_deref_atomic_comp_swap:
+       case nir_intrinsic_image_deref_atomic_inc_wrap:
+       case nir_intrinsic_image_deref_atomic_dec_wrap:
                result = visit_image_atomic(ctx, instr, false);
                break;
        case nir_intrinsic_bindless_image_size:

diff --git a/src/gallium/drivers/radeonsi/si_shader_nir.c b/src/gallium/drivers/radeonsi/si_shader_nir.c
index f0a9e8f..ebeaa63 100644 (file)
--- a/src/gallium/drivers/radeonsi/si_shader_nir.c
+++ b/src/gallium/drivers/radeonsi/si_shader_nir.c
@@ -306,6 +306,8 @@ static void scan_instruction(const struct nir_shader *nir,
                case nir_intrinsic_image_deref_atomic_xor:
                case nir_intrinsic_image_deref_atomic_exchange:
                case nir_intrinsic_image_deref_atomic_comp_swap:
+               case nir_intrinsic_image_deref_atomic_inc_wrap:
+               case nir_intrinsic_image_deref_atomic_dec_wrap:
                        info->writes_memory = true;
                        info->num_memory_instructions++; /* we only care about stores */
                        break;