nir: Handle unified atomics in simple cases

Lots of passes can be made unified-atomics-aware simply by adding extra cases in
their switch statements. This commit fixes a bunch of passes.

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Reviewed-by: Jesse Natalie <jenatali@microsoft.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/22914>

diff --git a/src/compiler/nir/nir_gather_info.c b/src/compiler/nir/nir_gather_info.c
index d3374ac..738b64d 100644 (file)
--- a/src/compiler/nir/nir_gather_info.c
+++ b/src/compiler/nir/nir_gather_info.c
@@ -340,6 +340,8 @@ nir_intrinsic_writes_external_memory(const nir_intrinsic_instr *instr)
    case nir_intrinsic_atomic_counter_exchange_deref:
    case nir_intrinsic_atomic_counter_comp_swap:
    case nir_intrinsic_atomic_counter_comp_swap_deref:
+   case nir_intrinsic_bindless_image_atomic:
+   case nir_intrinsic_bindless_image_atomic_swap:
    case nir_intrinsic_bindless_image_atomic_add:
    case nir_intrinsic_bindless_image_atomic_and:
    case nir_intrinsic_bindless_image_atomic_comp_swap:
@@ -355,6 +357,8 @@ nir_intrinsic_writes_external_memory(const nir_intrinsic_instr *instr)
    case nir_intrinsic_bindless_image_atomic_xor:
    case nir_intrinsic_bindless_image_store:
    case nir_intrinsic_bindless_image_store_raw_intel:
+   case nir_intrinsic_global_atomic:
+   case nir_intrinsic_global_atomic_swap:
    case nir_intrinsic_global_atomic_add:
    case nir_intrinsic_global_atomic_and:
    case nir_intrinsic_global_atomic_comp_swap:
@@ -371,6 +375,8 @@ nir_intrinsic_writes_external_memory(const nir_intrinsic_instr *instr)
    case nir_intrinsic_global_atomic_xor:
    case nir_intrinsic_global_atomic_ir3:
    case nir_intrinsic_global_atomic_swap_ir3:
+   case nir_intrinsic_image_atomic:
+   case nir_intrinsic_image_atomic_swap:
    case nir_intrinsic_image_atomic_add:
    case nir_intrinsic_image_atomic_and:
    case nir_intrinsic_image_atomic_comp_swap:
@@ -384,6 +390,8 @@ nir_intrinsic_writes_external_memory(const nir_intrinsic_instr *instr)
    case nir_intrinsic_image_atomic_umax:
    case nir_intrinsic_image_atomic_umin:
    case nir_intrinsic_image_atomic_xor:
+   case nir_intrinsic_image_deref_atomic:
+   case nir_intrinsic_image_deref_atomic_swap:
    case nir_intrinsic_image_deref_atomic_add:
    case nir_intrinsic_image_deref_atomic_and:
    case nir_intrinsic_image_deref_atomic_comp_swap:
@@ -427,6 +435,8 @@ nir_intrinsic_writes_external_memory(const nir_intrinsic_instr *instr)
       return true;
 
    case nir_intrinsic_store_deref:
+   case nir_intrinsic_deref_atomic:
+   case nir_intrinsic_deref_atomic_swap:
    case nir_intrinsic_deref_atomic_add:
    case nir_intrinsic_deref_atomic_imin:
    case nir_intrinsic_deref_atomic_umin:
@@ -453,6 +463,8 @@ static bool
 intrinsic_is_bindless(nir_intrinsic_instr *instr)
 {
    switch (instr->intrinsic) {
+   case nir_intrinsic_bindless_image_atomic:
+   case nir_intrinsic_bindless_image_atomic_swap:
    case nir_intrinsic_bindless_image_atomic_add:
    case nir_intrinsic_bindless_image_atomic_and:
    case nir_intrinsic_bindless_image_atomic_comp_swap:

diff --git a/src/compiler/nir/nir_lower_amul.c b/src/compiler/nir/nir_lower_amul.c
index cbea520..dfb8a0b 100644 (file)
--- a/src/compiler/nir/nir_lower_amul.c
+++ b/src/compiler/nir/nir_lower_amul.c
@@ -142,6 +142,8 @@ lower_intrinsic(lower_state *state, nir_intrinsic_instr *intr)
          lower_large_src(&intr->src[2], state);
       return;
 
+   case nir_intrinsic_ssbo_atomic:
+   case nir_intrinsic_ssbo_atomic_swap:
    case nir_intrinsic_ssbo_atomic_add:
    case nir_intrinsic_ssbo_atomic_imin:
    case nir_intrinsic_ssbo_atomic_umin:
@@ -163,6 +165,8 @@ lower_intrinsic(lower_state *state, nir_intrinsic_instr *intr)
          lower_large_src(&intr->src[1], state);
       return;
 
+   case nir_intrinsic_global_atomic:
+   case nir_intrinsic_global_atomic_swap:
    case nir_intrinsic_global_atomic_add:
    case nir_intrinsic_global_atomic_imin:
    case nir_intrinsic_global_atomic_umin:
@@ -189,6 +193,8 @@ lower_intrinsic(lower_state *state, nir_intrinsic_instr *intr)
       return;
 
    /* These should all be small enough to unconditionally use imul24: */
+   case nir_intrinsic_shared_atomic:
+   case nir_intrinsic_shared_atomic_swap:
    case nir_intrinsic_shared_atomic_add:
    case nir_intrinsic_shared_atomic_imin:
    case nir_intrinsic_shared_atomic_umin:

diff --git a/src/compiler/nir/nir_lower_cl_images.c b/src/compiler/nir/nir_lower_cl_images.c
index 738a9c9..2751947 100644 (file)
--- a/src/compiler/nir/nir_lower_cl_images.c
+++ b/src/compiler/nir/nir_lower_cl_images.c
@@ -248,6 +248,8 @@ nir_lower_cl_images(nir_shader *shader, bool lower_image_derefs, bool lower_samp
             switch (intrin->intrinsic) {
             case nir_intrinsic_image_deref_load:
             case nir_intrinsic_image_deref_store:
+            case nir_intrinsic_image_deref_atomic:
+            case nir_intrinsic_image_deref_atomic_swap:
             case nir_intrinsic_image_deref_atomic_add:
             case nir_intrinsic_image_deref_atomic_imin:
             case nir_intrinsic_image_deref_atomic_umin:

diff --git a/src/compiler/nir/nir_lower_helper_writes.c b/src/compiler/nir/nir_lower_helper_writes.c
index d961771..cb82856 100644 (file)
--- a/src/compiler/nir/nir_lower_helper_writes.c
+++ b/src/compiler/nir/nir_lower_helper_writes.c
@@ -35,6 +35,8 @@ lower(nir_builder *b, nir_instr *instr, void *data)
    bool *lower_plain_stores = data;
 
    switch (intr->intrinsic) {
+   case nir_intrinsic_global_atomic:
+   case nir_intrinsic_global_atomic_swap:
    case nir_intrinsic_global_atomic_add:
    case nir_intrinsic_global_atomic_and:
    case nir_intrinsic_global_atomic_comp_swap:
@@ -49,6 +51,8 @@ lower(nir_builder *b, nir_instr *instr, void *data)
    case nir_intrinsic_global_atomic_umax:
    case nir_intrinsic_global_atomic_umin:
    case nir_intrinsic_global_atomic_xor:
+   case nir_intrinsic_image_atomic:
+   case nir_intrinsic_image_atomic_swap:
    case nir_intrinsic_image_atomic_add:
    case nir_intrinsic_image_atomic_and:
    case nir_intrinsic_image_atomic_comp_swap:

diff --git a/src/compiler/nir/nir_lower_mediump.c b/src/compiler/nir/nir_lower_mediump.c
index 8698b0d..6f61a3e 100644 (file)
--- a/src/compiler/nir/nir_lower_mediump.c
+++ b/src/compiler/nir/nir_lower_mediump.c
@@ -568,6 +568,8 @@ nir_lower_mediump_vars(nir_shader *shader, nir_variable_mode modes)
                continue;
             nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
             switch (intr->intrinsic) {
+            case nir_intrinsic_deref_atomic:
+            case nir_intrinsic_deref_atomic_swap:
             case nir_intrinsic_deref_atomic_add:
             case nir_intrinsic_deref_atomic_imin:
             case nir_intrinsic_deref_atomic_umin:
@@ -1047,6 +1049,8 @@ fold_16bit_tex_image(nir_builder *b, nir_instr *instr, void *params)
          if (options->fold_image_srcs)
             progress |= fold_16bit_image_srcs(b, intrinsic, 3);
          break;
+      case nir_intrinsic_bindless_image_atomic:
+      case nir_intrinsic_bindless_image_atomic_swap:
       case nir_intrinsic_bindless_image_atomic_add:
       case nir_intrinsic_bindless_image_atomic_imin:
       case nir_intrinsic_bindless_image_atomic_umin:
@@ -1062,6 +1066,8 @@ fold_16bit_tex_image(nir_builder *b, nir_instr *instr, void *params)
       case nir_intrinsic_bindless_image_atomic_fmax:
       case nir_intrinsic_bindless_image_atomic_inc_wrap:
       case nir_intrinsic_bindless_image_atomic_dec_wrap:
+      case nir_intrinsic_image_deref_atomic:
+      case nir_intrinsic_image_deref_atomic_swap:
       case nir_intrinsic_image_deref_atomic_add:
       case nir_intrinsic_image_deref_atomic_umin:
       case nir_intrinsic_image_deref_atomic_imin:
@@ -1077,6 +1083,8 @@ fold_16bit_tex_image(nir_builder *b, nir_instr *instr, void *params)
       case nir_intrinsic_image_deref_atomic_fmax:
       case nir_intrinsic_image_deref_atomic_inc_wrap:
       case nir_intrinsic_image_deref_atomic_dec_wrap:
+      case nir_intrinsic_image_atomic:
+      case nir_intrinsic_image_atomic_swap:
       case nir_intrinsic_image_atomic_add:
       case nir_intrinsic_image_atomic_imin:
       case nir_intrinsic_image_atomic_umin:

diff --git a/src/compiler/nir/nir_lower_memory_model.c b/src/compiler/nir/nir_lower_memory_model.c
index b466cee..a8c20e1 100644 (file)
--- a/src/compiler/nir/nir_lower_memory_model.c
+++ b/src/compiler/nir/nir_lower_memory_model.c
@@ -44,6 +44,8 @@ get_intrinsic_info(nir_intrinsic_instr *intrin, nir_variable_mode *modes,
       *modes = nir_src_as_deref(intrin->src[0])->modes;
       *writes = true;
       break;
+   case nir_intrinsic_image_deref_atomic:
+   case nir_intrinsic_image_deref_atomic_swap:
    case nir_intrinsic_image_deref_atomic_add:
    case nir_intrinsic_image_deref_atomic_fadd:
    case nir_intrinsic_image_deref_atomic_umin:
@@ -69,6 +71,8 @@ get_intrinsic_info(nir_intrinsic_instr *intrin, nir_variable_mode *modes,
       *modes = nir_var_mem_ssbo;
       *writes = true;
       break;
+   case nir_intrinsic_ssbo_atomic:
+   case nir_intrinsic_ssbo_atomic_swap:
    case nir_intrinsic_ssbo_atomic_add:
    case nir_intrinsic_ssbo_atomic_imin:
    case nir_intrinsic_ssbo_atomic_umin:
@@ -95,6 +99,8 @@ get_intrinsic_info(nir_intrinsic_instr *intrin, nir_variable_mode *modes,
       *modes = nir_var_mem_global;
       *writes = true;
       break;
+   case nir_intrinsic_global_atomic:
+   case nir_intrinsic_global_atomic_swap:
    case nir_intrinsic_global_atomic_add:
    case nir_intrinsic_global_atomic_imin:
    case nir_intrinsic_global_atomic_umin:
@@ -121,6 +127,8 @@ get_intrinsic_info(nir_intrinsic_instr *intrin, nir_variable_mode *modes,
       *modes = nir_src_as_deref(intrin->src[0])->modes;
       *writes = true;
       break;
+   case nir_intrinsic_deref_atomic:
+   case nir_intrinsic_deref_atomic_swap:
    case nir_intrinsic_deref_atomic_add:
    case nir_intrinsic_deref_atomic_imin:
    case nir_intrinsic_deref_atomic_umin:

diff --git a/src/compiler/nir/nir_lower_multiview.c b/src/compiler/nir/nir_lower_multiview.c
index 82fea39..3cf02e4 100644 (file)
--- a/src/compiler/nir/nir_lower_multiview.c
+++ b/src/compiler/nir/nir_lower_multiview.c
@@ -55,6 +55,8 @@ shader_writes_to_memory(nir_shader *shader)
          nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
 
          switch (intrin->intrinsic) {
+         case nir_intrinsic_deref_atomic:
+         case nir_intrinsic_deref_atomic_swap:
          case nir_intrinsic_deref_atomic_add:
          case nir_intrinsic_deref_atomic_imin:
          case nir_intrinsic_deref_atomic_umin:
@@ -66,6 +68,8 @@ shader_writes_to_memory(nir_shader *shader)
          case nir_intrinsic_deref_atomic_exchange:
          case nir_intrinsic_deref_atomic_comp_swap:
          case nir_intrinsic_store_ssbo:
+         case nir_intrinsic_ssbo_atomic:
+         case nir_intrinsic_ssbo_atomic_swap:
          case nir_intrinsic_ssbo_atomic_add:
          case nir_intrinsic_ssbo_atomic_imin:
          case nir_intrinsic_ssbo_atomic_umin:
@@ -78,6 +82,8 @@ shader_writes_to_memory(nir_shader *shader)
          case nir_intrinsic_ssbo_atomic_comp_swap:
          case nir_intrinsic_store_shared:
          case nir_intrinsic_store_shared2_amd:
+         case nir_intrinsic_shared_atomic:
+         case nir_intrinsic_shared_atomic_swap:
          case nir_intrinsic_shared_atomic_add:
          case nir_intrinsic_shared_atomic_imin:
          case nir_intrinsic_shared_atomic_umin:
@@ -88,6 +94,8 @@ shader_writes_to_memory(nir_shader *shader)
          case nir_intrinsic_shared_atomic_xor:
          case nir_intrinsic_shared_atomic_exchange:
          case nir_intrinsic_shared_atomic_comp_swap:
+         case nir_intrinsic_task_payload_atomic:
+         case nir_intrinsic_task_payload_atomic_swap:
          case nir_intrinsic_task_payload_atomic_add:
          case nir_intrinsic_task_payload_atomic_imin:
          case nir_intrinsic_task_payload_atomic_umin:
@@ -103,6 +111,8 @@ shader_writes_to_memory(nir_shader *shader)
          case nir_intrinsic_task_payload_atomic_fmax:
          case nir_intrinsic_task_payload_atomic_fcomp_swap:
          case nir_intrinsic_image_deref_store:
+         case nir_intrinsic_image_deref_atomic:
+         case nir_intrinsic_image_deref_atomic_swap:
          case nir_intrinsic_image_deref_atomic_add:
          case nir_intrinsic_image_deref_atomic_fadd:
          case nir_intrinsic_image_deref_atomic_umin:

diff --git a/src/compiler/nir/nir_lower_non_uniform_access.c b/src/compiler/nir/nir_lower_non_uniform_access.c
index 1116158..75a1ab7 100644 (file)
--- a/src/compiler/nir/nir_lower_non_uniform_access.c
+++ b/src/compiler/nir/nir_lower_non_uniform_access.c
@@ -228,6 +228,8 @@ nir_lower_non_uniform_access_impl(nir_function_impl *impl,
                break;
 
             case nir_intrinsic_load_ssbo:
+            case nir_intrinsic_ssbo_atomic:
+            case nir_intrinsic_ssbo_atomic_swap:
             case nir_intrinsic_ssbo_atomic_add:
             case nir_intrinsic_ssbo_atomic_imin:
             case nir_intrinsic_ssbo_atomic_umin:
@@ -263,6 +265,8 @@ nir_lower_non_uniform_access_impl(nir_function_impl *impl,
             case nir_intrinsic_image_load:
             case nir_intrinsic_image_sparse_load:
             case nir_intrinsic_image_store:
+            case nir_intrinsic_image_atomic:
+            case nir_intrinsic_image_atomic_swap:
             case nir_intrinsic_image_atomic_add:
             case nir_intrinsic_image_atomic_imin:
             case nir_intrinsic_image_atomic_umin:
@@ -285,6 +289,8 @@ nir_lower_non_uniform_access_impl(nir_function_impl *impl,
             case nir_intrinsic_bindless_image_load:
             case nir_intrinsic_bindless_image_sparse_load:
             case nir_intrinsic_bindless_image_store:
+            case nir_intrinsic_bindless_image_atomic:
+            case nir_intrinsic_bindless_image_atomic_swap:
             case nir_intrinsic_bindless_image_atomic_add:
             case nir_intrinsic_bindless_image_atomic_imin:
             case nir_intrinsic_bindless_image_atomic_umin:
@@ -307,6 +313,8 @@ nir_lower_non_uniform_access_impl(nir_function_impl *impl,
             case nir_intrinsic_image_deref_load:
             case nir_intrinsic_image_deref_sparse_load:
             case nir_intrinsic_image_deref_store:
+            case nir_intrinsic_image_deref_atomic:
+            case nir_intrinsic_image_deref_atomic_swap:
             case nir_intrinsic_image_deref_atomic_add:
             case nir_intrinsic_image_deref_atomic_umin:
             case nir_intrinsic_image_deref_atomic_imin:

diff --git a/src/compiler/nir/nir_lower_task_shader.c b/src/compiler/nir/nir_lower_task_shader.c
index c4013a0..6d2c0b5 100644 (file)
--- a/src/compiler/nir/nir_lower_task_shader.c
+++ b/src/compiler/nir/nir_lower_task_shader.c
@@ -355,6 +355,8 @@ lower_task_intrin(nir_builder *b,
    nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
 
    switch (intrin->intrinsic) {
+   case nir_intrinsic_task_payload_atomic:
+   case nir_intrinsic_task_payload_atomic_swap:
    case nir_intrinsic_task_payload_atomic_add:
    case nir_intrinsic_task_payload_atomic_imin:
    case nir_intrinsic_task_payload_atomic_umin:
@@ -395,6 +397,8 @@ requires_payload_in_shared(nir_shader *shader, bool atomics, bool small_types)
 
             nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
             switch (intrin->intrinsic) {
+               case nir_intrinsic_task_payload_atomic:
+               case nir_intrinsic_task_payload_atomic_swap:
                case nir_intrinsic_task_payload_atomic_add:
                case nir_intrinsic_task_payload_atomic_imin:
                case nir_intrinsic_task_payload_atomic_umin:

diff --git a/src/compiler/nir/nir_opt_access.c b/src/compiler/nir/nir_opt_access.c
index d8ca876..db0b4eb 100644 (file)
--- a/src/compiler/nir/nir_opt_access.c
+++ b/src/compiler/nir/nir_opt_access.c
@@ -83,6 +83,8 @@ gather_intrinsic(struct access_state *state, nir_intrinsic_instr *instr)
    case nir_intrinsic_image_deref_load:
    case nir_intrinsic_image_deref_store:
    case nir_intrinsic_image_deref_sparse_load:
+   case nir_intrinsic_image_deref_atomic:
+   case nir_intrinsic_image_deref_atomic_swap:
    case nir_intrinsic_image_deref_atomic_add:
    case nir_intrinsic_image_deref_atomic_imin:
    case nir_intrinsic_image_deref_atomic_umin:
@@ -128,6 +130,8 @@ gather_intrinsic(struct access_state *state, nir_intrinsic_instr *instr)
    case nir_intrinsic_bindless_image_load:
    case nir_intrinsic_bindless_image_store:
    case nir_intrinsic_bindless_image_sparse_load:
+   case nir_intrinsic_bindless_image_atomic:
+   case nir_intrinsic_bindless_image_atomic_swap:
    case nir_intrinsic_bindless_image_atomic_add:
    case nir_intrinsic_bindless_image_atomic_imin:
    case nir_intrinsic_bindless_image_atomic_umin:
@@ -159,6 +163,8 @@ gather_intrinsic(struct access_state *state, nir_intrinsic_instr *instr)
 
    case nir_intrinsic_load_deref:
    case nir_intrinsic_store_deref:
+   case nir_intrinsic_deref_atomic:
+   case nir_intrinsic_deref_atomic_swap:
    case nir_intrinsic_deref_atomic_add:
    case nir_intrinsic_deref_atomic_imin:
    case nir_intrinsic_deref_atomic_umin:

diff --git a/src/compiler/nir/nir_opt_combine_stores.c b/src/compiler/nir/nir_opt_combine_stores.c
index d66f435..87d4cc1 100644 (file)
--- a/src/compiler/nir/nir_opt_combine_stores.c
+++ b/src/compiler/nir/nir_opt_combine_stores.c
@@ -402,6 +402,8 @@ combine_stores_block(struct combine_stores_state *state, nir_block *block)
          break;
       }
 
+      case nir_intrinsic_deref_atomic:
+      case nir_intrinsic_deref_atomic_swap:
       case nir_intrinsic_deref_atomic_add:
       case nir_intrinsic_deref_atomic_imin:
       case nir_intrinsic_deref_atomic_umin:

diff --git a/src/compiler/nir/nir_opt_copy_prop_vars.c b/src/compiler/nir/nir_opt_copy_prop_vars.c
index 8cc99d5..9fdb4c6 100644 (file)
--- a/src/compiler/nir/nir_opt_copy_prop_vars.c
+++ b/src/compiler/nir/nir_opt_copy_prop_vars.c
@@ -239,6 +239,8 @@ gather_vars_written(struct copy_prop_var_state *state,
                               nir_var_shader_call_data;
             break;
 
+         case nir_intrinsic_deref_atomic:
+         case nir_intrinsic_deref_atomic_swap:
          case nir_intrinsic_deref_atomic_add:
          case nir_intrinsic_deref_atomic_fadd:
          case nir_intrinsic_deref_atomic_imin:
@@ -1312,6 +1314,8 @@ copy_prop_vars_block(struct copy_prop_var_state *state,
       }
 
       case nir_intrinsic_memcpy_deref:
+      case nir_intrinsic_deref_atomic:
+      case nir_intrinsic_deref_atomic_swap:
       case nir_intrinsic_deref_atomic_add:
       case nir_intrinsic_deref_atomic_fadd:
       case nir_intrinsic_deref_atomic_imin:

diff --git a/src/compiler/nir/nir_opt_non_uniform_access.c b/src/compiler/nir/nir_opt_non_uniform_access.c
index 182450a..0aadf18 100644 (file)
--- a/src/compiler/nir/nir_opt_non_uniform_access.c
+++ b/src/compiler/nir/nir_opt_non_uniform_access.c
@@ -36,6 +36,8 @@ is_ssbo_intrinsic(nir_intrinsic_instr *intrin)
    switch (intrin->intrinsic) {
    case nir_intrinsic_load_ssbo:
    case nir_intrinsic_store_ssbo:
+   case nir_intrinsic_ssbo_atomic:
+   case nir_intrinsic_ssbo_atomic_swap:
    case nir_intrinsic_ssbo_atomic_add:
    case nir_intrinsic_ssbo_atomic_imin:
    case nir_intrinsic_ssbo_atomic_umin:
@@ -64,6 +66,8 @@ is_image_intrinsic(nir_intrinsic_instr *intrin)
    case nir_intrinsic_image_load:
    case nir_intrinsic_image_sparse_load:
    case nir_intrinsic_image_store:
+   case nir_intrinsic_image_atomic:
+   case nir_intrinsic_image_atomic_swap:
    case nir_intrinsic_image_atomic_add:
    case nir_intrinsic_image_atomic_imin:
    case nir_intrinsic_image_atomic_umin:
@@ -85,6 +89,8 @@ is_image_intrinsic(nir_intrinsic_instr *intrin)
    case nir_intrinsic_bindless_image_load:
    case nir_intrinsic_bindless_image_sparse_load:
    case nir_intrinsic_bindless_image_store:
+   case nir_intrinsic_bindless_image_atomic:
+   case nir_intrinsic_bindless_image_atomic_swap:
    case nir_intrinsic_bindless_image_atomic_add:
    case nir_intrinsic_bindless_image_atomic_imin:
    case nir_intrinsic_bindless_image_atomic_umin:
@@ -106,6 +112,8 @@ is_image_intrinsic(nir_intrinsic_instr *intrin)
    case nir_intrinsic_image_deref_load:
    case nir_intrinsic_image_deref_sparse_load:
    case nir_intrinsic_image_deref_store:
+   case nir_intrinsic_image_deref_atomic:
+   case nir_intrinsic_image_deref_atomic_swap:
    case nir_intrinsic_image_deref_atomic_add:
    case nir_intrinsic_image_deref_atomic_umin:
    case nir_intrinsic_image_deref_atomic_imin: