From 97857731ce875c846cca37059864f08ff47358a5 Mon Sep 17 00:00:00 2001 From: Alyssa Rosenzweig Date: Fri, 12 May 2023 10:52:57 -0400 Subject: [PATCH] nir: Handle unified atomics in simple cases Lots of passes can be made unified-atomics-aware simply by adding extra cases in their switch statements. This commit fixes a bunch of passes. Signed-off-by: Alyssa Rosenzweig Reviewed-by: Jesse Natalie Part-of: --- src/compiler/nir/nir_gather_info.c | 12 ++++++++++++ src/compiler/nir/nir_lower_amul.c | 6 ++++++ src/compiler/nir/nir_lower_cl_images.c | 2 ++ src/compiler/nir/nir_lower_helper_writes.c | 4 ++++ src/compiler/nir/nir_lower_mediump.c | 8 ++++++++ src/compiler/nir/nir_lower_memory_model.c | 8 ++++++++ src/compiler/nir/nir_lower_multiview.c | 10 ++++++++++ src/compiler/nir/nir_lower_non_uniform_access.c | 8 ++++++++ src/compiler/nir/nir_lower_task_shader.c | 4 ++++ src/compiler/nir/nir_opt_access.c | 6 ++++++ src/compiler/nir/nir_opt_combine_stores.c | 2 ++ src/compiler/nir/nir_opt_copy_prop_vars.c | 4 ++++ src/compiler/nir/nir_opt_non_uniform_access.c | 8 ++++++++ 13 files changed, 82 insertions(+) diff --git a/src/compiler/nir/nir_gather_info.c b/src/compiler/nir/nir_gather_info.c index d3374ac..738b64d 100644 --- a/src/compiler/nir/nir_gather_info.c +++ b/src/compiler/nir/nir_gather_info.c @@ -340,6 +340,8 @@ nir_intrinsic_writes_external_memory(const nir_intrinsic_instr *instr) case nir_intrinsic_atomic_counter_exchange_deref: case nir_intrinsic_atomic_counter_comp_swap: case nir_intrinsic_atomic_counter_comp_swap_deref: + case nir_intrinsic_bindless_image_atomic: + case nir_intrinsic_bindless_image_atomic_swap: case nir_intrinsic_bindless_image_atomic_add: case nir_intrinsic_bindless_image_atomic_and: case nir_intrinsic_bindless_image_atomic_comp_swap: @@ -355,6 +357,8 @@ nir_intrinsic_writes_external_memory(const nir_intrinsic_instr *instr) case nir_intrinsic_bindless_image_atomic_xor: case nir_intrinsic_bindless_image_store: case nir_intrinsic_bindless_image_store_raw_intel: + case nir_intrinsic_global_atomic: + case nir_intrinsic_global_atomic_swap: case nir_intrinsic_global_atomic_add: case nir_intrinsic_global_atomic_and: case nir_intrinsic_global_atomic_comp_swap: @@ -371,6 +375,8 @@ nir_intrinsic_writes_external_memory(const nir_intrinsic_instr *instr) case nir_intrinsic_global_atomic_xor: case nir_intrinsic_global_atomic_ir3: case nir_intrinsic_global_atomic_swap_ir3: + case nir_intrinsic_image_atomic: + case nir_intrinsic_image_atomic_swap: case nir_intrinsic_image_atomic_add: case nir_intrinsic_image_atomic_and: case nir_intrinsic_image_atomic_comp_swap: @@ -384,6 +390,8 @@ nir_intrinsic_writes_external_memory(const nir_intrinsic_instr *instr) case nir_intrinsic_image_atomic_umax: case nir_intrinsic_image_atomic_umin: case nir_intrinsic_image_atomic_xor: + case nir_intrinsic_image_deref_atomic: + case nir_intrinsic_image_deref_atomic_swap: case nir_intrinsic_image_deref_atomic_add: case nir_intrinsic_image_deref_atomic_and: case nir_intrinsic_image_deref_atomic_comp_swap: @@ -427,6 +435,8 @@ nir_intrinsic_writes_external_memory(const nir_intrinsic_instr *instr) return true; case nir_intrinsic_store_deref: + case nir_intrinsic_deref_atomic: + case nir_intrinsic_deref_atomic_swap: case nir_intrinsic_deref_atomic_add: case nir_intrinsic_deref_atomic_imin: case nir_intrinsic_deref_atomic_umin: @@ -453,6 +463,8 @@ static bool intrinsic_is_bindless(nir_intrinsic_instr *instr) { switch (instr->intrinsic) { + case nir_intrinsic_bindless_image_atomic: + case nir_intrinsic_bindless_image_atomic_swap: case nir_intrinsic_bindless_image_atomic_add: case nir_intrinsic_bindless_image_atomic_and: case nir_intrinsic_bindless_image_atomic_comp_swap: diff --git a/src/compiler/nir/nir_lower_amul.c b/src/compiler/nir/nir_lower_amul.c index cbea520..dfb8a0b 100644 --- a/src/compiler/nir/nir_lower_amul.c +++ b/src/compiler/nir/nir_lower_amul.c @@ -142,6 +142,8 @@ lower_intrinsic(lower_state *state, nir_intrinsic_instr *intr) lower_large_src(&intr->src[2], state); return; + case nir_intrinsic_ssbo_atomic: + case nir_intrinsic_ssbo_atomic_swap: case nir_intrinsic_ssbo_atomic_add: case nir_intrinsic_ssbo_atomic_imin: case nir_intrinsic_ssbo_atomic_umin: @@ -163,6 +165,8 @@ lower_intrinsic(lower_state *state, nir_intrinsic_instr *intr) lower_large_src(&intr->src[1], state); return; + case nir_intrinsic_global_atomic: + case nir_intrinsic_global_atomic_swap: case nir_intrinsic_global_atomic_add: case nir_intrinsic_global_atomic_imin: case nir_intrinsic_global_atomic_umin: @@ -189,6 +193,8 @@ lower_intrinsic(lower_state *state, nir_intrinsic_instr *intr) return; /* These should all be small enough to unconditionally use imul24: */ + case nir_intrinsic_shared_atomic: + case nir_intrinsic_shared_atomic_swap: case nir_intrinsic_shared_atomic_add: case nir_intrinsic_shared_atomic_imin: case nir_intrinsic_shared_atomic_umin: diff --git a/src/compiler/nir/nir_lower_cl_images.c b/src/compiler/nir/nir_lower_cl_images.c index 738a9c9..2751947 100644 --- a/src/compiler/nir/nir_lower_cl_images.c +++ b/src/compiler/nir/nir_lower_cl_images.c @@ -248,6 +248,8 @@ nir_lower_cl_images(nir_shader *shader, bool lower_image_derefs, bool lower_samp switch (intrin->intrinsic) { case nir_intrinsic_image_deref_load: case nir_intrinsic_image_deref_store: + case nir_intrinsic_image_deref_atomic: + case nir_intrinsic_image_deref_atomic_swap: case nir_intrinsic_image_deref_atomic_add: case nir_intrinsic_image_deref_atomic_imin: case nir_intrinsic_image_deref_atomic_umin: diff --git a/src/compiler/nir/nir_lower_helper_writes.c b/src/compiler/nir/nir_lower_helper_writes.c index d961771..cb82856 100644 --- a/src/compiler/nir/nir_lower_helper_writes.c +++ b/src/compiler/nir/nir_lower_helper_writes.c @@ -35,6 +35,8 @@ lower(nir_builder *b, nir_instr *instr, void *data) bool *lower_plain_stores = data; switch (intr->intrinsic) { + case nir_intrinsic_global_atomic: + case nir_intrinsic_global_atomic_swap: case nir_intrinsic_global_atomic_add: case nir_intrinsic_global_atomic_and: case nir_intrinsic_global_atomic_comp_swap: @@ -49,6 +51,8 @@ lower(nir_builder *b, nir_instr *instr, void *data) case nir_intrinsic_global_atomic_umax: case nir_intrinsic_global_atomic_umin: case nir_intrinsic_global_atomic_xor: + case nir_intrinsic_image_atomic: + case nir_intrinsic_image_atomic_swap: case nir_intrinsic_image_atomic_add: case nir_intrinsic_image_atomic_and: case nir_intrinsic_image_atomic_comp_swap: diff --git a/src/compiler/nir/nir_lower_mediump.c b/src/compiler/nir/nir_lower_mediump.c index 8698b0d..6f61a3e 100644 --- a/src/compiler/nir/nir_lower_mediump.c +++ b/src/compiler/nir/nir_lower_mediump.c @@ -568,6 +568,8 @@ nir_lower_mediump_vars(nir_shader *shader, nir_variable_mode modes) continue; nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); switch (intr->intrinsic) { + case nir_intrinsic_deref_atomic: + case nir_intrinsic_deref_atomic_swap: case nir_intrinsic_deref_atomic_add: case nir_intrinsic_deref_atomic_imin: case nir_intrinsic_deref_atomic_umin: @@ -1047,6 +1049,8 @@ fold_16bit_tex_image(nir_builder *b, nir_instr *instr, void *params) if (options->fold_image_srcs) progress |= fold_16bit_image_srcs(b, intrinsic, 3); break; + case nir_intrinsic_bindless_image_atomic: + case nir_intrinsic_bindless_image_atomic_swap: case nir_intrinsic_bindless_image_atomic_add: case nir_intrinsic_bindless_image_atomic_imin: case nir_intrinsic_bindless_image_atomic_umin: @@ -1062,6 +1066,8 @@ fold_16bit_tex_image(nir_builder *b, nir_instr *instr, void *params) case nir_intrinsic_bindless_image_atomic_fmax: case nir_intrinsic_bindless_image_atomic_inc_wrap: case nir_intrinsic_bindless_image_atomic_dec_wrap: + case nir_intrinsic_image_deref_atomic: + case nir_intrinsic_image_deref_atomic_swap: case nir_intrinsic_image_deref_atomic_add: case nir_intrinsic_image_deref_atomic_umin: case nir_intrinsic_image_deref_atomic_imin: @@ -1077,6 +1083,8 @@ fold_16bit_tex_image(nir_builder *b, nir_instr *instr, void *params) case nir_intrinsic_image_deref_atomic_fmax: case nir_intrinsic_image_deref_atomic_inc_wrap: case nir_intrinsic_image_deref_atomic_dec_wrap: + case nir_intrinsic_image_atomic: + case nir_intrinsic_image_atomic_swap: case nir_intrinsic_image_atomic_add: case nir_intrinsic_image_atomic_imin: case nir_intrinsic_image_atomic_umin: diff --git a/src/compiler/nir/nir_lower_memory_model.c b/src/compiler/nir/nir_lower_memory_model.c index b466cee..a8c20e1 100644 --- a/src/compiler/nir/nir_lower_memory_model.c +++ b/src/compiler/nir/nir_lower_memory_model.c @@ -44,6 +44,8 @@ get_intrinsic_info(nir_intrinsic_instr *intrin, nir_variable_mode *modes, *modes = nir_src_as_deref(intrin->src[0])->modes; *writes = true; break; + case nir_intrinsic_image_deref_atomic: + case nir_intrinsic_image_deref_atomic_swap: case nir_intrinsic_image_deref_atomic_add: case nir_intrinsic_image_deref_atomic_fadd: case nir_intrinsic_image_deref_atomic_umin: @@ -69,6 +71,8 @@ get_intrinsic_info(nir_intrinsic_instr *intrin, nir_variable_mode *modes, *modes = nir_var_mem_ssbo; *writes = true; break; + case nir_intrinsic_ssbo_atomic: + case nir_intrinsic_ssbo_atomic_swap: case nir_intrinsic_ssbo_atomic_add: case nir_intrinsic_ssbo_atomic_imin: case nir_intrinsic_ssbo_atomic_umin: @@ -95,6 +99,8 @@ get_intrinsic_info(nir_intrinsic_instr *intrin, nir_variable_mode *modes, *modes = nir_var_mem_global; *writes = true; break; + case nir_intrinsic_global_atomic: + case nir_intrinsic_global_atomic_swap: case nir_intrinsic_global_atomic_add: case nir_intrinsic_global_atomic_imin: case nir_intrinsic_global_atomic_umin: @@ -121,6 +127,8 @@ get_intrinsic_info(nir_intrinsic_instr *intrin, nir_variable_mode *modes, *modes = nir_src_as_deref(intrin->src[0])->modes; *writes = true; break; + case nir_intrinsic_deref_atomic: + case nir_intrinsic_deref_atomic_swap: case nir_intrinsic_deref_atomic_add: case nir_intrinsic_deref_atomic_imin: case nir_intrinsic_deref_atomic_umin: diff --git a/src/compiler/nir/nir_lower_multiview.c b/src/compiler/nir/nir_lower_multiview.c index 82fea39..3cf02e4 100644 --- a/src/compiler/nir/nir_lower_multiview.c +++ b/src/compiler/nir/nir_lower_multiview.c @@ -55,6 +55,8 @@ shader_writes_to_memory(nir_shader *shader) nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); switch (intrin->intrinsic) { + case nir_intrinsic_deref_atomic: + case nir_intrinsic_deref_atomic_swap: case nir_intrinsic_deref_atomic_add: case nir_intrinsic_deref_atomic_imin: case nir_intrinsic_deref_atomic_umin: @@ -66,6 +68,8 @@ shader_writes_to_memory(nir_shader *shader) case nir_intrinsic_deref_atomic_exchange: case nir_intrinsic_deref_atomic_comp_swap: case nir_intrinsic_store_ssbo: + case nir_intrinsic_ssbo_atomic: + case nir_intrinsic_ssbo_atomic_swap: case nir_intrinsic_ssbo_atomic_add: case nir_intrinsic_ssbo_atomic_imin: case nir_intrinsic_ssbo_atomic_umin: @@ -78,6 +82,8 @@ shader_writes_to_memory(nir_shader *shader) case nir_intrinsic_ssbo_atomic_comp_swap: case nir_intrinsic_store_shared: case nir_intrinsic_store_shared2_amd: + case nir_intrinsic_shared_atomic: + case nir_intrinsic_shared_atomic_swap: case nir_intrinsic_shared_atomic_add: case nir_intrinsic_shared_atomic_imin: case nir_intrinsic_shared_atomic_umin: @@ -88,6 +94,8 @@ shader_writes_to_memory(nir_shader *shader) case nir_intrinsic_shared_atomic_xor: case nir_intrinsic_shared_atomic_exchange: case nir_intrinsic_shared_atomic_comp_swap: + case nir_intrinsic_task_payload_atomic: + case nir_intrinsic_task_payload_atomic_swap: case nir_intrinsic_task_payload_atomic_add: case nir_intrinsic_task_payload_atomic_imin: case nir_intrinsic_task_payload_atomic_umin: @@ -103,6 +111,8 @@ shader_writes_to_memory(nir_shader *shader) case nir_intrinsic_task_payload_atomic_fmax: case nir_intrinsic_task_payload_atomic_fcomp_swap: case nir_intrinsic_image_deref_store: + case nir_intrinsic_image_deref_atomic: + case nir_intrinsic_image_deref_atomic_swap: case nir_intrinsic_image_deref_atomic_add: case nir_intrinsic_image_deref_atomic_fadd: case nir_intrinsic_image_deref_atomic_umin: diff --git a/src/compiler/nir/nir_lower_non_uniform_access.c b/src/compiler/nir/nir_lower_non_uniform_access.c index 1116158..75a1ab7 100644 --- a/src/compiler/nir/nir_lower_non_uniform_access.c +++ b/src/compiler/nir/nir_lower_non_uniform_access.c @@ -228,6 +228,8 @@ nir_lower_non_uniform_access_impl(nir_function_impl *impl, break; case nir_intrinsic_load_ssbo: + case nir_intrinsic_ssbo_atomic: + case nir_intrinsic_ssbo_atomic_swap: case nir_intrinsic_ssbo_atomic_add: case nir_intrinsic_ssbo_atomic_imin: case nir_intrinsic_ssbo_atomic_umin: @@ -263,6 +265,8 @@ nir_lower_non_uniform_access_impl(nir_function_impl *impl, case nir_intrinsic_image_load: case nir_intrinsic_image_sparse_load: case nir_intrinsic_image_store: + case nir_intrinsic_image_atomic: + case nir_intrinsic_image_atomic_swap: case nir_intrinsic_image_atomic_add: case nir_intrinsic_image_atomic_imin: case nir_intrinsic_image_atomic_umin: @@ -285,6 +289,8 @@ nir_lower_non_uniform_access_impl(nir_function_impl *impl, case nir_intrinsic_bindless_image_load: case nir_intrinsic_bindless_image_sparse_load: case nir_intrinsic_bindless_image_store: + case nir_intrinsic_bindless_image_atomic: + case nir_intrinsic_bindless_image_atomic_swap: case nir_intrinsic_bindless_image_atomic_add: case nir_intrinsic_bindless_image_atomic_imin: case nir_intrinsic_bindless_image_atomic_umin: @@ -307,6 +313,8 @@ nir_lower_non_uniform_access_impl(nir_function_impl *impl, case nir_intrinsic_image_deref_load: case nir_intrinsic_image_deref_sparse_load: case nir_intrinsic_image_deref_store: + case nir_intrinsic_image_deref_atomic: + case nir_intrinsic_image_deref_atomic_swap: case nir_intrinsic_image_deref_atomic_add: case nir_intrinsic_image_deref_atomic_umin: case nir_intrinsic_image_deref_atomic_imin: diff --git a/src/compiler/nir/nir_lower_task_shader.c b/src/compiler/nir/nir_lower_task_shader.c index c4013a0..6d2c0b5 100644 --- a/src/compiler/nir/nir_lower_task_shader.c +++ b/src/compiler/nir/nir_lower_task_shader.c @@ -355,6 +355,8 @@ lower_task_intrin(nir_builder *b, nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); switch (intrin->intrinsic) { + case nir_intrinsic_task_payload_atomic: + case nir_intrinsic_task_payload_atomic_swap: case nir_intrinsic_task_payload_atomic_add: case nir_intrinsic_task_payload_atomic_imin: case nir_intrinsic_task_payload_atomic_umin: @@ -395,6 +397,8 @@ requires_payload_in_shared(nir_shader *shader, bool atomics, bool small_types) nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); switch (intrin->intrinsic) { + case nir_intrinsic_task_payload_atomic: + case nir_intrinsic_task_payload_atomic_swap: case nir_intrinsic_task_payload_atomic_add: case nir_intrinsic_task_payload_atomic_imin: case nir_intrinsic_task_payload_atomic_umin: diff --git a/src/compiler/nir/nir_opt_access.c b/src/compiler/nir/nir_opt_access.c index d8ca876..db0b4eb 100644 --- a/src/compiler/nir/nir_opt_access.c +++ b/src/compiler/nir/nir_opt_access.c @@ -83,6 +83,8 @@ gather_intrinsic(struct access_state *state, nir_intrinsic_instr *instr) case nir_intrinsic_image_deref_load: case nir_intrinsic_image_deref_store: case nir_intrinsic_image_deref_sparse_load: + case nir_intrinsic_image_deref_atomic: + case nir_intrinsic_image_deref_atomic_swap: case nir_intrinsic_image_deref_atomic_add: case nir_intrinsic_image_deref_atomic_imin: case nir_intrinsic_image_deref_atomic_umin: @@ -128,6 +130,8 @@ gather_intrinsic(struct access_state *state, nir_intrinsic_instr *instr) case nir_intrinsic_bindless_image_load: case nir_intrinsic_bindless_image_store: case nir_intrinsic_bindless_image_sparse_load: + case nir_intrinsic_bindless_image_atomic: + case nir_intrinsic_bindless_image_atomic_swap: case nir_intrinsic_bindless_image_atomic_add: case nir_intrinsic_bindless_image_atomic_imin: case nir_intrinsic_bindless_image_atomic_umin: @@ -159,6 +163,8 @@ gather_intrinsic(struct access_state *state, nir_intrinsic_instr *instr) case nir_intrinsic_load_deref: case nir_intrinsic_store_deref: + case nir_intrinsic_deref_atomic: + case nir_intrinsic_deref_atomic_swap: case nir_intrinsic_deref_atomic_add: case nir_intrinsic_deref_atomic_imin: case nir_intrinsic_deref_atomic_umin: diff --git a/src/compiler/nir/nir_opt_combine_stores.c b/src/compiler/nir/nir_opt_combine_stores.c index d66f435..87d4cc1 100644 --- a/src/compiler/nir/nir_opt_combine_stores.c +++ b/src/compiler/nir/nir_opt_combine_stores.c @@ -402,6 +402,8 @@ combine_stores_block(struct combine_stores_state *state, nir_block *block) break; } + case nir_intrinsic_deref_atomic: + case nir_intrinsic_deref_atomic_swap: case nir_intrinsic_deref_atomic_add: case nir_intrinsic_deref_atomic_imin: case nir_intrinsic_deref_atomic_umin: diff --git a/src/compiler/nir/nir_opt_copy_prop_vars.c b/src/compiler/nir/nir_opt_copy_prop_vars.c index 8cc99d5..9fdb4c6 100644 --- a/src/compiler/nir/nir_opt_copy_prop_vars.c +++ b/src/compiler/nir/nir_opt_copy_prop_vars.c @@ -239,6 +239,8 @@ gather_vars_written(struct copy_prop_var_state *state, nir_var_shader_call_data; break; + case nir_intrinsic_deref_atomic: + case nir_intrinsic_deref_atomic_swap: case nir_intrinsic_deref_atomic_add: case nir_intrinsic_deref_atomic_fadd: case nir_intrinsic_deref_atomic_imin: @@ -1312,6 +1314,8 @@ copy_prop_vars_block(struct copy_prop_var_state *state, } case nir_intrinsic_memcpy_deref: + case nir_intrinsic_deref_atomic: + case nir_intrinsic_deref_atomic_swap: case nir_intrinsic_deref_atomic_add: case nir_intrinsic_deref_atomic_fadd: case nir_intrinsic_deref_atomic_imin: diff --git a/src/compiler/nir/nir_opt_non_uniform_access.c b/src/compiler/nir/nir_opt_non_uniform_access.c index 182450a..0aadf18 100644 --- a/src/compiler/nir/nir_opt_non_uniform_access.c +++ b/src/compiler/nir/nir_opt_non_uniform_access.c @@ -36,6 +36,8 @@ is_ssbo_intrinsic(nir_intrinsic_instr *intrin) switch (intrin->intrinsic) { case nir_intrinsic_load_ssbo: case nir_intrinsic_store_ssbo: + case nir_intrinsic_ssbo_atomic: + case nir_intrinsic_ssbo_atomic_swap: case nir_intrinsic_ssbo_atomic_add: case nir_intrinsic_ssbo_atomic_imin: case nir_intrinsic_ssbo_atomic_umin: @@ -64,6 +66,8 @@ is_image_intrinsic(nir_intrinsic_instr *intrin) case nir_intrinsic_image_load: case nir_intrinsic_image_sparse_load: case nir_intrinsic_image_store: + case nir_intrinsic_image_atomic: + case nir_intrinsic_image_atomic_swap: case nir_intrinsic_image_atomic_add: case nir_intrinsic_image_atomic_imin: case nir_intrinsic_image_atomic_umin: @@ -85,6 +89,8 @@ is_image_intrinsic(nir_intrinsic_instr *intrin) case nir_intrinsic_bindless_image_load: case nir_intrinsic_bindless_image_sparse_load: case nir_intrinsic_bindless_image_store: + case nir_intrinsic_bindless_image_atomic: + case nir_intrinsic_bindless_image_atomic_swap: case nir_intrinsic_bindless_image_atomic_add: case nir_intrinsic_bindless_image_atomic_imin: case nir_intrinsic_bindless_image_atomic_umin: @@ -106,6 +112,8 @@ is_image_intrinsic(nir_intrinsic_instr *intrin) case nir_intrinsic_image_deref_load: case nir_intrinsic_image_deref_sparse_load: case nir_intrinsic_image_deref_store: + case nir_intrinsic_image_deref_atomic: + case nir_intrinsic_image_deref_atomic_swap: case nir_intrinsic_image_deref_atomic_add: case nir_intrinsic_image_deref_atomic_umin: case nir_intrinsic_image_deref_atomic_imin: -- 2.7.4