aco: insert waitcnt before/after ds_ordered_count

The LLVM backend does this when lowering ordered_xfb_counter_add_amd. I
guess there is some missing dependency checking or something.

Signed-off-by: Rhys Perry <pendingchaos02@gmail.com>
Reviewed-by: Daniel Schürmann <daniel@schuermann.dev>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/19345>

diff --git a/src/amd/compiler/aco_insert_waitcnt.cpp b/src/amd/compiler/aco_insert_waitcnt.cpp
index 3a1fddc..bb3e362 100644 (file)
--- a/src/amd/compiler/aco_insert_waitcnt.cpp
+++ b/src/amd/compiler/aco_insert_waitcnt.cpp
@@ -393,6 +393,11 @@ kill(wait_imm& imm, Instruction* instr, wait_ctx& ctx, memory_sync_info sync_inf
       }
    }
 
+   if (instr->opcode == aco_opcode::ds_ordered_count &&
+       ((instr->ds().offset1 | (instr->ds().offset0 >> 8)) & 0x1)) {
+      imm.combine(ctx.barrier_imm[ffs(storage_gds) - 1]);
+   }
+
    if (ctx.program->early_rast && instr->opcode == aco_opcode::exp) {
       if (instr->exp().dest >= V_008DFC_SQ_EXP_POS && instr->exp().dest < V_008DFC_SQ_EXP_PRIM) {
 
@@ -774,8 +779,15 @@ handle_block(Program* program, Block& block, wait_ctx& ctx)
          if (!queued_imm.empty())
             emit_waitcnt(ctx, new_instructions, queued_imm);
 
+         bool is_ordered_count_acquire =
+            instr->opcode == aco_opcode::ds_ordered_count &&
+            !((instr->ds().offset1 | (instr->ds().offset0 >> 8)) & 0x1);
+
          new_instructions.emplace_back(std::move(instr));
          perform_barrier(ctx, queued_imm, sync_info, semantic_acquire);
+
+         if (is_ordered_count_acquire)
+            queued_imm.combine(ctx.barrier_imm[ffs(storage_gds) - 1]);
       }
    }
 

diff --git a/src/amd/compiler/tests/helpers.cpp b/src/amd/compiler/tests/helpers.cpp
index 88dc95b..211284b 100644 (file)
--- a/src/amd/compiler/tests/helpers.cpp
+++ b/src/amd/compiler/tests/helpers.cpp
@@ -214,6 +214,13 @@ void finish_to_hw_instr_test()
    aco_print_program(program.get(), output);
 }
 
+void finish_waitcnt_test()
+{
+   finish_program(program.get());
+   aco::insert_wait_states(program.get());
+   aco_print_program(program.get(), output);
+}
+
 void finish_insert_nops_test()
 {
    finish_program(program.get());

diff --git a/src/amd/compiler/tests/helpers.h b/src/amd/compiler/tests/helpers.h
index e2282b7..8f1f272 100644 (file)
--- a/src/amd/compiler/tests/helpers.h
+++ b/src/amd/compiler/tests/helpers.h
@@ -83,6 +83,7 @@ void finish_opt_test();
 void finish_ra_test(aco::ra_test_policy, bool lower=false);
 void finish_optimizer_postRA_test();
 void finish_to_hw_instr_test();
+void finish_waitcnt_test();
 void finish_insert_nops_test();
 void finish_form_hard_clause_test();
 void finish_assembler_test();

diff --git a/src/amd/compiler/tests/meson.build b/src/amd/compiler/tests/meson.build
index 7659df2..5734700 100644 (file)
--- a/src/amd/compiler/tests/meson.build
+++ b/src/amd/compiler/tests/meson.build
@@ -26,6 +26,7 @@ aco_tests_files = files(
   'test_builder.cpp',
   'test_hard_clause.cpp',
   'test_insert_nops.cpp',
+  'test_insert_waitcnt.cpp',
   'test_isel.cpp',
   'test_optimizer.cpp',
   'test_regalloc.cpp',

diff --git a/src/amd/compiler/tests/test_insert_waitcnt.cpp b/src/amd/compiler/tests/test_insert_waitcnt.cpp
new file mode 100644 (file)
index 0000000..b6c2e83
--- /dev/null
+++ b/src/amd/compiler/tests/test_insert_waitcnt.cpp
@@ -0,0 +1,56 @@
+/*
+ * Copyright © 2022 Valve Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+#include "helpers.h"
+
+using namespace aco;
+
+BEGIN_TEST(insert_waitcnt.ds_ordered_count)
+   if (!setup_cs(NULL, GFX10_3))
+      return;
+
+   Operand def0(PhysReg(256), v1);
+   Operand def1(PhysReg(257), v1);
+   Operand def2(PhysReg(258), v1);
+   Operand gds_base(PhysReg(259), v1);
+   Operand chan_counter(PhysReg(260), v1);
+   Operand m(m0, s1);
+
+   Instruction *ds_instr;
+   //>> ds_ordered_count %0:v[0], %0:v[3], %0:m0 offset0:3072 gds storage:gds semantics:volatile
+   //! s_waitcnt lgkmcnt(0)
+   ds_instr = bld.ds(aco_opcode::ds_ordered_count, def0, gds_base, m, 3072u, 0u, true);
+   ds_instr->ds().sync = memory_sync_info(storage_gds, semantic_volatile);
+
+   //! ds_add_rtn_u32 %0:v[1], %0:v[3], %0:v[4], %0:m0 gds storage:gds semantics:volatile,atomic,rmw
+   ds_instr = bld.ds(aco_opcode::ds_add_rtn_u32, def1,
+                     gds_base, chan_counter, m, 0u, 0u, true);
+   ds_instr->ds().sync = memory_sync_info(storage_gds, semantic_atomicrmw);
+
+   //! s_waitcnt lgkmcnt(0)
+   //! ds_ordered_count %0:v[2], %0:v[3], %0:m0 offset0:3840 gds storage:gds semantics:volatile
+   ds_instr = bld.ds(aco_opcode::ds_ordered_count, def2, gds_base, m, 3840u, 0u, true);
+   ds_instr->ds().sync = memory_sync_info(storage_gds, semantic_volatile);
+
+   finish_waitcnt_test();
+END_TEST