nv50/ir: Use unified atomics

Reviewed-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Reviewed-by: Jesse Natalie <jenatali@microsoft.com>
Reviewed-by: M Henning <drawoc@darkrefraction.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/22914>

diff --git a/src/nouveau/codegen/nv50_ir_from_nir.cpp b/src/nouveau/codegen/nv50_ir_from_nir.cpp
index 7b63920..3ab457a 100644 (file)
--- a/src/nouveau/codegen/nv50_ir_from_nir.cpp
+++ b/src/nouveau/codegen/nv50_ir_from_nir.cpp
@@ -141,6 +141,7 @@ private:
 
    int getSubOp(nir_intrinsic_op);
    int getSubOp(nir_op);
+   int getAtomicSubOp(nir_atomic_op);
 
    CondCode getCondCode(nir_op);
 
@@ -268,27 +269,16 @@ Converter::getDType(nir_intrinsic_instr *insn)
 {
    bool isFloat, isSigned;
    switch (insn->intrinsic) {
-   case nir_intrinsic_bindless_image_atomic_fadd:
-   case nir_intrinsic_global_atomic_fadd:
-   case nir_intrinsic_image_atomic_fadd:
-   case nir_intrinsic_shared_atomic_fadd:
-   case nir_intrinsic_ssbo_atomic_fadd:
-      isFloat = true;
-      isSigned = false;
-      break;
-   case nir_intrinsic_bindless_image_atomic_imax:
-   case nir_intrinsic_bindless_image_atomic_imin:
-   case nir_intrinsic_global_atomic_imax:
-   case nir_intrinsic_global_atomic_imin:
-   case nir_intrinsic_image_atomic_imax:
-   case nir_intrinsic_image_atomic_imin:
-   case nir_intrinsic_shared_atomic_imax:
-   case nir_intrinsic_shared_atomic_imin:
-   case nir_intrinsic_ssbo_atomic_imax:
-   case nir_intrinsic_ssbo_atomic_imin:
-      isFloat = false;
-      isSigned = true;
+   case nir_intrinsic_bindless_image_atomic:
+   case nir_intrinsic_global_atomic:
+   case nir_intrinsic_image_atomic:
+   case nir_intrinsic_shared_atomic:
+   case nir_intrinsic_ssbo_atomic: {
+      nir_alu_type type = nir_atomic_op_type(nir_intrinsic_atomic_op(insn));
+      isFloat = type == nir_type_float;
+      isSigned = type == nir_type_int;
       break;
+   }
    default:
       isFloat = false;
       isSigned = false;
@@ -564,30 +554,10 @@ Converter::getOperation(nir_intrinsic_op op)
       return OP_EMIT;
    case nir_intrinsic_end_primitive:
       return OP_RESTART;
-   case nir_intrinsic_bindless_image_atomic_add:
-   case nir_intrinsic_image_atomic_add:
-   case nir_intrinsic_bindless_image_atomic_and:
-   case nir_intrinsic_image_atomic_and:
-   case nir_intrinsic_bindless_image_atomic_comp_swap:
-   case nir_intrinsic_image_atomic_comp_swap:
-   case nir_intrinsic_bindless_image_atomic_exchange:
-   case nir_intrinsic_image_atomic_exchange:
-   case nir_intrinsic_bindless_image_atomic_imax:
-   case nir_intrinsic_image_atomic_imax:
-   case nir_intrinsic_bindless_image_atomic_umax:
-   case nir_intrinsic_image_atomic_umax:
-   case nir_intrinsic_bindless_image_atomic_imin:
-   case nir_intrinsic_image_atomic_imin:
-   case nir_intrinsic_bindless_image_atomic_umin:
-   case nir_intrinsic_image_atomic_umin:
-   case nir_intrinsic_bindless_image_atomic_or:
-   case nir_intrinsic_image_atomic_or:
-   case nir_intrinsic_bindless_image_atomic_xor:
-   case nir_intrinsic_image_atomic_xor:
-   case nir_intrinsic_bindless_image_atomic_inc_wrap:
-   case nir_intrinsic_image_atomic_inc_wrap:
-   case nir_intrinsic_bindless_image_atomic_dec_wrap:
-   case nir_intrinsic_image_atomic_dec_wrap:
+   case nir_intrinsic_bindless_image_atomic:
+   case nir_intrinsic_image_atomic:
+   case nir_intrinsic_bindless_image_atomic_swap:
+   case nir_intrinsic_image_atomic_swap:
       return OP_SUREDP;
    case nir_intrinsic_bindless_image_load:
    case nir_intrinsic_image_load:
@@ -636,80 +606,43 @@ Converter::getSubOp(nir_op op)
 }
 
 int
-Converter::getSubOp(nir_intrinsic_op op)
+Converter::getAtomicSubOp(nir_atomic_op op)
 {
    switch (op) {
-   case nir_intrinsic_bindless_image_atomic_add:
-   case nir_intrinsic_global_atomic_add:
-   case nir_intrinsic_image_atomic_add:
-   case nir_intrinsic_shared_atomic_add:
-   case nir_intrinsic_ssbo_atomic_add:
-      return  NV50_IR_SUBOP_ATOM_ADD;
-   case nir_intrinsic_bindless_image_atomic_fadd:
-   case nir_intrinsic_global_atomic_fadd:
-   case nir_intrinsic_image_atomic_fadd:
-   case nir_intrinsic_shared_atomic_fadd:
-   case nir_intrinsic_ssbo_atomic_fadd:
-      return  NV50_IR_SUBOP_ATOM_ADD;
-   case nir_intrinsic_bindless_image_atomic_and:
-   case nir_intrinsic_global_atomic_and:
-   case nir_intrinsic_image_atomic_and:
-   case nir_intrinsic_shared_atomic_and:
-   case nir_intrinsic_ssbo_atomic_and:
-      return  NV50_IR_SUBOP_ATOM_AND;
-   case nir_intrinsic_bindless_image_atomic_comp_swap:
-   case nir_intrinsic_global_atomic_comp_swap:
-   case nir_intrinsic_image_atomic_comp_swap:
-   case nir_intrinsic_shared_atomic_comp_swap:
-   case nir_intrinsic_ssbo_atomic_comp_swap:
-      return  NV50_IR_SUBOP_ATOM_CAS;
-   case nir_intrinsic_bindless_image_atomic_exchange:
-   case nir_intrinsic_global_atomic_exchange:
-   case nir_intrinsic_image_atomic_exchange:
-   case nir_intrinsic_shared_atomic_exchange:
-   case nir_intrinsic_ssbo_atomic_exchange:
-      return  NV50_IR_SUBOP_ATOM_EXCH;
-   case nir_intrinsic_bindless_image_atomic_or:
-   case nir_intrinsic_global_atomic_or:
-   case nir_intrinsic_image_atomic_or:
-   case nir_intrinsic_shared_atomic_or:
-   case nir_intrinsic_ssbo_atomic_or:
-      return  NV50_IR_SUBOP_ATOM_OR;
-   case nir_intrinsic_bindless_image_atomic_imax:
-   case nir_intrinsic_bindless_image_atomic_umax:
-   case nir_intrinsic_global_atomic_imax:
-   case nir_intrinsic_global_atomic_umax:
-   case nir_intrinsic_image_atomic_imax:
-   case nir_intrinsic_image_atomic_umax:
-   case nir_intrinsic_shared_atomic_imax:
-   case nir_intrinsic_shared_atomic_umax:
-   case nir_intrinsic_ssbo_atomic_imax:
-   case nir_intrinsic_ssbo_atomic_umax:
-      return  NV50_IR_SUBOP_ATOM_MAX;
-   case nir_intrinsic_bindless_image_atomic_imin:
-   case nir_intrinsic_bindless_image_atomic_umin:
-   case nir_intrinsic_global_atomic_imin:
-   case nir_intrinsic_global_atomic_umin:
-   case nir_intrinsic_image_atomic_imin:
-   case nir_intrinsic_image_atomic_umin:
-   case nir_intrinsic_shared_atomic_imin:
-   case nir_intrinsic_shared_atomic_umin:
-   case nir_intrinsic_ssbo_atomic_imin:
-   case nir_intrinsic_ssbo_atomic_umin:
-      return  NV50_IR_SUBOP_ATOM_MIN;
-   case nir_intrinsic_bindless_image_atomic_xor:
-   case nir_intrinsic_global_atomic_xor:
-   case nir_intrinsic_image_atomic_xor:
-   case nir_intrinsic_shared_atomic_xor:
-   case nir_intrinsic_ssbo_atomic_xor:
-      return  NV50_IR_SUBOP_ATOM_XOR;
-   case nir_intrinsic_bindless_image_atomic_inc_wrap:
-   case nir_intrinsic_image_atomic_inc_wrap:
-      return NV50_IR_SUBOP_ATOM_INC;
-   case nir_intrinsic_bindless_image_atomic_dec_wrap:
-   case nir_intrinsic_image_atomic_dec_wrap:
+   case nir_atomic_op_fadd:
+   case nir_atomic_op_iadd:
+      return NV50_IR_SUBOP_ATOM_ADD;
+   case nir_atomic_op_iand:
+      return NV50_IR_SUBOP_ATOM_AND;
+   case nir_atomic_op_cmpxchg:
+      return NV50_IR_SUBOP_ATOM_CAS;
+   case nir_atomic_op_imax:
+   case nir_atomic_op_umax:
+      return NV50_IR_SUBOP_ATOM_MAX;
+   case nir_atomic_op_imin:
+   case nir_atomic_op_umin:
+      return NV50_IR_SUBOP_ATOM_MIN;
+   case nir_atomic_op_xchg:
+      return NV50_IR_SUBOP_ATOM_EXCH;
+   case nir_atomic_op_ior:
+      return NV50_IR_SUBOP_ATOM_OR;
+   case nir_atomic_op_ixor:
+      return NV50_IR_SUBOP_ATOM_XOR;
+   case nir_atomic_op_dec_wrap:
       return NV50_IR_SUBOP_ATOM_DEC;
+   case nir_atomic_op_inc_wrap:
+      return NV50_IR_SUBOP_ATOM_INC;
+   default:
+      ERROR("couldn't get SubOp for atomic\n");
+      assert(false);
+      return 0;
+   }
+}
 
+int
+Converter::getSubOp(nir_intrinsic_op op)
+{
+   switch (op) {
    case nir_intrinsic_memory_barrier:
    case nir_intrinsic_memory_barrier_buffer:
    case nir_intrinsic_memory_barrier_image:
@@ -2117,40 +2050,22 @@ Converter::visit(nir_intrinsic_instr *insn)
       info_out->io.globalAccess |= 0x1;
       break;
    }
-   case nir_intrinsic_shared_atomic_add:
-   case nir_intrinsic_shared_atomic_fadd:
-   case nir_intrinsic_shared_atomic_and:
-   case nir_intrinsic_shared_atomic_comp_swap:
-   case nir_intrinsic_shared_atomic_exchange:
-   case nir_intrinsic_shared_atomic_or:
-   case nir_intrinsic_shared_atomic_imax:
-   case nir_intrinsic_shared_atomic_imin:
-   case nir_intrinsic_shared_atomic_umax:
-   case nir_intrinsic_shared_atomic_umin:
-   case nir_intrinsic_shared_atomic_xor: {
+   case nir_intrinsic_shared_atomic:
+   case nir_intrinsic_shared_atomic_swap: {
       const DataType dType = getDType(insn);
       LValues &newDefs = convert(&insn->dest);
       Value *indirectOffset;
       uint32_t offset = getIndirect(&insn->src[0], 0, indirectOffset);
       Symbol *sym = mkSymbol(FILE_MEMORY_SHARED, 0, dType, offset);
       Instruction *atom = mkOp2(OP_ATOM, dType, newDefs[0], sym, getSrc(&insn->src[1], 0));
-      if (op == nir_intrinsic_shared_atomic_comp_swap)
+      if (op == nir_intrinsic_shared_atomic_swap)
          atom->setSrc(2, getSrc(&insn->src[2], 0));
       atom->setIndirect(0, 0, indirectOffset);
-      atom->subOp = getSubOp(op);
-      break;
-   }
-   case nir_intrinsic_ssbo_atomic_add:
-   case nir_intrinsic_ssbo_atomic_fadd:
-   case nir_intrinsic_ssbo_atomic_and:
-   case nir_intrinsic_ssbo_atomic_comp_swap:
-   case nir_intrinsic_ssbo_atomic_exchange:
-   case nir_intrinsic_ssbo_atomic_or:
-   case nir_intrinsic_ssbo_atomic_imax:
-   case nir_intrinsic_ssbo_atomic_imin:
-   case nir_intrinsic_ssbo_atomic_umax:
-   case nir_intrinsic_ssbo_atomic_umin:
-   case nir_intrinsic_ssbo_atomic_xor: {
+      atom->subOp = getAtomicSubOp(nir_intrinsic_atomic_op(insn));
+      break;
+   }
+   case nir_intrinsic_ssbo_atomic:
+   case nir_intrinsic_ssbo_atomic_swap: {
       const DataType dType = getDType(insn);
       LValues &newDefs = convert(&insn->dest);
       Value *indirectBuffer;
@@ -2161,26 +2076,17 @@ Converter::visit(nir_intrinsic_instr *insn)
       Symbol *sym = mkSymbol(FILE_MEMORY_BUFFER, buffer, dType, offset);
       Instruction *atom = mkOp2(OP_ATOM, dType, newDefs[0], sym,
                                 getSrc(&insn->src[2], 0));
-      if (op == nir_intrinsic_ssbo_atomic_comp_swap)
+      if (op == nir_intrinsic_ssbo_atomic_swap)
          atom->setSrc(2, getSrc(&insn->src[3], 0));
       atom->setIndirect(0, 0, indirectOffset);
       atom->setIndirect(0, 1, indirectBuffer);
-      atom->subOp = getSubOp(op);
+      atom->subOp = getAtomicSubOp(nir_intrinsic_atomic_op(insn));
 
       info_out->io.globalAccess |= 0x2;
       break;
    }
-   case nir_intrinsic_global_atomic_add:
-   case nir_intrinsic_global_atomic_fadd:
-   case nir_intrinsic_global_atomic_and:
-   case nir_intrinsic_global_atomic_comp_swap:
-   case nir_intrinsic_global_atomic_exchange:
-   case nir_intrinsic_global_atomic_or:
-   case nir_intrinsic_global_atomic_imax:
-   case nir_intrinsic_global_atomic_imin:
-   case nir_intrinsic_global_atomic_umax:
-   case nir_intrinsic_global_atomic_umin:
-   case nir_intrinsic_global_atomic_xor: {
+   case nir_intrinsic_global_atomic:
+   case nir_intrinsic_global_atomic_swap: {
       const DataType dType = getDType(insn);
       LValues &newDefs = convert(&insn->dest);
       Value *address;
@@ -2189,44 +2095,22 @@ Converter::visit(nir_intrinsic_instr *insn)
       Symbol *sym = mkSymbol(FILE_MEMORY_GLOBAL, 0, dType, offset);
       Instruction *atom =
          mkOp2(OP_ATOM, dType, newDefs[0], sym, getSrc(&insn->src[1], 0));
-      if (op == nir_intrinsic_global_atomic_comp_swap)
+      if (op == nir_intrinsic_global_atomic_swap)
          atom->setSrc(2, getSrc(&insn->src[2], 0));
       atom->setIndirect(0, 0, address);
-      atom->subOp = getSubOp(op);
+      atom->subOp = getAtomicSubOp(nir_intrinsic_atomic_op(insn));
 
       info_out->io.globalAccess |= 0x2;
       break;
    }
-   case nir_intrinsic_bindless_image_atomic_add:
-   case nir_intrinsic_bindless_image_atomic_fadd:
-   case nir_intrinsic_bindless_image_atomic_and:
-   case nir_intrinsic_bindless_image_atomic_comp_swap:
-   case nir_intrinsic_bindless_image_atomic_exchange:
-   case nir_intrinsic_bindless_image_atomic_imax:
-   case nir_intrinsic_bindless_image_atomic_umax:
-   case nir_intrinsic_bindless_image_atomic_imin:
-   case nir_intrinsic_bindless_image_atomic_umin:
-   case nir_intrinsic_bindless_image_atomic_or:
-   case nir_intrinsic_bindless_image_atomic_xor:
-   case nir_intrinsic_bindless_image_atomic_inc_wrap:
-   case nir_intrinsic_bindless_image_atomic_dec_wrap:
+   case nir_intrinsic_bindless_image_atomic:
+   case nir_intrinsic_bindless_image_atomic_swap:
    case nir_intrinsic_bindless_image_load:
    case nir_intrinsic_bindless_image_samples:
    case nir_intrinsic_bindless_image_size:
    case nir_intrinsic_bindless_image_store:
-   case nir_intrinsic_image_atomic_add:
-   case nir_intrinsic_image_atomic_fadd:
-   case nir_intrinsic_image_atomic_and:
-   case nir_intrinsic_image_atomic_comp_swap:
-   case nir_intrinsic_image_atomic_exchange:
-   case nir_intrinsic_image_atomic_imax:
-   case nir_intrinsic_image_atomic_umax:
-   case nir_intrinsic_image_atomic_imin:
-   case nir_intrinsic_image_atomic_umin:
-   case nir_intrinsic_image_atomic_or:
-   case nir_intrinsic_image_atomic_xor:
-   case nir_intrinsic_image_atomic_inc_wrap:
-   case nir_intrinsic_image_atomic_dec_wrap:
+   case nir_intrinsic_image_atomic:
+   case nir_intrinsic_image_atomic_swap:
    case nir_intrinsic_image_load:
    case nir_intrinsic_image_samples:
    case nir_intrinsic_image_size:
@@ -2234,6 +2118,7 @@ Converter::visit(nir_intrinsic_instr *insn)
       std::vector<Value*> srcs, defs;
       Value *indirect;
       DataType ty;
+      int subOp = 0;
 
       uint32_t mask = 0;
       TexInstruction::Target target =
@@ -2252,41 +2137,21 @@ Converter::visit(nir_intrinsic_instr *insn)
       int lod_src = -1;
       bool bindless = false;
       switch (op) {
-      case nir_intrinsic_bindless_image_atomic_add:
-      case nir_intrinsic_bindless_image_atomic_fadd:
-      case nir_intrinsic_bindless_image_atomic_and:
-      case nir_intrinsic_bindless_image_atomic_comp_swap:
-      case nir_intrinsic_bindless_image_atomic_exchange:
-      case nir_intrinsic_bindless_image_atomic_imax:
-      case nir_intrinsic_bindless_image_atomic_umax:
-      case nir_intrinsic_bindless_image_atomic_imin:
-      case nir_intrinsic_bindless_image_atomic_umin:
-      case nir_intrinsic_bindless_image_atomic_or:
-      case nir_intrinsic_bindless_image_atomic_xor:
-      case nir_intrinsic_bindless_image_atomic_inc_wrap:
-      case nir_intrinsic_bindless_image_atomic_dec_wrap:
+      case nir_intrinsic_bindless_image_atomic:
+      case nir_intrinsic_bindless_image_atomic_swap:
          ty = getDType(insn);
          bindless = true;
          info_out->io.globalAccess |= 0x2;
          mask = 0x1;
+         subOp = getAtomicSubOp(nir_intrinsic_atomic_op(insn));
          break;
-      case nir_intrinsic_image_atomic_add:
-      case nir_intrinsic_image_atomic_fadd:
-      case nir_intrinsic_image_atomic_and:
-      case nir_intrinsic_image_atomic_comp_swap:
-      case nir_intrinsic_image_atomic_exchange:
-      case nir_intrinsic_image_atomic_imax:
-      case nir_intrinsic_image_atomic_umax:
-      case nir_intrinsic_image_atomic_imin:
-      case nir_intrinsic_image_atomic_umin:
-      case nir_intrinsic_image_atomic_or:
-      case nir_intrinsic_image_atomic_xor:
-      case nir_intrinsic_image_atomic_inc_wrap:
-      case nir_intrinsic_image_atomic_dec_wrap:
+      case nir_intrinsic_image_atomic:
+      case nir_intrinsic_image_atomic_swap:
          ty = getDType(insn);
          bindless = false;
          info_out->io.globalAccess |= 0x2;
          mask = 0x1;
+         subOp = getAtomicSubOp(nir_intrinsic_atomic_op(insn));
          break;
       case nir_intrinsic_bindless_image_load:
       case nir_intrinsic_image_load:
@@ -2361,7 +2226,7 @@ Converter::visit(nir_intrinsic_instr *insn)
       texi->tex.mask = mask;
       texi->cache = convert(nir_intrinsic_access(insn));
       texi->setType(ty);
-      texi->subOp = getSubOp(op);
+      texi->subOp = subOp;
 
       if (indirect)
          texi->setIndirectR(indirect);
@@ -3341,6 +3206,7 @@ Converter::run()
               type_size, (nir_lower_io_options)0);
 
    NIR_PASS_V(nir, nir_lower_subgroups, &subgroup_options);
+   NIR_PASS_V(nir, nir_lower_legacy_atomics);
 
    struct nir_lower_tex_options tex_options = {};
    tex_options.lower_txp = ~0;