}
static RatInstr::ERatOp
-get_rat_opcode(const nir_intrinsic_op opcode, pipe_format format)
+get_rat_opcode(const nir_atomic_op opcode)
{
switch (opcode) {
- case nir_intrinsic_image_load:
- return RatInstr::NOP_RTN;
- case nir_intrinsic_ssbo_atomic_add:
- case nir_intrinsic_image_atomic_add:
+ case nir_atomic_op_iadd:
return RatInstr::ADD_RTN;
- case nir_intrinsic_ssbo_atomic_and:
- case nir_intrinsic_image_atomic_and:
+ case nir_atomic_op_iand:
return RatInstr::AND_RTN;
- case nir_intrinsic_ssbo_atomic_or:
- case nir_intrinsic_image_atomic_or:
+ case nir_atomic_op_ior:
return RatInstr::OR_RTN;
- case nir_intrinsic_ssbo_atomic_imin:
- case nir_intrinsic_image_atomic_imin:
+ case nir_atomic_op_imin:
return RatInstr::MIN_INT_RTN;
- case nir_intrinsic_ssbo_atomic_imax:
- case nir_intrinsic_image_atomic_imax:
+ case nir_atomic_op_imax:
return RatInstr::MAX_INT_RTN;
- case nir_intrinsic_ssbo_atomic_umin:
- case nir_intrinsic_image_atomic_umin:
+ case nir_atomic_op_umin:
return RatInstr::MIN_UINT_RTN;
- case nir_intrinsic_ssbo_atomic_umax:
- case nir_intrinsic_image_atomic_umax:
+ case nir_atomic_op_umax:
return RatInstr::MAX_UINT_RTN;
- case nir_intrinsic_ssbo_atomic_xor:
- case nir_intrinsic_image_atomic_xor:
+ case nir_atomic_op_ixor:
return RatInstr::XOR_RTN;
- case nir_intrinsic_ssbo_atomic_comp_swap:
- case nir_intrinsic_image_atomic_comp_swap:
- if (util_format_is_float(format))
- return RatInstr::CMPXCHG_FLT_RTN;
- else
- return RatInstr::CMPXCHG_INT_RTN;
- case nir_intrinsic_ssbo_atomic_exchange:
- case nir_intrinsic_image_atomic_exchange:
+ case nir_atomic_op_cmpxchg:
+ return RatInstr::CMPXCHG_INT_RTN;
+ case nir_atomic_op_xchg:
return RatInstr::XCHG_RTN;
default:
- unreachable("Unsupported WO RAT instruction");
+ unreachable("Unsupported atomic");
}
}
static RatInstr::ERatOp
-get_rat_opcode_wo(const nir_intrinsic_op opcode, pipe_format format)
+get_rat_opcode_wo(const nir_atomic_op opcode)
{
switch (opcode) {
- case nir_intrinsic_ssbo_atomic_add:
- case nir_intrinsic_image_atomic_add:
+ case nir_atomic_op_iadd:
return RatInstr::ADD;
- case nir_intrinsic_ssbo_atomic_and:
- case nir_intrinsic_image_atomic_and:
+ case nir_atomic_op_iand:
return RatInstr::AND;
- case nir_intrinsic_ssbo_atomic_or:
- case nir_intrinsic_image_atomic_or:
+ case nir_atomic_op_ior:
return RatInstr::OR;
- case nir_intrinsic_ssbo_atomic_imin:
- case nir_intrinsic_image_atomic_imin:
+ case nir_atomic_op_imin:
return RatInstr::MIN_INT;
- case nir_intrinsic_ssbo_atomic_imax:
- case nir_intrinsic_image_atomic_imax:
+ case nir_atomic_op_imax:
return RatInstr::MAX_INT;
- case nir_intrinsic_ssbo_atomic_umin:
- case nir_intrinsic_image_atomic_umin:
+ case nir_atomic_op_umin:
return RatInstr::MIN_UINT;
- case nir_intrinsic_ssbo_atomic_umax:
- case nir_intrinsic_image_atomic_umax:
+ case nir_atomic_op_umax:
return RatInstr::MAX_UINT;
- case nir_intrinsic_ssbo_atomic_xor:
- case nir_intrinsic_image_atomic_xor:
+ case nir_atomic_op_ixor:
return RatInstr::XOR;
- case nir_intrinsic_ssbo_atomic_comp_swap:
- case nir_intrinsic_image_atomic_comp_swap:
- if (util_format_is_float(format))
- return RatInstr::CMPXCHG_FLT;
- else
- return RatInstr::CMPXCHG_INT;
- case nir_intrinsic_ssbo_atomic_exchange:
- case nir_intrinsic_image_atomic_exchange:
+ case nir_atomic_op_cmpxchg:
+ return RatInstr::CMPXCHG_INT;
+ case nir_atomic_op_xchg:
return RatInstr::XCHG_RTN;
default:
- unreachable("Unsupported WO RAT instruction");
+ unreachable("Unsupported atomic");
}
}
return emit_ssbo_load(intr, shader);
case nir_intrinsic_store_ssbo:
return emit_ssbo_store(intr, shader);
- case nir_intrinsic_ssbo_atomic_add:
- case nir_intrinsic_ssbo_atomic_comp_swap:
- case nir_intrinsic_ssbo_atomic_or:
- case nir_intrinsic_ssbo_atomic_xor:
- case nir_intrinsic_ssbo_atomic_imax:
- case nir_intrinsic_ssbo_atomic_imin:
- case nir_intrinsic_ssbo_atomic_umax:
- case nir_intrinsic_ssbo_atomic_umin:
- case nir_intrinsic_ssbo_atomic_and:
- case nir_intrinsic_ssbo_atomic_exchange:
+ case nir_intrinsic_ssbo_atomic:
+ case nir_intrinsic_ssbo_atomic_swap:
return emit_ssbo_atomic_op(intr, shader);
case nir_intrinsic_image_store:
return emit_image_store(intr, shader);
case nir_intrinsic_image_load:
- case nir_intrinsic_image_atomic_add:
- case nir_intrinsic_image_atomic_and:
- case nir_intrinsic_image_atomic_or:
- case nir_intrinsic_image_atomic_xor:
- case nir_intrinsic_image_atomic_exchange:
- case nir_intrinsic_image_atomic_comp_swap:
- case nir_intrinsic_image_atomic_umin:
- case nir_intrinsic_image_atomic_umax:
- case nir_intrinsic_image_atomic_imin:
- case nir_intrinsic_image_atomic_imax:
+ case nir_intrinsic_image_atomic:
+ case nir_intrinsic_image_atomic_swap:
return emit_image_load_or_atomic(intr, shader);
case nir_intrinsic_image_size:
return emit_image_size(intr, shader);
}
bool read_result = !intr->dest.is_ssa || !list_is_empty(&intr->dest.ssa.uses);
- auto opcode = read_result ? get_rat_opcode(intr->intrinsic, PIPE_FORMAT_R32_UINT)
- : get_rat_opcode_wo(intr->intrinsic, PIPE_FORMAT_R32_UINT);
+ auto opcode = read_result ? get_rat_opcode(nir_intrinsic_atomic_op(intr))
+ : get_rat_opcode_wo(nir_intrinsic_atomic_op(intr));
auto coord_orig = vf.src(intr->src[1], 0);
auto coord = vf.temp_register(0);
shader.emit_instruction(
new AluInstr(op1_mov, data_vec4[1], shader.rat_return_address(), AluInstr::write));
- if (intr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap) {
+ if (intr->intrinsic == nir_intrinsic_ssbo_atomic_swap) {
shader.emit_instruction(
new AluInstr(op1_mov, data_vec4[0], vf.src(intr->src[3], 0), AluInstr::write));
shader.emit_instruction(
}
bool read_result = !intrin->dest.is_ssa || !list_is_empty(&intrin->dest.ssa.uses);
- auto opcode = read_result ? get_rat_opcode(intrin->intrinsic, PIPE_FORMAT_R32_UINT)
- : get_rat_opcode_wo(intrin->intrinsic, PIPE_FORMAT_R32_UINT);
+ bool image_load = (intrin->intrinsic == nir_intrinsic_image_load);
+ auto opcode = image_load ? RatInstr::NOP_RTN :
+ read_result ? get_rat_opcode(nir_intrinsic_atomic_op(intrin))
+ : get_rat_opcode_wo(nir_intrinsic_atomic_op(intrin));
auto coord_orig = vf.src_vec4(intrin->src[1], pin_chan);
auto coord = vf.temp_vec4(pin_chgr);
shader.emit_instruction(
new AluInstr(op1_mov, data_vec4[1], shader.rat_return_address(), AluInstr::write));
- if (intrin->intrinsic == nir_intrinsic_image_atomic_comp_swap) {
+ if (intrin->intrinsic == nir_intrinsic_image_atomic_swap) {
shader.emit_instruction(
new AluInstr(op1_mov, data_vec4[0], vf.src(intrin->src[4], 0), AluInstr::write));
shader.emit_instruction(
// handle unhandled instructions
switch (intr->intrinsic) {
- case nir_intrinsic_ssbo_atomic_add:
- case nir_intrinsic_ssbo_atomic_comp_swap:
- case nir_intrinsic_ssbo_atomic_or:
- case nir_intrinsic_ssbo_atomic_xor:
- case nir_intrinsic_ssbo_atomic_imax:
- case nir_intrinsic_ssbo_atomic_imin:
- case nir_intrinsic_ssbo_atomic_umax:
- case nir_intrinsic_ssbo_atomic_umin:
- case nir_intrinsic_ssbo_atomic_and:
- case nir_intrinsic_ssbo_atomic_exchange:
+ case nir_intrinsic_ssbo_atomic:
+ case nir_intrinsic_ssbo_atomic_swap:
case nir_intrinsic_image_load:
- case nir_intrinsic_image_atomic_add:
- case nir_intrinsic_image_atomic_and:
- case nir_intrinsic_image_atomic_or:
- case nir_intrinsic_image_atomic_xor:
- case nir_intrinsic_image_atomic_exchange:
- case nir_intrinsic_image_atomic_comp_swap:
- case nir_intrinsic_image_atomic_umin:
- case nir_intrinsic_image_atomic_umax:
- case nir_intrinsic_image_atomic_imin:
- case nir_intrinsic_image_atomic_imax:
+ case nir_intrinsic_image_atomic:
+ case nir_intrinsic_image_atomic_swap:
m_flags.set(sh_needs_sbo_ret_address);
FALLTHROUGH;
case nir_intrinsic_image_store:
case nir_intrinsic_memory_barrier:
return emit_wait_ack();
- case nir_intrinsic_shared_atomic_add:
- case nir_intrinsic_shared_atomic_and:
- case nir_intrinsic_shared_atomic_or:
- case nir_intrinsic_shared_atomic_imax:
- case nir_intrinsic_shared_atomic_umax:
- case nir_intrinsic_shared_atomic_imin:
- case nir_intrinsic_shared_atomic_umin:
- case nir_intrinsic_shared_atomic_xor:
- case nir_intrinsic_shared_atomic_exchange:
- case nir_intrinsic_shared_atomic_comp_swap:
+ case nir_intrinsic_shared_atomic:
+ case nir_intrinsic_shared_atomic_swap:
return emit_atomic_local_shared(intr);
case nir_intrinsic_shader_clock:
return emit_shader_clock(intr);
}
static ESDOp
-lds_op_from_intrinsic(nir_intrinsic_op op, bool ret)
+lds_op_from_intrinsic(nir_atomic_op op, bool ret)
{
switch (op) {
- case nir_intrinsic_shared_atomic_add:
+ case nir_atomic_op_iadd:
return ret ? LDS_ADD_RET : LDS_ADD;
- case nir_intrinsic_shared_atomic_and:
+ case nir_atomic_op_iand:
return ret ? LDS_AND_RET : LDS_AND;
- case nir_intrinsic_shared_atomic_or:
+ case nir_atomic_op_ior:
return ret ? LDS_OR_RET : LDS_OR;
- case nir_intrinsic_shared_atomic_imax:
+ case nir_atomic_op_imax:
return ret ? LDS_MAX_INT_RET : LDS_MAX_INT;
- case nir_intrinsic_shared_atomic_umax:
+ case nir_atomic_op_umax:
return ret ? LDS_MAX_UINT_RET : LDS_MAX_UINT;
- case nir_intrinsic_shared_atomic_imin:
+ case nir_atomic_op_imin:
return ret ? LDS_MIN_INT_RET : LDS_MIN_INT;
- case nir_intrinsic_shared_atomic_umin:
+ case nir_atomic_op_umin:
return ret ? LDS_MIN_UINT_RET : LDS_MIN_UINT;
- case nir_intrinsic_shared_atomic_xor:
+ case nir_atomic_op_ixor:
return ret ? LDS_XOR_RET : LDS_XOR;
- case nir_intrinsic_shared_atomic_exchange:
+ case nir_atomic_op_xchg:
return LDS_XCHG_RET;
- case nir_intrinsic_shared_atomic_comp_swap:
+ case nir_atomic_op_cmpxchg:
return LDS_CMP_XCHG_RET;
default:
- unreachable("Unsupported shared atomic opcode");
+ unreachable("Unsupported shared atomic_op opcode");
}
}
auto dest_value = uses_retval ? vf.dest(instr->dest, 0, pin_free) : nullptr;
- auto op = lds_op_from_intrinsic(instr->intrinsic, uses_retval);
+ auto op = lds_op_from_intrinsic(nir_intrinsic_atomic_op(instr), uses_retval);
auto address = vf.src(instr->src[0], 0);
AluInstr::SrcValues src;
src.push_back(vf.src(instr->src[1], 0));
- if (unlikely(instr->intrinsic == nir_intrinsic_shared_atomic_comp_swap))
+ if (unlikely(instr->intrinsic == nir_intrinsic_shared_atomic_swap))
src.push_back(vf.src(instr->src[2], 0));
emit_instruction(new LDSAtomicInstr(op, dest_value, address, src));
return true;