void visit(const Block& instr) override;
void visit(const IfInstr& instr) override;
void visit(const ControlFlowInstr& instr) override;
- void visit(const WriteScratchInstr& instr) override;
+ void visit(const ScratchIOInstr& instr) override;
void visit(const StreamOutInstr& instr) override;
void visit(const MemRingOutInstr& instr) override;
void visit(const EmitVertexInstr& instr) override;
}
}
-void AssamblerVisitor::visit(const WriteScratchInstr& instr)
+void AssamblerVisitor::visit(const ScratchIOInstr& instr)
{
clear_states(sf_all);
cf.op = CF_OP_MEM_SCRATCH;
cf.elem_size = 3;
cf.gpr = instr.value().sel();
- cf.mark = 1;
- cf.comp_mask = instr.write_mask();
+ cf.mark = !instr.is_read();
+ cf.comp_mask = instr.is_read() ? 0xf : instr.write_mask();
cf.swizzle_x = 0;
cf.swizzle_y = 1;
cf.swizzle_z = 2;
cf.swizzle_w = 3;
cf.burst_count = 1;
+ assert(!instr.is_read() || m_bc->gfx_level < R700);
+
if (instr.address()) {
- cf.type = 3;
+ cf.type = instr.is_read() || m_bc->gfx_level > R600 ? 3 : 1;
cf.index_gpr = instr.address()->sel();
/* The docu seems to be wrong here: In indirect addressing the
* address_base seems to be the array_size */
cf.array_size = instr.array_size();
} else {
- cf.type = 2;
+ cf.type = instr.is_read() || m_bc->gfx_level > R600 ? 2 : 0;
cf.array_base = instr.location();
}
- /* This should be 0, but the address calculation is apparently wrong */
-
if (r600_bytecode_add_output(m_bc, &cf)){
R600_ERR("shader_from_nir: Error creating SCRATCH_WR assembly instruction\n");
DECLARE_MEMBER(Block);
DECLARE_MEMBER(ControlFlowInstr);
DECLARE_MEMBER(IfInstr);
- DECLARE_MEMBER(WriteScratchInstr);
+ DECLARE_MEMBER(ScratchIOInstr);
DECLARE_MEMBER(StreamOutInstr);
DECLARE_MEMBER(MemRingOutInstr);
DECLARE_MEMBER(EmitVertexInstr);
m_comparer = InstrComparer(&instr);
}
- void visit(const WriteScratchInstr& instr) override {
+ void visit(const ScratchIOInstr& instr) override {
m_comparer = InstrComparer(&instr);
}
class FetchInstr;
class ControlFlowInstr;
class IfInstr;
-class WriteScratchInstr;
+class ScratchIOInstr;
class StreamOutInstr;
class MemRingOutInstr;
class EmitVertexInstr;
virtual void visit(const Block& instr) = 0;
virtual void visit(const ControlFlowInstr& instr) = 0;
virtual void visit(const IfInstr& instr) = 0;
- virtual void visit(const WriteScratchInstr& instr) = 0;
+ virtual void visit(const ScratchIOInstr& instr) = 0;
virtual void visit(const StreamOutInstr& instr) = 0;
virtual void visit(const MemRingOutInstr& instr) = 0;
virtual void visit(const EmitVertexInstr& instr) = 0;
virtual void visit(Block *instr) = 0;
virtual void visit(ControlFlowInstr *instr) = 0;
virtual void visit(IfInstr *instr) = 0;
- virtual void visit(WriteScratchInstr *instr) = 0;
+ virtual void visit(ScratchIOInstr *instr) = 0;
virtual void visit(StreamOutInstr *instr) = 0;
virtual void visit(MemRingOutInstr *instr) = 0;
virtual void visit(EmitVertexInstr *instr) = 0;
void visit(ExportInstr *instr) override {(void)instr;}
void visit(FetchInstr *instr) override {(void)instr;}
void visit(ControlFlowInstr *instr) override {(void)instr;}
- void visit(WriteScratchInstr *instr) override {(void)instr;}
+ void visit(ScratchIOInstr *instr) override {(void)instr;}
void visit(StreamOutInstr *instr) override {(void)instr;}
void visit(MemRingOutInstr *instr) override {(void)instr;}
void visit(EmitVertexInstr *instr) override {(void)instr;}
return new ExportInstr( type, pos, value);
}
-WriteScratchInstr::WriteScratchInstr(const RegisterVec4& value, PRegister addr,
- int align, int align_offset, int writemask, int array_size):
+ScratchIOInstr::ScratchIOInstr(const RegisterVec4& value, PRegister addr,
+ int align, int align_offset, int writemask,
+ int array_size, bool is_read):
WriteOutInstr(value),
m_address(addr),
m_align(align),
m_align_offset(align_offset),
m_writemask(writemask),
- m_array_size(array_size - 1)
+ m_array_size(array_size - 1),
+ m_read(is_read)
{
addr->add_use(this);
+ if (m_read) {
+ for (int i = 0; i < 4; ++i)
+ value[i]->add_parent(this);
+ }
}
-WriteScratchInstr::WriteScratchInstr(const RegisterVec4& value, int loc,
- int align, int align_offset,int writemask):
+ScratchIOInstr::ScratchIOInstr(const RegisterVec4& value, int loc,
+ int align, int align_offset,int writemask,
+ bool is_read):
WriteOutInstr(value),
m_loc(loc),
m_align(align),
m_align_offset(align_offset),
- m_writemask(writemask)
+ m_writemask(writemask),
+ m_read(is_read)
{
+ if (m_read) {
+ for (int i = 0; i < 4; ++i)
+ value[i]->add_parent(this);
+ }
}
-void WriteScratchInstr::accept(ConstInstrVisitor& visitor) const
+void ScratchIOInstr::accept(ConstInstrVisitor& visitor) const
{
visitor.visit(*this);
}
-void WriteScratchInstr::accept(InstrVisitor& visitor)
+void ScratchIOInstr::accept(InstrVisitor& visitor)
{
visitor.visit(this);
}
-bool WriteScratchInstr::is_equal_to(const WriteScratchInstr& lhs) const
+bool ScratchIOInstr::is_equal_to(const ScratchIOInstr& lhs) const
{
if (m_address) {
if (!lhs.m_address)
value().sel() == lhs.value().sel();
}
-bool WriteScratchInstr::do_ready() const
+bool ScratchIOInstr::do_ready() const
{
- return value().ready(block_id(), index()) &&
- (!m_address || m_address->ready(block_id(), index()));
+ bool address_ready = !m_address || m_address->ready(block_id(), index());
+ if (is_read())
+ return address_ready;
+ else
+ return address_ready && value().ready(block_id(), index());
}
-void WriteScratchInstr::do_print(std::ostream& os) const
+void ScratchIOInstr::do_print(std::ostream& os) const
{
char buf[6] = {0};
- os << "WRITE_SCRATCH ";
+ os << (is_read() ? "READ_SCRATCH " : "WRITE_SCRATCH ");
+
+ if (is_read()) {
+ os << (value()[0]->is_ssa() ? " S" : " R")
+ << value().sel() << "." << writemask_to_swizzle(m_writemask, buf)
+ << " ";
+ }
+
if (m_address)
os << "@" << *m_address << "[" << m_array_size + 1<<"]";
else
os << m_loc;
- os << (value()[0]->is_ssa() ? " S" : " R")
- << value().sel() << "." << writemask_to_swizzle(m_writemask, buf)
- << " " << "AL:" << m_align << " ALO:" << m_align_offset;
+ if (!is_read())
+ os << (value()[0]->is_ssa() ? " S" : " R")
+ << value().sel() << "." << writemask_to_swizzle(m_writemask, buf);
+
+ os << " " << "AL:" << m_align << " ALO:" << m_align_offset;
}
-auto WriteScratchInstr::from_string(std::istream& is, ValueFactory &vf) -> Pointer
+auto ScratchIOInstr::from_string(std::istream& is, ValueFactory &vf) -> Pointer
{
string loc_str;
string value_str;
loc_ss >> array_size;
loc_ss >> c;
assert(c == ']');
- return new WriteScratchInstr(value, addr_reg->as_register(), align, align_offset, writemask, array_size);
+ return new ScratchIOInstr(value, addr_reg->as_register(), align, align_offset, writemask, array_size);
} else {
loc_ss >> offset;
- return new WriteScratchInstr(value, offset, align, align_offset, writemask);
+ return new ScratchIOInstr(value, offset, align, align_offset, writemask);
}
}
bool m_is_last;
};
-class WriteScratchInstr : public WriteOutInstr {
+class ScratchIOInstr : public WriteOutInstr {
public:
- WriteScratchInstr(const RegisterVec4& value, PRegister addr,
- int align, int align_offset, int writemask, int array_size);
- WriteScratchInstr(const RegisterVec4& value, int addr, int align, int align_offset,
- int writemask);
+ ScratchIOInstr(const RegisterVec4& value, PRegister addr,
+ int align, int align_offset, int writemask, int array_size,
+ bool is_read = false);
+ ScratchIOInstr(const RegisterVec4& value, int addr, int align, int align_offset,
+ int writemask, bool is_read = false);
void accept(ConstInstrVisitor& visitor) const override;
void accept(InstrVisitor& visitor) override;
- bool is_equal_to(const WriteScratchInstr& lhs) const;
+ bool is_equal_to(const ScratchIOInstr& lhs) const;
unsigned location() const { return m_loc;};
int write_mask() const { return m_writemask;}
auto address() const { return m_address;}
bool indirect() const { return !!m_address;}
int array_size() const { return m_array_size;}
+ bool is_read() const {return m_read; }
static auto from_string(std::istream& is, ValueFactory &vf) -> Pointer;
private:
unsigned m_align_offset;
unsigned m_writemask;
int m_array_size{0};
+ bool m_read{false};
};
class StreamOutInstr: public WriteOutInstr {
} else if (type == "IF") {
result = IfInstr::from_string(is, m_value_factory);
} else if (type == "WRITE_SCRATCH") {
- result = WriteScratchInstr::from_string(is, m_value_factory);
+ result = ScratchIOInstr::from_string(is, m_value_factory);
} else if (type == "MEM_RING") {
result = MemRingOutInstr::from_string(is, m_value_factory);
} else if (type == "EMIT_VERTEX") {
void visit(Block *instr) override;
void visit(ControlFlowInstr *instr) override;
void visit(IfInstr *instr) override;
- void visit(WriteScratchInstr *instr) override;
+ void visit(ScratchIOInstr *instr) override;
void visit(StreamOutInstr *instr) override;
void visit(MemRingOutInstr *instr) override;
void visit(EmitVertexInstr *instr) override {(void)instr;}
sfn_log << SfnLog::merge << "End block\n";
}
-void LiveRangeInstrVisitor::visit(WriteScratchInstr *instr)
+void LiveRangeInstrVisitor::visit(ScratchIOInstr *instr)
{
auto& src = instr->value();
for (int i = 0; i < 4; ++i) {
if ((1 << i) & instr->write_mask()) {
- record_read(src[i], LiveRangeEntry::use_unspecified);
+ if (instr->is_read())
+ record_write(src[i]);
+ else
+ record_read(src[i], LiveRangeEntry::use_unspecified);
}
}
void visit(ControlFlowInstr *instr) override {(void)instr;};
void visit(IfInstr *instr) override {(void)instr;};
- void visit(WriteScratchInstr *instr) override {(void)instr;};
+ void visit(ScratchIOInstr *instr) override {(void)instr;};
void visit(StreamOutInstr *instr) override {(void)instr;};
void visit(MemRingOutInstr *instr) override {(void)instr;};
void visit(EmitVertexInstr *instr) override {(void)instr;};
void visit(Block *instr) override;
void visit(ControlFlowInstr *instr) override {(void)instr;}
void visit(IfInstr *instr) override {(void)instr;}
- void visit(WriteScratchInstr *instr) override {(void)instr;}
+ void visit(ScratchIOInstr *instr) override {(void)instr;}
void visit(StreamOutInstr *instr) override {(void)instr;}
void visit(MemRingOutInstr *instr) override {(void)instr;}
void visit(EmitVertexInstr *instr) override {(void)instr;}
void visit(Block *instr) override;
void visit(ControlFlowInstr *instr) override {(void)instr;}
void visit(IfInstr *instr) override {(void)instr;}
- void visit(WriteScratchInstr *instr) override {(void)instr;}
+ void visit(ScratchIOInstr *instr) override {(void)instr;}
void visit(StreamOutInstr *instr) override {(void)instr;}
void visit(MemRingOutInstr *instr) override {(void)instr;}
void visit(EmitVertexInstr *instr) override {(void)instr;}
void visit(Block *instr) override;
void visit(ControlFlowInstr *instr) override;
void visit(IfInstr *instr) override;
- void visit(WriteScratchInstr *instr) override;
+ void visit(ScratchIOInstr *instr) override;
void visit(StreamOutInstr *instr) override;
void visit(MemRingOutInstr *instr) override;
void visit(EmitVertexInstr *instr) override {(void)instr;}
}
}
-void SimplifySourceVecVisitor::visit(WriteScratchInstr *instr)
+void SimplifySourceVecVisitor::visit(ScratchIOInstr *instr)
{
(void) instr;
}
void visit(Block *instr) override;
void visit(ControlFlowInstr *instr) override {(void)instr;}
void visit(IfInstr *instr) override;
- void visit(WriteScratchInstr *instr) override {(void)instr;}
+ void visit(ScratchIOInstr *instr) override {(void)instr;}
void visit(StreamOutInstr *instr) override {(void)instr;}
void visit(MemRingOutInstr *instr) override {(void)instr;}
void visit(EmitVertexInstr *instr) override {(void)instr;}
m_cf_instr = instr;
}
- void visit(WriteScratchInstr *instr) override {
+ void visit(ScratchIOInstr *instr) override {
mem_write_instr.push_back(instr);
}
int align = nir_intrinsic_align_mul(intr);
int align_offset = nir_intrinsic_align_offset(intr);
- WriteScratchInstr *ws_ir = nullptr;
+ ScratchIOInstr *ws_ir = nullptr;
int offset = -1;
if (address->as_literal()) {
}
if (offset >= 0) {
- ws_ir = new WriteScratchInstr(value, offset, align, align_offset, writemask);
+ ws_ir = new ScratchIOInstr(value, offset, align, align_offset, writemask);
} else {
auto addr_temp = vf.temp_register(0);
auto load_addr = new AluInstr(op1_mov, addr_temp, address, AluInstr::last_write);
load_addr->set_alu_flag(alu_no_schedule_bias);
emit_instruction(load_addr);
- ws_ir = new WriteScratchInstr(value, addr_temp, align, align_offset, writemask, m_scratch_size);
+ ws_ir = new ScratchIOInstr(value, addr_temp, align, align_offset, writemask, m_scratch_size);
}
emit_instruction(ws_ir);
bool Shader::emit_load_scratch(nir_intrinsic_instr *intr)
{
auto addr = value_factory().src(intr->src[0], 0);
+ auto dest = value_factory().dest_vec4(intr->dest, pin_group);
- RegisterVec4::Swizzle dest_swz = {7,7,7,7};
+ if (chip_class() >= ISA_CC_R700) {
+ RegisterVec4::Swizzle dest_swz = {7,7,7,7};
- for (unsigned i = 0; i < intr->num_components; ++i)
- dest_swz[i] = i;
+ for (unsigned i = 0; i < intr->num_components; ++i)
+ dest_swz[i] = i;
- auto dest = value_factory().dest_vec4(intr->dest, pin_group);
+ auto *ir = new LoadFromScratch(dest, dest_swz, addr, m_scratch_size);
+ emit_instruction(ir);
+ chain_scratch_read(ir);
+ } else {
+ int align = nir_intrinsic_align_mul(intr);
+ int align_offset = nir_intrinsic_align_offset(intr);
+
+
+ int offset = -1;
+ if (addr->as_literal()) {
+ offset = addr->as_literal()->value();
+ } else if (addr->as_inline_const()) {
+ auto il = addr->as_inline_const();
+ if (il->sel() == ALU_SRC_0)
+ offset = 0;
+ else if (il->sel() == ALU_SRC_1_INT)
+ offset = 1;
+ }
- auto ir = new LoadFromScratch(dest, dest_swz, addr, m_scratch_size);
- emit_instruction(ir);
+ ScratchIOInstr *ir = nullptr;
+ if (offset >= 0) {
+ ir = new ScratchIOInstr(dest, offset, align, align_offset, 0xf, true);
+ } else {
+ auto addr_temp = value_factory().temp_register(0);
+ auto load_addr = new AluInstr(op1_mov, addr_temp, addr, AluInstr::last_write);
+ load_addr->set_alu_flag(alu_no_schedule_bias);
+ emit_instruction(load_addr);
+
+ ir = new ScratchIOInstr(dest, addr_temp, align, align_offset, 0xf,
+ m_scratch_size, true);
+ }
+ emit_instruction(ir);
+ }
- chain_scratch_read(ir);
m_flags.set(sh_needs_scratch_space);
return true;
}
-void Shader::InstructionChain::visit(WriteScratchInstr *instr)
+void Shader::InstructionChain::visit(ScratchIOInstr *instr)
{
apply(instr, &last_scratch_instr);
}
void visit(LDSAtomicInstr *instr) override {(void) instr;}
void visit(LDSReadInstr *instr) override {(void) instr;}
- void visit(WriteScratchInstr *instr) override;
+ void visit(ScratchIOInstr *instr) override;
void visit(GDSInstr *instr) override;
void visit(RatInstr *instr) override;
{
add_dest_vec4_from_string("R1.xyzw");
string init = "WRITE_SCRATCH 20 R1.xyzw AL:4 ALO:16";
- WriteScratchInstr expect(RegisterVec4(1), 20, 4, 16, 0xf);
+ ScratchIOInstr expect(RegisterVec4(1), 20, 4, 16, 0xf);
check(init, expect);
add_dest_vec4_from_string("R2.xyzw");
string init2 = "WRITE_SCRATCH 10 R2.xy_w AL:8 ALO:8";
- WriteScratchInstr expect2(RegisterVec4(2), 10, 8, 8, 0xb);
+ ScratchIOInstr expect2(RegisterVec4(2), 10, 8, 8, 0xb);
check(init2, expect2);
}
add_dest_vec4_from_string("R1.xyzw");
add_dest_from_string("R3.x");
string init = "WRITE_SCRATCH @R3.x[10] R1.xyzw AL:4 ALO:16";
- WriteScratchInstr expect(RegisterVec4(1), new Register(3, 0, pin_none), 4, 16, 0xf, 10);
+ ScratchIOInstr expect(RegisterVec4(1), new Register(3, 0, pin_none), 4, 16, 0xf, 10);
check(init, expect);
add_dest_vec4_from_string("R2.xyzw");
add_dest_from_string("R4.x");
string init2 = "WRITE_SCRATCH @R4.x[20] R2.xy__ AL:4 ALO:16";
- WriteScratchInstr expect2(RegisterVec4(2), new Register(4, 0, pin_none), 4, 16, 0x3, 20);
+ ScratchIOInstr expect2(RegisterVec4(2), new Register(4, 0, pin_none), 4, 16, 0x3, 20);
check(init2, expect2);