Use the data port message "Untyped Atomic Operation" to implement openCL atomic functions.
This message can handle both global memory and SLM.
For all atomic functions, the operands will be treated as unsigned int, except imax/imin.
Only add one opcode Atomic for all atomic functions in ir, and identify the atomic function
by ir::AtomicOps.
Signed-off-by: Yang Rong <rong.r.yang@intel.com>
Reviewed-by: Zhigang Gong <zhigang.gong@linux.intel.com>
}
}
+ void GenContext::emitAtomicInstruction(const SelectionInstruction &insn) {
+ const GenRegister src = ra->genReg(insn.src(0));
+ const GenRegister dst = ra->genReg(insn.dst(0));
+ const uint32_t function = insn.extra.function;
+ const uint32_t bti = insn.extra.elem;
+
+ p->ATOMIC(dst, function, src, bti, insn.srcNum);
+ }
+
void GenContext::emitIndirectMoveInstruction(const SelectionInstruction &insn) {
GenRegister src = ra->genReg(insn.src(0));
if(isScalarReg(src.reg()))
void emitWriteFloat64Instruction(const SelectionInstruction &insn);
void emitUntypedReadInstruction(const SelectionInstruction &insn);
void emitUntypedWriteInstruction(const SelectionInstruction &insn);
+ void emitAtomicInstruction(const SelectionInstruction &insn);
void emitByteGatherInstruction(const SelectionInstruction &insn);
void emitByteScatterInstruction(const SelectionInstruction &insn);
void emitSampleInstruction(const SelectionInstruction &insn);
-/*
+/*
* Copyright © 2012 Intel Corporation
*
* This library is free software; you can redistribute it and/or
Copyright (C) Intel Corp. 2006. All Rights Reserved.
Intel funded Tungsten Graphics (http://www.tungstengraphics.com) to
develop this 3D driver.
-
+
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
-
+
The above copyright notice and this permission notice (including the
next paragraph) shall be included in all copies or substantial
portions of the Software.
-
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
+
**********************************************************************/
/*
* Authors:
GEN_OPCODE_NOP = 126,
};
+#define GEN_ATOMIC_SIMD16 0
+#define GEN_ATOMIC_SIMD8 1
+
+enum GenAtomicOpCode {
+ GEN_ATOMIC_OP_CMPWR8B = 0,
+ GEN_ATOMIC_OP_AND = 1,
+ GEN_ATOMIC_OP_OR = 2,
+ GEN_ATOMIC_OP_XOR = 3,
+ GEN_ATOMIC_OP_MOV = 4,
+ GEN_ATOMIC_OP_INC = 5,
+ GEN_ATOMIC_OP_DEC = 6,
+ GEN_ATOMIC_OP_ADD = 7,
+ GEN_ATOMIC_OP_SUB = 8,
+ GEN_ATOMIC_OP_REVSUB = 9,
+ GEN_ATOMIC_OP_IMAX = 10,
+ GEN_ATOMIC_OP_IMIN = 11,
+ GEN_ATOMIC_OP_UMAX = 12,
+ GEN_ATOMIC_OP_UMIN = 13,
+ GEN_ATOMIC_OP_CMPWR = 14,
+ GEN_ATOMIC_OP_PREDEC = 15
+};
+
/*! Gen SFID */
enum GenMessageTarget {
GEN_SFID_NULL = 0,
/*! Memory fence */
struct {
uint32_t bti:8;
- uint32_t ingored:5;
+ uint32_t pad:5;
uint32_t commit_enable:1;
uint32_t msg_type:4;
uint32_t pad2:1;
uint32_t end_of_thread:1;
} gen7_memory_fence;
+ /*! atomic messages */
+ struct {
+ uint32_t bti:8;
+ uint32_t aop_type:4;
+ uint32_t simd_mode:1;
+ uint32_t return_data:1;
+ uint32_t msg_type:4;
+ uint32_t category:1;
+ uint32_t header_present:1;
+ uint32_t response_length:5;
+ uint32_t msg_length:4;
+ uint32_t pad3:2;
+ uint32_t end_of_thread:1;
+ } gen7_atomic_op;
+
struct {
uint32_t src1_subreg_nr_high:1;
uint32_t src1_reg_nr:8;
response_length);
}
+ void GenEncoder::ATOMIC(GenRegister dst, uint32_t function, GenRegister src, uint32_t bti, uint32_t srcNum) {
+ GenInstruction *insn = this->next(GEN_OPCODE_SEND);
+ uint32_t msg_length = 0;
+ uint32_t response_length = 0;
+
+ if (this->curr.execWidth == 8) {
+ msg_length = srcNum;
+ response_length = 1;
+ } else if (this->curr.execWidth == 16) {
+ msg_length = 2*srcNum;
+ response_length = 2;
+ } else
+ NOT_IMPLEMENTED;
+
+ this->setHeader(insn);
+ this->setDst(insn, GenRegister::uw16grf(dst.nr, 0));
+ this->setSrc0(insn, GenRegister::ud8grf(src.nr, 0));
+ this->setSrc1(insn, GenRegister::immud(0));
+
+ const GenMessageTarget sfid = GEN_SFID_DATAPORT_DATA_CACHE;
+ setMessageDescriptor(this, insn, sfid, msg_length, response_length);
+ insn->bits3.gen7_atomic_op.msg_type = GEN_UNTYPED_ATOMIC_READ;
+ insn->bits3.gen7_atomic_op.bti = bti;
+ insn->bits3.gen7_atomic_op.return_data = 1;
+ insn->bits3.gen7_atomic_op.aop_type = function;
+
+ if (this->curr.execWidth == 8)
+ insn->bits3.gen7_atomic_op.simd_mode = GEN_ATOMIC_SIMD8;
+ else if (this->curr.execWidth == 16)
+ insn->bits3.gen7_atomic_op.simd_mode = GEN_ATOMIC_SIMD16;
+ else
+ NOT_SUPPORTED;
+
+ }
+
GenInstruction *GenEncoder::next(uint32_t opcode) {
GenInstruction insn;
std::memset(&insn, 0, sizeof(GenInstruction));
void NOP(void);
/*! Wait instruction (used for the barrier) */
void WAIT(void);
+ /*! Atomic instructions */
+ void ATOMIC(GenRegister dst, uint32_t function, GenRegister src, uint32_t bti, uint32_t srcNum);
/*! Read 64-bits float arrays */
void READ_FLOAT64(GenRegister dst, GenRegister src, uint32_t bti, uint32_t elemNum);
/*! Write 64-bits float arrays */
DECL_GEN7_SCHEDULE(Sample, 80, 1, 1)
DECL_GEN7_SCHEDULE(TypedWrite, 80, 1, 1)
DECL_GEN7_SCHEDULE(GetImageInfo, 20, 4, 2)
-
+DECL_GEN7_SCHEDULE(Atomic, 80, 1, 1)
bool SelectionInstruction::isRead(void) const {
return this->opcode == SEL_OP_UNTYPED_READ ||
this->opcode == SEL_OP_READ_FLOAT64 ||
+ this->opcode == SEL_OP_ATOMIC ||
this->opcode == SEL_OP_BYTE_GATHER;
}
bool SelectionInstruction::isWrite(void) const {
return this->opcode == SEL_OP_UNTYPED_WRITE ||
this->opcode == SEL_OP_WRITE_FLOAT64 ||
+ this->opcode == SEL_OP_ATOMIC ||
this->opcode == SEL_OP_BYTE_SCATTER;
}
void NOP(void);
/*! Wait instruction (used for the barrier) */
void WAIT(void);
+ /*! Atomic instruction */
+ void ATOMIC(Reg dst, uint32_t function, uint32_t srcNum, Reg src0, Reg src1, Reg src2, uint32_t bti);
/*! Read 64 bits float array */
void READ_FLOAT64(Reg addr, const GenRegister *dst, uint32_t elemNum, uint32_t bti);
/*! Write 64 bits float array */
insn->src(0) = src;
}
+ void Selection::Opaque::ATOMIC(Reg dst, uint32_t function,
+ uint32_t srcNum, Reg src0,
+ Reg src1, Reg src2, uint32_t bti) {
+ SelectionInstruction *insn = this->appendInsn(SEL_OP_ATOMIC, 1, srcNum);
+ insn->dst(0) = dst;
+ insn->src(0) = src0;
+ if(srcNum > 1) insn->src(1) = src1;
+ if(srcNum > 2) insn->src(2) = src2;
+ insn->extra.function = function;
+ insn->extra.elem = bti;
+ SelectionVector *vector = this->appendVector();
+
+ vector->regNum = srcNum;
+ vector->reg = &insn->src(0);
+ vector->isSrc = 1;
+ }
+
void Selection::Opaque::EOT(void) { this->appendInsn(SEL_OP_EOT, 0, 0); }
void Selection::Opaque::NOP(void) { this->appendInsn(SEL_OP_NOP, 0, 0); }
void Selection::Opaque::WAIT(void) { this->appendInsn(SEL_OP_WAIT, 0, 0); }
DECL_CTOR(ConvertInstruction, 1, 1);
};
+ /*! Convert instruction pattern */
+ DECL_PATTERN(AtomicInstruction)
+ {
+ INLINE bool emitOne(Selection::Opaque &sel, const ir::AtomicInstruction &insn) const
+ {
+ using namespace ir;
+ const AtomicOps atomicOp = insn.getAtomicOpcode();
+ const AddressSpace space = insn.getAddressSpace();
+ const uint32_t bti = space == MEM_LOCAL ? 0xfe : 0x01;
+ const uint32_t srcNum = insn.getSrcNum();
+ const GenRegister src0 = sel.selReg(insn.getSrc(0), TYPE_U32); //address
+ GenRegister src1 = src0, src2 = src0;
+ if(srcNum > 1) src1 = sel.selReg(insn.getSrc(1), TYPE_U32);
+ if(srcNum > 2) src2 = sel.selReg(insn.getSrc(2), TYPE_U32);
+ GenRegister dst = sel.selReg(insn.getDst(0), TYPE_U32);
+ GenAtomicOpCode genAtomicOp = (GenAtomicOpCode)atomicOp;
+ sel.ATOMIC(dst, genAtomicOp, srcNum, src0, src1, src2, bti);
+ return true;
+ }
+ DECL_CTOR(AtomicInstruction, 1, 1);
+ };
+
/*! Select instruction pattern */
class SelectInstructionPattern : public SelectionPattern
{
this->insert<SelectInstructionPattern>();
this->insert<CompareInstructionPattern>();
this->insert<ConvertInstructionPattern>();
+ this->insert<AtomicInstructionPattern>();
this->insert<LabelInstructionPattern>();
this->insert<BranchInstructionPattern>();
this->insert<Int32x32MulInstructionPattern>();
GenInstructionState state;
union {
struct {
- /*! Store bti for loads/stores and function for math and compares */
+ /*! Store bti for loads/stores and function for math, atomic and compares */
uint16_t function:8;
- /*! elemSize for byte scatters / gathers, elemNum for untyped msg */
+ /*! elemSize for byte scatters / gathers, elemNum for untyped msg, bti for atomic */
uint16_t elem:8;
};
struct {
DECL_SELECTION_IR(ASR, BinaryInstruction)
DECL_SELECTION_IR(ADD, BinaryInstruction)
DECL_SELECTION_IR(MUL, BinaryInstruction)
+DECL_SELECTION_IR(ATOMIC, AtomicInstruction)
DECL_SELECTION_IR(MACH, BinaryInstruction)
DECL_SELECTION_IR(CMP, CompareInstruction)
DECL_SELECTION_IR(SEL_CMP, CompareInstruction)
-/*
+/*
* Copyright © 2012 Intel Corporation
*
* This library is free software; you can redistribute it and/or
};
/*! For regular n source instructions */
- template <typename T, uint32_t srcNum>
+ template <typename T, uint32_t srcNum>
struct NSrcPolicy {
INLINE uint32_t getSrcNum(void) const { return srcNum; }
INLINE Register getSrc(const Function &fn, uint32_t ID) const {
Type srcType; //!< Type to convert from
};
+ class ALIGNED_INSTRUCTION AtomicInstruction :
+ public BasePolicy,
+ public TupleSrcPolicy<AtomicInstruction>,
+ public NDstPolicy<AtomicInstruction, 1>
+ {
+ public:
+ AtomicInstruction(AtomicOps atomicOp,
+ Register dst,
+ AddressSpace addrSpace,
+ Tuple src)
+ {
+ this->opcode = OP_ATOMIC;
+ this->atomicOp = atomicOp;
+ this->dst[0] = dst;
+ this->src = src;
+ this->addrSpace = addrSpace;
+ srcNum = 2;
+ if((atomicOp == ATOMIC_OP_INC) ||
+ (atomicOp == ATOMIC_OP_DEC))
+ srcNum = 1;
+ if(atomicOp == ATOMIC_OP_CMPXCHG)
+ srcNum = 3;
+ }
+ INLINE AddressSpace getAddressSpace(void) const { return this->addrSpace; }
+ INLINE AtomicOps getAtomicOpcode(void) const { return this->atomicOp; }
+ INLINE bool wellFormed(const Function &fn, std::string &whyNot) const;
+ INLINE void out(std::ostream &out, const Function &fn) const;
+ Register dst[1];
+ Tuple src;
+ AddressSpace addrSpace; //!< Address space
+ uint8_t srcNum:2; //!<Source Number
+ AtomicOps atomicOp:6; //!<Source Number
+ };
+
class ALIGNED_INSTRUCTION BranchInstruction :
public BasePolicy,
public NDstPolicy<BranchInstruction, 0>
return true;
}
+ // We can convert anything to anything, but types and families must match
+ INLINE bool AtomicInstruction::wellFormed(const Function &fn, std::string &whyNot) const
+ {
+ if (UNLIKELY(checkSpecialRegForWrite(dst[0], fn, whyNot) == false))
+ return false;
+ if (UNLIKELY(checkRegisterData(FAMILY_DWORD, dst[0], fn, whyNot) == false))
+ return false;
+ for (uint32_t srcID = 0; srcID < srcNum; ++srcID)
+ if (UNLIKELY(checkRegisterData(FAMILY_DWORD, getSrc(fn, srcID), fn, whyNot) == false))
+ return false;
+
+ return true;
+ }
+
/*! Loads and stores follow the same restrictions */
template <typename T>
INLINE bool wellFormedLoadStore(const T &insn, const Function &fn, std::string &whyNot)
ternaryOrSelectOut(*this, out, fn);
}
+ INLINE void AtomicInstruction::out(std::ostream &out, const Function &fn) const {
+ this->outOpcode(out);
+ out << "." << addrSpace;
+ out << " %" << this->getDst(fn, 0);
+ out << " {" << "%" << this->getSrc(fn, 0) << "}";
+ for (uint32_t i = 1; i < srcNum; ++i)
+ out << " %" << this->getSrc(fn, i);
+ }
+
INLINE void ConvertInstruction::out(std::ostream &out, const Function &fn) const {
this->outOpcode(out);
out << "." << this->getDstType()
#include "ir/instruction.hxx"
END_INTROSPECTION(ConvertInstruction)
+START_INTROSPECTION(AtomicInstruction)
+#include "ir/instruction.hxx"
+END_INTROSPECTION(AtomicInstruction)
+
START_INTROSPECTION(SelectInstruction)
#include "ir/instruction.hxx"
END_INTROSPECTION(SelectInstruction)
}
bool Instruction::hasSideEffect(void) const {
- return opcode == OP_STORE ||
+ return opcode == OP_STORE ||
opcode == OP_TYPED_WRITE ||
- opcode == OP_SYNC;
+ opcode == OP_SYNC ||
+ opcode == OP_ATOMIC;
}
#define DECL_MEM_FN(CLASS, RET, PROTOTYPE, CALL) \
DECL_MEM_FN(CompareInstruction, Type, getType(void), getType())
DECL_MEM_FN(ConvertInstruction, Type, getSrcType(void), getSrcType())
DECL_MEM_FN(ConvertInstruction, Type, getDstType(void), getDstType())
+DECL_MEM_FN(AtomicInstruction, AddressSpace, getAddressSpace(void), getAddressSpace())
+DECL_MEM_FN(AtomicInstruction, AtomicOps, getAtomicOpcode(void), getAtomicOpcode())
DECL_MEM_FN(StoreInstruction, Type, getValueType(void), getValueType())
DECL_MEM_FN(StoreInstruction, uint32_t, getValueNum(void), getValueNum())
DECL_MEM_FN(StoreInstruction, AddressSpace, getAddressSpace(void), getAddressSpace())
return internal::ConvertInstruction(dstType, srcType, dst, src).convert();
}
+ // For all unary functions with given opcode
+ Instruction ATOMIC(AtomicOps atomicOp, Register dst, AddressSpace space, Tuple src) {
+ return internal::AtomicInstruction(atomicOp, dst, space, src).convert();
+ }
+
// BRA
Instruction BRA(LabelIndex labelIndex) {
return internal::BranchInstruction(OP_BRA, labelIndex).convert();
-/*
+/*
* Copyright © 2012 Intel Corporation
*
* This library is free software; you can redistribute it and/or
MEM_INVALID
};
+ enum AtomicOps {
+ ATOMIC_OP_AND = 1,
+ ATOMIC_OP_OR = 2,
+ ATOMIC_OP_XOR = 3,
+ ATOMIC_OP_XCHG = 4,
+ ATOMIC_OP_INC = 5,
+ ATOMIC_OP_DEC = 6,
+ ATOMIC_OP_ADD = 7,
+ ATOMIC_OP_SUB = 8,
+ ATOMIC_OP_IMAX = 10,
+ ATOMIC_OP_IMIN = 11,
+ ATOMIC_OP_UMAX = 12,
+ ATOMIC_OP_UMIN = 13,
+ ATOMIC_OP_CMPXCHG = 14,
+ ATOMIC_OP_INVALID
+ };
+
/* Vote function per hardware thread */
enum VotePredicate : uint8_t {
VOTE_ALL = 0,
static bool isClassOf(const Instruction &insn);
};
+ /*! Atomic instruction */
+ class AtomicInstruction : public Instruction {
+ public:
+ /*! Where the address register goes */
+ static const uint32_t addressIndex = 0;
+ /*! Address space that is manipulated here */
+ AddressSpace getAddressSpace(void) const;
+ /*! Return the atomic function code */
+ AtomicOps getAtomicOpcode(void) const;
+ /*! Return the register that contains the addresses */
+ INLINE Register getAddress(void) const { return this->getSrc(addressIndex); }
+ /*! Return true if the given instruction is an instance of this class */
+ static bool isClassOf(const Instruction &insn);
+ };
+
/*! Store instruction. First source is the address. Next sources are the
* values to store contiguously at the given address
*/
Instruction GT(Type type, Register dst, Register src0, Register src1);
/*! cvt.{dstType <- srcType} dst src */
Instruction CVT(Type dstType, Type srcType, Register dst, Register src);
+ /*! atomic dst addr.space {src1 {src2}} */
+ Instruction ATOMIC(AtomicOps opcode, Register dst, AddressSpace space, Tuple src);
/*! bra labelIndex */
Instruction BRA(LabelIndex labelIndex);
/*! (pred) bra labelIndex */
DECL_INSN(GE, CompareInstruction)
DECL_INSN(GT, CompareInstruction)
DECL_INSN(CVT, ConvertInstruction)
+DECL_INSN(ATOMIC, AtomicInstruction)
DECL_INSN(BRA, BranchInstruction)
DECL_INSN(RET, BranchInstruction)
DECL_INSN(LOADI, LoadImmInstruction)
projects / contrib / beignet.git / commitdiff