// TargetLowering Implementation
//===----------------------------------------------------------------------===//
+TargetLowering::AtomicExpansionKind SparcTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
+ if (AI->getOperation() == AtomicRMWInst::Xchg &&
+ AI->getType()->getPrimitiveSizeInBits() == 32)
+ return AtomicExpansionKind::None; // Uses xchg instruction
+
+ return AtomicExpansionKind::CmpXChg;
+}
+
/// IntCondCCodeToICC - Convert a DAG integer condition code to a SPARC ICC
/// condition.
static SPCC::CondCodes IntCondCCodeToICC(ISD::CondCode CC) {
case SP::SELECT_CC_DFP_FCC:
case SP::SELECT_CC_QFP_FCC:
return expandSelectCC(MI, BB, SP::FBCOND);
-
- case SP::ATOMIC_LOAD_ADD_32:
- return expandAtomicRMW(MI, BB, SP::ADDrr);
- case SP::ATOMIC_LOAD_ADD_64:
- return expandAtomicRMW(MI, BB, SP::ADDXrr);
- case SP::ATOMIC_LOAD_SUB_32:
- return expandAtomicRMW(MI, BB, SP::SUBrr);
- case SP::ATOMIC_LOAD_SUB_64:
- return expandAtomicRMW(MI, BB, SP::SUBXrr);
- case SP::ATOMIC_LOAD_AND_32:
- return expandAtomicRMW(MI, BB, SP::ANDrr);
- case SP::ATOMIC_LOAD_AND_64:
- return expandAtomicRMW(MI, BB, SP::ANDXrr);
- case SP::ATOMIC_LOAD_OR_32:
- return expandAtomicRMW(MI, BB, SP::ORrr);
- case SP::ATOMIC_LOAD_OR_64:
- return expandAtomicRMW(MI, BB, SP::ORXrr);
- case SP::ATOMIC_LOAD_XOR_32:
- return expandAtomicRMW(MI, BB, SP::XORrr);
- case SP::ATOMIC_LOAD_XOR_64:
- return expandAtomicRMW(MI, BB, SP::XORXrr);
- case SP::ATOMIC_LOAD_NAND_32:
- return expandAtomicRMW(MI, BB, SP::ANDrr);
- case SP::ATOMIC_LOAD_NAND_64:
- return expandAtomicRMW(MI, BB, SP::ANDXrr);
-
- case SP::ATOMIC_SWAP_64:
- return expandAtomicRMW(MI, BB, 0);
-
- case SP::ATOMIC_LOAD_MAX_32:
- return expandAtomicRMW(MI, BB, SP::MOVICCrr, SPCC::ICC_G);
- case SP::ATOMIC_LOAD_MAX_64:
- return expandAtomicRMW(MI, BB, SP::MOVXCCrr, SPCC::ICC_G);
- case SP::ATOMIC_LOAD_MIN_32:
- return expandAtomicRMW(MI, BB, SP::MOVICCrr, SPCC::ICC_LE);
- case SP::ATOMIC_LOAD_MIN_64:
- return expandAtomicRMW(MI, BB, SP::MOVXCCrr, SPCC::ICC_LE);
- case SP::ATOMIC_LOAD_UMAX_32:
- return expandAtomicRMW(MI, BB, SP::MOVICCrr, SPCC::ICC_GU);
- case SP::ATOMIC_LOAD_UMAX_64:
- return expandAtomicRMW(MI, BB, SP::MOVXCCrr, SPCC::ICC_GU);
- case SP::ATOMIC_LOAD_UMIN_32:
- return expandAtomicRMW(MI, BB, SP::MOVICCrr, SPCC::ICC_LEU);
- case SP::ATOMIC_LOAD_UMIN_64:
- return expandAtomicRMW(MI, BB, SP::MOVXCCrr, SPCC::ICC_LEU);
}
}
return BB;
}
-MachineBasicBlock*
-SparcTargetLowering::expandAtomicRMW(MachineInstr *MI,
- MachineBasicBlock *MBB,
- unsigned Opcode,
- unsigned CondCode) const {
- const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
- MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
- DebugLoc DL = MI->getDebugLoc();
-
- // MI is an atomic read-modify-write instruction of the form:
- //
- // rd = atomicrmw<op> addr, rs2
- //
- // All three operands are registers.
- unsigned DestReg = MI->getOperand(0).getReg();
- unsigned AddrReg = MI->getOperand(1).getReg();
- unsigned Rs2Reg = MI->getOperand(2).getReg();
-
- // SelectionDAG has already inserted memory barriers before and after MI, so
- // we simply have to implement the operatiuon in terms of compare-and-swap.
- //
- // %val0 = load %addr
- // loop:
- // %val = phi %val0, %dest
- // %upd = op %val, %rs2
- // %dest = cas %addr, %val, %upd
- // cmp %val, %dest
- // bne loop
- // done:
- //
- bool is64Bit = SP::I64RegsRegClass.hasSubClassEq(MRI.getRegClass(DestReg));
- const TargetRegisterClass *ValueRC =
- is64Bit ? &SP::I64RegsRegClass : &SP::IntRegsRegClass;
- unsigned Val0Reg = MRI.createVirtualRegister(ValueRC);
-
- BuildMI(*MBB, MI, DL, TII.get(is64Bit ? SP::LDXri : SP::LDri), Val0Reg)
- .addReg(AddrReg).addImm(0);
-
- // Split the basic block MBB before MI and insert the loop block in the hole.
- MachineFunction::iterator MFI = MBB->getIterator();
- const BasicBlock *LLVM_BB = MBB->getBasicBlock();
- MachineFunction *MF = MBB->getParent();
- MachineBasicBlock *LoopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
- MachineBasicBlock *DoneMBB = MF->CreateMachineBasicBlock(LLVM_BB);
- ++MFI;
- MF->insert(MFI, LoopMBB);
- MF->insert(MFI, DoneMBB);
-
- // Move MI and following instructions to DoneMBB.
- DoneMBB->splice(DoneMBB->begin(), MBB, MI, MBB->end());
- DoneMBB->transferSuccessorsAndUpdatePHIs(MBB);
-
- // Connect the CFG again.
- MBB->addSuccessor(LoopMBB);
- LoopMBB->addSuccessor(LoopMBB);
- LoopMBB->addSuccessor(DoneMBB);
-
- // Build the loop block.
- unsigned ValReg = MRI.createVirtualRegister(ValueRC);
- // Opcode == 0 means try to write Rs2Reg directly (ATOMIC_SWAP).
- unsigned UpdReg = (Opcode ? MRI.createVirtualRegister(ValueRC) : Rs2Reg);
-
- BuildMI(LoopMBB, DL, TII.get(SP::PHI), ValReg)
- .addReg(Val0Reg).addMBB(MBB)
- .addReg(DestReg).addMBB(LoopMBB);
-
- if (CondCode) {
- // This is one of the min/max operations. We need a CMPrr followed by a
- // MOVXCC/MOVICC.
- BuildMI(LoopMBB, DL, TII.get(SP::CMPrr)).addReg(ValReg).addReg(Rs2Reg);
- BuildMI(LoopMBB, DL, TII.get(Opcode), UpdReg)
- .addReg(ValReg).addReg(Rs2Reg).addImm(CondCode);
- } else if (Opcode) {
- BuildMI(LoopMBB, DL, TII.get(Opcode), UpdReg)
- .addReg(ValReg).addReg(Rs2Reg);
- }
-
- if (MI->getOpcode() == SP::ATOMIC_LOAD_NAND_32 ||
- MI->getOpcode() == SP::ATOMIC_LOAD_NAND_64) {
- unsigned TmpReg = UpdReg;
- UpdReg = MRI.createVirtualRegister(ValueRC);
- BuildMI(LoopMBB, DL, TII.get(SP::XORri), UpdReg).addReg(TmpReg).addImm(-1);
- }
-
- BuildMI(LoopMBB, DL, TII.get(is64Bit ? SP::CASXrr : SP::CASrr), DestReg)
- .addReg(AddrReg).addReg(ValReg).addReg(UpdReg)
- .setMemRefs(MI->memoperands_begin(), MI->memoperands_end());
- BuildMI(LoopMBB, DL, TII.get(SP::CMPrr)).addReg(ValReg).addReg(DestReg);
- BuildMI(LoopMBB, DL, TII.get(is64Bit ? SP::BPXCC : SP::BCOND))
- .addMBB(LoopMBB).addImm(SPCC::ICC_NE);
-
- MI->eraseFromParent();
- return DoneMBB;
-}
-
//===----------------------------------------------------------------------===//
// Sparc Inline Assembly Support
//===----------------------------------------------------------------------===//
} // Predicates = [Is64Bit]
-let usesCustomInserter = 1, hasCtrlDep = 1, mayLoad = 1, mayStore = 1,
- Defs = [ICC] in
-multiclass AtomicRMW<SDPatternOperator op32, SDPatternOperator op64> {
-
- def _32 : Pseudo<(outs IntRegs:$rd),
- (ins ptr_rc:$addr, IntRegs:$rs2), "",
- [(set i32:$rd, (op32 iPTR:$addr, i32:$rs2))]>;
-
- let Predicates = [Is64Bit] in
- def _64 : Pseudo<(outs I64Regs:$rd),
- (ins ptr_rc:$addr, I64Regs:$rs2), "",
- [(set i64:$rd, (op64 iPTR:$addr, i64:$rs2))]>;
-}
-
-defm ATOMIC_LOAD_ADD : AtomicRMW<atomic_load_add_32, atomic_load_add_64>;
-defm ATOMIC_LOAD_SUB : AtomicRMW<atomic_load_sub_32, atomic_load_sub_64>;
-defm ATOMIC_LOAD_AND : AtomicRMW<atomic_load_and_32, atomic_load_and_64>;
-defm ATOMIC_LOAD_OR : AtomicRMW<atomic_load_or_32, atomic_load_or_64>;
-defm ATOMIC_LOAD_XOR : AtomicRMW<atomic_load_xor_32, atomic_load_xor_64>;
-defm ATOMIC_LOAD_NAND : AtomicRMW<atomic_load_nand_32, atomic_load_nand_64>;
-defm ATOMIC_LOAD_MIN : AtomicRMW<atomic_load_min_32, atomic_load_min_64>;
-defm ATOMIC_LOAD_MAX : AtomicRMW<atomic_load_max_32, atomic_load_max_64>;
-defm ATOMIC_LOAD_UMIN : AtomicRMW<atomic_load_umin_32, atomic_load_umin_64>;
-defm ATOMIC_LOAD_UMAX : AtomicRMW<atomic_load_umax_32, atomic_load_umax_64>;
-
-// There is no 64-bit variant of SWAP, so use a pseudo.
-let usesCustomInserter = 1, hasCtrlDep = 1, mayLoad = 1, mayStore = 1,
- Defs = [ICC], Predicates = [Is64Bit] in
-def ATOMIC_SWAP_64 : Pseudo<(outs I64Regs:$rd),
- (ins ptr_rc:$addr, I64Regs:$rs2), "",
- [(set i64:$rd,
- (atomic_swap_64 iPTR:$addr, i64:$rs2))]>;
-
let Predicates = [Is64Bit], hasSideEffects = 1, Uses = [ICC], cc = 0b10 in
defm TXCC : TRAP<"%xcc">;