From 24539f1ef2471d07bd87f833cb0288fc0f251f4b Mon Sep 17 00:00:00 2001 From: James Y Knight Date: Mon, 8 Feb 2021 23:07:12 -0500 Subject: [PATCH] Add Alignment argument to IRBuilder CreateAtomicRMW and CreateAtomicCmpXchg. And then push those change throughout LLVM. Keep the old signature in Clang's CGBuilder for now -- that will be updated in a follow-on patch (D97224). The MLIR LLVM-IR dialect is not updated to support the new alignment attribute, but preserves its existing behavior. Differential Revision: https://reviews.llvm.org/D97223 --- clang/lib/CodeGen/CGBuilder.h | 22 ++++ llvm/include/llvm/CodeGen/AtomicExpandUtils.h | 4 +- llvm/include/llvm/IR/IRBuilder.h | 31 +++-- llvm/lib/CodeGen/AtomicExpandPass.cpp | 130 +++++++++++---------- llvm/lib/IR/AutoUpgrade.cpp | 2 +- llvm/lib/IR/Core.cpp | 16 +-- .../Transforms/Instrumentation/GCOVProfiling.cpp | 2 +- .../Transforms/Instrumentation/InstrOrderFile.cpp | 2 +- .../Transforms/Instrumentation/InstrProfiling.cpp | 3 +- mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td | 5 +- 10 files changed, 132 insertions(+), 85 deletions(-) diff --git a/clang/lib/CodeGen/CGBuilder.h b/clang/lib/CodeGen/CGBuilder.h index 38e96c0..4cdf9fc 100644 --- a/clang/lib/CodeGen/CGBuilder.h +++ b/clang/lib/CodeGen/CGBuilder.h @@ -132,6 +132,28 @@ public: return CreateAlignedStore(getInt1(Value), Addr, CharUnits::One()); } + // Temporarily use old signature; clang will be updated to an Address overload + // in a subsequent patch. + llvm::AtomicCmpXchgInst * + CreateAtomicCmpXchg(llvm::Value *Ptr, llvm::Value *Cmp, llvm::Value *New, + llvm::AtomicOrdering SuccessOrdering, + llvm::AtomicOrdering FailureOrdering, + llvm::SyncScope::ID SSID = llvm::SyncScope::System) { + return CGBuilderBaseTy::CreateAtomicCmpXchg( + Ptr, Cmp, New, llvm::MaybeAlign(), SuccessOrdering, FailureOrdering, + SSID); + } + + // Temporarily use old signature; clang will be updated to an Address overload + // in a subsequent patch. + llvm::AtomicRMWInst * + CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, llvm::Value *Ptr, + llvm::Value *Val, llvm::AtomicOrdering Ordering, + llvm::SyncScope::ID SSID = llvm::SyncScope::System) { + return CGBuilderBaseTy::CreateAtomicRMW(Op, Ptr, Val, llvm::MaybeAlign(), + Ordering, SSID); + } + using CGBuilderBaseTy::CreateBitCast; Address CreateBitCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name = "") { diff --git a/llvm/include/llvm/CodeGen/AtomicExpandUtils.h b/llvm/include/llvm/CodeGen/AtomicExpandUtils.h index 8a46c6e..1471ec7 100644 --- a/llvm/include/llvm/CodeGen/AtomicExpandUtils.h +++ b/llvm/include/llvm/CodeGen/AtomicExpandUtils.h @@ -22,8 +22,8 @@ class Value; /// (the builder, %addr, %loaded, %new_val, ordering, /// /* OUT */ %success, /* OUT */ %new_loaded) using CreateCmpXchgInstFun = - function_ref &, Value *, Value *, Value *, AtomicOrdering, - Value *&, Value *&)>; + function_ref &, Value *, Value *, Value *, Align, + AtomicOrdering, Value *&, Value *&)>; /// Expand an atomic RMW instruction into a loop utilizing /// cmpxchg. You'll want to make sure your target machine likes cmpxchg diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h index c12159f..8525e2e 100644 --- a/llvm/include/llvm/IR/IRBuilder.h +++ b/llvm/include/llvm/IR/IRBuilder.h @@ -1704,21 +1704,30 @@ public: return Insert(new FenceInst(Context, Ordering, SSID), Name); } - AtomicCmpXchgInst *CreateAtomicCmpXchg( - Value *Ptr, Value *Cmp, Value *New, AtomicOrdering SuccessOrdering, - AtomicOrdering FailureOrdering, SyncScope::ID SSID = SyncScope::System) { - const DataLayout &DL = BB->getModule()->getDataLayout(); - Align Alignment(DL.getTypeStoreSize(New->getType())); - return Insert(new AtomicCmpXchgInst( - Ptr, Cmp, New, Alignment, SuccessOrdering, FailureOrdering, SSID)); + AtomicCmpXchgInst * + CreateAtomicCmpXchg(Value *Ptr, Value *Cmp, Value *New, MaybeAlign Align, + AtomicOrdering SuccessOrdering, + AtomicOrdering FailureOrdering, + SyncScope::ID SSID = SyncScope::System) { + if (!Align) { + const DataLayout &DL = BB->getModule()->getDataLayout(); + Align = llvm::Align(DL.getTypeStoreSize(New->getType())); + } + + return Insert(new AtomicCmpXchgInst(Ptr, Cmp, New, *Align, SuccessOrdering, + FailureOrdering, SSID)); } - AtomicRMWInst *CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, + AtomicRMWInst *CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, + Value *Val, MaybeAlign Align, AtomicOrdering Ordering, SyncScope::ID SSID = SyncScope::System) { - const DataLayout &DL = BB->getModule()->getDataLayout(); - Align Alignment(DL.getTypeStoreSize(Val->getType())); - return Insert(new AtomicRMWInst(Op, Ptr, Val, Alignment, Ordering, SSID)); + if (!Align) { + const DataLayout &DL = BB->getModule()->getDataLayout(); + Align = llvm::Align(DL.getTypeStoreSize(Val->getType())); + } + + return Insert(new AtomicRMWInst(Op, Ptr, Val, *Align, Ordering, SSID)); } Value *CreateGEP(Value *Ptr, ArrayRef IdxList, diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp index 4026022..f04a8d2 100644 --- a/llvm/lib/CodeGen/AtomicExpandPass.cpp +++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp @@ -80,10 +80,11 @@ namespace { bool tryExpandAtomicRMW(AtomicRMWInst *AI); Value * insertRMWLLSCLoop(IRBuilder<> &Builder, Type *ResultTy, Value *Addr, - AtomicOrdering MemOpOrder, + Align AddrAlign, AtomicOrdering MemOpOrder, function_ref &, Value *)> PerformOp); void expandAtomicOpToLLSC( - Instruction *I, Type *ResultTy, Value *Addr, AtomicOrdering MemOpOrder, + Instruction *I, Type *ResultTy, Value *Addr, Align AddrAlign, + AtomicOrdering MemOpOrder, function_ref &, Value *)> PerformOp); void expandPartwordAtomicRMW( AtomicRMWInst *I, @@ -95,7 +96,7 @@ namespace { AtomicCmpXchgInst *convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI); static Value *insertRMWCmpXchgLoop( - IRBuilder<> &Builder, Type *ResultType, Value *Addr, + IRBuilder<> &Builder, Type *ResultType, Value *Addr, Align AddrAlign, AtomicOrdering MemOpOrder, function_ref &, Value *)> PerformOp, CreateCmpXchgInstFun CreateCmpXchg); @@ -368,7 +369,8 @@ bool AtomicExpand::tryExpandAtomicLoad(LoadInst *LI) { return false; case TargetLoweringBase::AtomicExpansionKind::LLSC: expandAtomicOpToLLSC( - LI, LI->getType(), LI->getPointerOperand(), LI->getOrdering(), + LI, LI->getType(), LI->getPointerOperand(), LI->getAlign(), + LI->getOrdering(), [](IRBuilder<> &Builder, Value *Loaded) { return Loaded; }); return true; case TargetLoweringBase::AtomicExpansionKind::LLOnly: @@ -407,7 +409,7 @@ bool AtomicExpand::expandAtomicLoadToCmpXchg(LoadInst *LI) { Constant *DummyVal = Constant::getNullValue(Ty); Value *Pair = Builder.CreateAtomicCmpXchg( - Addr, DummyVal, DummyVal, Order, + Addr, DummyVal, DummyVal, LI->getAlign(), Order, AtomicCmpXchgInst::getStrongestFailureOrdering(Order)); Value *Loaded = Builder.CreateExtractValue(Pair, 0, "loaded"); @@ -454,9 +456,9 @@ bool AtomicExpand::expandAtomicStore(StoreInst *SI) { // It is the responsibility of the target to only signal expansion via // shouldExpandAtomicRMW in cases where this is required and possible. IRBuilder<> Builder(SI); - AtomicRMWInst *AI = - Builder.CreateAtomicRMW(AtomicRMWInst::Xchg, SI->getPointerOperand(), - SI->getValueOperand(), SI->getOrdering()); + AtomicRMWInst *AI = Builder.CreateAtomicRMW( + AtomicRMWInst::Xchg, SI->getPointerOperand(), SI->getValueOperand(), + SI->getAlign(), SI->getOrdering()); SI->eraseFromParent(); // Now we have an appropriate swap instruction, lower it as usual. @@ -464,9 +466,9 @@ bool AtomicExpand::expandAtomicStore(StoreInst *SI) { } static void createCmpXchgInstFun(IRBuilder<> &Builder, Value *Addr, - Value *Loaded, Value *NewVal, - AtomicOrdering MemOpOrder, - Value *&Success, Value *&NewLoaded) { + Value *Loaded, Value *NewVal, Align AddrAlign, + AtomicOrdering MemOpOrder, Value *&Success, + Value *&NewLoaded) { Type *OrigTy = NewVal->getType(); // This code can go away when cmpxchg supports FP types. @@ -479,8 +481,8 @@ static void createCmpXchgInstFun(IRBuilder<> &Builder, Value *Addr, Loaded = Builder.CreateBitCast(Loaded, IntTy); } - Value* Pair = Builder.CreateAtomicCmpXchg( - Addr, Loaded, NewVal, MemOpOrder, + Value *Pair = Builder.CreateAtomicCmpXchg( + Addr, Loaded, NewVal, AddrAlign, MemOpOrder, AtomicCmpXchgInst::getStrongestFailureOrdering(MemOpOrder)); Success = Builder.CreateExtractValue(Pair, 1, "success"); NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded"); @@ -546,7 +548,7 @@ bool AtomicExpand::tryExpandAtomicRMW(AtomicRMWInst *AI) { AI->getValOperand()); }; expandAtomicOpToLLSC(AI, AI->getType(), AI->getPointerOperand(), - AI->getOrdering(), PerformOp); + AI->getAlign(), AI->getOrdering(), PerformOp); } return true; } @@ -581,6 +583,7 @@ struct PartwordMaskValues { Type *WordType = nullptr; Type *ValueType = nullptr; Value *AlignedAddr = nullptr; + Align AlignedAddrAlignment; // The remaining fields can be null. Value *ShiftAmt = nullptr; Value *Mask = nullptr; @@ -603,6 +606,7 @@ raw_ostream &operator<<(raw_ostream &O, const PartwordMaskValues &PMV) { PrintObj(PMV.ValueType); O << " AlignedAddr: "; PrintObj(PMV.AlignedAddr); + O << " AlignedAddrAlignment: " << PMV.AlignedAddrAlignment.value() << '\n'; O << " ShiftAmt: "; PrintObj(PMV.ShiftAmt); O << " Mask: "; @@ -633,6 +637,7 @@ raw_ostream &operator<<(raw_ostream &O, const PartwordMaskValues &PMV) { /// Inv_Mask: The inverse of Mask. static PartwordMaskValues createMaskInstrs(IRBuilder<> &Builder, Instruction *I, Type *ValueType, Value *Addr, + Align AddrAlign, unsigned MinWordSize) { PartwordMaskValues PMV; @@ -646,6 +651,7 @@ static PartwordMaskValues createMaskInstrs(IRBuilder<> &Builder, Instruction *I, : ValueType; if (PMV.ValueType == PMV.WordType) { PMV.AlignedAddr = Addr; + PMV.AlignedAddrAlignment = AddrAlign; return PMV; } @@ -654,10 +660,12 @@ static PartwordMaskValues createMaskInstrs(IRBuilder<> &Builder, Instruction *I, Type *WordPtrType = PMV.WordType->getPointerTo(Addr->getType()->getPointerAddressSpace()); + // TODO: we could skip some of this if AddrAlign >= MinWordSize. Value *AddrInt = Builder.CreatePtrToInt(Addr, DL.getIntPtrType(Ctx)); PMV.AlignedAddr = Builder.CreateIntToPtr( Builder.CreateAnd(AddrInt, ~(uint64_t)(MinWordSize - 1)), WordPtrType, "AlignedAddr"); + PMV.AlignedAddrAlignment = Align(MinWordSize); Value *PtrLSB = Builder.CreateAnd(AddrInt, MinWordSize - 1, "PtrLSB"); if (DL.isLittleEndian()) { @@ -765,7 +773,7 @@ void AtomicExpand::expandPartwordAtomicRMW( PartwordMaskValues PMV = createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(), - TLI->getMinCmpXchgSizeInBits() / 8); + AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8); Value *ValOperand_Shifted = Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType), @@ -778,13 +786,14 @@ void AtomicExpand::expandPartwordAtomicRMW( Value *OldResult; if (ExpansionKind == TargetLoweringBase::AtomicExpansionKind::CmpXChg) { - OldResult = - insertRMWCmpXchgLoop(Builder, PMV.WordType, PMV.AlignedAddr, MemOpOrder, - PerformPartwordOp, createCmpXchgInstFun); + OldResult = insertRMWCmpXchgLoop(Builder, PMV.WordType, PMV.AlignedAddr, + PMV.AlignedAddrAlignment, MemOpOrder, + PerformPartwordOp, createCmpXchgInstFun); } else { assert(ExpansionKind == TargetLoweringBase::AtomicExpansionKind::LLSC); OldResult = insertRMWLLSCLoop(Builder, PMV.WordType, PMV.AlignedAddr, - MemOpOrder, PerformPartwordOp); + PMV.AlignedAddrAlignment, MemOpOrder, + PerformPartwordOp); } Value *FinalOldResult = extractMaskedValue(Builder, OldResult, PMV); @@ -803,7 +812,7 @@ AtomicRMWInst *AtomicExpand::widenPartwordAtomicRMW(AtomicRMWInst *AI) { PartwordMaskValues PMV = createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(), - TLI->getMinCmpXchgSizeInBits() / 8); + AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8); Value *ValOperand_Shifted = Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType), @@ -817,8 +826,9 @@ AtomicRMWInst *AtomicExpand::widenPartwordAtomicRMW(AtomicRMWInst *AI) { else NewOperand = ValOperand_Shifted; - AtomicRMWInst *NewAI = Builder.CreateAtomicRMW(Op, PMV.AlignedAddr, - NewOperand, AI->getOrdering()); + AtomicRMWInst *NewAI = + Builder.CreateAtomicRMW(Op, PMV.AlignedAddr, NewOperand, + PMV.AlignedAddrAlignment, AI->getOrdering()); Value *FinalOldResult = extractMaskedValue(Builder, NewAI, PMV); AI->replaceAllUsesWith(FinalOldResult); @@ -871,8 +881,6 @@ bool AtomicExpand::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) { IRBuilder<> Builder(CI); LLVMContext &Ctx = Builder.getContext(); - const int WordSize = TLI->getMinCmpXchgSizeInBits() / 8; - BasicBlock *EndBB = BB->splitBasicBlock(CI->getIterator(), "partword.cmpxchg.end"); auto FailureBB = @@ -884,8 +892,9 @@ bool AtomicExpand::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) { std::prev(BB->end())->eraseFromParent(); Builder.SetInsertPoint(BB); - PartwordMaskValues PMV = createMaskInstrs( - Builder, CI, CI->getCompareOperand()->getType(), Addr, WordSize); + PartwordMaskValues PMV = + createMaskInstrs(Builder, CI, CI->getCompareOperand()->getType(), Addr, + CI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8); // Shift the incoming values over, into the right location in the word. Value *NewVal_Shifted = @@ -909,8 +918,8 @@ bool AtomicExpand::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) { Value *FullWord_NewVal = Builder.CreateOr(Loaded_MaskOut, NewVal_Shifted); Value *FullWord_Cmp = Builder.CreateOr(Loaded_MaskOut, Cmp_Shifted); AtomicCmpXchgInst *NewCI = Builder.CreateAtomicCmpXchg( - PMV.AlignedAddr, FullWord_Cmp, FullWord_NewVal, CI->getSuccessOrdering(), - CI->getFailureOrdering(), CI->getSyncScopeID()); + PMV.AlignedAddr, FullWord_Cmp, FullWord_NewVal, PMV.AlignedAddrAlignment, + CI->getSuccessOrdering(), CI->getFailureOrdering(), CI->getSyncScopeID()); NewCI->setVolatile(CI->isVolatile()); // When we're building a strong cmpxchg, we need a loop, so you // might think we could use a weak cmpxchg inside. But, using strong @@ -953,11 +962,12 @@ bool AtomicExpand::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) { } void AtomicExpand::expandAtomicOpToLLSC( - Instruction *I, Type *ResultType, Value *Addr, AtomicOrdering MemOpOrder, + Instruction *I, Type *ResultType, Value *Addr, Align AddrAlign, + AtomicOrdering MemOpOrder, function_ref &, Value *)> PerformOp) { IRBuilder<> Builder(I); - Value *Loaded = - insertRMWLLSCLoop(Builder, ResultType, Addr, MemOpOrder, PerformOp); + Value *Loaded = insertRMWLLSCLoop(Builder, ResultType, Addr, AddrAlign, + MemOpOrder, PerformOp); I->replaceAllUsesWith(Loaded); I->eraseFromParent(); @@ -968,7 +978,7 @@ void AtomicExpand::expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI) { PartwordMaskValues PMV = createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(), - TLI->getMinCmpXchgSizeInBits() / 8); + AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8); // The value operand must be sign-extended for signed min/max so that the // target's signed comparison instructions can be used. Otherwise, just @@ -994,7 +1004,7 @@ void AtomicExpand::expandAtomicCmpXchgToMaskedIntrinsic(AtomicCmpXchgInst *CI) { PartwordMaskValues PMV = createMaskInstrs( Builder, CI, CI->getCompareOperand()->getType(), CI->getPointerOperand(), - TLI->getMinCmpXchgSizeInBits() / 8); + CI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8); Value *CmpVal_Shifted = Builder.CreateShl( Builder.CreateZExt(CI->getCompareOperand(), PMV.WordType), PMV.ShiftAmt, @@ -1017,13 +1027,16 @@ void AtomicExpand::expandAtomicCmpXchgToMaskedIntrinsic(AtomicCmpXchgInst *CI) { } Value *AtomicExpand::insertRMWLLSCLoop( - IRBuilder<> &Builder, Type *ResultTy, Value *Addr, + IRBuilder<> &Builder, Type *ResultTy, Value *Addr, Align AddrAlign, AtomicOrdering MemOpOrder, function_ref &, Value *)> PerformOp) { LLVMContext &Ctx = Builder.getContext(); BasicBlock *BB = Builder.GetInsertBlock(); Function *F = BB->getParent(); + assert(AddrAlign >= ResultTy->getPrimitiveSizeInBits() / 8 && + "Expected at least natural alignment at this point."); + // Given: atomicrmw some_op iN* %addr, iN %incr ordering // // The standard expansion we produce is: @@ -1082,11 +1095,9 @@ AtomicCmpXchgInst *AtomicExpand::convertCmpXchgToIntegerType(AtomicCmpXchgInst * Value *NewCmp = Builder.CreatePtrToInt(CI->getCompareOperand(), NewTy); Value *NewNewVal = Builder.CreatePtrToInt(CI->getNewValOperand(), NewTy); - - auto *NewCI = Builder.CreateAtomicCmpXchg(NewAddr, NewCmp, NewNewVal, - CI->getSuccessOrdering(), - CI->getFailureOrdering(), - CI->getSyncScopeID()); + auto *NewCI = Builder.CreateAtomicCmpXchg( + NewAddr, NewCmp, NewNewVal, CI->getAlign(), CI->getSuccessOrdering(), + CI->getFailureOrdering(), CI->getSyncScopeID()); NewCI->setVolatile(CI->isVolatile()); NewCI->setWeak(CI->isWeak()); LLVM_DEBUG(dbgs() << "Replaced " << *CI << " with " << *NewCI << "\n"); @@ -1211,7 +1222,7 @@ bool AtomicExpand::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) { PartwordMaskValues PMV = createMaskInstrs(Builder, CI, CI->getCompareOperand()->getType(), Addr, - TLI->getMinCmpXchgSizeInBits() / 8); + CI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8); Builder.CreateBr(StartBB); // Start the main loop block now that we've taken care of the preliminaries. @@ -1379,7 +1390,7 @@ bool AtomicExpand::simplifyIdempotentRMW(AtomicRMWInst* RMWI) { } Value *AtomicExpand::insertRMWCmpXchgLoop( - IRBuilder<> &Builder, Type *ResultTy, Value *Addr, + IRBuilder<> &Builder, Type *ResultTy, Value *Addr, Align AddrAlign, AtomicOrdering MemOpOrder, function_ref &, Value *)> PerformOp, CreateCmpXchgInstFun CreateCmpXchg) { @@ -1411,9 +1422,7 @@ Value *AtomicExpand::insertRMWCmpXchgLoop( // the branch entirely. std::prev(BB->end())->eraseFromParent(); Builder.SetInsertPoint(BB); - LoadInst *InitLoaded = Builder.CreateLoad(ResultTy, Addr); - // Atomics require at least natural alignment. - InitLoaded->setAlignment(Align(ResultTy->getPrimitiveSizeInBits() / 8)); + LoadInst *InitLoaded = Builder.CreateAlignedLoad(ResultTy, Addr, AddrAlign); Builder.CreateBr(LoopBB); // Start the main loop block now that we've taken care of the preliminaries. @@ -1426,7 +1435,7 @@ Value *AtomicExpand::insertRMWCmpXchgLoop( Value *NewLoaded = nullptr; Value *Success = nullptr; - CreateCmpXchg(Builder, Addr, Loaded, NewVal, + CreateCmpXchg(Builder, Addr, Loaded, NewVal, AddrAlign, MemOpOrder == AtomicOrdering::Unordered ? AtomicOrdering::Monotonic : MemOpOrder, @@ -1466,7 +1475,8 @@ bool llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI, CreateCmpXchgInstFun CreateCmpXchg) { IRBuilder<> Builder(AI); Value *Loaded = AtomicExpand::insertRMWCmpXchgLoop( - Builder, AI->getType(), AI->getPointerOperand(), AI->getOrdering(), + Builder, AI->getType(), AI->getPointerOperand(), AI->getAlign(), + AI->getOrdering(), [&](IRBuilder<> &Builder, Value *Loaded) { return performAtomicOp(AI->getOperation(), Builder, Loaded, AI->getValOperand()); @@ -1614,20 +1624,20 @@ void AtomicExpand::expandAtomicRMWToLibcall(AtomicRMWInst *I) { // libcalls (add/sub/etc) and we needed a generic. So, expand to a // CAS libcall, via a CAS loop, instead. if (!Success) { - expandAtomicRMWToCmpXchg(I, [this](IRBuilder<> &Builder, Value *Addr, - Value *Loaded, Value *NewVal, - AtomicOrdering MemOpOrder, - Value *&Success, Value *&NewLoaded) { - // Create the CAS instruction normally... - AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg( - Addr, Loaded, NewVal, MemOpOrder, - AtomicCmpXchgInst::getStrongestFailureOrdering(MemOpOrder)); - Success = Builder.CreateExtractValue(Pair, 1, "success"); - NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded"); - - // ...and then expand the CAS into a libcall. - expandAtomicCASToLibcall(Pair); - }); + expandAtomicRMWToCmpXchg( + I, [this](IRBuilder<> &Builder, Value *Addr, Value *Loaded, + Value *NewVal, Align Alignment, AtomicOrdering MemOpOrder, + Value *&Success, Value *&NewLoaded) { + // Create the CAS instruction normally... + AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg( + Addr, Loaded, NewVal, Alignment, MemOpOrder, + AtomicCmpXchgInst::getStrongestFailureOrdering(MemOpOrder)); + Success = Builder.CreateExtractValue(Pair, 1, "success"); + NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded"); + + // ...and then expand the CAS into a libcall. + expandAtomicCASToLibcall(Pair); + }); } } diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp index a4ec538..e0d152b 100644 --- a/llvm/lib/IR/AutoUpgrade.cpp +++ b/llvm/lib/IR/AutoUpgrade.cpp @@ -3594,7 +3594,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Name.startswith("atomic.load.add.f64.p"))) { Value *Ptr = CI->getArgOperand(0); Value *Val = CI->getArgOperand(1); - Rep = Builder.CreateAtomicRMW(AtomicRMWInst::FAdd, Ptr, Val, + Rep = Builder.CreateAtomicRMW(AtomicRMWInst::FAdd, Ptr, Val, MaybeAlign(), AtomicOrdering::SequentiallyConsistent); } else if (IsNVVM && (Name == "max.i" || Name == "max.ll" || Name == "max.ui" || Name == "max.ull")) { diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp index d38cd6f..fdcd643 100644 --- a/llvm/lib/IR/Core.cpp +++ b/llvm/lib/IR/Core.cpp @@ -3977,9 +3977,10 @@ LLVMValueRef LLVMBuildAtomicRMW(LLVMBuilderRef B,LLVMAtomicRMWBinOp op, LLVMAtomicOrdering ordering, LLVMBool singleThread) { AtomicRMWInst::BinOp intop = mapFromLLVMRMWBinOp(op); - return wrap(unwrap(B)->CreateAtomicRMW(intop, unwrap(PTR), unwrap(Val), - mapFromLLVMOrdering(ordering), singleThread ? SyncScope::SingleThread - : SyncScope::System)); + return wrap(unwrap(B)->CreateAtomicRMW( + intop, unwrap(PTR), unwrap(Val), MaybeAlign(), + mapFromLLVMOrdering(ordering), + singleThread ? SyncScope::SingleThread : SyncScope::System)); } LLVMValueRef LLVMBuildAtomicCmpXchg(LLVMBuilderRef B, LLVMValueRef Ptr, @@ -3988,10 +3989,11 @@ LLVMValueRef LLVMBuildAtomicCmpXchg(LLVMBuilderRef B, LLVMValueRef Ptr, LLVMAtomicOrdering FailureOrdering, LLVMBool singleThread) { - return wrap(unwrap(B)->CreateAtomicCmpXchg(unwrap(Ptr), unwrap(Cmp), - unwrap(New), mapFromLLVMOrdering(SuccessOrdering), - mapFromLLVMOrdering(FailureOrdering), - singleThread ? SyncScope::SingleThread : SyncScope::System)); + return wrap(unwrap(B)->CreateAtomicCmpXchg( + unwrap(Ptr), unwrap(Cmp), unwrap(New), MaybeAlign(), + mapFromLLVMOrdering(SuccessOrdering), + mapFromLLVMOrdering(FailureOrdering), + singleThread ? SyncScope::SingleThread : SyncScope::System)); } unsigned LLVMGetNumMaskElements(LLVMValueRef SVInst) { diff --git a/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp b/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp index bc1068f..c2a3b91 100644 --- a/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp @@ -965,7 +965,7 @@ bool GCOVProfiler::emitProfileNotes( Counters->getValueType(), Counters, 0, I); if (Options.Atomic) { Builder.CreateAtomicRMW(AtomicRMWInst::Add, V, Builder.getInt64(1), - AtomicOrdering::Monotonic); + MaybeAlign(), AtomicOrdering::Monotonic); } else { Value *Count = Builder.CreateLoad(Builder.getInt64Ty(), V, "gcov_ctr"); diff --git a/llvm/lib/Transforms/Instrumentation/InstrOrderFile.cpp b/llvm/lib/Transforms/Instrumentation/InstrOrderFile.cpp index 853385f..071feb8 100644 --- a/llvm/lib/Transforms/Instrumentation/InstrOrderFile.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrOrderFile.cpp @@ -144,7 +144,7 @@ public: // Fill up UpdateOrderFileBB: grab the index, update the buffer! Value *IdxVal = updateB.CreateAtomicRMW( AtomicRMWInst::Add, BufferIdx, ConstantInt::get(Int32Ty, 1), - AtomicOrdering::SequentiallyConsistent); + MaybeAlign(), AtomicOrdering::SequentiallyConsistent); // We need to wrap around the index to fit it inside the buffer. Value *WrappedIdx = updateB.CreateAnd( IdxVal, ConstantInt::get(Int32Ty, INSTR_ORDER_FILE_BUFFER_MASK)); diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp index 5aa8a15..70fef46 100644 --- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -205,6 +205,7 @@ public: // automic update currently can only be promoted across the current // loop, not the whole loop nest. Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue, + MaybeAlign(), AtomicOrdering::SequentiallyConsistent); else { LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted"); @@ -702,7 +703,7 @@ void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) { if (Options.Atomic || AtomicCounterUpdateAll || (Index == 0 && AtomicFirstCounter)) { Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(), - AtomicOrdering::Monotonic); + MaybeAlign(), AtomicOrdering::Monotonic); } else { Value *IncStep = Inc->getStep(); Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount"); diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td index 9359937..076b8ed 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td @@ -1526,6 +1526,7 @@ def AtomicOrdering : I64EnumAttr< def LLVM_AtomicRMWType : AnyTypeOf<[LLVM_AnyFloat, LLVM_AnyInteger]>; +// FIXME: Need to add alignment attribute to MLIR atomicrmw operation. def LLVM_AtomicRMWOp : LLVM_Op<"atomicrmw"> { let arguments = (ins AtomicBinOp:$bin_op, LLVM_PointerTo:$ptr, @@ -1533,6 +1534,7 @@ def LLVM_AtomicRMWOp : LLVM_Op<"atomicrmw"> { let results = (outs LLVM_AtomicRMWType:$res); let llvmBuilder = [{ $res = builder.CreateAtomicRMW(getLLVMAtomicBinOp($bin_op), $ptr, $val, + llvm::MaybeAlign(), getLLVMAtomicOrdering($ordering)); }]; let parser = [{ return parseAtomicRMWOp(parser, result); }]; @@ -1553,6 +1555,7 @@ def LLVM_AtomicCmpXchgResultType : Type; +// FIXME: Need to add alignment attribute to MLIR cmpxchg operation. def LLVM_AtomicCmpXchgOp : LLVM_Op<"cmpxchg"> { let arguments = (ins LLVM_PointerTo:$ptr, LLVM_AtomicCmpXchgType:$cmp, LLVM_AtomicCmpXchgType:$val, @@ -1560,7 +1563,7 @@ def LLVM_AtomicCmpXchgOp : LLVM_Op<"cmpxchg"> { AtomicOrdering:$failure_ordering); let results = (outs LLVM_AtomicCmpXchgResultType:$res); let llvmBuilder = [{ - $res = builder.CreateAtomicCmpXchg($ptr, $cmp, $val, + $res = builder.CreateAtomicCmpXchg($ptr, $cmp, $val, llvm::MaybeAlign(), getLLVMAtomicOrdering($success_ordering), getLLVMAtomicOrdering($failure_ordering)); }]; -- 2.7.4