From 4d7df43ffdb460dddb2877a886f75f45c3fee188 Mon Sep 17 00:00:00 2001 From: Pavel Iliin Date: Thu, 19 Nov 2020 19:52:46 +0000 Subject: [PATCH] [AArch64] Out-of-line atomics (-moutline-atomics) implementation. This patch implements out of line atomics for LSE deployment mechanism. Details how it works can be found in llvm/docs/Atomics.rst Options -moutline-atomics and -mno-outline-atomics to enable and disable it were added to clang driver. This is clang and llvm part of out-of-line atomics interface, library part is already supported by libgcc. Compiler-rt support is provided in separate patch. Differential Revision: https://reviews.llvm.org/D91157 --- clang/include/clang/Basic/DiagnosticDriverKinds.td | 4 + clang/include/clang/Driver/Options.td | 4 + clang/lib/Driver/ToolChains/Clang.cpp | 17 + llvm/docs/Atomics.rst | 20 + llvm/include/llvm/CodeGen/RuntimeLibcalls.h | 5 + llvm/include/llvm/IR/RuntimeLibcalls.def | 17 + llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 19 +- .../CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp | 18 +- llvm/lib/CodeGen/TargetLoweringBase.cpp | 77 + llvm/lib/Target/AArch64/AArch64.td | 3 + llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 77 +- llvm/lib/Target/AArch64/AArch64Subtarget.h | 3 + llvm/test/CodeGen/AArch64/arm64-atomic-128.ll | 2 + llvm/test/CodeGen/AArch64/arm64-atomic.ll | 8 + llvm/test/CodeGen/AArch64/arm64_32-atomics.ll | 2 + llvm/test/CodeGen/AArch64/atomic-ops-lse.ll | 4112 ++++++++++++++++++++ .../CodeGen/AArch64/atomic-ops-not-barriers.ll | 2 + llvm/test/CodeGen/AArch64/atomic-ops.ll | 537 ++- llvm/test/CodeGen/AArch64/cmpxchg-O0.ll | 7 + llvm/test/CodeGen/AArch64/cmpxchg-idioms.ll | 5 + llvm/test/CodeGen/AArch64/cmpxchg-lse-even-regs.ll | 1 + .../AArch64/expand-atomicrmw-xchg-fp.ll | 13 + 22 files changed, 4938 insertions(+), 15 deletions(-) diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td index 43ee496..781fcfc 100644 --- a/clang/include/clang/Basic/DiagnosticDriverKinds.td +++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td @@ -491,6 +491,10 @@ def warn_drv_moutline_unsupported_opt : Warning< "The '%0' architecture does not support -moutline; flag ignored">, InGroup; +def warn_drv_moutline_atomics_unsupported_opt : Warning< + "The '%0' architecture does not support -moutline-atomics; flag ignored">, + InGroup; + def warn_drv_darwin_sdk_invalid_settings : Warning< "SDK settings were ignored as 'SDKSettings.json' could not be parsed">, InGroup>; diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index d4aa4aa..b1aaa2f 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -2664,6 +2664,10 @@ def msmall_data_threshold_EQ : Joined <["-"], "msmall-data-threshold=">, Group, Alias; def msoft_float : Flag<["-"], "msoft-float">, Group, Flags<[CC1Option]>, HelpText<"Use software floating point">; +def moutline_atomics : Flag<["-"], "moutline-atomics">, Group, Flags<[CC1Option]>, + HelpText<"Generate local calls to out-of-line atomic operations">; +def mno_outline_atomics : Flag<["-"], "mno-outline-atomics">, Group, Flags<[CC1Option]>, + HelpText<"Don't generate local calls to out-of-line atomic operations">; def mno_implicit_float : Flag<["-"], "mno-implicit-float">, Group, HelpText<"Don't generate implicit floating point instructions">; def mimplicit_float : Flag<["-"], "mimplicit-float">, Group; diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index ad242d2..174701b 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -6367,6 +6367,23 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, } } + if (Arg *A = Args.getLastArg(options::OPT_moutline_atomics, + options::OPT_mno_outline_atomics)) { + if (A->getOption().matches(options::OPT_moutline_atomics)) { + // Option -moutline-atomics supported for AArch64 target only. + if (!Triple.isAArch64()) { + D.Diag(diag::warn_drv_moutline_atomics_unsupported_opt) + << Triple.getArchName(); + } else { + CmdArgs.push_back("-target-feature"); + CmdArgs.push_back("+outline-atomics"); + } + } else { + CmdArgs.push_back("-target-feature"); + CmdArgs.push_back("-outline-atomics"); + } + } + if (Args.hasFlag(options::OPT_faddrsig, options::OPT_fno_addrsig, (TC.getTriple().isOSBinFormatELF() || TC.getTriple().isOSBinFormatCOFF()) && diff --git a/llvm/docs/Atomics.rst b/llvm/docs/Atomics.rst index 7d57740..d091bf5 100644 --- a/llvm/docs/Atomics.rst +++ b/llvm/docs/Atomics.rst @@ -621,3 +621,23 @@ fence on either side of a normal load or store.) There's also, somewhat separately, the possibility to lower ``ATOMIC_FENCE`` to ``__sync_synchronize()``. This may happen or not happen independent of all the above, controlled purely by ``setOperationAction(ISD::ATOMIC_FENCE, ...)``. + +On AArch64, a variant of the __sync_* routines is used which contain the memory +order as part of the function name. These routines may determine at runtime +whether the single-instruction atomic operations which were introduced as part +of AArch64 Large System Extensions "LSE" instruction set are available, or if +it needs to fall back to an LL/SC loop. The following helper functions are +implemented in both ``compiler-rt`` and ``libgcc`` libraries +(``N`` is one of 1, 2, 4, 8, and ``M`` is one of 1, 2, 4, 8 and 16, and +``ORDER`` is one of 'relax', 'acq', 'rel', 'acq_rel'):: + + iM __aarch64_casM_ORDER(iM expected, iM desired, iM *ptr) + iN __aarch64_swpN_ORDER(iN val, iN *ptr) + iN __aarch64_ldaddN_ORDER(iN val, iN *ptr) + iN __aarch64_ldclrN_ORDER(iN val, iN *ptr) + iN __aarch64_ldeorN_ORDER(iN val, iN *ptr) + iN __aarch64_ldsetN_ORDER(iN val, iN *ptr) + +Please note, if LSE instruction set is specified for AArch64 target then +out-of-line atomics calls are not generated and single-instruction atomic +operations are used in place. diff --git a/llvm/include/llvm/CodeGen/RuntimeLibcalls.h b/llvm/include/llvm/CodeGen/RuntimeLibcalls.h index f71f39e..86e24ca 100644 --- a/llvm/include/llvm/CodeGen/RuntimeLibcalls.h +++ b/llvm/include/llvm/CodeGen/RuntimeLibcalls.h @@ -15,6 +15,7 @@ #define LLVM_CODEGEN_RUNTIMELIBCALLS_H #include "llvm/CodeGen/ValueTypes.h" +#include "llvm/Support/AtomicOrdering.h" namespace llvm { namespace RTLIB { @@ -60,6 +61,10 @@ namespace RTLIB { /// UNKNOWN_LIBCALL if there is none. Libcall getSYNC(unsigned Opc, MVT VT); + /// Return the outline atomics value for the given opcode, atomic ordering + /// and type, or UNKNOWN_LIBCALL if there is none. + Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT); + /// getMEMCPY_ELEMENT_UNORDERED_ATOMIC - Return /// MEMCPY_ELEMENT_UNORDERED_ATOMIC_* value for the given element size or /// UNKNOW_LIBCALL if there is none. diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.def b/llvm/include/llvm/IR/RuntimeLibcalls.def index 301bb04..3329fa8 100644 --- a/llvm/include/llvm/IR/RuntimeLibcalls.def +++ b/llvm/include/llvm/IR/RuntimeLibcalls.def @@ -545,6 +545,23 @@ HANDLE_LIBCALL(ATOMIC_FETCH_NAND_4, "__atomic_fetch_nand_4") HANDLE_LIBCALL(ATOMIC_FETCH_NAND_8, "__atomic_fetch_nand_8") HANDLE_LIBCALL(ATOMIC_FETCH_NAND_16, "__atomic_fetch_nand_16") +// Out-of-line atomics libcalls +#define HLCALLS(A, N) \ + HANDLE_LIBCALL(A##N##_RELAX, nullptr) \ + HANDLE_LIBCALL(A##N##_ACQ, nullptr) \ + HANDLE_LIBCALL(A##N##_REL, nullptr) \ + HANDLE_LIBCALL(A##N##_ACQ_REL, nullptr) +#define HLCALL5(A) \ + HLCALLS(A, 1) HLCALLS(A, 2) HLCALLS(A, 4) HLCALLS(A, 8) HLCALLS(A, 16) +HLCALL5(OUTLINE_ATOMIC_CAS) +HLCALL5(OUTLINE_ATOMIC_SWP) +HLCALL5(OUTLINE_ATOMIC_LDADD) +HLCALL5(OUTLINE_ATOMIC_LDSET) +HLCALL5(OUTLINE_ATOMIC_LDCLR) +HLCALL5(OUTLINE_ATOMIC_LDEOR) +#undef HLCALLS +#undef HLCALL5 + // Stack Protector Fail HANDLE_LIBCALL(STACKPROTECTOR_CHECK_FAIL, "__stack_chk_fail") diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index 74bfabf..31a9928 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -4052,12 +4052,23 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) { case ISD::ATOMIC_LOAD_UMAX: case ISD::ATOMIC_CMP_SWAP: { MVT VT = cast(Node)->getMemoryVT().getSimpleVT(); - RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT); - assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!"); - + AtomicOrdering Order = cast(Node)->getOrdering(); + RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT); EVT RetVT = Node->getValueType(0); - SmallVector Ops(Node->op_begin() + 1, Node->op_end()); TargetLowering::MakeLibCallOptions CallOptions; + SmallVector Ops; + if (TLI.getLibcallName(LC)) { + // If outline atomic available, prepare its arguments and expand. + Ops.append(Node->op_begin() + 2, Node->op_end()); + Ops.push_back(Node->getOperand(1)); + + } else { + LC = RTLIB::getSYNC(Opc, VT); + assert(LC != RTLIB::UNKNOWN_LIBCALL && + "Unexpected atomic op or value type!"); + // Arguments for expansion to sync libcall + Ops.append(Node->op_begin() + 1, Node->op_end()); + } std::pair Tmp = TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, SDLoc(Node), diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp index 2b6471e..414e769 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -2169,12 +2169,22 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { std::pair DAGTypeLegalizer::ExpandAtomic(SDNode *Node) { unsigned Opc = Node->getOpcode(); MVT VT = cast(Node)->getMemoryVT().getSimpleVT(); - RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT); - assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!"); - + AtomicOrdering order = cast(Node)->getOrdering(); + // Lower to outline atomic libcall if outline atomics enabled, + // or to sync libcall otherwise + RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, order, VT); EVT RetVT = Node->getValueType(0); - SmallVector Ops(Node->op_begin() + 1, Node->op_end()); TargetLowering::MakeLibCallOptions CallOptions; + SmallVector Ops; + if (TLI.getLibcallName(LC)) { + Ops.append(Node->op_begin() + 2, Node->op_end()); + Ops.push_back(Node->getOperand(1)); + } else { + LC = RTLIB::getSYNC(Opc, VT); + assert(LC != RTLIB::UNKNOWN_LIBCALL && + "Unexpected atomic op or value type!"); + Ops.append(Node->op_begin() + 1, Node->op_end()); + } return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, SDLoc(Node), Node->getOperand(0)); } diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp index 8b44885..e4d9b46 100644 --- a/llvm/lib/CodeGen/TargetLoweringBase.cpp +++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp @@ -448,6 +448,83 @@ RTLIB::Libcall RTLIB::getUINTTOFP(EVT OpVT, EVT RetVT) { return UNKNOWN_LIBCALL; } +RTLIB::Libcall RTLIB::getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, + MVT VT) { + unsigned ModeN, ModelN; + switch (VT.SimpleTy) { + case MVT::i8: + ModeN = 0; + break; + case MVT::i16: + ModeN = 1; + break; + case MVT::i32: + ModeN = 2; + break; + case MVT::i64: + ModeN = 3; + break; + case MVT::i128: + ModeN = 4; + break; + default: + return UNKNOWN_LIBCALL; + } + + switch (Order) { + case AtomicOrdering::Monotonic: + ModelN = 0; + break; + case AtomicOrdering::Acquire: + ModelN = 1; + break; + case AtomicOrdering::Release: + ModelN = 2; + break; + case AtomicOrdering::AcquireRelease: + case AtomicOrdering::SequentiallyConsistent: + ModelN = 3; + break; + default: + return UNKNOWN_LIBCALL; + } + +#define LCALLS(A, B) \ + { A##B##_RELAX, A##B##_ACQ, A##B##_REL, A##B##_ACQ_REL } +#define LCALL5(A) \ + LCALLS(A, 1), LCALLS(A, 2), LCALLS(A, 4), LCALLS(A, 8), LCALLS(A, 16) + switch (Opc) { + case ISD::ATOMIC_CMP_SWAP: { + const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_CAS)}; + return LC[ModeN][ModelN]; + } + case ISD::ATOMIC_SWAP: { + const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_SWP)}; + return LC[ModeN][ModelN]; + } + case ISD::ATOMIC_LOAD_ADD: { + const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDADD)}; + return LC[ModeN][ModelN]; + } + case ISD::ATOMIC_LOAD_OR: { + const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDSET)}; + return LC[ModeN][ModelN]; + } + case ISD::ATOMIC_LOAD_CLR: { + const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDCLR)}; + return LC[ModeN][ModelN]; + } + case ISD::ATOMIC_LOAD_XOR: { + const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDEOR)}; + return LC[ModeN][ModelN]; + } + default: + return UNKNOWN_LIBCALL; + } +#undef LCALLS +#undef LCALL5 +} + RTLIB::Libcall RTLIB::getSYNC(unsigned Opc, MVT VT) { #define OP_TO_LIBCALL(Name, Enum) \ case Name: \ diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td index 8bf85de..d73ded1 100644 --- a/llvm/lib/Target/AArch64/AArch64.td +++ b/llvm/lib/Target/AArch64/AArch64.td @@ -61,6 +61,9 @@ def FeatureRAS : SubtargetFeature<"ras", "HasRAS", "true", def FeatureLSE : SubtargetFeature<"lse", "HasLSE", "true", "Enable ARMv8.1 Large System Extension (LSE) atomic instructions">; +def FeatureOutlineAtomics : SubtargetFeature<"outline-atomics", "OutlineAtomics", "true", + "Enable out of line atomics to support LSE instructions">; + def FeatureRDM : SubtargetFeature<"rdm", "HasRDM", "true", "Enable ARMv8.1 Rounding Double Multiply Add/Subtract instructions">; diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 463580c..b92eb1d 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -662,6 +662,57 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom); setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i64, Custom); + // Generate outline atomics library calls only if LSE was not specified for + // subtarget + if (Subtarget->outlineAtomics() && !Subtarget->hasLSE()) { + setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i8, LibCall); + setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i16, LibCall); + setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, LibCall); + setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i64, LibCall); + setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, LibCall); + setOperationAction(ISD::ATOMIC_SWAP, MVT::i8, LibCall); + setOperationAction(ISD::ATOMIC_SWAP, MVT::i16, LibCall); + setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, LibCall); + setOperationAction(ISD::ATOMIC_SWAP, MVT::i64, LibCall); + setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i8, LibCall); + setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i16, LibCall); + setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, LibCall); + setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i64, LibCall); + setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i8, LibCall); + setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i16, LibCall); + setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, LibCall); + setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i64, LibCall); + setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i8, LibCall); + setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i16, LibCall); + setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i32, LibCall); + setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i64, LibCall); + setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i8, LibCall); + setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i16, LibCall); + setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, LibCall); + setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i64, LibCall); +#define LCALLNAMES(A, B, N) \ + setLibcallName(A##N##_RELAX, #B #N "_relax"); \ + setLibcallName(A##N##_ACQ, #B #N "_acq"); \ + setLibcallName(A##N##_REL, #B #N "_rel"); \ + setLibcallName(A##N##_ACQ_REL, #B #N "_acq_rel"); +#define LCALLNAME4(A, B) \ + LCALLNAMES(A, B, 1) \ + LCALLNAMES(A, B, 2) LCALLNAMES(A, B, 4) LCALLNAMES(A, B, 8) +#define LCALLNAME5(A, B) \ + LCALLNAMES(A, B, 1) \ + LCALLNAMES(A, B, 2) \ + LCALLNAMES(A, B, 4) LCALLNAMES(A, B, 8) LCALLNAMES(A, B, 16) + LCALLNAME5(RTLIB::OUTLINE_ATOMIC_CAS, __aarch64_cas) + LCALLNAME4(RTLIB::OUTLINE_ATOMIC_SWP, __aarch64_swp) + LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDADD, __aarch64_ldadd) + LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDSET, __aarch64_ldset) + LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDCLR, __aarch64_ldclr) + LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDEOR, __aarch64_ldeor) +#undef LCALLNAMES +#undef LCALLNAME4 +#undef LCALLNAME5 + } + // 128-bit loads and stores can be done without expanding setOperationAction(ISD::LOAD, MVT::i128, Custom); setOperationAction(ISD::STORE, MVT::i128, Custom); @@ -10078,7 +10129,7 @@ SDValue AArch64TargetLowering::LowerVECREDUCE(SDValue Op, SDValue AArch64TargetLowering::LowerATOMIC_LOAD_SUB(SDValue Op, SelectionDAG &DAG) const { auto &Subtarget = static_cast(DAG.getSubtarget()); - if (!Subtarget.hasLSE()) + if (!Subtarget.hasLSE() && !Subtarget.outlineAtomics()) return SDValue(); // LSE has an atomic load-add instruction, but not a load-sub. @@ -10095,7 +10146,7 @@ SDValue AArch64TargetLowering::LowerATOMIC_LOAD_SUB(SDValue Op, SDValue AArch64TargetLowering::LowerATOMIC_LOAD_AND(SDValue Op, SelectionDAG &DAG) const { auto &Subtarget = static_cast(DAG.getSubtarget()); - if (!Subtarget.hasLSE()) + if (!Subtarget.hasLSE() && !Subtarget.outlineAtomics()) return SDValue(); // LSE has an atomic load-clear instruction, but not a load-and. @@ -15549,7 +15600,7 @@ static void ReplaceCMP_SWAP_128Results(SDNode *N, assert(N->getValueType(0) == MVT::i128 && "AtomicCmpSwap on types less than 128 should be legal"); - if (Subtarget->hasLSE()) { + if (Subtarget->hasLSE() || Subtarget->outlineAtomics()) { // LSE has a 128-bit compare and swap (CASP), but i128 is not a legal type, // so lower it here, wrapped in REG_SEQUENCE and EXTRACT_SUBREG. SDValue Ops[] = { @@ -15778,14 +15829,30 @@ AArch64TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const { // Nand not supported in LSE. if (AI->getOperation() == AtomicRMWInst::Nand) return AtomicExpansionKind::LLSC; // Leave 128 bits to LLSC. - return (Subtarget->hasLSE() && Size < 128) ? AtomicExpansionKind::None : AtomicExpansionKind::LLSC; + if (Subtarget->hasLSE() && Size < 128) + return AtomicExpansionKind::None; + if (Subtarget->outlineAtomics() && Size < 128) { + // [U]Min/[U]Max RWM atomics are used in __sync_fetch_ libcalls so far. + // Don't outline them unless + // (1) high level support approved: + // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p0493r1.pdf + // (2) low level libgcc and compiler-rt support implemented by: + // min/max outline atomics helpers + if (AI->getOperation() != AtomicRMWInst::Min && + AI->getOperation() != AtomicRMWInst::Max && + AI->getOperation() != AtomicRMWInst::UMin && + AI->getOperation() != AtomicRMWInst::UMax) { + return AtomicExpansionKind::None; + } + } + return AtomicExpansionKind::LLSC; } TargetLowering::AtomicExpansionKind AArch64TargetLowering::shouldExpandAtomicCmpXchgInIR( AtomicCmpXchgInst *AI) const { // If subtarget has LSE, leave cmpxchg intact for codegen. - if (Subtarget->hasLSE()) + if (Subtarget->hasLSE() || Subtarget->outlineAtomics()) return AtomicExpansionKind::None; // At -O0, fast-regalloc cannot cope with the live vregs necessary to // implement cmpxchg without spilling. If the address being exchanged is also diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h index 4eb4843..631d18a 100644 --- a/llvm/lib/Target/AArch64/AArch64Subtarget.h +++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h @@ -195,6 +195,7 @@ protected: // Enable 64-bit vectorization in SLP. unsigned MinVectorRegisterBitWidth = 64; + bool OutlineAtomics = false; bool UseAA = false; bool PredictableSelectIsExpensive = false; bool BalanceFPOps = false; @@ -471,6 +472,8 @@ public: bool useAA() const override { return UseAA; } + bool outlineAtomics() const { return OutlineAtomics; } + bool hasVH() const { return HasVH; } bool hasPAN() const { return HasPAN; } bool hasLOR() const { return HasLOR; } diff --git a/llvm/test/CodeGen/AArch64/arm64-atomic-128.ll b/llvm/test/CodeGen/AArch64/arm64-atomic-128.ll index 21e3c76..2409a34 100644 --- a/llvm/test/CodeGen/AArch64/arm64-atomic-128.ll +++ b/llvm/test/CodeGen/AArch64/arm64-atomic-128.ll @@ -1,8 +1,10 @@ ; RUN: llc < %s -mtriple=arm64-linux-gnu -verify-machineinstrs -mcpu=cyclone | FileCheck %s +; RUN: llc < %s -mtriple=arm64-linux-gnu -verify-machineinstrs -mcpu=cyclone -mattr=+outline-atomics | FileCheck %s -check-prefix=OUTLINE-ATOMICS @var = global i128 0 define i128 @val_compare_and_swap(i128* %p, i128 %oldval, i128 %newval) { +; OUTLINE-ATOMICS: bl __aarch64_cas16_acq ; CHECK-LABEL: val_compare_and_swap: ; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: ; CHECK: ldaxp [[RESULTLO:x[0-9]+]], [[RESULTHI:x[0-9]+]], [x[[ADDR:[0-9]+]]] diff --git a/llvm/test/CodeGen/AArch64/arm64-atomic.ll b/llvm/test/CodeGen/AArch64/arm64-atomic.ll index db343e9..cfdbcf9 100644 --- a/llvm/test/CodeGen/AArch64/arm64-atomic.ll +++ b/llvm/test/CodeGen/AArch64/arm64-atomic.ll @@ -1,6 +1,8 @@ ; RUN: llc < %s -mtriple=arm64-eabi -asm-verbose=false -verify-machineinstrs -mcpu=cyclone | FileCheck -enable-var-scope %s +; RUN: llc < %s -mtriple=arm64-eabi -asm-verbose=false -verify-machineinstrs -mcpu=cyclone -mattr=+outline-atomics | FileCheck -enable-var-scope %s -check-prefix=OUTLINE-ATOMICS define i32 @val_compare_and_swap(i32* %p, i32 %cmp, i32 %new) #0 { +; OUTLINE-ATOMICS: bl __aarch64_cas4_acq ; CHECK-LABEL: val_compare_and_swap: ; CHECK-NEXT: mov x[[ADDR:[0-9]+]], x0 ; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]: @@ -19,6 +21,7 @@ define i32 @val_compare_and_swap(i32* %p, i32 %cmp, i32 %new) #0 { } define i32 @val_compare_and_swap_from_load(i32* %p, i32 %cmp, i32* %pnew) #0 { +; OUTLINE-ATOMICS: bl __aarch64_cas4_acq ; CHECK-LABEL: val_compare_and_swap_from_load: ; CHECK-NEXT: ldr [[NEW:w[0-9]+]], [x2] ; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]: @@ -40,6 +43,7 @@ define i32 @val_compare_and_swap_from_load(i32* %p, i32 %cmp, i32* %pnew) #0 { } define i32 @val_compare_and_swap_rel(i32* %p, i32 %cmp, i32 %new) #0 { +; OUTLINE-ATOMICS: bl __aarch64_cas4_acq_rel ; CHECK-LABEL: val_compare_and_swap_rel: ; CHECK-NEXT: mov x[[ADDR:[0-9]+]], x0 ; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]: @@ -58,6 +62,7 @@ define i32 @val_compare_and_swap_rel(i32* %p, i32 %cmp, i32 %new) #0 { } define i64 @val_compare_and_swap_64(i64* %p, i64 %cmp, i64 %new) #0 { +; OUTLINE-ATOMICS: bl __aarch64_cas8_relax ; CHECK-LABEL: val_compare_and_swap_64: ; CHECK-NEXT: mov x[[ADDR:[0-9]+]], x0 ; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]: @@ -104,6 +109,7 @@ define i64 @fetch_and_nand_64(i64* %p) #0 { } define i32 @fetch_and_or(i32* %p) #0 { +; OUTLINE-ATOMICS: bl __aarch64_ldset4_acq_rel ; CHECK-LABEL: fetch_and_or: ; CHECK: mov [[OLDVAL_REG:w[0-9]+]], #5 ; CHECK: [[TRYBB:.?LBB[0-9_]+]]: @@ -118,6 +124,7 @@ define i32 @fetch_and_or(i32* %p) #0 { } define i64 @fetch_and_or_64(i64* %p) #0 { +; OUTLINE-ATOMICS: bl __aarch64_ldset8_relax ; CHECK: fetch_and_or_64: ; CHECK: mov x[[ADDR:[0-9]+]], x0 ; CHECK: [[TRYBB:.?LBB[0-9_]+]]: @@ -332,6 +339,7 @@ define void @atomic_store_relaxed_32(i32* %p, i32 %off32, i32 %val) #0 { } define void @atomic_store_relaxed_64(i64* %p, i32 %off32, i64 %val) #0 { +; OUTLINE-ATOMICS: bl __aarch64_ldadd4_acq_rel ; CHECK-LABEL: atomic_store_relaxed_64: %ptr_unsigned = getelementptr i64, i64* %p, i32 4095 store atomic i64 %val, i64* %ptr_unsigned monotonic, align 8 diff --git a/llvm/test/CodeGen/AArch64/arm64_32-atomics.ll b/llvm/test/CodeGen/AArch64/arm64_32-atomics.ll index 34682e8..bd7452f 100644 --- a/llvm/test/CodeGen/AArch64/arm64_32-atomics.ll +++ b/llvm/test/CodeGen/AArch64/arm64_32-atomics.ll @@ -1,4 +1,5 @@ ; RUN: llc -mtriple=arm64_32-apple-ios7.0 -o - %s | FileCheck %s +; RUN: llc -mtriple=arm64_32-apple-ios7.0 -mattr=+outline-atomics -o - %s | FileCheck %s -check-prefix=OUTLINE-ATOMICS define i8 @test_load_8(i8* %addr) { ; CHECK-LABAL: test_load_8: @@ -239,6 +240,7 @@ define i32 @test_stlxr_64(i64* %addr, i64 %val) { } define {i8*, i1} @test_cmpxchg_ptr(i8** %addr, i8* %cmp, i8* %new) { +; OUTLINE-ATOMICS: bl ___aarch64_cas4_acq_rel ; CHECK-LABEL: test_cmpxchg_ptr: ; CHECK: [[LOOP:LBB[0-9]+_[0-9]+]]: ; CHECK: ldaxr [[OLD:w[0-9]+]], [x0] diff --git a/llvm/test/CodeGen/AArch64/atomic-ops-lse.ll b/llvm/test/CodeGen/AArch64/atomic-ops-lse.ll index a75614d..c7d3dc0 100644 --- a/llvm/test/CodeGen/AArch64/atomic-ops-lse.ll +++ b/llvm/test/CodeGen/AArch64/atomic-ops-lse.ll @@ -1,4 +1,6 @@ ; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs -mattr=+lse < %s | FileCheck %s +; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs -mattr=+lse -mattr=+outline-atomics < %s | FileCheck %s +; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs -mattr=+outline-atomics < %s | FileCheck %s --check-prefix=OUTLINE-ATOMICS ; RUN: llc -mtriple=aarch64_be-none-linux-gnu -disable-post-ra -verify-machineinstrs -mattr=+lse < %s | FileCheck %s ; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs -mattr=+lse < %s | FileCheck %s --check-prefix=CHECK-REG ; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs -mcpu=saphira < %s | FileCheck %s @@ -16,6 +18,14 @@ define i8 @test_atomic_load_add_i8(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i8: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i8: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw add i8* @var8, i8 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -29,6 +39,14 @@ define i8 @test_atomic_load_add_i8(i8 %offset) nounwind { define i16 @test_atomic_load_add_i16(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i16: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i16: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw add i16* @var16, i16 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -42,6 +60,14 @@ define i16 @test_atomic_load_add_i16(i16 %offset) nounwind { define i32 @test_atomic_load_add_i32(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i32: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i32: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw add i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -55,6 +81,14 @@ define i32 @test_atomic_load_add_i32(i32 %offset) nounwind { define i64 @test_atomic_load_add_i64(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i64: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i64: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw add i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -68,6 +102,14 @@ define i64 @test_atomic_load_add_i64(i64 %offset) nounwind { define void @test_atomic_load_add_i32_noret(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i32_noret: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i32_noret: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw add i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -80,6 +122,14 @@ define void @test_atomic_load_add_i32_noret(i32 %offset) nounwind { define void @test_atomic_load_add_i64_noret(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i64_noret: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i64_noret: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw add i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -92,6 +142,14 @@ define void @test_atomic_load_add_i64_noret(i64 %offset) nounwind { define i8 @test_atomic_load_or_i8(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i8: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i8: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset1_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw or i8* @var8, i8 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -105,6 +163,14 @@ define i8 @test_atomic_load_or_i8(i8 %offset) nounwind { define i16 @test_atomic_load_or_i16(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i16: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i16: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset2_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw or i16* @var16, i16 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -118,6 +184,14 @@ define i16 @test_atomic_load_or_i16(i16 %offset) nounwind { define i32 @test_atomic_load_or_i32(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i32: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i32: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw or i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -131,6 +205,14 @@ define i32 @test_atomic_load_or_i32(i32 %offset) nounwind { define i64 @test_atomic_load_or_i64(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i64: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i64: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw or i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -144,6 +226,14 @@ define i64 @test_atomic_load_or_i64(i64 %offset) nounwind { define void @test_atomic_load_or_i32_noret(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i32_noret: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i32_noret: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw or i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -156,6 +246,14 @@ define void @test_atomic_load_or_i32_noret(i32 %offset) nounwind { define void @test_atomic_load_or_i64_noret(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i64_noret: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i64_noret: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw or i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -168,6 +266,14 @@ define void @test_atomic_load_or_i64_noret(i64 %offset) nounwind { define i8 @test_atomic_load_xor_i8(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i8: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i8: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor1_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xor i8* @var8, i8 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -181,6 +287,14 @@ define i8 @test_atomic_load_xor_i8(i8 %offset) nounwind { define i16 @test_atomic_load_xor_i16(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i16: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i16: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor2_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xor i16* @var16, i16 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -194,6 +308,14 @@ define i16 @test_atomic_load_xor_i16(i16 %offset) nounwind { define i32 @test_atomic_load_xor_i32(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i32: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i32: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xor i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -207,6 +329,14 @@ define i32 @test_atomic_load_xor_i32(i32 %offset) nounwind { define i64 @test_atomic_load_xor_i64(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i64: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i64: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xor i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -220,6 +350,14 @@ define i64 @test_atomic_load_xor_i64(i64 %offset) nounwind { define void @test_atomic_load_xor_i32_noret(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i32_noret: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i32_noret: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw xor i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -232,6 +370,14 @@ define void @test_atomic_load_xor_i32_noret(i32 %offset) nounwind { define void @test_atomic_load_xor_i64_noret(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i64_noret: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i64_noret: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw xor i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -244,6 +390,21 @@ define void @test_atomic_load_xor_i64_noret(i64 %offset) nounwind { define i8 @test_atomic_load_min_i8(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i8: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i8: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var8 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: .LBB18_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9] +; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb +; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le +; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB18_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw min i8* @var8, i8 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -257,6 +418,21 @@ define i8 @test_atomic_load_min_i8(i8 %offset) nounwind { define i16 @test_atomic_load_min_i16(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i16: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i16: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var16 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: .LBB19_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9] +; OUTLINE-ATOMICS-NEXT: sxth w8, w10 +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth +; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le +; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB19_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw min i16* @var16, i16 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -270,6 +446,20 @@ define i16 @test_atomic_load_min_i16(i16 %offset) nounwind { define i32 @test_atomic_load_min_i32(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i32: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var32 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB20_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0 +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, le +; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB20_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw min i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -283,6 +473,20 @@ define i32 @test_atomic_load_min_i32(i32 %offset) nounwind { define i64 @test_atomic_load_min_i64(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i64: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var64 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB21_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp x8, x0 +; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, le +; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB21_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov x0, x8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw min i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -296,6 +500,19 @@ define i64 @test_atomic_load_min_i64(i64 %offset) nounwind { define void @test_atomic_load_min_i32_noret(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i32_noret: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_noret: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var32 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB22_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp w9, w0 +; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, le +; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB22_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw min i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -308,6 +525,19 @@ define void @test_atomic_load_min_i32_noret(i32 %offset) nounwind { define void @test_atomic_load_min_i64_noret(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i64_noret: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_noret: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var64 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB23_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp x9, x0 +; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, le +; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB23_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw min i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -320,6 +550,20 @@ define void @test_atomic_load_min_i64_noret(i64 %offset) nounwind { define i8 @test_atomic_load_umin_i8(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i8: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i8: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var8 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: .LBB24_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrb w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls +; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB24_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umin i8* @var8, i8 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -333,6 +577,20 @@ define i8 @test_atomic_load_umin_i8(i8 %offset) nounwind { define i16 @test_atomic_load_umin_i16(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i16: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i16: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var16 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: .LBB25_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrh w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls +; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB25_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umin i16* @var16, i16 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -346,6 +604,20 @@ define i16 @test_atomic_load_umin_i16(i16 %offset) nounwind { define i32 @test_atomic_load_umin_i32(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i32: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var32 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB26_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0 +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls +; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB26_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umin i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -359,6 +631,20 @@ define i32 @test_atomic_load_umin_i32(i32 %offset) nounwind { define i64 @test_atomic_load_umin_i64(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i64: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var64 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB27_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp x8, x0 +; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, ls +; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB27_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov x0, x8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umin i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -372,6 +658,19 @@ define i64 @test_atomic_load_umin_i64(i64 %offset) nounwind { define void @test_atomic_load_umin_i32_noret(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i32_noret: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_noret: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var32 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB28_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp w9, w0 +; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, ls +; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB28_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw umin i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -384,6 +683,19 @@ define void @test_atomic_load_umin_i32_noret(i32 %offset) nounwind { define void @test_atomic_load_umin_i64_noret(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i64_noret: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_noret: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var64 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB29_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp x9, x0 +; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, ls +; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB29_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw umin i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -396,6 +708,21 @@ define void @test_atomic_load_umin_i64_noret(i64 %offset) nounwind { define i8 @test_atomic_load_max_i8(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i8: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i8: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var8 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: .LBB30_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9] +; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb +; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt +; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB30_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw max i8* @var8, i8 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -409,6 +736,21 @@ define i8 @test_atomic_load_max_i8(i8 %offset) nounwind { define i16 @test_atomic_load_max_i16(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i16: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i16: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var16 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: .LBB31_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9] +; OUTLINE-ATOMICS-NEXT: sxth w8, w10 +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth +; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt +; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB31_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw max i16* @var16, i16 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -422,6 +764,20 @@ define i16 @test_atomic_load_max_i16(i16 %offset) nounwind { define i32 @test_atomic_load_max_i32(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i32: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var32 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB32_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0 +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, gt +; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB32_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw max i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -435,6 +791,20 @@ define i32 @test_atomic_load_max_i32(i32 %offset) nounwind { define i64 @test_atomic_load_max_i64(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i64: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var64 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB33_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp x8, x0 +; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, gt +; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB33_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov x0, x8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw max i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -448,6 +818,19 @@ define i64 @test_atomic_load_max_i64(i64 %offset) nounwind { define void @test_atomic_load_max_i32_noret(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i32_noret: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_noret: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var32 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB34_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp w9, w0 +; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, gt +; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB34_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw max i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -460,6 +843,19 @@ define void @test_atomic_load_max_i32_noret(i32 %offset) nounwind { define void @test_atomic_load_max_i64_noret(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i64_noret: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_noret: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var64 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB35_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp x9, x0 +; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, gt +; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB35_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw max i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -472,6 +868,20 @@ define void @test_atomic_load_max_i64_noret(i64 %offset) nounwind { define i8 @test_atomic_load_umax_i8(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i8: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i8: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var8 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: .LBB36_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrb w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi +; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB36_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umax i8* @var8, i8 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -485,6 +895,20 @@ define i8 @test_atomic_load_umax_i8(i8 %offset) nounwind { define i16 @test_atomic_load_umax_i16(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i16: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i16: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var16 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: .LBB37_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrh w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi +; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB37_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umax i16* @var16, i16 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -498,6 +922,20 @@ define i16 @test_atomic_load_umax_i16(i16 %offset) nounwind { define i32 @test_atomic_load_umax_i32(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i32: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var32 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB38_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0 +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi +; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB38_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umax i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -511,6 +949,20 @@ define i32 @test_atomic_load_umax_i32(i32 %offset) nounwind { define i64 @test_atomic_load_umax_i64(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i64: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var64 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB39_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp x8, x0 +; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, hi +; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB39_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov x0, x8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umax i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -524,6 +976,19 @@ define i64 @test_atomic_load_umax_i64(i64 %offset) nounwind { define void @test_atomic_load_umax_i32_noret(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i32_noret: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_noret: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var32 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB40_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp w9, w0 +; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, hi +; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB40_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw umax i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -536,6 +1001,19 @@ define void @test_atomic_load_umax_i32_noret(i32 %offset) nounwind { define void @test_atomic_load_umax_i64_noret(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i64_noret: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_noret: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var64 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB41_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp x9, x0 +; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, hi +; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB41_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw umax i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -548,6 +1026,14 @@ define void @test_atomic_load_umax_i64_noret(i64 %offset) nounwind { define i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i8: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i8: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp1_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xchg i8* @var8, i8 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -561,6 +1047,14 @@ define i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind { define i16 @test_atomic_load_xchg_i16(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i16: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i16: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp2_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xchg i16* @var16, i16 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -574,6 +1068,14 @@ define i16 @test_atomic_load_xchg_i16(i16 %offset) nounwind { define i32 @test_atomic_load_xchg_i32(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i32: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i32: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xchg i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -587,6 +1089,14 @@ define i32 @test_atomic_load_xchg_i32(i32 %offset) nounwind { define i64 @test_atomic_load_xchg_i64(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i64: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i64: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xchg i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -600,6 +1110,14 @@ define i64 @test_atomic_load_xchg_i64(i64 %offset) nounwind { define void @test_atomic_load_xchg_i32_noret(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i32_noret: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i32_noret: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw xchg i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -613,6 +1131,14 @@ define void @test_atomic_load_xchg_i32_noret(i32 %offset) nounwind { define void @test_atomic_load_xchg_i64_noret(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i64_noret: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i64_noret: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw xchg i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -626,6 +1152,14 @@ define void @test_atomic_load_xchg_i64_noret(i64 %offset) nounwind { define i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i8: +; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i8: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x2, var8 +; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas1_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %pair = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire acquire %old = extractvalue { i8, i1 } %pair, 0 @@ -640,6 +1174,17 @@ define i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind { define i1 @test_atomic_cmpxchg_i8_1(i8 %wanted, i8 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i8_1: +; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i8_1: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: stp x30, x19, [sp, #-16]! // 16-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mov w19, w0 +; OUTLINE-ATOMICS-NEXT: adrp x2, var8 +; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas1_acq +; OUTLINE-ATOMICS-NEXT: cmp w0, w19, uxtb +; OUTLINE-ATOMICS-NEXT: cset w0, eq +; OUTLINE-ATOMICS-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %pair = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire acquire %success = extractvalue { i8, i1 } %pair, 1 @@ -656,6 +1201,14 @@ define i1 @test_atomic_cmpxchg_i8_1(i8 %wanted, i8 %new) nounwind { define i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i16: +; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i16: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x2, var16 +; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas2_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %pair = cmpxchg i16* @var16, i16 %wanted, i16 %new acquire acquire %old = extractvalue { i16, i1 } %pair, 0 @@ -670,6 +1223,17 @@ define i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind { define i1 @test_atomic_cmpxchg_i16_1(i16 %wanted, i16 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i16_1: +; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i16_1: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: stp x30, x19, [sp, #-16]! // 16-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mov w19, w0 +; OUTLINE-ATOMICS-NEXT: adrp x2, var16 +; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas2_acq +; OUTLINE-ATOMICS-NEXT: cmp w0, w19, uxth +; OUTLINE-ATOMICS-NEXT: cset w0, eq +; OUTLINE-ATOMICS-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %pair = cmpxchg i16* @var16, i16 %wanted, i16 %new acquire acquire %success = extractvalue { i16, i1 } %pair, 1 @@ -687,6 +1251,14 @@ define i1 @test_atomic_cmpxchg_i16_1(i16 %wanted, i16 %new) nounwind { define i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i32: +; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i32: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x2, var32 +; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas4_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %pair = cmpxchg i32* @var32, i32 %wanted, i32 %new acquire acquire %old = extractvalue { i32, i1 } %pair, 0 @@ -702,6 +1274,14 @@ define i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind { define i64 @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i64: +; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i64: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x2, var64 +; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas8_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %pair = cmpxchg i64* @var64, i64 %wanted, i64 %new acquire acquire %old = extractvalue { i64, i1 } %pair, 0 @@ -717,6 +1297,14 @@ define i64 @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind { define i128 @test_atomic_cmpxchg_i128(i128 %wanted, i128 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i128: +; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i128: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x4, var128 +; OUTLINE-ATOMICS-NEXT: add x4, x4, :lo12:var128 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas16_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %pair = cmpxchg i128* @var128, i128 %wanted, i128 %new acquire acquire %old = extractvalue { i128, i1 } %pair, 0 @@ -732,6 +1320,15 @@ define i128 @test_atomic_cmpxchg_i128(i128 %wanted, i128 %new) nounwind { define i8 @test_atomic_load_sub_i8(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i8: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i8: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i8* @var8, i8 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] @@ -746,6 +1343,15 @@ define i8 @test_atomic_load_sub_i8(i8 %offset) nounwind { define i16 @test_atomic_load_sub_i16(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i16: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i16: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i16* @var16, i16 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] @@ -760,6 +1366,15 @@ define i16 @test_atomic_load_sub_i16(i16 %offset) nounwind { define i32 @test_atomic_load_sub_i32(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i32: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] @@ -774,6 +1389,15 @@ define i32 @test_atomic_load_sub_i32(i32 %offset) nounwind { define i64 @test_atomic_load_sub_i64(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i64: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg x0, x0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]] @@ -788,6 +1412,15 @@ define i64 @test_atomic_load_sub_i64(i64 %offset) nounwind { define void @test_atomic_load_sub_i32_noret(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i32_noret: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32_noret: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw sub i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] @@ -802,6 +1435,15 @@ define void @test_atomic_load_sub_i32_noret(i32 %offset) nounwind { define void @test_atomic_load_sub_i64_noret(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i64_noret: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64_noret: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg x0, x0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw sub i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]] @@ -816,6 +1458,15 @@ define void @test_atomic_load_sub_i64_noret(i64 %offset) nounwind { define i8 @test_atomic_load_sub_i8_neg_imm() nounwind { ; CHECK-LABEL: test_atomic_load_sub_i8_neg_imm: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i8_neg_imm: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: mov w0, #1 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i8* @var8, i8 -1 seq_cst ; CHECK-NOT: dmb @@ -830,6 +1481,15 @@ define i8 @test_atomic_load_sub_i8_neg_imm() nounwind { define i16 @test_atomic_load_sub_i16_neg_imm() nounwind { ; CHECK-LABEL: test_atomic_load_sub_i16_neg_imm: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i16_neg_imm: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: mov w0, #1 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i16* @var16, i16 -1 seq_cst ; CHECK-NOT: dmb @@ -844,6 +1504,15 @@ define i16 @test_atomic_load_sub_i16_neg_imm() nounwind { define i32 @test_atomic_load_sub_i32_neg_imm() nounwind { ; CHECK-LABEL: test_atomic_load_sub_i32_neg_imm: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32_neg_imm: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: mov w0, #1 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i32* @var32, i32 -1 seq_cst ; CHECK-NOT: dmb @@ -858,6 +1527,15 @@ define i32 @test_atomic_load_sub_i32_neg_imm() nounwind { define i64 @test_atomic_load_sub_i64_neg_imm() nounwind { ; CHECK-LABEL: test_atomic_load_sub_i64_neg_imm: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64_neg_imm: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: mov w0, #1 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i64* @var64, i64 -1 seq_cst ; CHECK-NOT: dmb @@ -872,6 +1550,14 @@ define i64 @test_atomic_load_sub_i64_neg_imm() nounwind { define i8 @test_atomic_load_sub_i8_neg_arg(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i8_neg_arg: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i8_neg_arg: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %neg = sub i8 0, %offset %old = atomicrmw sub i8* @var8, i8 %neg seq_cst @@ -886,6 +1572,14 @@ define i8 @test_atomic_load_sub_i8_neg_arg(i8 %offset) nounwind { define i16 @test_atomic_load_sub_i16_neg_arg(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i16_neg_arg: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i16_neg_arg: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %neg = sub i16 0, %offset %old = atomicrmw sub i16* @var16, i16 %neg seq_cst @@ -900,6 +1594,14 @@ define i16 @test_atomic_load_sub_i16_neg_arg(i16 %offset) nounwind { define i32 @test_atomic_load_sub_i32_neg_arg(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i32_neg_arg: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32_neg_arg: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %neg = sub i32 0, %offset %old = atomicrmw sub i32* @var32, i32 %neg seq_cst @@ -914,6 +1616,14 @@ define i32 @test_atomic_load_sub_i32_neg_arg(i32 %offset) nounwind { define i64 @test_atomic_load_sub_i64_neg_arg(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i64_neg_arg: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64_neg_arg: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %neg = sub i64 0, %offset %old = atomicrmw sub i64* @var64, i64 %neg seq_cst @@ -928,6 +1638,15 @@ define i64 @test_atomic_load_sub_i64_neg_arg(i64 %offset) nounwind { define i8 @test_atomic_load_and_i8(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i8: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i8: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr1_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i8* @var8, i8 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] @@ -941,6 +1660,15 @@ define i8 @test_atomic_load_and_i8(i8 %offset) nounwind { define i16 @test_atomic_load_and_i16(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i16: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i16: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr2_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i16* @var16, i16 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] @@ -954,6 +1682,15 @@ define i16 @test_atomic_load_and_i16(i16 %offset) nounwind { define i32 @test_atomic_load_and_i32(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i32: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] @@ -967,6 +1704,15 @@ define i32 @test_atomic_load_and_i32(i32 %offset) nounwind { define i64 @test_atomic_load_and_i64(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i64: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn x0, x0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]] @@ -980,6 +1726,15 @@ define i64 @test_atomic_load_and_i64(i64 %offset) nounwind { define i8 @test_atomic_load_and_i8_inv_imm() nounwind { ; CHECK-LABEL: test_atomic_load_and_i8_inv_imm: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i8_inv_imm: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: mov w0, #1 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr1_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i8* @var8, i8 -2 seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -992,6 +1747,15 @@ define i8 @test_atomic_load_and_i8_inv_imm() nounwind { define i16 @test_atomic_load_and_i16_inv_imm() nounwind { ; CHECK-LABEL: test_atomic_load_and_i16_inv_imm: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i16_inv_imm: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: mov w0, #1 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr2_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i16* @var16, i16 -2 seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -1004,6 +1768,15 @@ define i16 @test_atomic_load_and_i16_inv_imm() nounwind { define i32 @test_atomic_load_and_i32_inv_imm() nounwind { ; CHECK-LABEL: test_atomic_load_and_i32_inv_imm: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32_inv_imm: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: mov w0, #1 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i32* @var32, i32 -2 seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -1016,6 +1789,15 @@ define i32 @test_atomic_load_and_i32_inv_imm() nounwind { define i64 @test_atomic_load_and_i64_inv_imm() nounwind { ; CHECK-LABEL: test_atomic_load_and_i64_inv_imm: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64_inv_imm: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: mov w0, #1 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i64* @var64, i64 -2 seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -1028,6 +1810,14 @@ define i64 @test_atomic_load_and_i64_inv_imm() nounwind { define i8 @test_atomic_load_and_i8_inv_arg(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i8_inv_arg: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i8_inv_arg: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr1_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %inv = xor i8 %offset, -1 %old = atomicrmw and i8* @var8, i8 %inv seq_cst ; CHECK-NOT: dmb @@ -1040,6 +1830,14 @@ define i8 @test_atomic_load_and_i8_inv_arg(i8 %offset) nounwind { define i16 @test_atomic_load_and_i16_inv_arg(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i16_inv_arg: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i16_inv_arg: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr2_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %inv = xor i16 %offset, -1 %old = atomicrmw and i16* @var16, i16 %inv seq_cst ; CHECK-NOT: dmb @@ -1052,6 +1850,14 @@ define i16 @test_atomic_load_and_i16_inv_arg(i16 %offset) nounwind { define i32 @test_atomic_load_and_i32_inv_arg(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i32_inv_arg: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32_inv_arg: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %inv = xor i32 %offset, -1 %old = atomicrmw and i32* @var32, i32 %inv seq_cst ; CHECK-NOT: dmb @@ -1064,6 +1870,14 @@ define i32 @test_atomic_load_and_i32_inv_arg(i32 %offset) nounwind { define i64 @test_atomic_load_and_i64_inv_arg(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i64_inv_arg: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64_inv_arg: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %inv = xor i64 %offset, -1 %old = atomicrmw and i64* @var64, i64 %inv seq_cst ; CHECK-NOT: dmb @@ -1076,6 +1890,15 @@ define i64 @test_atomic_load_and_i64_inv_arg(i64 %offset) nounwind { define void @test_atomic_load_and_i32_noret(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i32_noret: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32_noret: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw and i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] @@ -1089,6 +1912,15 @@ define void @test_atomic_load_and_i32_noret(i32 %offset) nounwind { define void @test_atomic_load_and_i64_noret(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i64_noret: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64_noret: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn x0, x0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw and i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]] @@ -1102,6 +1934,14 @@ define void @test_atomic_load_and_i64_noret(i64 %offset) nounwind { define i8 @test_atomic_load_add_i8_acq_rel(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i8_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i8_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw add i8* @var8, i8 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -1115,6 +1955,14 @@ define i8 @test_atomic_load_add_i8_acq_rel(i8 %offset) nounwind { define i16 @test_atomic_load_add_i16_acq_rel(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i16_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i16_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw add i16* @var16, i16 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -1128,6 +1976,14 @@ define i16 @test_atomic_load_add_i16_acq_rel(i16 %offset) nounwind { define i32 @test_atomic_load_add_i32_acq_rel(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i32_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i32_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw add i32* @var32, i32 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -1141,6 +1997,14 @@ define i32 @test_atomic_load_add_i32_acq_rel(i32 %offset) nounwind { define i64 @test_atomic_load_add_i64_acq_rel(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i64_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i64_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw add i64* @var64, i64 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -1154,6 +2018,14 @@ define i64 @test_atomic_load_add_i64_acq_rel(i64 %offset) nounwind { define void @test_atomic_load_add_i32_noret_acq_rel(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i32_noret_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i32_noret_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw add i32* @var32, i32 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -1166,6 +2038,14 @@ define void @test_atomic_load_add_i32_noret_acq_rel(i32 %offset) nounwind { define void @test_atomic_load_add_i64_noret_acq_rel(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i64_noret_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i64_noret_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw add i64* @var64, i64 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -1178,6 +2058,14 @@ define void @test_atomic_load_add_i64_noret_acq_rel(i64 %offset) nounwind { define i8 @test_atomic_load_add_i8_acquire(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i8_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i8_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw add i8* @var8, i8 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -1191,6 +2079,14 @@ define i8 @test_atomic_load_add_i8_acquire(i8 %offset) nounwind { define i16 @test_atomic_load_add_i16_acquire(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i16_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i16_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw add i16* @var16, i16 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -1204,6 +2100,14 @@ define i16 @test_atomic_load_add_i16_acquire(i16 %offset) nounwind { define i32 @test_atomic_load_add_i32_acquire(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i32_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i32_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw add i32* @var32, i32 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -1217,6 +2121,14 @@ define i32 @test_atomic_load_add_i32_acquire(i32 %offset) nounwind { define i64 @test_atomic_load_add_i64_acquire(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i64_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i64_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw add i64* @var64, i64 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -1230,6 +2142,14 @@ define i64 @test_atomic_load_add_i64_acquire(i64 %offset) nounwind { define void @test_atomic_load_add_i32_noret_acquire(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i32_noret_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i32_noret_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw add i32* @var32, i32 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -1242,6 +2162,14 @@ define void @test_atomic_load_add_i32_noret_acquire(i32 %offset) nounwind { define void @test_atomic_load_add_i64_noret_acquire(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i64_noret_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i64_noret_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw add i64* @var64, i64 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -1254,6 +2182,14 @@ define void @test_atomic_load_add_i64_noret_acquire(i64 %offset) nounwind { define i8 @test_atomic_load_add_i8_monotonic(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i8_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i8_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw add i8* @var8, i8 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -1267,6 +2203,14 @@ define i8 @test_atomic_load_add_i8_monotonic(i8 %offset) nounwind { define i16 @test_atomic_load_add_i16_monotonic(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i16_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i16_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw add i16* @var16, i16 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -1280,6 +2224,14 @@ define i16 @test_atomic_load_add_i16_monotonic(i16 %offset) nounwind { define i32 @test_atomic_load_add_i32_monotonic(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i32_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i32_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw add i32* @var32, i32 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -1293,6 +2245,14 @@ define i32 @test_atomic_load_add_i32_monotonic(i32 %offset) nounwind { define i64 @test_atomic_load_add_i64_monotonic(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i64_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i64_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw add i64* @var64, i64 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -1306,6 +2266,14 @@ define i64 @test_atomic_load_add_i64_monotonic(i64 %offset) nounwind { define void @test_atomic_load_add_i32_noret_monotonic(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i32_noret_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i32_noret_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw add i32* @var32, i32 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -1318,6 +2286,14 @@ define void @test_atomic_load_add_i32_noret_monotonic(i32 %offset) nounwind { define void @test_atomic_load_add_i64_noret_monotonic(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i64_noret_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i64_noret_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw add i64* @var64, i64 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -1330,6 +2306,14 @@ define void @test_atomic_load_add_i64_noret_monotonic(i64 %offset) nounwind { define i8 @test_atomic_load_add_i8_release(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i8_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i8_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw add i8* @var8, i8 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -1343,6 +2327,14 @@ define i8 @test_atomic_load_add_i8_release(i8 %offset) nounwind { define i16 @test_atomic_load_add_i16_release(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i16_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i16_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw add i16* @var16, i16 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -1356,6 +2348,14 @@ define i16 @test_atomic_load_add_i16_release(i16 %offset) nounwind { define i32 @test_atomic_load_add_i32_release(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i32_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i32_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw add i32* @var32, i32 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -1369,6 +2369,14 @@ define i32 @test_atomic_load_add_i32_release(i32 %offset) nounwind { define i64 @test_atomic_load_add_i64_release(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i64_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i64_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw add i64* @var64, i64 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -1382,6 +2390,14 @@ define i64 @test_atomic_load_add_i64_release(i64 %offset) nounwind { define void @test_atomic_load_add_i32_noret_release(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i32_noret_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i32_noret_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw add i32* @var32, i32 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -1394,6 +2410,14 @@ define void @test_atomic_load_add_i32_noret_release(i32 %offset) nounwind { define void @test_atomic_load_add_i64_noret_release(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i64_noret_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i64_noret_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw add i64* @var64, i64 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -1406,6 +2430,14 @@ define void @test_atomic_load_add_i64_noret_release(i64 %offset) nounwind { define i8 @test_atomic_load_add_i8_seq_cst(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i8_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i8_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw add i8* @var8, i8 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -1419,6 +2451,14 @@ define i8 @test_atomic_load_add_i8_seq_cst(i8 %offset) nounwind { define i16 @test_atomic_load_add_i16_seq_cst(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i16_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i16_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw add i16* @var16, i16 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -1432,6 +2472,14 @@ define i16 @test_atomic_load_add_i16_seq_cst(i16 %offset) nounwind { define i32 @test_atomic_load_add_i32_seq_cst(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i32_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i32_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw add i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -1445,6 +2493,14 @@ define i32 @test_atomic_load_add_i32_seq_cst(i32 %offset) nounwind { define i64 @test_atomic_load_add_i64_seq_cst(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i64_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i64_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw add i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -1458,6 +2514,14 @@ define i64 @test_atomic_load_add_i64_seq_cst(i64 %offset) nounwind { define void @test_atomic_load_add_i32_noret_seq_cst(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i32_noret_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i32_noret_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw add i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -1470,6 +2534,14 @@ define void @test_atomic_load_add_i32_noret_seq_cst(i32 %offset) nounwind { define void @test_atomic_load_add_i64_noret_seq_cst(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i64_noret_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i64_noret_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw add i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -1482,6 +2554,15 @@ define void @test_atomic_load_add_i64_noret_seq_cst(i64 %offset) nounwind { define i8 @test_atomic_load_and_i8_acq_rel(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i8_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i8_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr1_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i8* @var8, i8 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] @@ -1495,6 +2576,15 @@ define i8 @test_atomic_load_and_i8_acq_rel(i8 %offset) nounwind { define i16 @test_atomic_load_and_i16_acq_rel(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i16_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i16_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr2_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i16* @var16, i16 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] @@ -1508,6 +2598,15 @@ define i16 @test_atomic_load_and_i16_acq_rel(i16 %offset) nounwind { define i32 @test_atomic_load_and_i32_acq_rel(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i32_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i32* @var32, i32 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] @@ -1521,6 +2620,15 @@ define i32 @test_atomic_load_and_i32_acq_rel(i32 %offset) nounwind { define i64 @test_atomic_load_and_i64_acq_rel(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i64_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn x0, x0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i64* @var64, i64 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]] @@ -1534,6 +2642,15 @@ define i64 @test_atomic_load_and_i64_acq_rel(i64 %offset) nounwind { define void @test_atomic_load_and_i32_noret_acq_rel(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i32_noret_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32_noret_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw and i32* @var32, i32 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] @@ -1547,6 +2664,15 @@ define void @test_atomic_load_and_i32_noret_acq_rel(i32 %offset) nounwind { define void @test_atomic_load_and_i64_noret_acq_rel(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i64_noret_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64_noret_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn x0, x0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw and i64* @var64, i64 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]] @@ -1560,6 +2686,15 @@ define void @test_atomic_load_and_i64_noret_acq_rel(i64 %offset) nounwind { define i8 @test_atomic_load_and_i8_acquire(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i8_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i8_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr1_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i8* @var8, i8 %offset acquire ; CHECK-NOT: dmb ; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] @@ -1573,6 +2708,15 @@ define i8 @test_atomic_load_and_i8_acquire(i8 %offset) nounwind { define i16 @test_atomic_load_and_i16_acquire(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i16_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i16_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr2_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i16* @var16, i16 %offset acquire ; CHECK-NOT: dmb ; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] @@ -1586,6 +2730,15 @@ define i16 @test_atomic_load_and_i16_acquire(i16 %offset) nounwind { define i32 @test_atomic_load_and_i32_acquire(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i32_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i32* @var32, i32 %offset acquire ; CHECK-NOT: dmb ; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] @@ -1599,6 +2752,15 @@ define i32 @test_atomic_load_and_i32_acquire(i32 %offset) nounwind { define i64 @test_atomic_load_and_i64_acquire(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i64_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn x0, x0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i64* @var64, i64 %offset acquire ; CHECK-NOT: dmb ; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]] @@ -1612,6 +2774,15 @@ define i64 @test_atomic_load_and_i64_acquire(i64 %offset) nounwind { define void @test_atomic_load_and_i32_noret_acquire(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i32_noret_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32_noret_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw and i32* @var32, i32 %offset acquire ; CHECK-NOT: dmb ; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] @@ -1625,6 +2796,15 @@ define void @test_atomic_load_and_i32_noret_acquire(i32 %offset) nounwind { define void @test_atomic_load_and_i64_noret_acquire(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i64_noret_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64_noret_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn x0, x0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw and i64* @var64, i64 %offset acquire ; CHECK-NOT: dmb ; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]] @@ -1638,6 +2818,15 @@ define void @test_atomic_load_and_i64_noret_acquire(i64 %offset) nounwind { define i8 @test_atomic_load_and_i8_monotonic(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i8_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i8_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr1_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i8* @var8, i8 %offset monotonic ; CHECK-NOT: dmb ; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] @@ -1651,6 +2840,15 @@ define i8 @test_atomic_load_and_i8_monotonic(i8 %offset) nounwind { define i16 @test_atomic_load_and_i16_monotonic(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i16_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i16_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr2_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i16* @var16, i16 %offset monotonic ; CHECK-NOT: dmb ; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] @@ -1664,6 +2862,15 @@ define i16 @test_atomic_load_and_i16_monotonic(i16 %offset) nounwind { define i32 @test_atomic_load_and_i32_monotonic(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i32_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i32* @var32, i32 %offset monotonic ; CHECK-NOT: dmb ; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] @@ -1677,6 +2884,15 @@ define i32 @test_atomic_load_and_i32_monotonic(i32 %offset) nounwind { define i64 @test_atomic_load_and_i64_monotonic(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i64_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn x0, x0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i64* @var64, i64 %offset monotonic ; CHECK-NOT: dmb ; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]] @@ -1690,6 +2906,15 @@ define i64 @test_atomic_load_and_i64_monotonic(i64 %offset) nounwind { define void @test_atomic_load_and_i32_noret_monotonic(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i32_noret_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32_noret_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw and i32* @var32, i32 %offset monotonic ; CHECK-NOT: dmb ; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] @@ -1703,6 +2928,15 @@ define void @test_atomic_load_and_i32_noret_monotonic(i32 %offset) nounwind { define void @test_atomic_load_and_i64_noret_monotonic(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i64_noret_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64_noret_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn x0, x0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw and i64* @var64, i64 %offset monotonic ; CHECK-NOT: dmb ; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]] @@ -1716,6 +2950,15 @@ define void @test_atomic_load_and_i64_noret_monotonic(i64 %offset) nounwind { define i8 @test_atomic_load_and_i8_release(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i8_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i8_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr1_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i8* @var8, i8 %offset release ; CHECK-NOT: dmb ; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] @@ -1729,6 +2972,15 @@ define i8 @test_atomic_load_and_i8_release(i8 %offset) nounwind { define i16 @test_atomic_load_and_i16_release(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i16_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i16_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr2_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i16* @var16, i16 %offset release ; CHECK-NOT: dmb ; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] @@ -1742,6 +2994,15 @@ define i16 @test_atomic_load_and_i16_release(i16 %offset) nounwind { define i32 @test_atomic_load_and_i32_release(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i32_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i32* @var32, i32 %offset release ; CHECK-NOT: dmb ; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] @@ -1755,6 +3016,15 @@ define i32 @test_atomic_load_and_i32_release(i32 %offset) nounwind { define i64 @test_atomic_load_and_i64_release(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i64_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn x0, x0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i64* @var64, i64 %offset release ; CHECK-NOT: dmb ; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]] @@ -1768,6 +3038,15 @@ define i64 @test_atomic_load_and_i64_release(i64 %offset) nounwind { define void @test_atomic_load_and_i32_noret_release(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i32_noret_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32_noret_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw and i32* @var32, i32 %offset release ; CHECK-NOT: dmb ; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] @@ -1781,6 +3060,15 @@ define void @test_atomic_load_and_i32_noret_release(i32 %offset) nounwind { define void @test_atomic_load_and_i64_noret_release(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i64_noret_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64_noret_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn x0, x0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw and i64* @var64, i64 %offset release ; CHECK-NOT: dmb ; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]] @@ -1794,6 +3082,15 @@ define void @test_atomic_load_and_i64_noret_release(i64 %offset) nounwind { define i8 @test_atomic_load_and_i8_seq_cst(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i8_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i8_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr1_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i8* @var8, i8 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] @@ -1807,6 +3104,15 @@ define i8 @test_atomic_load_and_i8_seq_cst(i8 %offset) nounwind { define i16 @test_atomic_load_and_i16_seq_cst(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i16_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i16_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr2_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i16* @var16, i16 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] @@ -1820,6 +3126,15 @@ define i16 @test_atomic_load_and_i16_seq_cst(i16 %offset) nounwind { define i32 @test_atomic_load_and_i32_seq_cst(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i32_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] @@ -1833,6 +3148,15 @@ define i32 @test_atomic_load_and_i32_seq_cst(i32 %offset) nounwind { define i64 @test_atomic_load_and_i64_seq_cst(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i64_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn x0, x0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw and i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]] @@ -1846,6 +3170,15 @@ define i64 @test_atomic_load_and_i64_seq_cst(i64 %offset) nounwind { define void @test_atomic_load_and_i32_noret_seq_cst(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i32_noret_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32_noret_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw and i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] @@ -1859,6 +3192,15 @@ define void @test_atomic_load_and_i32_noret_seq_cst(i32 %offset) nounwind { define void @test_atomic_load_and_i64_noret_seq_cst(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i64_noret_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64_noret_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: mvn x0, x0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw and i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]] @@ -1872,6 +3214,14 @@ define void @test_atomic_load_and_i64_noret_seq_cst(i64 %offset) nounwind { define i8 @test_atomic_cmpxchg_i8_acquire(i8 %wanted, i8 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i8_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i8_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x2, var8 +; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas1_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %pair = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire acquire %old = extractvalue { i8, i1 } %pair, 0 @@ -1887,6 +3237,14 @@ define i8 @test_atomic_cmpxchg_i8_acquire(i8 %wanted, i8 %new) nounwind { define i16 @test_atomic_cmpxchg_i16_acquire(i16 %wanted, i16 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i16_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i16_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x2, var16 +; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas2_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %pair = cmpxchg i16* @var16, i16 %wanted, i16 %new acquire acquire %old = extractvalue { i16, i1 } %pair, 0 @@ -1902,6 +3260,14 @@ define i16 @test_atomic_cmpxchg_i16_acquire(i16 %wanted, i16 %new) nounwind { define i32 @test_atomic_cmpxchg_i32_acquire(i32 %wanted, i32 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i32_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i32_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x2, var32 +; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas4_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %pair = cmpxchg i32* @var32, i32 %wanted, i32 %new acquire acquire %old = extractvalue { i32, i1 } %pair, 0 @@ -1917,6 +3283,14 @@ define i32 @test_atomic_cmpxchg_i32_acquire(i32 %wanted, i32 %new) nounwind { define i64 @test_atomic_cmpxchg_i64_acquire(i64 %wanted, i64 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i64_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i64_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x2, var64 +; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas8_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %pair = cmpxchg i64* @var64, i64 %wanted, i64 %new acquire acquire %old = extractvalue { i64, i1 } %pair, 0 @@ -1932,6 +3306,14 @@ define i64 @test_atomic_cmpxchg_i64_acquire(i64 %wanted, i64 %new) nounwind { define i128 @test_atomic_cmpxchg_i128_acquire(i128 %wanted, i128 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i128_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i128_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x4, var128 +; OUTLINE-ATOMICS-NEXT: add x4, x4, :lo12:var128 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas16_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %pair = cmpxchg i128* @var128, i128 %wanted, i128 %new acquire acquire %old = extractvalue { i128, i1 } %pair, 0 @@ -1947,6 +3329,14 @@ define i128 @test_atomic_cmpxchg_i128_acquire(i128 %wanted, i128 %new) nounwind define i8 @test_atomic_cmpxchg_i8_monotonic(i8 %wanted, i8 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i8_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i8_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x2, var8 +; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas1_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %pair = cmpxchg i8* @var8, i8 %wanted, i8 %new monotonic monotonic %old = extractvalue { i8, i1 } %pair, 0 @@ -1962,6 +3352,14 @@ define i8 @test_atomic_cmpxchg_i8_monotonic(i8 %wanted, i8 %new) nounwind { define i16 @test_atomic_cmpxchg_i16_monotonic(i16 %wanted, i16 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i16_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i16_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x2, var16 +; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas2_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %pair = cmpxchg i16* @var16, i16 %wanted, i16 %new monotonic monotonic %old = extractvalue { i16, i1 } %pair, 0 @@ -1977,6 +3375,14 @@ define i16 @test_atomic_cmpxchg_i16_monotonic(i16 %wanted, i16 %new) nounwind { define i32 @test_atomic_cmpxchg_i32_monotonic(i32 %wanted, i32 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i32_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i32_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x2, var32 +; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas4_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %pair = cmpxchg i32* @var32, i32 %wanted, i32 %new monotonic monotonic %old = extractvalue { i32, i1 } %pair, 0 @@ -1992,6 +3398,14 @@ define i32 @test_atomic_cmpxchg_i32_monotonic(i32 %wanted, i32 %new) nounwind { define i64 @test_atomic_cmpxchg_i64_monotonic(i64 %wanted, i64 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i64_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i64_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x2, var64 +; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas8_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %pair = cmpxchg i64* @var64, i64 %wanted, i64 %new monotonic monotonic %old = extractvalue { i64, i1 } %pair, 0 @@ -2007,6 +3421,14 @@ define i64 @test_atomic_cmpxchg_i64_monotonic(i64 %wanted, i64 %new) nounwind { define i128 @test_atomic_cmpxchg_i128_monotonic(i128 %wanted, i128 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i128_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i128_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x4, var128 +; OUTLINE-ATOMICS-NEXT: add x4, x4, :lo12:var128 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas16_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %pair = cmpxchg i128* @var128, i128 %wanted, i128 %new monotonic monotonic %old = extractvalue { i128, i1 } %pair, 0 @@ -2022,6 +3444,14 @@ define i128 @test_atomic_cmpxchg_i128_monotonic(i128 %wanted, i128 %new) nounwin define i8 @test_atomic_cmpxchg_i8_seq_cst(i8 %wanted, i8 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i8_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i8_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x2, var8 +; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas1_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %pair = cmpxchg i8* @var8, i8 %wanted, i8 %new seq_cst seq_cst %old = extractvalue { i8, i1 } %pair, 0 @@ -2037,6 +3467,14 @@ define i8 @test_atomic_cmpxchg_i8_seq_cst(i8 %wanted, i8 %new) nounwind { define i16 @test_atomic_cmpxchg_i16_seq_cst(i16 %wanted, i16 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i16_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i16_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x2, var16 +; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas2_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %pair = cmpxchg i16* @var16, i16 %wanted, i16 %new seq_cst seq_cst %old = extractvalue { i16, i1 } %pair, 0 @@ -2052,6 +3490,14 @@ define i16 @test_atomic_cmpxchg_i16_seq_cst(i16 %wanted, i16 %new) nounwind { define i32 @test_atomic_cmpxchg_i32_seq_cst(i32 %wanted, i32 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i32_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i32_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x2, var32 +; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %pair = cmpxchg i32* @var32, i32 %wanted, i32 %new seq_cst seq_cst %old = extractvalue { i32, i1 } %pair, 0 @@ -2067,6 +3513,14 @@ define i32 @test_atomic_cmpxchg_i32_seq_cst(i32 %wanted, i32 %new) nounwind { define i64 @test_atomic_cmpxchg_i64_seq_cst(i64 %wanted, i64 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i64_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i64_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x2, var64 +; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %pair = cmpxchg i64* @var64, i64 %wanted, i64 %new seq_cst seq_cst %old = extractvalue { i64, i1 } %pair, 0 @@ -2082,6 +3536,14 @@ define i64 @test_atomic_cmpxchg_i64_seq_cst(i64 %wanted, i64 %new) nounwind { define i128 @test_atomic_cmpxchg_i128_seq_cst(i128 %wanted, i128 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i128_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i128_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x4, var128 +; OUTLINE-ATOMICS-NEXT: add x4, x4, :lo12:var128 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas16_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %pair = cmpxchg i128* @var128, i128 %wanted, i128 %new seq_cst seq_cst %old = extractvalue { i128, i1 } %pair, 0 @@ -2097,6 +3559,21 @@ define i128 @test_atomic_cmpxchg_i128_seq_cst(i128 %wanted, i128 %new) nounwind define i8 @test_atomic_load_max_i8_acq_rel(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i8_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i8_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var8 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: .LBB158_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9] +; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb +; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt +; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB158_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw max i8* @var8, i8 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -2110,6 +3587,21 @@ define i8 @test_atomic_load_max_i8_acq_rel(i8 %offset) nounwind { define i16 @test_atomic_load_max_i16_acq_rel(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i16_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i16_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var16 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: .LBB159_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9] +; OUTLINE-ATOMICS-NEXT: sxth w8, w10 +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth +; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt +; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB159_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw max i16* @var16, i16 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -2123,6 +3615,20 @@ define i16 @test_atomic_load_max_i16_acq_rel(i16 %offset) nounwind { define i32 @test_atomic_load_max_i32_acq_rel(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i32_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var32 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB160_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0 +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, gt +; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB160_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw max i32* @var32, i32 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -2136,6 +3642,20 @@ define i32 @test_atomic_load_max_i32_acq_rel(i32 %offset) nounwind { define i64 @test_atomic_load_max_i64_acq_rel(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i64_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var64 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB161_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp x8, x0 +; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, gt +; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB161_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov x0, x8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw max i64* @var64, i64 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -2149,6 +3669,19 @@ define i64 @test_atomic_load_max_i64_acq_rel(i64 %offset) nounwind { define void @test_atomic_load_max_i32_noret_acq_rel(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i32_noret_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_noret_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var32 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB162_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp w9, w0 +; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, gt +; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB162_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw max i32* @var32, i32 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -2161,6 +3694,19 @@ define void @test_atomic_load_max_i32_noret_acq_rel(i32 %offset) nounwind { define void @test_atomic_load_max_i64_noret_acq_rel(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i64_noret_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_noret_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var64 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB163_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp x9, x0 +; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, gt +; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB163_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw max i64* @var64, i64 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -2173,6 +3719,21 @@ define void @test_atomic_load_max_i64_noret_acq_rel(i64 %offset) nounwind { define i8 @test_atomic_load_max_i8_acquire(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i8_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i8_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var8 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: .LBB164_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9] +; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb +; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt +; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB164_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw max i8* @var8, i8 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -2186,6 +3747,21 @@ define i8 @test_atomic_load_max_i8_acquire(i8 %offset) nounwind { define i16 @test_atomic_load_max_i16_acquire(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i16_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i16_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var16 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: .LBB165_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9] +; OUTLINE-ATOMICS-NEXT: sxth w8, w10 +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth +; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt +; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB165_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw max i16* @var16, i16 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -2199,6 +3775,20 @@ define i16 @test_atomic_load_max_i16_acquire(i16 %offset) nounwind { define i32 @test_atomic_load_max_i32_acquire(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i32_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var32 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB166_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0 +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, gt +; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB166_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw max i32* @var32, i32 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -2212,6 +3802,20 @@ define i32 @test_atomic_load_max_i32_acquire(i32 %offset) nounwind { define i64 @test_atomic_load_max_i64_acquire(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i64_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var64 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB167_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp x8, x0 +; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, gt +; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB167_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov x0, x8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw max i64* @var64, i64 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -2225,6 +3829,19 @@ define i64 @test_atomic_load_max_i64_acquire(i64 %offset) nounwind { define void @test_atomic_load_max_i32_noret_acquire(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i32_noret_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_noret_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var32 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB168_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp w9, w0 +; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, gt +; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB168_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw max i32* @var32, i32 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -2237,6 +3854,19 @@ define void @test_atomic_load_max_i32_noret_acquire(i32 %offset) nounwind { define void @test_atomic_load_max_i64_noret_acquire(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i64_noret_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_noret_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var64 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB169_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp x9, x0 +; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, gt +; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB169_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw max i64* @var64, i64 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -2249,6 +3879,21 @@ define void @test_atomic_load_max_i64_noret_acquire(i64 %offset) nounwind { define i8 @test_atomic_load_max_i8_monotonic(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i8_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i8_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var8 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: .LBB170_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxrb w10, [x9] +; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb +; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt +; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB170_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw max i8* @var8, i8 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -2262,6 +3907,21 @@ define i8 @test_atomic_load_max_i8_monotonic(i8 %offset) nounwind { define i16 @test_atomic_load_max_i16_monotonic(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i16_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i16_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var16 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: .LBB171_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxrh w10, [x9] +; OUTLINE-ATOMICS-NEXT: sxth w8, w10 +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth +; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt +; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB171_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw max i16* @var16, i16 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -2275,6 +3935,20 @@ define i16 @test_atomic_load_max_i16_monotonic(i16 %offset) nounwind { define i32 @test_atomic_load_max_i32_monotonic(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i32_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var32 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB172_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0 +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, gt +; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB172_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw max i32* @var32, i32 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -2288,6 +3962,20 @@ define i32 @test_atomic_load_max_i32_monotonic(i32 %offset) nounwind { define i64 @test_atomic_load_max_i64_monotonic(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i64_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var64 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB173_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp x8, x0 +; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, gt +; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB173_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov x0, x8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw max i64* @var64, i64 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -2301,6 +3989,19 @@ define i64 @test_atomic_load_max_i64_monotonic(i64 %offset) nounwind { define void @test_atomic_load_max_i32_noret_monotonic(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i32_noret_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_noret_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var32 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB174_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp w9, w0 +; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, gt +; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB174_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw max i32* @var32, i32 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -2313,6 +4014,19 @@ define void @test_atomic_load_max_i32_noret_monotonic(i32 %offset) nounwind { define void @test_atomic_load_max_i64_noret_monotonic(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i64_noret_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_noret_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var64 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB175_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp x9, x0 +; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, gt +; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB175_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw max i64* @var64, i64 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -2325,6 +4039,21 @@ define void @test_atomic_load_max_i64_noret_monotonic(i64 %offset) nounwind { define i8 @test_atomic_load_max_i8_release(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i8_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i8_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var8 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: .LBB176_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxrb w10, [x9] +; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb +; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt +; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB176_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw max i8* @var8, i8 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -2338,6 +4067,21 @@ define i8 @test_atomic_load_max_i8_release(i8 %offset) nounwind { define i16 @test_atomic_load_max_i16_release(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i16_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i16_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var16 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: .LBB177_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxrh w10, [x9] +; OUTLINE-ATOMICS-NEXT: sxth w8, w10 +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth +; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt +; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB177_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw max i16* @var16, i16 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -2351,6 +4095,20 @@ define i16 @test_atomic_load_max_i16_release(i16 %offset) nounwind { define i32 @test_atomic_load_max_i32_release(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i32_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var32 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB178_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0 +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, gt +; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB178_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw max i32* @var32, i32 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -2364,6 +4122,20 @@ define i32 @test_atomic_load_max_i32_release(i32 %offset) nounwind { define i64 @test_atomic_load_max_i64_release(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i64_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var64 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB179_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp x8, x0 +; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, gt +; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB179_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov x0, x8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw max i64* @var64, i64 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -2377,6 +4149,19 @@ define i64 @test_atomic_load_max_i64_release(i64 %offset) nounwind { define void @test_atomic_load_max_i32_noret_release(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i32_noret_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_noret_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var32 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB180_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp w9, w0 +; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, gt +; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB180_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw max i32* @var32, i32 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -2389,6 +4174,19 @@ define void @test_atomic_load_max_i32_noret_release(i32 %offset) nounwind { define void @test_atomic_load_max_i64_noret_release(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i64_noret_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_noret_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var64 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB181_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp x9, x0 +; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, gt +; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB181_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw max i64* @var64, i64 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -2401,6 +4199,21 @@ define void @test_atomic_load_max_i64_noret_release(i64 %offset) nounwind { define i8 @test_atomic_load_max_i8_seq_cst(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i8_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i8_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var8 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: .LBB182_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9] +; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb +; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt +; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB182_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw max i8* @var8, i8 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -2414,6 +4227,21 @@ define i8 @test_atomic_load_max_i8_seq_cst(i8 %offset) nounwind { define i16 @test_atomic_load_max_i16_seq_cst(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i16_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i16_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var16 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: .LBB183_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9] +; OUTLINE-ATOMICS-NEXT: sxth w8, w10 +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth +; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt +; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB183_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw max i16* @var16, i16 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -2427,6 +4255,20 @@ define i16 @test_atomic_load_max_i16_seq_cst(i16 %offset) nounwind { define i32 @test_atomic_load_max_i32_seq_cst(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i32_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var32 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB184_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0 +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, gt +; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB184_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw max i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -2440,6 +4282,20 @@ define i32 @test_atomic_load_max_i32_seq_cst(i32 %offset) nounwind { define i64 @test_atomic_load_max_i64_seq_cst(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i64_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var64 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB185_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp x8, x0 +; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, gt +; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB185_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov x0, x8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw max i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -2453,6 +4309,19 @@ define i64 @test_atomic_load_max_i64_seq_cst(i64 %offset) nounwind { define void @test_atomic_load_max_i32_noret_seq_cst(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i32_noret_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_noret_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var32 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB186_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp w9, w0 +; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, gt +; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB186_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw max i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -2465,6 +4334,19 @@ define void @test_atomic_load_max_i32_noret_seq_cst(i32 %offset) nounwind { define void @test_atomic_load_max_i64_noret_seq_cst(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i64_noret_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_noret_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var64 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB187_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp x9, x0 +; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, gt +; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB187_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw max i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -2477,6 +4359,21 @@ define void @test_atomic_load_max_i64_noret_seq_cst(i64 %offset) nounwind { define i8 @test_atomic_load_min_i8_acq_rel(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i8_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i8_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var8 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: .LBB188_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9] +; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb +; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le +; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB188_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw min i8* @var8, i8 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -2490,6 +4387,21 @@ define i8 @test_atomic_load_min_i8_acq_rel(i8 %offset) nounwind { define i16 @test_atomic_load_min_i16_acq_rel(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i16_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i16_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var16 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: .LBB189_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9] +; OUTLINE-ATOMICS-NEXT: sxth w8, w10 +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth +; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le +; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB189_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw min i16* @var16, i16 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -2503,6 +4415,20 @@ define i16 @test_atomic_load_min_i16_acq_rel(i16 %offset) nounwind { define i32 @test_atomic_load_min_i32_acq_rel(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i32_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var32 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB190_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0 +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, le +; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB190_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw min i32* @var32, i32 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -2516,6 +4442,20 @@ define i32 @test_atomic_load_min_i32_acq_rel(i32 %offset) nounwind { define i64 @test_atomic_load_min_i64_acq_rel(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i64_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var64 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB191_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp x8, x0 +; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, le +; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB191_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov x0, x8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw min i64* @var64, i64 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -2529,6 +4469,19 @@ define i64 @test_atomic_load_min_i64_acq_rel(i64 %offset) nounwind { define void @test_atomic_load_min_i32_noret_acq_rel(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i32_noret_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_noret_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var32 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB192_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp w9, w0 +; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, le +; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB192_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw min i32* @var32, i32 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -2541,6 +4494,19 @@ define void @test_atomic_load_min_i32_noret_acq_rel(i32 %offset) nounwind { define void @test_atomic_load_min_i64_noret_acq_rel(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i64_noret_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_noret_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var64 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB193_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp x9, x0 +; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, le +; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB193_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw min i64* @var64, i64 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -2553,6 +4519,21 @@ define void @test_atomic_load_min_i64_noret_acq_rel(i64 %offset) nounwind { define i8 @test_atomic_load_min_i8_acquire(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i8_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i8_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var8 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: .LBB194_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9] +; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb +; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le +; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB194_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw min i8* @var8, i8 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -2566,6 +4547,21 @@ define i8 @test_atomic_load_min_i8_acquire(i8 %offset) nounwind { define i16 @test_atomic_load_min_i16_acquire(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i16_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i16_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var16 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: .LBB195_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9] +; OUTLINE-ATOMICS-NEXT: sxth w8, w10 +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth +; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le +; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB195_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw min i16* @var16, i16 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -2579,6 +4575,20 @@ define i16 @test_atomic_load_min_i16_acquire(i16 %offset) nounwind { define i32 @test_atomic_load_min_i32_acquire(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i32_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var32 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB196_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0 +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, le +; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB196_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw min i32* @var32, i32 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -2592,6 +4602,20 @@ define i32 @test_atomic_load_min_i32_acquire(i32 %offset) nounwind { define i64 @test_atomic_load_min_i64_acquire(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i64_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var64 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB197_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp x8, x0 +; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, le +; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB197_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov x0, x8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw min i64* @var64, i64 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -2605,6 +4629,19 @@ define i64 @test_atomic_load_min_i64_acquire(i64 %offset) nounwind { define void @test_atomic_load_min_i32_noret_acquire(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i32_noret_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_noret_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var32 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB198_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp w9, w0 +; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, le +; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB198_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw min i32* @var32, i32 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -2617,6 +4654,19 @@ define void @test_atomic_load_min_i32_noret_acquire(i32 %offset) nounwind { define void @test_atomic_load_min_i64_noret_acquire(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i64_noret_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_noret_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var64 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB199_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp x9, x0 +; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, le +; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB199_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw min i64* @var64, i64 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -2629,6 +4679,21 @@ define void @test_atomic_load_min_i64_noret_acquire(i64 %offset) nounwind { define i8 @test_atomic_load_min_i8_monotonic(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i8_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i8_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var8 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: .LBB200_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxrb w10, [x9] +; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb +; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le +; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB200_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw min i8* @var8, i8 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -2642,6 +4707,21 @@ define i8 @test_atomic_load_min_i8_monotonic(i8 %offset) nounwind { define i16 @test_atomic_load_min_i16_monotonic(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i16_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i16_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var16 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: .LBB201_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxrh w10, [x9] +; OUTLINE-ATOMICS-NEXT: sxth w8, w10 +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth +; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le +; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB201_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw min i16* @var16, i16 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -2655,6 +4735,20 @@ define i16 @test_atomic_load_min_i16_monotonic(i16 %offset) nounwind { define i32 @test_atomic_load_min_i32_monotonic(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i32_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var32 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB202_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0 +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, le +; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB202_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw min i32* @var32, i32 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -2668,6 +4762,20 @@ define i32 @test_atomic_load_min_i32_monotonic(i32 %offset) nounwind { define i64 @test_atomic_load_min_i64_monotonic(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i64_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var64 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB203_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp x8, x0 +; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, le +; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB203_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov x0, x8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw min i64* @var64, i64 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -2681,6 +4789,19 @@ define i64 @test_atomic_load_min_i64_monotonic(i64 %offset) nounwind { define void @test_atomic_load_min_i32_noret_monotonic(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i32_noret_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_noret_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var32 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB204_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp w9, w0 +; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, le +; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB204_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw min i32* @var32, i32 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -2693,6 +4814,19 @@ define void @test_atomic_load_min_i32_noret_monotonic(i32 %offset) nounwind { define void @test_atomic_load_min_i64_noret_monotonic(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i64_noret_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_noret_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var64 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB205_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp x9, x0 +; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, le +; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB205_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw min i64* @var64, i64 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -2705,6 +4839,21 @@ define void @test_atomic_load_min_i64_noret_monotonic(i64 %offset) nounwind { define i8 @test_atomic_load_min_i8_release(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i8_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i8_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var8 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: .LBB206_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxrb w10, [x9] +; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb +; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le +; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB206_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw min i8* @var8, i8 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -2718,6 +4867,21 @@ define i8 @test_atomic_load_min_i8_release(i8 %offset) nounwind { define i16 @test_atomic_load_min_i16_release(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i16_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i16_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var16 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: .LBB207_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxrh w10, [x9] +; OUTLINE-ATOMICS-NEXT: sxth w8, w10 +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth +; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le +; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB207_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw min i16* @var16, i16 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -2731,6 +4895,20 @@ define i16 @test_atomic_load_min_i16_release(i16 %offset) nounwind { define i32 @test_atomic_load_min_i32_release(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i32_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var32 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB208_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0 +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, le +; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB208_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw min i32* @var32, i32 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -2744,6 +4922,20 @@ define i32 @test_atomic_load_min_i32_release(i32 %offset) nounwind { define i64 @test_atomic_load_min_i64_release(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i64_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var64 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB209_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp x8, x0 +; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, le +; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB209_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov x0, x8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw min i64* @var64, i64 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -2757,6 +4949,19 @@ define i64 @test_atomic_load_min_i64_release(i64 %offset) nounwind { define void @test_atomic_load_min_i32_noret_release(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i32_noret_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_noret_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var32 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB210_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp w9, w0 +; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, le +; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB210_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw min i32* @var32, i32 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -2769,6 +4974,19 @@ define void @test_atomic_load_min_i32_noret_release(i32 %offset) nounwind { define void @test_atomic_load_min_i64_noret_release(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i64_noret_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_noret_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var64 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB211_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp x9, x0 +; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, le +; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB211_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw min i64* @var64, i64 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -2781,6 +4999,21 @@ define void @test_atomic_load_min_i64_noret_release(i64 %offset) nounwind { define i8 @test_atomic_load_min_i8_seq_cst(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i8_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i8_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var8 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: .LBB212_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9] +; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb +; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le +; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB212_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw min i8* @var8, i8 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -2794,6 +5027,21 @@ define i8 @test_atomic_load_min_i8_seq_cst(i8 %offset) nounwind { define i16 @test_atomic_load_min_i16_seq_cst(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i16_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i16_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var16 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: .LBB213_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9] +; OUTLINE-ATOMICS-NEXT: sxth w8, w10 +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth +; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le +; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB213_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw min i16* @var16, i16 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -2807,6 +5055,20 @@ define i16 @test_atomic_load_min_i16_seq_cst(i16 %offset) nounwind { define i32 @test_atomic_load_min_i32_seq_cst(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i32_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var32 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB214_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0 +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, le +; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB214_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw min i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -2820,6 +5082,20 @@ define i32 @test_atomic_load_min_i32_seq_cst(i32 %offset) nounwind { define i64 @test_atomic_load_min_i64_seq_cst(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i64_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var64 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB215_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp x8, x0 +; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, le +; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB215_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov x0, x8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw min i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -2833,6 +5109,19 @@ define i64 @test_atomic_load_min_i64_seq_cst(i64 %offset) nounwind { define void @test_atomic_load_min_i32_noret_seq_cst(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i32_noret_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_noret_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var32 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB216_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp w9, w0 +; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, le +; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB216_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw min i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -2845,6 +5134,19 @@ define void @test_atomic_load_min_i32_noret_seq_cst(i32 %offset) nounwind { define void @test_atomic_load_min_i64_noret_seq_cst(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i64_noret_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_noret_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var64 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB217_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp x9, x0 +; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, le +; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB217_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw min i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -2857,6 +5159,14 @@ define void @test_atomic_load_min_i64_noret_seq_cst(i64 %offset) nounwind { define i8 @test_atomic_load_or_i8_acq_rel(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i8_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i8_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset1_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw or i8* @var8, i8 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -2870,6 +5180,14 @@ define i8 @test_atomic_load_or_i8_acq_rel(i8 %offset) nounwind { define i16 @test_atomic_load_or_i16_acq_rel(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i16_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i16_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset2_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw or i16* @var16, i16 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -2883,6 +5201,14 @@ define i16 @test_atomic_load_or_i16_acq_rel(i16 %offset) nounwind { define i32 @test_atomic_load_or_i32_acq_rel(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i32_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i32_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw or i32* @var32, i32 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -2896,6 +5222,14 @@ define i32 @test_atomic_load_or_i32_acq_rel(i32 %offset) nounwind { define i64 @test_atomic_load_or_i64_acq_rel(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i64_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i64_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw or i64* @var64, i64 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -2909,6 +5243,14 @@ define i64 @test_atomic_load_or_i64_acq_rel(i64 %offset) nounwind { define void @test_atomic_load_or_i32_noret_acq_rel(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i32_noret_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i32_noret_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw or i32* @var32, i32 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -2921,6 +5263,14 @@ define void @test_atomic_load_or_i32_noret_acq_rel(i32 %offset) nounwind { define void @test_atomic_load_or_i64_noret_acq_rel(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i64_noret_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i64_noret_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw or i64* @var64, i64 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -2933,6 +5283,14 @@ define void @test_atomic_load_or_i64_noret_acq_rel(i64 %offset) nounwind { define i8 @test_atomic_load_or_i8_acquire(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i8_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i8_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset1_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw or i8* @var8, i8 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -2946,6 +5304,14 @@ define i8 @test_atomic_load_or_i8_acquire(i8 %offset) nounwind { define i16 @test_atomic_load_or_i16_acquire(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i16_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i16_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset2_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw or i16* @var16, i16 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -2959,6 +5325,14 @@ define i16 @test_atomic_load_or_i16_acquire(i16 %offset) nounwind { define i32 @test_atomic_load_or_i32_acquire(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i32_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i32_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset4_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw or i32* @var32, i32 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -2972,6 +5346,14 @@ define i32 @test_atomic_load_or_i32_acquire(i32 %offset) nounwind { define i64 @test_atomic_load_or_i64_acquire(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i64_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i64_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset8_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw or i64* @var64, i64 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -2985,6 +5367,14 @@ define i64 @test_atomic_load_or_i64_acquire(i64 %offset) nounwind { define void @test_atomic_load_or_i32_noret_acquire(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i32_noret_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i32_noret_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset4_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw or i32* @var32, i32 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -2997,6 +5387,14 @@ define void @test_atomic_load_or_i32_noret_acquire(i32 %offset) nounwind { define void @test_atomic_load_or_i64_noret_acquire(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i64_noret_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i64_noret_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset8_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw or i64* @var64, i64 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -3009,6 +5407,14 @@ define void @test_atomic_load_or_i64_noret_acquire(i64 %offset) nounwind { define i8 @test_atomic_load_or_i8_monotonic(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i8_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i8_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset1_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw or i8* @var8, i8 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -3022,6 +5428,14 @@ define i8 @test_atomic_load_or_i8_monotonic(i8 %offset) nounwind { define i16 @test_atomic_load_or_i16_monotonic(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i16_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i16_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset2_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw or i16* @var16, i16 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -3035,6 +5449,14 @@ define i16 @test_atomic_load_or_i16_monotonic(i16 %offset) nounwind { define i32 @test_atomic_load_or_i32_monotonic(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i32_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i32_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset4_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw or i32* @var32, i32 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -3048,6 +5470,14 @@ define i32 @test_atomic_load_or_i32_monotonic(i32 %offset) nounwind { define i64 @test_atomic_load_or_i64_monotonic(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i64_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i64_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset8_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw or i64* @var64, i64 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -3061,6 +5491,14 @@ define i64 @test_atomic_load_or_i64_monotonic(i64 %offset) nounwind { define void @test_atomic_load_or_i32_noret_monotonic(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i32_noret_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i32_noret_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset4_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw or i32* @var32, i32 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -3073,6 +5511,14 @@ define void @test_atomic_load_or_i32_noret_monotonic(i32 %offset) nounwind { define void @test_atomic_load_or_i64_noret_monotonic(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i64_noret_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i64_noret_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset8_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw or i64* @var64, i64 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -3085,6 +5531,14 @@ define void @test_atomic_load_or_i64_noret_monotonic(i64 %offset) nounwind { define i8 @test_atomic_load_or_i8_release(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i8_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i8_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset1_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw or i8* @var8, i8 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -3098,6 +5552,14 @@ define i8 @test_atomic_load_or_i8_release(i8 %offset) nounwind { define i16 @test_atomic_load_or_i16_release(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i16_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i16_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset2_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw or i16* @var16, i16 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -3111,6 +5573,14 @@ define i16 @test_atomic_load_or_i16_release(i16 %offset) nounwind { define i32 @test_atomic_load_or_i32_release(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i32_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i32_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset4_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw or i32* @var32, i32 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -3124,6 +5594,14 @@ define i32 @test_atomic_load_or_i32_release(i32 %offset) nounwind { define i64 @test_atomic_load_or_i64_release(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i64_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i64_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset8_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw or i64* @var64, i64 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -3137,6 +5615,14 @@ define i64 @test_atomic_load_or_i64_release(i64 %offset) nounwind { define void @test_atomic_load_or_i32_noret_release(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i32_noret_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i32_noret_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset4_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw or i32* @var32, i32 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -3149,6 +5635,14 @@ define void @test_atomic_load_or_i32_noret_release(i32 %offset) nounwind { define void @test_atomic_load_or_i64_noret_release(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i64_noret_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i64_noret_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset8_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw or i64* @var64, i64 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -3161,6 +5655,14 @@ define void @test_atomic_load_or_i64_noret_release(i64 %offset) nounwind { define i8 @test_atomic_load_or_i8_seq_cst(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i8_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i8_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset1_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw or i8* @var8, i8 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -3174,6 +5676,14 @@ define i8 @test_atomic_load_or_i8_seq_cst(i8 %offset) nounwind { define i16 @test_atomic_load_or_i16_seq_cst(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i16_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i16_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset2_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw or i16* @var16, i16 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -3187,6 +5697,14 @@ define i16 @test_atomic_load_or_i16_seq_cst(i16 %offset) nounwind { define i32 @test_atomic_load_or_i32_seq_cst(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i32_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i32_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw or i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -3200,6 +5718,14 @@ define i32 @test_atomic_load_or_i32_seq_cst(i32 %offset) nounwind { define i64 @test_atomic_load_or_i64_seq_cst(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i64_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i64_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw or i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -3213,6 +5739,14 @@ define i64 @test_atomic_load_or_i64_seq_cst(i64 %offset) nounwind { define void @test_atomic_load_or_i32_noret_seq_cst(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i32_noret_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i32_noret_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw or i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -3225,6 +5759,14 @@ define void @test_atomic_load_or_i32_noret_seq_cst(i32 %offset) nounwind { define void @test_atomic_load_or_i64_noret_seq_cst(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i64_noret_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i64_noret_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw or i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -3237,6 +5779,15 @@ define void @test_atomic_load_or_i64_noret_seq_cst(i64 %offset) nounwind { define i8 @test_atomic_load_sub_i8_acq_rel(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i8_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i8_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i8* @var8, i8 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] @@ -3251,6 +5802,15 @@ define i8 @test_atomic_load_sub_i8_acq_rel(i8 %offset) nounwind { define i16 @test_atomic_load_sub_i16_acq_rel(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i16_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i16_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i16* @var16, i16 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] @@ -3265,6 +5825,15 @@ define i16 @test_atomic_load_sub_i16_acq_rel(i16 %offset) nounwind { define i32 @test_atomic_load_sub_i32_acq_rel(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i32_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i32* @var32, i32 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] @@ -3279,6 +5848,15 @@ define i32 @test_atomic_load_sub_i32_acq_rel(i32 %offset) nounwind { define i64 @test_atomic_load_sub_i64_acq_rel(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i64_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg x0, x0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i64* @var64, i64 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]] @@ -3293,6 +5871,15 @@ define i64 @test_atomic_load_sub_i64_acq_rel(i64 %offset) nounwind { define void @test_atomic_load_sub_i32_noret_acq_rel(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i32_noret_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32_noret_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw sub i32* @var32, i32 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] @@ -3307,6 +5894,15 @@ define void @test_atomic_load_sub_i32_noret_acq_rel(i32 %offset) nounwind { define void @test_atomic_load_sub_i64_noret_acq_rel(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i64_noret_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64_noret_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg x0, x0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw sub i64* @var64, i64 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]] @@ -3321,6 +5917,15 @@ define void @test_atomic_load_sub_i64_noret_acq_rel(i64 %offset) nounwind { define i8 @test_atomic_load_sub_i8_acquire(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i8_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i8_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i8* @var8, i8 %offset acquire ; CHECK-NOT: dmb ; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] @@ -3335,6 +5940,15 @@ define i8 @test_atomic_load_sub_i8_acquire(i8 %offset) nounwind { define i16 @test_atomic_load_sub_i16_acquire(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i16_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i16_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i16* @var16, i16 %offset acquire ; CHECK-NOT: dmb ; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] @@ -3349,6 +5963,15 @@ define i16 @test_atomic_load_sub_i16_acquire(i16 %offset) nounwind { define i32 @test_atomic_load_sub_i32_acquire(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i32_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i32* @var32, i32 %offset acquire ; CHECK-NOT: dmb ; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] @@ -3363,6 +5986,15 @@ define i32 @test_atomic_load_sub_i32_acquire(i32 %offset) nounwind { define i64 @test_atomic_load_sub_i64_acquire(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i64_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg x0, x0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i64* @var64, i64 %offset acquire ; CHECK-NOT: dmb ; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]] @@ -3377,6 +6009,15 @@ define i64 @test_atomic_load_sub_i64_acquire(i64 %offset) nounwind { define void @test_atomic_load_sub_i32_noret_acquire(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i32_noret_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32_noret_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw sub i32* @var32, i32 %offset acquire ; CHECK-NOT: dmb ; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] @@ -3391,6 +6032,15 @@ define void @test_atomic_load_sub_i32_noret_acquire(i32 %offset) nounwind { define void @test_atomic_load_sub_i64_noret_acquire(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i64_noret_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64_noret_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg x0, x0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw sub i64* @var64, i64 %offset acquire ; CHECK-NOT: dmb ; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]] @@ -3405,6 +6055,15 @@ define void @test_atomic_load_sub_i64_noret_acquire(i64 %offset) nounwind { define i8 @test_atomic_load_sub_i8_monotonic(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i8_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i8_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i8* @var8, i8 %offset monotonic ; CHECK-NOT: dmb ; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] @@ -3419,6 +6078,15 @@ define i8 @test_atomic_load_sub_i8_monotonic(i8 %offset) nounwind { define i16 @test_atomic_load_sub_i16_monotonic(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i16_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i16_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i16* @var16, i16 %offset monotonic ; CHECK-NOT: dmb ; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] @@ -3433,6 +6101,15 @@ define i16 @test_atomic_load_sub_i16_monotonic(i16 %offset) nounwind { define i32 @test_atomic_load_sub_i32_monotonic(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i32_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i32* @var32, i32 %offset monotonic ; CHECK-NOT: dmb ; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] @@ -3447,6 +6124,15 @@ define i32 @test_atomic_load_sub_i32_monotonic(i32 %offset) nounwind { define i64 @test_atomic_load_sub_i64_monotonic(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i64_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg x0, x0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i64* @var64, i64 %offset monotonic ; CHECK-NOT: dmb ; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]] @@ -3461,6 +6147,15 @@ define i64 @test_atomic_load_sub_i64_monotonic(i64 %offset) nounwind { define void @test_atomic_load_sub_i32_noret_monotonic(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i32_noret_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32_noret_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw sub i32* @var32, i32 %offset monotonic ; CHECK-NOT: dmb ; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] @@ -3475,6 +6170,15 @@ define void @test_atomic_load_sub_i32_noret_monotonic(i32 %offset) nounwind { define void @test_atomic_load_sub_i64_noret_monotonic(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i64_noret_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64_noret_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg x0, x0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw sub i64* @var64, i64 %offset monotonic ; CHECK-NOT: dmb ; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]] @@ -3489,6 +6193,15 @@ define void @test_atomic_load_sub_i64_noret_monotonic(i64 %offset) nounwind { define i8 @test_atomic_load_sub_i8_release(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i8_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i8_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i8* @var8, i8 %offset release ; CHECK-NOT: dmb ; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] @@ -3503,6 +6216,15 @@ define i8 @test_atomic_load_sub_i8_release(i8 %offset) nounwind { define i16 @test_atomic_load_sub_i16_release(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i16_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i16_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i16* @var16, i16 %offset release ; CHECK-NOT: dmb ; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] @@ -3517,6 +6239,15 @@ define i16 @test_atomic_load_sub_i16_release(i16 %offset) nounwind { define i32 @test_atomic_load_sub_i32_release(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i32_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i32* @var32, i32 %offset release ; CHECK-NOT: dmb ; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] @@ -3531,6 +6262,15 @@ define i32 @test_atomic_load_sub_i32_release(i32 %offset) nounwind { define i64 @test_atomic_load_sub_i64_release(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i64_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg x0, x0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i64* @var64, i64 %offset release ; CHECK-NOT: dmb ; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]] @@ -3545,6 +6285,15 @@ define i64 @test_atomic_load_sub_i64_release(i64 %offset) nounwind { define void @test_atomic_load_sub_i32_noret_release(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i32_noret_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32_noret_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw sub i32* @var32, i32 %offset release ; CHECK-NOT: dmb ; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] @@ -3559,6 +6308,15 @@ define void @test_atomic_load_sub_i32_noret_release(i32 %offset) nounwind { define void @test_atomic_load_sub_i64_noret_release(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i64_noret_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64_noret_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg x0, x0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw sub i64* @var64, i64 %offset release ; CHECK-NOT: dmb ; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]] @@ -3573,6 +6331,15 @@ define void @test_atomic_load_sub_i64_noret_release(i64 %offset) nounwind { define i8 @test_atomic_load_sub_i8_seq_cst(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i8_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i8_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i8* @var8, i8 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] @@ -3587,6 +6354,15 @@ define i8 @test_atomic_load_sub_i8_seq_cst(i8 %offset) nounwind { define i16 @test_atomic_load_sub_i16_seq_cst(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i16_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i16_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i16* @var16, i16 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] @@ -3601,6 +6377,15 @@ define i16 @test_atomic_load_sub_i16_seq_cst(i16 %offset) nounwind { define i32 @test_atomic_load_sub_i32_seq_cst(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i32_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] @@ -3615,6 +6400,15 @@ define i32 @test_atomic_load_sub_i32_seq_cst(i32 %offset) nounwind { define i64 @test_atomic_load_sub_i64_seq_cst(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i64_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg x0, x0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw sub i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]] @@ -3629,6 +6423,15 @@ define i64 @test_atomic_load_sub_i64_seq_cst(i64 %offset) nounwind { define void @test_atomic_load_sub_i32_noret_seq_cst(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i32_noret_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32_noret_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg w0, w0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw sub i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] @@ -3643,6 +6446,15 @@ define void @test_atomic_load_sub_i32_noret_seq_cst(i32 %offset) nounwind { define void @test_atomic_load_sub_i64_noret_seq_cst(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i64_noret_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64_noret_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: neg x0, x0 +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw sub i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]] @@ -3657,6 +6469,14 @@ define void @test_atomic_load_sub_i64_noret_seq_cst(i64 %offset) nounwind { define i8 @test_atomic_load_xchg_i8_acq_rel(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i8_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i8_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp1_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xchg i8* @var8, i8 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -3670,6 +6490,14 @@ define i8 @test_atomic_load_xchg_i8_acq_rel(i8 %offset) nounwind { define i16 @test_atomic_load_xchg_i16_acq_rel(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i16_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i16_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp2_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xchg i16* @var16, i16 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -3683,6 +6511,14 @@ define i16 @test_atomic_load_xchg_i16_acq_rel(i16 %offset) nounwind { define i32 @test_atomic_load_xchg_i32_acq_rel(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i32_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i32_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xchg i32* @var32, i32 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -3696,6 +6532,14 @@ define i32 @test_atomic_load_xchg_i32_acq_rel(i32 %offset) nounwind { define i64 @test_atomic_load_xchg_i64_acq_rel(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i64_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i64_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xchg i64* @var64, i64 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -3709,6 +6553,14 @@ define i64 @test_atomic_load_xchg_i64_acq_rel(i64 %offset) nounwind { define void @test_atomic_load_xchg_i32_noret_acq_rel(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i32_noret_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i32_noret_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw xchg i32* @var32, i32 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -3722,6 +6574,14 @@ define void @test_atomic_load_xchg_i32_noret_acq_rel(i32 %offset) nounwind { define void @test_atomic_load_xchg_i64_noret_acq_rel(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i64_noret_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i64_noret_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw xchg i64* @var64, i64 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -3735,6 +6595,14 @@ define void @test_atomic_load_xchg_i64_noret_acq_rel(i64 %offset) nounwind { define i8 @test_atomic_load_xchg_i8_acquire(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i8_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i8_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp1_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xchg i8* @var8, i8 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -3748,6 +6616,14 @@ define i8 @test_atomic_load_xchg_i8_acquire(i8 %offset) nounwind { define i16 @test_atomic_load_xchg_i16_acquire(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i16_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i16_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp2_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xchg i16* @var16, i16 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -3761,6 +6637,14 @@ define i16 @test_atomic_load_xchg_i16_acquire(i16 %offset) nounwind { define i32 @test_atomic_load_xchg_i32_acquire(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i32_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i32_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp4_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xchg i32* @var32, i32 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -3774,6 +6658,14 @@ define i32 @test_atomic_load_xchg_i32_acquire(i32 %offset) nounwind { define i64 @test_atomic_load_xchg_i64_acquire(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i64_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i64_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp8_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xchg i64* @var64, i64 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -3787,6 +6679,14 @@ define i64 @test_atomic_load_xchg_i64_acquire(i64 %offset) nounwind { define void @test_atomic_load_xchg_i32_noret_acquire(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i32_noret_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i32_noret_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp4_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw xchg i32* @var32, i32 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -3800,6 +6700,14 @@ define void @test_atomic_load_xchg_i32_noret_acquire(i32 %offset) nounwind { define void @test_atomic_load_xchg_i64_noret_acquire(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i64_noret_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i64_noret_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp8_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw xchg i64* @var64, i64 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -3813,6 +6721,14 @@ define void @test_atomic_load_xchg_i64_noret_acquire(i64 %offset) nounwind { define i8 @test_atomic_load_xchg_i8_monotonic(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i8_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i8_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp1_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xchg i8* @var8, i8 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -3826,6 +6742,14 @@ define i8 @test_atomic_load_xchg_i8_monotonic(i8 %offset) nounwind { define i16 @test_atomic_load_xchg_i16_monotonic(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i16_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i16_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp2_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xchg i16* @var16, i16 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -3839,6 +6763,14 @@ define i16 @test_atomic_load_xchg_i16_monotonic(i16 %offset) nounwind { define i32 @test_atomic_load_xchg_i32_monotonic(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i32_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i32_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp4_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xchg i32* @var32, i32 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -3852,6 +6784,14 @@ define i32 @test_atomic_load_xchg_i32_monotonic(i32 %offset) nounwind { define i64 @test_atomic_load_xchg_i64_monotonic(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i64_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i64_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp8_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xchg i64* @var64, i64 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -3865,6 +6805,14 @@ define i64 @test_atomic_load_xchg_i64_monotonic(i64 %offset) nounwind { define void @test_atomic_load_xchg_i32_noret_monotonic(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i32_noret_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i32_noret_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp4_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw xchg i32* @var32, i32 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -3878,6 +6826,14 @@ define void @test_atomic_load_xchg_i32_noret_monotonic(i32 %offset) nounwind { define void @test_atomic_load_xchg_i64_noret_monotonic(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i64_noret_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i64_noret_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp8_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw xchg i64* @var64, i64 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -3891,6 +6847,14 @@ define void @test_atomic_load_xchg_i64_noret_monotonic(i64 %offset) nounwind { define i8 @test_atomic_load_xchg_i8_release(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i8_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i8_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp1_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xchg i8* @var8, i8 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -3904,6 +6868,14 @@ define i8 @test_atomic_load_xchg_i8_release(i8 %offset) nounwind { define i16 @test_atomic_load_xchg_i16_release(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i16_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i16_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp2_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xchg i16* @var16, i16 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -3917,6 +6889,14 @@ define i16 @test_atomic_load_xchg_i16_release(i16 %offset) nounwind { define i32 @test_atomic_load_xchg_i32_release(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i32_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i32_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp4_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xchg i32* @var32, i32 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -3930,6 +6910,14 @@ define i32 @test_atomic_load_xchg_i32_release(i32 %offset) nounwind { define i64 @test_atomic_load_xchg_i64_release(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i64_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i64_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp8_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xchg i64* @var64, i64 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -3943,6 +6931,14 @@ define i64 @test_atomic_load_xchg_i64_release(i64 %offset) nounwind { define void @test_atomic_load_xchg_i32_noret_release(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i32_noret_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i32_noret_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp4_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw xchg i32* @var32, i32 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -3956,6 +6952,14 @@ define void @test_atomic_load_xchg_i32_noret_release(i32 %offset) nounwind { define void @test_atomic_load_xchg_i64_noret_release(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i64_noret_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i64_noret_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp8_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw xchg i64* @var64, i64 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -3969,6 +6973,14 @@ define void @test_atomic_load_xchg_i64_noret_release(i64 %offset) nounwind { define i8 @test_atomic_load_xchg_i8_seq_cst(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i8_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i8_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp1_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xchg i8* @var8, i8 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -3982,6 +6994,14 @@ define i8 @test_atomic_load_xchg_i8_seq_cst(i8 %offset) nounwind { define i16 @test_atomic_load_xchg_i16_seq_cst(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i16_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i16_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp2_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xchg i16* @var16, i16 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -3995,6 +7015,14 @@ define i16 @test_atomic_load_xchg_i16_seq_cst(i16 %offset) nounwind { define i32 @test_atomic_load_xchg_i32_seq_cst(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i32_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i32_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xchg i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4008,6 +7036,14 @@ define i32 @test_atomic_load_xchg_i32_seq_cst(i32 %offset) nounwind { define i64 @test_atomic_load_xchg_i64_seq_cst(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i64_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i64_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xchg i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -4021,6 +7057,14 @@ define i64 @test_atomic_load_xchg_i64_seq_cst(i64 %offset) nounwind { define void @test_atomic_load_xchg_i32_noret_seq_cst(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i32_noret_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i32_noret_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw xchg i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4034,6 +7078,14 @@ define void @test_atomic_load_xchg_i32_noret_seq_cst(i32 %offset) nounwind { define void @test_atomic_load_xchg_i64_noret_seq_cst(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i64_noret_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i64_noret_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw xchg i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -4047,6 +7099,20 @@ define void @test_atomic_load_xchg_i64_noret_seq_cst(i64 %offset) nounwind { define i8 @test_atomic_load_umax_i8_acq_rel(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i8_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i8_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var8 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: .LBB308_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrb w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi +; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB308_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umax i8* @var8, i8 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -4060,6 +7126,20 @@ define i8 @test_atomic_load_umax_i8_acq_rel(i8 %offset) nounwind { define i16 @test_atomic_load_umax_i16_acq_rel(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i16_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i16_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var16 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: .LBB309_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrh w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi +; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB309_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umax i16* @var16, i16 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -4073,6 +7153,20 @@ define i16 @test_atomic_load_umax_i16_acq_rel(i16 %offset) nounwind { define i32 @test_atomic_load_umax_i32_acq_rel(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i32_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var32 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB310_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0 +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi +; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB310_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umax i32* @var32, i32 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4086,6 +7180,20 @@ define i32 @test_atomic_load_umax_i32_acq_rel(i32 %offset) nounwind { define i64 @test_atomic_load_umax_i64_acq_rel(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i64_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var64 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB311_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp x8, x0 +; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, hi +; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB311_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov x0, x8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umax i64* @var64, i64 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -4099,6 +7207,19 @@ define i64 @test_atomic_load_umax_i64_acq_rel(i64 %offset) nounwind { define void @test_atomic_load_umax_i32_noret_acq_rel(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i32_noret_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_noret_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var32 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB312_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp w9, w0 +; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, hi +; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB312_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw umax i32* @var32, i32 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4111,6 +7232,19 @@ define void @test_atomic_load_umax_i32_noret_acq_rel(i32 %offset) nounwind { define void @test_atomic_load_umax_i64_noret_acq_rel(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i64_noret_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_noret_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var64 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB313_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp x9, x0 +; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, hi +; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB313_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw umax i64* @var64, i64 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -4123,6 +7257,20 @@ define void @test_atomic_load_umax_i64_noret_acq_rel(i64 %offset) nounwind { define i8 @test_atomic_load_umax_i8_acquire(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i8_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i8_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var8 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: .LBB314_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrb w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi +; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB314_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umax i8* @var8, i8 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -4136,6 +7284,20 @@ define i8 @test_atomic_load_umax_i8_acquire(i8 %offset) nounwind { define i16 @test_atomic_load_umax_i16_acquire(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i16_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i16_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var16 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: .LBB315_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrh w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi +; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB315_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umax i16* @var16, i16 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -4149,6 +7311,20 @@ define i16 @test_atomic_load_umax_i16_acquire(i16 %offset) nounwind { define i32 @test_atomic_load_umax_i32_acquire(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i32_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var32 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB316_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0 +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi +; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB316_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umax i32* @var32, i32 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4162,6 +7338,20 @@ define i32 @test_atomic_load_umax_i32_acquire(i32 %offset) nounwind { define i64 @test_atomic_load_umax_i64_acquire(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i64_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var64 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB317_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp x8, x0 +; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, hi +; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB317_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov x0, x8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umax i64* @var64, i64 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -4175,6 +7365,19 @@ define i64 @test_atomic_load_umax_i64_acquire(i64 %offset) nounwind { define void @test_atomic_load_umax_i32_noret_acquire(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i32_noret_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_noret_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var32 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB318_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp w9, w0 +; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, hi +; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB318_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw umax i32* @var32, i32 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4187,6 +7390,19 @@ define void @test_atomic_load_umax_i32_noret_acquire(i32 %offset) nounwind { define void @test_atomic_load_umax_i64_noret_acquire(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i64_noret_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_noret_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var64 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB319_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp x9, x0 +; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, hi +; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB319_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw umax i64* @var64, i64 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -4199,6 +7415,20 @@ define void @test_atomic_load_umax_i64_noret_acquire(i64 %offset) nounwind { define i8 @test_atomic_load_umax_i8_monotonic(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i8_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i8_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var8 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: .LBB320_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxrb w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi +; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB320_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umax i8* @var8, i8 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -4212,6 +7442,20 @@ define i8 @test_atomic_load_umax_i8_monotonic(i8 %offset) nounwind { define i16 @test_atomic_load_umax_i16_monotonic(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i16_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i16_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var16 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: .LBB321_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxrh w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi +; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB321_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umax i16* @var16, i16 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -4225,6 +7469,20 @@ define i16 @test_atomic_load_umax_i16_monotonic(i16 %offset) nounwind { define i32 @test_atomic_load_umax_i32_monotonic(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i32_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var32 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB322_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0 +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi +; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB322_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umax i32* @var32, i32 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4238,6 +7496,20 @@ define i32 @test_atomic_load_umax_i32_monotonic(i32 %offset) nounwind { define i64 @test_atomic_load_umax_i64_monotonic(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i64_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var64 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB323_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp x8, x0 +; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, hi +; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB323_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov x0, x8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umax i64* @var64, i64 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -4251,6 +7523,19 @@ define i64 @test_atomic_load_umax_i64_monotonic(i64 %offset) nounwind { define void @test_atomic_load_umax_i32_noret_monotonic(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i32_noret_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_noret_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var32 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB324_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp w9, w0 +; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, hi +; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB324_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw umax i32* @var32, i32 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4263,6 +7548,19 @@ define void @test_atomic_load_umax_i32_noret_monotonic(i32 %offset) nounwind { define void @test_atomic_load_umax_i64_noret_monotonic(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i64_noret_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_noret_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var64 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB325_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp x9, x0 +; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, hi +; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB325_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw umax i64* @var64, i64 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -4275,6 +7573,20 @@ define void @test_atomic_load_umax_i64_noret_monotonic(i64 %offset) nounwind { define i8 @test_atomic_load_umax_i8_release(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i8_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i8_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var8 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: .LBB326_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxrb w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi +; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB326_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umax i8* @var8, i8 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -4288,6 +7600,20 @@ define i8 @test_atomic_load_umax_i8_release(i8 %offset) nounwind { define i16 @test_atomic_load_umax_i16_release(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i16_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i16_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var16 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: .LBB327_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxrh w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi +; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB327_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umax i16* @var16, i16 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -4301,6 +7627,20 @@ define i16 @test_atomic_load_umax_i16_release(i16 %offset) nounwind { define i32 @test_atomic_load_umax_i32_release(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i32_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var32 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB328_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0 +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi +; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB328_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umax i32* @var32, i32 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4314,6 +7654,20 @@ define i32 @test_atomic_load_umax_i32_release(i32 %offset) nounwind { define i64 @test_atomic_load_umax_i64_release(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i64_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var64 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB329_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp x8, x0 +; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, hi +; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB329_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov x0, x8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umax i64* @var64, i64 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -4327,6 +7681,19 @@ define i64 @test_atomic_load_umax_i64_release(i64 %offset) nounwind { define void @test_atomic_load_umax_i32_noret_release(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i32_noret_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_noret_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var32 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB330_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp w9, w0 +; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, hi +; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB330_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw umax i32* @var32, i32 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4339,6 +7706,19 @@ define void @test_atomic_load_umax_i32_noret_release(i32 %offset) nounwind { define void @test_atomic_load_umax_i64_noret_release(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i64_noret_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_noret_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var64 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB331_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp x9, x0 +; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, hi +; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB331_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw umax i64* @var64, i64 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -4351,6 +7731,20 @@ define void @test_atomic_load_umax_i64_noret_release(i64 %offset) nounwind { define i8 @test_atomic_load_umax_i8_seq_cst(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i8_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i8_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var8 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: .LBB332_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrb w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi +; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB332_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umax i8* @var8, i8 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -4364,6 +7758,20 @@ define i8 @test_atomic_load_umax_i8_seq_cst(i8 %offset) nounwind { define i16 @test_atomic_load_umax_i16_seq_cst(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i16_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i16_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var16 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: .LBB333_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrh w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi +; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB333_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umax i16* @var16, i16 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -4377,6 +7785,20 @@ define i16 @test_atomic_load_umax_i16_seq_cst(i16 %offset) nounwind { define i32 @test_atomic_load_umax_i32_seq_cst(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i32_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var32 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB334_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0 +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi +; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB334_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umax i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4390,6 +7812,20 @@ define i32 @test_atomic_load_umax_i32_seq_cst(i32 %offset) nounwind { define i64 @test_atomic_load_umax_i64_seq_cst(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i64_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var64 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB335_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp x8, x0 +; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, hi +; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB335_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov x0, x8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umax i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -4403,6 +7839,19 @@ define i64 @test_atomic_load_umax_i64_seq_cst(i64 %offset) nounwind { define void @test_atomic_load_umax_i32_noret_seq_cst(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i32_noret_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_noret_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var32 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB336_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp w9, w0 +; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, hi +; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB336_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw umax i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4415,6 +7864,19 @@ define void @test_atomic_load_umax_i32_noret_seq_cst(i32 %offset) nounwind { define void @test_atomic_load_umax_i64_noret_seq_cst(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i64_noret_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_noret_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var64 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB337_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp x9, x0 +; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, hi +; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB337_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw umax i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -4427,6 +7889,20 @@ define void @test_atomic_load_umax_i64_noret_seq_cst(i64 %offset) nounwind { define i8 @test_atomic_load_umin_i8_acq_rel(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i8_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i8_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var8 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: .LBB338_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrb w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls +; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB338_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umin i8* @var8, i8 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -4440,6 +7916,20 @@ define i8 @test_atomic_load_umin_i8_acq_rel(i8 %offset) nounwind { define i16 @test_atomic_load_umin_i16_acq_rel(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i16_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i16_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var16 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: .LBB339_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrh w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls +; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB339_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umin i16* @var16, i16 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -4453,6 +7943,20 @@ define i16 @test_atomic_load_umin_i16_acq_rel(i16 %offset) nounwind { define i32 @test_atomic_load_umin_i32_acq_rel(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i32_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var32 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB340_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0 +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls +; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB340_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umin i32* @var32, i32 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4466,6 +7970,20 @@ define i32 @test_atomic_load_umin_i32_acq_rel(i32 %offset) nounwind { define i64 @test_atomic_load_umin_i64_acq_rel(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i64_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var64 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB341_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp x8, x0 +; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, ls +; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB341_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov x0, x8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umin i64* @var64, i64 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -4479,6 +7997,19 @@ define i64 @test_atomic_load_umin_i64_acq_rel(i64 %offset) nounwind { define void @test_atomic_load_umin_i32_noret_acq_rel(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i32_noret_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_noret_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var32 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB342_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp w9, w0 +; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, ls +; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB342_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw umin i32* @var32, i32 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4491,6 +8022,19 @@ define void @test_atomic_load_umin_i32_noret_acq_rel(i32 %offset) nounwind { define void @test_atomic_load_umin_i64_noret_acq_rel(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i64_noret_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_noret_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var64 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB343_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp x9, x0 +; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, ls +; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB343_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw umin i64* @var64, i64 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -4503,6 +8047,20 @@ define void @test_atomic_load_umin_i64_noret_acq_rel(i64 %offset) nounwind { define i8 @test_atomic_load_umin_i8_acquire(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i8_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i8_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var8 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: .LBB344_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrb w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls +; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB344_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umin i8* @var8, i8 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -4516,6 +8074,20 @@ define i8 @test_atomic_load_umin_i8_acquire(i8 %offset) nounwind { define i16 @test_atomic_load_umin_i16_acquire(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i16_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i16_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var16 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: .LBB345_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrh w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls +; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB345_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umin i16* @var16, i16 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -4529,6 +8101,20 @@ define i16 @test_atomic_load_umin_i16_acquire(i16 %offset) nounwind { define i32 @test_atomic_load_umin_i32_acquire(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i32_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var32 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB346_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0 +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls +; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB346_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umin i32* @var32, i32 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4542,6 +8128,20 @@ define i32 @test_atomic_load_umin_i32_acquire(i32 %offset) nounwind { define i64 @test_atomic_load_umin_i64_acquire(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i64_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var64 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB347_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp x8, x0 +; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, ls +; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB347_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov x0, x8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umin i64* @var64, i64 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -4555,6 +8155,19 @@ define i64 @test_atomic_load_umin_i64_acquire(i64 %offset) nounwind { define void @test_atomic_load_umin_i32_noret_acquire(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i32_noret_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_noret_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var32 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB348_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp w9, w0 +; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, ls +; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB348_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw umin i32* @var32, i32 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4567,6 +8180,19 @@ define void @test_atomic_load_umin_i32_noret_acquire(i32 %offset) nounwind { define void @test_atomic_load_umin_i64_noret_acquire(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i64_noret_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_noret_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var64 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB349_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp x9, x0 +; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, ls +; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB349_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw umin i64* @var64, i64 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -4579,6 +8205,20 @@ define void @test_atomic_load_umin_i64_noret_acquire(i64 %offset) nounwind { define i8 @test_atomic_load_umin_i8_monotonic(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i8_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i8_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var8 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: .LBB350_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxrb w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls +; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB350_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umin i8* @var8, i8 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -4592,6 +8232,20 @@ define i8 @test_atomic_load_umin_i8_monotonic(i8 %offset) nounwind { define i16 @test_atomic_load_umin_i16_monotonic(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i16_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i16_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var16 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: .LBB351_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxrh w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls +; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB351_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umin i16* @var16, i16 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -4605,6 +8259,20 @@ define i16 @test_atomic_load_umin_i16_monotonic(i16 %offset) nounwind { define i32 @test_atomic_load_umin_i32_monotonic(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i32_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var32 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB352_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0 +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls +; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB352_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umin i32* @var32, i32 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4618,6 +8286,20 @@ define i32 @test_atomic_load_umin_i32_monotonic(i32 %offset) nounwind { define i64 @test_atomic_load_umin_i64_monotonic(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i64_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var64 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB353_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp x8, x0 +; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, ls +; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB353_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov x0, x8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umin i64* @var64, i64 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -4631,6 +8313,19 @@ define i64 @test_atomic_load_umin_i64_monotonic(i64 %offset) nounwind { define void @test_atomic_load_umin_i32_noret_monotonic(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i32_noret_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_noret_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var32 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB354_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp w9, w0 +; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, ls +; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB354_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw umin i32* @var32, i32 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4643,6 +8338,19 @@ define void @test_atomic_load_umin_i32_noret_monotonic(i32 %offset) nounwind { define void @test_atomic_load_umin_i64_noret_monotonic(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i64_noret_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_noret_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var64 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB355_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp x9, x0 +; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, ls +; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB355_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw umin i64* @var64, i64 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -4655,6 +8363,20 @@ define void @test_atomic_load_umin_i64_noret_monotonic(i64 %offset) nounwind { define i8 @test_atomic_load_umin_i8_release(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i8_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i8_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var8 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: .LBB356_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxrb w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls +; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB356_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umin i8* @var8, i8 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -4668,6 +8390,20 @@ define i8 @test_atomic_load_umin_i8_release(i8 %offset) nounwind { define i16 @test_atomic_load_umin_i16_release(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i16_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i16_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var16 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: .LBB357_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxrh w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls +; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB357_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umin i16* @var16, i16 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -4681,6 +8417,20 @@ define i16 @test_atomic_load_umin_i16_release(i16 %offset) nounwind { define i32 @test_atomic_load_umin_i32_release(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i32_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var32 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB358_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0 +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls +; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB358_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umin i32* @var32, i32 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4694,6 +8444,20 @@ define i32 @test_atomic_load_umin_i32_release(i32 %offset) nounwind { define i64 @test_atomic_load_umin_i64_release(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i64_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var64 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB359_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp x8, x0 +; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, ls +; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB359_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov x0, x8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umin i64* @var64, i64 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -4707,6 +8471,19 @@ define i64 @test_atomic_load_umin_i64_release(i64 %offset) nounwind { define void @test_atomic_load_umin_i32_noret_release(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i32_noret_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_noret_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var32 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB360_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp w9, w0 +; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, ls +; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB360_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw umin i32* @var32, i32 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4719,6 +8496,19 @@ define void @test_atomic_load_umin_i32_noret_release(i32 %offset) nounwind { define void @test_atomic_load_umin_i64_noret_release(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i64_noret_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_noret_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var64 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB361_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp x9, x0 +; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, ls +; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB361_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw umin i64* @var64, i64 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -4731,6 +8521,20 @@ define void @test_atomic_load_umin_i64_noret_release(i64 %offset) nounwind { define i8 @test_atomic_load_umin_i8_seq_cst(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i8_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i8_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var8 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: .LBB362_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrb w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxtb +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls +; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB362_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umin i8* @var8, i8 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -4744,6 +8548,20 @@ define i8 @test_atomic_load_umin_i8_seq_cst(i8 %offset) nounwind { define i16 @test_atomic_load_umin_i16_seq_cst(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i16_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i16_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var16 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: .LBB363_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxrh w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0, uxth +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls +; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB363_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umin i16* @var16, i16 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -4757,6 +8575,20 @@ define i16 @test_atomic_load_umin_i16_seq_cst(i16 %offset) nounwind { define i32 @test_atomic_load_umin_i32_seq_cst(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i32_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var32 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB364_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp w8, w0 +; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls +; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB364_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov w0, w8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umin i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4770,6 +8602,20 @@ define i32 @test_atomic_load_umin_i32_seq_cst(i32 %offset) nounwind { define i64 @test_atomic_load_umin_i64_seq_cst(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i64_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x9, var64 +; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB365_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] +; OUTLINE-ATOMICS-NEXT: cmp x8, x0 +; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, ls +; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] +; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB365_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: mov x0, x8 +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw umin i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -4783,6 +8629,19 @@ define i64 @test_atomic_load_umin_i64_seq_cst(i64 %offset) nounwind { define void @test_atomic_load_umin_i32_noret_seq_cst(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i32_noret_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_noret_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var32 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: .LBB366_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp w9, w0 +; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, ls +; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB366_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw umin i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4795,6 +8654,19 @@ define void @test_atomic_load_umin_i32_noret_seq_cst(i32 %offset) nounwind { define void @test_atomic_load_umin_i64_noret_seq_cst(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i64_noret_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_noret_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: adrp x8, var64 +; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: .LBB367_1: // %atomicrmw.start +; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] +; OUTLINE-ATOMICS-NEXT: cmp x9, x0 +; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, ls +; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] +; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB367_1 +; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE-ATOMICS-NEXT: ret atomicrmw umin i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -4807,6 +8679,14 @@ define void @test_atomic_load_umin_i64_noret_seq_cst(i64 %offset) nounwind { define i8 @test_atomic_load_xor_i8_acq_rel(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i8_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i8_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor1_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xor i8* @var8, i8 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -4820,6 +8700,14 @@ define i8 @test_atomic_load_xor_i8_acq_rel(i8 %offset) nounwind { define i16 @test_atomic_load_xor_i16_acq_rel(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i16_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i16_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor2_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xor i16* @var16, i16 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -4833,6 +8721,14 @@ define i16 @test_atomic_load_xor_i16_acq_rel(i16 %offset) nounwind { define i32 @test_atomic_load_xor_i32_acq_rel(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i32_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i32_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xor i32* @var32, i32 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4846,6 +8742,14 @@ define i32 @test_atomic_load_xor_i32_acq_rel(i32 %offset) nounwind { define i64 @test_atomic_load_xor_i64_acq_rel(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i64_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i64_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xor i64* @var64, i64 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -4859,6 +8763,14 @@ define i64 @test_atomic_load_xor_i64_acq_rel(i64 %offset) nounwind { define void @test_atomic_load_xor_i32_noret_acq_rel(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i32_noret_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i32_noret_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw xor i32* @var32, i32 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4871,6 +8783,14 @@ define void @test_atomic_load_xor_i32_noret_acq_rel(i32 %offset) nounwind { define void @test_atomic_load_xor_i64_noret_acq_rel(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i64_noret_acq_rel: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i64_noret_acq_rel: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw xor i64* @var64, i64 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -4883,6 +8803,14 @@ define void @test_atomic_load_xor_i64_noret_acq_rel(i64 %offset) nounwind { define i8 @test_atomic_load_xor_i8_acquire(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i8_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i8_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor1_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xor i8* @var8, i8 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -4896,6 +8824,14 @@ define i8 @test_atomic_load_xor_i8_acquire(i8 %offset) nounwind { define i16 @test_atomic_load_xor_i16_acquire(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i16_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i16_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor2_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xor i16* @var16, i16 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -4909,6 +8845,14 @@ define i16 @test_atomic_load_xor_i16_acquire(i16 %offset) nounwind { define i32 @test_atomic_load_xor_i32_acquire(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i32_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i32_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor4_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xor i32* @var32, i32 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4922,6 +8866,14 @@ define i32 @test_atomic_load_xor_i32_acquire(i32 %offset) nounwind { define i64 @test_atomic_load_xor_i64_acquire(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i64_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i64_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor8_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xor i64* @var64, i64 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -4935,6 +8887,14 @@ define i64 @test_atomic_load_xor_i64_acquire(i64 %offset) nounwind { define void @test_atomic_load_xor_i32_noret_acquire(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i32_noret_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i32_noret_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor4_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw xor i32* @var32, i32 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4947,6 +8907,14 @@ define void @test_atomic_load_xor_i32_noret_acquire(i32 %offset) nounwind { define void @test_atomic_load_xor_i64_noret_acquire(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i64_noret_acquire: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i64_noret_acquire: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor8_acq +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw xor i64* @var64, i64 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -4959,6 +8927,14 @@ define void @test_atomic_load_xor_i64_noret_acquire(i64 %offset) nounwind { define i8 @test_atomic_load_xor_i8_monotonic(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i8_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i8_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor1_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xor i8* @var8, i8 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -4972,6 +8948,14 @@ define i8 @test_atomic_load_xor_i8_monotonic(i8 %offset) nounwind { define i16 @test_atomic_load_xor_i16_monotonic(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i16_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i16_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor2_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xor i16* @var16, i16 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -4985,6 +8969,14 @@ define i16 @test_atomic_load_xor_i16_monotonic(i16 %offset) nounwind { define i32 @test_atomic_load_xor_i32_monotonic(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i32_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i32_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor4_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xor i32* @var32, i32 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -4998,6 +8990,14 @@ define i32 @test_atomic_load_xor_i32_monotonic(i32 %offset) nounwind { define i64 @test_atomic_load_xor_i64_monotonic(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i64_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i64_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor8_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xor i64* @var64, i64 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -5011,6 +9011,14 @@ define i64 @test_atomic_load_xor_i64_monotonic(i64 %offset) nounwind { define void @test_atomic_load_xor_i32_noret_monotonic(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i32_noret_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i32_noret_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor4_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw xor i32* @var32, i32 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -5023,6 +9031,14 @@ define void @test_atomic_load_xor_i32_noret_monotonic(i32 %offset) nounwind { define void @test_atomic_load_xor_i64_noret_monotonic(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i64_noret_monotonic: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i64_noret_monotonic: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor8_relax +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw xor i64* @var64, i64 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -5035,6 +9051,14 @@ define void @test_atomic_load_xor_i64_noret_monotonic(i64 %offset) nounwind { define i8 @test_atomic_load_xor_i8_release(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i8_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i8_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor1_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xor i8* @var8, i8 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -5048,6 +9072,14 @@ define i8 @test_atomic_load_xor_i8_release(i8 %offset) nounwind { define i16 @test_atomic_load_xor_i16_release(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i16_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i16_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor2_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xor i16* @var16, i16 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -5061,6 +9093,14 @@ define i16 @test_atomic_load_xor_i16_release(i16 %offset) nounwind { define i32 @test_atomic_load_xor_i32_release(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i32_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i32_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor4_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xor i32* @var32, i32 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -5074,6 +9114,14 @@ define i32 @test_atomic_load_xor_i32_release(i32 %offset) nounwind { define i64 @test_atomic_load_xor_i64_release(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i64_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i64_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor8_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xor i64* @var64, i64 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -5087,6 +9135,14 @@ define i64 @test_atomic_load_xor_i64_release(i64 %offset) nounwind { define void @test_atomic_load_xor_i32_noret_release(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i32_noret_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i32_noret_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor4_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw xor i32* @var32, i32 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -5099,6 +9155,14 @@ define void @test_atomic_load_xor_i32_noret_release(i32 %offset) nounwind { define void @test_atomic_load_xor_i64_noret_release(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i64_noret_release: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i64_noret_release: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor8_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw xor i64* @var64, i64 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -5111,6 +9175,14 @@ define void @test_atomic_load_xor_i64_noret_release(i64 %offset) nounwind { define i8 @test_atomic_load_xor_i8_seq_cst(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i8_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i8_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var8 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor1_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xor i8* @var8, i8 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -5124,6 +9196,14 @@ define i8 @test_atomic_load_xor_i8_seq_cst(i8 %offset) nounwind { define i16 @test_atomic_load_xor_i16_seq_cst(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i16_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i16_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var16 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor2_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xor i16* @var16, i16 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -5137,6 +9217,14 @@ define i16 @test_atomic_load_xor_i16_seq_cst(i16 %offset) nounwind { define i32 @test_atomic_load_xor_i32_seq_cst(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i32_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i32_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xor i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -5150,6 +9238,14 @@ define i32 @test_atomic_load_xor_i32_seq_cst(i32 %offset) nounwind { define i64 @test_atomic_load_xor_i64_seq_cst(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i64_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i64_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret %old = atomicrmw xor i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -5163,6 +9259,14 @@ define i64 @test_atomic_load_xor_i64_seq_cst(i64 %offset) nounwind { define void @test_atomic_load_xor_i32_noret_seq_cst(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i32_noret_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i32_noret_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var32 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor4_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw xor i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -5175,6 +9279,14 @@ define void @test_atomic_load_xor_i32_noret_seq_cst(i32 %offset) nounwind { define void @test_atomic_load_xor_i64_noret_seq_cst(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i64_noret_seq_cst: +; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i64_noret_seq_cst: +; OUTLINE-ATOMICS: // %bb.0: +; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-ATOMICS-NEXT: adrp x1, var64 +; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor8_acq_rel +; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-ATOMICS-NEXT: ret atomicrmw xor i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 diff --git a/llvm/test/CodeGen/AArch64/atomic-ops-not-barriers.ll b/llvm/test/CodeGen/AArch64/atomic-ops-not-barriers.ll index 44e7771..25fb9b8 100644 --- a/llvm/test/CodeGen/AArch64/atomic-ops-not-barriers.ll +++ b/llvm/test/CodeGen/AArch64/atomic-ops-not-barriers.ll @@ -1,6 +1,8 @@ ; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs -mattr=+outline-atomics < %s | FileCheck %s --check-prefix=OUTLINE-ATOMICS define i32 @foo(i32* %var, i1 %cond) { +; OUTLINE-ATOMICS: bl __aarch64_ldadd4_relax ; CHECK-LABEL: foo: br i1 %cond, label %atomic_ver, label %simple_ver simple_ver: diff --git a/llvm/test/CodeGen/AArch64/atomic-ops.ll b/llvm/test/CodeGen/AArch64/atomic-ops.ll index b763e06..1a52e31 100644 --- a/llvm/test/CodeGen/AArch64/atomic-ops.ll +++ b/llvm/test/CodeGen/AArch64/atomic-ops.ll @@ -1,5 +1,6 @@ ; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs < %s | FileCheck %s ; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK-REG +; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs -mattr=+outline-atomics < %s | FileCheck %s --check-prefix=OUTLINE_ATOMICS ; Point of CHECK-REG is to make sure UNPREDICTABLE instructions aren't created @@ -14,6 +15,14 @@ define i8 @test_atomic_load_add_i8(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i8: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_add_i8: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: adrp x1, var8 +; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd1_acq_rel +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw add i8* @var8, i8 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -34,6 +43,14 @@ define i8 @test_atomic_load_add_i8(i8 %offset) nounwind { define i16 @test_atomic_load_add_i16(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i16: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_add_i16: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: adrp x1, var16 +; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd2_acq +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw add i16* @var16, i16 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -54,6 +71,14 @@ define i16 @test_atomic_load_add_i16(i16 %offset) nounwind { define i32 @test_atomic_load_add_i32(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i32: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_add_i32: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: adrp x1, var32 +; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd4_rel +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw add i32* @var32, i32 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -74,6 +99,14 @@ define i32 @test_atomic_load_add_i32(i32 %offset) nounwind { define i64 @test_atomic_load_add_i64(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_add_i64: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_add_i64: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: adrp x1, var64 +; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd8_relax +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw add i64* @var64, i64 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -94,6 +127,15 @@ define i64 @test_atomic_load_add_i64(i64 %offset) nounwind { define i8 @test_atomic_load_sub_i8(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i8: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_sub_i8: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: neg w0, w0 +; OUTLINE_ATOMICS-NEXT: adrp x1, var8 +; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd1_relax +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw sub i8* @var8, i8 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -114,6 +156,15 @@ define i8 @test_atomic_load_sub_i8(i8 %offset) nounwind { define i16 @test_atomic_load_sub_i16(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i16: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_sub_i16: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: neg w0, w0 +; OUTLINE_ATOMICS-NEXT: adrp x1, var16 +; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd2_rel +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw sub i16* @var16, i16 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -134,6 +185,15 @@ define i16 @test_atomic_load_sub_i16(i16 %offset) nounwind { define i32 @test_atomic_load_sub_i32(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i32: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_sub_i32: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: neg w0, w0 +; OUTLINE_ATOMICS-NEXT: adrp x1, var32 +; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd4_acq +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw sub i32* @var32, i32 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -154,6 +214,15 @@ define i32 @test_atomic_load_sub_i32(i32 %offset) nounwind { define i64 @test_atomic_load_sub_i64(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_sub_i64: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_sub_i64: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: neg x0, x0 +; OUTLINE_ATOMICS-NEXT: adrp x1, var64 +; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw sub i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -174,6 +243,15 @@ define i64 @test_atomic_load_sub_i64(i64 %offset) nounwind { define i8 @test_atomic_load_and_i8(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i8: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_and_i8: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: mvn w0, w0 +; OUTLINE_ATOMICS-NEXT: adrp x1, var8 +; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldclr1_rel +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw and i8* @var8, i8 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -194,6 +272,15 @@ define i8 @test_atomic_load_and_i8(i8 %offset) nounwind { define i16 @test_atomic_load_and_i16(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i16: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_and_i16: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: mvn w0, w0 +; OUTLINE_ATOMICS-NEXT: adrp x1, var16 +; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldclr2_relax +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw and i16* @var16, i16 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -214,6 +301,15 @@ define i16 @test_atomic_load_and_i16(i16 %offset) nounwind { define i32 @test_atomic_load_and_i32(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i32: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_and_i32: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: mvn w0, w0 +; OUTLINE_ATOMICS-NEXT: adrp x1, var32 +; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldclr4_acq_rel +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw and i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -234,6 +330,15 @@ define i32 @test_atomic_load_and_i32(i32 %offset) nounwind { define i64 @test_atomic_load_and_i64(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_and_i64: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_and_i64: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: mvn x0, x0 +; OUTLINE_ATOMICS-NEXT: adrp x1, var64 +; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldclr8_acq +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw and i64* @var64, i64 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -254,6 +359,14 @@ define i64 @test_atomic_load_and_i64(i64 %offset) nounwind { define i8 @test_atomic_load_or_i8(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i8: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_or_i8: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: adrp x1, var8 +; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldset1_acq_rel +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw or i8* @var8, i8 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -274,6 +387,14 @@ define i8 @test_atomic_load_or_i8(i8 %offset) nounwind { define i16 @test_atomic_load_or_i16(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i16: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_or_i16: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: adrp x1, var16 +; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldset2_relax +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw or i16* @var16, i16 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -294,6 +415,14 @@ define i16 @test_atomic_load_or_i16(i16 %offset) nounwind { define i32 @test_atomic_load_or_i32(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i32: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_or_i32: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: adrp x1, var32 +; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldset4_acq +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw or i32* @var32, i32 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -314,6 +443,14 @@ define i32 @test_atomic_load_or_i32(i32 %offset) nounwind { define i64 @test_atomic_load_or_i64(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_or_i64: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_or_i64: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: adrp x1, var64 +; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldset8_rel +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw or i64* @var64, i64 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -334,6 +471,14 @@ define i64 @test_atomic_load_or_i64(i64 %offset) nounwind { define i8 @test_atomic_load_xor_i8(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i8: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_xor_i8: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: adrp x1, var8 +; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldeor1_acq +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw xor i8* @var8, i8 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -354,6 +499,14 @@ define i8 @test_atomic_load_xor_i8(i8 %offset) nounwind { define i16 @test_atomic_load_xor_i16(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i16: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_xor_i16: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: adrp x1, var16 +; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldeor2_rel +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw xor i16* @var16, i16 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -374,6 +527,14 @@ define i16 @test_atomic_load_xor_i16(i16 %offset) nounwind { define i32 @test_atomic_load_xor_i32(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i32: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_xor_i32: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: adrp x1, var32 +; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldeor4_acq_rel +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw xor i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -394,6 +555,14 @@ define i32 @test_atomic_load_xor_i32(i32 %offset) nounwind { define i64 @test_atomic_load_xor_i64(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xor_i64: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_xor_i64: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: adrp x1, var64 +; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldeor8_relax +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw xor i64* @var64, i64 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -414,6 +583,14 @@ define i64 @test_atomic_load_xor_i64(i64 %offset) nounwind { define i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i8: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_xchg_i8: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: adrp x1, var8 +; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var8 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_swp1_relax +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw xchg i8* @var8, i8 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -433,6 +610,14 @@ define i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind { define i16 @test_atomic_load_xchg_i16(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i16: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_xchg_i16: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: adrp x1, var16 +; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var16 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_swp2_acq_rel +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw xchg i16* @var16, i16 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -453,6 +638,14 @@ define i16 @test_atomic_load_xchg_i16(i16 %offset) nounwind { define i32 @test_atomic_load_xchg_i32(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i32: ; CHECK: mov {{[xw]}}8, w[[OLD:[0-9]+]] +; OUTLINE_ATOMICS-LABEL: test_atomic_load_xchg_i32: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: adrp x1, var32 +; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var32 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_swp4_rel +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw xchg i32* @var32, i32 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -470,6 +663,14 @@ define i32 @test_atomic_load_xchg_i32(i32 %offset) nounwind { define i64 @test_atomic_load_xchg_i64(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_xchg_i64: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_xchg_i64: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: adrp x1, var64 +; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var64 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_swp8_acq +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw xchg i64* @var64, i64 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -490,6 +691,21 @@ define i64 @test_atomic_load_xchg_i64(i64 %offset) nounwind { define i8 @test_atomic_load_min_i8(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i8: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_min_i8: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: adrp x9, var8 +; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE_ATOMICS-NEXT: .LBB24_1: // %atomicrmw.start +; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE_ATOMICS-NEXT: ldaxrb w10, [x9] +; OUTLINE_ATOMICS-NEXT: sxtb w8, w10 +; OUTLINE_ATOMICS-NEXT: cmp w8, w0, sxtb +; OUTLINE_ATOMICS-NEXT: csel w10, w10, w0, le +; OUTLINE_ATOMICS-NEXT: stxrb w11, w10, [x9] +; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB24_1 +; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE_ATOMICS-NEXT: mov w0, w8 +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw min i8* @var8, i8 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -514,6 +730,21 @@ define i8 @test_atomic_load_min_i8(i8 %offset) nounwind { define i16 @test_atomic_load_min_i16(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i16: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_min_i16: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: adrp x9, var16 +; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE_ATOMICS-NEXT: .LBB25_1: // %atomicrmw.start +; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE_ATOMICS-NEXT: ldxrh w10, [x9] +; OUTLINE_ATOMICS-NEXT: sxth w8, w10 +; OUTLINE_ATOMICS-NEXT: cmp w8, w0, sxth +; OUTLINE_ATOMICS-NEXT: csel w10, w10, w0, le +; OUTLINE_ATOMICS-NEXT: stlxrh w11, w10, [x9] +; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB25_1 +; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE_ATOMICS-NEXT: mov w0, w8 +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw min i16* @var16, i16 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -539,6 +770,20 @@ define i16 @test_atomic_load_min_i16(i16 %offset) nounwind { define i32 @test_atomic_load_min_i32(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i32: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_min_i32: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: adrp x9, var32 +; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE_ATOMICS-NEXT: .LBB26_1: // %atomicrmw.start +; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE_ATOMICS-NEXT: ldxr w8, [x9] +; OUTLINE_ATOMICS-NEXT: cmp w8, w0 +; OUTLINE_ATOMICS-NEXT: csel w10, w8, w0, le +; OUTLINE_ATOMICS-NEXT: stxr w11, w10, [x9] +; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB26_1 +; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE_ATOMICS-NEXT: mov w0, w8 +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw min i32* @var32, i32 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -563,6 +808,20 @@ define i32 @test_atomic_load_min_i32(i32 %offset) nounwind { define i64 @test_atomic_load_min_i64(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_min_i64: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_min_i64: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: adrp x9, var64 +; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE_ATOMICS-NEXT: .LBB27_1: // %atomicrmw.start +; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE_ATOMICS-NEXT: ldaxr x8, [x9] +; OUTLINE_ATOMICS-NEXT: cmp x8, x0 +; OUTLINE_ATOMICS-NEXT: csel x10, x8, x0, le +; OUTLINE_ATOMICS-NEXT: stlxr w11, x10, [x9] +; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB27_1 +; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE_ATOMICS-NEXT: mov x0, x8 +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw min i64* @var64, i64 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -587,6 +846,21 @@ define i64 @test_atomic_load_min_i64(i64 %offset) nounwind { define i8 @test_atomic_load_max_i8(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i8: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_max_i8: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: adrp x9, var8 +; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE_ATOMICS-NEXT: .LBB28_1: // %atomicrmw.start +; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE_ATOMICS-NEXT: ldaxrb w10, [x9] +; OUTLINE_ATOMICS-NEXT: sxtb w8, w10 +; OUTLINE_ATOMICS-NEXT: cmp w8, w0, sxtb +; OUTLINE_ATOMICS-NEXT: csel w10, w10, w0, gt +; OUTLINE_ATOMICS-NEXT: stlxrb w11, w10, [x9] +; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB28_1 +; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE_ATOMICS-NEXT: mov w0, w8 +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw max i8* @var8, i8 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -612,6 +886,21 @@ define i8 @test_atomic_load_max_i8(i8 %offset) nounwind { define i16 @test_atomic_load_max_i16(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i16: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_max_i16: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: adrp x9, var16 +; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE_ATOMICS-NEXT: .LBB29_1: // %atomicrmw.start +; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE_ATOMICS-NEXT: ldaxrh w10, [x9] +; OUTLINE_ATOMICS-NEXT: sxth w8, w10 +; OUTLINE_ATOMICS-NEXT: cmp w8, w0, sxth +; OUTLINE_ATOMICS-NEXT: csel w10, w10, w0, gt +; OUTLINE_ATOMICS-NEXT: stxrh w11, w10, [x9] +; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB29_1 +; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE_ATOMICS-NEXT: mov w0, w8 +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw max i16* @var16, i16 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -637,6 +926,20 @@ define i16 @test_atomic_load_max_i16(i16 %offset) nounwind { define i32 @test_atomic_load_max_i32(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i32: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_max_i32: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: adrp x9, var32 +; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE_ATOMICS-NEXT: .LBB30_1: // %atomicrmw.start +; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE_ATOMICS-NEXT: ldxr w8, [x9] +; OUTLINE_ATOMICS-NEXT: cmp w8, w0 +; OUTLINE_ATOMICS-NEXT: csel w10, w8, w0, gt +; OUTLINE_ATOMICS-NEXT: stlxr w11, w10, [x9] +; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB30_1 +; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE_ATOMICS-NEXT: mov w0, w8 +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw max i32* @var32, i32 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -661,6 +964,20 @@ define i32 @test_atomic_load_max_i32(i32 %offset) nounwind { define i64 @test_atomic_load_max_i64(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_max_i64: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_max_i64: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: adrp x9, var64 +; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE_ATOMICS-NEXT: .LBB31_1: // %atomicrmw.start +; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE_ATOMICS-NEXT: ldxr x8, [x9] +; OUTLINE_ATOMICS-NEXT: cmp x8, x0 +; OUTLINE_ATOMICS-NEXT: csel x10, x8, x0, gt +; OUTLINE_ATOMICS-NEXT: stxr w11, x10, [x9] +; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB31_1 +; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE_ATOMICS-NEXT: mov x0, x8 +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw max i64* @var64, i64 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -685,6 +1002,20 @@ define i64 @test_atomic_load_max_i64(i64 %offset) nounwind { define i8 @test_atomic_load_umin_i8(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i8: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_umin_i8: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: adrp x9, var8 +; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE_ATOMICS-NEXT: .LBB32_1: // %atomicrmw.start +; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE_ATOMICS-NEXT: ldxrb w8, [x9] +; OUTLINE_ATOMICS-NEXT: cmp w8, w0, uxtb +; OUTLINE_ATOMICS-NEXT: csel w10, w8, w0, ls +; OUTLINE_ATOMICS-NEXT: stxrb w11, w10, [x9] +; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB32_1 +; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE_ATOMICS-NEXT: mov w0, w8 +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw umin i8* @var8, i8 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -709,6 +1040,20 @@ define i8 @test_atomic_load_umin_i8(i8 %offset) nounwind { define i16 @test_atomic_load_umin_i16(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i16: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_umin_i16: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: adrp x9, var16 +; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE_ATOMICS-NEXT: .LBB33_1: // %atomicrmw.start +; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE_ATOMICS-NEXT: ldaxrh w8, [x9] +; OUTLINE_ATOMICS-NEXT: cmp w8, w0, uxth +; OUTLINE_ATOMICS-NEXT: csel w10, w8, w0, ls +; OUTLINE_ATOMICS-NEXT: stxrh w11, w10, [x9] +; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB33_1 +; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE_ATOMICS-NEXT: mov w0, w8 +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw umin i16* @var16, i16 %offset acquire ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -733,6 +1078,20 @@ define i16 @test_atomic_load_umin_i16(i16 %offset) nounwind { define i32 @test_atomic_load_umin_i32(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i32: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_umin_i32: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: adrp x9, var32 +; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE_ATOMICS-NEXT: .LBB34_1: // %atomicrmw.start +; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE_ATOMICS-NEXT: ldaxr w8, [x9] +; OUTLINE_ATOMICS-NEXT: cmp w8, w0 +; OUTLINE_ATOMICS-NEXT: csel w10, w8, w0, ls +; OUTLINE_ATOMICS-NEXT: stlxr w11, w10, [x9] +; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB34_1 +; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE_ATOMICS-NEXT: mov w0, w8 +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw umin i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -757,6 +1116,20 @@ define i32 @test_atomic_load_umin_i32(i32 %offset) nounwind { define i64 @test_atomic_load_umin_i64(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umin_i64: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_umin_i64: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: adrp x9, var64 +; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE_ATOMICS-NEXT: .LBB35_1: // %atomicrmw.start +; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE_ATOMICS-NEXT: ldaxr x8, [x9] +; OUTLINE_ATOMICS-NEXT: cmp x8, x0 +; OUTLINE_ATOMICS-NEXT: csel x10, x8, x0, ls +; OUTLINE_ATOMICS-NEXT: stlxr w11, x10, [x9] +; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB35_1 +; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE_ATOMICS-NEXT: mov x0, x8 +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw umin i64* @var64, i64 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -781,6 +1154,20 @@ define i64 @test_atomic_load_umin_i64(i64 %offset) nounwind { define i8 @test_atomic_load_umax_i8(i8 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i8: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_umax_i8: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: adrp x9, var8 +; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var8 +; OUTLINE_ATOMICS-NEXT: .LBB36_1: // %atomicrmw.start +; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE_ATOMICS-NEXT: ldaxrb w8, [x9] +; OUTLINE_ATOMICS-NEXT: cmp w8, w0, uxtb +; OUTLINE_ATOMICS-NEXT: csel w10, w8, w0, hi +; OUTLINE_ATOMICS-NEXT: stlxrb w11, w10, [x9] +; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB36_1 +; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE_ATOMICS-NEXT: mov w0, w8 +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw umax i8* @var8, i8 %offset acq_rel ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -805,6 +1192,20 @@ define i8 @test_atomic_load_umax_i8(i8 %offset) nounwind { define i16 @test_atomic_load_umax_i16(i16 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i16: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_umax_i16: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: adrp x9, var16 +; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var16 +; OUTLINE_ATOMICS-NEXT: .LBB37_1: // %atomicrmw.start +; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE_ATOMICS-NEXT: ldxrh w8, [x9] +; OUTLINE_ATOMICS-NEXT: cmp w8, w0, uxth +; OUTLINE_ATOMICS-NEXT: csel w10, w8, w0, hi +; OUTLINE_ATOMICS-NEXT: stxrh w11, w10, [x9] +; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB37_1 +; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE_ATOMICS-NEXT: mov w0, w8 +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw umax i16* @var16, i16 %offset monotonic ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 @@ -829,6 +1230,20 @@ define i16 @test_atomic_load_umax_i16(i16 %offset) nounwind { define i32 @test_atomic_load_umax_i32(i32 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i32: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_umax_i32: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: adrp x9, var32 +; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var32 +; OUTLINE_ATOMICS-NEXT: .LBB38_1: // %atomicrmw.start +; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE_ATOMICS-NEXT: ldaxr w8, [x9] +; OUTLINE_ATOMICS-NEXT: cmp w8, w0 +; OUTLINE_ATOMICS-NEXT: csel w10, w8, w0, hi +; OUTLINE_ATOMICS-NEXT: stlxr w11, w10, [x9] +; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB38_1 +; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE_ATOMICS-NEXT: mov w0, w8 +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw umax i32* @var32, i32 %offset seq_cst ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 @@ -853,6 +1268,20 @@ define i32 @test_atomic_load_umax_i32(i32 %offset) nounwind { define i64 @test_atomic_load_umax_i64(i64 %offset) nounwind { ; CHECK-LABEL: test_atomic_load_umax_i64: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_umax_i64: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: adrp x9, var64 +; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var64 +; OUTLINE_ATOMICS-NEXT: .LBB39_1: // %atomicrmw.start +; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 +; OUTLINE_ATOMICS-NEXT: ldxr x8, [x9] +; OUTLINE_ATOMICS-NEXT: cmp x8, x0 +; OUTLINE_ATOMICS-NEXT: csel x10, x8, x0, hi +; OUTLINE_ATOMICS-NEXT: stlxr w11, x10, [x9] +; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB39_1 +; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end +; OUTLINE_ATOMICS-NEXT: mov x0, x8 +; OUTLINE_ATOMICS-NEXT: ret %old = atomicrmw umax i64* @var64, i64 %offset release ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 @@ -877,6 +1306,14 @@ define i64 @test_atomic_load_umax_i64(i64 %offset) nounwind { define i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i8: +; OUTLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i8: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: adrp x2, var8 +; OUTLINE_ATOMICS-NEXT: add x2, x2, :lo12:var8 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_cas1_acq +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %pair = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire acquire %old = extractvalue { i8, i1 } %pair, 0 @@ -902,6 +1339,14 @@ define i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind { define i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i16: +; OUTLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i16: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: adrp x2, var16 +; OUTLINE_ATOMICS-NEXT: add x2, x2, :lo12:var16 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_cas2_acq_rel +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %pair = cmpxchg i16* @var16, i16 %wanted, i16 %new seq_cst seq_cst %old = extractvalue { i16, i1 } %pair, 0 @@ -927,6 +1372,14 @@ define i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind { define i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i32: +; OUTLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i32: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: adrp x2, var32 +; OUTLINE_ATOMICS-NEXT: add x2, x2, :lo12:var32 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_cas4_rel +; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %pair = cmpxchg i32* @var32, i32 %wanted, i32 %new release monotonic %old = extractvalue { i32, i1 } %pair, 0 @@ -950,6 +1403,16 @@ define i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind { define void @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind { ; CHECK-LABEL: test_atomic_cmpxchg_i64: +; OUTLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i64: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: stp x30, x19, [sp, #-16]! // 16-byte Folded Spill +; OUTLINE_ATOMICS-NEXT: adrp x19, var64 +; OUTLINE_ATOMICS-NEXT: add x19, x19, :lo12:var64 +; OUTLINE_ATOMICS-NEXT: mov x2, x19 +; OUTLINE_ATOMICS-NEXT: bl __aarch64_cas8_relax +; OUTLINE_ATOMICS-NEXT: str x0, [x19] +; OUTLINE_ATOMICS-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload +; OUTLINE_ATOMICS-NEXT: ret %pair = cmpxchg i64* @var64, i64 %wanted, i64 %new monotonic monotonic %old = extractvalue { i64, i1 } %pair, 0 @@ -977,6 +1440,11 @@ define void @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind { define i8 @test_atomic_load_monotonic_i8() nounwind { ; CHECK-LABEL: test_atomic_load_monotonic_i8: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_monotonic_i8: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: adrp x8, var8 +; OUTLINE_ATOMICS-NEXT: ldrb w0, [x8, :lo12:var8] +; OUTLINE_ATOMICS-NEXT: ret %val = load atomic i8, i8* @var8 monotonic, align 1 ; CHECK-NOT: dmb ; CHECK: adrp x[[HIADDR:[0-9]+]], var8 @@ -988,6 +1456,10 @@ define i8 @test_atomic_load_monotonic_i8() nounwind { define i8 @test_atomic_load_monotonic_regoff_i8(i64 %base, i64 %off) nounwind { ; CHECK-LABEL: test_atomic_load_monotonic_regoff_i8: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_monotonic_regoff_i8: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: ldrb w0, [x0, x1] +; OUTLINE_ATOMICS-NEXT: ret %addr_int = add i64 %base, %off %addr = inttoptr i64 %addr_int to i8* @@ -1001,6 +1473,12 @@ define i8 @test_atomic_load_monotonic_regoff_i8(i64 %base, i64 %off) nounwind { define i8 @test_atomic_load_acquire_i8() nounwind { ; CHECK-LABEL: test_atomic_load_acquire_i8: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_acquire_i8: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: adrp x8, var8 +; OUTLINE_ATOMICS-NEXT: add x8, x8, :lo12:var8 +; OUTLINE_ATOMICS-NEXT: ldarb w0, [x8] +; OUTLINE_ATOMICS-NEXT: ret %val = load atomic i8, i8* @var8 acquire, align 1 ; CHECK-NOT: dmb ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 @@ -1014,6 +1492,12 @@ define i8 @test_atomic_load_acquire_i8() nounwind { define i8 @test_atomic_load_seq_cst_i8() nounwind { ; CHECK-LABEL: test_atomic_load_seq_cst_i8: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_seq_cst_i8: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: adrp x8, var8 +; OUTLINE_ATOMICS-NEXT: add x8, x8, :lo12:var8 +; OUTLINE_ATOMICS-NEXT: ldarb w0, [x8] +; OUTLINE_ATOMICS-NEXT: ret %val = load atomic i8, i8* @var8 seq_cst, align 1 ; CHECK-NOT: dmb ; CHECK: adrp [[HIADDR:x[0-9]+]], var8 @@ -1027,6 +1511,11 @@ define i8 @test_atomic_load_seq_cst_i8() nounwind { define i16 @test_atomic_load_monotonic_i16() nounwind { ; CHECK-LABEL: test_atomic_load_monotonic_i16: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_monotonic_i16: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: adrp x8, var16 +; OUTLINE_ATOMICS-NEXT: ldrh w0, [x8, :lo12:var16] +; OUTLINE_ATOMICS-NEXT: ret %val = load atomic i16, i16* @var16 monotonic, align 2 ; CHECK-NOT: dmb ; CHECK: adrp x[[HIADDR:[0-9]+]], var16 @@ -1039,6 +1528,10 @@ define i16 @test_atomic_load_monotonic_i16() nounwind { define i32 @test_atomic_load_monotonic_regoff_i32(i64 %base, i64 %off) nounwind { ; CHECK-LABEL: test_atomic_load_monotonic_regoff_i32: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_monotonic_regoff_i32: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: ldr w0, [x0, x1] +; OUTLINE_ATOMICS-NEXT: ret %addr_int = add i64 %base, %off %addr = inttoptr i64 %addr_int to i32* @@ -1052,6 +1545,12 @@ define i32 @test_atomic_load_monotonic_regoff_i32(i64 %base, i64 %off) nounwind define i64 @test_atomic_load_seq_cst_i64() nounwind { ; CHECK-LABEL: test_atomic_load_seq_cst_i64: +; OUTLINE_ATOMICS-LABEL: test_atomic_load_seq_cst_i64: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: adrp x8, var64 +; OUTLINE_ATOMICS-NEXT: add x8, x8, :lo12:var64 +; OUTLINE_ATOMICS-NEXT: ldar x0, [x8] +; OUTLINE_ATOMICS-NEXT: ret %val = load atomic i64, i64* @var64 seq_cst, align 8 ; CHECK-NOT: dmb ; CHECK: adrp [[HIADDR:x[0-9]+]], var64 @@ -1065,6 +1564,11 @@ define i64 @test_atomic_load_seq_cst_i64() nounwind { define void @test_atomic_store_monotonic_i8(i8 %val) nounwind { ; CHECK-LABEL: test_atomic_store_monotonic_i8: +; OUTLINE_ATOMICS-LABEL: test_atomic_store_monotonic_i8: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: adrp x8, var8 +; OUTLINE_ATOMICS-NEXT: strb w0, [x8, :lo12:var8] +; OUTLINE_ATOMICS-NEXT: ret store atomic i8 %val, i8* @var8 monotonic, align 1 ; CHECK: adrp x[[HIADDR:[0-9]+]], var8 ; CHECK: strb w0, [x[[HIADDR]], {{#?}}:lo12:var8] @@ -1074,7 +1578,10 @@ define void @test_atomic_store_monotonic_i8(i8 %val) nounwind { define void @test_atomic_store_monotonic_regoff_i8(i64 %base, i64 %off, i8 %val) nounwind { ; CHECK-LABEL: test_atomic_store_monotonic_regoff_i8: - +; OUTLINE_ATOMICS-LABEL: test_atomic_store_monotonic_regoff_i8: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: strb w2, [x0, x1] +; OUTLINE_ATOMICS-NEXT: ret %addr_int = add i64 %base, %off %addr = inttoptr i64 %addr_int to i8* @@ -1085,6 +1592,12 @@ define void @test_atomic_store_monotonic_regoff_i8(i64 %base, i64 %off, i8 %val) } define void @test_atomic_store_release_i8(i8 %val) nounwind { ; CHECK-LABEL: test_atomic_store_release_i8: +; OUTLINE_ATOMICS-LABEL: test_atomic_store_release_i8: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: adrp x8, var8 +; OUTLINE_ATOMICS-NEXT: add x8, x8, :lo12:var8 +; OUTLINE_ATOMICS-NEXT: stlrb w0, [x8] +; OUTLINE_ATOMICS-NEXT: ret store atomic i8 %val, i8* @var8 release, align 1 ; CHECK-NOT: dmb ; CHECK: adrp [[HIADDR:x[0-9]+]], var8 @@ -1098,6 +1611,12 @@ define void @test_atomic_store_release_i8(i8 %val) nounwind { define void @test_atomic_store_seq_cst_i8(i8 %val) nounwind { ; CHECK-LABEL: test_atomic_store_seq_cst_i8: +; OUTLINE_ATOMICS-LABEL: test_atomic_store_seq_cst_i8: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: adrp x8, var8 +; OUTLINE_ATOMICS-NEXT: add x8, x8, :lo12:var8 +; OUTLINE_ATOMICS-NEXT: stlrb w0, [x8] +; OUTLINE_ATOMICS-NEXT: ret store atomic i8 %val, i8* @var8 seq_cst, align 1 ; CHECK-NOT: dmb ; CHECK: adrp [[HIADDR:x[0-9]+]], var8 @@ -1112,6 +1631,11 @@ define void @test_atomic_store_seq_cst_i8(i8 %val) nounwind { define void @test_atomic_store_monotonic_i16(i16 %val) nounwind { ; CHECK-LABEL: test_atomic_store_monotonic_i16: +; OUTLINE_ATOMICS-LABEL: test_atomic_store_monotonic_i16: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: adrp x8, var16 +; OUTLINE_ATOMICS-NEXT: strh w0, [x8, :lo12:var16] +; OUTLINE_ATOMICS-NEXT: ret store atomic i16 %val, i16* @var16 monotonic, align 2 ; CHECK-NOT: dmb ; CHECK: adrp x[[HIADDR:[0-9]+]], var16 @@ -1123,7 +1647,10 @@ define void @test_atomic_store_monotonic_i16(i16 %val) nounwind { define void @test_atomic_store_monotonic_regoff_i32(i64 %base, i64 %off, i32 %val) nounwind { ; CHECK-LABEL: test_atomic_store_monotonic_regoff_i32: - +; OUTLINE_ATOMICS-LABEL: test_atomic_store_monotonic_regoff_i32: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: str w2, [x0, x1] +; OUTLINE_ATOMICS-NEXT: ret %addr_int = add i64 %base, %off %addr = inttoptr i64 %addr_int to i32* @@ -1137,6 +1664,12 @@ define void @test_atomic_store_monotonic_regoff_i32(i64 %base, i64 %off, i32 %va define void @test_atomic_store_release_i64(i64 %val) nounwind { ; CHECK-LABEL: test_atomic_store_release_i64: +; OUTLINE_ATOMICS-LABEL: test_atomic_store_release_i64: +; OUTLINE_ATOMICS: // %bb.0: +; OUTLINE_ATOMICS-NEXT: adrp x8, var64 +; OUTLINE_ATOMICS-NEXT: add x8, x8, :lo12:var64 +; OUTLINE_ATOMICS-NEXT: stlr x0, [x8] +; OUTLINE_ATOMICS-NEXT: ret store atomic i64 %val, i64* @var64 release, align 8 ; CHECK-NOT: dmb ; CHECK: adrp [[HIADDR:x[0-9]+]], var64 diff --git a/llvm/test/CodeGen/AArch64/cmpxchg-O0.ll b/llvm/test/CodeGen/AArch64/cmpxchg-O0.ll index 43e36dd..def4159 100644 --- a/llvm/test/CodeGen/AArch64/cmpxchg-O0.ll +++ b/llvm/test/CodeGen/AArch64/cmpxchg-O0.ll @@ -1,6 +1,8 @@ ; RUN: llc -verify-machineinstrs -mtriple=aarch64-linux-gnu -O0 -fast-isel=0 -global-isel=false %s -o - | FileCheck -enable-var-scope %s +; RUN: llc -verify-machineinstrs -mtriple=aarch64-linux-gnu -O0 -fast-isel=0 -global-isel=false -mattr=+outline-atomics %s -o - | FileCheck -enable-var-scope %s --check-prefix=OUTLINE-ATOMICS define { i8, i1 } @test_cmpxchg_8(i8* %addr, i8 %desired, i8 %new) nounwind { +; OUTLINE-ATOMICS: bl __aarch64_cas1_acq_rel ; CHECK-LABEL: test_cmpxchg_8: ; CHECK: mov [[ADDR:x[0-9]+]], x0 ; CHECK: [[RETRY:.LBB[0-9]+_[0-9]+]]: @@ -17,6 +19,7 @@ define { i8, i1 } @test_cmpxchg_8(i8* %addr, i8 %desired, i8 %new) nounwind { } define { i16, i1 } @test_cmpxchg_16(i16* %addr, i16 %desired, i16 %new) nounwind { +; OUTLINE-ATOMICS: bl __aarch64_cas2_acq_rel ; CHECK-LABEL: test_cmpxchg_16: ; CHECK: mov [[ADDR:x[0-9]+]], x0 ; CHECK: [[RETRY:.LBB[0-9]+_[0-9]+]]: @@ -33,6 +36,7 @@ define { i16, i1 } @test_cmpxchg_16(i16* %addr, i16 %desired, i16 %new) nounwind } define { i32, i1 } @test_cmpxchg_32(i32* %addr, i32 %desired, i32 %new) nounwind { +; OUTLINE-ATOMICS: bl __aarch64_cas4_acq_rel ; CHECK-LABEL: test_cmpxchg_32: ; CHECK: mov [[ADDR:x[0-9]+]], x0 ; CHECK: [[RETRY:.LBB[0-9]+_[0-9]+]]: @@ -49,6 +53,7 @@ define { i32, i1 } @test_cmpxchg_32(i32* %addr, i32 %desired, i32 %new) nounwind } define { i64, i1 } @test_cmpxchg_64(i64* %addr, i64 %desired, i64 %new) nounwind { +; OUTLINE-ATOMICS: bl __aarch64_cas8_acq_rel ; CHECK-LABEL: test_cmpxchg_64: ; CHECK: mov [[ADDR:x[0-9]+]], x0 ; CHECK: [[RETRY:.LBB[0-9]+_[0-9]+]]: @@ -65,6 +70,7 @@ define { i64, i1 } @test_cmpxchg_64(i64* %addr, i64 %desired, i64 %new) nounwind } define { i128, i1 } @test_cmpxchg_128(i128* %addr, i128 %desired, i128 %new) nounwind { +; OUTLINE-ATOMICS: bl __aarch64_cas16_acq_rel ; CHECK-LABEL: test_cmpxchg_128: ; CHECK: mov [[ADDR:x[0-9]+]], x0 ; CHECK: [[RETRY:.LBB[0-9]+_[0-9]+]]: @@ -86,6 +92,7 @@ define { i128, i1 } @test_cmpxchg_128(i128* %addr, i128 %desired, i128 %new) nou ; was false. @var128 = global i128 0 define {i128, i1} @test_cmpxchg_128_unsplit(i128* %addr) { +; OUTLINE-ATOMICS: bl __aarch64_cas16_acq_rel ; CHECK-LABEL: test_cmpxchg_128_unsplit: ; CHECK: mov [[ADDR:x[0-9]+]], x0 ; CHECK: add x[[VAR128:[0-9]+]], {{x[0-9]+}}, :lo12:var128 diff --git a/llvm/test/CodeGen/AArch64/cmpxchg-idioms.ll b/llvm/test/CodeGen/AArch64/cmpxchg-idioms.ll index 3bafd8c..2b356c5 100644 --- a/llvm/test/CodeGen/AArch64/cmpxchg-idioms.ll +++ b/llvm/test/CodeGen/AArch64/cmpxchg-idioms.ll @@ -1,6 +1,8 @@ ; RUN: llc -mtriple=aarch64-apple-ios7.0 -o - %s | FileCheck %s +; RUN: llc -mtriple=aarch64-apple-ios7.0 -mattr=+outline-atomics -o - %s | FileCheck %s --check-prefix=OUTLINE-ATOMICS define i32 @test_return(i32* %p, i32 %oldval, i32 %newval) { +; OUTLINE-ATOMICS: bl ___aarch64_cas4_acq_rel ; CHECK-LABEL: test_return: ; CHECK: [[LOOP:LBB[0-9]+_[0-9]+]]: @@ -27,6 +29,7 @@ define i32 @test_return(i32* %p, i32 %oldval, i32 %newval) { } define i1 @test_return_bool(i8* %value, i8 %oldValue, i8 %newValue) { +; OUTLINE-ATOMICS: bl ___aarch64_cas1_acq_rel ; CHECK-LABEL: test_return_bool: ; CHECK: [[LOOP:LBB[0-9]+_[0-9]+]]: @@ -55,6 +58,7 @@ define i1 @test_return_bool(i8* %value, i8 %oldValue, i8 %newValue) { } define void @test_conditional(i32* %p, i32 %oldval, i32 %newval) { +; OUTLINE-ATOMICS: bl ___aarch64_cas4_acq_rel ; CHECK-LABEL: test_conditional: ; CHECK: [[LOOP:LBB[0-9]+_[0-9]+]]: @@ -92,6 +96,7 @@ declare void @bar() declare void @baz() define i1 @test_conditional2(i32 %a, i32 %b, i32* %c) { +; OUTLINE-ATOMICS: bl ___aarch64_cas4_acq_rel ; CHECK-LABEL: test_conditional2: ; CHECK: [[LOOP:LBB[0-9]+_[0-9]+]]: ; CHECK: ldaxr [[LOADED:w[0-9]+]], [x19] diff --git a/llvm/test/CodeGen/AArch64/cmpxchg-lse-even-regs.ll b/llvm/test/CodeGen/AArch64/cmpxchg-lse-even-regs.ll index 9c6d8cc..ab083b3 100644 --- a/llvm/test/CodeGen/AArch64/cmpxchg-lse-even-regs.ll +++ b/llvm/test/CodeGen/AArch64/cmpxchg-lse-even-regs.ll @@ -1,4 +1,5 @@ ; RUN: llc -mtriple arm64-apple-ios -mattr=+lse %s -o - | FileCheck %s +; RUN: llc -mtriple arm64-apple-ios -mattr=+lse -mattr=+outline-atomics %s -o - | FileCheck %s ; Only "even,even+1" pairs are valid for CASP instructions. Make sure LLVM ; doesn't allocate odd ones and that it can copy them around properly. N.b. we diff --git a/llvm/test/Transforms/AtomicExpand/AArch64/expand-atomicrmw-xchg-fp.ll b/llvm/test/Transforms/AtomicExpand/AArch64/expand-atomicrmw-xchg-fp.ll index d3d36eb..ece757f 100644 --- a/llvm/test/Transforms/AtomicExpand/AArch64/expand-atomicrmw-xchg-fp.ll +++ b/llvm/test/Transforms/AtomicExpand/AArch64/expand-atomicrmw-xchg-fp.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt -S -mtriple=aarch64-- -atomic-expand %s | FileCheck %s +; RUN: opt -S -mtriple=aarch64-- -mattr=+outline-atomics -atomic-expand %s | FileCheck %s --check-prefix=OUTLINE-ATOMICS define void @atomic_swap_f16(half* %ptr, half %val) nounwind { ; CHECK-LABEL: @atomic_swap_f16( @@ -16,6 +17,10 @@ define void @atomic_swap_f16(half* %ptr, half %val) nounwind { ; CHECK: atomicrmw.end: ; CHECK-NEXT: ret void ; +; OUTLINE-ATOMICS-LABEL: @atomic_swap_f16( +; OUTLINE-ATOMICS-NEXT: [[T1:%.*]] = atomicrmw xchg half* [[PTR:%.*]], half [[VAL:%.*]] acquire +; OUTLINE-ATOMICS-NEXT: ret void +; %t1 = atomicrmw xchg half* %ptr, half %val acquire ret void } @@ -35,6 +40,10 @@ define void @atomic_swap_f32(float* %ptr, float %val) nounwind { ; CHECK: atomicrmw.end: ; CHECK-NEXT: ret void ; +; OUTLINE-ATOMICS-LABEL: @atomic_swap_f32( +; OUTLINE-ATOMICS-NEXT: [[T1:%.*]] = atomicrmw xchg float* [[PTR:%.*]], float [[VAL:%.*]] acquire +; OUTLINE-ATOMICS-NEXT: ret void +; %t1 = atomicrmw xchg float* %ptr, float %val acquire ret void } @@ -52,6 +61,10 @@ define void @atomic_swap_f64(double* %ptr, double %val) nounwind { ; CHECK: atomicrmw.end: ; CHECK-NEXT: ret void ; +; OUTLINE-ATOMICS-LABEL: @atomic_swap_f64( +; OUTLINE-ATOMICS-NEXT: [[T1:%.*]] = atomicrmw xchg double* [[PTR:%.*]], double [[VAL:%.*]] acquire +; OUTLINE-ATOMICS-NEXT: ret void +; %t1 = atomicrmw xchg double* %ptr, double %val acquire ret void } -- 2.7.4