--- /dev/null
+//===- X86AvoidStoreForwardingBlockis.cpp - Avoid HW Store Forward Block --===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// If a load follows a store and reloads data that the store has written to
+// memory, Intel microarchitectures can in many cases forward the data directly
+// from the store to the load, This "store forwarding" saves cycles by enabling
+// the load to directly obtain the data instead of accessing the data from
+// cache or memory.
+// A "store forward block" occurs in cases that a store cannot be forwarded to
+// the load. The most typical case of store forward block on Intel Core
+// microarchitecture that a small store cannot be forwarded to a large load.
+// The estimated penalty for a store forward block is ~13 cycles.
+//
+// This pass tries to recognize and handle cases where "store forward block"
+// is created by the compiler when lowering memcpy calls to a sequence
+// of a load and a store.
+//
+// The pass currently only handles cases where memcpy is lowered to
+// XMM/YMM registers, it tries to break the memcpy into smaller copies.
+// breaking the memcpy should be possible since there is no atomicity
+// guarantee for loads and stores to XMM/YMM.
+//
+// It could be better for performance to solve the problem by loading
+// to XMM/YMM then inserting the partial store before storing back from XMM/YMM
+// to memory, but this will result in a more conservative optimization since it
+// requires we prove that all memory accesses between the blocking store and the
+// load must alias/don't alias before we can move the store, whereas the
+// transformation done here is correct regardless to other memory accesses.
+//===----------------------------------------------------------------------===//
+
+#include "X86InstrInfo.h"
+#include "X86Subtarget.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/DebugLoc.h"
+#include "llvm/IR/Function.h"
+#include "llvm/MC/MCInstrDesc.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "x86-avoid-SFB"
+
+namespace llvm {
+void initializeX86AvoidSFBPassPass(PassRegistry &);
+} // end namespace llvm
+
+static cl::opt<bool> DisableX86AvoidStoreForwardBlocks(
+ "x86-disable-avoid-SFB", cl::Hidden,
+ cl::desc("X86: Disable Store Forwarding Blocks fixup."), cl::init(false));
+
+static cl::opt<unsigned> X86AvoidSFBInspectionLimit(
+ "x86-sfb-inspection-limit",
+ cl::desc("X86: Number of instructions backward to "
+ "inspect for store forwarding blocks."),
+ cl::init(20), cl::Hidden);
+
+namespace {
+
+using DisplacementSizeMap = std::map<int64_t, unsigned>;
+
+class X86AvoidSFBPass : public MachineFunctionPass {
+public:
+ static char ID;
+ X86AvoidSFBPass() : MachineFunctionPass(ID) {
+ initializeX86AvoidSFBPassPass(*PassRegistry::getPassRegistry());
+ }
+
+ StringRef getPassName() const override {
+ return "X86 Avoid Store Forwarding Blocks";
+ }
+
+ bool runOnMachineFunction(MachineFunction &MF) override;
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ MachineFunctionPass::getAnalysisUsage(AU);
+ AU.addRequired<AAResultsWrapperPass>();
+ }
+
+private:
+ MachineRegisterInfo *MRI;
+ const X86InstrInfo *TII;
+ const X86RegisterInfo *TRI;
+ SmallVector<std::pair<MachineInstr *, MachineInstr *>, 2>
+ BlockedLoadsStoresPairs;
+ SmallVector<MachineInstr *, 2> ForRemoval;
+ AliasAnalysis *AA;
+
+ /// \brief Returns couples of Load then Store to memory which look
+ /// like a memcpy.
+ void findPotentiallylBlockedCopies(MachineFunction &MF);
+ /// \brief Break the memcpy's load and store into smaller copies
+ /// such that each memory load that was blocked by a smaller store
+ /// would now be copied separately.
+ void breakBlockedCopies(MachineInstr *LoadInst, MachineInstr *StoreInst,
+ const DisplacementSizeMap &BlockingStoresDispSizeMap);
+ /// \brief Break a copy of size Size to smaller copies.
+ void buildCopies(int Size, MachineInstr *LoadInst, int64_t LdDispImm,
+ MachineInstr *StoreInst, int64_t StDispImm,
+ int64_t LMMOffset, int64_t SMMOffset);
+
+ void buildCopy(MachineInstr *LoadInst, unsigned NLoadOpcode, int64_t LoadDisp,
+ MachineInstr *StoreInst, unsigned NStoreOpcode,
+ int64_t StoreDisp, unsigned Size, int64_t LMMOffset,
+ int64_t SMMOffset);
+
+ bool alias(const MachineMemOperand &Op1, const MachineMemOperand &Op2) const;
+
+ unsigned getRegSizeInBytes(MachineInstr *Inst);
+};
+
+} // end anonymous namespace
+
+char X86AvoidSFBPass::ID = 0;
+
+INITIALIZE_PASS_BEGIN(X86AvoidSFBPass, DEBUG_TYPE, "Machine code sinking",
+ false, false)
+INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
+INITIALIZE_PASS_END(X86AvoidSFBPass, DEBUG_TYPE, "Machine code sinking", false,
+ false)
+
+FunctionPass *llvm::createX86AvoidStoreForwardingBlocks() {
+ return new X86AvoidSFBPass();
+}
+
+static bool isXMMLoadOpcode(unsigned Opcode) {
+ return Opcode == X86::MOVUPSrm || Opcode == X86::MOVAPSrm ||
+ Opcode == X86::VMOVUPSrm || Opcode == X86::VMOVAPSrm ||
+ Opcode == X86::VMOVUPDrm || Opcode == X86::VMOVAPDrm ||
+ Opcode == X86::VMOVDQUrm || Opcode == X86::VMOVDQArm ||
+ Opcode == X86::VMOVUPSZ128rm || Opcode == X86::VMOVAPSZ128rm ||
+ Opcode == X86::VMOVUPDZ128rm || Opcode == X86::VMOVAPDZ128rm ||
+ Opcode == X86::VMOVDQU64Z128rm || Opcode == X86::VMOVDQA64Z128rm ||
+ Opcode == X86::VMOVDQU32Z128rm || Opcode == X86::VMOVDQA32Z128rm;
+}
+static bool isYMMLoadOpcode(unsigned Opcode) {
+ return Opcode == X86::VMOVUPSYrm || Opcode == X86::VMOVAPSYrm ||
+ Opcode == X86::VMOVUPDYrm || Opcode == X86::VMOVAPDYrm ||
+ Opcode == X86::VMOVDQUYrm || Opcode == X86::VMOVDQAYrm ||
+ Opcode == X86::VMOVUPSZ256rm || Opcode == X86::VMOVAPSZ256rm ||
+ Opcode == X86::VMOVUPDZ256rm || Opcode == X86::VMOVAPDZ256rm ||
+ Opcode == X86::VMOVDQU64Z256rm || Opcode == X86::VMOVDQA64Z256rm ||
+ Opcode == X86::VMOVDQU32Z256rm || Opcode == X86::VMOVDQA32Z256rm;
+}
+
+static bool isPotentialBlockedMemCpyLd(unsigned Opcode) {
+ return isXMMLoadOpcode(Opcode) || isYMMLoadOpcode(Opcode);
+}
+
+static bool isPotentialBlockedMemCpyPair(int LdOpcode, int StOpcode) {
+ switch (LdOpcode) {
+ case X86::MOVUPSrm:
+ case X86::MOVAPSrm:
+ return StOpcode == X86::MOVUPSmr || StOpcode == X86::MOVAPSmr;
+ case X86::VMOVUPSrm:
+ case X86::VMOVAPSrm:
+ return StOpcode == X86::VMOVUPSmr || StOpcode == X86::VMOVAPSmr;
+ case X86::VMOVUPDrm:
+ case X86::VMOVAPDrm:
+ return StOpcode == X86::VMOVUPDmr || StOpcode == X86::VMOVAPDmr;
+ case X86::VMOVDQUrm:
+ case X86::VMOVDQArm:
+ return StOpcode == X86::VMOVDQUmr || StOpcode == X86::VMOVDQAmr;
+ case X86::VMOVUPSZ128rm:
+ case X86::VMOVAPSZ128rm:
+ return StOpcode == X86::VMOVUPSZ128mr || StOpcode == X86::VMOVAPSZ128mr;
+ case X86::VMOVUPDZ128rm:
+ case X86::VMOVAPDZ128rm:
+ return StOpcode == X86::VMOVUPDZ128mr || StOpcode == X86::VMOVAPDZ128mr;
+ case X86::VMOVUPSYrm:
+ case X86::VMOVAPSYrm:
+ return StOpcode == X86::VMOVUPSYmr || StOpcode == X86::VMOVAPSYmr;
+ case X86::VMOVUPDYrm:
+ case X86::VMOVAPDYrm:
+ return StOpcode == X86::VMOVUPDYmr || StOpcode == X86::VMOVAPDYmr;
+ case X86::VMOVDQUYrm:
+ case X86::VMOVDQAYrm:
+ return StOpcode == X86::VMOVDQUYmr || StOpcode == X86::VMOVDQAYmr;
+ case X86::VMOVUPSZ256rm:
+ case X86::VMOVAPSZ256rm:
+ return StOpcode == X86::VMOVUPSZ256mr || StOpcode == X86::VMOVAPSZ256mr;
+ case X86::VMOVUPDZ256rm:
+ case X86::VMOVAPDZ256rm:
+ return StOpcode == X86::VMOVUPDZ256mr || StOpcode == X86::VMOVAPDZ256mr;
+ case X86::VMOVDQU64Z128rm:
+ case X86::VMOVDQA64Z128rm:
+ return StOpcode == X86::VMOVDQU64Z128mr || StOpcode == X86::VMOVDQA64Z128mr;
+ case X86::VMOVDQU32Z128rm:
+ case X86::VMOVDQA32Z128rm:
+ return StOpcode == X86::VMOVDQU32Z128mr || StOpcode == X86::VMOVDQA32Z128mr;
+ case X86::VMOVDQU64Z256rm:
+ case X86::VMOVDQA64Z256rm:
+ return StOpcode == X86::VMOVDQU64Z256mr || StOpcode == X86::VMOVDQA64Z256mr;
+ case X86::VMOVDQU32Z256rm:
+ case X86::VMOVDQA32Z256rm:
+ return StOpcode == X86::VMOVDQU32Z256mr || StOpcode == X86::VMOVDQA32Z256mr;
+ default:
+ return false;
+ }
+}
+
+static bool isPotentialBlockingStoreInst(int Opcode, int LoadOpcode) {
+ bool PBlock = false;
+ PBlock |= Opcode == X86::MOV64mr || Opcode == X86::MOV64mi32 ||
+ Opcode == X86::MOV32mr || Opcode == X86::MOV32mi ||
+ Opcode == X86::MOV16mr || Opcode == X86::MOV16mi ||
+ Opcode == X86::MOV8mr || Opcode == X86::MOV8mi;
+ if (isYMMLoadOpcode(LoadOpcode))
+ PBlock |= Opcode == X86::VMOVUPSmr || Opcode == X86::VMOVAPSmr ||
+ Opcode == X86::VMOVUPDmr || Opcode == X86::VMOVAPDmr ||
+ Opcode == X86::VMOVDQUmr || Opcode == X86::VMOVDQAmr ||
+ Opcode == X86::VMOVUPSZ128mr || Opcode == X86::VMOVAPSZ128mr ||
+ Opcode == X86::VMOVUPDZ128mr || Opcode == X86::VMOVAPDZ128mr ||
+ Opcode == X86::VMOVDQU64Z128mr ||
+ Opcode == X86::VMOVDQA64Z128mr ||
+ Opcode == X86::VMOVDQU32Z128mr || Opcode == X86::VMOVDQA32Z128mr;
+ return PBlock;
+}
+
+static const int MOV128SZ = 16;
+static const int MOV64SZ = 8;
+static const int MOV32SZ = 4;
+static const int MOV16SZ = 2;
+static const int MOV8SZ = 1;
+
+static unsigned getYMMtoXMMLoadOpcode(unsigned LoadOpcode) {
+ switch (LoadOpcode) {
+ case X86::VMOVUPSYrm:
+ case X86::VMOVAPSYrm:
+ return X86::VMOVUPSrm;
+ case X86::VMOVUPDYrm:
+ case X86::VMOVAPDYrm:
+ return X86::VMOVUPDrm;
+ case X86::VMOVDQUYrm:
+ case X86::VMOVDQAYrm:
+ return X86::VMOVDQUrm;
+ case X86::VMOVUPSZ256rm:
+ case X86::VMOVAPSZ256rm:
+ return X86::VMOVUPSZ128rm;
+ case X86::VMOVUPDZ256rm:
+ case X86::VMOVAPDZ256rm:
+ return X86::VMOVUPDZ128rm;
+ case X86::VMOVDQU64Z256rm:
+ case X86::VMOVDQA64Z256rm:
+ return X86::VMOVDQU64Z128rm;
+ case X86::VMOVDQU32Z256rm:
+ case X86::VMOVDQA32Z256rm:
+ return X86::VMOVDQU32Z128rm;
+ default:
+ llvm_unreachable("Unexpected Load Instruction Opcode");
+ }
+ return 0;
+}
+
+static unsigned getYMMtoXMMStoreOpcode(unsigned StoreOpcode) {
+ switch (StoreOpcode) {
+ case X86::VMOVUPSYmr:
+ case X86::VMOVAPSYmr:
+ return X86::VMOVUPSmr;
+ case X86::VMOVUPDYmr:
+ case X86::VMOVAPDYmr:
+ return X86::VMOVUPDmr;
+ case X86::VMOVDQUYmr:
+ case X86::VMOVDQAYmr:
+ return X86::VMOVDQUmr;
+ case X86::VMOVUPSZ256mr:
+ case X86::VMOVAPSZ256mr:
+ return X86::VMOVUPSZ128mr;
+ case X86::VMOVUPDZ256mr:
+ case X86::VMOVAPDZ256mr:
+ return X86::VMOVUPDZ128mr;
+ case X86::VMOVDQU64Z256mr:
+ case X86::VMOVDQA64Z256mr:
+ return X86::VMOVDQU64Z128mr;
+ case X86::VMOVDQU32Z256mr:
+ case X86::VMOVDQA32Z256mr:
+ return X86::VMOVDQU32Z128mr;
+ default:
+ llvm_unreachable("Unexpected Load Instruction Opcode");
+ }
+ return 0;
+}
+
+static int getAddrOffset(MachineInstr *MI) {
+ const MCInstrDesc &Descl = MI->getDesc();
+ int AddrOffset = X86II::getMemoryOperandNo(Descl.TSFlags);
+ assert(AddrOffset != -1 && "Expected Memory Operand");
+ AddrOffset += X86II::getOperandBias(Descl);
+ return AddrOffset;
+}
+
+static MachineOperand &getBaseOperand(MachineInstr *MI) {
+ int AddrOffset = getAddrOffset(MI);
+ return MI->getOperand(AddrOffset + X86::AddrBaseReg);
+}
+
+static MachineOperand &getDispOperand(MachineInstr *MI) {
+ int AddrOffset = getAddrOffset(MI);
+ return MI->getOperand(AddrOffset + X86::AddrDisp);
+}
+
+// Relevant addressing modes contain only base register and immediate
+// displacement or frameindex and immediate displacement.
+// TODO: Consider expanding to other addressing modes in the future
+static bool isRelevantAddressingMode(MachineInstr *MI) {
+ int AddrOffset = getAddrOffset(MI);
+ MachineOperand &Base = getBaseOperand(MI);
+ MachineOperand &Disp = getDispOperand(MI);
+ MachineOperand &Scale = MI->getOperand(AddrOffset + X86::AddrScaleAmt);
+ MachineOperand &Index = MI->getOperand(AddrOffset + X86::AddrIndexReg);
+ MachineOperand &Segment = MI->getOperand(AddrOffset + X86::AddrSegmentReg);
+
+ if (!((Base.isReg() && Base.getReg() != X86::NoRegister) || Base.isFI()))
+ return false;
+ if (!Disp.isImm())
+ return false;
+ if (Scale.getImm() != 1)
+ return false;
+ if (!(Index.isReg() && Index.getReg() == X86::NoRegister))
+ return false;
+ if (!(Segment.isReg() && Segment.getReg() == X86::NoRegister))
+ return false;
+ return true;
+}
+
+// Collect potentially blocking stores.
+// Limit the number of instructions backwards we want to inspect
+// since the effect of store block won't be visible if the store
+// and load instructions have enough instructions in between to
+// keep the core busy.
+static SmallVector<MachineInstr *, 2>
+findPotentialBlockers(MachineInstr *LoadInst) {
+ SmallVector<MachineInstr *, 2> PotentialBlockers;
+ unsigned BlockCount = 0;
+ const unsigned InspectionLimit = X86AvoidSFBInspectionLimit;
+ for (auto PBInst = std::next(MachineBasicBlock::reverse_iterator(LoadInst)),
+ E = LoadInst->getParent()->rend();
+ PBInst != E; ++PBInst) {
+ BlockCount++;
+ if (BlockCount >= InspectionLimit)
+ break;
+ MachineInstr &MI = *PBInst;
+ if (MI.getDesc().isCall())
+ return PotentialBlockers;
+ PotentialBlockers.push_back(&MI);
+ }
+ // If we didn't get to the instructions limit try predecessing blocks.
+ // Ideally we should traverse the predecessor blocks in depth with some
+ // coloring algorithm, but for now let's just look at the first order
+ // predecessors.
+ if (BlockCount < InspectionLimit) {
+ MachineBasicBlock *MBB = LoadInst->getParent();
+ int LimitLeft = InspectionLimit - BlockCount;
+ for (MachineBasicBlock::pred_iterator PB = MBB->pred_begin(),
+ PE = MBB->pred_end();
+ PB != PE; ++PB) {
+ MachineBasicBlock *PMBB = *PB;
+ int PredCount = 0;
+ for (MachineBasicBlock::reverse_iterator PBInst = PMBB->rbegin(),
+ PME = PMBB->rend();
+ PBInst != PME; ++PBInst) {
+ PredCount++;
+ if (PredCount >= LimitLeft)
+ break;
+ if (PBInst->getDesc().isCall())
+ break;
+ PotentialBlockers.push_back(&*PBInst);
+ }
+ }
+ }
+ return PotentialBlockers;
+}
+
+void X86AvoidSFBPass::buildCopy(MachineInstr *LoadInst, unsigned NLoadOpcode,
+ int64_t LoadDisp, MachineInstr *StoreInst,
+ unsigned NStoreOpcode, int64_t StoreDisp,
+ unsigned Size, int64_t LMMOffset,
+ int64_t SMMOffset) {
+ MachineOperand &LoadBase = getBaseOperand(LoadInst);
+ MachineOperand &StoreBase = getBaseOperand(StoreInst);
+ MachineBasicBlock *MBB = LoadInst->getParent();
+ MachineMemOperand *LMMO = *LoadInst->memoperands_begin();
+ MachineMemOperand *SMMO = *StoreInst->memoperands_begin();
+
+ unsigned Reg1 = MRI->createVirtualRegister(
+ TII->getRegClass(TII->get(NLoadOpcode), 0, TRI, *(MBB->getParent())));
+ BuildMI(*MBB, LoadInst, LoadInst->getDebugLoc(), TII->get(NLoadOpcode), Reg1)
+ .add(LoadBase)
+ .addImm(1)
+ .addReg(X86::NoRegister)
+ .addImm(LoadDisp)
+ .addReg(X86::NoRegister)
+ .addMemOperand(
+ MBB->getParent()->getMachineMemOperand(LMMO, LMMOffset, Size));
+ DEBUG(LoadInst->getPrevNode()->dump());
+ // If the load and store are consecutive, use the loadInst location to
+ // reduce register pressure.
+ MachineInstr *StInst = StoreInst;
+ if (StoreInst->getPrevNode() == LoadInst)
+ StInst = LoadInst;
+ BuildMI(*MBB, StInst, StInst->getDebugLoc(), TII->get(NStoreOpcode))
+ .add(StoreBase)
+ .addImm(1)
+ .addReg(X86::NoRegister)
+ .addImm(StoreDisp)
+ .addReg(X86::NoRegister)
+ .addReg(Reg1)
+ .addMemOperand(
+ MBB->getParent()->getMachineMemOperand(SMMO, SMMOffset, Size));
+ DEBUG(StInst->getPrevNode()->dump());
+}
+
+void X86AvoidSFBPass::buildCopies(int Size, MachineInstr *LoadInst,
+ int64_t LdDispImm, MachineInstr *StoreInst,
+ int64_t StDispImm, int64_t LMMOffset,
+ int64_t SMMOffset) {
+ int LdDisp = LdDispImm;
+ int StDisp = StDispImm;
+ while (Size > 0) {
+ if ((Size - MOV128SZ >= 0) && isYMMLoadOpcode(LoadInst->getOpcode())) {
+ Size = Size - MOV128SZ;
+ buildCopy(LoadInst, getYMMtoXMMLoadOpcode(LoadInst->getOpcode()), LdDisp,
+ StoreInst, getYMMtoXMMStoreOpcode(StoreInst->getOpcode()),
+ StDisp, MOV128SZ, LMMOffset, SMMOffset);
+ LdDisp += MOV128SZ;
+ StDisp += MOV128SZ;
+ LMMOffset += MOV128SZ;
+ SMMOffset += MOV128SZ;
+ continue;
+ }
+ if (Size - MOV64SZ >= 0) {
+ Size = Size - MOV64SZ;
+ buildCopy(LoadInst, X86::MOV64rm, LdDisp, StoreInst, X86::MOV64mr, StDisp,
+ MOV64SZ, LMMOffset, SMMOffset);
+ LdDisp += MOV64SZ;
+ StDisp += MOV64SZ;
+ LMMOffset += MOV64SZ;
+ SMMOffset += MOV64SZ;
+ continue;
+ }
+ if (Size - MOV32SZ >= 0) {
+ Size = Size - MOV32SZ;
+ buildCopy(LoadInst, X86::MOV32rm, LdDisp, StoreInst, X86::MOV32mr, StDisp,
+ MOV32SZ, LMMOffset, SMMOffset);
+ LdDisp += MOV32SZ;
+ StDisp += MOV32SZ;
+ LMMOffset += MOV32SZ;
+ SMMOffset += MOV32SZ;
+ continue;
+ }
+ if (Size - MOV16SZ >= 0) {
+ Size = Size - MOV16SZ;
+ buildCopy(LoadInst, X86::MOV16rm, LdDisp, StoreInst, X86::MOV16mr, StDisp,
+ MOV16SZ, LMMOffset, SMMOffset);
+ LdDisp += MOV16SZ;
+ StDisp += MOV16SZ;
+ LMMOffset += MOV16SZ;
+ SMMOffset += MOV16SZ;
+ continue;
+ }
+ if (Size - MOV8SZ >= 0) {
+ Size = Size - MOV8SZ;
+ buildCopy(LoadInst, X86::MOV8rm, LdDisp, StoreInst, X86::MOV8mr, StDisp,
+ MOV8SZ, LMMOffset, SMMOffset);
+ LdDisp += MOV8SZ;
+ StDisp += MOV8SZ;
+ LMMOffset += MOV8SZ;
+ SMMOffset += MOV8SZ;
+ continue;
+ }
+ }
+ assert(Size == 0 && "Wrong size division");
+}
+
+static void updateKillStatus(MachineInstr *LoadInst, MachineInstr *StoreInst) {
+ MachineOperand &LoadBase = getBaseOperand(LoadInst);
+ MachineOperand &StoreBase = getBaseOperand(StoreInst);
+ if (LoadBase.isReg()) {
+ MachineInstr *LastLoad = LoadInst->getPrevNode();
+ // If the original load and store to xmm/ymm were consecutive
+ // then the partial copies were also created in
+ // a consecutive order to reduce register pressure,
+ // and the location of the last load is before the last store.
+ if (StoreInst->getPrevNode() == LoadInst)
+ LastLoad = LoadInst->getPrevNode()->getPrevNode();
+ getBaseOperand(LastLoad).setIsKill(LoadBase.isKill());
+ }
+ if (StoreBase.isReg()) {
+ MachineInstr *StInst = StoreInst;
+ if (StoreInst->getPrevNode() == LoadInst)
+ StInst = LoadInst;
+ getBaseOperand(StInst->getPrevNode()).setIsKill(StoreBase.isKill());
+ }
+}
+
+bool X86AvoidSFBPass::alias(const MachineMemOperand &Op1,
+ const MachineMemOperand &Op2) const {
+ if (!Op1.getValue() || !Op2.getValue())
+ return true;
+
+ int64_t MinOffset = std::min(Op1.getOffset(), Op2.getOffset());
+ int64_t Overlapa = Op1.getSize() + Op1.getOffset() - MinOffset;
+ int64_t Overlapb = Op2.getSize() + Op2.getOffset() - MinOffset;
+
+ AliasResult AAResult =
+ AA->alias(MemoryLocation(Op1.getValue(), Overlapa, Op1.getAAInfo()),
+ MemoryLocation(Op2.getValue(), Overlapb, Op2.getAAInfo()));
+ return AAResult != NoAlias;
+}
+
+void X86AvoidSFBPass::findPotentiallylBlockedCopies(MachineFunction &MF) {
+ for (auto &MBB : MF)
+ for (auto &MI : MBB) {
+ if (!isPotentialBlockedMemCpyLd(MI.getOpcode()))
+ continue;
+ int DefVR = MI.getOperand(0).getReg();
+ if (!MRI->hasOneUse(DefVR))
+ continue;
+ for (auto UI = MRI->use_nodbg_begin(DefVR), UE = MRI->use_nodbg_end();
+ UI != UE;) {
+ MachineOperand &StoreMO = *UI++;
+ MachineInstr &StoreMI = *StoreMO.getParent();
+ // Skip cases where the memcpy may overlap.
+ if (StoreMI.getParent() == MI.getParent() &&
+ isPotentialBlockedMemCpyPair(MI.getOpcode(), StoreMI.getOpcode()) &&
+ isRelevantAddressingMode(&MI) &&
+ isRelevantAddressingMode(&StoreMI)) {
+ assert(MI.hasOneMemOperand() &&
+ "Expected one memory operand for load instruction");
+ assert(StoreMI.hasOneMemOperand() &&
+ "Expected one memory operand for store instruction");
+ if (!alias(**MI.memoperands_begin(), **StoreMI.memoperands_begin()))
+ BlockedLoadsStoresPairs.push_back(std::make_pair(&MI, &StoreMI));
+ }
+ }
+ }
+}
+
+unsigned X86AvoidSFBPass::getRegSizeInBytes(MachineInstr *LoadInst) {
+ auto TRC = TII->getRegClass(TII->get(LoadInst->getOpcode()), 0, TRI,
+ *LoadInst->getParent()->getParent());
+ return TRI->getRegSizeInBits(*TRC) / 8;
+}
+
+void X86AvoidSFBPass::breakBlockedCopies(
+ MachineInstr *LoadInst, MachineInstr *StoreInst,
+ const DisplacementSizeMap &BlockingStoresDispSizeMap) {
+ int64_t LdDispImm = getDispOperand(LoadInst).getImm();
+ int64_t StDispImm = getDispOperand(StoreInst).getImm();
+ int64_t LMMOffset = (*LoadInst->memoperands_begin())->getOffset();
+ int64_t SMMOffset = (*StoreInst->memoperands_begin())->getOffset();
+
+ int64_t LdDisp1 = LdDispImm;
+ int64_t LdDisp2 = 0;
+ int64_t StDisp1 = StDispImm;
+ int64_t StDisp2 = 0;
+ unsigned Size1 = 0;
+ unsigned Size2 = 0;
+ int64_t LdStDelta = StDispImm - LdDispImm;
+
+ for (auto DispSizePair : BlockingStoresDispSizeMap) {
+ LdDisp2 = DispSizePair.first;
+ StDisp2 = DispSizePair.first + LdStDelta;
+ Size2 = DispSizePair.second;
+ // Avoid copying overlapping areas.
+ if (LdDisp2 < LdDisp1) {
+ int OverlapDelta = LdDisp1 - LdDisp2;
+ LdDisp2 += OverlapDelta;
+ StDisp2 += OverlapDelta;
+ Size2 -= OverlapDelta;
+ }
+ Size1 = std::abs(std::abs(LdDisp2) - std::abs(LdDisp1));
+
+ // Build a copy for the point until the current blocking store's
+ // displacement.
+ buildCopies(Size1, LoadInst, LdDisp1, StoreInst, StDisp1, LMMOffset,
+ SMMOffset);
+ // Build a copy for the current blocking store.
+ buildCopies(Size2, LoadInst, LdDisp2, StoreInst, StDisp2, LMMOffset + Size1,
+ SMMOffset + Size1);
+ LdDisp1 = LdDisp2 + Size2;
+ StDisp1 = StDisp2 + Size2;
+ LMMOffset += Size1 + Size2;
+ SMMOffset += Size1 + Size2;
+ }
+ unsigned Size3 = (LdDispImm + getRegSizeInBytes(LoadInst)) - LdDisp1;
+ buildCopies(Size3, LoadInst, LdDisp1, StoreInst, StDisp1, LMMOffset,
+ LMMOffset);
+}
+
+static bool hasSameBaseOpValue(MachineInstr *LoadInst,
+ MachineInstr *StoreInst) {
+ MachineOperand &LoadBase = getBaseOperand(LoadInst);
+ MachineOperand &StoreBase = getBaseOperand(StoreInst);
+ if (LoadBase.isReg() != StoreBase.isReg())
+ return false;
+ if (LoadBase.isReg())
+ return LoadBase.getReg() == StoreBase.getReg();
+ return LoadBase.getIndex() == StoreBase.getIndex();
+}
+
+static bool isBlockingStore(int64_t LoadDispImm, unsigned LoadSize,
+ int64_t StoreDispImm, unsigned StoreSize) {
+ return ((StoreDispImm >= LoadDispImm) &&
+ (StoreDispImm <= LoadDispImm + (LoadSize - StoreSize)));
+}
+
+// Keep track of all stores blocking a load
+static void
+updateBlockingStoresDispSizeMap(DisplacementSizeMap &BlockingStoresDispSizeMap,
+ int64_t DispImm, unsigned Size) {
+ if (BlockingStoresDispSizeMap.count(DispImm)) {
+ // Choose the smallest blocking store starting at this displacement.
+ if (BlockingStoresDispSizeMap[DispImm] > Size)
+ BlockingStoresDispSizeMap[DispImm] = Size;
+
+ } else
+ BlockingStoresDispSizeMap[DispImm] = Size;
+}
+
+// Remove blocking stores contained in each other.
+static void
+removeRedundantBlockingStores(DisplacementSizeMap &BlockingStoresDispSizeMap) {
+ if (BlockingStoresDispSizeMap.size() <= 1)
+ return;
+
+ int64_t PrevDisp = BlockingStoresDispSizeMap.begin()->first;
+ unsigned PrevSize = BlockingStoresDispSizeMap.begin()->second;
+ SmallVector<int64_t, 2> ForRemoval;
+ for (auto DispSizePair = std::next(BlockingStoresDispSizeMap.begin());
+ DispSizePair != BlockingStoresDispSizeMap.end(); ++DispSizePair) {
+ int64_t CurrDisp = DispSizePair->first;
+ unsigned CurrSize = DispSizePair->second;
+ if (CurrDisp + CurrSize <= PrevDisp + PrevSize) {
+ ForRemoval.push_back(PrevDisp);
+ }
+ PrevDisp = CurrDisp;
+ PrevSize = CurrSize;
+ }
+ for (auto Disp : ForRemoval)
+ BlockingStoresDispSizeMap.erase(Disp);
+}
+
+bool X86AvoidSFBPass::runOnMachineFunction(MachineFunction &MF) {
+ bool Changed = false;
+
+ if (DisableX86AvoidStoreForwardBlocks || skipFunction(MF.getFunction()) ||
+ !MF.getSubtarget<X86Subtarget>().is64Bit())
+ return false;
+
+ MRI = &MF.getRegInfo();
+ assert(MRI->isSSA() && "Expected MIR to be in SSA form");
+ TII = MF.getSubtarget<X86Subtarget>().getInstrInfo();
+ TRI = MF.getSubtarget<X86Subtarget>().getRegisterInfo();
+ AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
+ DEBUG(dbgs() << "Start X86AvoidStoreForwardBlocks\n";);
+ // Look for a load then a store to XMM/YMM which look like a memcpy
+ findPotentiallylBlockedCopies(MF);
+
+ for (auto LoadStoreInstPair : BlockedLoadsStoresPairs) {
+ MachineInstr *LoadInst = LoadStoreInstPair.first;
+ int64_t LdDispImm = getDispOperand(LoadInst).getImm();
+ DisplacementSizeMap BlockingStoresDispSizeMap;
+
+ SmallVector<MachineInstr *, 2> PotentialBlockers =
+ findPotentialBlockers(LoadInst);
+ for (auto PBInst : PotentialBlockers) {
+ if (!isPotentialBlockingStoreInst(PBInst->getOpcode(),
+ LoadInst->getOpcode()) ||
+ !isRelevantAddressingMode(PBInst))
+ continue;
+ int64_t PBstDispImm = getDispOperand(PBInst).getImm();
+ assert(PBInst->hasOneMemOperand() && "Expected One Memory Operand");
+ unsigned PBstSize = (*PBInst->memoperands_begin())->getSize();
+ // This check doesn't cover all cases, but it will suffice for now.
+ // TODO: take branch probability into consideration, if the blocking
+ // store is in an unreached block, breaking the memcopy could lose
+ // performance.
+ if (hasSameBaseOpValue(LoadInst, PBInst) &&
+ isBlockingStore(LdDispImm, getRegSizeInBytes(LoadInst), PBstDispImm,
+ PBstSize))
+ updateBlockingStoresDispSizeMap(BlockingStoresDispSizeMap, PBstDispImm,
+ PBstSize);
+ }
+
+ if (BlockingStoresDispSizeMap.empty())
+ continue;
+
+ // We found a store forward block, break the memcpy's load and store
+ // into smaller copies such that each smaller store that was causing
+ // a store block would now be copied separately.
+ MachineInstr *StoreInst = LoadStoreInstPair.second;
+ DEBUG(dbgs() << "Blocked load and store instructions: \n");
+ DEBUG(LoadInst->dump());
+ DEBUG(StoreInst->dump());
+ DEBUG(dbgs() << "Replaced with:\n");
+ removeRedundantBlockingStores(BlockingStoresDispSizeMap);
+ breakBlockedCopies(LoadInst, StoreInst, BlockingStoresDispSizeMap);
+ updateKillStatus(LoadInst, StoreInst);
+ ForRemoval.push_back(LoadInst);
+ ForRemoval.push_back(StoreInst);
+ }
+ for (auto RemovedInst : ForRemoval) {
+ RemovedInst->eraseFromParent();
+ }
+ ForRemoval.clear();
+ BlockedLoadsStoresPairs.clear();
+ DEBUG(dbgs() << "End X86AvoidStoreForwardBlocks\n";);
+
+ return Changed;
+}
--- /dev/null
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s -check-prefix=CHECK
+; RUN: llc < %s -mtriple=x86_64-linux --x86-disable-avoid-SFB | FileCheck %s --check-prefix=DISABLED
+; RUN: llc < %s -mtriple=x86_64-linux -mcpu=core-avx2 | FileCheck %s -check-prefix=CHECK-AVX2
+; RUN: llc < %s -mtriple=x86_64-linux -mcpu=skx | FileCheck %s -check-prefix=CHECK-AVX512
+
+; ModuleID = '../testSFB/testOverlapBlocks.c'
+source_filename = "../testSFB/testOverlapBlocks.c"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: nounwind uwtable
+define dso_local void @test_overlap_1(i8* nocapture %A, i32 %x) local_unnamed_addr #0 {
+; CHECK-LABEL: test_overlap_1:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movl $7, -8(%rdi)
+; CHECK-NEXT: movq -16(%rdi), %rax
+; CHECK-NEXT: movq %rax, (%rdi)
+; CHECK-NEXT: movl -8(%rdi), %eax
+; CHECK-NEXT: movl %eax, 8(%rdi)
+; CHECK-NEXT: movl -4(%rdi), %eax
+; CHECK-NEXT: movl %eax, 12(%rdi)
+; CHECK-NEXT: movslq %esi, %rax
+; CHECK-NEXT: movq %rax, -9(%rdi)
+; CHECK-NEXT: movq %rax, -16(%rdi)
+; CHECK-NEXT: movb $0, -1(%rdi)
+; CHECK-NEXT: movq -16(%rdi), %rax
+; CHECK-NEXT: movq %rax, 16(%rdi)
+; CHECK-NEXT: movl -8(%rdi), %eax
+; CHECK-NEXT: movl %eax, 24(%rdi)
+; CHECK-NEXT: movzwl -4(%rdi), %eax
+; CHECK-NEXT: movw %ax, 28(%rdi)
+; CHECK-NEXT: movb -2(%rdi), %al
+; CHECK-NEXT: movb %al, 30(%rdi)
+; CHECK-NEXT: movb -1(%rdi), %al
+; CHECK-NEXT: movb %al, 31(%rdi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_overlap_1:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: movl $7, -8(%rdi)
+; DISABLED-NEXT: movups -16(%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rdi)
+; DISABLED-NEXT: movslq %esi, %rax
+; DISABLED-NEXT: movq %rax, -9(%rdi)
+; DISABLED-NEXT: movq %rax, -16(%rdi)
+; DISABLED-NEXT: movb $0, -1(%rdi)
+; DISABLED-NEXT: movups -16(%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, 16(%rdi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_overlap_1:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: movl $7, -8(%rdi)
+; CHECK-AVX2-NEXT: movq -16(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, (%rdi)
+; CHECK-AVX2-NEXT: movl -8(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 8(%rdi)
+; CHECK-AVX2-NEXT: movl -4(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 12(%rdi)
+; CHECK-AVX2-NEXT: movslq %esi, %rax
+; CHECK-AVX2-NEXT: movq %rax, -9(%rdi)
+; CHECK-AVX2-NEXT: movq %rax, -16(%rdi)
+; CHECK-AVX2-NEXT: movb $0, -1(%rdi)
+; CHECK-AVX2-NEXT: movq -16(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, 16(%rdi)
+; CHECK-AVX2-NEXT: movl -8(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 24(%rdi)
+; CHECK-AVX2-NEXT: movzwl -4(%rdi), %eax
+; CHECK-AVX2-NEXT: movw %ax, 28(%rdi)
+; CHECK-AVX2-NEXT: movb -2(%rdi), %al
+; CHECK-AVX2-NEXT: movb %al, 30(%rdi)
+; CHECK-AVX2-NEXT: movb -1(%rdi), %al
+; CHECK-AVX2-NEXT: movb %al, 31(%rdi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_overlap_1:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: movl $7, -8(%rdi)
+; CHECK-AVX512-NEXT: movq -16(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, (%rdi)
+; CHECK-AVX512-NEXT: movl -8(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 8(%rdi)
+; CHECK-AVX512-NEXT: movl -4(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 12(%rdi)
+; CHECK-AVX512-NEXT: movslq %esi, %rax
+; CHECK-AVX512-NEXT: movq %rax, -9(%rdi)
+; CHECK-AVX512-NEXT: movq %rax, -16(%rdi)
+; CHECK-AVX512-NEXT: movb $0, -1(%rdi)
+; CHECK-AVX512-NEXT: movq -16(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, 16(%rdi)
+; CHECK-AVX512-NEXT: movl -8(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 24(%rdi)
+; CHECK-AVX512-NEXT: movzwl -4(%rdi), %eax
+; CHECK-AVX512-NEXT: movw %ax, 28(%rdi)
+; CHECK-AVX512-NEXT: movb -2(%rdi), %al
+; CHECK-AVX512-NEXT: movb %al, 30(%rdi)
+; CHECK-AVX512-NEXT: movb -1(%rdi), %al
+; CHECK-AVX512-NEXT: movb %al, 31(%rdi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %add.ptr = getelementptr inbounds i8, i8* %A, i64 -16
+ %add.ptr1 = getelementptr inbounds i8, i8* %A, i64 -8
+ %0 = bitcast i8* %add.ptr1 to i32*
+ store i32 7, i32* %0, align 4
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %A, i8* nonnull align 4 %add.ptr, i64 16, i1 false)
+ %conv = sext i32 %x to i64
+ %add.ptr2 = getelementptr inbounds i8, i8* %A, i64 -9
+ %1 = bitcast i8* %add.ptr2 to i64*
+ store i64 %conv, i64* %1, align 8
+ %2 = bitcast i8* %add.ptr to i64*
+ store i64 %conv, i64* %2, align 8
+ %add.ptr5 = getelementptr inbounds i8, i8* %A, i64 -1
+ store i8 0, i8* %add.ptr5, align 1
+ %add.ptr6 = getelementptr inbounds i8, i8* %A, i64 16
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 %add.ptr6, i8* nonnull align 4 %add.ptr, i64 16, i1 false)
+ ret void
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1) #1
+
+; Function Attrs: nounwind uwtable
+define dso_local void @test_overlap_2(i8* nocapture %A, i32 %x) local_unnamed_addr #0 {
+; CHECK-LABEL: test_overlap_2:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movslq %esi, %rax
+; CHECK-NEXT: movq %rax, -16(%rdi)
+; CHECK-NEXT: movq -16(%rdi), %rcx
+; CHECK-NEXT: movq %rcx, (%rdi)
+; CHECK-NEXT: movq -8(%rdi), %rcx
+; CHECK-NEXT: movq %rcx, 8(%rdi)
+; CHECK-NEXT: movq %rax, -8(%rdi)
+; CHECK-NEXT: movl $7, -12(%rdi)
+; CHECK-NEXT: movl -16(%rdi), %eax
+; CHECK-NEXT: movl %eax, 16(%rdi)
+; CHECK-NEXT: movl -12(%rdi), %eax
+; CHECK-NEXT: movl %eax, 20(%rdi)
+; CHECK-NEXT: movq -8(%rdi), %rax
+; CHECK-NEXT: movq %rax, 24(%rdi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_overlap_2:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: movslq %esi, %rax
+; DISABLED-NEXT: movq %rax, -16(%rdi)
+; DISABLED-NEXT: movups -16(%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rdi)
+; DISABLED-NEXT: movq %rax, -8(%rdi)
+; DISABLED-NEXT: movl $7, -12(%rdi)
+; DISABLED-NEXT: movups -16(%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, 16(%rdi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_overlap_2:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: movslq %esi, %rax
+; CHECK-AVX2-NEXT: movq %rax, -16(%rdi)
+; CHECK-AVX2-NEXT: movq -16(%rdi), %rcx
+; CHECK-AVX2-NEXT: movq %rcx, (%rdi)
+; CHECK-AVX2-NEXT: movq -8(%rdi), %rcx
+; CHECK-AVX2-NEXT: movq %rcx, 8(%rdi)
+; CHECK-AVX2-NEXT: movq %rax, -8(%rdi)
+; CHECK-AVX2-NEXT: movl $7, -12(%rdi)
+; CHECK-AVX2-NEXT: movl -16(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 16(%rdi)
+; CHECK-AVX2-NEXT: movl -12(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 20(%rdi)
+; CHECK-AVX2-NEXT: movq -8(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, 24(%rdi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_overlap_2:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: movslq %esi, %rax
+; CHECK-AVX512-NEXT: movq %rax, -16(%rdi)
+; CHECK-AVX512-NEXT: movq -16(%rdi), %rcx
+; CHECK-AVX512-NEXT: movq %rcx, (%rdi)
+; CHECK-AVX512-NEXT: movq -8(%rdi), %rcx
+; CHECK-AVX512-NEXT: movq %rcx, 8(%rdi)
+; CHECK-AVX512-NEXT: movq %rax, -8(%rdi)
+; CHECK-AVX512-NEXT: movl $7, -12(%rdi)
+; CHECK-AVX512-NEXT: movl -16(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 16(%rdi)
+; CHECK-AVX512-NEXT: movl -12(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 20(%rdi)
+; CHECK-AVX512-NEXT: movq -8(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, 24(%rdi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %add.ptr = getelementptr inbounds i8, i8* %A, i64 -16
+ %conv = sext i32 %x to i64
+ %0 = bitcast i8* %add.ptr to i64*
+ store i64 %conv, i64* %0, align 8
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %A, i8* nonnull align 4 %add.ptr, i64 16, i1 false)
+ %add.ptr3 = getelementptr inbounds i8, i8* %A, i64 -8
+ %1 = bitcast i8* %add.ptr3 to i64*
+ store i64 %conv, i64* %1, align 8
+ %add.ptr4 = getelementptr inbounds i8, i8* %A, i64 -12
+ %2 = bitcast i8* %add.ptr4 to i32*
+ store i32 7, i32* %2, align 4
+ %add.ptr5 = getelementptr inbounds i8, i8* %A, i64 16
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 %add.ptr5, i8* nonnull align 4 %add.ptr, i64 16, i1 false)
+ ret void
+}
+
+; Function Attrs: nounwind uwtable
+define dso_local void @test_overlap_3(i8* nocapture %A, i32 %x) local_unnamed_addr #0 {
+; CHECK-LABEL: test_overlap_3:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movl $7, -10(%rdi)
+; CHECK-NEXT: movl -16(%rdi), %eax
+; CHECK-NEXT: movl %eax, (%rdi)
+; CHECK-NEXT: movzwl -12(%rdi), %eax
+; CHECK-NEXT: movw %ax, 4(%rdi)
+; CHECK-NEXT: movl -10(%rdi), %eax
+; CHECK-NEXT: movl %eax, 6(%rdi)
+; CHECK-NEXT: movl -6(%rdi), %eax
+; CHECK-NEXT: movl %eax, 10(%rdi)
+; CHECK-NEXT: movzwl -2(%rdi), %eax
+; CHECK-NEXT: movw %ax, 14(%rdi)
+; CHECK-NEXT: movslq %esi, %rax
+; CHECK-NEXT: movq %rax, -9(%rdi)
+; CHECK-NEXT: movq %rax, -16(%rdi)
+; CHECK-NEXT: movb $0, -1(%rdi)
+; CHECK-NEXT: movq -16(%rdi), %rax
+; CHECK-NEXT: movq %rax, 16(%rdi)
+; CHECK-NEXT: movzwl -8(%rdi), %eax
+; CHECK-NEXT: movw %ax, 24(%rdi)
+; CHECK-NEXT: movl -6(%rdi), %eax
+; CHECK-NEXT: movl %eax, 26(%rdi)
+; CHECK-NEXT: movb -2(%rdi), %al
+; CHECK-NEXT: movb %al, 30(%rdi)
+; CHECK-NEXT: movb -1(%rdi), %al
+; CHECK-NEXT: movb %al, 31(%rdi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_overlap_3:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: movl $7, -10(%rdi)
+; DISABLED-NEXT: movups -16(%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rdi)
+; DISABLED-NEXT: movslq %esi, %rax
+; DISABLED-NEXT: movq %rax, -9(%rdi)
+; DISABLED-NEXT: movq %rax, -16(%rdi)
+; DISABLED-NEXT: movb $0, -1(%rdi)
+; DISABLED-NEXT: movups -16(%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, 16(%rdi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_overlap_3:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: movl $7, -10(%rdi)
+; CHECK-AVX2-NEXT: movl -16(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, (%rdi)
+; CHECK-AVX2-NEXT: movzwl -12(%rdi), %eax
+; CHECK-AVX2-NEXT: movw %ax, 4(%rdi)
+; CHECK-AVX2-NEXT: movl -10(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 6(%rdi)
+; CHECK-AVX2-NEXT: movl -6(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 10(%rdi)
+; CHECK-AVX2-NEXT: movzwl -2(%rdi), %eax
+; CHECK-AVX2-NEXT: movw %ax, 14(%rdi)
+; CHECK-AVX2-NEXT: movslq %esi, %rax
+; CHECK-AVX2-NEXT: movq %rax, -9(%rdi)
+; CHECK-AVX2-NEXT: movq %rax, -16(%rdi)
+; CHECK-AVX2-NEXT: movb $0, -1(%rdi)
+; CHECK-AVX2-NEXT: movq -16(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, 16(%rdi)
+; CHECK-AVX2-NEXT: movzwl -8(%rdi), %eax
+; CHECK-AVX2-NEXT: movw %ax, 24(%rdi)
+; CHECK-AVX2-NEXT: movl -6(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 26(%rdi)
+; CHECK-AVX2-NEXT: movb -2(%rdi), %al
+; CHECK-AVX2-NEXT: movb %al, 30(%rdi)
+; CHECK-AVX2-NEXT: movb -1(%rdi), %al
+; CHECK-AVX2-NEXT: movb %al, 31(%rdi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_overlap_3:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: movl $7, -10(%rdi)
+; CHECK-AVX512-NEXT: movl -16(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, (%rdi)
+; CHECK-AVX512-NEXT: movzwl -12(%rdi), %eax
+; CHECK-AVX512-NEXT: movw %ax, 4(%rdi)
+; CHECK-AVX512-NEXT: movl -10(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 6(%rdi)
+; CHECK-AVX512-NEXT: movl -6(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 10(%rdi)
+; CHECK-AVX512-NEXT: movzwl -2(%rdi), %eax
+; CHECK-AVX512-NEXT: movw %ax, 14(%rdi)
+; CHECK-AVX512-NEXT: movslq %esi, %rax
+; CHECK-AVX512-NEXT: movq %rax, -9(%rdi)
+; CHECK-AVX512-NEXT: movq %rax, -16(%rdi)
+; CHECK-AVX512-NEXT: movb $0, -1(%rdi)
+; CHECK-AVX512-NEXT: movq -16(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, 16(%rdi)
+; CHECK-AVX512-NEXT: movzwl -8(%rdi), %eax
+; CHECK-AVX512-NEXT: movw %ax, 24(%rdi)
+; CHECK-AVX512-NEXT: movl -6(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 26(%rdi)
+; CHECK-AVX512-NEXT: movb -2(%rdi), %al
+; CHECK-AVX512-NEXT: movb %al, 30(%rdi)
+; CHECK-AVX512-NEXT: movb -1(%rdi), %al
+; CHECK-AVX512-NEXT: movb %al, 31(%rdi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %add.ptr = getelementptr inbounds i8, i8* %A, i64 -16
+ %add.ptr1 = getelementptr inbounds i8, i8* %A, i64 -10
+ %0 = bitcast i8* %add.ptr1 to i32*
+ store i32 7, i32* %0, align 4
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %A, i8* nonnull align 4 %add.ptr, i64 16, i1 false)
+ %conv = sext i32 %x to i64
+ %add.ptr2 = getelementptr inbounds i8, i8* %A, i64 -9
+ %1 = bitcast i8* %add.ptr2 to i64*
+ store i64 %conv, i64* %1, align 8
+ %2 = bitcast i8* %add.ptr to i64*
+ store i64 %conv, i64* %2, align 8
+ %add.ptr5 = getelementptr inbounds i8, i8* %A, i64 -1
+ store i8 0, i8* %add.ptr5, align 1
+ %add.ptr6 = getelementptr inbounds i8, i8* %A, i64 16
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 %add.ptr6, i8* nonnull align 4 %add.ptr, i64 16, i1 false)
+ ret void
+}
+
+; Function Attrs: nounwind uwtable
+define dso_local void @test_overlap_4(i8* nocapture %A, i32 %x) local_unnamed_addr #0 {
+; CHECK-LABEL: test_overlap_4:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movups -16(%rdi), %xmm0
+; CHECK-NEXT: movups %xmm0, (%rdi)
+; CHECK-NEXT: movslq %esi, %rax
+; CHECK-NEXT: movq %rax, -8(%rdi)
+; CHECK-NEXT: movl %eax, -16(%rdi)
+; CHECK-NEXT: movl $0, -11(%rdi)
+; CHECK-NEXT: movl -16(%rdi), %eax
+; CHECK-NEXT: movl %eax, 16(%rdi)
+; CHECK-NEXT: movb -12(%rdi), %al
+; CHECK-NEXT: movb %al, 20(%rdi)
+; CHECK-NEXT: movl -11(%rdi), %eax
+; CHECK-NEXT: movl %eax, 21(%rdi)
+; CHECK-NEXT: movl -7(%rdi), %eax
+; CHECK-NEXT: movl %eax, 25(%rdi)
+; CHECK-NEXT: movzwl -3(%rdi), %eax
+; CHECK-NEXT: movw %ax, 29(%rdi)
+; CHECK-NEXT: movb -1(%rdi), %al
+; CHECK-NEXT: movb %al, 31(%rdi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_overlap_4:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: movups -16(%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rdi)
+; DISABLED-NEXT: movslq %esi, %rax
+; DISABLED-NEXT: movq %rax, -8(%rdi)
+; DISABLED-NEXT: movl %eax, -16(%rdi)
+; DISABLED-NEXT: movl $0, -11(%rdi)
+; DISABLED-NEXT: movups -16(%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, 16(%rdi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_overlap_4:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: vmovups -16(%rdi), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, (%rdi)
+; CHECK-AVX2-NEXT: movslq %esi, %rax
+; CHECK-AVX2-NEXT: movq %rax, -8(%rdi)
+; CHECK-AVX2-NEXT: movl %eax, -16(%rdi)
+; CHECK-AVX2-NEXT: movl $0, -11(%rdi)
+; CHECK-AVX2-NEXT: movl -16(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 16(%rdi)
+; CHECK-AVX2-NEXT: movb -12(%rdi), %al
+; CHECK-AVX2-NEXT: movb %al, 20(%rdi)
+; CHECK-AVX2-NEXT: movl -11(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 21(%rdi)
+; CHECK-AVX2-NEXT: movl -7(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 25(%rdi)
+; CHECK-AVX2-NEXT: movzwl -3(%rdi), %eax
+; CHECK-AVX2-NEXT: movw %ax, 29(%rdi)
+; CHECK-AVX2-NEXT: movb -1(%rdi), %al
+; CHECK-AVX2-NEXT: movb %al, 31(%rdi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_overlap_4:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: vmovups -16(%rdi), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, (%rdi)
+; CHECK-AVX512-NEXT: movslq %esi, %rax
+; CHECK-AVX512-NEXT: movq %rax, -8(%rdi)
+; CHECK-AVX512-NEXT: movl %eax, -16(%rdi)
+; CHECK-AVX512-NEXT: movl $0, -11(%rdi)
+; CHECK-AVX512-NEXT: movl -16(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 16(%rdi)
+; CHECK-AVX512-NEXT: movb -12(%rdi), %al
+; CHECK-AVX512-NEXT: movb %al, 20(%rdi)
+; CHECK-AVX512-NEXT: movl -11(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 21(%rdi)
+; CHECK-AVX512-NEXT: movl -7(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 25(%rdi)
+; CHECK-AVX512-NEXT: movzwl -3(%rdi), %eax
+; CHECK-AVX512-NEXT: movw %ax, 29(%rdi)
+; CHECK-AVX512-NEXT: movb -1(%rdi), %al
+; CHECK-AVX512-NEXT: movb %al, 31(%rdi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %add.ptr = getelementptr inbounds i8, i8* %A, i64 -16
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %A, i8* nonnull align 4 %add.ptr, i64 16, i1 false)
+ %conv = sext i32 %x to i64
+ %add.ptr1 = getelementptr inbounds i8, i8* %A, i64 -8
+ %0 = bitcast i8* %add.ptr1 to i64*
+ store i64 %conv, i64* %0, align 8
+ %1 = bitcast i8* %add.ptr to i32*
+ store i32 %x, i32* %1, align 4
+ %add.ptr3 = getelementptr inbounds i8, i8* %A, i64 -11
+ %2 = bitcast i8* %add.ptr3 to i32*
+ store i32 0, i32* %2, align 4
+ %add.ptr4 = getelementptr inbounds i8, i8* %A, i64 16
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 %add.ptr4, i8* nonnull align 4 %add.ptr, i64 16, i1 false)
+ ret void
+}
+
+; Function Attrs: nounwind uwtable
+define dso_local void @test_overlap_5(i8* nocapture %A, i32 %x) local_unnamed_addr #0 {
+; CHECK-LABEL: test_overlap_5:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movups -16(%rdi), %xmm0
+; CHECK-NEXT: movups %xmm0, (%rdi)
+; CHECK-NEXT: movslq %esi, %rax
+; CHECK-NEXT: movq %rax, -16(%rdi)
+; CHECK-NEXT: movb %al, -14(%rdi)
+; CHECK-NEXT: movb $0, -11(%rdi)
+; CHECK-NEXT: movzwl -16(%rdi), %eax
+; CHECK-NEXT: movw %ax, 16(%rdi)
+; CHECK-NEXT: movb -14(%rdi), %al
+; CHECK-NEXT: movb %al, 18(%rdi)
+; CHECK-NEXT: movzwl -13(%rdi), %eax
+; CHECK-NEXT: movw %ax, 19(%rdi)
+; CHECK-NEXT: movb -11(%rdi), %al
+; CHECK-NEXT: movb %al, 21(%rdi)
+; CHECK-NEXT: movq -10(%rdi), %rax
+; CHECK-NEXT: movq %rax, 22(%rdi)
+; CHECK-NEXT: movzwl -2(%rdi), %eax
+; CHECK-NEXT: movw %ax, 30(%rdi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_overlap_5:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: movups -16(%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rdi)
+; DISABLED-NEXT: movslq %esi, %rax
+; DISABLED-NEXT: movq %rax, -16(%rdi)
+; DISABLED-NEXT: movb %al, -14(%rdi)
+; DISABLED-NEXT: movb $0, -11(%rdi)
+; DISABLED-NEXT: movups -16(%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, 16(%rdi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_overlap_5:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: vmovups -16(%rdi), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, (%rdi)
+; CHECK-AVX2-NEXT: movslq %esi, %rax
+; CHECK-AVX2-NEXT: movq %rax, -16(%rdi)
+; CHECK-AVX2-NEXT: movb %al, -14(%rdi)
+; CHECK-AVX2-NEXT: movb $0, -11(%rdi)
+; CHECK-AVX2-NEXT: movzwl -16(%rdi), %eax
+; CHECK-AVX2-NEXT: movw %ax, 16(%rdi)
+; CHECK-AVX2-NEXT: movb -14(%rdi), %al
+; CHECK-AVX2-NEXT: movb %al, 18(%rdi)
+; CHECK-AVX2-NEXT: movzwl -13(%rdi), %eax
+; CHECK-AVX2-NEXT: movw %ax, 19(%rdi)
+; CHECK-AVX2-NEXT: movb -11(%rdi), %al
+; CHECK-AVX2-NEXT: movb %al, 21(%rdi)
+; CHECK-AVX2-NEXT: movq -10(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, 22(%rdi)
+; CHECK-AVX2-NEXT: movzwl -2(%rdi), %eax
+; CHECK-AVX2-NEXT: movw %ax, 30(%rdi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_overlap_5:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: vmovups -16(%rdi), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, (%rdi)
+; CHECK-AVX512-NEXT: movslq %esi, %rax
+; CHECK-AVX512-NEXT: movq %rax, -16(%rdi)
+; CHECK-AVX512-NEXT: movb %al, -14(%rdi)
+; CHECK-AVX512-NEXT: movb $0, -11(%rdi)
+; CHECK-AVX512-NEXT: movzwl -16(%rdi), %eax
+; CHECK-AVX512-NEXT: movw %ax, 16(%rdi)
+; CHECK-AVX512-NEXT: movb -14(%rdi), %al
+; CHECK-AVX512-NEXT: movb %al, 18(%rdi)
+; CHECK-AVX512-NEXT: movzwl -13(%rdi), %eax
+; CHECK-AVX512-NEXT: movw %ax, 19(%rdi)
+; CHECK-AVX512-NEXT: movb -11(%rdi), %al
+; CHECK-AVX512-NEXT: movb %al, 21(%rdi)
+; CHECK-AVX512-NEXT: movq -10(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, 22(%rdi)
+; CHECK-AVX512-NEXT: movzwl -2(%rdi), %eax
+; CHECK-AVX512-NEXT: movw %ax, 30(%rdi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %add.ptr = getelementptr inbounds i8, i8* %A, i64 -16
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %A, i8* nonnull align 4 %add.ptr, i64 16, i1 false)
+ %conv = sext i32 %x to i64
+ %0 = bitcast i8* %add.ptr to i64*
+ store i64 %conv, i64* %0, align 8
+ %conv2 = trunc i32 %x to i8
+ %add.ptr3 = getelementptr inbounds i8, i8* %A, i64 -14
+ store i8 %conv2, i8* %add.ptr3, align 1
+ %add.ptr4 = getelementptr inbounds i8, i8* %A, i64 -11
+ store i8 0, i8* %add.ptr4, align 1
+ %add.ptr5 = getelementptr inbounds i8, i8* %A, i64 16
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 %add.ptr5, i8* nonnull align 4 %add.ptr, i64 16, i1 false)
+ ret void
+}
+
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind }
+
+
--- /dev/null
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s -check-prefix=CHECK
+; RUN: llc < %s -mtriple=x86_64-linux --x86-disable-avoid-SFB | FileCheck %s --check-prefix=DISABLED
+; RUN: llc < %s -mtriple=x86_64-linux -mcpu=core-avx2 | FileCheck %s -check-prefix=CHECK-AVX2
+; RUN: llc < %s -mtriple=x86_64-linux -mcpu=skx | FileCheck %s -check-prefix=CHECK-AVX512
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.S = type { i32, i32, i32, i32 }
+
+; Function Attrs: nounwind uwtable
+define void @test_conditional_block(%struct.S* nocapture noalias %s1 , %struct.S* nocapture noalias %s2, i32 %x, %struct.S* nocapture noalias %s3, %struct.S* nocapture noalias readonly %s4) local_unnamed_addr #0 {
+; CHECK-LABEL: test_conditional_block:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: cmpl $18, %edx
+; CHECK-NEXT: jl .LBB0_2
+; CHECK-NEXT: # %bb.1: # %if.then
+; CHECK-NEXT: movl %edx, 4(%rdi)
+; CHECK-NEXT: .LBB0_2: # %if.end
+; CHECK-NEXT: movups (%r8), %xmm0
+; CHECK-NEXT: movups %xmm0, (%rcx)
+; CHECK-NEXT: movl (%rdi), %eax
+; CHECK-NEXT: movl %eax, (%rsi)
+; CHECK-NEXT: movl 4(%rdi), %eax
+; CHECK-NEXT: movl %eax, 4(%rsi)
+; CHECK-NEXT: movq 8(%rdi), %rax
+; CHECK-NEXT: movq %rax, 8(%rsi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_conditional_block:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: cmpl $18, %edx
+; DISABLED-NEXT: jl .LBB0_2
+; DISABLED-NEXT: # %bb.1: # %if.then
+; DISABLED-NEXT: movl %edx, 4(%rdi)
+; DISABLED-NEXT: .LBB0_2: # %if.end
+; DISABLED-NEXT: movups (%r8), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rcx)
+; DISABLED-NEXT: movups (%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rsi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_conditional_block:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: cmpl $18, %edx
+; CHECK-AVX2-NEXT: jl .LBB0_2
+; CHECK-AVX2-NEXT: # %bb.1: # %if.then
+; CHECK-AVX2-NEXT: movl %edx, 4(%rdi)
+; CHECK-AVX2-NEXT: .LBB0_2: # %if.end
+; CHECK-AVX2-NEXT: vmovups (%r8), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, (%rcx)
+; CHECK-AVX2-NEXT: movl (%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, (%rsi)
+; CHECK-AVX2-NEXT: movl 4(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 4(%rsi)
+; CHECK-AVX2-NEXT: movq 8(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, 8(%rsi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_conditional_block:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: cmpl $18, %edx
+; CHECK-AVX512-NEXT: jl .LBB0_2
+; CHECK-AVX512-NEXT: # %bb.1: # %if.then
+; CHECK-AVX512-NEXT: movl %edx, 4(%rdi)
+; CHECK-AVX512-NEXT: .LBB0_2: # %if.end
+; CHECK-AVX512-NEXT: vmovups (%r8), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, (%rcx)
+; CHECK-AVX512-NEXT: movl (%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, (%rsi)
+; CHECK-AVX512-NEXT: movl 4(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 4(%rsi)
+; CHECK-AVX512-NEXT: movq 8(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, 8(%rsi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %cmp = icmp sgt i32 %x, 17
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then: ; preds = %entry
+ %b = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 1
+ store i32 %x, i32* %b, align 4
+ br label %if.end
+
+if.end: ; preds = %if.then, %entry
+ %0 = bitcast %struct.S* %s3 to i8*
+ %1 = bitcast %struct.S* %s4 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 4, i1 false)
+ %2 = bitcast %struct.S* %s2 to i8*
+ %3 = bitcast %struct.S* %s1 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 16, i32 4, i1 false)
+ ret void
+}
+
+; Function Attrs: nounwind uwtable
+define void @test_imm_store(%struct.S* nocapture noalias %s1, %struct.S* nocapture %s2, i32 %x, %struct.S* nocapture %s3) local_unnamed_addr #0 {
+; CHECK-LABEL: test_imm_store:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movl $0, (%rdi)
+; CHECK-NEXT: movl $1, (%rcx)
+; CHECK-NEXT: movl (%rdi), %eax
+; CHECK-NEXT: movl %eax, (%rsi)
+; CHECK-NEXT: movq 4(%rdi), %rax
+; CHECK-NEXT: movq %rax, 4(%rsi)
+; CHECK-NEXT: movl 12(%rdi), %eax
+; CHECK-NEXT: movl %eax, 12(%rsi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_imm_store:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: movl $0, (%rdi)
+; DISABLED-NEXT: movl $1, (%rcx)
+; DISABLED-NEXT: movups (%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rsi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_imm_store:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: movl $0, (%rdi)
+; CHECK-AVX2-NEXT: movl $1, (%rcx)
+; CHECK-AVX2-NEXT: movl (%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, (%rsi)
+; CHECK-AVX2-NEXT: movq 4(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, 4(%rsi)
+; CHECK-AVX2-NEXT: movl 12(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 12(%rsi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_imm_store:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: movl $0, (%rdi)
+; CHECK-AVX512-NEXT: movl $1, (%rcx)
+; CHECK-AVX512-NEXT: movl (%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, (%rsi)
+; CHECK-AVX512-NEXT: movq 4(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, 4(%rsi)
+; CHECK-AVX512-NEXT: movl 12(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 12(%rsi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %a = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 0
+ store i32 0, i32* %a, align 4
+ %a1 = getelementptr inbounds %struct.S, %struct.S* %s3, i64 0, i32 0
+ store i32 1, i32* %a1, align 4
+ %0 = bitcast %struct.S* %s2 to i8*
+ %1 = bitcast %struct.S* %s1 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 4, i1 false)
+ ret void
+}
+
+; Function Attrs: nounwind uwtable
+define void @test_nondirect_br(%struct.S* nocapture noalias %s1, %struct.S* nocapture %s2, i32 %x, %struct.S* nocapture %s3, %struct.S* nocapture readonly %s4, i32 %x2) local_unnamed_addr #0 {
+; CHECK-LABEL: test_nondirect_br:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: cmpl $18, %edx
+; CHECK-NEXT: jl .LBB2_2
+; CHECK-NEXT: # %bb.1: # %if.then
+; CHECK-NEXT: movl %edx, 4(%rdi)
+; CHECK-NEXT: .LBB2_2: # %if.end
+; CHECK-NEXT: cmpl $14, %r9d
+; CHECK-NEXT: jl .LBB2_4
+; CHECK-NEXT: # %bb.3: # %if.then2
+; CHECK-NEXT: movl %r9d, 12(%rdi)
+; CHECK-NEXT: .LBB2_4: # %if.end3
+; CHECK-NEXT: movups (%r8), %xmm0
+; CHECK-NEXT: movups %xmm0, (%rcx)
+; CHECK-NEXT: movq (%rdi), %rax
+; CHECK-NEXT: movq %rax, (%rsi)
+; CHECK-NEXT: movl 8(%rdi), %eax
+; CHECK-NEXT: movl %eax, 8(%rsi)
+; CHECK-NEXT: movl 12(%rdi), %eax
+; CHECK-NEXT: movl %eax, 12(%rsi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_nondirect_br:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: cmpl $18, %edx
+; DISABLED-NEXT: jl .LBB2_2
+; DISABLED-NEXT: # %bb.1: # %if.then
+; DISABLED-NEXT: movl %edx, 4(%rdi)
+; DISABLED-NEXT: .LBB2_2: # %if.end
+; DISABLED-NEXT: cmpl $14, %r9d
+; DISABLED-NEXT: jl .LBB2_4
+; DISABLED-NEXT: # %bb.3: # %if.then2
+; DISABLED-NEXT: movl %r9d, 12(%rdi)
+; DISABLED-NEXT: .LBB2_4: # %if.end3
+; DISABLED-NEXT: movups (%r8), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rcx)
+; DISABLED-NEXT: movups (%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rsi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_nondirect_br:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: cmpl $18, %edx
+; CHECK-AVX2-NEXT: jl .LBB2_2
+; CHECK-AVX2-NEXT: # %bb.1: # %if.then
+; CHECK-AVX2-NEXT: movl %edx, 4(%rdi)
+; CHECK-AVX2-NEXT: .LBB2_2: # %if.end
+; CHECK-AVX2-NEXT: cmpl $14, %r9d
+; CHECK-AVX2-NEXT: jl .LBB2_4
+; CHECK-AVX2-NEXT: # %bb.3: # %if.then2
+; CHECK-AVX2-NEXT: movl %r9d, 12(%rdi)
+; CHECK-AVX2-NEXT: .LBB2_4: # %if.end3
+; CHECK-AVX2-NEXT: vmovups (%r8), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, (%rcx)
+; CHECK-AVX2-NEXT: movq (%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, (%rsi)
+; CHECK-AVX2-NEXT: movl 8(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 8(%rsi)
+; CHECK-AVX2-NEXT: movl 12(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 12(%rsi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_nondirect_br:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: cmpl $18, %edx
+; CHECK-AVX512-NEXT: jl .LBB2_2
+; CHECK-AVX512-NEXT: # %bb.1: # %if.then
+; CHECK-AVX512-NEXT: movl %edx, 4(%rdi)
+; CHECK-AVX512-NEXT: .LBB2_2: # %if.end
+; CHECK-AVX512-NEXT: cmpl $14, %r9d
+; CHECK-AVX512-NEXT: jl .LBB2_4
+; CHECK-AVX512-NEXT: # %bb.3: # %if.then2
+; CHECK-AVX512-NEXT: movl %r9d, 12(%rdi)
+; CHECK-AVX512-NEXT: .LBB2_4: # %if.end3
+; CHECK-AVX512-NEXT: vmovups (%r8), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, (%rcx)
+; CHECK-AVX512-NEXT: movq (%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, (%rsi)
+; CHECK-AVX512-NEXT: movl 8(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 8(%rsi)
+; CHECK-AVX512-NEXT: movl 12(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 12(%rsi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %cmp = icmp sgt i32 %x, 17
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then: ; preds = %entry
+ %b = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 1
+ store i32 %x, i32* %b, align 4
+ br label %if.end
+
+if.end: ; preds = %if.then, %entry
+ %cmp1 = icmp sgt i32 %x2, 13
+ br i1 %cmp1, label %if.then2, label %if.end3
+
+if.then2: ; preds = %if.end
+ %d = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 3
+ store i32 %x2, i32* %d, align 4
+ br label %if.end3
+
+if.end3: ; preds = %if.then2, %if.end
+ %0 = bitcast %struct.S* %s3 to i8*
+ %1 = bitcast %struct.S* %s4 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 4, i1 false)
+ %2 = bitcast %struct.S* %s2 to i8*
+ %3 = bitcast %struct.S* %s1 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 16, i32 4, i1 false)
+ ret void
+}
+
+; Function Attrs: nounwind uwtable
+define void @test_2preds_block(%struct.S* nocapture noalias %s1, %struct.S* nocapture %s2, i32 %x, %struct.S* nocapture %s3, %struct.S* nocapture readonly %s4, i32 %x2) local_unnamed_addr #0 {
+; CHECK-LABEL: test_2preds_block:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movl %r9d, 12(%rdi)
+; CHECK-NEXT: cmpl $18, %edx
+; CHECK-NEXT: jl .LBB3_2
+; CHECK-NEXT: # %bb.1: # %if.then
+; CHECK-NEXT: movl %edx, 4(%rdi)
+; CHECK-NEXT: .LBB3_2: # %if.end
+; CHECK-NEXT: movups (%r8), %xmm0
+; CHECK-NEXT: movups %xmm0, (%rcx)
+; CHECK-NEXT: movl (%rdi), %eax
+; CHECK-NEXT: movl %eax, (%rsi)
+; CHECK-NEXT: movl 4(%rdi), %eax
+; CHECK-NEXT: movl %eax, 4(%rsi)
+; CHECK-NEXT: movl 8(%rdi), %eax
+; CHECK-NEXT: movl %eax, 8(%rsi)
+; CHECK-NEXT: movl 12(%rdi), %eax
+; CHECK-NEXT: movl %eax, 12(%rsi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_2preds_block:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: movl %r9d, 12(%rdi)
+; DISABLED-NEXT: cmpl $18, %edx
+; DISABLED-NEXT: jl .LBB3_2
+; DISABLED-NEXT: # %bb.1: # %if.then
+; DISABLED-NEXT: movl %edx, 4(%rdi)
+; DISABLED-NEXT: .LBB3_2: # %if.end
+; DISABLED-NEXT: movups (%r8), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rcx)
+; DISABLED-NEXT: movups (%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rsi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_2preds_block:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: movl %r9d, 12(%rdi)
+; CHECK-AVX2-NEXT: cmpl $18, %edx
+; CHECK-AVX2-NEXT: jl .LBB3_2
+; CHECK-AVX2-NEXT: # %bb.1: # %if.then
+; CHECK-AVX2-NEXT: movl %edx, 4(%rdi)
+; CHECK-AVX2-NEXT: .LBB3_2: # %if.end
+; CHECK-AVX2-NEXT: vmovups (%r8), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, (%rcx)
+; CHECK-AVX2-NEXT: movl (%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, (%rsi)
+; CHECK-AVX2-NEXT: movl 4(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 4(%rsi)
+; CHECK-AVX2-NEXT: movl 8(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 8(%rsi)
+; CHECK-AVX2-NEXT: movl 12(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 12(%rsi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_2preds_block:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: movl %r9d, 12(%rdi)
+; CHECK-AVX512-NEXT: cmpl $18, %edx
+; CHECK-AVX512-NEXT: jl .LBB3_2
+; CHECK-AVX512-NEXT: # %bb.1: # %if.then
+; CHECK-AVX512-NEXT: movl %edx, 4(%rdi)
+; CHECK-AVX512-NEXT: .LBB3_2: # %if.end
+; CHECK-AVX512-NEXT: vmovups (%r8), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, (%rcx)
+; CHECK-AVX512-NEXT: movl (%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, (%rsi)
+; CHECK-AVX512-NEXT: movl 4(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 4(%rsi)
+; CHECK-AVX512-NEXT: movl 8(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 8(%rsi)
+; CHECK-AVX512-NEXT: movl 12(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 12(%rsi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %d = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 3
+ store i32 %x2, i32* %d, align 4
+ %cmp = icmp sgt i32 %x, 17
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then: ; preds = %entry
+ %b = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 1
+ store i32 %x, i32* %b, align 4
+ br label %if.end
+
+if.end: ; preds = %if.then, %entry
+ %0 = bitcast %struct.S* %s3 to i8*
+ %1 = bitcast %struct.S* %s4 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 4, i1 false)
+ %2 = bitcast %struct.S* %s2 to i8*
+ %3 = bitcast %struct.S* %s1 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 16, i32 4, i1 false)
+ ret void
+}
+%struct.S2 = type { i64, i64 }
+
+; Function Attrs: nounwind uwtable
+define void @test_type64(%struct.S2* nocapture noalias %s1, %struct.S2* nocapture %s2, i32 %x, %struct.S2* nocapture %s3, %struct.S2* nocapture readonly %s4) local_unnamed_addr #0 {
+; CHECK-LABEL: test_type64:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: cmpl $18, %edx
+; CHECK-NEXT: jl .LBB4_2
+; CHECK-NEXT: # %bb.1: # %if.then
+; CHECK-NEXT: movslq %edx, %rax
+; CHECK-NEXT: movq %rax, 8(%rdi)
+; CHECK-NEXT: .LBB4_2: # %if.end
+; CHECK-NEXT: movups (%r8), %xmm0
+; CHECK-NEXT: movups %xmm0, (%rcx)
+; CHECK-NEXT: movq (%rdi), %rax
+; CHECK-NEXT: movq %rax, (%rsi)
+; CHECK-NEXT: movq 8(%rdi), %rax
+; CHECK-NEXT: movq %rax, 8(%rsi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_type64:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: cmpl $18, %edx
+; DISABLED-NEXT: jl .LBB4_2
+; DISABLED-NEXT: # %bb.1: # %if.then
+; DISABLED-NEXT: movslq %edx, %rax
+; DISABLED-NEXT: movq %rax, 8(%rdi)
+; DISABLED-NEXT: .LBB4_2: # %if.end
+; DISABLED-NEXT: movups (%r8), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rcx)
+; DISABLED-NEXT: movups (%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rsi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_type64:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: cmpl $18, %edx
+; CHECK-AVX2-NEXT: jl .LBB4_2
+; CHECK-AVX2-NEXT: # %bb.1: # %if.then
+; CHECK-AVX2-NEXT: movslq %edx, %rax
+; CHECK-AVX2-NEXT: movq %rax, 8(%rdi)
+; CHECK-AVX2-NEXT: .LBB4_2: # %if.end
+; CHECK-AVX2-NEXT: vmovups (%r8), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, (%rcx)
+; CHECK-AVX2-NEXT: movq (%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, (%rsi)
+; CHECK-AVX2-NEXT: movq 8(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, 8(%rsi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_type64:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: cmpl $18, %edx
+; CHECK-AVX512-NEXT: jl .LBB4_2
+; CHECK-AVX512-NEXT: # %bb.1: # %if.then
+; CHECK-AVX512-NEXT: movslq %edx, %rax
+; CHECK-AVX512-NEXT: movq %rax, 8(%rdi)
+; CHECK-AVX512-NEXT: .LBB4_2: # %if.end
+; CHECK-AVX512-NEXT: vmovups (%r8), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, (%rcx)
+; CHECK-AVX512-NEXT: movq (%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, (%rsi)
+; CHECK-AVX512-NEXT: movq 8(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, 8(%rsi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %cmp = icmp sgt i32 %x, 17
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then: ; preds = %entry
+ %conv = sext i32 %x to i64
+ %b = getelementptr inbounds %struct.S2, %struct.S2* %s1, i64 0, i32 1
+ store i64 %conv, i64* %b, align 8
+ br label %if.end
+
+if.end: ; preds = %if.then, %entry
+ %0 = bitcast %struct.S2* %s3 to i8*
+ %1 = bitcast %struct.S2* %s4 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 8, i1 false)
+ %2 = bitcast %struct.S2* %s2 to i8*
+ %3 = bitcast %struct.S2* %s1 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 16, i32 8, i1 false)
+ ret void
+}
+%struct.S3 = type { i64, i8, i8, i16, i32 }
+
+; Function Attrs: noinline nounwind uwtable
+define void @test_mixed_type(%struct.S3* nocapture noalias %s1, %struct.S3* nocapture %s2, i32 %x, %struct.S3* nocapture readnone %s3, %struct.S3* nocapture readnone %s4) local_unnamed_addr #0 {
+; CHECK-LABEL: test_mixed_type:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: cmpl $18, %edx
+; CHECK-NEXT: jl .LBB5_2
+; CHECK-NEXT: # %bb.1: # %if.then
+; CHECK-NEXT: movslq %edx, %rax
+; CHECK-NEXT: movq %rax, (%rdi)
+; CHECK-NEXT: movb %dl, 8(%rdi)
+; CHECK-NEXT: .LBB5_2: # %if.end
+; CHECK-NEXT: movq (%rdi), %rax
+; CHECK-NEXT: movq %rax, (%rsi)
+; CHECK-NEXT: movb 8(%rdi), %al
+; CHECK-NEXT: movb %al, 8(%rsi)
+; CHECK-NEXT: movl 9(%rdi), %eax
+; CHECK-NEXT: movl %eax, 9(%rsi)
+; CHECK-NEXT: movzwl 13(%rdi), %eax
+; CHECK-NEXT: movw %ax, 13(%rsi)
+; CHECK-NEXT: movb 15(%rdi), %al
+; CHECK-NEXT: movb %al, 15(%rsi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_mixed_type:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: cmpl $18, %edx
+; DISABLED-NEXT: jl .LBB5_2
+; DISABLED-NEXT: # %bb.1: # %if.then
+; DISABLED-NEXT: movslq %edx, %rax
+; DISABLED-NEXT: movq %rax, (%rdi)
+; DISABLED-NEXT: movb %dl, 8(%rdi)
+; DISABLED-NEXT: .LBB5_2: # %if.end
+; DISABLED-NEXT: movups (%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rsi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_mixed_type:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: cmpl $18, %edx
+; CHECK-AVX2-NEXT: jl .LBB5_2
+; CHECK-AVX2-NEXT: # %bb.1: # %if.then
+; CHECK-AVX2-NEXT: movslq %edx, %rax
+; CHECK-AVX2-NEXT: movq %rax, (%rdi)
+; CHECK-AVX2-NEXT: movb %dl, 8(%rdi)
+; CHECK-AVX2-NEXT: .LBB5_2: # %if.end
+; CHECK-AVX2-NEXT: movq (%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, (%rsi)
+; CHECK-AVX2-NEXT: movb 8(%rdi), %al
+; CHECK-AVX2-NEXT: movb %al, 8(%rsi)
+; CHECK-AVX2-NEXT: movl 9(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 9(%rsi)
+; CHECK-AVX2-NEXT: movzwl 13(%rdi), %eax
+; CHECK-AVX2-NEXT: movw %ax, 13(%rsi)
+; CHECK-AVX2-NEXT: movb 15(%rdi), %al
+; CHECK-AVX2-NEXT: movb %al, 15(%rsi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_mixed_type:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: cmpl $18, %edx
+; CHECK-AVX512-NEXT: jl .LBB5_2
+; CHECK-AVX512-NEXT: # %bb.1: # %if.then
+; CHECK-AVX512-NEXT: movslq %edx, %rax
+; CHECK-AVX512-NEXT: movq %rax, (%rdi)
+; CHECK-AVX512-NEXT: movb %dl, 8(%rdi)
+; CHECK-AVX512-NEXT: .LBB5_2: # %if.end
+; CHECK-AVX512-NEXT: movq (%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, (%rsi)
+; CHECK-AVX512-NEXT: movb 8(%rdi), %al
+; CHECK-AVX512-NEXT: movb %al, 8(%rsi)
+; CHECK-AVX512-NEXT: movl 9(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 9(%rsi)
+; CHECK-AVX512-NEXT: movzwl 13(%rdi), %eax
+; CHECK-AVX512-NEXT: movw %ax, 13(%rsi)
+; CHECK-AVX512-NEXT: movb 15(%rdi), %al
+; CHECK-AVX512-NEXT: movb %al, 15(%rsi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %cmp = icmp sgt i32 %x, 17
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then: ; preds = %entry
+ %conv = sext i32 %x to i64
+ %a = getelementptr inbounds %struct.S3, %struct.S3* %s1, i64 0, i32 0
+ store i64 %conv, i64* %a, align 8
+ %conv1 = trunc i32 %x to i8
+ %b = getelementptr inbounds %struct.S3, %struct.S3* %s1, i64 0, i32 1
+ store i8 %conv1, i8* %b, align 8
+ br label %if.end
+
+if.end: ; preds = %if.then, %entry
+ %0 = bitcast %struct.S3* %s2 to i8*
+ %1 = bitcast %struct.S3* %s1 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 8, i1 false)
+ ret void
+}
+%struct.S4 = type { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 }
+
+; Function Attrs: nounwind uwtable
+define void @test_multiple_blocks(%struct.S4* nocapture noalias %s1, %struct.S4* nocapture %s2) local_unnamed_addr #0 {
+; CHECK-LABEL: test_multiple_blocks:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movl $0, 4(%rdi)
+; CHECK-NEXT: movl $0, 36(%rdi)
+; CHECK-NEXT: movups 16(%rdi), %xmm0
+; CHECK-NEXT: movups %xmm0, 16(%rsi)
+; CHECK-NEXT: movl 32(%rdi), %eax
+; CHECK-NEXT: movl %eax, 32(%rsi)
+; CHECK-NEXT: movl 36(%rdi), %eax
+; CHECK-NEXT: movl %eax, 36(%rsi)
+; CHECK-NEXT: movq 40(%rdi), %rax
+; CHECK-NEXT: movq %rax, 40(%rsi)
+; CHECK-NEXT: movl (%rdi), %eax
+; CHECK-NEXT: movl %eax, (%rsi)
+; CHECK-NEXT: movl 4(%rdi), %eax
+; CHECK-NEXT: movl %eax, 4(%rsi)
+; CHECK-NEXT: movq 8(%rdi), %rax
+; CHECK-NEXT: movq %rax, 8(%rsi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_multiple_blocks:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: movl $0, 4(%rdi)
+; DISABLED-NEXT: movl $0, 36(%rdi)
+; DISABLED-NEXT: movups 16(%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, 16(%rsi)
+; DISABLED-NEXT: movups 32(%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, 32(%rsi)
+; DISABLED-NEXT: movups (%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rsi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_multiple_blocks:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: movl $0, 4(%rdi)
+; CHECK-AVX2-NEXT: movl $0, 36(%rdi)
+; CHECK-AVX2-NEXT: vmovups 16(%rdi), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, 16(%rsi)
+; CHECK-AVX2-NEXT: movl 32(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 32(%rsi)
+; CHECK-AVX2-NEXT: movl 36(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 36(%rsi)
+; CHECK-AVX2-NEXT: movq 40(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, 40(%rsi)
+; CHECK-AVX2-NEXT: movl (%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, (%rsi)
+; CHECK-AVX2-NEXT: movl 4(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 4(%rsi)
+; CHECK-AVX2-NEXT: vmovups 8(%rdi), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, 8(%rsi)
+; CHECK-AVX2-NEXT: movq 24(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, 24(%rsi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_multiple_blocks:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: movl $0, 4(%rdi)
+; CHECK-AVX512-NEXT: movl $0, 36(%rdi)
+; CHECK-AVX512-NEXT: vmovups 16(%rdi), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, 16(%rsi)
+; CHECK-AVX512-NEXT: movl 32(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 32(%rsi)
+; CHECK-AVX512-NEXT: movl 36(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 36(%rsi)
+; CHECK-AVX512-NEXT: movq 40(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, 40(%rsi)
+; CHECK-AVX512-NEXT: movl (%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, (%rsi)
+; CHECK-AVX512-NEXT: movl 4(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 4(%rsi)
+; CHECK-AVX512-NEXT: vmovups 8(%rdi), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, 8(%rsi)
+; CHECK-AVX512-NEXT: movq 24(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, 24(%rsi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %b = getelementptr inbounds %struct.S4, %struct.S4* %s1, i64 0, i32 1
+ store i32 0, i32* %b, align 4
+ %b3 = getelementptr inbounds %struct.S4, %struct.S4* %s1, i64 0, i32 9
+ store i32 0, i32* %b3, align 4
+ %0 = bitcast %struct.S4* %s2 to i8*
+ %1 = bitcast %struct.S4* %s1 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 48, i32 4, i1 false)
+ ret void
+}
+%struct.S5 = type { i16, i16, i16, i16, i16, i16, i16, i16 }
+
+; Function Attrs: nounwind uwtable
+define void @test_type16(%struct.S5* nocapture noalias %s1, %struct.S5* nocapture %s2, i32 %x, %struct.S5* nocapture %s3, %struct.S5* nocapture readonly %s4) local_unnamed_addr #0 {
+; CHECK-LABEL: test_type16:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: cmpl $18, %edx
+; CHECK-NEXT: jl .LBB7_2
+; CHECK-NEXT: # %bb.1: # %if.then
+; CHECK-NEXT: movw %dx, 2(%rdi)
+; CHECK-NEXT: .LBB7_2: # %if.end
+; CHECK-NEXT: movups (%r8), %xmm0
+; CHECK-NEXT: movups %xmm0, (%rcx)
+; CHECK-NEXT: movzwl (%rdi), %eax
+; CHECK-NEXT: movw %ax, (%rsi)
+; CHECK-NEXT: movzwl 2(%rdi), %eax
+; CHECK-NEXT: movw %ax, 2(%rsi)
+; CHECK-NEXT: movq 4(%rdi), %rax
+; CHECK-NEXT: movq %rax, 4(%rsi)
+; CHECK-NEXT: movl 12(%rdi), %eax
+; CHECK-NEXT: movl %eax, 12(%rsi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_type16:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: cmpl $18, %edx
+; DISABLED-NEXT: jl .LBB7_2
+; DISABLED-NEXT: # %bb.1: # %if.then
+; DISABLED-NEXT: movw %dx, 2(%rdi)
+; DISABLED-NEXT: .LBB7_2: # %if.end
+; DISABLED-NEXT: movups (%r8), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rcx)
+; DISABLED-NEXT: movups (%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rsi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_type16:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: cmpl $18, %edx
+; CHECK-AVX2-NEXT: jl .LBB7_2
+; CHECK-AVX2-NEXT: # %bb.1: # %if.then
+; CHECK-AVX2-NEXT: movw %dx, 2(%rdi)
+; CHECK-AVX2-NEXT: .LBB7_2: # %if.end
+; CHECK-AVX2-NEXT: vmovups (%r8), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, (%rcx)
+; CHECK-AVX2-NEXT: movzwl (%rdi), %eax
+; CHECK-AVX2-NEXT: movw %ax, (%rsi)
+; CHECK-AVX2-NEXT: movzwl 2(%rdi), %eax
+; CHECK-AVX2-NEXT: movw %ax, 2(%rsi)
+; CHECK-AVX2-NEXT: movq 4(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, 4(%rsi)
+; CHECK-AVX2-NEXT: movl 12(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 12(%rsi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_type16:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: cmpl $18, %edx
+; CHECK-AVX512-NEXT: jl .LBB7_2
+; CHECK-AVX512-NEXT: # %bb.1: # %if.then
+; CHECK-AVX512-NEXT: movw %dx, 2(%rdi)
+; CHECK-AVX512-NEXT: .LBB7_2: # %if.end
+; CHECK-AVX512-NEXT: vmovups (%r8), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, (%rcx)
+; CHECK-AVX512-NEXT: movzwl (%rdi), %eax
+; CHECK-AVX512-NEXT: movw %ax, (%rsi)
+; CHECK-AVX512-NEXT: movzwl 2(%rdi), %eax
+; CHECK-AVX512-NEXT: movw %ax, 2(%rsi)
+; CHECK-AVX512-NEXT: movq 4(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, 4(%rsi)
+; CHECK-AVX512-NEXT: movl 12(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 12(%rsi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %cmp = icmp sgt i32 %x, 17
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then: ; preds = %entry
+ %conv = trunc i32 %x to i16
+ %b = getelementptr inbounds %struct.S5, %struct.S5* %s1, i64 0, i32 1
+ store i16 %conv, i16* %b, align 2
+ br label %if.end
+
+if.end: ; preds = %if.then, %entry
+ %0 = bitcast %struct.S5* %s3 to i8*
+ %1 = bitcast %struct.S5* %s4 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 2, i1 false)
+ %2 = bitcast %struct.S5* %s2 to i8*
+ %3 = bitcast %struct.S5* %s1 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 16, i32 2, i1 false)
+ ret void
+}
+
+%struct.S6 = type { [4 x i32], i32, i32, i32, i32 }
+
+; Function Attrs: nounwind uwtable
+define void @test_stack(%struct.S6* noalias nocapture sret %agg.result, %struct.S6* byval nocapture readnone align 8 %s1, %struct.S6* byval nocapture align 8 %s2, i32 %x) local_unnamed_addr #0 {
+; CHECK-LABEL: test_stack:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movl %esi, {{[0-9]+}}(%rsp)
+; CHECK-NEXT: movaps {{[0-9]+}}(%rsp), %xmm0
+; CHECK-NEXT: movups %xmm0, (%rdi)
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT: movq %rax, 16(%rdi)
+; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT: movl %eax, 24(%rdi)
+; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT: movl %eax, 28(%rdi)
+; CHECK-NEXT: movaps {{[0-9]+}}(%rsp), %xmm0
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %ecx
+; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %edx
+; CHECK-NEXT: movaps %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT: movq %rax, {{[0-9]+}}(%rsp)
+; CHECK-NEXT: movl %ecx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT: movl %edx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_stack:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: movl %esi, {{[0-9]+}}(%rsp)
+; DISABLED-NEXT: movaps {{[0-9]+}}(%rsp), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rdi)
+; DISABLED-NEXT: movaps {{[0-9]+}}(%rsp), %xmm0
+; DISABLED-NEXT: movups %xmm0, 16(%rdi)
+; DISABLED-NEXT: movaps {{[0-9]+}}(%rsp), %xmm0
+; DISABLED-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1
+; DISABLED-NEXT: movaps %xmm0, {{[0-9]+}}(%rsp)
+; DISABLED-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
+; DISABLED-NEXT: movq %rdi, %rax
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_stack:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: movl %esi, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-NEXT: vmovups {{[0-9]+}}(%rsp), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, (%rdi)
+; CHECK-AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-AVX2-NEXT: movq %rax, 16(%rdi)
+; CHECK-AVX2-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-AVX2-NEXT: movl %eax, 24(%rdi)
+; CHECK-AVX2-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-AVX2-NEXT: movl %eax, 28(%rdi)
+; CHECK-AVX2-NEXT: vmovups {{[0-9]+}}(%rsp), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-AVX2-NEXT: movq %rax, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-AVX2-NEXT: movl %eax, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-AVX2-NEXT: movl %eax, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-NEXT: movq %rdi, %rax
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_stack:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: movl %esi, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-NEXT: vmovups {{[0-9]+}}(%rsp), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, (%rdi)
+; CHECK-AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-AVX512-NEXT: movq %rax, 16(%rdi)
+; CHECK-AVX512-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-AVX512-NEXT: movl %eax, 24(%rdi)
+; CHECK-AVX512-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-AVX512-NEXT: movl %eax, 28(%rdi)
+; CHECK-AVX512-NEXT: vmovups {{[0-9]+}}(%rsp), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-AVX512-NEXT: movq %rax, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-AVX512-NEXT: movl %eax, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-AVX512-NEXT: movl %eax, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-NEXT: movq %rdi, %rax
+; CHECK-AVX512-NEXT: retq
+entry:
+ %s6.sroa.0.0..sroa_cast1 = bitcast %struct.S6* %s2 to i8*
+ %s6.sroa.3.0..sroa_idx4 = getelementptr inbounds %struct.S6, %struct.S6* %s2, i64 0, i32 3
+ store i32 %x, i32* %s6.sroa.3.0..sroa_idx4, align 8
+ %0 = bitcast %struct.S6* %agg.result to i8*
+ %s6.sroa.0.0..sroa_cast2 = bitcast %struct.S6* %s1 to i8*
+ call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* nonnull %s6.sroa.0.0..sroa_cast1, i64 32, i32 4, i1 false)
+ call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull %s6.sroa.0.0..sroa_cast2, i8* nonnull %s6.sroa.0.0..sroa_cast1, i64 32, i32 4, i1 false)
+
+ ret void
+}
+
+; Function Attrs: nounwind uwtable
+define void @test_limit_all(%struct.S* noalias %s1, %struct.S* nocapture %s2, i32 %x, %struct.S* nocapture %s3, %struct.S* nocapture readonly %s4, i32 %x2) local_unnamed_addr #0 {
+; CHECK-LABEL: test_limit_all:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: pushq %rbp
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: pushq %r15
+; CHECK-NEXT: .cfi_def_cfa_offset 24
+; CHECK-NEXT: pushq %r14
+; CHECK-NEXT: .cfi_def_cfa_offset 32
+; CHECK-NEXT: pushq %r12
+; CHECK-NEXT: .cfi_def_cfa_offset 40
+; CHECK-NEXT: pushq %rbx
+; CHECK-NEXT: .cfi_def_cfa_offset 48
+; CHECK-NEXT: .cfi_offset %rbx, -48
+; CHECK-NEXT: .cfi_offset %r12, -40
+; CHECK-NEXT: .cfi_offset %r14, -32
+; CHECK-NEXT: .cfi_offset %r15, -24
+; CHECK-NEXT: .cfi_offset %rbp, -16
+; CHECK-NEXT: movq %r8, %r15
+; CHECK-NEXT: movq %rcx, %r14
+; CHECK-NEXT: movl %edx, %ebp
+; CHECK-NEXT: movq %rsi, %r12
+; CHECK-NEXT: movq %rdi, %rbx
+; CHECK-NEXT: movl %r9d, 12(%rdi)
+; CHECK-NEXT: callq bar
+; CHECK-NEXT: cmpl $18, %ebp
+; CHECK-NEXT: jl .LBB9_2
+; CHECK-NEXT: # %bb.1: # %if.then
+; CHECK-NEXT: movl %ebp, 4(%rbx)
+; CHECK-NEXT: movq %rbx, %rdi
+; CHECK-NEXT: callq bar
+; CHECK-NEXT: .LBB9_2: # %if.end
+; CHECK-NEXT: movups (%r15), %xmm0
+; CHECK-NEXT: movups %xmm0, (%r14)
+; CHECK-NEXT: movups (%rbx), %xmm0
+; CHECK-NEXT: movups %xmm0, (%r12)
+; CHECK-NEXT: popq %rbx
+; CHECK-NEXT: popq %r12
+; CHECK-NEXT: popq %r14
+; CHECK-NEXT: popq %r15
+; CHECK-NEXT: popq %rbp
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_limit_all:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: pushq %rbp
+; DISABLED-NEXT: .cfi_def_cfa_offset 16
+; DISABLED-NEXT: pushq %r15
+; DISABLED-NEXT: .cfi_def_cfa_offset 24
+; DISABLED-NEXT: pushq %r14
+; DISABLED-NEXT: .cfi_def_cfa_offset 32
+; DISABLED-NEXT: pushq %r12
+; DISABLED-NEXT: .cfi_def_cfa_offset 40
+; DISABLED-NEXT: pushq %rbx
+; DISABLED-NEXT: .cfi_def_cfa_offset 48
+; DISABLED-NEXT: .cfi_offset %rbx, -48
+; DISABLED-NEXT: .cfi_offset %r12, -40
+; DISABLED-NEXT: .cfi_offset %r14, -32
+; DISABLED-NEXT: .cfi_offset %r15, -24
+; DISABLED-NEXT: .cfi_offset %rbp, -16
+; DISABLED-NEXT: movq %r8, %r15
+; DISABLED-NEXT: movq %rcx, %r14
+; DISABLED-NEXT: movl %edx, %ebp
+; DISABLED-NEXT: movq %rsi, %r12
+; DISABLED-NEXT: movq %rdi, %rbx
+; DISABLED-NEXT: movl %r9d, 12(%rdi)
+; DISABLED-NEXT: callq bar
+; DISABLED-NEXT: cmpl $18, %ebp
+; DISABLED-NEXT: jl .LBB9_2
+; DISABLED-NEXT: # %bb.1: # %if.then
+; DISABLED-NEXT: movl %ebp, 4(%rbx)
+; DISABLED-NEXT: movq %rbx, %rdi
+; DISABLED-NEXT: callq bar
+; DISABLED-NEXT: .LBB9_2: # %if.end
+; DISABLED-NEXT: movups (%r15), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%r14)
+; DISABLED-NEXT: movups (%rbx), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%r12)
+; DISABLED-NEXT: popq %rbx
+; DISABLED-NEXT: popq %r12
+; DISABLED-NEXT: popq %r14
+; DISABLED-NEXT: popq %r15
+; DISABLED-NEXT: popq %rbp
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_limit_all:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: pushq %rbp
+; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 16
+; CHECK-AVX2-NEXT: pushq %r15
+; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 24
+; CHECK-AVX2-NEXT: pushq %r14
+; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 32
+; CHECK-AVX2-NEXT: pushq %r12
+; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 40
+; CHECK-AVX2-NEXT: pushq %rbx
+; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 48
+; CHECK-AVX2-NEXT: .cfi_offset %rbx, -48
+; CHECK-AVX2-NEXT: .cfi_offset %r12, -40
+; CHECK-AVX2-NEXT: .cfi_offset %r14, -32
+; CHECK-AVX2-NEXT: .cfi_offset %r15, -24
+; CHECK-AVX2-NEXT: .cfi_offset %rbp, -16
+; CHECK-AVX2-NEXT: movq %r8, %r15
+; CHECK-AVX2-NEXT: movq %rcx, %r14
+; CHECK-AVX2-NEXT: movl %edx, %ebp
+; CHECK-AVX2-NEXT: movq %rsi, %r12
+; CHECK-AVX2-NEXT: movq %rdi, %rbx
+; CHECK-AVX2-NEXT: movl %r9d, 12(%rdi)
+; CHECK-AVX2-NEXT: callq bar
+; CHECK-AVX2-NEXT: cmpl $18, %ebp
+; CHECK-AVX2-NEXT: jl .LBB9_2
+; CHECK-AVX2-NEXT: # %bb.1: # %if.then
+; CHECK-AVX2-NEXT: movl %ebp, 4(%rbx)
+; CHECK-AVX2-NEXT: movq %rbx, %rdi
+; CHECK-AVX2-NEXT: callq bar
+; CHECK-AVX2-NEXT: .LBB9_2: # %if.end
+; CHECK-AVX2-NEXT: vmovups (%r15), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, (%r14)
+; CHECK-AVX2-NEXT: vmovups (%rbx), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, (%r12)
+; CHECK-AVX2-NEXT: popq %rbx
+; CHECK-AVX2-NEXT: popq %r12
+; CHECK-AVX2-NEXT: popq %r14
+; CHECK-AVX2-NEXT: popq %r15
+; CHECK-AVX2-NEXT: popq %rbp
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_limit_all:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: pushq %rbp
+; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 16
+; CHECK-AVX512-NEXT: pushq %r15
+; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 24
+; CHECK-AVX512-NEXT: pushq %r14
+; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 32
+; CHECK-AVX512-NEXT: pushq %r12
+; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 40
+; CHECK-AVX512-NEXT: pushq %rbx
+; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 48
+; CHECK-AVX512-NEXT: .cfi_offset %rbx, -48
+; CHECK-AVX512-NEXT: .cfi_offset %r12, -40
+; CHECK-AVX512-NEXT: .cfi_offset %r14, -32
+; CHECK-AVX512-NEXT: .cfi_offset %r15, -24
+; CHECK-AVX512-NEXT: .cfi_offset %rbp, -16
+; CHECK-AVX512-NEXT: movq %r8, %r15
+; CHECK-AVX512-NEXT: movq %rcx, %r14
+; CHECK-AVX512-NEXT: movl %edx, %ebp
+; CHECK-AVX512-NEXT: movq %rsi, %r12
+; CHECK-AVX512-NEXT: movq %rdi, %rbx
+; CHECK-AVX512-NEXT: movl %r9d, 12(%rdi)
+; CHECK-AVX512-NEXT: callq bar
+; CHECK-AVX512-NEXT: cmpl $18, %ebp
+; CHECK-AVX512-NEXT: jl .LBB9_2
+; CHECK-AVX512-NEXT: # %bb.1: # %if.then
+; CHECK-AVX512-NEXT: movl %ebp, 4(%rbx)
+; CHECK-AVX512-NEXT: movq %rbx, %rdi
+; CHECK-AVX512-NEXT: callq bar
+; CHECK-AVX512-NEXT: .LBB9_2: # %if.end
+; CHECK-AVX512-NEXT: vmovups (%r15), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, (%r14)
+; CHECK-AVX512-NEXT: vmovups (%rbx), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, (%r12)
+; CHECK-AVX512-NEXT: popq %rbx
+; CHECK-AVX512-NEXT: popq %r12
+; CHECK-AVX512-NEXT: popq %r14
+; CHECK-AVX512-NEXT: popq %r15
+; CHECK-AVX512-NEXT: popq %rbp
+; CHECK-AVX512-NEXT: retq
+entry:
+ %d = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 3
+ store i32 %x2, i32* %d, align 4
+ tail call void @bar(%struct.S* %s1) #3
+ %cmp = icmp sgt i32 %x, 17
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then: ; preds = %entry
+ %b = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 1
+ store i32 %x, i32* %b, align 4
+ tail call void @bar(%struct.S* nonnull %s1) #3
+ br label %if.end
+
+if.end: ; preds = %if.then, %entry
+ %0 = bitcast %struct.S* %s3 to i8*
+ %1 = bitcast %struct.S* %s4 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 4, i1 false)
+ %2 = bitcast %struct.S* %s2 to i8*
+ %3 = bitcast %struct.S* %s1 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 16, i32 4, i1 false)
+ ret void
+}
+
+; Function Attrs: nounwind uwtable
+define void @test_limit_one_pred(%struct.S* noalias %s1, %struct.S* nocapture %s2, i32 %x, %struct.S* nocapture %s3, %struct.S* nocapture readonly %s4, i32 %x2) local_unnamed_addr #0 {
+; CHECK-LABEL: test_limit_one_pred:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: pushq %r15
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: pushq %r14
+; CHECK-NEXT: .cfi_def_cfa_offset 24
+; CHECK-NEXT: pushq %r12
+; CHECK-NEXT: .cfi_def_cfa_offset 32
+; CHECK-NEXT: pushq %rbx
+; CHECK-NEXT: .cfi_def_cfa_offset 40
+; CHECK-NEXT: pushq %rax
+; CHECK-NEXT: .cfi_def_cfa_offset 48
+; CHECK-NEXT: .cfi_offset %rbx, -40
+; CHECK-NEXT: .cfi_offset %r12, -32
+; CHECK-NEXT: .cfi_offset %r14, -24
+; CHECK-NEXT: .cfi_offset %r15, -16
+; CHECK-NEXT: movq %r8, %r12
+; CHECK-NEXT: movq %rcx, %r15
+; CHECK-NEXT: movq %rsi, %r14
+; CHECK-NEXT: movq %rdi, %rbx
+; CHECK-NEXT: movl %r9d, 12(%rdi)
+; CHECK-NEXT: cmpl $18, %edx
+; CHECK-NEXT: jl .LBB10_2
+; CHECK-NEXT: # %bb.1: # %if.then
+; CHECK-NEXT: movl %edx, 4(%rbx)
+; CHECK-NEXT: movq %rbx, %rdi
+; CHECK-NEXT: callq bar
+; CHECK-NEXT: .LBB10_2: # %if.end
+; CHECK-NEXT: movups (%r12), %xmm0
+; CHECK-NEXT: movups %xmm0, (%r15)
+; CHECK-NEXT: movq (%rbx), %rax
+; CHECK-NEXT: movq %rax, (%r14)
+; CHECK-NEXT: movl 8(%rbx), %eax
+; CHECK-NEXT: movl %eax, 8(%r14)
+; CHECK-NEXT: movl 12(%rbx), %eax
+; CHECK-NEXT: movl %eax, 12(%r14)
+; CHECK-NEXT: addq $8, %rsp
+; CHECK-NEXT: popq %rbx
+; CHECK-NEXT: popq %r12
+; CHECK-NEXT: popq %r14
+; CHECK-NEXT: popq %r15
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_limit_one_pred:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: pushq %r15
+; DISABLED-NEXT: .cfi_def_cfa_offset 16
+; DISABLED-NEXT: pushq %r14
+; DISABLED-NEXT: .cfi_def_cfa_offset 24
+; DISABLED-NEXT: pushq %r12
+; DISABLED-NEXT: .cfi_def_cfa_offset 32
+; DISABLED-NEXT: pushq %rbx
+; DISABLED-NEXT: .cfi_def_cfa_offset 40
+; DISABLED-NEXT: pushq %rax
+; DISABLED-NEXT: .cfi_def_cfa_offset 48
+; DISABLED-NEXT: .cfi_offset %rbx, -40
+; DISABLED-NEXT: .cfi_offset %r12, -32
+; DISABLED-NEXT: .cfi_offset %r14, -24
+; DISABLED-NEXT: .cfi_offset %r15, -16
+; DISABLED-NEXT: movq %r8, %r15
+; DISABLED-NEXT: movq %rcx, %r14
+; DISABLED-NEXT: movq %rsi, %r12
+; DISABLED-NEXT: movq %rdi, %rbx
+; DISABLED-NEXT: movl %r9d, 12(%rdi)
+; DISABLED-NEXT: cmpl $18, %edx
+; DISABLED-NEXT: jl .LBB10_2
+; DISABLED-NEXT: # %bb.1: # %if.then
+; DISABLED-NEXT: movl %edx, 4(%rbx)
+; DISABLED-NEXT: movq %rbx, %rdi
+; DISABLED-NEXT: callq bar
+; DISABLED-NEXT: .LBB10_2: # %if.end
+; DISABLED-NEXT: movups (%r15), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%r14)
+; DISABLED-NEXT: movups (%rbx), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%r12)
+; DISABLED-NEXT: addq $8, %rsp
+; DISABLED-NEXT: popq %rbx
+; DISABLED-NEXT: popq %r12
+; DISABLED-NEXT: popq %r14
+; DISABLED-NEXT: popq %r15
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_limit_one_pred:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: pushq %r15
+; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 16
+; CHECK-AVX2-NEXT: pushq %r14
+; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 24
+; CHECK-AVX2-NEXT: pushq %r12
+; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 32
+; CHECK-AVX2-NEXT: pushq %rbx
+; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 40
+; CHECK-AVX2-NEXT: pushq %rax
+; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 48
+; CHECK-AVX2-NEXT: .cfi_offset %rbx, -40
+; CHECK-AVX2-NEXT: .cfi_offset %r12, -32
+; CHECK-AVX2-NEXT: .cfi_offset %r14, -24
+; CHECK-AVX2-NEXT: .cfi_offset %r15, -16
+; CHECK-AVX2-NEXT: movq %r8, %r12
+; CHECK-AVX2-NEXT: movq %rcx, %r15
+; CHECK-AVX2-NEXT: movq %rsi, %r14
+; CHECK-AVX2-NEXT: movq %rdi, %rbx
+; CHECK-AVX2-NEXT: movl %r9d, 12(%rdi)
+; CHECK-AVX2-NEXT: cmpl $18, %edx
+; CHECK-AVX2-NEXT: jl .LBB10_2
+; CHECK-AVX2-NEXT: # %bb.1: # %if.then
+; CHECK-AVX2-NEXT: movl %edx, 4(%rbx)
+; CHECK-AVX2-NEXT: movq %rbx, %rdi
+; CHECK-AVX2-NEXT: callq bar
+; CHECK-AVX2-NEXT: .LBB10_2: # %if.end
+; CHECK-AVX2-NEXT: vmovups (%r12), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, (%r15)
+; CHECK-AVX2-NEXT: movq (%rbx), %rax
+; CHECK-AVX2-NEXT: movq %rax, (%r14)
+; CHECK-AVX2-NEXT: movl 8(%rbx), %eax
+; CHECK-AVX2-NEXT: movl %eax, 8(%r14)
+; CHECK-AVX2-NEXT: movl 12(%rbx), %eax
+; CHECK-AVX2-NEXT: movl %eax, 12(%r14)
+; CHECK-AVX2-NEXT: addq $8, %rsp
+; CHECK-AVX2-NEXT: popq %rbx
+; CHECK-AVX2-NEXT: popq %r12
+; CHECK-AVX2-NEXT: popq %r14
+; CHECK-AVX2-NEXT: popq %r15
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_limit_one_pred:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: pushq %r15
+; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 16
+; CHECK-AVX512-NEXT: pushq %r14
+; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 24
+; CHECK-AVX512-NEXT: pushq %r12
+; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 32
+; CHECK-AVX512-NEXT: pushq %rbx
+; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 40
+; CHECK-AVX512-NEXT: pushq %rax
+; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 48
+; CHECK-AVX512-NEXT: .cfi_offset %rbx, -40
+; CHECK-AVX512-NEXT: .cfi_offset %r12, -32
+; CHECK-AVX512-NEXT: .cfi_offset %r14, -24
+; CHECK-AVX512-NEXT: .cfi_offset %r15, -16
+; CHECK-AVX512-NEXT: movq %r8, %r12
+; CHECK-AVX512-NEXT: movq %rcx, %r15
+; CHECK-AVX512-NEXT: movq %rsi, %r14
+; CHECK-AVX512-NEXT: movq %rdi, %rbx
+; CHECK-AVX512-NEXT: movl %r9d, 12(%rdi)
+; CHECK-AVX512-NEXT: cmpl $18, %edx
+; CHECK-AVX512-NEXT: jl .LBB10_2
+; CHECK-AVX512-NEXT: # %bb.1: # %if.then
+; CHECK-AVX512-NEXT: movl %edx, 4(%rbx)
+; CHECK-AVX512-NEXT: movq %rbx, %rdi
+; CHECK-AVX512-NEXT: callq bar
+; CHECK-AVX512-NEXT: .LBB10_2: # %if.end
+; CHECK-AVX512-NEXT: vmovups (%r12), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, (%r15)
+; CHECK-AVX512-NEXT: movq (%rbx), %rax
+; CHECK-AVX512-NEXT: movq %rax, (%r14)
+; CHECK-AVX512-NEXT: movl 8(%rbx), %eax
+; CHECK-AVX512-NEXT: movl %eax, 8(%r14)
+; CHECK-AVX512-NEXT: movl 12(%rbx), %eax
+; CHECK-AVX512-NEXT: movl %eax, 12(%r14)
+; CHECK-AVX512-NEXT: addq $8, %rsp
+; CHECK-AVX512-NEXT: popq %rbx
+; CHECK-AVX512-NEXT: popq %r12
+; CHECK-AVX512-NEXT: popq %r14
+; CHECK-AVX512-NEXT: popq %r15
+; CHECK-AVX512-NEXT: retq
+entry:
+ %d = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 3
+ store i32 %x2, i32* %d, align 4
+ %cmp = icmp sgt i32 %x, 17
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then: ; preds = %entry
+ %b = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 1
+ store i32 %x, i32* %b, align 4
+ tail call void @bar(%struct.S* nonnull %s1) #3
+ br label %if.end
+
+if.end: ; preds = %if.then, %entry
+ %0 = bitcast %struct.S* %s3 to i8*
+ %1 = bitcast %struct.S* %s4 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 4, i1 false)
+ %2 = bitcast %struct.S* %s2 to i8*
+ %3 = bitcast %struct.S* %s1 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 16, i32 4, i1 false)
+ ret void
+}
+
+
+declare void @bar(%struct.S*) local_unnamed_addr #1
+
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i32, i1) #1
+
+attributes #0 = { nounwind uwtable "target-cpu"="x86-64" }
+
+%struct.S7 = type { float, float, float , float, float, float, float, float }
+
+; Function Attrs: nounwind uwtable
+define void @test_conditional_block_float(%struct.S7* nocapture noalias %s1, %struct.S7* nocapture %s2, i32 %x, %struct.S7* nocapture %s3, %struct.S7* nocapture readonly %s4, float %y) local_unnamed_addr #0 {
+; CHECK-LABEL: test_conditional_block_float:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: cmpl $18, %edx
+; CHECK-NEXT: jl .LBB11_2
+; CHECK-NEXT: # %bb.1: # %if.then
+; CHECK-NEXT: movl $1065353216, 4(%rdi) # imm = 0x3F800000
+; CHECK-NEXT: .LBB11_2: # %if.end
+; CHECK-NEXT: movups (%r8), %xmm0
+; CHECK-NEXT: movups 16(%r8), %xmm1
+; CHECK-NEXT: movups %xmm1, 16(%rcx)
+; CHECK-NEXT: movups %xmm0, (%rcx)
+; CHECK-NEXT: movl (%rdi), %eax
+; CHECK-NEXT: movl 4(%rdi), %ecx
+; CHECK-NEXT: movq 8(%rdi), %rdx
+; CHECK-NEXT: movups 16(%rdi), %xmm0
+; CHECK-NEXT: movups %xmm0, 16(%rsi)
+; CHECK-NEXT: movl %eax, (%rsi)
+; CHECK-NEXT: movl %ecx, 4(%rsi)
+; CHECK-NEXT: movq %rdx, 8(%rsi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_conditional_block_float:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: cmpl $18, %edx
+; DISABLED-NEXT: jl .LBB11_2
+; DISABLED-NEXT: # %bb.1: # %if.then
+; DISABLED-NEXT: movl $1065353216, 4(%rdi) # imm = 0x3F800000
+; DISABLED-NEXT: .LBB11_2: # %if.end
+; DISABLED-NEXT: movups (%r8), %xmm0
+; DISABLED-NEXT: movups 16(%r8), %xmm1
+; DISABLED-NEXT: movups %xmm1, 16(%rcx)
+; DISABLED-NEXT: movups %xmm0, (%rcx)
+; DISABLED-NEXT: movups (%rdi), %xmm0
+; DISABLED-NEXT: movups 16(%rdi), %xmm1
+; DISABLED-NEXT: movups %xmm1, 16(%rsi)
+; DISABLED-NEXT: movups %xmm0, (%rsi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_conditional_block_float:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: cmpl $18, %edx
+; CHECK-AVX2-NEXT: jl .LBB11_2
+; CHECK-AVX2-NEXT: # %bb.1: # %if.then
+; CHECK-AVX2-NEXT: movl $1065353216, 4(%rdi) # imm = 0x3F800000
+; CHECK-AVX2-NEXT: .LBB11_2: # %if.end
+; CHECK-AVX2-NEXT: vmovups (%r8), %ymm0
+; CHECK-AVX2-NEXT: vmovups %ymm0, (%rcx)
+; CHECK-AVX2-NEXT: movl (%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, (%rsi)
+; CHECK-AVX2-NEXT: movl 4(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 4(%rsi)
+; CHECK-AVX2-NEXT: vmovups 8(%rdi), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, 8(%rsi)
+; CHECK-AVX2-NEXT: movq 24(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, 24(%rsi)
+; CHECK-AVX2-NEXT: vzeroupper
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_conditional_block_float:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: cmpl $18, %edx
+; CHECK-AVX512-NEXT: jl .LBB11_2
+; CHECK-AVX512-NEXT: # %bb.1: # %if.then
+; CHECK-AVX512-NEXT: movl $1065353216, 4(%rdi) # imm = 0x3F800000
+; CHECK-AVX512-NEXT: .LBB11_2: # %if.end
+; CHECK-AVX512-NEXT: vmovups (%r8), %ymm0
+; CHECK-AVX512-NEXT: vmovups %ymm0, (%rcx)
+; CHECK-AVX512-NEXT: movl (%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, (%rsi)
+; CHECK-AVX512-NEXT: movl 4(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 4(%rsi)
+; CHECK-AVX512-NEXT: vmovups 8(%rdi), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, 8(%rsi)
+; CHECK-AVX512-NEXT: movq 24(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, 24(%rsi)
+; CHECK-AVX512-NEXT: vzeroupper
+; CHECK-AVX512-NEXT: retq
+entry:
+ %cmp = icmp sgt i32 %x, 17
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then: ; preds = %entry
+ %b = getelementptr inbounds %struct.S7, %struct.S7* %s1, i64 0, i32 1
+ store float 1.0, float* %b, align 4
+ br label %if.end
+
+if.end: ; preds = %if.then, %entry
+ %0 = bitcast %struct.S7* %s3 to i8*
+ %1 = bitcast %struct.S7* %s4 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 32, i32 4, i1 false)
+ %2 = bitcast %struct.S7* %s2 to i8*
+ %3 = bitcast %struct.S7* %s1 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 32, i32 4, i1 false)
+ ret void
+}
+
+%struct.S8 = type { i64, i64, i64, i64, i64, i64 }
+
+; Function Attrs: nounwind uwtable
+define void @test_conditional_block_ymm(%struct.S8* nocapture noalias %s1, %struct.S8* nocapture %s2, i32 %x, %struct.S8* nocapture %s3, %struct.S8* nocapture readonly %s4) local_unnamed_addr #0 {
+; CHECK-LABEL: test_conditional_block_ymm:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: cmpl $18, %edx
+; CHECK-NEXT: jl .LBB12_2
+; CHECK-NEXT: # %bb.1: # %if.then
+; CHECK-NEXT: movq $1, 8(%rdi)
+; CHECK-NEXT: .LBB12_2: # %if.end
+; CHECK-NEXT: movups (%r8), %xmm0
+; CHECK-NEXT: movups 16(%r8), %xmm1
+; CHECK-NEXT: movups %xmm1, 16(%rcx)
+; CHECK-NEXT: movups %xmm0, (%rcx)
+; CHECK-NEXT: movq (%rdi), %rax
+; CHECK-NEXT: movq 8(%rdi), %rcx
+; CHECK-NEXT: movups 16(%rdi), %xmm0
+; CHECK-NEXT: movups %xmm0, 16(%rsi)
+; CHECK-NEXT: movq %rax, (%rsi)
+; CHECK-NEXT: movq %rcx, 8(%rsi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_conditional_block_ymm:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: cmpl $18, %edx
+; DISABLED-NEXT: jl .LBB12_2
+; DISABLED-NEXT: # %bb.1: # %if.then
+; DISABLED-NEXT: movq $1, 8(%rdi)
+; DISABLED-NEXT: .LBB12_2: # %if.end
+; DISABLED-NEXT: movups (%r8), %xmm0
+; DISABLED-NEXT: movups 16(%r8), %xmm1
+; DISABLED-NEXT: movups %xmm1, 16(%rcx)
+; DISABLED-NEXT: movups %xmm0, (%rcx)
+; DISABLED-NEXT: movups (%rdi), %xmm0
+; DISABLED-NEXT: movups 16(%rdi), %xmm1
+; DISABLED-NEXT: movups %xmm1, 16(%rsi)
+; DISABLED-NEXT: movups %xmm0, (%rsi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_conditional_block_ymm:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: cmpl $18, %edx
+; CHECK-AVX2-NEXT: jl .LBB12_2
+; CHECK-AVX2-NEXT: # %bb.1: # %if.then
+; CHECK-AVX2-NEXT: movq $1, 8(%rdi)
+; CHECK-AVX2-NEXT: .LBB12_2: # %if.end
+; CHECK-AVX2-NEXT: vmovups (%r8), %ymm0
+; CHECK-AVX2-NEXT: vmovups %ymm0, (%rcx)
+; CHECK-AVX2-NEXT: movq (%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, (%rsi)
+; CHECK-AVX2-NEXT: movq 8(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, 8(%rsi)
+; CHECK-AVX2-NEXT: vmovups 16(%rdi), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, 16(%rsi)
+; CHECK-AVX2-NEXT: vzeroupper
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_conditional_block_ymm:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: cmpl $18, %edx
+; CHECK-AVX512-NEXT: jl .LBB12_2
+; CHECK-AVX512-NEXT: # %bb.1: # %if.then
+; CHECK-AVX512-NEXT: movq $1, 8(%rdi)
+; CHECK-AVX512-NEXT: .LBB12_2: # %if.end
+; CHECK-AVX512-NEXT: vmovups (%r8), %ymm0
+; CHECK-AVX512-NEXT: vmovups %ymm0, (%rcx)
+; CHECK-AVX512-NEXT: movq (%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, (%rsi)
+; CHECK-AVX512-NEXT: movq 8(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, 8(%rsi)
+; CHECK-AVX512-NEXT: vmovups 16(%rdi), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, 16(%rsi)
+; CHECK-AVX512-NEXT: vzeroupper
+; CHECK-AVX512-NEXT: retq
+entry:
+ %cmp = icmp sgt i32 %x, 17
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then: ; preds = %entry
+ %b = getelementptr inbounds %struct.S8, %struct.S8* %s1, i64 0, i32 1
+ store i64 1, i64* %b, align 4
+ br label %if.end
+
+if.end: ; preds = %if.then, %entry
+ %0 = bitcast %struct.S8* %s3 to i8*
+ %1 = bitcast %struct.S8* %s4 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 32, i32 4, i1 false)
+ %2 = bitcast %struct.S8* %s2 to i8*
+ %3 = bitcast %struct.S8* %s1 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 32, i32 4, i1 false)
+ ret void
+}
+
+define dso_local void @test_alias(i8* nocapture %A, i32 %x) local_unnamed_addr #0 {
+; CHECK-LABEL: test_alias:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movl %esi, (%rdi)
+; CHECK-NEXT: movups (%rdi), %xmm0
+; CHECK-NEXT: movups %xmm0, 4(%rdi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_alias:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: movl %esi, (%rdi)
+; DISABLED-NEXT: movups (%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, 4(%rdi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_alias:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: movl %esi, (%rdi)
+; CHECK-AVX2-NEXT: vmovups (%rdi), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, 4(%rdi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_alias:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: movl %esi, (%rdi)
+; CHECK-AVX512-NEXT: vmovups (%rdi), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, 4(%rdi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %a = bitcast i8* %A to i32*
+ store i32 %x, i32* %a, align 4
+ %add.ptr = getelementptr inbounds i8, i8* %A, i64 4
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 %add.ptr, i8* align 4 %A, i64 16, i32 4, i1 false)
+ ret void
+}
+
+; Function Attrs: nounwind uwtable
+define dso_local void @test_noalias(i8* nocapture %A, i32 %x) local_unnamed_addr #0 {
+; CHECK-LABEL: test_noalias:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movl %esi, (%rdi)
+; CHECK-NEXT: movl (%rdi), %eax
+; CHECK-NEXT: movl %eax, 20(%rdi)
+; CHECK-NEXT: movq 4(%rdi), %rax
+; CHECK-NEXT: movq %rax, 24(%rdi)
+; CHECK-NEXT: movl 12(%rdi), %eax
+; CHECK-NEXT: movl %eax, 32(%rdi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_noalias:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: movl %esi, (%rdi)
+; DISABLED-NEXT: movups (%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, 20(%rdi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_noalias:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: movl %esi, (%rdi)
+; CHECK-AVX2-NEXT: movl (%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 20(%rdi)
+; CHECK-AVX2-NEXT: movq 4(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, 24(%rdi)
+; CHECK-AVX2-NEXT: movl 12(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 32(%rdi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_noalias:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: movl %esi, (%rdi)
+; CHECK-AVX512-NEXT: movl (%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 20(%rdi)
+; CHECK-AVX512-NEXT: movq 4(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, 24(%rdi)
+; CHECK-AVX512-NEXT: movl 12(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 32(%rdi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %a = bitcast i8* %A to i32*
+ store i32 %x, i32* %a, align 4
+ %add.ptr = getelementptr inbounds i8, i8* %A, i64 20
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 %add.ptr, i8* align 4 %A, i64 16, i32 4, i1 false)
+ ret void
+}
+
+
+