llvm_unreachable("Unknown libcall function");
}
+/// True if an instruction is in tail position in its caller. Intended for
+/// legalizing libcalls as tail calls when possible.
+static bool isLibCallInTailPosition(MachineInstr &MI) {
+ const Function &F = MI.getParent()->getParent()->getFunction();
+
+ // Conservatively require the attributes of the call to match those of
+ // the return. Ignore NoAlias and NonNull because they don't affect the
+ // call sequence.
+ AttributeList CallerAttrs = F.getAttributes();
+ if (AttrBuilder(CallerAttrs, AttributeList::ReturnIndex)
+ .removeAttribute(Attribute::NoAlias)
+ .removeAttribute(Attribute::NonNull)
+ .hasAttributes())
+ return false;
+
+ // It's not safe to eliminate the sign / zero extension of the return value.
+ if (CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt) ||
+ CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt))
+ return false;
+
+ // Only tail call if the following instruction is a standard return.
+ auto &TII = *MI.getMF()->getSubtarget().getInstrInfo();
+ MachineInstr *Next = MI.getNextNode();
+ if (!Next || TII.isTailCall(*Next) || !Next->isReturn())
+ return false;
+
+ return true;
+}
+
LegalizerHelper::LegalizeResult
llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
const CallLowering::ArgInfo &Result,
Info.CallConv = TLI.getLibcallCallingConv(RTLibcall);
Info.Callee = MachineOperand::CreateES(Name);
Info.OrigRet = CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx));
+ Info.IsTailCall = isLibCallInTailPosition(MI);
+
std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
if (!CLI.lowerCall(MIRBuilder, Info))
return LegalizerHelper::UnableToLegalize;
+ if (Info.LoweredTailCall) {
+ assert(Info.IsTailCall && "Lowered tail call when it wasn't a tail call?");
+ // We must have a return following the call to get past
+ // isLibCallInTailPosition.
+ assert(MI.getNextNode() && MI.getNextNode()->isReturn() &&
+ "Expected instr following MI to be a return?");
+
+ // We lowered a tail call, so the call is now the return from the block.
+ // Delete the old return.
+ MI.getNextNode()->eraseFromParent();
+ }
+
return LegalizerHelper::Legalized;
}
; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $w2
; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY2]](s32)
- ; CHECK: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
; CHECK: $x0 = COPY [[COPY]](p0)
; CHECK: $x1 = COPY [[COPY1]](p0)
; CHECK: $x2 = COPY [[ZEXT]](s64)
- ; CHECK: BL &memcpy, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $x1, implicit $x2
- ; CHECK: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
- ; CHECK: RET_ReallyLR
+ ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+ ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[C1]], [[C]]
+ ; CHECK: $w3 = COPY [[AND]](s32)
+ ; CHECK: TCRETURNdi &memcpy, 0, csr_aarch64_aapcs, implicit $sp, implicit $x0, implicit $x1, implicit $x2, implicit $w3
%0:_(p0) = COPY $x0
%1:_(p0) = COPY $x1
%2:_(s32) = COPY $w2
; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $w2
; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY2]](s32)
- ; CHECK: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
; CHECK: $x0 = COPY [[COPY]](p0)
; CHECK: $x1 = COPY [[COPY1]](p0)
; CHECK: $x2 = COPY [[ZEXT]](s64)
- ; CHECK: BL &memmove, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $x1, implicit $x2
- ; CHECK: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
- ; CHECK: RET_ReallyLR
+ ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+ ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[C1]], [[C]]
+ ; CHECK: $w3 = COPY [[AND]](s32)
+ ; CHECK: TCRETURNdi &memmove, 0, csr_aarch64_aapcs, implicit $sp, implicit $x0, implicit $x1, implicit $x2, implicit $w3
%0:_(p0) = COPY $x0
%1:_(p0) = COPY $x1
%2:_(s32) = COPY $w2
; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $w2
; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY2]](s32)
- ; CHECK: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
; CHECK: $x0 = COPY [[COPY]](p0)
; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
; CHECK: $w1 = COPY [[COPY3]](s32)
; CHECK: $x2 = COPY [[ZEXT]](s64)
- ; CHECK: BL &memset, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $w1, implicit $x2
- ; CHECK: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
- ; CHECK: RET_ReallyLR
+ ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+ ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[C1]], [[C]]
+ ; CHECK: $w3 = COPY [[AND]](s32)
+ ; CHECK: TCRETURNdi &memset, 0, csr_aarch64_aapcs, implicit $sp, implicit $x0, implicit $w1, implicit $x2, implicit $w3
%0:_(p0) = COPY $x0
%1:_(s32) = COPY $w1
%2:_(s32) = COPY $w2
RET_ReallyLR
...
+---
+name: no_tail_call
+tracksRegLiveness: true
+body: |
+ bb.1:
+ liveins: $w2, $x0, $x1
+
+ ; CHECK-LABEL: name: no_tail_call
+ ; CHECK: liveins: $w2, $x0, $x1
+ ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+ ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
+ ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $w2
+ ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY2]](s32)
+ ; CHECK: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+ ; CHECK: $x0 = COPY [[COPY]](p0)
+ ; CHECK: $x1 = COPY [[COPY1]](p0)
+ ; CHECK: $x2 = COPY [[ZEXT]](s64)
+ ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+ ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[C1]], [[C]]
+ ; CHECK: $w3 = COPY [[AND]](s32)
+ ; CHECK: BL &memcpy, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $x1, implicit $x2, implicit $w3
+ ; CHECK: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+ ; CHECK: $x0 = COPY [[ZEXT]](s64)
+ ; CHECK: RET_ReallyLR implicit $x0
+ %0:_(p0) = COPY $x0
+ %1:_(p0) = COPY $x1
+ %2:_(s32) = COPY $w2
+ %4:_(s1) = G_CONSTANT i1 false
+ %3:_(s64) = G_ZEXT %2(s32)
+ G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %3(s64), %4(s1)
+ $x0 = COPY %3
+ RET_ReallyLR implicit $x0
+
+...
+---
+name: dont_tc_twice
+tracksRegLiveness: true
+body: |
+ bb.1:
+ liveins: $w2, $x0, $x1
+ ; CHECK-LABEL: name: dont_tc_twice
+ ; CHECK: liveins: $w2, $x0, $x1
+ ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+ ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
+ ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $w2
+ ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY2]](s32)
+ ; CHECK: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+ ; CHECK: $x0 = COPY [[COPY]](p0)
+ ; CHECK: $x1 = COPY [[COPY1]](p0)
+ ; CHECK: $x2 = COPY [[ZEXT]](s64)
+ ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+ ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[C1]], [[C]]
+ ; CHECK: $w3 = COPY [[AND]](s32)
+ ; CHECK: BL &memcpy, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $x1, implicit $x2, implicit $w3
+ ; CHECK: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+ ; CHECK: TCRETURNdi &memset, 0, csr_aarch64_aapcs, implicit $sp
+ %0:_(p0) = COPY $x0
+ %1:_(p0) = COPY $x1
+ %2:_(s32) = COPY $w2
+ %4:_(s1) = G_CONSTANT i1 false
+ %3:_(s64) = G_ZEXT %2(s32)
+ G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %3(s64), %4(s1)
+ TCRETURNdi &memset, 0, csr_aarch64_aapcs, implicit $sp