- The call must immediately precede a :ref:`ret <i_ret>` instruction,
or a pointer bitcast followed by a ret instruction.
- The ret instruction must return the (possibly bitcasted) value
- produced by the call or void.
- - The caller and callee prototypes must match. Pointer types of
- parameters or return types may differ in pointee type, but not
- in address space.
+ produced by the call, undef, or void.
- The calling conventions of the caller and callee must match.
- - All ABI-impacting function attributes, such as sret, byval, inreg,
- returned, and inalloca, must match.
- The callee must be varargs iff the caller is varargs. Bitcasting a
non-varargs function to the appropriate varargs type is legal so
long as the non-varargs prefixes obey the other rules.
+ - The return type must not undergo automatic conversion to an `sret` pointer.
+
+ In addition, if the calling convention is not `swifttailcc` or `tailcc`:
+
+ - All ABI-impacting function attributes, such as sret, byval, inreg,
+ returned, and inalloca, must match.
+ - The caller and callee prototypes must match. Pointer types of parameters
+ or return types may differ in pointee type, but not in address space.
+
+ On the other hand, if the calling convention is `swifttailcc` or `swiftcc`:
+
+ - Only these ABI-impacting attributes attributes are allowed: sret, byval,
+ swiftself, and swiftasync.
+ - Prototypes are not required to match.
Tail call optimization for calls marked ``tail`` is guaranteed to occur if
the following conditions are met:
// with deopt state.
LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
} else {
- LowerCallTo(I, getValue(Callee), false, EHPadBB);
+ LowerCallTo(I, getValue(Callee), false, false, EHPadBB);
}
// If the value of the invoke is used outside of its defining block, make it
SDValue Callee = DAG.getExternalSymbol(
FunctionName,
DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()));
- LowerCallTo(I, Callee, I.isTailCall());
+ LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
}
/// Given a @llvm.call.preallocated.setup, return the corresponding
void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee,
bool isTailCall,
+ bool isMustTailCall,
const BasicBlock *EHPadBB) {
auto &DL = DAG.getDataLayout();
FunctionType *FTy = CB.getFunctionType();
// attribute.
auto *Caller = CB.getParent()->getParent();
if (Caller->getFnAttribute("disable-tail-calls").getValueAsString() ==
- "true")
+ "true" && !isMustTailCall)
isTailCall = false;
// We can't tail call inside a function with a swifterror argument. Lowering
// Check if we can potentially perform a tail call. More detailed checking
// is be done within LowerCallTo, after more information about the call is
// known.
- LowerCallTo(I, Callee, I.isTailCall());
+ LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
}
namespace {
void CopyToExportRegsIfNeeded(const Value *V);
void ExportFromCurrentBlock(const Value *V);
void LowerCallTo(const CallBase &CB, SDValue Callee, bool IsTailCall,
- const BasicBlock *EHPadBB = nullptr);
+ bool IsMustTailCall, const BasicBlock *EHPadBB = nullptr);
// Lower range metadata from 0 to N to assert zext to an integer of nearest
// floor power of two.
void verifySwiftErrorCall(CallBase &Call, const Value *SwiftErrorVal);
void verifySwiftErrorValue(const Value *SwiftErrorVal);
+ void verifyTailCCMustTailAttrs(AttrBuilder Attrs, StringRef Context);
void verifyMustTailCall(CallInst &CI);
bool verifyAttributeCount(AttributeList Attrs, unsigned Params);
void verifyAttributeTypes(AttributeSet Attrs, bool IsFunction,
visitInstruction(Call);
}
+void Verifier::verifyTailCCMustTailAttrs(AttrBuilder Attrs,
+ StringRef Context) {
+ Assert(!Attrs.contains(Attribute::InAlloca),
+ Twine("inalloca attribute not allowed in ") + Context);
+ Assert(!Attrs.contains(Attribute::InReg),
+ Twine("inreg attribute not allowed in ") + Context);
+ Assert(!Attrs.contains(Attribute::SwiftError),
+ Twine("swifterror attribute not allowed in ") + Context);
+ Assert(!Attrs.contains(Attribute::Preallocated),
+ Twine("preallocated attribute not allowed in ") + Context);
+ Assert(!Attrs.contains(Attribute::ByRef),
+ Twine("byref attribute not allowed in ") + Context);
+}
+
/// Two types are "congruent" if they are identical, or if they are both pointer
/// types with different pointee types and the same address space.
static bool isTypeCongruent(Type *L, Type *R) {
void Verifier::verifyMustTailCall(CallInst &CI) {
Assert(!CI.isInlineAsm(), "cannot use musttail call with inline asm", &CI);
- // - The caller and callee prototypes must match. Pointer types of
- // parameters or return types may differ in pointee type, but not
- // address space.
Function *F = CI.getParent()->getParent();
FunctionType *CallerTy = F->getFunctionType();
FunctionType *CalleeTy = CI.getFunctionType();
- if (!CI.getCalledFunction() || !CI.getCalledFunction()->isIntrinsic()) {
- Assert(CallerTy->getNumParams() == CalleeTy->getNumParams(),
- "cannot guarantee tail call due to mismatched parameter counts",
- &CI);
- for (int I = 0, E = CallerTy->getNumParams(); I != E; ++I) {
- Assert(
- isTypeCongruent(CallerTy->getParamType(I), CalleeTy->getParamType(I)),
- "cannot guarantee tail call due to mismatched parameter types", &CI);
- }
- }
Assert(CallerTy->isVarArg() == CalleeTy->isVarArg(),
"cannot guarantee tail call due to mismatched varargs", &CI);
Assert(isTypeCongruent(CallerTy->getReturnType(), CalleeTy->getReturnType()),
Assert(F->getCallingConv() == CI.getCallingConv(),
"cannot guarantee tail call due to mismatched calling conv", &CI);
- // - All ABI-impacting function attributes, such as sret, byval, inreg,
- // returned, preallocated, and inalloca, must match.
- AttributeList CallerAttrs = F->getAttributes();
- AttributeList CalleeAttrs = CI.getAttributes();
- for (int I = 0, E = CallerTy->getNumParams(); I != E; ++I) {
- AttrBuilder CallerABIAttrs = getParameterABIAttributes(I, CallerAttrs);
- AttrBuilder CalleeABIAttrs = getParameterABIAttributes(I, CalleeAttrs);
- Assert(CallerABIAttrs == CalleeABIAttrs,
- "cannot guarantee tail call due to mismatched ABI impacting "
- "function attributes",
- &CI, CI.getOperand(I));
- }
-
// - The call must immediately precede a :ref:`ret <i_ret>` instruction,
// or a pointer bitcast followed by a ret instruction.
// - The ret instruction must return the (possibly bitcasted) value
ReturnInst *Ret = dyn_cast_or_null<ReturnInst>(Next);
Assert(Ret, "musttail call must precede a ret with an optional bitcast",
&CI);
- Assert(!Ret->getReturnValue() || Ret->getReturnValue() == RetVal,
+ Assert(!Ret->getReturnValue() || Ret->getReturnValue() == RetVal ||
+ isa<UndefValue>(Ret->getReturnValue()),
"musttail call result must be returned", Ret);
+
+ AttributeList CallerAttrs = F->getAttributes();
+ AttributeList CalleeAttrs = CI.getAttributes();
+ if (CI.getCallingConv() == CallingConv::SwiftTail ||
+ CI.getCallingConv() == CallingConv::Tail) {
+ StringRef CCName =
+ CI.getCallingConv() == CallingConv::Tail ? "tailcc" : "swifttailcc";
+
+ // - Only sret, byval, swiftself, and swiftasync ABI-impacting attributes
+ // are allowed in swifttailcc call
+ for (int I = 0, E = CallerTy->getNumParams(); I != E; ++I) {
+ AttrBuilder ABIAttrs = getParameterABIAttributes(I, CallerAttrs);
+ SmallString<32> Context{CCName, StringRef(" musttail caller")};
+ verifyTailCCMustTailAttrs(ABIAttrs, Context);
+ }
+ for (int I = 0, E = CalleeTy->getNumParams(); I != E; ++I) {
+ AttrBuilder ABIAttrs = getParameterABIAttributes(I, CalleeAttrs);
+ SmallString<32> Context{CCName, StringRef(" musttail callee")};
+ verifyTailCCMustTailAttrs(ABIAttrs, Context);
+ }
+ // - Varargs functions are not allowed
+ Assert(!CallerTy->isVarArg(), Twine("cannot guarantee ") + CCName +
+ " tail call for varargs function");
+ return;
+ }
+
+ // - The caller and callee prototypes must match. Pointer types of
+ // parameters or return types may differ in pointee type, but not
+ // address space.
+ if (!CI.getCalledFunction() || !CI.getCalledFunction()->isIntrinsic()) {
+ Assert(CallerTy->getNumParams() == CalleeTy->getNumParams(),
+ "cannot guarantee tail call due to mismatched parameter counts",
+ &CI);
+ for (int I = 0, E = CallerTy->getNumParams(); I != E; ++I) {
+ Assert(
+ isTypeCongruent(CallerTy->getParamType(I), CalleeTy->getParamType(I)),
+ "cannot guarantee tail call due to mismatched parameter types", &CI);
+ }
+ }
+
+ // - All ABI-impacting function attributes, such as sret, byval, inreg,
+ // returned, preallocated, and inalloca, must match.
+ for (int I = 0, E = CallerTy->getNumParams(); I != E; ++I) {
+ AttrBuilder CallerABIAttrs = getParameterABIAttributes(I, CallerAttrs);
+ AttrBuilder CalleeABIAttrs = getParameterABIAttributes(I, CalleeAttrs);
+ Assert(CallerABIAttrs == CalleeABIAttrs,
+ "cannot guarantee tail call due to mismatched ABI impacting "
+ "function attributes",
+ &CI, CI.getOperand(I));
+ }
}
void Verifier::visitCallInst(CallInst &CI) {
// Check if it's really possible to do a tail call.
IsTailCall = isEligibleForTailCallOptimization(
Callee, CallConv, IsVarArg, Outs, OutVals, Ins, DAG);
- if (!IsTailCall && CLI.CB && CLI.CB->isMustTailCall())
- report_fatal_error("failed to perform tail call elimination on a call "
- "site marked musttail");
// A sibling call is one where we're under the usual C ABI and not planning
// to change that but can still do a tail call:
++NumTailCalls;
}
+ if (!IsTailCall && CLI.CB && CLI.CB->isMustTailCall())
+ report_fatal_error("failed to perform tail call elimination on a call "
+ "site marked musttail");
+
// Analyze operands of the call, assigning locations to each operand.
SmallVector<CCValAssign, 16> ArgLocs;
CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), ArgLocs,
Callee, CallConv, isVarArg, isStructRet,
MF.getFunction().hasStructRetAttr(), Outs, OutVals, Ins, DAG,
PreferIndirect);
- if (!isTailCall && CLI.CB && CLI.CB->isMustTailCall())
- report_fatal_error("failed to perform tail call elimination on a call "
- "site marked musttail");
if (isTailCall && !getTargetMachine().Options.GuaranteedTailCallOpt &&
CallConv != CallingConv::Tail && CallConv != CallingConv::SwiftTail)
++NumTailCalls;
}
+ if (!isTailCall && CLI.CB && CLI.CB->isMustTailCall())
+ report_fatal_error("failed to perform tail call elimination on a call "
+ "site marked musttail");
// Analyze operands of the call, assigning locations to each operand.
SmallVector<CCValAssign, 16> ArgLocs;
CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
if (CallConv == CallingConv::X86_INTR)
report_fatal_error("X86 interrupts may not be called directly");
- if (Subtarget.isPICStyleGOT() && !IsGuaranteeTCO) {
+ bool IsMustTail = CLI.CB && CLI.CB->isMustTailCall();
+ if (Subtarget.isPICStyleGOT() && !IsGuaranteeTCO && !IsMustTail) {
// If we are using a GOT, disable tail calls to external symbols with
// default visibility. Tail calling such a symbol requires using a GOT
// relocation, which forces early binding of the symbol. This breaks code
isTailCall = false;
}
- bool IsMustTail = CLI.CB && CLI.CB->isMustTailCall();
- if (IsMustTail) {
- // Force this to be a tail call. The verifier rules are enough to ensure
- // that we can lower this successfully without moving the return address
- // around.
- isTailCall = true;
- } else if (isTailCall) {
+
+ if (isTailCall && !IsMustTail) {
// Check if it's really possible to do a tail call.
isTailCall = IsEligibleForTailCallOptimization(Callee, CallConv,
isVarArg, SR != NotStructReturn,
++NumTailCalls;
}
+ if (IsMustTail && !isTailCall)
+ report_fatal_error("failed to perform tail call elimination on a call "
+ "site marked musttail");
+
assert(!(isVarArg && canGuaranteeTCO(CallConv)) &&
"Var args not supported with calling convention fastcc, ghc or hipe");
NumBytes = GetAlignedArgumentStackSize(NumBytes, DAG);
int FPDiff = 0;
- if (isTailCall && !IsSibcall && !IsMustTail) {
+ if (isTailCall &&
+ shouldGuaranteeTCO(CallConv,
+ MF.getTarget().Options.GuaranteedTailCallOpt)) {
// Lower arguments at fp - stackoffset + fpdiff.
unsigned NumBytesCallerPushed = X86Info->getBytesToPopOnReturn();
; COMMON-LABEL: caller_to0_from0:
; COMMON-NEXT: // %bb.
- tail call swifttailcc void @callee_stack0()
+ musttail call swifttailcc void @callee_stack0()
ret void
; COMMON-NEXT: b callee_stack0
define swifttailcc void @caller_to0_from8([8 x i64], i64) {
; COMMON-LABEL: caller_to0_from8:
- tail call swifttailcc void @callee_stack0()
+ musttail call swifttailcc void @callee_stack0()
ret void
; COMMON: add sp, sp, #16
; Key point is that the "42" should go #16 below incoming stack
; pointer (we didn't have arg space to reuse).
- tail call swifttailcc void @callee_stack8([8 x i64] undef, i64 42)
+ musttail call swifttailcc void @callee_stack8([8 x i64] undef, i64 42)
ret void
; COMMON: str {{x[0-9]+}}, [sp, #-16]!
; COMMON-NOT: sub sp,
; Key point is that the "%a" should go where at SP on entry.
- tail call swifttailcc void @callee_stack8([8 x i64] undef, i64 42)
+ musttail call swifttailcc void @callee_stack8([8 x i64] undef, i64 42)
ret void
; COMMON: str {{x[0-9]+}}, [sp]
; Important point is that the call reuses the "dead" argument space
; above %a on the stack. If it tries to go below incoming-SP then the
; callee will not deallocate the space, even in swifttailcc.
- tail call swifttailcc void @callee_stack16([8 x i64] undef, i64 42, i64 2)
+ musttail call swifttailcc void @callee_stack16([8 x i64] undef, i64 42, i64 2)
; COMMON: stp {{x[0-9]+}}, {{x[0-9]+}}, [sp]
; COMMON-NEXT: b callee_stack16
; COMMON-NOT: sub sp,
; Key point is that the "%a" should go where at #16 above SP on entry.
- tail call swifttailcc void @callee_stack8([8 x i64] undef, i64 42)
+ musttail call swifttailcc void @callee_stack8([8 x i64] undef, i64 42)
ret void
; COMMON: str {{x[0-9]+}}, [sp, #16]!
; Here we want to make sure that both loads happen before the stores:
; otherwise either %a or %b will be wrongly clobbered.
- tail call swifttailcc void @callee_stack16([8 x i64] undef, i64 %b, i64 %a)
+ musttail call swifttailcc void @callee_stack16([8 x i64] undef, i64 %b, i64 %a)
ret void
; COMMON: ldp {{x[0-9]+}}, {{x[0-9]+}}, [sp]
--- /dev/null
+; RUN: not --crash llc -mtriple=arm64-apple-ios %s -o - 2>&1 | FileCheck %s
+
+; CHECK: LLVM ERROR: failed to perform tail call elimination on a call site marked musttail
+declare tailcc [16 x i64] @callee()
+define tailcc [16 x i64] @caller() {
+ %res = musttail call tailcc [16 x i64] @callee()
+ ret [16 x i64] %res
+}
--- /dev/null
+; RUN: not --crash llc -mtriple=thumbv7k-apple-ios %s -o - 2>&1 | FileCheck %s
+
+; CHECK: LLVM ERROR: failed to perform tail call elimination on a call site marked musttail
+declare tailcc [16 x i64] @callee()
+define tailcc [16 x i64] @caller() {
+ %res = musttail call tailcc [16 x i64] @callee()
+ ret [16 x i64] %res
+}
; CHECK-NOT: addq
; CHECK: jmp tailcallee
entry:
- %tmp11 = tail call swifttailcc i32 @tailcallee(i32 %in1, i32 %in2, i32 %in1, i32 %in2)
+ %tmp11 = musttail call swifttailcc i32 @tailcallee(i32 %in1, i32 %in2, i32 %in1, i32 %in2)
ret i32 %tmp11
}
define dso_local swifttailcc i8* @alias_caller() nounwind {
; CHECK-LABEL: alias_caller:
; CHECK: jmp noalias_callee # TAILCALL
- %p = tail call swifttailcc noalias i8* @noalias_callee()
+ %p = musttail call swifttailcc noalias i8* @noalias_callee()
ret i8* %p
}
define dso_local swifttailcc i32 @ret_undef() nounwind {
; CHECK-LABEL: ret_undef:
; CHECK: jmp i32_callee # TAILCALL
- %p = tail call swifttailcc i32 @i32_callee()
+ %p = musttail call swifttailcc i32 @i32_callee()
ret i32 undef
}
; CHECK-LABEL: void_test:
; CHECK: jmp void_test
entry:
- tail call swifttailcc void @void_test( i32 %0, i32 %1, i32 %2, i32 %3)
+ musttail call swifttailcc void @void_test( i32 %0, i32 %1, i32 %2, i32 %3)
ret void
}
; CHECK-LABEL: i1test:
; CHECK: jmp i1test
entry:
- %4 = tail call swifttailcc i1 @i1test( i32 %0, i32 %1, i32 %2, i32 %3)
+ %4 = musttail call swifttailcc i1 @i1test( i32 %0, i32 %1, i32 %2, i32 %3)
ret i1 %4
}
--- /dev/null
+; RUN: not --crash llc -mtriple=x86_64-linux-gnu %s -o - 2>&1 | FileCheck %s
+
+; CHECK: LLVM ERROR: failed to perform tail call elimination on a call site marked musttail
+declare tailcc [16 x i64] @callee()
+define tailcc [16 x i64] @caller() {
+ %res = musttail call tailcc [16 x i64] @callee()
+ ret [16 x i64] %res
+}
--- /dev/null
+; RUN: opt -verify %s
+
+define swifttailcc void @valid_attrs(i64* sret(i64) %ret, i8* byval(i8) %byval, i8* swiftself %self, i8* swiftasync %ctx) {
+ musttail call swifttailcc void @valid_attrs(i64* sret(i64) %ret, i8* byval(i8) %byval, i8* swiftself %self, i8* swiftasync %ctx)
+ ret void
+}
+
+define swifttailcc void @mismatch_parms() {
+ musttail call swifttailcc void @valid_attrs(i64* sret(i64) undef, i8* byval(i8) undef, i8* swiftself undef, i8* swiftasync undef)
+ ret void
+}
--- /dev/null
+; RUN: not opt -verify %s 2>&1 | FileCheck %s
+
+declare swifttailcc void @simple()
+
+define swifttailcc void @inreg(i8* inreg) {
+; CHECK: inreg attribute not allowed in swifttailcc musttail caller
+ musttail call swifttailcc void @simple()
+ ret void
+}
+
+define swifttailcc void @inalloca(i8* inalloca(i8)) {
+; CHECK: inalloca attribute not allowed in swifttailcc musttail caller
+ musttail call swifttailcc void @simple()
+ ret void
+}
+
+define swifttailcc void @swifterror(i8** swifterror) {
+; CHECK: swifterror attribute not allowed in swifttailcc musttail caller
+ musttail call swifttailcc void @simple()
+ ret void
+}
+
+define swifttailcc void @preallocated(i8* preallocated(i8)) {
+; CHECK: preallocated attribute not allowed in swifttailcc musttail caller
+ musttail call swifttailcc void @simple()
+ ret void
+}
+
+define swifttailcc void @byref(i8* byref(i8)) {
+; CHECK: byref attribute not allowed in swifttailcc musttail caller
+ musttail call swifttailcc void @simple()
+ ret void
+}
+
+define swifttailcc void @call_inreg() {
+; CHECK: inreg attribute not allowed in swifttailcc musttail callee
+ musttail call swifttailcc void @inreg(i8* inreg undef)
+ ret void
+}
+
+define swifttailcc void @call_inalloca() {
+; CHECK: inalloca attribute not allowed in swifttailcc musttail callee
+ musttail call swifttailcc void @inalloca(i8* inalloca(i8) undef)
+ ret void
+}
+
+define swifttailcc void @call_swifterror() {
+; CHECK: swifterror attribute not allowed in swifttailcc musttail callee
+ %err = alloca swifterror i8*
+ musttail call swifttailcc void @swifterror(i8** swifterror %err)
+ ret void
+}
+
+define swifttailcc void @call_preallocated() {
+; CHECK: preallocated attribute not allowed in swifttailcc musttail callee
+ musttail call swifttailcc void @preallocated(i8* preallocated(i8) undef)
+ ret void
+}
+
+define swifttailcc void @call_byref() {
+; CHECK: byref attribute not allowed in swifttailcc musttail callee
+ musttail call swifttailcc void @byref(i8* byref(i8) undef)
+ ret void
+}
+
+
+declare swifttailcc void @varargs(...)
+define swifttailcc void @call_varargs(...) {
+; CHECK: cannot guarantee swifttailcc tail call for varargs function
+ musttail call swifttailcc void(...) @varargs(...)
+ ret void
+}
--- /dev/null
+; RUN: not opt -verify %s 2>&1 | FileCheck %s
+
+declare tailcc void @simple()
+
+define tailcc void @inreg(i8* inreg) {
+; CHECK: inreg attribute not allowed in tailcc musttail caller
+ musttail call tailcc void @simple()
+ ret void
+}
+
+define tailcc void @inalloca(i8* inalloca(i8)) {
+; CHECK: inalloca attribute not allowed in tailcc musttail caller
+ musttail call tailcc void @simple()
+ ret void
+}
+
+define tailcc void @swifterror(i8** swifterror) {
+; CHECK: swifterror attribute not allowed in tailcc musttail caller
+ musttail call tailcc void @simple()
+ ret void
+}
+
+define tailcc void @preallocated(i8* preallocated(i8)) {
+; CHECK: preallocated attribute not allowed in tailcc musttail caller
+ musttail call tailcc void @simple()
+ ret void
+}
+
+define tailcc void @byref(i8* byref(i8)) {
+; CHECK: byref attribute not allowed in tailcc musttail caller
+ musttail call tailcc void @simple()
+ ret void
+}
+
+define tailcc void @call_inreg() {
+; CHECK: inreg attribute not allowed in tailcc musttail callee
+ musttail call tailcc void @inreg(i8* inreg undef)
+ ret void
+}
+
+define tailcc void @call_inalloca() {
+; CHECK: inalloca attribute not allowed in tailcc musttail callee
+ musttail call tailcc void @inalloca(i8* inalloca(i8) undef)
+ ret void
+}
+
+define tailcc void @call_swifterror() {
+; CHECK: swifterror attribute not allowed in tailcc musttail callee
+ %err = alloca swifterror i8*
+ musttail call tailcc void @swifterror(i8** swifterror %err)
+ ret void
+}
+
+define tailcc void @call_preallocated() {
+; CHECK: preallocated attribute not allowed in tailcc musttail callee
+ musttail call tailcc void @preallocated(i8* preallocated(i8) undef)
+ ret void
+}
+
+define tailcc void @call_byref() {
+; CHECK: byref attribute not allowed in tailcc musttail callee
+ musttail call tailcc void @byref(i8* byref(i8) undef)
+ ret void
+}
+
+
+declare tailcc void @varargs(...)
+define tailcc void @call_varargs(...) {
+; CHECK: cannot guarantee tailcc tail call for varargs function
+ musttail call tailcc void(...) @varargs(...)
+ ret void
+}