[Analysis] improve function signature checking for calloc

This would crash later if we thought the parameters were
valid for the standard library call as shown in:
https://llvm.org/PR50846

diff --git a/llvm/lib/Analysis/TargetLibraryInfo.cpp b/llvm/lib/Analysis/TargetLibraryInfo.cpp
index 6fd4dbf..812200c 100644 (file)
--- a/llvm/lib/Analysis/TargetLibraryInfo.cpp
+++ b/llvm/lib/Analysis/TargetLibraryInfo.cpp
@@ -996,7 +996,8 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy,
     return (NumParams == 2 && FTy.getParamType(0)->isPointerTy());
   case LibFunc_calloc:
   case LibFunc_vec_calloc:
-    return (NumParams == 2 && FTy.getReturnType()->isPointerTy());
+    return (NumParams == 2 && FTy.getReturnType()->isPointerTy() &&
+            FTy.getParamType(0) == FTy.getParamType(1));
 
   case LibFunc_atof:
   case LibFunc_atoi:

diff --git a/llvm/test/Transforms/InstCombine/calloc-mismatch.ll b/llvm/test/Transforms/InstCombine/calloc-mismatch.ll
new file mode 100644 (file)
index 0000000..f81bde3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/calloc-mismatch.ll
@@ -0,0 +1,16 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; The argument types should match if it is the standard library calloc.
+; Don't crash analyzing an imposter.
+
+declare i8* @calloc(i64, i32)
+
+define void @PR50846() {
+; CHECK-LABEL: @PR50846(
+; CHECK-NEXT:    [[CALL:%.*]] = call i8* @calloc(i64 1, i32 1)
+; CHECK-NEXT:    ret void
+;
+  %call = call i8* @calloc(i64 1, i32 1)
+  ret void
+}