[Analysis][SimplifyLibCalls] improve function signature check for memcmp

This would assert/crash as shown in:
https://llvm.org/PR50850

The matching for bcmp/bcopy should probably also be updated,
but that's another patch.

diff --git a/llvm/lib/Analysis/TargetLibraryInfo.cpp b/llvm/lib/Analysis/TargetLibraryInfo.cpp
index cfd9aff..1e377df 100644 (file)
--- a/llvm/lib/Analysis/TargetLibraryInfo.cpp
+++ b/llvm/lib/Analysis/TargetLibraryInfo.cpp
@@ -922,9 +922,9 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy,
   case LibFunc_vec_malloc:
     return (NumParams == 1 && FTy.getReturnType()->isPointerTy());
   case LibFunc_memcmp:
-    return (NumParams == 3 && FTy.getReturnType()->isIntegerTy(32) &&
-            FTy.getParamType(0)->isPointerTy() &&
-            FTy.getParamType(1)->isPointerTy());
+    return NumParams == 3 && FTy.getReturnType()->isIntegerTy(32) &&
+           FTy.getParamType(0)->isPointerTy() &&
+           FTy.getParamType(1)->isPointerTy() && IsSizeTTy(FTy.getParamType(2));
 
   case LibFunc_memchr:
   case LibFunc_memrchr:

diff --git a/llvm/test/Transforms/InstCombine/X86/simplify-libcalls-memcmp.ll b/llvm/test/Transforms/InstCombine/X86/simplify-libcalls-memcmp.ll
new file mode 100644 (file)
index 0000000..0bb82d8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/simplify-libcalls-memcmp.ll
@@ -0,0 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S < %s -instcombine | FileCheck %s
+
+; This test requires a target with 'bcmp' in its library.
+; It used to crash/assert because the function signature does
+; not match the expected libcall signature for memcmp -
+; the last arg should have type size_t -> i64 for this target.
+
+target triple = "x86_64-unknown-linux-gnu"
+
+@str = private unnamed_addr constant [6 x i8] c"abcde\00", align 1
+
+declare i32 @memcmp(i8*, i8*, i32)
+
+define void @PR50850() {
+; CHECK-LABEL: @PR50850(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @memcmp(i8* bitcast (void ()* @PR50850 to i8*), i8* getelementptr inbounds ([6 x i8], [6 x i8]* @str, i64 0, i64 0), i32 6)
+; CHECK-NEXT:    ret void
+;
+  %call = call i32 @memcmp(i8* bitcast (void ()* @PR50850 to i8*), i8* bitcast ([6 x i8]* @str to i8*), i32 6)
+  ret void
+}