[Analysis] improve function matching for strlen libcall

The return type of strlen is size_t, not just any integer.

This is a partial fix for an example based on:
https://llvm.org/PR50836

There's another bug here because we can still crash
processing a real strlen or something that looks like it.

diff --git a/llvm/lib/Analysis/TargetLibraryInfo.cpp b/llvm/lib/Analysis/TargetLibraryInfo.cpp
index 0a2031d..a7027bc 100644 (file)
--- a/llvm/lib/Analysis/TargetLibraryInfo.cpp
+++ b/llvm/lib/Analysis/TargetLibraryInfo.cpp
@@ -760,8 +760,8 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy,
       return false;
     LLVM_FALLTHROUGH;
   case LibFunc_strlen:
-    return (NumParams == 1 && FTy.getParamType(0)->isPointerTy() &&
-            FTy.getReturnType()->isIntegerTy());
+    return NumParams == 1 && FTy.getParamType(0)->isPointerTy() &&
+           IsSizeTTy(FTy.getReturnType());
 
   case LibFunc_strchr:
   case LibFunc_strrchr:

diff --git a/llvm/test/Transforms/InstCombine/call-callconv.ll b/llvm/test/Transforms/InstCombine/call-callconv.ll
index 58a0cf2..48fdea4 100644 (file)
--- a/llvm/test/Transforms/InstCombine/call-callconv.ll
+++ b/llvm/test/Transforms/InstCombine/call-callconv.ll
@@ -2,6 +2,11 @@
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 ; Verify that the non-default calling conv doesn't prevent the libcall simplification
 
+; Layout specifies type of pointer which determines "size_t"
+; which is used to identify libcalls such as "size_t strlen(char*)".
+
+target datalayout = "p:32:32"
+
 @.str = private unnamed_addr constant [4 x i8] c"abc\00", align 1
 
 define arm_aapcscc i32 @_abs(i32 %i) nounwind readnone {

diff --git a/llvm/test/Transforms/InstCombine/simplify-libcalls.ll b/llvm/test/Transforms/InstCombine/simplify-libcalls.ll
index f80286a..e4db9d7 100644 (file)
--- a/llvm/test/Transforms/InstCombine/simplify-libcalls.ll
+++ b/llvm/test/Transforms/InstCombine/simplify-libcalls.ll
@@ -230,5 +230,17 @@ define i32 @fake_snprintf(i32 %buf, double %len, i32 * %str) {
   ret i32 %call
 }
 
+; Wrong return type for the real strlen.
+; https://llvm.org/PR50836
+
+define i4 @strlen(i8* %s) {
+; CHECK-LABEL: @strlen(
+; CHECK-NEXT:    [[R:%.*]] = call i4 @strlen(i8* [[S:%.*]])
+; CHECK-NEXT:    ret i4 0
+;
+  %r = call i4 @strlen(i8* %s)
+  ret i4 0
+}
+
 attributes #0 = { nobuiltin }
 attributes #1 = { builtin }