From 2adb13c10026a983eee9e6a4d75df005d283c6a8 Mon Sep 17 00:00:00 2001 From: Paul Redmond Date: Sun, 9 Dec 2012 20:42:17 +0000 Subject: [PATCH] LoopVectorize: support vectorizing intrinsic calls - added function to VectorTargetTransformInfo to query cost of intrinsics - vectorize trivially vectorizable intrinsic calls such as sin, cos, log, etc. Reviewed by: Nadav llvm-svn: 169711 --- llvm/include/llvm/Target/TargetTransformImpl.h | 3 + llvm/include/llvm/TargetTransformInfo.h | 8 + llvm/lib/Target/TargetTransformImpl.cpp | 19 + llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 57 +- llvm/test/Transforms/LoopVectorize/intrinsic.ll | 851 ++++++++++++++++++++++++ 5 files changed, 936 insertions(+), 2 deletions(-) create mode 100644 llvm/test/Transforms/LoopVectorize/intrinsic.ll diff --git a/llvm/include/llvm/Target/TargetTransformImpl.h b/llvm/include/llvm/Target/TargetTransformImpl.h index f4ec97a..a94278c 100644 --- a/llvm/include/llvm/Target/TargetTransformImpl.h +++ b/llvm/include/llvm/Target/TargetTransformImpl.h @@ -90,6 +90,9 @@ public: unsigned Alignment, unsigned AddressSpace) const; + virtual unsigned getIntrinsicInstrCost(Intrinsic::ID, Type *RetTy, + ArrayRef Tys) const; + virtual unsigned getNumberOfParts(Type *Tp) const; }; diff --git a/llvm/include/llvm/TargetTransformInfo.h b/llvm/include/llvm/TargetTransformInfo.h index a18cef6d..519ccb9 100644 --- a/llvm/include/llvm/TargetTransformInfo.h +++ b/llvm/include/llvm/TargetTransformInfo.h @@ -23,6 +23,7 @@ #define LLVM_TRANSFORMS_TARGET_TRANSFORM_INTERFACE #include "llvm/AddressingMode.h" +#include "llvm/Intrinsics.h" #include "llvm/Pass.h" #include "llvm/Support/DataTypes.h" #include "llvm/Type.h" @@ -209,6 +210,13 @@ public: return 1; } + /// Returns the cost of Intrinsic instructions. + virtual unsigned getIntrinsicInstrCost(Intrinsic::ID, + Type *RetTy, + ArrayRef Tys) const { + return 1; + } + /// Returns the number of pieces into which the provided type must be /// split during legalization. Zero is returned when the answer is unknown. virtual unsigned getNumberOfParts(Type *Tp) const { diff --git a/llvm/lib/Target/TargetTransformImpl.cpp b/llvm/lib/Target/TargetTransformImpl.cpp index 2df7263..7f9fdfc 100644 --- a/llvm/lib/Target/TargetTransformImpl.cpp +++ b/llvm/lib/Target/TargetTransformImpl.cpp @@ -347,6 +347,25 @@ VectorTargetTransformImpl::getMemoryOpCost(unsigned Opcode, Type *Src, } unsigned +VectorTargetTransformImpl::getIntrinsicInstrCost(Intrinsic::ID, Type *RetTy, + ArrayRef Tys) const { + // assume that we need to scalarize this intrinsic. + unsigned ScalarizationCost = 0; + unsigned ScalarCalls = 1; + if (RetTy->isVectorTy()) { + ScalarizationCost = getScalarizationOverhead(RetTy, true, false); + ScalarCalls = std::max(ScalarCalls, RetTy->getVectorNumElements()); + } + for (unsigned i = 0, ie = Tys.size(); i != ie; ++i) { + if (Tys[i]->isVectorTy()) { + ScalarizationCost += getScalarizationOverhead(Tys[i], false, true); + ScalarCalls = std::max(ScalarCalls, RetTy->getVectorNumElements()); + } + } + return ScalarCalls + ScalarizationCost; +} + +unsigned VectorTargetTransformImpl::getNumberOfParts(Type *Tp) const { std::pair LT = getTypeLegalizationCost(Tp); return LT.first; diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 66ac0b4..c93c2bf 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -62,6 +62,7 @@ #include "llvm/DerivedTypes.h" #include "llvm/Function.h" #include "llvm/Instructions.h" +#include "llvm/IntrinsicInst.h" #include "llvm/LLVMContext.h" #include "llvm/Module.h" #include "llvm/Pass.h" @@ -1051,6 +1052,35 @@ getReductionIdentity(LoopVectorizationLegality::ReductionKind K) { } } +static bool +isTriviallyVectorizableIntrinsic(Instruction *Inst) { + IntrinsicInst *II = dyn_cast(Inst); + if (!II) + return false; + switch (II->getIntrinsicID()) { + case Intrinsic::sqrt: + case Intrinsic::sin: + case Intrinsic::cos: + case Intrinsic::exp: + case Intrinsic::exp2: + case Intrinsic::log: + case Intrinsic::log10: + case Intrinsic::log2: + case Intrinsic::fabs: + case Intrinsic::floor: + case Intrinsic::ceil: + case Intrinsic::trunc: + case Intrinsic::rint: + case Intrinsic::nearbyint: + case Intrinsic::pow: + case Intrinsic::fma: + return true; + default: + return false; + } + return false; +} + void InnerLoopVectorizer::vectorizeLoop(LoopVectorizationLegality *Legal) { //===------------------------------------------------===// @@ -1509,8 +1539,22 @@ InnerLoopVectorizer::vectorizeBlockInLoop(LoopVectorizationLegality *Legal, break; } + case Instruction::Call: { + assert(isTriviallyVectorizableIntrinsic(it)); + Module *M = BB->getParent()->getParent(); + IntrinsicInst *II = cast(it); + Intrinsic::ID ID = II->getIntrinsicID(); + SmallVector Args; + for (unsigned i = 0, ie = II->getNumArgOperands(); i != ie; ++i) + Args.push_back(getVectorValue(II->getArgOperand(i))); + Type *Tys[] = { VectorType::get(II->getType()->getScalarType(), VF) }; + Function *F = Intrinsic::getDeclaration(M, ID, Tys); + WidenMap[it] = Builder.CreateCall(F, Args); + break; + } + default: - /// All other instructions are unsupported. Scalarize them. + // All other instructions are unsupported. Scalarize them. scalarizeInstruction(it); break; }// end of switch. @@ -1706,7 +1750,7 @@ bool LoopVectorizationLegality::canVectorizeInstrs() { // We still don't handle functions. CallInst *CI = dyn_cast(it); - if (CI) { + if (CI && !isTriviallyVectorizableIntrinsic(it)) { DEBUG(dbgs() << "LV: Found a call site.\n"); return false; } @@ -2326,6 +2370,15 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I, unsigned VF) { Type *SrcVecTy = ToVectorTy(I->getOperand(0)->getType(), VF); return VTTI->getCastInstrCost(I->getOpcode(), VectorTy, SrcVecTy); } + case Instruction::Call: { + assert(isTriviallyVectorizableIntrinsic(I)); + IntrinsicInst *II = cast(I); + Type *RetTy = ToVectorTy(II->getType(), VF); + SmallVector Tys; + for (unsigned i = 0, ie = II->getNumArgOperands(); i != ie; ++i) + Tys.push_back(ToVectorTy(II->getArgOperand(i)->getType(), VF)); + return VTTI->getIntrinsicInstrCost(II->getIntrinsicID(), RetTy, Tys); + } default: { // We are scalarizing the instruction. Return the cost of the scalar // instruction, plus the cost of insert and extract into vector diff --git a/llvm/test/Transforms/LoopVectorize/intrinsic.ll b/llvm/test/Transforms/LoopVectorize/intrinsic.ll new file mode 100644 index 0000000..54e3c69 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/intrinsic.ll @@ -0,0 +1,851 @@ +; RUN: opt < %s -loop-vectorize -force-vector-width=4 -dce -instcombine -licm -S | FileCheck %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +;CHECK: @sqrt_f32 +;CHECK: llvm.sqrt.v4f32 +;CHECK: ret void +define void @sqrt_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds float* %y, i64 %indvars.iv + %0 = load float* %arrayidx, align 4, !tbaa !0 + %call = tail call float @llvm.sqrt.f32(float %0) nounwind readnone + %arrayidx2 = getelementptr inbounds float* %x, i64 %indvars.iv + store float %call, float* %arrayidx2, align 4, !tbaa !0 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare float @llvm.sqrt.f32(float) nounwind readnone + +;CHECK: @sqrt_f64 +;CHECK: llvm.sqrt.v4f64 +;CHECK: ret void +define void @sqrt_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds double* %y, i64 %indvars.iv + %0 = load double* %arrayidx, align 8, !tbaa !3 + %call = tail call double @llvm.sqrt.f64(double %0) nounwind readnone + %arrayidx2 = getelementptr inbounds double* %x, i64 %indvars.iv + store double %call, double* %arrayidx2, align 8, !tbaa !3 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare double @llvm.sqrt.f64(double) nounwind readnone + +;CHECK: @sin_f32 +;CHECK: llvm.sin.v4f32 +;CHECK: ret void +define void @sin_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds float* %y, i64 %indvars.iv + %0 = load float* %arrayidx, align 4, !tbaa !0 + %call = tail call float @llvm.sin.f32(float %0) nounwind readnone + %arrayidx2 = getelementptr inbounds float* %x, i64 %indvars.iv + store float %call, float* %arrayidx2, align 4, !tbaa !0 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare float @llvm.sin.f32(float) nounwind readnone + +;CHECK: @sin_f64 +;CHECK: llvm.sin.v4f64 +;CHECK: ret void +define void @sin_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds double* %y, i64 %indvars.iv + %0 = load double* %arrayidx, align 8, !tbaa !3 + %call = tail call double @llvm.sin.f64(double %0) nounwind readnone + %arrayidx2 = getelementptr inbounds double* %x, i64 %indvars.iv + store double %call, double* %arrayidx2, align 8, !tbaa !3 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare double @llvm.sin.f64(double) nounwind readnone + +;CHECK: @cos_f32 +;CHECK: llvm.cos.v4f32 +;CHECK: ret void +define void @cos_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds float* %y, i64 %indvars.iv + %0 = load float* %arrayidx, align 4, !tbaa !0 + %call = tail call float @llvm.cos.f32(float %0) nounwind readnone + %arrayidx2 = getelementptr inbounds float* %x, i64 %indvars.iv + store float %call, float* %arrayidx2, align 4, !tbaa !0 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare float @llvm.cos.f32(float) nounwind readnone + +;CHECK: @cos_f64 +;CHECK: llvm.cos.v4f64 +;CHECK: ret void +define void @cos_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds double* %y, i64 %indvars.iv + %0 = load double* %arrayidx, align 8, !tbaa !3 + %call = tail call double @llvm.cos.f64(double %0) nounwind readnone + %arrayidx2 = getelementptr inbounds double* %x, i64 %indvars.iv + store double %call, double* %arrayidx2, align 8, !tbaa !3 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare double @llvm.cos.f64(double) nounwind readnone + +;CHECK: @exp_f32 +;CHECK: llvm.exp.v4f32 +;CHECK: ret void +define void @exp_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds float* %y, i64 %indvars.iv + %0 = load float* %arrayidx, align 4, !tbaa !0 + %call = tail call float @llvm.exp.f32(float %0) nounwind readnone + %arrayidx2 = getelementptr inbounds float* %x, i64 %indvars.iv + store float %call, float* %arrayidx2, align 4, !tbaa !0 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare float @llvm.exp.f32(float) nounwind readnone + +;CHECK: @exp_f64 +;CHECK: llvm.exp.v4f64 +;CHECK: ret void +define void @exp_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds double* %y, i64 %indvars.iv + %0 = load double* %arrayidx, align 8, !tbaa !3 + %call = tail call double @llvm.exp.f64(double %0) nounwind readnone + %arrayidx2 = getelementptr inbounds double* %x, i64 %indvars.iv + store double %call, double* %arrayidx2, align 8, !tbaa !3 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare double @llvm.exp.f64(double) nounwind readnone + +;CHECK: @exp2_f32 +;CHECK: llvm.exp2.v4f32 +;CHECK: ret void +define void @exp2_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds float* %y, i64 %indvars.iv + %0 = load float* %arrayidx, align 4, !tbaa !0 + %call = tail call float @llvm.exp2.f32(float %0) nounwind readnone + %arrayidx2 = getelementptr inbounds float* %x, i64 %indvars.iv + store float %call, float* %arrayidx2, align 4, !tbaa !0 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare float @llvm.exp2.f32(float) nounwind readnone + +;CHECK: @exp2_f64 +;CHECK: llvm.exp2.v4f64 +;CHECK: ret void +define void @exp2_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds double* %y, i64 %indvars.iv + %0 = load double* %arrayidx, align 8, !tbaa !3 + %call = tail call double @llvm.exp2.f64(double %0) nounwind readnone + %arrayidx2 = getelementptr inbounds double* %x, i64 %indvars.iv + store double %call, double* %arrayidx2, align 8, !tbaa !3 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare double @llvm.exp2.f64(double) nounwind readnone + +;CHECK: @log_f32 +;CHECK: llvm.log.v4f32 +;CHECK: ret void +define void @log_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds float* %y, i64 %indvars.iv + %0 = load float* %arrayidx, align 4, !tbaa !0 + %call = tail call float @llvm.log.f32(float %0) nounwind readnone + %arrayidx2 = getelementptr inbounds float* %x, i64 %indvars.iv + store float %call, float* %arrayidx2, align 4, !tbaa !0 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare float @llvm.log.f32(float) nounwind readnone + +;CHECK: @log_f64 +;CHECK: llvm.log.v4f64 +;CHECK: ret void +define void @log_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds double* %y, i64 %indvars.iv + %0 = load double* %arrayidx, align 8, !tbaa !3 + %call = tail call double @llvm.log.f64(double %0) nounwind readnone + %arrayidx2 = getelementptr inbounds double* %x, i64 %indvars.iv + store double %call, double* %arrayidx2, align 8, !tbaa !3 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare double @llvm.log.f64(double) nounwind readnone + +;CHECK: @log10_f32 +;CHECK: llvm.log10.v4f32 +;CHECK: ret void +define void @log10_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds float* %y, i64 %indvars.iv + %0 = load float* %arrayidx, align 4, !tbaa !0 + %call = tail call float @llvm.log10.f32(float %0) nounwind readnone + %arrayidx2 = getelementptr inbounds float* %x, i64 %indvars.iv + store float %call, float* %arrayidx2, align 4, !tbaa !0 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare float @llvm.log10.f32(float) nounwind readnone + +;CHECK: @log10_f64 +;CHECK: llvm.log10.v4f64 +;CHECK: ret void +define void @log10_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds double* %y, i64 %indvars.iv + %0 = load double* %arrayidx, align 8, !tbaa !3 + %call = tail call double @llvm.log10.f64(double %0) nounwind readnone + %arrayidx2 = getelementptr inbounds double* %x, i64 %indvars.iv + store double %call, double* %arrayidx2, align 8, !tbaa !3 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare double @llvm.log10.f64(double) nounwind readnone + +;CHECK: @log2_f32 +;CHECK: llvm.log2.v4f32 +;CHECK: ret void +define void @log2_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds float* %y, i64 %indvars.iv + %0 = load float* %arrayidx, align 4, !tbaa !0 + %call = tail call float @llvm.log2.f32(float %0) nounwind readnone + %arrayidx2 = getelementptr inbounds float* %x, i64 %indvars.iv + store float %call, float* %arrayidx2, align 4, !tbaa !0 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare float @llvm.log2.f32(float) nounwind readnone + +;CHECK: @log2_f64 +;CHECK: llvm.log2.v4f64 +;CHECK: ret void +define void @log2_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds double* %y, i64 %indvars.iv + %0 = load double* %arrayidx, align 8, !tbaa !3 + %call = tail call double @llvm.log2.f64(double %0) nounwind readnone + %arrayidx2 = getelementptr inbounds double* %x, i64 %indvars.iv + store double %call, double* %arrayidx2, align 8, !tbaa !3 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare double @llvm.log2.f64(double) nounwind readnone + +;CHECK: @fabs_f32 +;CHECK: llvm.fabs.v4f32 +;CHECK: ret void +define void @fabs_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds float* %y, i64 %indvars.iv + %0 = load float* %arrayidx, align 4, !tbaa !0 + %call = tail call float @llvm.fabs.f32(float %0) nounwind readnone + %arrayidx2 = getelementptr inbounds float* %x, i64 %indvars.iv + store float %call, float* %arrayidx2, align 4, !tbaa !0 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare float @llvm.fabs.f32(float) nounwind readnone + +define void @fabs_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds double* %y, i64 %indvars.iv + %0 = load double* %arrayidx, align 8, !tbaa !3 + %call = tail call double @llvm.fabs(double %0) nounwind readnone + %arrayidx2 = getelementptr inbounds double* %x, i64 %indvars.iv + store double %call, double* %arrayidx2, align 8, !tbaa !3 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare double @llvm.fabs(double) nounwind readnone + +;CHECK: @floor_f32 +;CHECK: llvm.floor.v4f32 +;CHECK: ret void +define void @floor_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds float* %y, i64 %indvars.iv + %0 = load float* %arrayidx, align 4, !tbaa !0 + %call = tail call float @llvm.floor.f32(float %0) nounwind readnone + %arrayidx2 = getelementptr inbounds float* %x, i64 %indvars.iv + store float %call, float* %arrayidx2, align 4, !tbaa !0 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare float @llvm.floor.f32(float) nounwind readnone + +;CHECK: @floor_f64 +;CHECK: llvm.floor.v4f64 +;CHECK: ret void +define void @floor_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds double* %y, i64 %indvars.iv + %0 = load double* %arrayidx, align 8, !tbaa !3 + %call = tail call double @llvm.floor.f64(double %0) nounwind readnone + %arrayidx2 = getelementptr inbounds double* %x, i64 %indvars.iv + store double %call, double* %arrayidx2, align 8, !tbaa !3 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare double @llvm.floor.f64(double) nounwind readnone + +;CHECK: @ceil_f32 +;CHECK: llvm.ceil.v4f32 +;CHECK: ret void +define void @ceil_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds float* %y, i64 %indvars.iv + %0 = load float* %arrayidx, align 4, !tbaa !0 + %call = tail call float @llvm.ceil.f32(float %0) nounwind readnone + %arrayidx2 = getelementptr inbounds float* %x, i64 %indvars.iv + store float %call, float* %arrayidx2, align 4, !tbaa !0 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare float @llvm.ceil.f32(float) nounwind readnone + +;CHECK: @ceil_f64 +;CHECK: llvm.ceil.v4f64 +;CHECK: ret void +define void @ceil_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds double* %y, i64 %indvars.iv + %0 = load double* %arrayidx, align 8, !tbaa !3 + %call = tail call double @llvm.ceil.f64(double %0) nounwind readnone + %arrayidx2 = getelementptr inbounds double* %x, i64 %indvars.iv + store double %call, double* %arrayidx2, align 8, !tbaa !3 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare double @llvm.ceil.f64(double) nounwind readnone + +;CHECK: @trunc_f32 +;CHECK: llvm.trunc.v4f32 +;CHECK: ret void +define void @trunc_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds float* %y, i64 %indvars.iv + %0 = load float* %arrayidx, align 4, !tbaa !0 + %call = tail call float @llvm.trunc.f32(float %0) nounwind readnone + %arrayidx2 = getelementptr inbounds float* %x, i64 %indvars.iv + store float %call, float* %arrayidx2, align 4, !tbaa !0 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare float @llvm.trunc.f32(float) nounwind readnone + +;CHECK: @trunc_f64 +;CHECK: llvm.trunc.v4f64 +;CHECK: ret void +define void @trunc_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds double* %y, i64 %indvars.iv + %0 = load double* %arrayidx, align 8, !tbaa !3 + %call = tail call double @llvm.trunc.f64(double %0) nounwind readnone + %arrayidx2 = getelementptr inbounds double* %x, i64 %indvars.iv + store double %call, double* %arrayidx2, align 8, !tbaa !3 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare double @llvm.trunc.f64(double) nounwind readnone + +;CHECK: @rint_f32 +;CHECK: llvm.rint.v4f32 +;CHECK: ret void +define void @rint_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds float* %y, i64 %indvars.iv + %0 = load float* %arrayidx, align 4, !tbaa !0 + %call = tail call float @llvm.rint.f32(float %0) nounwind readnone + %arrayidx2 = getelementptr inbounds float* %x, i64 %indvars.iv + store float %call, float* %arrayidx2, align 4, !tbaa !0 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare float @llvm.rint.f32(float) nounwind readnone + +;CHECK: @rint_f64 +;CHECK: llvm.rint.v4f64 +;CHECK: ret void +define void @rint_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds double* %y, i64 %indvars.iv + %0 = load double* %arrayidx, align 8, !tbaa !3 + %call = tail call double @llvm.rint.f64(double %0) nounwind readnone + %arrayidx2 = getelementptr inbounds double* %x, i64 %indvars.iv + store double %call, double* %arrayidx2, align 8, !tbaa !3 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare double @llvm.rint.f64(double) nounwind readnone + +;CHECK: @nearbyint_f32 +;CHECK: llvm.nearbyint.v4f32 +;CHECK: ret void +define void @nearbyint_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds float* %y, i64 %indvars.iv + %0 = load float* %arrayidx, align 4, !tbaa !0 + %call = tail call float @llvm.nearbyint.f32(float %0) nounwind readnone + %arrayidx2 = getelementptr inbounds float* %x, i64 %indvars.iv + store float %call, float* %arrayidx2, align 4, !tbaa !0 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare float @llvm.nearbyint.f32(float) nounwind readnone + +;CHECK: @nearbyint_f64 +;CHECK: llvm.nearbyint.v4f64 +;CHECK: ret void +define void @nearbyint_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds double* %y, i64 %indvars.iv + %0 = load double* %arrayidx, align 8, !tbaa !3 + %call = tail call double @llvm.nearbyint.f64(double %0) nounwind readnone + %arrayidx2 = getelementptr inbounds double* %x, i64 %indvars.iv + store double %call, double* %arrayidx2, align 8, !tbaa !3 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare double @llvm.nearbyint.f64(double) nounwind readnone + +;CHECK: @fma_f32 +;CHECK: llvm.fma.v4f32 +;CHECK: ret void +define void @fma_f32(i32 %n, float* noalias %y, float* noalias %x, float* noalias %z, float* noalias %w) nounwind uwtable { +entry: + %cmp12 = icmp sgt i32 %n, 0 + br i1 %cmp12, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds float* %y, i64 %indvars.iv + %0 = load float* %arrayidx, align 4, !tbaa !0 + %arrayidx2 = getelementptr inbounds float* %w, i64 %indvars.iv + %1 = load float* %arrayidx2, align 4, !tbaa !0 + %arrayidx4 = getelementptr inbounds float* %z, i64 %indvars.iv + %2 = load float* %arrayidx4, align 4, !tbaa !0 + %3 = tail call float @llvm.fma.f32(float %0, float %2, float %1) + %arrayidx6 = getelementptr inbounds float* %x, i64 %indvars.iv + store float %3, float* %arrayidx6, align 4, !tbaa !0 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare float @llvm.fma.f32(float, float, float) nounwind readnone + +;CHECK: @fma_f64 +;CHECK: llvm.fma.v4f64 +;CHECK: ret void +define void @fma_f64(i32 %n, double* noalias %y, double* noalias %x, double* noalias %z, double* noalias %w) nounwind uwtable { +entry: + %cmp12 = icmp sgt i32 %n, 0 + br i1 %cmp12, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds double* %y, i64 %indvars.iv + %0 = load double* %arrayidx, align 8, !tbaa !3 + %arrayidx2 = getelementptr inbounds double* %w, i64 %indvars.iv + %1 = load double* %arrayidx2, align 8, !tbaa !3 + %arrayidx4 = getelementptr inbounds double* %z, i64 %indvars.iv + %2 = load double* %arrayidx4, align 8, !tbaa !3 + %3 = tail call double @llvm.fma.f64(double %0, double %2, double %1) + %arrayidx6 = getelementptr inbounds double* %x, i64 %indvars.iv + store double %3, double* %arrayidx6, align 8, !tbaa !3 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare double @llvm.fma.f64(double, double, double) nounwind readnone + +;CHECK: @pow_f32 +;CHECK: llvm.pow.v4f32 +;CHECK: ret void +define void @pow_f32(i32 %n, float* noalias %y, float* noalias %x, float* noalias %z) nounwind uwtable { +entry: + %cmp9 = icmp sgt i32 %n, 0 + br i1 %cmp9, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds float* %y, i64 %indvars.iv + %0 = load float* %arrayidx, align 4, !tbaa !0 + %arrayidx2 = getelementptr inbounds float* %z, i64 %indvars.iv + %1 = load float* %arrayidx2, align 4, !tbaa !0 + %call = tail call float @llvm.pow.f32(float %0, float %1) nounwind readnone + %arrayidx4 = getelementptr inbounds float* %x, i64 %indvars.iv + store float %call, float* %arrayidx4, align 4, !tbaa !0 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare float @llvm.pow.f32(float, float) nounwind readnone + +;CHECK: @pow_f64 +;CHECK: llvm.pow.v4f64 +;CHECK: ret void +define void @pow_f64(i32 %n, double* noalias %y, double* noalias %x, double* noalias %z) nounwind uwtable { +entry: + %cmp9 = icmp sgt i32 %n, 0 + br i1 %cmp9, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds double* %y, i64 %indvars.iv + %0 = load double* %arrayidx, align 8, !tbaa !3 + %arrayidx2 = getelementptr inbounds double* %z, i64 %indvars.iv + %1 = load double* %arrayidx2, align 8, !tbaa !3 + %call = tail call double @llvm.pow.f64(double %0, double %1) nounwind readnone + %arrayidx4 = getelementptr inbounds double* %x, i64 %indvars.iv + store double %call, double* %arrayidx4, align 8, !tbaa !3 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare double @llvm.pow.f64(double, double) nounwind readnone + +!0 = metadata !{metadata !"float", metadata !1} +!1 = metadata !{metadata !"omnipotent char", metadata !2} +!2 = metadata !{metadata !"Simple C/C++ TBAA"} +!3 = metadata !{metadata !"double", metadata !1} +!4 = metadata !{metadata !"int", metadata !1} -- 2.7.4