From e2d0b44a7cd261218c9e527d23eb5d13425afe8b Mon Sep 17 00:00:00 2001 From: Matt Morehouse Date: Wed, 29 Jul 2020 18:58:14 +0000 Subject: [PATCH] [DFSan] Add efficient fast16labels instrumentation mode. Adds the -fast-16-labels flag, which enables efficient instrumentation for DFSan when the user needs <=16 labels. The instrumentation eliminates most branches and most calls to __dfsan_union or __dfsan_union_load. Reviewed By: vitalybuka Differential Revision: https://reviews.llvm.org/D84371 --- clang/docs/DataFlowSanitizer.rst | 52 +++++++++++ compiler-rt/lib/dfsan/dfsan.cpp | 34 +++---- compiler-rt/lib/dfsan/dfsan_flags.inc | 4 - compiler-rt/lib/dfsan/done_abilist.txt | 2 + compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.cpp | 2 +- compiler-rt/lib/fuzzer/dataflow/DataFlow.cpp | 6 +- compiler-rt/test/dfsan/fast16labels.c | 30 +------ compiler-rt/test/fuzzer/dataflow.test | 8 +- compiler-rt/test/fuzzer/only-some-bytes-fork.test | 4 +- compiler-rt/test/fuzzer/only-some-bytes.test | 4 +- .../Instrumentation/DataFlowSanitizer.cpp | 54 ++++++++++- .../DataFlowSanitizer/fast16labels.ll | 100 +++++++++++++++++++++ 12 files changed, 235 insertions(+), 65 deletions(-) create mode 100644 llvm/test/Instrumentation/DataFlowSanitizer/fast16labels.ll diff --git a/clang/docs/DataFlowSanitizer.rst b/clang/docs/DataFlowSanitizer.rst index 4495603..cd8c586 100644 --- a/clang/docs/DataFlowSanitizer.rst +++ b/clang/docs/DataFlowSanitizer.rst @@ -174,6 +174,58 @@ the correct labels are propagated. return 0; } +fast16labels mode +================= + +If you need 16 or fewer labels, you can use fast16labels instrumentation for +less CPU and code size overhead. To use fast16labels instrumentation, you'll +need to specify `-fsanitize=dataflow -mllvm -dfsan-fast-16-labels` in your +compile and link commands and use a modified API for creating and managing +labels. + +In fast16labels mode, base labels are simply 16-bit unsigned integers that are +powers of 2 (i.e. 1, 2, 4, 8, ..., 32768), and union labels are created by ORing +base labels. In this mode DFSan does not manage any label metadata, so the +functions `dfsan_create_label`, `dfsan_union`, `dfsan_get_label_info`, +`dfsan_has_label`, `dfsan_has_label_with_desc`, `dfsan_get_label_count`, and +`dfsan_dump_labels` are unsupported. Instead of using them, the user should +maintain any necessary metadata about base labels themselves. + +For example: + +.. code-block:: c++ + + #include + #include + + int main(void) { + int i = 100; + int j = 200; + int k = 300; + dfsan_label i_label = 1; + dfsan_label j_label = 2; + dfsan_label k_label = 4; + dfsan_set_label(i_label, &i, sizeof(i)); + dfsan_set_label(j_label, &j, sizeof(j)); + dfsan_set_label(k_label, &k, sizeof(k)); + + dfsan_label ij_label = dfsan_get_label(i + j); + + assert(ij_label & i_label); // ij_label has i_label + assert(ij_label & j_label); // ij_label has j_label + assert(!(ij_label & k_label)); // ij_label doesn't have k_label + assert(ij_label == 3) // Verifies all of the above + + dfsan_label ijk_label = dfsan_get_label(i + j + k); + + assert(ijk_label & i_label); // ijk_label has i_label + assert(ijk_label & j_label); // ijk_label has j_label + assert(ijk_label & k_label); // ijk_label has k_label + assert(ijk_label == 7); // Verifies all of the above + + return 0; + } + Current status ============== diff --git a/compiler-rt/lib/dfsan/dfsan.cpp b/compiler-rt/lib/dfsan/dfsan.cpp index 105989c..678f6c1 100644 --- a/compiler-rt/lib/dfsan/dfsan.cpp +++ b/compiler-rt/lib/dfsan/dfsan.cpp @@ -18,15 +18,16 @@ // prefixed __dfsan_. //===----------------------------------------------------------------------===// +#include "dfsan/dfsan.h" + #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_file.h" -#include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_flag_parser.h" +#include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_internal_defs.h" #include "sanitizer_common/sanitizer_libc.h" -#include "dfsan/dfsan.h" - using namespace __dfsan; typedef atomic_uint16_t atomic_dfsan_label; @@ -158,18 +159,10 @@ static void dfsan_check_label(dfsan_label label) { } } -static void ReportUnsupportedFast16(const char *func) { - Report("FATAL: DataFlowSanitizer: %s is unsupported in fast16labels mode\n", - func); - Die(); -} - // Resolves the union of two unequal labels. Nonequality is a precondition for // this function (the instrumentation pass inlines the equality test). extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_label __dfsan_union(dfsan_label l1, dfsan_label l2) { - if (flags().fast16labels) - return l1 | l2; DCHECK_NE(l1, l2); if (l1 == 0) @@ -225,6 +218,14 @@ dfsan_label __dfsan_union_load(const dfsan_label *ls, uptr n) { } extern "C" SANITIZER_INTERFACE_ATTRIBUTE +dfsan_label __dfsan_union_load_fast16labels(const dfsan_label *ls, uptr n) { + dfsan_label label = ls[0]; + for (uptr i = 1; i != n; ++i) + label |= ls[i]; + return label; +} + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_unimplemented(char *fname) { if (flags().warn_unimplemented) Report("WARNING: DataFlowSanitizer: call to uninstrumented function %s\n", @@ -259,8 +260,6 @@ dfsan_union(dfsan_label l1, dfsan_label l2) { extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_label dfsan_create_label(const char *desc, void *userdata) { - if (flags().fast16labels) - ReportUnsupportedFast16("dfsan_create_label"); dfsan_label label = atomic_fetch_add(&__dfsan_last_label, 1, memory_order_relaxed) + 1; dfsan_check_label(label); @@ -319,15 +318,11 @@ dfsan_read_label(const void *addr, uptr size) { extern "C" SANITIZER_INTERFACE_ATTRIBUTE const struct dfsan_label_info *dfsan_get_label_info(dfsan_label label) { - if (flags().fast16labels) - ReportUnsupportedFast16("dfsan_get_label_info"); return &__dfsan_label_info[label]; } extern "C" SANITIZER_INTERFACE_ATTRIBUTE int dfsan_has_label(dfsan_label label, dfsan_label elem) { - if (flags().fast16labels) - return label & elem; if (label == elem) return true; const dfsan_label_info *info = dfsan_get_label_info(label); @@ -340,8 +335,6 @@ dfsan_has_label(dfsan_label label, dfsan_label elem) { extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_label dfsan_has_label_with_desc(dfsan_label label, const char *desc) { - if (flags().fast16labels) - ReportUnsupportedFast16("dfsan_has_label_with_desc"); const dfsan_label_info *info = dfsan_get_label_info(label); if (info->l1 != 0) { return dfsan_has_label_with_desc(info->l1, desc) || @@ -361,9 +354,6 @@ dfsan_get_label_count(void) { extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_dump_labels(int fd) { - if (flags().fast16labels) - return; - dfsan_label last_label = atomic_load(&__dfsan_last_label, memory_order_relaxed); for (uptr l = 1; l <= last_label; ++l) { diff --git a/compiler-rt/lib/dfsan/dfsan_flags.inc b/compiler-rt/lib/dfsan/dfsan_flags.inc index 29db73b..cdd0035 100644 --- a/compiler-rt/lib/dfsan/dfsan_flags.inc +++ b/compiler-rt/lib/dfsan/dfsan_flags.inc @@ -29,7 +29,3 @@ DFSAN_FLAG( DFSAN_FLAG(const char *, dump_labels_at_exit, "", "The path of the file where " "to dump the labels when the " "program terminates.") -DFSAN_FLAG(bool, fast16labels, false, - "Enables experimental mode where DFSan supports only 16 power-of-2 labels " - "(1, 2, 4, 8, ... 32768) and the label union is computed as a bit-wise OR." -) diff --git a/compiler-rt/lib/dfsan/done_abilist.txt b/compiler-rt/lib/dfsan/done_abilist.txt index 52f3ff5..540f0a4 100644 --- a/compiler-rt/lib/dfsan/done_abilist.txt +++ b/compiler-rt/lib/dfsan/done_abilist.txt @@ -28,6 +28,8 @@ fun:dfsan_set_write_callback=uninstrumented fun:dfsan_set_write_callback=custom fun:dfsan_flush=uninstrumented fun:dfsan_flush=discard +fun:dfsan_use_fast16labels=uninstrumented +fun:dfsan_use_fast16labels=discard ############################################################################### # glibc diff --git a/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.cpp b/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.cpp index 48df8e6..0e9cdf7 100644 --- a/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.cpp @@ -253,7 +253,7 @@ int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath, return 1; } - static char DFSanEnv[] = "DFSAN_OPTIONS=fast16labels=1:warn_unimplemented=0"; + static char DFSanEnv[] = "DFSAN_OPTIONS=warn_unimplemented=0"; putenv(DFSanEnv); MkDir(DirPath); for (auto &F : CorporaFiles) { diff --git a/compiler-rt/lib/fuzzer/dataflow/DataFlow.cpp b/compiler-rt/lib/fuzzer/dataflow/DataFlow.cpp index 8bf4e25..78b3f9a 100644 --- a/compiler-rt/lib/fuzzer/dataflow/DataFlow.cpp +++ b/compiler-rt/lib/fuzzer/dataflow/DataFlow.cpp @@ -17,9 +17,11 @@ // and also provides basic-block coverage for every input. // // Build: -// 1. Compile this file (DataFlow.cpp) with -fsanitize=dataflow and -O2. +// 1. Compile this file (DataFlow.cpp) with -fsanitize=dataflow -mllvm +// -dfsan-fast-16-labels and -O2. // 2. Compile DataFlowCallbacks.cpp with -O2 -fPIC. // 3. Build the fuzz target with -g -fsanitize=dataflow +// -mllvm -dfsan-fast-16-labels // -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp // 4. Link those together with -fsanitize=dataflow // @@ -36,7 +38,7 @@ // Run: // # Collect data flow and coverage for INPUT_FILE // # write to OUTPUT_FILE (default: stdout) -// export DFSAN_OPTIONS=fast16labels=1:warn_unimplemented=0 +// export DFSAN_OPTIONS=warn_unimplemented=0 // ./a.out INPUT_FILE [OUTPUT_FILE] // // # Print all instrumented functions. llvm-symbolizer must be present in PATH diff --git a/compiler-rt/test/dfsan/fast16labels.c b/compiler-rt/test/dfsan/fast16labels.c index 269d1e2..72aaf38 100644 --- a/compiler-rt/test/dfsan/fast16labels.c +++ b/compiler-rt/test/dfsan/fast16labels.c @@ -1,14 +1,8 @@ -// RUN: %clang_dfsan %s -o %t -// RUN: DFSAN_OPTIONS=fast16labels=1 %run %t -// RUN: DFSAN_OPTIONS=fast16labels=1 not %run %t dfsan_create_label 2>&1 \ -// RUN: | FileCheck %s --check-prefix=CREATE-LABEL -// RUN: DFSAN_OPTIONS=fast16labels=1 not %run %t dfsan_get_label_info 2>&1 \ -// RUN: | FileCheck %s --check-prefix=GET-LABEL-INFO -// RUN: DFSAN_OPTIONS=fast16labels=1 not %run %t dfsan_has_label_with_desc \ -// RUN: 2>&1 | FileCheck %s --check-prefix=HAS-LABEL-WITH-DESC -// -// Tests DFSAN_OPTIONS=fast16labels=1 +// RUN: %clang_dfsan %s -mllvm -dfsan-fast-16-labels -o %t +// RUN: %run %t // +// Tests fast16labels mode. + #include #include @@ -20,19 +14,6 @@ int foo(int a, int b) { } int main(int argc, char *argv[]) { - // Death tests for unsupported API usage. - const char *command = (argc < 2) ? "" : argv[1]; - // CREATE-LABEL: FATAL: DataFlowSanitizer: dfsan_create_label is unsupported - if (strcmp(command, "dfsan_create_label") == 0) - dfsan_create_label("", NULL); - // GET-LABEL-INFO: FATAL: DataFlowSanitizer: dfsan_get_label_info is unsupported - if (strcmp(command, "dfsan_get_label_info") == 0) - dfsan_get_label_info(1); - // HAS-LABEL-WITH-DESC: FATAL: DataFlowSanitizer: dfsan_has_label_with_desc is unsupported - if (strcmp(command, "dfsan_has_label_with_desc") == 0) - dfsan_has_label_with_desc(1, ""); - - // Supported usage. int a = 10; int b = 20; dfsan_set_label(8, &a, sizeof(a)); @@ -43,7 +24,4 @@ int main(int argc, char *argv[]) { dfsan_label l = dfsan_get_label(c); printf("C: 0x%x\n", l); assert(l == 520); // OR of the other two labels. - assert(dfsan_has_label(l, 8)); - assert(dfsan_has_label(l, 512)); - assert(!dfsan_has_label(l, 1)); } diff --git a/compiler-rt/test/fuzzer/dataflow.test b/compiler-rt/test/fuzzer/dataflow.test index 93cd580..391f160 100644 --- a/compiler-rt/test/fuzzer/dataflow.test +++ b/compiler-rt/test/fuzzer/dataflow.test @@ -2,10 +2,10 @@ REQUIRES: linux, x86_64 # Build the tracer and the test. -RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fsanitize=dataflow %S/../../lib/fuzzer/dataflow/DataFlow.cpp -o %t-DataFlow.o +RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fsanitize=dataflow -mllvm -dfsan-fast-16-labels %S/../../lib/fuzzer/dataflow/DataFlow.cpp -o %t-DataFlow.o RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fPIC %S/../../lib/fuzzer/dataflow/DataFlowCallbacks.cpp -o %t-DataFlowCallbacks.o -RUN: %no_fuzzer_cpp_compiler -fno-sanitize=all -fsanitize=dataflow -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp %S/ThreeFunctionsTest.cpp %t-DataFlow*.o -o %t-ThreeFunctionsTestDF -RUN: %no_fuzzer_cpp_compiler -fno-sanitize=all -fsanitize=dataflow -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp %S/Labels20Test.cpp %t-DataFlow*.o -o %t-Labels20TestDF +RUN: %no_fuzzer_cpp_compiler -fno-sanitize=all -fsanitize=dataflow -mllvm -dfsan-fast-16-labels -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp %S/ThreeFunctionsTest.cpp %t-DataFlow*.o -o %t-ThreeFunctionsTestDF +RUN: %no_fuzzer_cpp_compiler -fno-sanitize=all -fsanitize=dataflow -mllvm -dfsan-fast-16-labels -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp %S/Labels20Test.cpp %t-DataFlow*.o -o %t-Labels20TestDF RUN: %cpp_compiler %S/ThreeFunctionsTest.cpp -o %t-ThreeFunctionsTest # Dump the function list. @@ -30,7 +30,7 @@ RUN: echo -n FUZxxxxxxxxxxxxxxxxx > %t/IN20/FUZxxxxxxxxxxxxxxxxx RUN: echo -n FUxxxxxxxxxxxxxxxxxx > %t/IN20/FUxxxxxxxxxxxxxxxxxx -RUN: export DFSAN_OPTIONS=fast16labels=1:warn_unimplemented=0 +RUN: export DFSAN_OPTIONS=warn_unimplemented=0 # This test assumes that the functions in ThreeFunctionsTestDF are instrumented # in a specific order: diff --git a/compiler-rt/test/fuzzer/only-some-bytes-fork.test b/compiler-rt/test/fuzzer/only-some-bytes-fork.test index ef03c24..a4142db 100644 --- a/compiler-rt/test/fuzzer/only-some-bytes-fork.test +++ b/compiler-rt/test/fuzzer/only-some-bytes-fork.test @@ -2,9 +2,9 @@ REQUIRES: linux, x86_64 # Build the tracer and the test. -RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fsanitize=dataflow %S/../../lib/fuzzer/dataflow/DataFlow.cpp -o %t-DataFlow.o +RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fsanitize=dataflow -mllvm -dfsan-fast-16-labels %S/../../lib/fuzzer/dataflow/DataFlow.cpp -o %t-DataFlow.o RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fPIC %S/../../lib/fuzzer/dataflow/DataFlowCallbacks.cpp -o %t-DataFlowCallbacks.o -RUN: %no_fuzzer_cpp_compiler -fno-sanitize=all -fsanitize=dataflow -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp %S/OnlySomeBytesTest.cpp %t-DataFlow*.o -o %t-DFT +RUN: %no_fuzzer_cpp_compiler -fno-sanitize=all -fsanitize=dataflow -mllvm -dfsan-fast-16-labels -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp %S/OnlySomeBytesTest.cpp %t-DataFlow*.o -o %t-DFT RUN: %cpp_compiler %S/OnlySomeBytesTest.cpp -o %t-Fuzz # Test that the fork mode can collect and use the DFT diff --git a/compiler-rt/test/fuzzer/only-some-bytes.test b/compiler-rt/test/fuzzer/only-some-bytes.test index 62703f3..9a31ebf 100644 --- a/compiler-rt/test/fuzzer/only-some-bytes.test +++ b/compiler-rt/test/fuzzer/only-some-bytes.test @@ -2,9 +2,9 @@ REQUIRES: linux, x86_64 # Build the tracer and the test. -RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fsanitize=dataflow %S/../../lib/fuzzer/dataflow/DataFlow.cpp -o %t-DataFlow.o +RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fsanitize=dataflow -mllvm -dfsan-fast-16-labels %S/../../lib/fuzzer/dataflow/DataFlow.cpp -o %t-DataFlow.o RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fPIC %S/../../lib/fuzzer/dataflow/DataFlowCallbacks.cpp -o %t-DataFlowCallbacks.o -RUN: %no_fuzzer_cpp_compiler -fno-sanitize=all -fsanitize=dataflow -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp %S/OnlySomeBytesTest.cpp %t-DataFlow*.o -o %t-DFT +RUN: %no_fuzzer_cpp_compiler -fno-sanitize=all -fsanitize=dataflow -mllvm -dfsan-fast-16-labels -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp %S/OnlySomeBytesTest.cpp %t-DataFlow*.o -o %t-DFT RUN: %cpp_compiler %S/OnlySomeBytesTest.cpp -o %t-Fuzz # Prepare the inputs. diff --git a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp index 317b090..feb1abb 100644 --- a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp @@ -178,6 +178,14 @@ static cl::opt ClEventCallbacks( cl::desc("Insert calls to __dfsan_*_callback functions on data events."), cl::Hidden, cl::init(false)); +// Use a distinct bit for each base label, enabling faster unions with less +// instrumentation. Limits the max number of base labels to 16. +static cl::opt ClFast16Labels( + "dfsan-fast-16-labels", + cl::desc("Use more efficient instrumentation, limiting the number of " + "labels to 16."), + cl::Hidden, cl::init(false)); + static StringRef GetGlobalTypeString(const GlobalValue &G) { // Types of GlobalVariables are always pointer types. Type *GType = G.getValueType(); @@ -362,6 +370,7 @@ class DataFlowSanitizer { FunctionCallee DFSanUnionFn; FunctionCallee DFSanCheckedUnionFn; FunctionCallee DFSanUnionLoadFn; + FunctionCallee DFSanUnionLoadFast16LabelsFn; FunctionCallee DFSanUnimplementedFn; FunctionCallee DFSanSetLabelFn; FunctionCallee DFSanNonzeroLabelFn; @@ -748,6 +757,17 @@ void DataFlowSanitizer::initializeRuntimeFunctions(Module &M) { DFSanUnionLoadFn = Mod->getOrInsertFunction("__dfsan_union_load", DFSanUnionLoadFnTy, AL); } + { + AttributeList AL; + AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex, + Attribute::NoUnwind); + AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex, + Attribute::ReadOnly); + AL = AL.addAttribute(M.getContext(), AttributeList::ReturnIndex, + Attribute::ZExt); + DFSanUnionLoadFast16LabelsFn = Mod->getOrInsertFunction( + "__dfsan_union_load_fast16labels", DFSanUnionLoadFnTy, AL); + } DFSanUnimplementedFn = Mod->getOrInsertFunction("__dfsan_unimplemented", DFSanUnimplementedFnTy); { @@ -810,6 +830,7 @@ bool DataFlowSanitizer::runImpl(Module &M) { &i != DFSanUnionFn.getCallee()->stripPointerCasts() && &i != DFSanCheckedUnionFn.getCallee()->stripPointerCasts() && &i != DFSanUnionLoadFn.getCallee()->stripPointerCasts() && + &i != DFSanUnionLoadFast16LabelsFn.getCallee()->stripPointerCasts() && &i != DFSanUnimplementedFn.getCallee()->stripPointerCasts() && &i != DFSanSetLabelFn.getCallee()->stripPointerCasts() && &i != DFSanNonzeroLabelFn.getCallee()->stripPointerCasts() && @@ -1144,7 +1165,10 @@ Value *DFSanFunction::combineShadows(Value *V1, Value *V2, Instruction *Pos) { return CCS.Shadow; IRBuilder<> IRB(Pos); - if (AvoidNewBlocks) { + if (ClFast16Labels) { + CCS.Block = Pos->getParent(); + CCS.Shadow = IRB.CreateOr(V1, V2); + } else if (AvoidNewBlocks) { CallInst *Call = IRB.CreateCall(DFS.DFSanCheckedUnionFn, {V1, V2}); Call->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt); Call->addParamAttr(0, Attribute::ZExt); @@ -1254,6 +1278,30 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align, IRB.CreateAlignedLoad(DFS.ShadowTy, ShadowAddr1, ShadowAlign), Pos); } } + + if (ClFast16Labels && Size % (64 / DFS.ShadowWidthBits) == 0) { + // First OR all the WideShadows, then OR individual shadows within the + // combined WideShadow. This is fewer instructions than ORing shadows + // individually. + IRBuilder<> IRB(Pos); + Value *WideAddr = + IRB.CreateBitCast(ShadowAddr, Type::getInt64PtrTy(*DFS.Ctx)); + Value *CombinedWideShadow = + IRB.CreateAlignedLoad(IRB.getInt64Ty(), WideAddr, ShadowAlign); + for (uint64_t Ofs = 64 / DFS.ShadowWidthBits; Ofs != Size; + Ofs += 64 / DFS.ShadowWidthBits) { + WideAddr = IRB.CreateGEP(Type::getInt64Ty(*DFS.Ctx), WideAddr, + ConstantInt::get(DFS.IntptrTy, 1)); + Value *NextWideShadow = + IRB.CreateAlignedLoad(IRB.getInt64Ty(), WideAddr, ShadowAlign); + CombinedWideShadow = IRB.CreateOr(CombinedWideShadow, NextWideShadow); + } + for (unsigned Width = 32; Width >= DFS.ShadowWidthBits; Width >>= 1) { + Value *ShrShadow = IRB.CreateLShr(CombinedWideShadow, Width); + CombinedWideShadow = IRB.CreateOr(CombinedWideShadow, ShrShadow); + } + return IRB.CreateTrunc(CombinedWideShadow, DFS.ShadowTy); + } if (!AvoidNewBlocks && Size % (64 / DFS.ShadowWidthBits) == 0) { // Fast path for the common case where each byte has identical shadow: load // shadow 64 bits at a time, fall out to a __dfsan_union_load call if any @@ -1320,8 +1368,10 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align, } IRBuilder<> IRB(Pos); + FunctionCallee &UnionLoadFn = + ClFast16Labels ? DFS.DFSanUnionLoadFast16LabelsFn : DFS.DFSanUnionLoadFn; CallInst *FallbackCall = IRB.CreateCall( - DFS.DFSanUnionLoadFn, {ShadowAddr, ConstantInt::get(DFS.IntptrTy, Size)}); + UnionLoadFn, {ShadowAddr, ConstantInt::get(DFS.IntptrTy, Size)}); FallbackCall->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt); return FallbackCall; } diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/fast16labels.ll b/llvm/test/Instrumentation/DataFlowSanitizer/fast16labels.ll new file mode 100644 index 0000000..85f99f5 --- /dev/null +++ b/llvm/test/Instrumentation/DataFlowSanitizer/fast16labels.ll @@ -0,0 +1,100 @@ +; Test that -dfsan-fast-16-labels mode uses inline ORs rather than calling +; __dfsan_union or __dfsan_union_load. +; RUN: opt < %s -dfsan -dfsan-fast-16-labels -S | FileCheck %s --implicit-check-not="call{{.*}}__dfsan_union" +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define i8 @add(i8 %a, i8 %b) { + ; CHECK-LABEL: define i8 @"dfs$add" + ; CHECK-DAG: %[[ALABEL:.*]] = load{{.*}}__dfsan_arg_tls, i64 0, i64 0 + ; CHECK-DAG: %[[BLABEL:.*]] = load{{.*}}__dfsan_arg_tls, i64 0, i64 1 + ; CHECK: %[[ADDLABEL:.*]] = or i16 %[[ALABEL]], %[[BLABEL]] + ; CHECK: add i8 + ; CHECK: store i16 %[[ADDLABEL]], i16* @__dfsan_retval_tls + ; CHECK: ret i8 + %c = add i8 %a, %b + ret i8 %c +} + +define i8 @load8(i8* %p) { + ; CHECK-LABEL: define i8 @"dfs$load8" + ; CHECK: load i16, i16* + ; CHECK: ptrtoint i8* {{.*}} to i64 + ; CHECK: and i64 + ; CHECK: mul i64 + ; CHECK: inttoptr i64 + ; CHECK: load i16, i16* + ; CHECK: or i16 + ; CHECK: load i8, i8* + ; CHECK: store i16 {{.*}} @__dfsan_retval_tls + ; CHECK: ret i8 + + %a = load i8, i8* %p + ret i8 %a +} + +define i16 @load16(i16* %p) { + ; CHECK-LABEL: define i16 @"dfs$load16" + ; CHECK: ptrtoint i16* + ; CHECK: and i64 + ; CHECK: mul i64 + ; CHECK: inttoptr i64 {{.*}} i16* + ; CHECK: getelementptr i16 + ; CHECK: load i16, i16* + ; CHECK: load i16, i16* + ; CHECK: or i16 + ; CHECK: or i16 + ; CHECK: load i16, i16* + ; CHECK: store {{.*}} @__dfsan_retval_tls + ; CHECK: ret i16 + + %a = load i16, i16* %p + ret i16 %a +} + +define i32 @load32(i32* %p) { + ; CHECK-LABEL: define i32 @"dfs$load32" + ; CHECK: ptrtoint i32* + ; CHECK: and i64 + ; CHECK: mul i64 + ; CHECK: inttoptr i64 {{.*}} i16* + ; CHECK: bitcast i16* {{.*}} i64* + ; CHECK: load i64, i64* + ; CHECK: lshr i64 {{.*}}, 32 + ; CHECK: or i64 + ; CHECK: lshr i64 {{.*}}, 16 + ; CHECK: or i64 + ; CHECK: trunc i64 {{.*}} i16 + ; CHECK: or i16 + ; CHECK: load i32, i32* + ; CHECK: store i16 {{.*}} @__dfsan_retval_tls + ; CHECK: ret i32 + + %a = load i32, i32* %p + ret i32 %a +} + +define i64 @load64(i64* %p) { + ; CHECK-LABEL: define i64 @"dfs$load64" + ; CHECK: ptrtoint i64* + ; CHECK: and i64 + ; CHECK: mul i64 + ; CHECK: inttoptr i64 {{.*}} i16* + ; CHECK: bitcast i16* {{.*}} i64* + ; CHECK: load i64, i64* + ; CHECK: getelementptr i64, i64* {{.*}}, i64 1 + ; CHECK: load i64, i64* + ; CHECK: or i64 + ; CHECK: lshr i64 {{.*}}, 32 + ; CHECK: or i64 + ; CHECK: lshr i64 {{.*}}, 16 + ; CHECK: or i64 + ; CHECK: trunc i64 {{.*}} i16 + ; CHECK: or i16 + ; CHECK: load i64, i64* + ; CHECK: store i16 {{.*}} @__dfsan_retval_tls + ; CHECK: ret i64 + + %a = load i64, i64* %p + ret i64 %a +} -- 2.7.4