From c5325ed29d01d2d24121fcc57249c7939d085814 Mon Sep 17 00:00:00 2001 From: Kostya Serebryany Date: Sat, 8 Oct 2016 23:24:45 +0000 Subject: [PATCH] [libFuzzer] when shrinking the corpus, delete evicted files previously created by the current process llvm-svn: 283682 --- llvm/lib/Fuzzer/FuzzerCorpus.h | 27 +++++++++++++++++++-------- llvm/lib/Fuzzer/FuzzerDefs.h | 6 ++++-- llvm/lib/Fuzzer/FuzzerDriver.cpp | 5 +++-- llvm/lib/Fuzzer/FuzzerFlags.def | 2 +- llvm/lib/Fuzzer/FuzzerIO.cpp | 14 ++++++++++---- llvm/lib/Fuzzer/FuzzerLoop.cpp | 13 +++++++------ llvm/lib/Fuzzer/build.sh | 2 +- llvm/lib/Fuzzer/test/FuzzerUnittest.cpp | 2 +- 8 files changed, 46 insertions(+), 25 deletions(-) diff --git a/llvm/lib/Fuzzer/FuzzerCorpus.h b/llvm/lib/Fuzzer/FuzzerCorpus.h index 5899c67..714ca2c 100644 --- a/llvm/lib/Fuzzer/FuzzerCorpus.h +++ b/llvm/lib/Fuzzer/FuzzerCorpus.h @@ -30,12 +30,13 @@ struct InputInfo { // Stats. size_t NumExecutedMutations = 0; size_t NumSuccessfullMutations = 0; + bool MayDeleteFile = false; }; class InputCorpus { public: static const size_t kFeatureSetSize = 1 << 16; - InputCorpus() { + InputCorpus(const std::string &OutputCorpus) : OutputCorpus(OutputCorpus) { memset(InputSizesPerFeature, 0, sizeof(InputSizesPerFeature)); memset(SmallestElementPerFeature, 0, sizeof(SmallestElementPerFeature)); } @@ -58,7 +59,7 @@ class InputCorpus { } bool empty() const { return Inputs.empty(); } const Unit &operator[] (size_t Idx) const { return Inputs[Idx]->U; } - void AddToCorpus(const Unit &U, size_t NumFeatures) { + void AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile = false) { assert(!U.empty()); uint8_t Hash[kSHA1NumBytes]; if (FeatureDebug) @@ -69,6 +70,7 @@ class InputCorpus { InputInfo &II = *Inputs.back(); II.U = U; II.NumFeatures = NumFeatures; + II.MayDeleteFile = MayDeleteFile; memcpy(II.Sha1, Hash, kSHA1NumBytes); UpdateCorpusDistribution(); ValidateFeatureSet(); @@ -112,20 +114,27 @@ class InputCorpus { Printf("\n"); } + void DeleteInput(size_t Idx) { + InputInfo &II = *Inputs[Idx]; + if (!OutputCorpus.empty() && II.MayDeleteFile) + DeleteFile(DirPlusFile(OutputCorpus, Sha1ToString(II.Sha1))); + Unit().swap(II.U); + if (FeatureDebug) + Printf("EVICTED %zd\n", Idx); + } + bool AddFeature(size_t Idx, uint32_t NewSize, bool Shrink) { assert(NewSize); Idx = Idx % kFeatureSetSize; uint32_t OldSize = GetFeature(Idx); if (OldSize == 0 || (Shrink && OldSize > NewSize)) { if (OldSize > 0) { - InputInfo &II = *Inputs[SmallestElementPerFeature[Idx]]; + size_t OldIdx = SmallestElementPerFeature[Idx]; + InputInfo &II = *Inputs[OldIdx]; assert(II.NumFeatures > 0); II.NumFeatures--; - if (II.NumFeatures == 0) { - Unit().swap(II.U); - if (FeatureDebug) - Printf("EVICTED %zd\n", SmallestElementPerFeature[Idx]); - } + if (II.NumFeatures == 0) + DeleteInput(OldIdx); } if (FeatureDebug) Printf("ADD FEATURE %zd sz %d\n", Idx, NewSize); @@ -191,6 +200,8 @@ private: bool CountingFeatures = false; uint32_t InputSizesPerFeature[kFeatureSetSize]; uint32_t SmallestElementPerFeature[kFeatureSetSize]; + + std::string OutputCorpus; }; } // namespace fuzzer diff --git a/llvm/lib/Fuzzer/FuzzerDefs.h b/llvm/lib/Fuzzer/FuzzerDefs.h index 3fc62da..b282a83 100644 --- a/llvm/lib/Fuzzer/FuzzerDefs.h +++ b/llvm/lib/Fuzzer/FuzzerDefs.h @@ -60,11 +60,13 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback); bool IsFile(const std::string &Path); long GetEpoch(const std::string &Path); std::string FileToString(const std::string &Path); -Unit FileToVector(const std::string &Path, size_t MaxSize = 0); +Unit FileToVector(const std::string &Path, size_t MaxSize = 0, + bool ExitOnError = true); void ReadDirToVectorOfUnits(const char *Path, std::vector *V, - long *Epoch, size_t MaxSize); + long *Epoch, size_t MaxSize, bool ExitOnError); void WriteToFile(const Unit &U, const std::string &Path); void CopyFileToErr(const std::string &Path); +void DeleteFile(const std::string &Path); // Returns "Dir/FileName" or equivalent for the current OS. std::string DirPlusFile(const std::string &DirPath, const std::string &FileName); diff --git a/llvm/lib/Fuzzer/FuzzerDriver.cpp b/llvm/lib/Fuzzer/FuzzerDriver.cpp index a00e49e..974bbca 100644 --- a/llvm/lib/Fuzzer/FuzzerDriver.cpp +++ b/llvm/lib/Fuzzer/FuzzerDriver.cpp @@ -443,7 +443,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { Random Rand(Seed); MutationDispatcher MD(Rand, Options); - InputCorpus Corpus; + InputCorpus Corpus(Options.OutputCorpus); Fuzzer F(Callback, Corpus, MD, Options); for (auto &U: Dictionary) @@ -500,7 +500,8 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { UnitVector InitialCorpus; for (auto &Inp : *Inputs) { Printf("Loading corpus dir: %s\n", Inp.c_str()); - ReadDirToVectorOfUnits(Inp.c_str(), &InitialCorpus, nullptr, TemporaryMaxLen); + ReadDirToVectorOfUnits(Inp.c_str(), &InitialCorpus, nullptr, + TemporaryMaxLen, /*ExitOnError=*/false); } if (Options.MaxLen == 0) { diff --git a/llvm/lib/Fuzzer/FuzzerFlags.def b/llvm/lib/Fuzzer/FuzzerFlags.def index 03db9ad..4371276 100644 --- a/llvm/lib/Fuzzer/FuzzerFlags.def +++ b/llvm/lib/Fuzzer/FuzzerFlags.def @@ -56,7 +56,7 @@ FUZZER_FLAG_INT(jobs, 0, "Number of jobs to run. If jobs >= 1 we spawn" FUZZER_FLAG_INT(workers, 0, "Number of simultaneous worker processes to run the jobs." " If zero, \"min(jobs,NumberOfCpuCores()/2)\" is used.") -FUZZER_FLAG_INT(reload, 10, +FUZZER_FLAG_INT(reload, 1, "Reload the main corpus every seconds to get new units" " discovered by other processes. If 0, disabled") FUZZER_FLAG_INT(report_slow_units, 10, diff --git a/llvm/lib/Fuzzer/FuzzerIO.cpp b/llvm/lib/Fuzzer/FuzzerIO.cpp index e956b3e..a70af88 100644 --- a/llvm/lib/Fuzzer/FuzzerIO.cpp +++ b/llvm/lib/Fuzzer/FuzzerIO.cpp @@ -60,9 +60,9 @@ static void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, *Epoch = E; } -Unit FileToVector(const std::string &Path, size_t MaxSize) { +Unit FileToVector(const std::string &Path, size_t MaxSize, bool ExitOnError) { std::ifstream T(Path); - if (!T) { + if (ExitOnError && !T) { Printf("No such directory: %s; exiting\n", Path.c_str()); exit(1); } @@ -78,6 +78,10 @@ Unit FileToVector(const std::string &Path, size_t MaxSize) { return Res; } +void DeleteFile(const std::string &Path) { + unlink(Path.c_str()); +} + std::string FileToString(const std::string &Path) { std::ifstream T(Path); return std::string((std::istreambuf_iterator(T)), @@ -97,7 +101,7 @@ void WriteToFile(const Unit &U, const std::string &Path) { } void ReadDirToVectorOfUnits(const char *Path, std::vector *V, - long *Epoch, size_t MaxSize) { + long *Epoch, size_t MaxSize, bool ExitOnError) { long E = Epoch ? *Epoch : 0; std::vector Files; ListFilesInDirRecursive(Path, Epoch, &Files, /*TopDir*/true); @@ -108,7 +112,9 @@ void ReadDirToVectorOfUnits(const char *Path, std::vector *V, NumLoaded++; if ((NumLoaded & (NumLoaded - 1)) == 0 && NumLoaded >= 1024) Printf("Loaded %zd/%zd files from %s\n", NumLoaded, Files.size(), Path); - V->push_back(FileToVector(X, MaxSize)); + auto S = FileToVector(X, MaxSize, ExitOnError); + if (!S.empty()) + V->push_back(S); } } diff --git a/llvm/lib/Fuzzer/FuzzerLoop.cpp b/llvm/lib/Fuzzer/FuzzerLoop.cpp index be006ce..1b5e026 100644 --- a/llvm/lib/Fuzzer/FuzzerLoop.cpp +++ b/llvm/lib/Fuzzer/FuzzerLoop.cpp @@ -386,7 +386,8 @@ void Fuzzer::RereadOutputCorpus(size_t MaxSize) { if (Options.OutputCorpus.empty() || !Options.ReloadIntervalSec) return; std::vector AdditionalCorpus; ReadDirToVectorOfUnits(Options.OutputCorpus.c_str(), &AdditionalCorpus, - &EpochOfLastReadOfOutputCorpus, MaxSize); + &EpochOfLastReadOfOutputCorpus, MaxSize, + /*ExitOnError*/ false); if (Options.Verbosity >= 2) Printf("Reload: read %zd new units.\n", AdditionalCorpus.size()); bool Reloaded = false; @@ -605,9 +606,9 @@ void Fuzzer::Merge(const std::vector &Corpora) { assert(MaxInputLen > 0); UnitVector Initial, Extra; - ReadDirToVectorOfUnits(Corpora[0].c_str(), &Initial, nullptr, MaxInputLen); + ReadDirToVectorOfUnits(Corpora[0].c_str(), &Initial, nullptr, MaxInputLen, true); for (auto &C : ExtraCorpora) - ReadDirToVectorOfUnits(C.c_str(), &Extra, nullptr, MaxInputLen); + ReadDirToVectorOfUnits(C.c_str(), &Extra, nullptr, MaxInputLen, true); if (!Initial.empty()) { Printf("=== Minimizing the initial corpus of %zd units\n", Initial.size()); @@ -685,8 +686,8 @@ void Fuzzer::MutateAndTestOne() { StartTraceRecording(); II.NumExecutedMutations++; if (size_t NumFeatures = RunOne(CurrentUnitData, Size)) { - Corpus.AddToCorpus({CurrentUnitData, CurrentUnitData + Size}, - NumFeatures); + Corpus.AddToCorpus({CurrentUnitData, CurrentUnitData + Size}, NumFeatures, + /*MayDeleteFile=*/true); ReportNewCoverage(&II, {CurrentUnitData, CurrentUnitData + Size}); CheckExitOnItem(); } @@ -711,7 +712,7 @@ void Fuzzer::Loop() { if (duration_cast(Now - LastCorpusReload).count() >= Options.ReloadIntervalSec) { RereadOutputCorpus(MaxInputLen); - LastCorpusReload = Now; + LastCorpusReload = system_clock::now(); } if (TotalNumberOfRuns >= Options.MaxNumberOfRuns) break; diff --git a/llvm/lib/Fuzzer/build.sh b/llvm/lib/Fuzzer/build.sh index 3cbe39d..27c148a 100755 --- a/llvm/lib/Fuzzer/build.sh +++ b/llvm/lib/Fuzzer/build.sh @@ -1,7 +1,7 @@ #!/bin/bash LIBFUZZER_SRC_DIR=$(dirname $0) for f in $LIBFUZZER_SRC_DIR/*.cpp; do - clang -g -O2 -std=c++11 $f -c & + clang -g -O2 -fno-omit-frame-pointer -std=c++11 $f -c & done wait rm -f libFuzzer.a diff --git a/llvm/lib/Fuzzer/test/FuzzerUnittest.cpp b/llvm/lib/Fuzzer/test/FuzzerUnittest.cpp index ed37055..118ae57 100644 --- a/llvm/lib/Fuzzer/test/FuzzerUnittest.cpp +++ b/llvm/lib/Fuzzer/test/FuzzerUnittest.cpp @@ -579,7 +579,7 @@ TEST(FuzzerUtil, Base64) { TEST(Corpus, Distribution) { Random Rand(0); - InputCorpus C; + InputCorpus C(""); size_t N = 10; size_t TriesPerUnit = 1<<20; for (size_t i = 0; i < N; i++) -- 2.7.4