From bf29d7729e8934602c70026b7e4c5f2ef441a934 Mon Sep 17 00:00:00 2001 From: "R. Elliott Childre" Date: Mon, 19 Feb 2024 08:24:16 -0500 Subject: [PATCH] Update libfuzzer code to compiler-rt 18 Update the LLVM compiler-rt C++ code to the top of the 18.x branch The summary of the changes from compiler-rt 15 (the last time this code was updated) are: 1. Improvements to Ctrl+C detection and closing out of a fuzzing session 2. Loop optimizations, in preventing copy operations 3. Dynamic pagesize detection 4. Setting Worker thread names (on Linux) 5. Use of a bitset data structure to track rarity of features "improves fuzzer throughput rather dramatically (close to 2x) in early exploratory phases" The LLVM commits which make up the changes in this commit come from the squash of the following LLVM commits: ```sh git log --format=reference origin/release/15.x..release/18.x -- compiler-rt/lib/fuzzer/ ``` ``` 3747cde5e84f ([Fuzzer] Enable custom libc++ for Android (#70407), 2023-11-02) 747e0d9f0aad ([compiler-rt] Use std::clamp (NFC), 2023-10-22) bede46f57eec ([Fuzzer] Optimize UpdateFeatureFrequency (#65288), 2023-09-07) d3440304b110 ([Fuzzer] Update build.sh to fix build errors (#65496), 2023-09-07) dd3aa26fc8e9 (Revert "[Fuzzer] SetThreadName implementation for Windows", 2023-08-28) 151e33c768a6 ([NFC][fuzzer] Cleanup SetThreadName after D156317, 2023-08-28) 45eb6026d979 ([fuzzer] Try to fix windows build after D156317, 2023-08-28) e0e8695a608f (Revert "[fuzzer] Try to fix windows build after D156317", 2023-08-28) d364597f8a47 ([fuzzer] Try to fix windows build after D156317, 2023-08-28) cf76ddcbeb10 ([Fuzzer] SetThreadName implementation for Windows, 2023-08-28) 52526065bc80 ([NFC][compiler-rt] Add missing space in libfuzzer -help docs, 2023-07-28) cb924ddca514 (Revert "[Fuzzer] SetThreadName implementation for Windows", 2023-07-28) 885275bff215 ([Fuzzer] SetThreadName implementation for Windows, 2023-07-27) f4aa7b5b8d90 (Revert "[Fuzzer] SetThreadName implementation for Windows", 2023-07-27) e3f935c7f80c ([Fuzzer] SetThreadName implementation for Windows, 2023-07-27) b2a253855f4e ([Fuzzer] Assign names to workers, 2023-07-26) 9c2f792dceb6 ([fuzzer] Enable loongarch64, 2023-07-18) 42564f97e8ca ([fuzzer][fuchsia] Support RISC-V, 2023-06-15) facf22b8b07b ([fuzzer] Platfom specific version of PageSize, 2023-05-25) baa1488c1693 ([fuzzer] Don't hard-code page size in FuzzerUtil.h, 2023-05-25) cb9f2de2e802 (Revert "[fuzzer] Don't hard-code page size in FuzzerUtil.h", 2023-05-25) a2b677e81537 ([fuzzer] Don't hard-code page size in FuzzerUtil.h, 2023-05-25) f98ee40f4b5d ([NFC][Py Reformat] Reformat python files in the rest of the dirs, 2023-05-17) cdfcf1aa38a4 ([libfuzzer] avoid unneccessary auto-copy, 2023-04-26) 8a986abe98ff ([fuzzer] Fix -Wunreachable-code-aggressive warning in FuzzerDriver.cpp, 2023-04-11) f2c9d24e8c8d ([compiler-rt] [test] [fuzzer] Don't pass msvc/clang-cl specific flags to mingw tools, 2023-03-31) 90b4d1bcb201 ([fuzzer] Use puts() rather than printf() in CopyFileToErr(), 2023-03-15) c2df1d8a6d1c ([libfuzzer] add test of cov file-id in control file, 2023-03-22) d02ff3d57831 (Revert "[fuzzer] Use puts() rather than printf() in CopyFileToErr()", 2023-03-17) 03aa02adb03c ([fuzzer] Use puts() rather than printf() in CopyFileToErr(), 2023-03-15) 0b327814d86c ([libfuzzer] Fix build error due to out-of-line definition of 'Fuzzer' does not match any declaration in 'fuzzer::Fuzzer' (NFC), 2023-03-10) 513d9b9f3d67 ([libfuzzer] avoid unneccessary copy, 2023-03-09) e09b3be411b0 ([fuzzer] Make sure we never delete Fuzzer, 2023-03-07) 67f5b05cdcdc (Add test for Flags.data_flow_trace, 2023-03-07) 91985c2ee3b3 (Use the right printf format specifiers, 2023-03-07) 6c485409de52 (Adding missing colon, 2023-01-19) beb3fa2d2efb (Revert "Reland "[compiler-rt][test] Heed COMPILER_RT_DEBUG when compiling unittests"", 2022-12-13) 255c3e3dcb06 (Reland "[compiler-rt][test] Heed COMPILER_RT_DEBUG when compiling unittests", 2022-12-13) 68f4ceaf9b4e (Revert "[compiler-rt][test] Heed COMPILER_RT_DEBUG when compiling unittests", 2022-10-05) 93b1256e38f6 ([compiler-rt][test] Heed COMPILER_RT_DEBUG when compiling unittests, 2022-10-05) 98c2754abd8f ([Libfuzzer] Include signal.h for SIGINT, 2022-09-04) aa0e9046c16b (Libfuzzer fix for Ctrl + c not working with -fork and -ignore_crashes=1, 2022-08-15) ``` --- CHANGELOG.md | 2 +- libfuzzer/CMakeLists.txt | 4 +- libfuzzer/FuzzerCommand.h | 3 +- libfuzzer/FuzzerCorpus.h | 10 +- libfuzzer/FuzzerDataFlowTrace.cpp | 2 +- libfuzzer/FuzzerDriver.cpp | 20 ++-- libfuzzer/FuzzerFlags.def | 2 +- libfuzzer/FuzzerFork.cpp | 2 +- libfuzzer/FuzzerIO.cpp | 7 +- libfuzzer/FuzzerIO.h | 1 + libfuzzer/FuzzerInternal.h | 8 +- libfuzzer/FuzzerLoop.cpp | 22 +++-- libfuzzer/FuzzerMerge.cpp | 8 +- libfuzzer/FuzzerMutate.cpp | 2 +- libfuzzer/FuzzerTracePC.cpp | 4 +- libfuzzer/FuzzerUtil.h | 5 +- libfuzzer/FuzzerUtilDarwin.cpp | 5 + libfuzzer/FuzzerUtilFuchsia.cpp | 62 ++++++++++++- libfuzzer/FuzzerUtilLinux.cpp | 12 +++ libfuzzer/FuzzerUtilPosix.cpp | 5 + libfuzzer/FuzzerUtilWindows.cpp | 14 +++ libfuzzer/build.sh | 4 +- libfuzzer/scripts/unbalanced_allocs.py | 124 +++++++++++++------------ libfuzzer/tests/CMakeLists.txt | 2 +- 24 files changed, 226 insertions(+), 104 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c30095..83c8894 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ Released YYYY-MM-DD. ### Changed -* TODO (or remove section if none) +* Updated to `libFuzzer` commit `3747cde5e84f` (`release/18.x`). ### Deprecated diff --git a/libfuzzer/CMakeLists.txt b/libfuzzer/CMakeLists.txt index a9a10f7..fb5adf1 100644 --- a/libfuzzer/CMakeLists.txt +++ b/libfuzzer/CMakeLists.txt @@ -59,7 +59,7 @@ CHECK_CXX_SOURCE_COMPILES(" set(LIBFUZZER_CFLAGS ${COMPILER_RT_COMMON_CFLAGS}) -if(OS_NAME MATCHES "Linux|Fuchsia" AND +if(OS_NAME MATCHES "Android|Linux|Fuchsia" AND COMPILER_RT_LIBCXX_PATH AND COMPILER_RT_LIBCXXABI_PATH) list(APPEND LIBFUZZER_CFLAGS -D_LIBCPP_ABI_VERSION=Fuzzer) @@ -135,7 +135,7 @@ add_compiler_rt_runtime(clang_rt.fuzzer_interceptors CFLAGS ${LIBFUZZER_CFLAGS} PARENT_TARGET fuzzer) -if(OS_NAME MATCHES "Linux|Fuchsia" AND +if(OS_NAME MATCHES "Android|Linux|Fuchsia" AND COMPILER_RT_LIBCXX_PATH AND COMPILER_RT_LIBCXXABI_PATH) macro(partially_link_libcxx name dir arch) diff --git a/libfuzzer/FuzzerCommand.h b/libfuzzer/FuzzerCommand.h index f653fe3..718d7e9 100644 --- a/libfuzzer/FuzzerCommand.h +++ b/libfuzzer/FuzzerCommand.h @@ -19,6 +19,7 @@ #include #include #include +#include namespace fuzzer { @@ -139,7 +140,7 @@ class Command final { // be the equivalent command line. std::string toString() const { std::stringstream SS; - for (auto arg : getArguments()) + for (const auto &arg : getArguments()) SS << arg << " "; if (hasOutputFile()) SS << ">" << getOutputFile() << " "; diff --git a/libfuzzer/FuzzerCorpus.h b/libfuzzer/FuzzerCorpus.h index e01891e..48b5a2c 100644 --- a/libfuzzer/FuzzerCorpus.h +++ b/libfuzzer/FuzzerCorpus.h @@ -18,6 +18,7 @@ #include "FuzzerSHA1.h" #include "FuzzerTracePC.h" #include +#include #include #include #include @@ -77,7 +78,7 @@ struct InputInfo { SumIncidence = 0.0; // Apply add-one smoothing to locally discovered features. - for (auto F : FeatureFreqs) { + for (const auto &F : FeatureFreqs) { double LocalIncidence = F.second + 1; Energy -= LocalIncidence * log(LocalIncidence); SumIncidence += LocalIncidence; @@ -382,6 +383,7 @@ class InputCorpus { } // Remove most abundant rare feature. + IsRareFeature[Delete] = false; RareFeatures[Delete] = RareFeatures.back(); RareFeatures.pop_back(); @@ -397,6 +399,7 @@ class InputCorpus { // Add rare feature, handle collisions, and update energy. RareFeatures.push_back(Idx); + IsRareFeature[Idx] = true; GlobalFeatureFreqs[Idx] = 0; for (auto II : Inputs) { II->DeleteFeatureFreq(Idx); @@ -450,9 +453,7 @@ class InputCorpus { uint16_t Freq = GlobalFeatureFreqs[Idx32]++; // Skip if abundant. - if (Freq > FreqOfMostAbundantRareFeature || - std::find(RareFeatures.begin(), RareFeatures.end(), Idx32) == - RareFeatures.end()) + if (Freq > FreqOfMostAbundantRareFeature || !IsRareFeature[Idx32]) return; // Update global frequencies. @@ -581,6 +582,7 @@ class InputCorpus { uint16_t FreqOfMostAbundantRareFeature = 0; uint16_t GlobalFeatureFreqs[kFeatureSetSize] = {}; std::vector RareFeatures; + std::bitset IsRareFeature; std::string OutputCorpus; }; diff --git a/libfuzzer/FuzzerDataFlowTrace.cpp b/libfuzzer/FuzzerDataFlowTrace.cpp index 2f9a4d2..93bf817 100644 --- a/libfuzzer/FuzzerDataFlowTrace.cpp +++ b/libfuzzer/FuzzerDataFlowTrace.cpp @@ -88,7 +88,7 @@ bool BlockCoverage::AppendCoverage(std::istream &IN) { // * a function with a less frequently executed code gets bigger weight. std::vector BlockCoverage::FunctionWeights(size_t NumFunctions) const { std::vector Res(NumFunctions); - for (auto It : Functions) { + for (const auto &It : Functions) { auto FunctionID = It.first; auto Counters = It.second; assert(FunctionID < NumFunctions); diff --git a/libfuzzer/FuzzerDriver.cpp b/libfuzzer/FuzzerDriver.cpp index 6b007f2..8674d78 100644 --- a/libfuzzer/FuzzerDriver.cpp +++ b/libfuzzer/FuzzerDriver.cpp @@ -293,9 +293,12 @@ static int RunInMultipleProcesses(const std::vector &Args, std::vector V; std::thread Pulse(PulseThread); Pulse.detach(); - for (unsigned i = 0; i < NumWorkers; i++) - V.push_back(std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs, - &HasErrors)); + V.resize(NumWorkers); + for (unsigned i = 0; i < NumWorkers; i++) { + V[i] = std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs, + &HasErrors); + SetThreadName(V[i], "FuzzerWorker"); + } for (auto &T : V) T.join(); return HasErrors ? 1 : 0; @@ -463,7 +466,7 @@ int MinimizeCrashInput(const std::vector &Args, CurrentFilePath = Flags.exact_artifact_path; WriteToFile(U, CurrentFilePath); } - Printf("CRASH_MIN: failed to minimize beyond %s (%d bytes), exiting\n", + Printf("CRASH_MIN: failed to minimize beyond %s (%zu bytes), exiting\n", CurrentFilePath.c_str(), U.size()); break; } @@ -501,7 +504,6 @@ int MinimizeCrashInputInternalStep(Fuzzer *F, InputCorpus *Corpus) { F->MinimizeCrashLoop(U); Printf("INFO: Done MinimizeCrashInputInternalStep, no crashes found\n"); exit(0); - return 0; } void Merge(Fuzzer *F, FuzzingOptions &Options, @@ -535,7 +537,7 @@ void Merge(Fuzzer *F, FuzzingOptions &Options, int AnalyzeDictionary(Fuzzer *F, const std::vector &Dict, UnitVector &Corpus) { - Printf("Started dictionary minimization (up to %d tests)\n", + Printf("Started dictionary minimization (up to %zu tests)\n", Dict.size() * Corpus.size() * 2); // Scores and usage count for each dictionary unit. @@ -779,7 +781,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { if (!Options.FocusFunction.empty()) Options.Entropic = false; // FocusFunction overrides entropic scheduling. if (Options.Entropic) - Printf("INFO: Running with entropic power schedule (0x%X, %d).\n", + Printf("INFO: Running with entropic power schedule (0x%zX, %zu).\n", Options.EntropicFeatureFrequencyThreshold, Options.EntropicNumberOfRarestFeatures); struct EntropicOptions Entropic; @@ -797,7 +799,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { if (Flags.verbosity) Printf("INFO: Seed: %u\n", Seed); - if (Flags.collect_data_flow && !Flags.fork && + if (Flags.collect_data_flow && Flags.data_flow_trace && !Flags.fork && !(Flags.merge || Flags.set_cover_merge)) { if (RunIndividualFiles) return CollectDataFlow(Flags.collect_data_flow, Flags.data_flow_trace, @@ -860,7 +862,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { RunOneTest(F, Path.c_str(), Options.MaxLen); auto StopTime = system_clock::now(); auto MS = duration_cast(StopTime - StartTime).count(); - Printf("Executed %s in %zd ms\n", Path.c_str(), (long)MS); + Printf("Executed %s in %ld ms\n", Path.c_str(), (long)MS); } Printf("***\n" "*** NOTE: fuzzing was not performed, you have only\n" diff --git a/libfuzzer/FuzzerFlags.def b/libfuzzer/FuzzerFlags.def index 1181534..fc3b3aa 100644 --- a/libfuzzer/FuzzerFlags.def +++ b/libfuzzer/FuzzerFlags.def @@ -167,7 +167,7 @@ FUZZER_FLAG_INT(purge_allocator_interval, 1, "Purge allocator caches and " "purge_allocator_interval=-1 to disable this functionality.") FUZZER_FLAG_INT(trace_malloc, 0, "If >= 1 will print all mallocs/frees. " "If >= 2 will also print stack traces.") -FUZZER_FLAG_INT(rss_limit_mb, 2048, "If non-zero, the fuzzer will exit upon" +FUZZER_FLAG_INT(rss_limit_mb, 2048, "If non-zero, the fuzzer will exit upon " "reaching this limit of RSS memory usage.") FUZZER_FLAG_INT(malloc_limit_mb, 0, "If non-zero, the fuzzer will exit " "if the target tries to allocate this number of Mb with one malloc call. " diff --git a/libfuzzer/FuzzerFork.cpp b/libfuzzer/FuzzerFork.cpp index d59d513..c248a1d 100644 --- a/libfuzzer/FuzzerFork.cpp +++ b/libfuzzer/FuzzerFork.cpp @@ -220,7 +220,7 @@ struct GlobalEnv { } } // if (!FilesToAdd.empty() || Job->ExitCode != 0) - Printf("#%zd: cov: %zd ft: %zd corp: %zd exec/s %zd " + Printf("#%zd: cov: %zd ft: %zd corp: %zd exec/s: %zd " "oom/timeout/crash: %zd/%zd/%zd time: %zds job: %zd dft_time: %d\n", NumRuns, Cov.size(), Features.size(), Files.size(), Stats.average_exec_per_sec, NumOOMs, NumTimeouts, NumCrashes, diff --git a/libfuzzer/FuzzerIO.cpp b/libfuzzer/FuzzerIO.cpp index 0a58c53..54cc4ee 100644 --- a/libfuzzer/FuzzerIO.cpp +++ b/libfuzzer/FuzzerIO.cpp @@ -65,7 +65,7 @@ std::string FileToString(const std::string &Path) { } void CopyFileToErr(const std::string &Path) { - Printf("%s", FileToString(Path).c_str()); + Puts(FileToString(Path).c_str()); } void WriteToFile(const Unit &U, const std::string &Path) { @@ -151,6 +151,11 @@ void CloseStdout() { DiscardOutput(1); } +void Puts(const char *Str) { + fputs(Str, OutputFile); + fflush(OutputFile); +} + void Printf(const char *Fmt, ...) { va_list ap; va_start(ap, Fmt); diff --git a/libfuzzer/FuzzerIO.h b/libfuzzer/FuzzerIO.h index 401afa0..874caad 100644 --- a/libfuzzer/FuzzerIO.h +++ b/libfuzzer/FuzzerIO.h @@ -58,6 +58,7 @@ void CloseStdout(); FILE *GetOutputFile(); void SetOutputFile(FILE *NewOutputFile); +void Puts(const char *Str); void Printf(const char *Fmt, ...); void VPrintf(bool Verbose, const char *Fmt, ...); diff --git a/libfuzzer/FuzzerInternal.h b/libfuzzer/FuzzerInternal.h index 31f54ea..8850470 100644 --- a/libfuzzer/FuzzerInternal.h +++ b/libfuzzer/FuzzerInternal.h @@ -29,12 +29,11 @@ namespace fuzzer { using namespace std::chrono; -class Fuzzer { +class Fuzzer final { public: - Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, - FuzzingOptions Options); - ~Fuzzer(); + const FuzzingOptions &Options); + ~Fuzzer() = delete; void Loop(std::vector &CorporaFiles); void ReadAndExecuteSeedCorpora(std::vector &CorporaFiles); void MinimizeCrashLoop(const Unit &U); @@ -91,6 +90,7 @@ class Fuzzer { void HandleMalloc(size_t Size); static void MaybeExitGracefully(); + static int InterruptExitCode(); std::string WriteToOutputCorpus(const Unit &U); private: diff --git a/libfuzzer/FuzzerLoop.cpp b/libfuzzer/FuzzerLoop.cpp index f095757..935dd23 100644 --- a/libfuzzer/FuzzerLoop.cpp +++ b/libfuzzer/FuzzerLoop.cpp @@ -136,7 +136,7 @@ void Fuzzer::HandleMalloc(size_t Size) { } Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, - FuzzingOptions Options) + const FuzzingOptions &Options) : CB(CB), Corpus(Corpus), MD(MD), Options(Options) { if (EF->__sanitizer_set_death_callback) EF->__sanitizer_set_death_callback(StaticDeathCallback); @@ -160,8 +160,6 @@ Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, memset(BaseSha1, 0, sizeof(BaseSha1)); } -Fuzzer::~Fuzzer() {} - void Fuzzer::AllocateCurrentUnitData() { if (CurrentUnitData || MaxInputLen == 0) return; @@ -262,6 +260,11 @@ void Fuzzer::MaybeExitGracefully() { _Exit(0); } +int Fuzzer::InterruptExitCode() { + assert(F); + return F->Options.InterruptExitCode; +} + void Fuzzer::InterruptCallback() { Printf("==%lu== libFuzzer: run interrupted; exiting\n", GetPid()); PrintFinalStats(); @@ -296,7 +299,7 @@ void Fuzzer::AlarmCallback() { Printf(" and the timeout value is %d (use -timeout=N to change)\n", Options.UnitTimeoutSec); DumpCurrentUnit("timeout-"); - Printf("==%lu== ERROR: libFuzzer: timeout after %d seconds\n", GetPid(), + Printf("==%lu== ERROR: libFuzzer: timeout after %zu seconds\n", GetPid(), Seconds); PrintStackTrace(); Printf("SUMMARY: libFuzzer: timeout\n"); @@ -309,9 +312,8 @@ void Fuzzer::RssLimitCallback() { if (EF->__sanitizer_acquire_crash_state && !EF->__sanitizer_acquire_crash_state()) return; - Printf( - "==%lu== ERROR: libFuzzer: out-of-memory (used: %zdMb; limit: %zdMb)\n", - GetPid(), GetPeakRSSMb(), Options.RssLimitMb); + Printf("==%lu== ERROR: libFuzzer: out-of-memory (used: %zdMb; limit: %dMb)\n", + GetPid(), GetPeakRSSMb(), Options.RssLimitMb); Printf(" To change the out-of-memory limit use -rss_limit_mb=\n\n"); PrintMemoryProfile(); DumpCurrentUnit("oom-"); @@ -366,7 +368,7 @@ void Fuzzer::PrintFinalStats() { Printf("stat::number_of_executed_units: %zd\n", TotalNumberOfRuns); Printf("stat::average_exec_per_sec: %zd\n", ExecPerSec); Printf("stat::new_units_added: %zd\n", NumberOfNewUnitsAdded); - Printf("stat::slowest_unit_time_sec: %zd\n", TimeOfLongestUnitInSeconds); + Printf("stat::slowest_unit_time_sec: %ld\n", TimeOfLongestUnitInSeconds); Printf("stat::peak_rss_mb: %zd\n", GetPeakRSSMb()); } @@ -450,7 +452,7 @@ void Fuzzer::PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size) { static_cast(static_cast(TimeOfLongestUnitInSeconds) * 1.1); if (TimeOfUnit > Threshhold && TimeOfUnit >= Options.ReportSlowUnits) { TimeOfLongestUnitInSeconds = TimeOfUnit; - Printf("Slowest unit: %zd s:\n", TimeOfLongestUnitInSeconds); + Printf("Slowest unit: %ld s:\n", TimeOfLongestUnitInSeconds); WriteUnitToFileWithPrefix({Data, Data + Size}, "slow-unit-"); } } @@ -797,7 +799,7 @@ void Fuzzer::ReadAndExecuteSeedCorpora(std::vector &CorporaFiles) { TotalSize += File.Size; } if (Options.MaxLen == 0) - SetMaxInputLen(std::min(std::max(kMinDefaultLen, MaxSize), kMaxSaneLen)); + SetMaxInputLen(std::clamp(MaxSize, kMinDefaultLen, kMaxSaneLen)); assert(MaxInputLen > 0); // Test the callback with empty input and never try it again. diff --git a/libfuzzer/FuzzerMerge.cpp b/libfuzzer/FuzzerMerge.cpp index 24bd119..8c8806e 100644 --- a/libfuzzer/FuzzerMerge.cpp +++ b/libfuzzer/FuzzerMerge.cpp @@ -77,6 +77,7 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { size_t ExpectedStartMarker = 0; const size_t kInvalidStartMarker = -1; size_t LastSeenStartMarker = kInvalidStartMarker; + bool HaveFtMarker = true; std::vector TmpFeatures; std::set PCs; while (std::getline(IS, Line, '\n')) { @@ -93,12 +94,13 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { LastSeenStartMarker = ExpectedStartMarker; assert(ExpectedStartMarker < Files.size()); ExpectedStartMarker++; + HaveFtMarker = false; } else if (Marker == "FT") { // FT FILE_ID COV1 COV2 COV3 ... size_t CurrentFileIdx = N; if (CurrentFileIdx != LastSeenStartMarker) return false; - LastSeenStartMarker = kInvalidStartMarker; + HaveFtMarker = true; if (ParseCoverage) { TmpFeatures.clear(); // use a vector from outer scope to avoid resizes. while (ISS1 >> N) @@ -108,6 +110,8 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { } } else if (Marker == "COV") { size_t CurrentFileIdx = N; + if (CurrentFileIdx != LastSeenStartMarker) + return false; if (ParseCoverage) while (ISS1 >> N) if (PCs.insert(N).second) @@ -116,7 +120,7 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { return false; } } - if (LastSeenStartMarker != kInvalidStartMarker) + if (!HaveFtMarker && LastSeenStartMarker != kInvalidStartMarker) LastFailure = Files[LastSeenStartMarker].Name; FirstNotProcessedFile = ExpectedStartMarker; diff --git a/libfuzzer/FuzzerMutate.cpp b/libfuzzer/FuzzerMutate.cpp index d663900..1abce16 100644 --- a/libfuzzer/FuzzerMutate.cpp +++ b/libfuzzer/FuzzerMutate.cpp @@ -521,7 +521,7 @@ void MutationDispatcher::PrintMutationSequence(bool Verbose) { std::string MutationDispatcher::MutationSequence() { std::string MS; - for (auto M : CurrentMutatorSequence) { + for (const auto &M : CurrentMutatorSequence) { MS += M.Name; MS += "-"; } diff --git a/libfuzzer/FuzzerTracePC.cpp b/libfuzzer/FuzzerTracePC.cpp index f12f7aa..7f4e8ef 100644 --- a/libfuzzer/FuzzerTracePC.cpp +++ b/libfuzzer/FuzzerTracePC.cpp @@ -149,8 +149,8 @@ inline ALWAYS_INLINE uintptr_t GetPreviousInstructionPc(uintptr_t PC) { ALWAYS_INLINE uintptr_t TracePC::GetNextInstructionPc(uintptr_t PC) { #if defined(__mips__) return PC + 8; -#elif defined(__powerpc__) || defined(__sparc__) || defined(__arm__) || \ - defined(__aarch64__) +#elif defined(__powerpc__) || defined(__sparc__) || defined(__arm__) || \ + defined(__aarch64__) || defined(__loongarch__) return PC + 4; #else return PC + 1; diff --git a/libfuzzer/FuzzerUtil.h b/libfuzzer/FuzzerUtil.h index 71d4909..554567e 100644 --- a/libfuzzer/FuzzerUtil.h +++ b/libfuzzer/FuzzerUtil.h @@ -59,6 +59,8 @@ size_t GetPeakRSSMb(); int ExecuteCommand(const Command &Cmd); bool ExecuteCommand(const Command &Cmd, std::string *CmdOutput); +void SetThreadName(std::thread &thread, const std::string &name); + // Fuchsia does not have popen/pclose. FILE *OpenProcessPipe(const char *Command, const char *Mode); int CloseProcessPipe(FILE *F); @@ -94,7 +96,8 @@ inline size_t Log(size_t X) { return static_cast((sizeof(unsigned long long) * 8) - Clzll(X) - 1); } -inline size_t PageSize() { return 4096; } +size_t PageSize(); + inline uint8_t *RoundUpByPage(uint8_t *P) { uintptr_t X = reinterpret_cast(P); size_t Mask = PageSize() - 1; diff --git a/libfuzzer/FuzzerUtilDarwin.cpp b/libfuzzer/FuzzerUtilDarwin.cpp index a5bed65..6c3ece3 100644 --- a/libfuzzer/FuzzerUtilDarwin.cpp +++ b/libfuzzer/FuzzerUtilDarwin.cpp @@ -165,6 +165,11 @@ void DiscardOutput(int Fd) { fclose(Temp); } +void SetThreadName(std::thread &thread, const std::string &name) { + // TODO ? + // Darwin allows to set the name only on the current thread it seems +} + } // namespace fuzzer #endif // LIBFUZZER_APPLE diff --git a/libfuzzer/FuzzerUtilFuchsia.cpp b/libfuzzer/FuzzerUtilFuchsia.cpp index d80b80c..cfb81cd 100644 --- a/libfuzzer/FuzzerUtilFuchsia.cpp +++ b/libfuzzer/FuzzerUtilFuchsia.cpp @@ -87,6 +87,7 @@ void AlarmHandler(int Seconds) { // Alternatively, Fuchsia may in future actually implement basic signal // handling for the machine trap signals. #if defined(__x86_64__) + #define FOREACH_REGISTER(OP_REG, OP_NUM) \ OP_REG(rax) \ OP_REG(rbx) \ @@ -107,6 +108,7 @@ void AlarmHandler(int Seconds) { OP_REG(rip) #elif defined(__aarch64__) + #define FOREACH_REGISTER(OP_REG, OP_NUM) \ OP_NUM(0) \ OP_NUM(1) \ @@ -140,6 +142,41 @@ void AlarmHandler(int Seconds) { OP_NUM(29) \ OP_REG(sp) +#elif defined(__riscv) + +#define FOREACH_REGISTER(OP_REG, OP_NUM) \ + OP_REG(ra) \ + OP_REG(sp) \ + OP_REG(gp) \ + OP_REG(tp) \ + OP_REG(t0) \ + OP_REG(t1) \ + OP_REG(t2) \ + OP_REG(s0) \ + OP_REG(s1) \ + OP_REG(a0) \ + OP_REG(a1) \ + OP_REG(a2) \ + OP_REG(a3) \ + OP_REG(a4) \ + OP_REG(a5) \ + OP_REG(a6) \ + OP_REG(a7) \ + OP_REG(s2) \ + OP_REG(s3) \ + OP_REG(s4) \ + OP_REG(s5) \ + OP_REG(s6) \ + OP_REG(s7) \ + OP_REG(s8) \ + OP_REG(s9) \ + OP_REG(s10) \ + OP_REG(s11) \ + OP_REG(t3) \ + OP_REG(t4) \ + OP_REG(t5) \ + OP_REG(t6) \ + #else #error "Unsupported architecture for fuzzing on Fuchsia" #endif @@ -200,6 +237,13 @@ void MakeTrampoline() { ".cfi_offset 30, %c[lr]\n" "bl %c[StaticCrashHandler]\n" "brk 1\n" +#elif defined(__riscv) + ".cfi_return_column 64\n" + ".cfi_def_cfa sp, 0\n" + ".cfi_offset 64, %[pc]\n" + FOREACH_REGISTER(CFI_OFFSET_REG, CFI_OFFSET_NUM) + "call %c[StaticCrashHandler]\n" + "unimp\n" #else #error "Unsupported architecture for fuzzing on Fuchsia" #endif @@ -209,8 +253,11 @@ void MakeTrampoline() { ".cfi_startproc\n" : // No outputs : FOREACH_REGISTER(ASM_OPERAND_REG, ASM_OPERAND_NUM) +#if defined(__aarch64__) || defined(__riscv) + ASM_OPERAND_REG(pc) +#endif #if defined(__aarch64__) - ASM_OPERAND_REG(pc) ASM_OPERAND_REG(lr) + ASM_OPERAND_REG(lr) #endif [StaticCrashHandler] "i"(StaticCrashHandler)); } @@ -294,6 +341,7 @@ void CrashHandler() { // onto the stack and jump into a trampoline with CFI instructions on how // to restore it. #if defined(__x86_64__) + uintptr_t StackPtr = (GeneralRegisters.rsp - (128 + sizeof(GeneralRegisters))) & -(uintptr_t)16; @@ -302,7 +350,8 @@ void CrashHandler() { GeneralRegisters.rsp = StackPtr; GeneralRegisters.rip = reinterpret_cast(CrashTrampolineAsm); -#elif defined(__aarch64__) +#elif defined(__aarch64__) || defined(__riscv) + uintptr_t StackPtr = (GeneralRegisters.sp - sizeof(GeneralRegisters)) & -(uintptr_t)16; __unsanitized_memcpy(reinterpret_cast(StackPtr), &GeneralRegisters, @@ -551,6 +600,15 @@ void DiscardOutput(int Fd) { dup2(nullfd, Fd); } +size_t PageSize() { + static size_t PageSizeCached = _zx_system_get_page_size(); + return PageSizeCached; +} + +void SetThreadName(std::thread &thread, const std::string &name) { + // TODO ? +} + } // namespace fuzzer #endif // LIBFUZZER_FUCHSIA diff --git a/libfuzzer/FuzzerUtilLinux.cpp b/libfuzzer/FuzzerUtilLinux.cpp index 981f9a8..5729448 100644 --- a/libfuzzer/FuzzerUtilLinux.cpp +++ b/libfuzzer/FuzzerUtilLinux.cpp @@ -11,7 +11,9 @@ #if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FREEBSD || \ LIBFUZZER_EMSCRIPTEN #include "FuzzerCommand.h" +#include "FuzzerInternal.h" +#include #include #include #include @@ -25,6 +27,8 @@ int ExecuteCommand(const Command &Cmd) { int exit_code = system(CmdLine.c_str()); if (WIFEXITED(exit_code)) return WEXITSTATUS(exit_code); + if (WIFSIGNALED(exit_code) && WTERMSIG(exit_code) == SIGINT) + return Fuzzer::InterruptExitCode(); return exit_code; } @@ -36,6 +40,14 @@ void DiscardOutput(int Fd) { fclose(Temp); } +void SetThreadName(std::thread &thread, const std::string &name) { +#if LIBFUZZER_LINUX || LIBFUZZER_FREEBSD + (void)pthread_setname_np(thread.native_handle(), name.c_str()); +#elif LIBFUZZER_NETBSD + (void)pthread_set_name_np(thread.native_handle(), "%s", name.c_str()); +#endif +} + } // namespace fuzzer #endif diff --git a/libfuzzer/FuzzerUtilPosix.cpp b/libfuzzer/FuzzerUtilPosix.cpp index 0446d73..392c1e5 100644 --- a/libfuzzer/FuzzerUtilPosix.cpp +++ b/libfuzzer/FuzzerUtilPosix.cpp @@ -183,6 +183,11 @@ std::string SearchRegexCmd(const std::string &Regex) { return "grep '" + Regex + "'"; } +size_t PageSize() { + static size_t PageSizeCached = sysconf(_SC_PAGESIZE); + return PageSizeCached; +} + } // namespace fuzzer #endif // LIBFUZZER_POSIX diff --git a/libfuzzer/FuzzerUtilWindows.cpp b/libfuzzer/FuzzerUtilWindows.cpp index 3598758..7177016 100644 --- a/libfuzzer/FuzzerUtilWindows.cpp +++ b/libfuzzer/FuzzerUtilWindows.cpp @@ -224,6 +224,20 @@ void DiscardOutput(int Fd) { fclose(Temp); } +size_t PageSize() { + static size_t PageSizeCached = []() -> size_t { + SYSTEM_INFO si; + GetSystemInfo(&si); + return si.dwPageSize; + }(); + return PageSizeCached; +} + +void SetThreadName(std::thread &thread, const std::string &name) { + // TODO ? + // to UTF-8 then SetThreadDescription ? +} + } // namespace fuzzer #endif // LIBFUZZER_WINDOWS diff --git a/libfuzzer/build.sh b/libfuzzer/build.sh index 504e54e..f7f329c 100755 --- a/libfuzzer/build.sh +++ b/libfuzzer/build.sh @@ -2,10 +2,10 @@ LIBFUZZER_SRC_DIR=$(dirname $0) CXX="${CXX:-clang}" for f in $LIBFUZZER_SRC_DIR/*.cpp; do - $CXX -g -O2 -fno-omit-frame-pointer -std=c++11 $f -c & + $CXX -g -O2 -fno-omit-frame-pointer -std=c++14 $f -c & done wait rm -f libFuzzer.a -ar ru libFuzzer.a Fuzzer*.o +ar r libFuzzer.a Fuzzer*.o rm -f Fuzzer*.o diff --git a/libfuzzer/scripts/unbalanced_allocs.py b/libfuzzer/scripts/unbalanced_allocs.py index 579e481..7ba7e09 100755 --- a/libfuzzer/scripts/unbalanced_allocs.py +++ b/libfuzzer/scripts/unbalanced_allocs.py @@ -1,92 +1,100 @@ #!/usr/bin/env python -#===- lib/fuzzer/scripts/unbalanced_allocs.py ------------------------------===# +# ===- lib/fuzzer/scripts/unbalanced_allocs.py ------------------------------===# # # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # -#===------------------------------------------------------------------------===# +# ===------------------------------------------------------------------------===# # # Post-process -trace_malloc=2 output and printout only allocations and frees # unbalanced inside of fuzzer runs. # Usage: # my_fuzzer -trace_malloc=2 -runs=10 2>&1 | unbalanced_allocs.py -skip=5 # -#===------------------------------------------------------------------------===# +# ===------------------------------------------------------------------------===# import argparse import sys _skip = 0 + def PrintStack(line, stack): - global _skip - if _skip > 0: - return - print('Unbalanced ' + line.rstrip()); - for l in stack: - print(l.rstrip()) + global _skip + if _skip > 0: + return + print("Unbalanced " + line.rstrip()) + for l in stack: + print(l.rstrip()) + def ProcessStack(line, f): - stack = [] - while line and line.startswith(' #'): - stack += [line] - line = f.readline() - return line, stack + stack = [] + while line and line.startswith(" #"): + stack += [line] + line = f.readline() + return line, stack + def ProcessFree(line, f, allocs): - if not line.startswith('FREE['): - return f.readline() + if not line.startswith("FREE["): + return f.readline() + + addr = int(line.split()[1], 16) + next_line, stack = ProcessStack(f.readline(), f) + if addr in allocs: + del allocs[addr] + else: + PrintStack(line, stack) + return next_line - addr = int(line.split()[1], 16) - next_line, stack = ProcessStack(f.readline(), f) - if addr in allocs: - del allocs[addr] - else: - PrintStack(line, stack) - return next_line def ProcessMalloc(line, f, allocs): - if not line.startswith('MALLOC['): - return ProcessFree(line, f, allocs) + if not line.startswith("MALLOC["): + return ProcessFree(line, f, allocs) + + addr = int(line.split()[1], 16) + assert not addr in allocs - addr = int(line.split()[1], 16) - assert not addr in allocs + next_line, stack = ProcessStack(f.readline(), f) + allocs[addr] = (line, stack) + return next_line - next_line, stack = ProcessStack(f.readline(), f) - allocs[addr] = (line, stack) - return next_line def ProcessRun(line, f): - if not line.startswith('MallocFreeTracer: START'): - return ProcessMalloc(line, f, {}) - - allocs = {} - print(line.rstrip()) - line = f.readline() - while line: - if line.startswith('MallocFreeTracer: STOP'): - global _skip - _skip = _skip - 1 - for _, (l, s) in allocs.items(): - PrintStack(l, s) - print(line.rstrip()) - return f.readline() - line = ProcessMalloc(line, f, allocs) - return line + if not line.startswith("MallocFreeTracer: START"): + return ProcessMalloc(line, f, {}) + + allocs = {} + print(line.rstrip()) + line = f.readline() + while line: + if line.startswith("MallocFreeTracer: STOP"): + global _skip + _skip = _skip - 1 + for _, (l, s) in allocs.items(): + PrintStack(l, s) + print(line.rstrip()) + return f.readline() + line = ProcessMalloc(line, f, allocs) + return line + def ProcessFile(f): - line = f.readline() - while line: - line = ProcessRun(line, f); + line = f.readline() + while line: + line = ProcessRun(line, f) + def main(argv): - parser = argparse.ArgumentParser() - parser.add_argument('--skip', default=0, help='number of runs to ignore') - args = parser.parse_args() - global _skip - _skip = int(args.skip) + 1 - ProcessFile(sys.stdin) - -if __name__ == '__main__': - main(sys.argv) + parser = argparse.ArgumentParser() + parser.add_argument("--skip", default=0, help="number of runs to ignore") + args = parser.parse_args() + global _skip + _skip = int(args.skip) + 1 + ProcessFile(sys.stdin) + + +if __name__ == "__main__": + main(sys.argv) diff --git a/libfuzzer/tests/CMakeLists.txt b/libfuzzer/tests/CMakeLists.txt index 10fcfba..dd82c49 100644 --- a/libfuzzer/tests/CMakeLists.txt +++ b/libfuzzer/tests/CMakeLists.txt @@ -20,7 +20,7 @@ set_target_properties(FuzzedDataProviderUnitTests PROPERTIES FOLDER "Compiler-RT set(LIBFUZZER_UNITTEST_LINK_FLAGS ${COMPILER_RT_UNITTEST_LINK_FLAGS}) list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS --driver-mode=g++) -if(WIN32) +if(MSVC) list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS -Wl,-defaultlib:libcmt,-defaultlib:oldnames) else() if (APPLE)