diff options
Diffstat (limited to 'contrib/llvm-project/compiler-rt/lib/fuzzer')
39 files changed, 1006 insertions, 473 deletions
diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerBuiltins.h b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerBuiltins.h index 4c0ada82662d..ce0bd5cb47f4 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerBuiltins.h +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerBuiltins.h @@ -26,7 +26,6 @@ inline uint32_t Bswap(uint32_t x) { return __builtin_bswap32(x); } inline uint64_t Bswap(uint64_t x) { return __builtin_bswap64(x); } inline uint32_t Clzll(unsigned long long X) { return __builtin_clzll(X); } -inline uint32_t Clz(unsigned long long X) { return __builtin_clz(X); } inline int Popcountll(unsigned long long X) { return __builtin_popcountll(X); } } // namespace fuzzer diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerBuiltinsMsvc.h b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerBuiltinsMsvc.h index c5bec9787d8e..421dee7f6603 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerBuiltinsMsvc.h +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerBuiltinsMsvc.h @@ -41,7 +41,8 @@ inline uint32_t Clzll(uint64_t X) { #if !defined(_M_ARM) && !defined(_M_X64) // Scan the high 32 bits. if (_BitScanReverse(&LeadZeroIdx, static_cast<unsigned long>(X >> 32))) - return static_cast<int>(63 - (LeadZeroIdx + 32)); // Create a bit offset from the MSB. + return static_cast<int>( + 63 - (LeadZeroIdx + 32)); // Create a bit offset from the MSB. // Scan the low 32 bits. if (_BitScanReverse(&LeadZeroIdx, static_cast<unsigned long>(X))) return static_cast<int>(63 - LeadZeroIdx); @@ -52,12 +53,6 @@ inline uint32_t Clzll(uint64_t X) { return 64; } -inline uint32_t Clz(uint32_t X) { - unsigned long LeadZeroIdx = 0; - if (_BitScanReverse(&LeadZeroIdx, X)) return 31 - LeadZeroIdx; - return 32; -} - inline int Popcountll(unsigned long long X) { #if !defined(_M_ARM) && !defined(_M_X64) return __popcnt(X) + __popcnt(X >> 32); diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerCommand.h b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerCommand.h index 87308864af53..718d7e951fb1 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerCommand.h +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerCommand.h @@ -19,6 +19,7 @@ #include <sstream> #include <string> #include <vector> +#include <thread> namespace fuzzer { @@ -33,7 +34,7 @@ public: Command() : CombinedOutAndErr(false) {} - explicit Command(const Vector<std::string> &ArgsToAdd) + explicit Command(const std::vector<std::string> &ArgsToAdd) : Args(ArgsToAdd), CombinedOutAndErr(false) {} explicit Command(const Command &Other) @@ -58,7 +59,7 @@ public: // Gets all of the current command line arguments, **including** those after // "-ignore-remaining-args=1". - const Vector<std::string> &getArguments() const { return Args; } + const std::vector<std::string> &getArguments() const { return Args; } // Adds the given argument before "-ignore_remaining_args=1", or at the end // if that flag isn't present. @@ -68,7 +69,7 @@ public: // Adds all given arguments before "-ignore_remaining_args=1", or at the end // if that flag isn't present. - void addArguments(const Vector<std::string> &ArgsToAdd) { + void addArguments(const std::vector<std::string> &ArgsToAdd) { Args.insert(endMutableArgs(), ArgsToAdd.begin(), ArgsToAdd.end()); } @@ -139,7 +140,7 @@ public: // be the equivalent command line. std::string toString() const { std::stringstream SS; - for (auto arg : getArguments()) + for (const auto &arg : getArguments()) SS << arg << " "; if (hasOutputFile()) SS << ">" << getOutputFile() << " "; @@ -155,16 +156,16 @@ private: Command(Command &&Other) = delete; Command &operator=(Command &&Other) = delete; - Vector<std::string>::iterator endMutableArgs() { + std::vector<std::string>::iterator endMutableArgs() { return std::find(Args.begin(), Args.end(), ignoreRemainingArgs()); } - Vector<std::string>::const_iterator endMutableArgs() const { + std::vector<std::string>::const_iterator endMutableArgs() const { return std::find(Args.begin(), Args.end(), ignoreRemainingArgs()); } // The command arguments. Args[0] is the command name. - Vector<std::string> Args; + std::vector<std::string> Args; // True indicates stderr is redirected to stdout. bool CombinedOutAndErr; diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerCorpus.h b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerCorpus.h index daea4f5213b1..48b5a2cff02e 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerCorpus.h +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerCorpus.h @@ -18,6 +18,7 @@ #include "FuzzerSHA1.h" #include "FuzzerTracePC.h" #include <algorithm> +#include <bitset> #include <chrono> #include <numeric> #include <random> @@ -39,13 +40,13 @@ struct InputInfo { bool MayDeleteFile = false; bool Reduced = false; bool HasFocusFunction = false; - Vector<uint32_t> UniqFeatureSet; - Vector<uint8_t> DataFlowTraceForFocusFunction; + std::vector<uint32_t> UniqFeatureSet; + std::vector<uint8_t> DataFlowTraceForFocusFunction; // Power schedule. bool NeedsEnergyUpdate = false; double Energy = 0.0; - size_t SumIncidence = 0; - Vector<std::pair<uint32_t, uint16_t>> FeatureFreqs; + double SumIncidence = 0.0; + std::vector<std::pair<uint32_t, uint16_t>> FeatureFreqs; // Delete feature Idx and its frequency from FeatureFreqs. bool DeleteFeatureFreq(uint32_t Idx) { @@ -74,27 +75,28 @@ struct InputInfo { void UpdateEnergy(size_t GlobalNumberOfFeatures, bool ScalePerExecTime, std::chrono::microseconds AverageUnitExecutionTime) { Energy = 0.0; - SumIncidence = 0; + SumIncidence = 0.0; // Apply add-one smoothing to locally discovered features. - for (auto F : FeatureFreqs) { - size_t LocalIncidence = F.second + 1; - Energy -= LocalIncidence * logl(LocalIncidence); + for (const auto &F : FeatureFreqs) { + double LocalIncidence = F.second + 1; + Energy -= LocalIncidence * log(LocalIncidence); SumIncidence += LocalIncidence; } // Apply add-one smoothing to locally undiscovered features. - // PreciseEnergy -= 0; // since logl(1.0) == 0) - SumIncidence += (GlobalNumberOfFeatures - FeatureFreqs.size()); + // PreciseEnergy -= 0; // since log(1.0) == 0) + SumIncidence += + static_cast<double>(GlobalNumberOfFeatures - FeatureFreqs.size()); // Add a single locally abundant feature apply add-one smoothing. - size_t AbdIncidence = NumExecutedMutations + 1; - Energy -= AbdIncidence * logl(AbdIncidence); + double AbdIncidence = static_cast<double>(NumExecutedMutations + 1); + Energy -= AbdIncidence * log(AbdIncidence); SumIncidence += AbdIncidence; // Normalize. if (SumIncidence != 0) - Energy = (Energy / SumIncidence) + logl(SumIncidence); + Energy = Energy / SumIncidence + log(SumIncidence); if (ScalePerExecTime) { // Scaling to favor inputs with lower execution time. @@ -208,11 +210,13 @@ public: InputInfo *AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile, bool HasFocusFunction, bool NeverReduce, std::chrono::microseconds TimeOfUnit, - const Vector<uint32_t> &FeatureSet, + const std::vector<uint32_t> &FeatureSet, const DataFlowTrace &DFT, const InputInfo *BaseII) { assert(!U.empty()); if (FeatureDebug) Printf("ADD_TO_CORPUS %zd NF %zd\n", Inputs.size(), NumFeatures); + // Inputs.size() is cast to uint32_t below. + assert(Inputs.size() < std::numeric_limits<uint32_t>::max()); Inputs.push_back(new InputInfo()); InputInfo &II = *Inputs.back(); II.U = U; @@ -224,7 +228,7 @@ public: II.HasFocusFunction = HasFocusFunction; // Assign maximal energy to the new seed. II.Energy = RareFeatures.empty() ? 1.0 : log(RareFeatures.size()); - II.SumIncidence = RareFeatures.size(); + II.SumIncidence = static_cast<double>(RareFeatures.size()); II.NeedsEnergyUpdate = false; std::sort(II.UniqFeatureSet.begin(), II.UniqFeatureSet.end()); ComputeSHA1(U.data(), U.size(), II.Sha1); @@ -255,7 +259,7 @@ public: } // Debug-only - void PrintFeatureSet(const Vector<uint32_t> &FeatureSet) { + void PrintFeatureSet(const std::vector<uint32_t> &FeatureSet) { if (!FeatureDebug) return; Printf("{"); for (uint32_t Feature: FeatureSet) @@ -281,7 +285,8 @@ public: } } - void Replace(InputInfo *II, const Unit &U) { + void Replace(InputInfo *II, const Unit &U, + std::chrono::microseconds TimeOfUnit) { assert(II->U.size() > U.size()); Hashes.erase(Sha1ToString(II->Sha1)); DeleteFile(*II); @@ -289,6 +294,7 @@ public: Hashes.insert(Sha1ToString(II->Sha1)); II->U = U; II->Reduced = true; + II->TimeOfUnit = TimeOfUnit; DistributionNeedsUpdate = true; } @@ -322,7 +328,8 @@ public: const auto &II = *Inputs[i]; Printf(" [% 3zd %s] sz: % 5zd runs: % 5zd succ: % 5zd focus: %d\n", i, Sha1ToString(II.Sha1).c_str(), II.U.size(), - II.NumExecutedMutations, II.NumSuccessfullMutations, II.HasFocusFunction); + II.NumExecutedMutations, II.NumSuccessfullMutations, + II.HasFocusFunction); } } @@ -376,6 +383,7 @@ public: } // Remove most abundant rare feature. + IsRareFeature[Delete] = false; RareFeatures[Delete] = RareFeatures.back(); RareFeatures.pop_back(); @@ -391,6 +399,7 @@ public: // Add rare feature, handle collisions, and update energy. RareFeatures.push_back(Idx); + IsRareFeature[Idx] = true; GlobalFeatureFreqs[Idx] = 0; for (auto II : Inputs) { II->DeleteFeatureFreq(Idx); @@ -399,7 +408,7 @@ public: // Zero energy seeds will never be fuzzed and remain zero energy. if (II->Energy > 0.0) { II->SumIncidence += 1; - II->Energy += logl(II->SumIncidence) / II->SumIncidence; + II->Energy += log(II->SumIncidence) / II->SumIncidence; } } @@ -426,7 +435,8 @@ public: NumUpdatedFeatures++; if (FeatureDebug) Printf("ADD FEATURE %zd sz %d\n", Idx, NewSize); - SmallestElementPerFeature[Idx] = Inputs.size(); + // Inputs.size() is guaranteed to be less than UINT32_MAX by AddToCorpus. + SmallestElementPerFeature[Idx] = static_cast<uint32_t>(Inputs.size()); InputSizesPerFeature[Idx] = NewSize; return true; } @@ -443,9 +453,7 @@ public: uint16_t Freq = GlobalFeatureFreqs[Idx32]++; // Skip if abundant. - if (Freq > FreqOfMostAbundantRareFeature || - std::find(RareFeatures.begin(), RareFeatures.end(), Idx32) == - RareFeatures.end()) + if (Freq > FreqOfMostAbundantRareFeature || !IsRareFeature[Idx32]) return; // Update global frequencies. @@ -464,7 +472,7 @@ private: static const bool FeatureDebug = false; - size_t GetFeature(size_t Idx) const { return InputSizesPerFeature[Idx]; } + uint32_t GetFeature(size_t Idx) const { return InputSizesPerFeature[Idx]; } void ValidateFeatureSet() { if (FeatureDebug) @@ -539,9 +547,11 @@ private: if (VanillaSchedule) { for (size_t i = 0; i < N; i++) - Weights[i] = Inputs[i]->NumFeatures - ? (i + 1) * (Inputs[i]->HasFocusFunction ? 1000 : 1) - : 0.; + Weights[i] = + Inputs[i]->NumFeatures + ? static_cast<double>((i + 1) * + (Inputs[i]->HasFocusFunction ? 1000 : 1)) + : 0.; } if (FeatureDebug) { @@ -557,11 +567,11 @@ private: } std::piecewise_constant_distribution<double> CorpusDistribution; - Vector<double> Intervals; - Vector<double> Weights; + std::vector<double> Intervals; + std::vector<double> Weights; std::unordered_set<std::string> Hashes; - Vector<InputInfo*> Inputs; + std::vector<InputInfo *> Inputs; size_t NumAddedFeatures = 0; size_t NumUpdatedFeatures = 0; @@ -571,7 +581,8 @@ private: bool DistributionNeedsUpdate = true; uint16_t FreqOfMostAbundantRareFeature = 0; uint16_t GlobalFeatureFreqs[kFeatureSetSize] = {}; - Vector<uint32_t> RareFeatures; + std::vector<uint32_t> RareFeatures; + std::bitset<kFeatureSetSize> IsRareFeature; std::string OutputCorpus; }; diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.cpp b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.cpp index 0e9cdf7e66b1..93bf817a857b 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.cpp +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.cpp @@ -37,7 +37,7 @@ bool BlockCoverage::AppendCoverage(const std::string &S) { // Coverage lines have this form: // CN X Y Z T // where N is the number of the function, T is the total number of instrumented -// BBs, and X,Y,Z, if present, are the indecies of covered BB. +// BBs, and X,Y,Z, if present, are the indices of covered BB. // BB #0, which is the entry block, is not explicitly listed. bool BlockCoverage::AppendCoverage(std::istream &IN) { std::string L; @@ -52,7 +52,7 @@ bool BlockCoverage::AppendCoverage(std::istream &IN) { continue; } if (L[0] != 'C') continue; - Vector<uint32_t> CoveredBlocks; + std::vector<uint32_t> CoveredBlocks; while (true) { uint32_t BB = 0; SS >> BB; @@ -60,6 +60,7 @@ bool BlockCoverage::AppendCoverage(std::istream &IN) { CoveredBlocks.push_back(BB); } if (CoveredBlocks.empty()) return false; + // Ensures no CoverageVector is longer than UINT32_MAX. uint32_t NumBlocks = CoveredBlocks.back(); CoveredBlocks.pop_back(); for (auto BB : CoveredBlocks) @@ -67,7 +68,7 @@ bool BlockCoverage::AppendCoverage(std::istream &IN) { auto It = Functions.find(FunctionId); auto &Counters = It == Functions.end() - ? Functions.insert({FunctionId, Vector<uint32_t>(NumBlocks)}) + ? Functions.insert({FunctionId, std::vector<uint32_t>(NumBlocks)}) .first->second : It->second; @@ -85,9 +86,9 @@ bool BlockCoverage::AppendCoverage(std::istream &IN) { // * any uncovered function gets weight 0. // * a function with lots of uncovered blocks gets bigger weight. // * a function with a less frequently executed code gets bigger weight. -Vector<double> BlockCoverage::FunctionWeights(size_t NumFunctions) const { - Vector<double> Res(NumFunctions); - for (auto It : Functions) { +std::vector<double> BlockCoverage::FunctionWeights(size_t NumFunctions) const { + std::vector<double> Res(NumFunctions); + for (const auto &It : Functions) { auto FunctionID = It.first; auto Counters = It.second; assert(FunctionID < NumFunctions); @@ -103,7 +104,7 @@ Vector<double> BlockCoverage::FunctionWeights(size_t NumFunctions) const { } void DataFlowTrace::ReadCoverage(const std::string &DirPath) { - Vector<SizedFile> Files; + std::vector<SizedFile> Files; GetSizedFilesFromDir(DirPath, &Files); for (auto &SF : Files) { auto Name = Basename(SF.File); @@ -114,16 +115,16 @@ void DataFlowTrace::ReadCoverage(const std::string &DirPath) { } } -static void DFTStringAppendToVector(Vector<uint8_t> *DFT, +static void DFTStringAppendToVector(std::vector<uint8_t> *DFT, const std::string &DFTString) { assert(DFT->size() == DFTString.size()); for (size_t I = 0, Len = DFT->size(); I < Len; I++) (*DFT)[I] = DFTString[I] == '1'; } -// converts a string of '0' and '1' into a Vector<uint8_t> -static Vector<uint8_t> DFTStringToVector(const std::string &DFTString) { - Vector<uint8_t> DFT(DFTString.size()); +// converts a string of '0' and '1' into a std::vector<uint8_t> +static std::vector<uint8_t> DFTStringToVector(const std::string &DFTString) { + std::vector<uint8_t> DFT(DFTString.size()); DFTStringAppendToVector(&DFT, DFTString); return DFT; } @@ -158,14 +159,14 @@ static bool ParseDFTLine(const std::string &Line, size_t *FunctionNum, } bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, - Vector<SizedFile> &CorporaFiles, Random &Rand) { + std::vector<SizedFile> &CorporaFiles, Random &Rand) { if (DirPath.empty()) return false; Printf("INFO: DataFlowTrace: reading from '%s'\n", DirPath.c_str()); - Vector<SizedFile> Files; + std::vector<SizedFile> Files; GetSizedFilesFromDir(DirPath, &Files); std::string L; size_t FocusFuncIdx = SIZE_MAX; - Vector<std::string> FunctionNames; + std::vector<std::string> FunctionNames; // Collect the hashes of the corpus files. for (auto &SF : CorporaFiles) @@ -190,7 +191,7 @@ bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, // * chooses a random function according to the weights. ReadCoverage(DirPath); auto Weights = Coverage.FunctionWeights(NumFunctions); - Vector<double> Intervals(NumFunctions + 1); + std::vector<double> Intervals(NumFunctions + 1); std::iota(Intervals.begin(), Intervals.end(), 0); auto Distribution = std::piecewise_constant_distribution<double>( Intervals.begin(), Intervals.end(), Weights.begin()); @@ -200,7 +201,8 @@ bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, Printf("INFO: AUTOFOCUS: %zd %s\n", FocusFuncIdx, FunctionNames[FocusFuncIdx].c_str()); for (size_t i = 0; i < NumFunctions; i++) { - if (!Weights[i]) continue; + if (Weights[i] == 0.0) + continue; Printf(" [%zd] W %g\tBB-tot %u\tBB-cov %u\tEntryFreq %u:\t%s\n", i, Weights[i], Coverage.GetNumberOfBlocks(i), Coverage.GetNumberOfCoveredBlocks(i), Coverage.GetCounter(i, 0), @@ -245,7 +247,7 @@ bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, } int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath, - const Vector<SizedFile> &CorporaFiles) { + const std::vector<SizedFile> &CorporaFiles) { Printf("INFO: collecting data flow: bin: %s dir: %s files: %zd\n", DFTBinary.c_str(), DirPath.c_str(), CorporaFiles.size()); if (CorporaFiles.empty()) { @@ -263,7 +265,7 @@ int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath, // we then request tags in [0,Size/2) and [Size/2, Size), and so on. // Function number => DFT. auto OutPath = DirPlusFile(DirPath, Hash(FileToVector(F.File))); - std::unordered_map<size_t, Vector<uint8_t>> DFTMap; + std::unordered_map<size_t, std::vector<uint8_t>> DFTMap; std::unordered_set<std::string> Cov; Command Cmd; Cmd.addArgument(DFTBinary); diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.h b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.h index d6e3de30a4ef..054dce1bdcb6 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.h +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.h @@ -39,10 +39,11 @@ namespace fuzzer { int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath, - const Vector<SizedFile> &CorporaFiles); + const std::vector<SizedFile> &CorporaFiles); class BlockCoverage { - public: +public: + // These functions guarantee no CoverageVector is longer than UINT32_MAX. bool AppendCoverage(std::istream &IN); bool AppendCoverage(const std::string &S); @@ -50,7 +51,8 @@ class BlockCoverage { uint32_t GetCounter(size_t FunctionId, size_t BasicBlockId) { auto It = Functions.find(FunctionId); - if (It == Functions.end()) return 0; + if (It == Functions.end()) + return 0; const auto &Counters = It->second; if (BasicBlockId < Counters.size()) return Counters[BasicBlockId]; @@ -61,7 +63,7 @@ class BlockCoverage { auto It = Functions.find(FunctionId); if (It == Functions.end()) return 0; const auto &Counters = It->second; - return Counters.size(); + return static_cast<uint32_t>(Counters.size()); } uint32_t GetNumberOfCoveredBlocks(size_t FunctionId) { @@ -75,12 +77,11 @@ class BlockCoverage { return Result; } - Vector<double> FunctionWeights(size_t NumFunctions) const; + std::vector<double> FunctionWeights(size_t NumFunctions) const; void clear() { Functions.clear(); } - private: - - typedef Vector<uint32_t> CoverageVector; +private: + typedef std::vector<uint32_t> CoverageVector; uint32_t NumberOfCoveredBlocks(const CoverageVector &Counters) const { uint32_t Res = 0; @@ -91,7 +92,8 @@ class BlockCoverage { } uint32_t NumberOfUncoveredBlocks(const CoverageVector &Counters) const { - return Counters.size() - NumberOfCoveredBlocks(Counters); + return static_cast<uint32_t>(Counters.size()) - + NumberOfCoveredBlocks(Counters); } uint32_t SmallestNonZeroCounter(const CoverageVector &Counters) const { @@ -115,9 +117,9 @@ class DataFlowTrace { public: void ReadCoverage(const std::string &DirPath); bool Init(const std::string &DirPath, std::string *FocusFunction, - Vector<SizedFile> &CorporaFiles, Random &Rand); + std::vector<SizedFile> &CorporaFiles, Random &Rand); void Clear() { Traces.clear(); } - const Vector<uint8_t> *Get(const std::string &InputSha1) const { + const std::vector<uint8_t> *Get(const std::string &InputSha1) const { auto It = Traces.find(InputSha1); if (It != Traces.end()) return &It->second; @@ -126,9 +128,9 @@ class DataFlowTrace { private: // Input's sha1 => DFT for the FocusFunction. - std::unordered_map<std::string, Vector<uint8_t> > Traces; - BlockCoverage Coverage; - std::unordered_set<std::string> CorporaHashes; + std::unordered_map<std::string, std::vector<uint8_t>> Traces; + BlockCoverage Coverage; + std::unordered_set<std::string> CorporaHashes; }; } // namespace fuzzer diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerDefs.h b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerDefs.h index 1a2752af2f4d..db1f74a545e3 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerDefs.h +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerDefs.h @@ -38,28 +38,8 @@ struct ExternalFunctions; // Global interface to functions that may or may not be available. extern ExternalFunctions *EF; -// We are using a custom allocator to give a different symbol name to STL -// containers in order to avoid ODR violations. -template<typename T> - class fuzzer_allocator: public std::allocator<T> { - public: - fuzzer_allocator() = default; - - template<class U> - fuzzer_allocator(const fuzzer_allocator<U>&) {} - - template<class Other> - struct rebind { typedef fuzzer_allocator<Other> other; }; - }; - -template<typename T> -using Vector = std::vector<T, fuzzer_allocator<T>>; - -template<typename T> -using Set = std::set<T, std::less<T>, fuzzer_allocator<T>>; - -typedef Vector<uint8_t> Unit; -typedef Vector<Unit> UnitVector; +typedef std::vector<uint8_t> Unit; +typedef std::vector<Unit> UnitVector; typedef int (*UserCallback)(const uint8_t *Data, size_t Size); int FuzzerDriver(int *argc, char ***argv, UserCallback Callback); diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerDictionary.h b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerDictionary.h index 301c5d9afecf..48f063c7ee4e 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerDictionary.h +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerDictionary.h @@ -23,12 +23,14 @@ template <size_t kMaxSizeT> class FixedWord { public: static const size_t kMaxSize = kMaxSizeT; FixedWord() {} - FixedWord(const uint8_t *B, uint8_t S) { Set(B, S); } + FixedWord(const uint8_t *B, size_t S) { Set(B, S); } - void Set(const uint8_t *B, uint8_t S) { + void Set(const uint8_t *B, size_t S) { + static_assert(kMaxSizeT <= std::numeric_limits<uint8_t>::max(), + "FixedWord::kMaxSizeT cannot fit in a uint8_t."); assert(S <= kMaxSize); memcpy(Data, B, S); - Size = S; + Size = static_cast<uint8_t>(S); } bool operator==(const FixedWord<kMaxSize> &w) const { @@ -50,10 +52,13 @@ class DictionaryEntry { public: DictionaryEntry() {} DictionaryEntry(Word W) : W(W) {} - DictionaryEntry(Word W, size_t PositionHint) : W(W), PositionHint(PositionHint) {} + DictionaryEntry(Word W, size_t PositionHint) + : W(W), PositionHint(PositionHint) {} const Word &GetW() const { return W; } - bool HasPositionHint() const { return PositionHint != std::numeric_limits<size_t>::max(); } + bool HasPositionHint() const { + return PositionHint != std::numeric_limits<size_t>::max(); + } size_t GetPositionHint() const { assert(HasPositionHint()); return PositionHint; @@ -106,12 +111,12 @@ private: }; // Parses one dictionary entry. -// If successful, write the enty to Unit and returns true, +// If successful, writes the entry to Unit and returns true, // otherwise returns false. bool ParseOneDictionaryEntry(const std::string &Str, Unit *U); // Parses the dictionary file, fills Units, returns true iff all lines // were parsed successfully. -bool ParseDictionaryFile(const std::string &Text, Vector<Unit> *Units); +bool ParseDictionaryFile(const std::string &Text, std::vector<Unit> *Units); } // namespace fuzzer diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerDriver.cpp b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerDriver.cpp index 447cafce7fd4..8674d788932f 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerDriver.cpp +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerDriver.cpp @@ -86,7 +86,7 @@ static const FlagDescription FlagDescriptions [] { static const size_t kNumFlags = sizeof(FlagDescriptions) / sizeof(FlagDescriptions[0]); -static Vector<std::string> *Inputs; +static std::vector<std::string> *Inputs; static std::string *ProgName; static void PrintHelp() { @@ -159,14 +159,14 @@ static bool ParseOneFlag(const char *Param) { const char *Str = FlagValue(Param, Name); if (Str) { if (FlagDescriptions[F].IntFlag) { - int Val = MyStol(Str); - *FlagDescriptions[F].IntFlag = Val; + auto Val = MyStol(Str); + *FlagDescriptions[F].IntFlag = static_cast<int>(Val); if (Flags.verbosity >= 2) Printf("Flag: %s %d\n", Name, Val); return true; } else if (FlagDescriptions[F].UIntFlag) { - unsigned int Val = std::stoul(Str); - *FlagDescriptions[F].UIntFlag = Val; + auto Val = std::stoul(Str); + *FlagDescriptions[F].UIntFlag = static_cast<unsigned int>(Val); if (Flags.verbosity >= 2) Printf("Flag: %s %u\n", Name, Val); return true; @@ -187,7 +187,7 @@ static bool ParseOneFlag(const char *Param) { } // We don't use any library to minimize dependencies. -static void ParseFlags(const Vector<std::string> &Args, +static void ParseFlags(const std::vector<std::string> &Args, const ExternalFunctions *EF) { for (size_t F = 0; F < kNumFlags; F++) { if (FlagDescriptions[F].IntFlag) @@ -206,7 +206,7 @@ static void ParseFlags(const Vector<std::string> &Args, "Disabling -len_control by default.\n", EF->LLVMFuzzerCustomMutator); } - Inputs = new Vector<std::string>; + Inputs = new std::vector<std::string>; for (size_t A = 1; A < Args.size(); A++) { if (ParseOneFlag(Args[A].c_str())) { if (Flags.ignore_remaining_args) @@ -272,7 +272,7 @@ static void ValidateDirectoryExists(const std::string &Path, exit(1); } -std::string CloneArgsWithoutX(const Vector<std::string> &Args, +std::string CloneArgsWithoutX(const std::vector<std::string> &Args, const char *X1, const char *X2) { std::string Cmd; for (auto &S : Args) { @@ -283,18 +283,22 @@ std::string CloneArgsWithoutX(const Vector<std::string> &Args, return Cmd; } -static int RunInMultipleProcesses(const Vector<std::string> &Args, +static int RunInMultipleProcesses(const std::vector<std::string> &Args, unsigned NumWorkers, unsigned NumJobs) { std::atomic<unsigned> Counter(0); std::atomic<bool> HasErrors(false); Command Cmd(Args); Cmd.removeFlag("jobs"); Cmd.removeFlag("workers"); - Vector<std::thread> V; + std::vector<std::thread> V; std::thread Pulse(PulseThread); Pulse.detach(); - for (unsigned i = 0; i < NumWorkers; i++) - V.push_back(std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs, &HasErrors)); + V.resize(NumWorkers); + for (unsigned i = 0; i < NumWorkers; i++) { + V[i] = std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs, + &HasErrors); + SetThreadName(V[i], "FuzzerWorker"); + } for (auto &T : V) T.join(); return HasErrors ? 1 : 0; @@ -348,8 +352,8 @@ static std::string GetDedupTokenFromCmdOutput(const std::string &S) { return S.substr(Beg, End - Beg); } -int CleanseCrashInput(const Vector<std::string> &Args, - const FuzzingOptions &Options) { +int CleanseCrashInput(const std::vector<std::string> &Args, + const FuzzingOptions &Options) { if (Inputs->size() != 1 || !Flags.exact_artifact_path) { Printf("ERROR: -cleanse_crash should be given one input file and" " -exact_artifact_path\n"); @@ -372,7 +376,7 @@ int CleanseCrashInput(const Vector<std::string> &Args, auto U = FileToVector(CurrentFilePath); size_t Size = U.size(); - const Vector<uint8_t> ReplacementBytes = {' ', 0xff}; + const std::vector<uint8_t> ReplacementBytes = {' ', 0xff}; for (int NumAttempts = 0; NumAttempts < 5; NumAttempts++) { bool Changed = false; for (size_t Idx = 0; Idx < Size; Idx++) { @@ -403,7 +407,7 @@ int CleanseCrashInput(const Vector<std::string> &Args, return 0; } -int MinimizeCrashInput(const Vector<std::string> &Args, +int MinimizeCrashInput(const std::vector<std::string> &Args, const FuzzingOptions &Options) { if (Inputs->size() != 1) { Printf("ERROR: -minimize_crash should be given one input file\n"); @@ -462,7 +466,7 @@ int MinimizeCrashInput(const Vector<std::string> &Args, CurrentFilePath = Flags.exact_artifact_path; WriteToFile(U, CurrentFilePath); } - Printf("CRASH_MIN: failed to minimize beyond %s (%d bytes), exiting\n", + Printf("CRASH_MIN: failed to minimize beyond %s (%zu bytes), exiting\n", CurrentFilePath.c_str(), U.size()); break; } @@ -500,17 +504,17 @@ int MinimizeCrashInputInternalStep(Fuzzer *F, InputCorpus *Corpus) { F->MinimizeCrashLoop(U); Printf("INFO: Done MinimizeCrashInputInternalStep, no crashes found\n"); exit(0); - return 0; } -void Merge(Fuzzer *F, FuzzingOptions &Options, const Vector<std::string> &Args, - const Vector<std::string> &Corpora, const char *CFPathOrNull) { +void Merge(Fuzzer *F, FuzzingOptions &Options, + const std::vector<std::string> &Args, + const std::vector<std::string> &Corpora, const char *CFPathOrNull) { if (Corpora.size() < 2) { Printf("INFO: Merge requires two or more corpus dirs\n"); exit(0); } - Vector<SizedFile> OldCorpus, NewCorpus; + std::vector<SizedFile> OldCorpus, NewCorpus; GetSizedFilesFromDir(Corpora[0], &OldCorpus); for (size_t i = 1; i < Corpora.size(); i++) GetSizedFilesFromDir(Corpora[i], &NewCorpus); @@ -518,10 +522,10 @@ void Merge(Fuzzer *F, FuzzingOptions &Options, const Vector<std::string> &Args, std::sort(NewCorpus.begin(), NewCorpus.end()); std::string CFPath = CFPathOrNull ? CFPathOrNull : TempPath("Merge", ".txt"); - Vector<std::string> NewFiles; - Set<uint32_t> NewFeatures, NewCov; + std::vector<std::string> NewFiles; + std::set<uint32_t> NewFeatures, NewCov; CrashResistantMerge(Args, OldCorpus, NewCorpus, &NewFiles, {}, &NewFeatures, - {}, &NewCov, CFPath, true); + {}, &NewCov, CFPath, true, Flags.set_cover_merge); for (auto &Path : NewFiles) F->WriteToOutputCorpus(FileToVector(Path, Options.MaxLen)); // We are done, delete the control file if it was a temporary one. @@ -531,17 +535,17 @@ void Merge(Fuzzer *F, FuzzingOptions &Options, const Vector<std::string> &Args, exit(0); } -int AnalyzeDictionary(Fuzzer *F, const Vector<Unit>& Dict, - UnitVector& Corpus) { - Printf("Started dictionary minimization (up to %d tests)\n", +int AnalyzeDictionary(Fuzzer *F, const std::vector<Unit> &Dict, + UnitVector &Corpus) { + Printf("Started dictionary minimization (up to %zu tests)\n", Dict.size() * Corpus.size() * 2); // Scores and usage count for each dictionary unit. - Vector<int> Scores(Dict.size()); - Vector<int> Usages(Dict.size()); + std::vector<int> Scores(Dict.size()); + std::vector<int> Usages(Dict.size()); - Vector<size_t> InitialFeatures; - Vector<size_t> ModifiedFeatures; + std::vector<size_t> InitialFeatures; + std::vector<size_t> ModifiedFeatures; for (auto &C : Corpus) { // Get coverage for the testcase without modifications. F->ExecuteCallback(C.data(), C.size()); @@ -551,7 +555,7 @@ int AnalyzeDictionary(Fuzzer *F, const Vector<Unit>& Dict, }); for (size_t i = 0; i < Dict.size(); ++i) { - Vector<uint8_t> Data = C; + std::vector<uint8_t> Data = C; auto StartPos = std::search(Data.begin(), Data.end(), Dict[i].begin(), Dict[i].end()); // Skip dictionary unit, if the testcase does not contain it. @@ -597,9 +601,9 @@ int AnalyzeDictionary(Fuzzer *F, const Vector<Unit>& Dict, return 0; } -Vector<std::string> ParseSeedInuts(const char *seed_inputs) { +std::vector<std::string> ParseSeedInuts(const char *seed_inputs) { // Parse -seed_inputs=file1,file2,... or -seed_inputs=@seed_inputs_file - Vector<std::string> Files; + std::vector<std::string> Files; if (!seed_inputs) return Files; std::string SeedInputs; if (Flags.seed_inputs[0] == '@') @@ -620,9 +624,10 @@ Vector<std::string> ParseSeedInuts(const char *seed_inputs) { return Files; } -static Vector<SizedFile> ReadCorpora(const Vector<std::string> &CorpusDirs, - const Vector<std::string> &ExtraSeedFiles) { - Vector<SizedFile> SizedFiles; +static std::vector<SizedFile> +ReadCorpora(const std::vector<std::string> &CorpusDirs, + const std::vector<std::string> &ExtraSeedFiles) { + std::vector<SizedFile> SizedFiles; size_t LastNumFiles = 0; for (auto &Dir : CorpusDirs) { GetSizedFilesFromDir(Dir, &SizedFiles); @@ -645,7 +650,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { EF->LLVMFuzzerInitialize(argc, argv); if (EF->__msan_scoped_disable_interceptor_checks) EF->__msan_scoped_disable_interceptor_checks(); - const Vector<std::string> Args(*argv, *argv + *argc); + const std::vector<std::string> Args(*argv, *argv + *argc); assert(!Args.empty()); ProgName = new std::string(Args[0]); if (Argv0 != *ProgName) { @@ -734,7 +739,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { ValidateDirectoryExists(DirName(Options.ExactArtifactPath), Flags.create_missing_dirs); } - Vector<Unit> Dictionary; + std::vector<Unit> Dictionary; if (Flags.dict) if (!ParseDictionaryFile(FileToString(Flags.dict), &Dictionary)) return 1; @@ -776,7 +781,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { if (!Options.FocusFunction.empty()) Options.Entropic = false; // FocusFunction overrides entropic scheduling. if (Options.Entropic) - Printf("INFO: Running with entropic power schedule (0x%X, %d).\n", + Printf("INFO: Running with entropic power schedule (0x%zX, %zu).\n", Options.EntropicFeatureFrequencyThreshold, Options.EntropicNumberOfRarestFeatures); struct EntropicOptions Entropic; @@ -789,12 +794,13 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { unsigned Seed = Flags.seed; // Initialize Seed. if (Seed == 0) - Seed = - std::chrono::system_clock::now().time_since_epoch().count() + GetPid(); + Seed = static_cast<unsigned>( + std::chrono::system_clock::now().time_since_epoch().count() + GetPid()); if (Flags.verbosity) Printf("INFO: Seed: %u\n", Seed); - if (Flags.collect_data_flow && !Flags.fork && !Flags.merge) { + if (Flags.collect_data_flow && Flags.data_flow_trace && !Flags.fork && + !(Flags.merge || Flags.set_cover_merge)) { if (RunIndividualFiles) return CollectDataFlow(Flags.collect_data_flow, Flags.data_flow_trace, ReadCorpora({}, *Inputs)); @@ -856,7 +862,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { RunOneTest(F, Path.c_str(), Options.MaxLen); auto StopTime = system_clock::now(); auto MS = duration_cast<milliseconds>(StopTime - StartTime).count(); - Printf("Executed %s in %zd ms\n", Path.c_str(), (long)MS); + Printf("Executed %s in %ld ms\n", Path.c_str(), (long)MS); } Printf("***\n" "*** NOTE: fuzzing was not performed, you have only\n" @@ -866,10 +872,11 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { exit(0); } + Options.ForkCorpusGroups = Flags.fork_corpus_groups; if (Flags.fork) FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork); - if (Flags.merge) + if (Flags.merge || Flags.set_cover_merge) Merge(F, Options, Args, *Inputs, Flags.merge_control_file); if (Flags.merge_inner) { @@ -877,7 +884,8 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { if (Options.MaxLen == 0) F->SetMaxInputLen(kDefaultMaxMergeLen); assert(Flags.merge_control_file); - F->CrashResistantMergeInternalStep(Flags.merge_control_file); + F->CrashResistantMergeInternalStep(Flags.merge_control_file, + !strncmp(Flags.merge_inner, "2", 1)); exit(0); } diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerExtraCounters.cpp b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerExtraCounters.cpp index 04f569a1a879..54ecbf7c62f1 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerExtraCounters.cpp +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerExtraCounters.cpp @@ -31,12 +31,4 @@ void ClearExtraCounters() { // hand-written memset, don't asan-ify. } // namespace fuzzer -#else -// TODO: implement for other platforms. -namespace fuzzer { -uint8_t *ExtraCountersBegin() { return nullptr; } -uint8_t *ExtraCountersEnd() { return nullptr; } -void ClearExtraCounters() {} -} // namespace fuzzer - #endif diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerExtraCountersDarwin.cpp b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerExtraCountersDarwin.cpp new file mode 100644 index 000000000000..2321ba8a3d40 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerExtraCountersDarwin.cpp @@ -0,0 +1,22 @@ +//===- FuzzerExtraCountersDarwin.cpp - Extra coverage counters for Darwin -===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Extra coverage counters defined by user code for Darwin. +//===----------------------------------------------------------------------===// + +#include "FuzzerPlatform.h" +#include <cstdint> + +#if LIBFUZZER_APPLE + +namespace fuzzer { +uint8_t *ExtraCountersBegin() { return nullptr; } +uint8_t *ExtraCountersEnd() { return nullptr; } +void ClearExtraCounters() {} +} // namespace fuzzer + +#endif diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerExtraCountersWindows.cpp b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerExtraCountersWindows.cpp new file mode 100644 index 000000000000..102f5febdaec --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerExtraCountersWindows.cpp @@ -0,0 +1,80 @@ +//===- FuzzerExtraCountersWindows.cpp - Extra coverage counters for Win32 -===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Extra coverage counters defined by user code for Windows. +//===----------------------------------------------------------------------===// + +#include "FuzzerPlatform.h" +#include <cstdint> + +#if LIBFUZZER_WINDOWS +#include <windows.h> + +namespace fuzzer { + +// +// The __start___libfuzzer_extra_counters variable is align 16, size 16 to +// ensure the padding between it and the next variable in this section (either +// __libfuzzer_extra_counters or __stop___libfuzzer_extra_counters) will be +// located at (__start___libfuzzer_extra_counters + +// sizeof(__start___libfuzzer_extra_counters)). Otherwise, the calculation of +// (stop - (start + sizeof(start))) might be skewed. +// +// The section name, __libfuzzer_extra_countaaa ends with "aaa", so it sorts +// before __libfuzzer_extra_counters alphabetically. We want the start symbol to +// be placed in the section just before the user supplied counters (if present). +// +#pragma section(".data$__libfuzzer_extra_countaaa") +ATTRIBUTE_ALIGNED(16) +__declspec(allocate(".data$__libfuzzer_extra_countaaa")) uint8_t + __start___libfuzzer_extra_counters[16] = {0}; + +// +// Example of what the user-supplied counters should look like. First, the +// pragma to create the section name. It will fall alphabetically between +// ".data$__libfuzzer_extra_countaaa" and ".data$__libfuzzer_extra_countzzz". +// Next, the declspec to allocate the variable inside the specified section. +// Finally, some array, struct, whatever that is used to track the counter data. +// The size of this variable is computed at runtime by finding the difference of +// __stop___libfuzzer_extra_counters and __start___libfuzzer_extra_counters + +// sizeof(__start___libfuzzer_extra_counters). +// + +// +// #pragma section(".data$__libfuzzer_extra_counters") +// __declspec(allocate(".data$__libfuzzer_extra_counters")) +// uint8_t any_name_variable[64 * 1024]; +// + +// +// Here, the section name, __libfuzzer_extra_countzzz ends with "zzz", so it +// sorts after __libfuzzer_extra_counters alphabetically. We want the stop +// symbol to be placed in the section just after the user supplied counters (if +// present). Align to 1 so there isn't any padding placed between this and the +// previous variable. +// +#pragma section(".data$__libfuzzer_extra_countzzz") +ATTRIBUTE_ALIGNED(1) +__declspec(allocate(".data$__libfuzzer_extra_countzzz")) uint8_t + __stop___libfuzzer_extra_counters = 0; + +uint8_t *ExtraCountersBegin() { + return __start___libfuzzer_extra_counters + + sizeof(__start___libfuzzer_extra_counters); +} + +uint8_t *ExtraCountersEnd() { return &__stop___libfuzzer_extra_counters; } + +ATTRIBUTE_NO_SANITIZE_ALL +void ClearExtraCounters() { + uint8_t *Beg = ExtraCountersBegin(); + SecureZeroMemory(Beg, ExtraCountersEnd() - Beg); +} + +} // namespace fuzzer + +#endif diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerFlags.def b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerFlags.def index ab31da0ae5d6..fc3b3aa8c98a 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerFlags.def +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerFlags.def @@ -58,12 +58,21 @@ FUZZER_FLAG_INT(max_total_time, 0, "If positive, indicates the maximal total " FUZZER_FLAG_INT(help, 0, "Print help.") FUZZER_FLAG_INT(fork, 0, "Experimental mode where fuzzing happens " "in a subprocess") +FUZZER_FLAG_INT(fork_corpus_groups, 0, "For fork mode, enable the corpus-group " + "strategy, The main corpus will be grouped according to size, " + "and each sub-process will randomly select seeds from different " + "groups as the sub-corpus.") FUZZER_FLAG_INT(ignore_timeouts, 1, "Ignore timeouts in fork mode") FUZZER_FLAG_INT(ignore_ooms, 1, "Ignore OOMs in fork mode") FUZZER_FLAG_INT(ignore_crashes, 0, "Ignore crashes in fork mode") FUZZER_FLAG_INT(merge, 0, "If 1, the 2-nd, 3-rd, etc corpora will be " "merged into the 1-st corpus. Only interesting units will be taken. " "This flag can be used to minimize a corpus.") +FUZZER_FLAG_INT(set_cover_merge, 0, "If 1, the 2-nd, 3-rd, etc corpora will be " + "merged into the 1-st corpus. Same as the 'merge' flag, but uses the " + "standard greedy algorithm for the set cover problem to " + "compute an approximation of the minimum set of testcases that " + "provide the same coverage as the initial corpora") FUZZER_FLAG_STRING(stop_file, "Stop fuzzing ASAP if this file exists") FUZZER_FLAG_STRING(merge_inner, "internal flag") FUZZER_FLAG_STRING(merge_control_file, @@ -158,7 +167,7 @@ FUZZER_FLAG_INT(purge_allocator_interval, 1, "Purge allocator caches and " "purge_allocator_interval=-1 to disable this functionality.") FUZZER_FLAG_INT(trace_malloc, 0, "If >= 1 will print all mallocs/frees. " "If >= 2 will also print stack traces.") -FUZZER_FLAG_INT(rss_limit_mb, 2048, "If non-zero, the fuzzer will exit upon" +FUZZER_FLAG_INT(rss_limit_mb, 2048, "If non-zero, the fuzzer will exit upon " "reaching this limit of RSS memory usage.") FUZZER_FLAG_INT(malloc_limit_mb, 0, "If non-zero, the fuzzer will exit " "if the target tries to allocate this number of Mb with one malloc call. " diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerFork.cpp b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerFork.cpp index 84725d22a9c7..c248a1d246a3 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerFork.cpp +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerFork.cpp @@ -86,18 +86,21 @@ struct FuzzJob { }; struct GlobalEnv { - Vector<std::string> Args; - Vector<std::string> CorpusDirs; + std::vector<std::string> Args; + std::vector<std::string> CorpusDirs; std::string MainCorpusDir; std::string TempDir; std::string DFTDir; std::string DataFlowBinary; - Set<uint32_t> Features, Cov; - Set<std::string> FilesWithDFT; - Vector<std::string> Files; + std::set<uint32_t> Features, Cov; + std::set<std::string> FilesWithDFT; + std::vector<std::string> Files; + std::vector<std::size_t> FilesSizes; Random *Rand; std::chrono::system_clock::time_point ProcessStartTime; int Verbosity = 0; + int Group = 0; + int NumCorpuses = 8; size_t NumTimeouts = 0; size_t NumOOMs = 0; @@ -136,13 +139,29 @@ struct GlobalEnv { if (size_t CorpusSubsetSize = std::min(Files.size(), (size_t)sqrt(Files.size() + 2))) { auto Time1 = std::chrono::system_clock::now(); - for (size_t i = 0; i < CorpusSubsetSize; i++) { - auto &SF = Files[Rand->SkewTowardsLast(Files.size())]; - Seeds += (Seeds.empty() ? "" : ",") + SF; - CollectDFT(SF); + if (Group) { // whether to group the corpus. + size_t AverageCorpusSize = Files.size() / NumCorpuses + 1; + size_t StartIndex = ((JobId - 1) % NumCorpuses) * AverageCorpusSize; + for (size_t i = 0; i < CorpusSubsetSize; i++) { + size_t RandNum = (*Rand)(AverageCorpusSize); + size_t Index = RandNum + StartIndex; + Index = Index < Files.size() ? Index + : Rand->SkewTowardsLast(Files.size()); + auto &SF = Files[Index]; + Seeds += (Seeds.empty() ? "" : ",") + SF; + CollectDFT(SF); + } + } else { + for (size_t i = 0; i < CorpusSubsetSize; i++) { + auto &SF = Files[Rand->SkewTowardsLast(Files.size())]; + Seeds += (Seeds.empty() ? "" : ",") + SF; + CollectDFT(SF); + } } auto Time2 = std::chrono::system_clock::now(); - Job->DftTimeInSeconds = duration_cast<seconds>(Time2 - Time1).count(); + auto DftTimeInSeconds = duration_cast<seconds>(Time2 - Time1).count(); + assert(DftTimeInSeconds < std::numeric_limits<int>::max()); + Job->DftTimeInSeconds = static_cast<int>(DftTimeInSeconds); } if (!Seeds.empty()) { Job->SeedListPath = @@ -181,7 +200,7 @@ struct GlobalEnv { auto Stats = ParseFinalStatsFromLog(Job->LogPath); NumRuns += Stats.number_of_executed_units; - Vector<SizedFile> TempFiles, MergeCandidates; + std::vector<SizedFile> TempFiles, MergeCandidates; // Read all newly created inputs and their feature sets. // Choose only those inputs that have new features. GetSizedFilesFromDir(Job->CorpusDir, &TempFiles); @@ -191,7 +210,7 @@ struct GlobalEnv { FeatureFile.replace(0, Job->CorpusDir.size(), Job->FeaturesDir); auto FeatureBytes = FileToVector(FeatureFile, 0, false); assert((FeatureBytes.size() % sizeof(uint32_t)) == 0); - Vector<uint32_t> NewFeatures(FeatureBytes.size() / sizeof(uint32_t)); + std::vector<uint32_t> NewFeatures(FeatureBytes.size() / sizeof(uint32_t)); memcpy(NewFeatures.data(), FeatureBytes.data(), FeatureBytes.size()); for (auto Ft : NewFeatures) { if (!Features.count(Ft)) { @@ -201,7 +220,7 @@ struct GlobalEnv { } } // if (!FilesToAdd.empty() || Job->ExitCode != 0) - Printf("#%zd: cov: %zd ft: %zd corp: %zd exec/s %zd " + Printf("#%zd: cov: %zd ft: %zd corp: %zd exec/s: %zd " "oom/timeout/crash: %zd/%zd/%zd time: %zds job: %zd dft_time: %d\n", NumRuns, Cov.size(), Features.size(), Files.size(), Stats.average_exec_per_sec, NumOOMs, NumTimeouts, NumCrashes, @@ -209,15 +228,27 @@ struct GlobalEnv { if (MergeCandidates.empty()) return; - Vector<std::string> FilesToAdd; - Set<uint32_t> NewFeatures, NewCov; + std::vector<std::string> FilesToAdd; + std::set<uint32_t> NewFeatures, NewCov; + bool IsSetCoverMerge = + !Job->Cmd.getFlagValue("set_cover_merge").compare("1"); CrashResistantMerge(Args, {}, MergeCandidates, &FilesToAdd, Features, - &NewFeatures, Cov, &NewCov, Job->CFPath, false); + &NewFeatures, Cov, &NewCov, Job->CFPath, false, + IsSetCoverMerge); for (auto &Path : FilesToAdd) { auto U = FileToVector(Path); auto NewPath = DirPlusFile(MainCorpusDir, Hash(U)); WriteToFile(U, NewPath); - Files.push_back(NewPath); + if (Group) { // Insert the queue according to the size of the seed. + size_t UnitSize = U.size(); + auto Idx = + std::upper_bound(FilesSizes.begin(), FilesSizes.end(), UnitSize) - + FilesSizes.begin(); + FilesSizes.insert(FilesSizes.begin() + Idx, UnitSize); + Files.insert(Files.begin() + Idx, NewPath); + } else { + Files.push_back(NewPath); + } } Features.insert(NewFeatures.begin(), NewFeatures.end()); Cov.insert(NewCov.begin(), NewCov.end()); @@ -226,10 +257,8 @@ struct GlobalEnv { if (TPC.PcIsFuncEntry(TE)) PrintPC(" NEW_FUNC: %p %F %L\n", "", TPC.GetNextInstructionPc(TE->PC)); - } - void CollectDFT(const std::string &InputPath) { if (DataFlowBinary.empty()) return; if (!FilesWithDFT.insert(InputPath).second) return; @@ -281,8 +310,8 @@ void WorkerThread(JobQueue *FuzzQ, JobQueue *MergeQ) { // This is just a skeleton of an experimental -fork=1 feature. void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, - const Vector<std::string> &Args, - const Vector<std::string> &CorpusDirs, int NumJobs) { + const std::vector<std::string> &Args, + const std::vector<std::string> &CorpusDirs, int NumJobs) { Printf("INFO: -fork=%d: fuzzing in separate process(s)\n", NumJobs); GlobalEnv Env; @@ -292,8 +321,9 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, Env.Verbosity = Options.Verbosity; Env.ProcessStartTime = std::chrono::system_clock::now(); Env.DataFlowBinary = Options.CollectDataFlow; + Env.Group = Options.ForkCorpusGroups; - Vector<SizedFile> SeedFiles; + std::vector<SizedFile> SeedFiles; for (auto &Dir : CorpusDirs) GetSizedFilesFromDir(Dir, &SeedFiles); std::sort(SeedFiles.begin(), SeedFiles.end()); @@ -314,10 +344,20 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, Env.Files.push_back(File.File); } else { auto CFPath = DirPlusFile(Env.TempDir, "merge.txt"); - CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, {}, &Env.Features, - {}, &Env.Cov, CFPath, false); + std::set<uint32_t> NewFeatures, NewCov; + CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, Env.Features, + &NewFeatures, Env.Cov, &NewCov, CFPath, + /*Verbose=*/false, /*IsSetCoverMerge=*/false); + Env.Features.insert(NewFeatures.begin(), NewFeatures.end()); + Env.Cov.insert(NewFeatures.begin(), NewFeatures.end()); RemoveFile(CFPath); } + + if (Env.Group) { + for (auto &path : Env.Files) + Env.FilesSizes.push_back(FileSize(path)); + } + Printf("INFO: -fork=%d: %zd seed inputs, starting to fuzz in %s\n", NumJobs, Env.Files.size(), Env.TempDir.c_str()); @@ -332,8 +372,10 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, WriteToFile(Unit({1}), Env.StopFile()); }; + size_t MergeCycle = 20; + size_t JobExecuted = 0; size_t JobId = 1; - Vector<std::thread> Threads; + std::vector<std::thread> Threads; for (int t = 0; t < NumJobs; t++) { Threads.push_back(std::thread(WorkerThread, &FuzzQ, &MergeQ)); FuzzQ.Push(Env.CreateNewJob(JobId++)); @@ -353,7 +395,46 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, Env.RunOneMergeJob(Job.get()); - // Continue if our crash is one of the ignorred ones. + // merge the corpus . + JobExecuted++; + if (Env.Group && JobExecuted >= MergeCycle) { + std::vector<SizedFile> CurrentSeedFiles; + for (auto &Dir : CorpusDirs) + GetSizedFilesFromDir(Dir, &CurrentSeedFiles); + std::sort(CurrentSeedFiles.begin(), CurrentSeedFiles.end()); + + auto CFPath = DirPlusFile(Env.TempDir, "merge.txt"); + std::set<uint32_t> TmpNewFeatures, TmpNewCov; + std::set<uint32_t> TmpFeatures, TmpCov; + Env.Files.clear(); + Env.FilesSizes.clear(); + CrashResistantMerge(Env.Args, {}, CurrentSeedFiles, &Env.Files, + TmpFeatures, &TmpNewFeatures, TmpCov, &TmpNewCov, + CFPath, /*Verbose=*/false, /*IsSetCoverMerge=*/false); + for (auto &path : Env.Files) + Env.FilesSizes.push_back(FileSize(path)); + RemoveFile(CFPath); + JobExecuted = 0; + MergeCycle += 5; + } + + // Since the number of corpus seeds will gradually increase, in order to + // control the number in each group to be about three times the number of + // seeds selected each time, the number of groups is dynamically adjusted. + if (Env.Files.size() < 2000) + Env.NumCorpuses = 12; + else if (Env.Files.size() < 6000) + Env.NumCorpuses = 20; + else if (Env.Files.size() < 12000) + Env.NumCorpuses = 32; + else if (Env.Files.size() < 16000) + Env.NumCorpuses = 40; + else if (Env.Files.size() < 24000) + Env.NumCorpuses = 60; + else + Env.NumCorpuses = 80; + + // Continue if our crash is one of the ignored ones. if (Options.IgnoreTimeouts && ExitCode == Options.TimeoutExitCode) Env.NumTimeouts++; else if (Options.IgnoreOOMs && ExitCode == Options.OOMExitCode) diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerFork.h b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerFork.h index b29a43e13fbc..fc3e9d636cbc 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerFork.h +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerFork.h @@ -17,8 +17,8 @@ namespace fuzzer { void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, - const Vector<std::string> &Args, - const Vector<std::string> &CorpusDirs, int NumJobs); + const std::vector<std::string> &Args, + const std::vector<std::string> &CorpusDirs, int NumJobs); } // namespace fuzzer #endif // LLVM_FUZZER_FORK_H diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerIO.cpp b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerIO.cpp index 54a7219fc0e0..54cc4ee54be0 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerIO.cpp +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerIO.cpp @@ -23,6 +23,14 @@ namespace fuzzer { static FILE *OutputFile = stderr; +FILE *GetOutputFile() { + return OutputFile; +} + +void SetOutputFile(FILE *NewOutputFile) { + OutputFile = NewOutputFile; +} + long GetEpoch(const std::string &Path) { struct stat St; if (stat(Path.c_str(), &St)) @@ -57,7 +65,7 @@ std::string FileToString(const std::string &Path) { } void CopyFileToErr(const std::string &Path) { - Printf("%s", FileToString(Path).c_str()); + Puts(FileToString(Path).c_str()); } void WriteToFile(const Unit &U, const std::string &Path) { @@ -90,10 +98,11 @@ void AppendToFile(const uint8_t *Data, size_t Size, const std::string &Path) { fclose(Out); } -void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V, - long *Epoch, size_t MaxSize, bool ExitOnError) { +void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V, long *Epoch, + size_t MaxSize, bool ExitOnError, + std::vector<std::string> *VPaths) { long E = Epoch ? *Epoch : 0; - Vector<std::string> Files; + std::vector<std::string> Files; ListFilesInDirRecursive(Path, Epoch, &Files, /*TopDir*/true); size_t NumLoaded = 0; for (size_t i = 0; i < Files.size(); i++) { @@ -103,14 +112,16 @@ void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V, if ((NumLoaded & (NumLoaded - 1)) == 0 && NumLoaded >= 1024) Printf("Loaded %zd/%zd files from %s\n", NumLoaded, Files.size(), Path); auto S = FileToVector(X, MaxSize, ExitOnError); - if (!S.empty()) + if (!S.empty()) { V->push_back(S); + if (VPaths) + VPaths->push_back(X); + } } } - -void GetSizedFilesFromDir(const std::string &Dir, Vector<SizedFile> *V) { - Vector<std::string> Files; +void GetSizedFilesFromDir(const std::string &Dir, std::vector<SizedFile> *V) { + std::vector<std::string> Files; ListFilesInDirRecursive(Dir, 0, &Files, /*TopDir*/true); for (auto &File : Files) if (size_t Size = FileSize(File)) @@ -140,6 +151,11 @@ void CloseStdout() { DiscardOutput(1); } +void Puts(const char *Str) { + fputs(Str, OutputFile); + fflush(OutputFile); +} + void Printf(const char *Fmt, ...) { va_list ap; va_start(ap, Fmt); diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerIO.h b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerIO.h index abd25110d07d..874caad1baed 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerIO.h +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerIO.h @@ -32,8 +32,9 @@ void WriteToFile(const Unit &U, const std::string &Path); void AppendToFile(const uint8_t *Data, size_t Size, const std::string &Path); void AppendToFile(const std::string &Data, const std::string &Path); -void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V, - long *Epoch, size_t MaxSize, bool ExitOnError); +void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V, long *Epoch, + size_t MaxSize, bool ExitOnError, + std::vector<std::string> *VPaths = 0); // Returns "Dir/FileName" or equivalent for the current OS. std::string DirPlusFile(const std::string &DirPath, @@ -53,6 +54,11 @@ void DupAndCloseStderr(); void CloseStdout(); +// For testing. +FILE *GetOutputFile(); +void SetOutputFile(FILE *NewOutputFile); + +void Puts(const char *Str); void Printf(const char *Fmt, ...); void VPrintf(bool Verbose, const char *Fmt, ...); @@ -65,7 +71,7 @@ bool IsDirectory(const std::string &Path); size_t FileSize(const std::string &Path); void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, - Vector<std::string> *V, bool TopDir); + std::vector<std::string> *V, bool TopDir); bool MkDirRecursive(const std::string &Dir); void RmDirRecursive(const std::string &Dir); @@ -84,7 +90,7 @@ struct SizedFile { bool operator<(const SizedFile &B) const { return Size < B.Size; } }; -void GetSizedFilesFromDir(const std::string &Dir, Vector<SizedFile> *V); +void GetSizedFilesFromDir(const std::string &Dir, std::vector<SizedFile> *V); char GetSeparator(); bool IsSeparator(char C); diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerIOPosix.cpp b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerIOPosix.cpp index 4706a40959be..3700fb098e55 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerIOPosix.cpp +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerIOPosix.cpp @@ -53,7 +53,7 @@ std::string Basename(const std::string &Path) { } void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, - Vector<std::string> *V, bool TopDir) { + std::vector<std::string> *V, bool TopDir) { auto E = GetEpoch(Dir); if (Epoch) if (E && *Epoch >= E) return; @@ -78,7 +78,6 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, *Epoch = E; } - void IterateDirRecursive(const std::string &Dir, void (*DirPreCallback)(const std::string &Dir), void (*DirPostCallback)(const std::string &Dir), diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerIOWindows.cpp b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerIOWindows.cpp index 61ad35e281f5..6771fc173c91 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerIOWindows.cpp +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerIOWindows.cpp @@ -111,7 +111,7 @@ size_t FileSize(const std::string &Path) { } void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, - Vector<std::string> *V, bool TopDir) { + std::vector<std::string> *V, bool TopDir) { auto E = GetEpoch(Dir); if (Epoch) if (E && *Epoch >= E) return; @@ -159,7 +159,6 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, *Epoch = E; } - void IterateDirRecursive(const std::string &Dir, void (*DirPreCallback)(const std::string &Dir), void (*DirPostCallback)(const std::string &Dir), @@ -297,9 +296,8 @@ static size_t ParseServerAndShare(const std::string &FileName, return Pos - Offset; } -// Parse the given Ref string from the position Offset, to exactly match the given -// string Patt. -// Returns number of characters considered if successful. +// Parse the given Ref string from the position Offset, to exactly match the +// given string Patt. Returns number of characters considered if successful. static size_t ParseCustomString(const std::string &Ref, size_t Offset, const char *Patt) { size_t Len = strlen(Patt); diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerInterceptors.cpp b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerInterceptors.cpp index b87798603fda..d5b0a42fd3fb 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerInterceptors.cpp +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerInterceptors.cpp @@ -25,6 +25,7 @@ } #include <cassert> +#include <cstddef> // for size_t #include <cstdint> #include <dlfcn.h> // for dlsym() diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerInternal.h b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerInternal.h index 37c8a01dc3c6..88504705137a 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerInternal.h +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerInternal.h @@ -29,14 +29,13 @@ namespace fuzzer { using namespace std::chrono; -class Fuzzer { +class Fuzzer final { public: - Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, - FuzzingOptions Options); - ~Fuzzer(); - void Loop(Vector<SizedFile> &CorporaFiles); - void ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles); + const FuzzingOptions &Options); + ~Fuzzer() = delete; + void Loop(std::vector<SizedFile> &CorporaFiles); + void ReadAndExecuteSeedCorpora(std::vector<SizedFile> &CorporaFiles); void MinimizeCrashLoop(const Unit &U); void RereadOutputCorpus(size_t MaxSize); @@ -65,15 +64,19 @@ public: static void StaticFileSizeExceedCallback(); static void StaticGracefulExitCallback(); - void ExecuteCallback(const uint8_t *Data, size_t Size); + // Executes the target callback on {Data, Size} once. + // Returns false if the input was rejected by the target (target returned -1), + // and true otherwise. + bool ExecuteCallback(const uint8_t *Data, size_t Size); bool RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile = false, InputInfo *II = nullptr, bool ForceAddToCorpus = false, bool *FoundUniqFeatures = nullptr); void TPCUpdateObservedPCs(); // Merge Corpora[1:] into Corpora[0]. - void Merge(const Vector<std::string> &Corpora); - void CrashResistantMergeInternalStep(const std::string &ControlFilePath); + void Merge(const std::vector<std::string> &Corpora); + void CrashResistantMergeInternalStep(const std::string &ControlFilePath, + bool IsSetCoverMerge); MutationDispatcher &GetMD() { return MD; } void PrintFinalStats(); void SetMaxInputLen(size_t MaxInputLen); @@ -87,6 +90,7 @@ public: void HandleMalloc(size_t Size); static void MaybeExitGracefully(); + static int InterruptExitCode(); std::string WriteToOutputCorpus(const Unit &U); private: @@ -141,7 +145,7 @@ private: size_t MaxMutationLen = 0; size_t TmpMaxMutationLen = 0; - Vector<uint32_t> UniqFeatureSetTmp; + std::vector<uint32_t> UniqFeatureSetTmp; // Need to know our own thread. static thread_local bool IsMyThread; diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerLoop.cpp b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerLoop.cpp index 6e3bf44f8b45..935dd2342e18 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerLoop.cpp +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerLoop.cpp @@ -136,7 +136,7 @@ void Fuzzer::HandleMalloc(size_t Size) { } Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, - FuzzingOptions Options) + const FuzzingOptions &Options) : CB(CB), Corpus(Corpus), MD(MD), Options(Options) { if (EF->__sanitizer_set_death_callback) EF->__sanitizer_set_death_callback(StaticDeathCallback); @@ -160,8 +160,6 @@ Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, memset(BaseSha1, 0, sizeof(BaseSha1)); } -Fuzzer::~Fuzzer() {} - void Fuzzer::AllocateCurrentUnitData() { if (CurrentUnitData || MaxInputLen == 0) return; @@ -262,6 +260,11 @@ void Fuzzer::MaybeExitGracefully() { _Exit(0); } +int Fuzzer::InterruptExitCode() { + assert(F); + return F->Options.InterruptExitCode; +} + void Fuzzer::InterruptCallback() { Printf("==%lu== libFuzzer: run interrupted; exiting\n", GetPid()); PrintFinalStats(); @@ -296,7 +299,7 @@ void Fuzzer::AlarmCallback() { Printf(" and the timeout value is %d (use -timeout=N to change)\n", Options.UnitTimeoutSec); DumpCurrentUnit("timeout-"); - Printf("==%lu== ERROR: libFuzzer: timeout after %d seconds\n", GetPid(), + Printf("==%lu== ERROR: libFuzzer: timeout after %zu seconds\n", GetPid(), Seconds); PrintStackTrace(); Printf("SUMMARY: libFuzzer: timeout\n"); @@ -309,9 +312,8 @@ void Fuzzer::RssLimitCallback() { if (EF->__sanitizer_acquire_crash_state && !EF->__sanitizer_acquire_crash_state()) return; - Printf( - "==%lu== ERROR: libFuzzer: out-of-memory (used: %zdMb; limit: %zdMb)\n", - GetPid(), GetPeakRSSMb(), Options.RssLimitMb); + Printf("==%lu== ERROR: libFuzzer: out-of-memory (used: %zdMb; limit: %dMb)\n", + GetPid(), GetPeakRSSMb(), Options.RssLimitMb); Printf(" To change the out-of-memory limit use -rss_limit_mb=<N>\n\n"); PrintMemoryProfile(); DumpCurrentUnit("oom-"); @@ -366,7 +368,7 @@ void Fuzzer::PrintFinalStats() { Printf("stat::number_of_executed_units: %zd\n", TotalNumberOfRuns); Printf("stat::average_exec_per_sec: %zd\n", ExecPerSec); Printf("stat::new_units_added: %zd\n", NumberOfNewUnitsAdded); - Printf("stat::slowest_unit_time_sec: %zd\n", TimeOfLongestUnitInSeconds); + Printf("stat::slowest_unit_time_sec: %ld\n", TimeOfLongestUnitInSeconds); Printf("stat::peak_rss_mb: %zd\n", GetPeakRSSMb()); } @@ -388,7 +390,7 @@ void Fuzzer::SetMaxMutationLen(size_t MaxMutationLen) { void Fuzzer::CheckExitOnSrcPosOrItem() { if (!Options.ExitOnSrcPos.empty()) { - static auto *PCsSet = new Set<uintptr_t>; + static auto *PCsSet = new std::set<uintptr_t>; auto HandlePC = [&](const TracePC::PCTableEntry *TE) { if (!PCsSet->insert(TE->PC).second) return; @@ -413,20 +415,26 @@ void Fuzzer::CheckExitOnSrcPosOrItem() { void Fuzzer::RereadOutputCorpus(size_t MaxSize) { if (Options.OutputCorpus.empty() || !Options.ReloadIntervalSec) return; - Vector<Unit> AdditionalCorpus; - ReadDirToVectorOfUnits(Options.OutputCorpus.c_str(), &AdditionalCorpus, - &EpochOfLastReadOfOutputCorpus, MaxSize, - /*ExitOnError*/ false); + std::vector<Unit> AdditionalCorpus; + std::vector<std::string> AdditionalCorpusPaths; + ReadDirToVectorOfUnits( + Options.OutputCorpus.c_str(), &AdditionalCorpus, + &EpochOfLastReadOfOutputCorpus, MaxSize, + /*ExitOnError*/ false, + (Options.Verbosity >= 2 ? &AdditionalCorpusPaths : nullptr)); if (Options.Verbosity >= 2) Printf("Reload: read %zd new units.\n", AdditionalCorpus.size()); bool Reloaded = false; - for (auto &U : AdditionalCorpus) { + for (size_t i = 0; i != AdditionalCorpus.size(); ++i) { + auto &U = AdditionalCorpus[i]; if (U.size() > MaxSize) U.resize(MaxSize); if (!Corpus.HasUnit(U)) { if (RunOne(U.data(), U.size())) { CheckExitOnSrcPosOrItem(); Reloaded = true; + if (Options.Verbosity >= 2) + Printf("Reloaded %s\n", AdditionalCorpusPaths[i].c_str()); } } } @@ -440,17 +448,18 @@ void Fuzzer::PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size) { if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1)) && secondsSinceProcessStartUp() >= 2) PrintStats("pulse "); - if (TimeOfUnit > TimeOfLongestUnitInSeconds * 1.1 && - TimeOfUnit >= Options.ReportSlowUnits) { + auto Threshhold = + static_cast<long>(static_cast<double>(TimeOfLongestUnitInSeconds) * 1.1); + if (TimeOfUnit > Threshhold && TimeOfUnit >= Options.ReportSlowUnits) { TimeOfLongestUnitInSeconds = TimeOfUnit; - Printf("Slowest unit: %zd s:\n", TimeOfLongestUnitInSeconds); + Printf("Slowest unit: %ld s:\n", TimeOfLongestUnitInSeconds); WriteUnitToFileWithPrefix({Data, Data + Size}, "slow-unit-"); } } static void WriteFeatureSetToFile(const std::string &FeaturesDir, const std::string &FileName, - const Vector<uint32_t> &FeatureSet) { + const std::vector<uint32_t> &FeatureSet) { if (FeaturesDir.empty() || FeatureSet.empty()) return; WriteToFile(reinterpret_cast<const uint8_t *>(FeatureSet.data()), FeatureSet.size() * sizeof(FeatureSet[0]), @@ -501,15 +510,17 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, bool *FoundUniqFeatures) { if (!Size) return false; + // Largest input length should be INT_MAX. + assert(Size < std::numeric_limits<uint32_t>::max()); - ExecuteCallback(Data, Size); + if(!ExecuteCallback(Data, Size)) return false; auto TimeOfUnit = duration_cast<microseconds>(UnitStopTime - UnitStartTime); UniqFeatureSetTmp.clear(); size_t FoundUniqFeaturesOfII = 0; size_t NumUpdatesBefore = Corpus.NumFeatureUpdates(); - TPC.CollectFeatures([&](size_t Feature) { - if (Corpus.AddFeature(Feature, Size, Options.Shrink)) + TPC.CollectFeatures([&](uint32_t Feature) { + if (Corpus.AddFeature(Feature, static_cast<uint32_t>(Size), Options.Shrink)) UniqFeatureSetTmp.push_back(Feature); if (Options.Entropic) Corpus.UpdateFeatureFrequency(II, Feature); @@ -539,7 +550,7 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, FoundUniqFeaturesOfII == II->UniqFeatureSet.size() && II->U.size() > Size) { auto OldFeaturesFile = Sha1ToString(II->Sha1); - Corpus.Replace(II, {Data, Data + Size}); + Corpus.Replace(II, {Data, Data + Size}, TimeOfUnit); RenameFeatureSetFile(Options.FeaturesDir, OldFeaturesFile, Sha1ToString(II->Sha1)); return true; @@ -575,7 +586,10 @@ static bool LooseMemeq(const uint8_t *A, const uint8_t *B, size_t Size) { !memcmp(A + Size - Limit / 2, B + Size - Limit / 2, Limit / 2); } -void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { +// This method is not inlined because it would cause a test to fail where it +// is part of the stack unwinding. See D97975 for details. +ATTRIBUTE_NOINLINE bool Fuzzer::ExecuteCallback(const uint8_t *Data, + size_t Size) { TPC.RecordInitialStack(); TotalNumberOfRuns++; assert(InFuzzingThread()); @@ -590,23 +604,24 @@ void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { if (CurrentUnitData && CurrentUnitData != Data) memcpy(CurrentUnitData, Data, Size); CurrentUnitSize = Size; + int CBRes = 0; { ScopedEnableMsanInterceptorChecks S; AllocTracer.Start(Options.TraceMalloc); UnitStartTime = system_clock::now(); TPC.ResetMaps(); RunningUserCallback = true; - int Res = CB(DataCopy, Size); + CBRes = CB(DataCopy, Size); RunningUserCallback = false; UnitStopTime = system_clock::now(); - (void)Res; - assert(Res == 0); + assert(CBRes == 0 || CBRes == -1); HasMoreMallocsThanFrees = AllocTracer.Stop(); } if (!LooseMemeq(DataCopy, Data, Size)) CrashOnOverwrittenData(); CurrentUnitSize = 0; delete[] DataCopy; + return CBRes == 0; } std::string Fuzzer::WriteToOutputCorpus(const Unit &U) { @@ -772,7 +787,7 @@ void Fuzzer::PurgeAllocator() { LastAllocatorPurgeAttemptTime = system_clock::now(); } -void Fuzzer::ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles) { +void Fuzzer::ReadAndExecuteSeedCorpora(std::vector<SizedFile> &CorporaFiles) { const size_t kMaxSaneLen = 1 << 20; const size_t kMinDefaultLen = 4096; size_t MaxSize = 0; @@ -784,7 +799,7 @@ void Fuzzer::ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles) { TotalSize += File.Size; } if (Options.MaxLen == 0) - SetMaxInputLen(std::min(std::max(kMinDefaultLen, MaxSize), kMaxSaneLen)); + SetMaxInputLen(std::clamp(MaxSize, kMinDefaultLen, kMaxSaneLen)); assert(MaxInputLen > 0); // Test the callback with empty input and never try it again. @@ -831,13 +846,20 @@ void Fuzzer::ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles) { } if (Corpus.empty() && Options.MaxNumberOfRuns) { - Printf("ERROR: no interesting inputs were found. " - "Is the code instrumented for coverage? Exiting.\n"); - exit(1); + Printf("WARNING: no interesting inputs were found so far. " + "Is the code instrumented for coverage?\n" + "This may also happen if the target rejected all inputs we tried so " + "far\n"); + // The remaining logic requires that the corpus is not empty, + // so we add one fake input to the in-memory corpus. + Corpus.AddToCorpus({'\n'}, /*NumFeatures=*/1, /*MayDeleteFile=*/true, + /*HasFocusFunction=*/false, /*NeverReduce=*/false, + /*TimeOfUnit=*/duration_cast<microseconds>(0s), {0}, DFT, + /*BaseII*/ nullptr); } } -void Fuzzer::Loop(Vector<SizedFile> &CorporaFiles) { +void Fuzzer::Loop(std::vector<SizedFile> &CorporaFiles) { auto FocusFunctionOrAuto = Options.FocusFunction; DFT.Init(Options.DataFlowTrace, &FocusFunctionOrAuto, CorporaFiles, MD.GetRand()); diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerMerge.cpp b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerMerge.cpp index e3ad8b3851e7..8c8806e8aafd 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerMerge.cpp +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerMerge.cpp @@ -77,14 +77,15 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { size_t ExpectedStartMarker = 0; const size_t kInvalidStartMarker = -1; size_t LastSeenStartMarker = kInvalidStartMarker; - Vector<uint32_t> TmpFeatures; - Set<uint32_t> PCs; + bool HaveFtMarker = true; + std::vector<uint32_t> TmpFeatures; + std::set<uint32_t> PCs; while (std::getline(IS, Line, '\n')) { std::istringstream ISS1(Line); std::string Marker; - size_t N; - ISS1 >> Marker; - ISS1 >> N; + uint32_t N; + if (!(ISS1 >> Marker) || !(ISS1 >> N)) + return false; if (Marker == "STARTED") { // STARTED FILE_ID FILE_SIZE if (ExpectedStartMarker != N) @@ -93,12 +94,13 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { LastSeenStartMarker = ExpectedStartMarker; assert(ExpectedStartMarker < Files.size()); ExpectedStartMarker++; + HaveFtMarker = false; } else if (Marker == "FT") { // FT FILE_ID COV1 COV2 COV3 ... size_t CurrentFileIdx = N; if (CurrentFileIdx != LastSeenStartMarker) return false; - LastSeenStartMarker = kInvalidStartMarker; + HaveFtMarker = true; if (ParseCoverage) { TmpFeatures.clear(); // use a vector from outer scope to avoid resizes. while (ISS1 >> N) @@ -108,6 +110,8 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { } } else if (Marker == "COV") { size_t CurrentFileIdx = N; + if (CurrentFileIdx != LastSeenStartMarker) + return false; if (ParseCoverage) while (ISS1 >> N) if (PCs.insert(N).second) @@ -116,7 +120,7 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { return false; } } - if (LastSeenStartMarker != kInvalidStartMarker) + if (!HaveFtMarker && LastSeenStartMarker != kInvalidStartMarker) LastFailure = Files[LastSeenStartMarker].Name; FirstNotProcessedFile = ExpectedStartMarker; @@ -132,13 +136,16 @@ size_t Merger::ApproximateMemoryConsumption() const { // Decides which files need to be merged (add those to NewFiles). // Returns the number of new features added. -size_t Merger::Merge(const Set<uint32_t> &InitialFeatures, - Set<uint32_t> *NewFeatures, - const Set<uint32_t> &InitialCov, Set<uint32_t> *NewCov, - Vector<std::string> *NewFiles) { +size_t Merger::Merge(const std::set<uint32_t> &InitialFeatures, + std::set<uint32_t> *NewFeatures, + const std::set<uint32_t> &InitialCov, + std::set<uint32_t> *NewCov, + std::vector<std::string> *NewFiles) { NewFiles->clear(); + NewFeatures->clear(); + NewCov->clear(); assert(NumFilesInFirstCorpus <= Files.size()); - Set<uint32_t> AllFeatures = InitialFeatures; + std::set<uint32_t> AllFeatures = InitialFeatures; // What features are in the initial corpus? for (size_t i = 0; i < NumFilesInFirstCorpus; i++) { @@ -148,7 +155,7 @@ size_t Merger::Merge(const Set<uint32_t> &InitialFeatures, // Remove all features that we already know from all other inputs. for (size_t i = NumFilesInFirstCorpus; i < Files.size(); i++) { auto &Cur = Files[i].Features; - Vector<uint32_t> Tmp; + std::vector<uint32_t> Tmp; std::set_difference(Cur.begin(), Cur.end(), AllFeatures.begin(), AllFeatures.end(), std::inserter(Tmp, Tmp.begin())); Cur.swap(Tmp); @@ -186,15 +193,16 @@ size_t Merger::Merge(const Set<uint32_t> &InitialFeatures, return NewFeatures->size(); } -Set<uint32_t> Merger::AllFeatures() const { - Set<uint32_t> S; +std::set<uint32_t> Merger::AllFeatures() const { + std::set<uint32_t> S; for (auto &File : Files) S.insert(File.Features.begin(), File.Features.end()); return S; } // Inner process. May crash if the target crashes. -void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { +void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath, + bool IsSetCoverMerge) { Printf("MERGE-INNER: using the control file '%s'\n", CFPath.c_str()); Merger M; std::ifstream IF(CFPath); @@ -210,11 +218,11 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { M.Files.size() - M.FirstNotProcessedFile); std::ofstream OF(CFPath, std::ofstream::out | std::ofstream::app); - Set<size_t> AllFeatures; + std::set<size_t> AllFeatures; auto PrintStatsWrapper = [this, &AllFeatures](const char* Where) { this->PrintStats(Where, "\n", 0, AllFeatures.size()); }; - Set<const TracePC::PCTableEntry *> AllPCs; + std::set<const TracePC::PCTableEntry *> AllPCs; for (size_t i = M.FirstNotProcessedFile; i < M.Files.size(); i++) { Fuzzer::MaybeExitGracefully(); auto U = FileToVector(M.Files[i].Name); @@ -232,13 +240,14 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { // Collect coverage. We are iterating over the files in this order: // * First, files in the initial corpus ordered by size, smallest first. // * Then, all other files, smallest first. - // So it makes no sense to record all features for all files, instead we - // only record features that were not seen before. - Set<size_t> UniqFeatures; - TPC.CollectFeatures([&](size_t Feature) { - if (AllFeatures.insert(Feature).second) - UniqFeatures.insert(Feature); - }); + std::set<size_t> Features; + if (IsSetCoverMerge) + TPC.CollectFeatures([&](size_t Feature) { Features.insert(Feature); }); + else + TPC.CollectFeatures([&](size_t Feature) { + if (AllFeatures.insert(Feature).second) + Features.insert(Feature); + }); TPC.UpdateObservedPCs(); // Show stats. if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1))) @@ -247,7 +256,7 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { PrintStatsWrapper("LOADED"); // Write the post-run marker and the coverage. OF << "FT " << i; - for (size_t F : UniqFeatures) + for (size_t F : Features) OF << " " << F; OF << "\n"; OF << "COV " << i; @@ -261,15 +270,137 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { PrintStatsWrapper("DONE "); } -static size_t WriteNewControlFile(const std::string &CFPath, - const Vector<SizedFile> &OldCorpus, - const Vector<SizedFile> &NewCorpus, - const Vector<MergeFileInfo> &KnownFiles) { +// Merges all corpora into the first corpus. A file is added into +// the first corpus only if it adds new features. Unlike `Merger::Merge`, +// this implementation calculates an approximation of the minimum set +// of corpora files, that cover all known features (set cover problem). +// Generally, this means that files with more features are preferred for +// merge into the first corpus. When two files have the same number of +// features, the smaller one is preferred. +size_t Merger::SetCoverMerge(const std::set<uint32_t> &InitialFeatures, + std::set<uint32_t> *NewFeatures, + const std::set<uint32_t> &InitialCov, + std::set<uint32_t> *NewCov, + std::vector<std::string> *NewFiles) { + assert(NumFilesInFirstCorpus <= Files.size()); + NewFiles->clear(); + NewFeatures->clear(); + NewCov->clear(); + std::set<uint32_t> AllFeatures; + // 1 << 21 - 1 is the maximum feature index. + // See 'kFeatureSetSize' in 'FuzzerCorpus.h'. + const uint32_t kFeatureSetSize = 1 << 21; + std::vector<bool> Covered(kFeatureSetSize, false); + size_t NumCovered = 0; + + std::set<uint32_t> ExistingFeatures = InitialFeatures; + for (size_t i = 0; i < NumFilesInFirstCorpus; ++i) + ExistingFeatures.insert(Files[i].Features.begin(), Files[i].Features.end()); + + // Mark the existing features as covered. + for (const auto &F : ExistingFeatures) { + if (!Covered[F % kFeatureSetSize]) { + ++NumCovered; + Covered[F % kFeatureSetSize] = true; + } + // Calculate an underestimation of the set of covered features + // since the `Covered` bitvector is smaller than the feature range. + AllFeatures.insert(F % kFeatureSetSize); + } + + std::set<size_t> RemainingFiles; + for (size_t i = NumFilesInFirstCorpus; i < Files.size(); ++i) { + // Construct an incremental sequence which represent the + // indices to all files (excluding those in the initial corpus). + // RemainingFiles = range(NumFilesInFirstCorpus..Files.size()). + RemainingFiles.insert(i); + // Insert this file's unique features to all features. + for (const auto &F : Files[i].Features) + AllFeatures.insert(F % kFeatureSetSize); + } + + // Integrate files into Covered until set is complete. + while (NumCovered != AllFeatures.size()) { + // Index to file with largest number of unique features. + size_t MaxFeaturesIndex = NumFilesInFirstCorpus; + // Indices to remove from RemainingFiles. + std::set<size_t> RemoveIndices; + // Running max unique feature count. + // Updated upon finding a file with more features. + size_t MaxNumFeatures = 0; + + // Iterate over all files not yet integrated into Covered, + // to find the file which has the largest number of + // features that are not already in Covered. + for (const auto &i : RemainingFiles) { + const auto &File = Files[i]; + size_t CurrentUnique = 0; + // Count number of features in this file + // which are not yet in Covered. + for (const auto &F : File.Features) + if (!Covered[F % kFeatureSetSize]) + ++CurrentUnique; + + if (CurrentUnique == 0) { + // All features in this file are already in Covered: skip next time. + RemoveIndices.insert(i); + } else if (CurrentUnique > MaxNumFeatures || + (CurrentUnique == MaxNumFeatures && + File.Size < Files[MaxFeaturesIndex].Size)) { + // Update the max features file based on unique features + // Break ties by selecting smaller files. + MaxNumFeatures = CurrentUnique; + MaxFeaturesIndex = i; + } + } + // Must be a valid index/ + assert(MaxFeaturesIndex < Files.size()); + // Remove any feature-less files found. + for (const auto &i : RemoveIndices) + RemainingFiles.erase(i); + if (MaxNumFeatures == 0) { + // Did not find a file that adds unique features. + // This means that we should have no remaining files. + assert(RemainingFiles.size() == 0); + assert(NumCovered == AllFeatures.size()); + break; + } + + // MaxFeaturesIndex must be an element of Remaining. + assert(RemainingFiles.find(MaxFeaturesIndex) != RemainingFiles.end()); + // Remove the file with the most features from Remaining. + RemainingFiles.erase(MaxFeaturesIndex); + const auto &MaxFeatureFile = Files[MaxFeaturesIndex]; + // Add the features of the max feature file to Covered. + for (const auto &F : MaxFeatureFile.Features) { + if (!Covered[F % kFeatureSetSize]) { + ++NumCovered; + Covered[F % kFeatureSetSize] = true; + NewFeatures->insert(F); + } + } + // Add the index to this file to the result. + NewFiles->push_back(MaxFeatureFile.Name); + // Update NewCov with the additional coverage + // that MaxFeatureFile provides. + for (const auto &C : MaxFeatureFile.Cov) + if (InitialCov.find(C) == InitialCov.end()) + NewCov->insert(C); + } + + return NewFeatures->size(); +} + +static size_t +WriteNewControlFile(const std::string &CFPath, + const std::vector<SizedFile> &OldCorpus, + const std::vector<SizedFile> &NewCorpus, + const std::vector<MergeFileInfo> &KnownFiles) { std::unordered_set<std::string> FilesToSkip; for (auto &SF: KnownFiles) FilesToSkip.insert(SF.Name); - Vector<std::string> FilesToUse; + std::vector<std::string> FilesToUse; auto MaybeUseFile = [=, &FilesToUse](std::string Name) { if (FilesToSkip.find(Name) == FilesToSkip.end()) FilesToUse.push_back(Name); @@ -297,19 +428,19 @@ static size_t WriteNewControlFile(const std::string &CFPath, } // Outer process. Does not call the target code and thus should not fail. -void CrashResistantMerge(const Vector<std::string> &Args, - const Vector<SizedFile> &OldCorpus, - const Vector<SizedFile> &NewCorpus, - Vector<std::string> *NewFiles, - const Set<uint32_t> &InitialFeatures, - Set<uint32_t> *NewFeatures, - const Set<uint32_t> &InitialCov, - Set<uint32_t> *NewCov, - const std::string &CFPath, - bool V /*Verbose*/) { +void CrashResistantMerge(const std::vector<std::string> &Args, + const std::vector<SizedFile> &OldCorpus, + const std::vector<SizedFile> &NewCorpus, + std::vector<std::string> *NewFiles, + const std::set<uint32_t> &InitialFeatures, + std::set<uint32_t> *NewFeatures, + const std::set<uint32_t> &InitialCov, + std::set<uint32_t> *NewCov, const std::string &CFPath, + bool V, /*Verbose*/ + bool IsSetCoverMerge) { if (NewCorpus.empty() && OldCorpus.empty()) return; // Nothing to merge. size_t NumAttempts = 0; - Vector<MergeFileInfo> KnownFiles; + std::vector<MergeFileInfo> KnownFiles; if (FileSize(CFPath)) { VPrintf(V, "MERGE-OUTER: non-empty control file provided: '%s'\n", CFPath.c_str()); @@ -361,6 +492,7 @@ void CrashResistantMerge(const Vector<std::string> &Args, // Every inner process should execute at least one input. Command BaseCmd(Args); BaseCmd.removeFlag("merge"); + BaseCmd.removeFlag("set_cover_merge"); BaseCmd.removeFlag("fork"); BaseCmd.removeFlag("collect_data_flow"); for (size_t Attempt = 1; Attempt <= NumAttempts; Attempt++) { @@ -368,14 +500,16 @@ void CrashResistantMerge(const Vector<std::string> &Args, VPrintf(V, "MERGE-OUTER: attempt %zd\n", Attempt); Command Cmd(BaseCmd); Cmd.addFlag("merge_control_file", CFPath); - Cmd.addFlag("merge_inner", "1"); + // If we are going to use the set cover implementation for + // minimization add the merge_inner=2 internal flag. + Cmd.addFlag("merge_inner", IsSetCoverMerge ? "2" : "1"); if (!V) { Cmd.setOutputFile(getDevNull()); Cmd.combineOutAndErr(); } auto ExitCode = ExecuteCommand(Cmd); if (!ExitCode) { - VPrintf(V, "MERGE-OUTER: succesfull in %zd attempt(s)\n", Attempt); + VPrintf(V, "MERGE-OUTER: successful in %zd attempt(s)\n", Attempt); break; } } @@ -393,7 +527,10 @@ void CrashResistantMerge(const Vector<std::string> &Args, M.ApproximateMemoryConsumption() >> 20, GetPeakRSSMb()); M.Files.insert(M.Files.end(), KnownFiles.begin(), KnownFiles.end()); - M.Merge(InitialFeatures, NewFeatures, InitialCov, NewCov, NewFiles); + if (IsSetCoverMerge) + M.SetCoverMerge(InitialFeatures, NewFeatures, InitialCov, NewCov, NewFiles); + else + M.Merge(InitialFeatures, NewFeatures, InitialCov, NewCov, NewFiles); VPrintf(V, "MERGE-OUTER: %zd new files with %zd new features added; " "%zd new coverage edges\n", NewFiles->size(), NewFeatures->size(), NewCov->size()); diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerMerge.h b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerMerge.h index e0c6bc539bdb..42f798e1da18 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerMerge.h +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerMerge.h @@ -41,6 +41,7 @@ #define LLVM_FUZZER_MERGE_H #include "FuzzerDefs.h" +#include "FuzzerIO.h" #include <istream> #include <ostream> @@ -52,11 +53,11 @@ namespace fuzzer { struct MergeFileInfo { std::string Name; size_t Size = 0; - Vector<uint32_t> Features, Cov; + std::vector<uint32_t> Features, Cov; }; struct Merger { - Vector<MergeFileInfo> Files; + std::vector<MergeFileInfo> Files; size_t NumFilesInFirstCorpus = 0; size_t FirstNotProcessedFile = 0; std::string LastFailure; @@ -64,23 +65,28 @@ struct Merger { bool Parse(std::istream &IS, bool ParseCoverage); bool Parse(const std::string &Str, bool ParseCoverage); void ParseOrExit(std::istream &IS, bool ParseCoverage); - size_t Merge(const Set<uint32_t> &InitialFeatures, Set<uint32_t> *NewFeatures, - const Set<uint32_t> &InitialCov, Set<uint32_t> *NewCov, - Vector<std::string> *NewFiles); + size_t Merge(const std::set<uint32_t> &InitialFeatures, + std::set<uint32_t> *NewFeatures, + const std::set<uint32_t> &InitialCov, std::set<uint32_t> *NewCov, + std::vector<std::string> *NewFiles); + size_t SetCoverMerge(const std::set<uint32_t> &InitialFeatures, + std::set<uint32_t> *NewFeatures, + const std::set<uint32_t> &InitialCov, + std::set<uint32_t> *NewCov, + std::vector<std::string> *NewFiles); size_t ApproximateMemoryConsumption() const; - Set<uint32_t> AllFeatures() const; + std::set<uint32_t> AllFeatures() const; }; -void CrashResistantMerge(const Vector<std::string> &Args, - const Vector<SizedFile> &OldCorpus, - const Vector<SizedFile> &NewCorpus, - Vector<std::string> *NewFiles, - const Set<uint32_t> &InitialFeatures, - Set<uint32_t> *NewFeatures, - const Set<uint32_t> &InitialCov, - Set<uint32_t> *NewCov, - const std::string &CFPath, - bool Verbose); +void CrashResistantMerge(const std::vector<std::string> &Args, + const std::vector<SizedFile> &OldCorpus, + const std::vector<SizedFile> &NewCorpus, + std::vector<std::string> *NewFiles, + const std::set<uint32_t> &InitialFeatures, + std::set<uint32_t> *NewFeatures, + const std::set<uint32_t> &InitialCov, + std::set<uint32_t> *NewCov, const std::string &CFPath, + bool Verbose, bool IsSetCoverMerge); } // namespace fuzzer diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerMutate.cpp b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerMutate.cpp index cf34a9fe8e2e..1abce16d70d9 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerMutate.cpp +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerMutate.cpp @@ -61,14 +61,20 @@ MutationDispatcher::MutationDispatcher(Random &Rand, } static char RandCh(Random &Rand) { - if (Rand.RandBool()) return Rand(256); + if (Rand.RandBool()) + return static_cast<char>(Rand(256)); const char Special[] = "!*'();:@&=+$,/?%#[]012Az-`~.\xff\x00"; return Special[Rand(sizeof(Special) - 1)]; } size_t MutationDispatcher::Mutate_Custom(uint8_t *Data, size_t Size, size_t MaxSize) { - return EF->LLVMFuzzerCustomMutator(Data, Size, MaxSize, Rand.Rand()); + if (EF->__msan_unpoison) + EF->__msan_unpoison(Data, Size); + if (EF->__msan_unpoison_param) + EF->__msan_unpoison_param(4); + return EF->LLVMFuzzerCustomMutator(Data, Size, MaxSize, + Rand.Rand<unsigned int>()); } size_t MutationDispatcher::Mutate_CustomCrossOver(uint8_t *Data, size_t Size, @@ -81,8 +87,18 @@ size_t MutationDispatcher::Mutate_CustomCrossOver(uint8_t *Data, size_t Size, return 0; CustomCrossOverInPlaceHere.resize(MaxSize); auto &U = CustomCrossOverInPlaceHere; + + if (EF->__msan_unpoison) { + EF->__msan_unpoison(Data, Size); + EF->__msan_unpoison(Other.data(), Other.size()); + EF->__msan_unpoison(U.data(), U.size()); + } + if (EF->__msan_unpoison_param) + EF->__msan_unpoison_param(7); size_t NewSize = EF->LLVMFuzzerCustomCrossOver( - Data, Size, Other.data(), Other.size(), U.data(), U.size(), Rand.Rand()); + Data, Size, Other.data(), Other.size(), U.data(), U.size(), + Rand.Rand<unsigned int>()); + if (!NewSize) return 0; assert(NewSize <= MaxSize && "CustomCrossOver returned overisized unit"); @@ -135,7 +151,8 @@ size_t MutationDispatcher::Mutate_InsertRepeatedBytes(uint8_t *Data, // Insert new values at Data[Idx]. memmove(Data + Idx + N, Data + Idx, Size - Idx); // Give preference to 0x00 and 0xff. - uint8_t Byte = Rand.RandBool() ? Rand(256) : (Rand.RandBool() ? 0 : 255); + uint8_t Byte = static_cast<uint8_t>( + Rand.RandBool() ? Rand(256) : (Rand.RandBool() ? 0 : 255)); for (size_t i = 0; i < N; i++) Data[Idx + i] = Byte; return Size + N; @@ -178,7 +195,8 @@ size_t MutationDispatcher::ApplyDictionaryEntry(uint8_t *Data, size_t Size, Size += W.size(); } else { // Overwrite some bytes with W. if (W.size() > Size) return 0; - size_t Idx = UsePositionHint ? DE.GetPositionHint() : Rand(Size - W.size()); + size_t Idx = + UsePositionHint ? DE.GetPositionHint() : Rand(Size + 1 - W.size()); memcpy(Data + Idx, W.data(), W.size()); } return Size; @@ -227,8 +245,8 @@ DictionaryEntry MutationDispatcher::MakeDictionaryEntryFromCMP( T Arg1, T Arg2, const uint8_t *Data, size_t Size) { if (Rand.RandBool()) Arg1 = Bswap(Arg1); if (Rand.RandBool()) Arg2 = Bswap(Arg2); - T Arg1Mutation = Arg1 + Rand(-1, 1); - T Arg2Mutation = Arg2 + Rand(-1, 1); + T Arg1Mutation = static_cast<T>(Arg1 + Rand(-1, 1)); + T Arg2Mutation = static_cast<T>(Arg2 + Rand(-1, 1)); return MakeDictionaryEntryFromCMP(&Arg1, &Arg2, &Arg1Mutation, &Arg2Mutation, sizeof(Arg1), Data, Size); } @@ -245,23 +263,23 @@ size_t MutationDispatcher::Mutate_AddWordFromTORC( DictionaryEntry DE; switch (Rand(4)) { case 0: { - auto X = TPC.TORC8.Get(Rand.Rand()); + auto X = TPC.TORC8.Get(Rand.Rand<size_t>()); DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size); } break; case 1: { - auto X = TPC.TORC4.Get(Rand.Rand()); + auto X = TPC.TORC4.Get(Rand.Rand<size_t>()); if ((X.A >> 16) == 0 && (X.B >> 16) == 0 && Rand.RandBool()) DE = MakeDictionaryEntryFromCMP((uint16_t)X.A, (uint16_t)X.B, Data, Size); else DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size); } break; case 2: { - auto X = TPC.TORCW.Get(Rand.Rand()); + auto X = TPC.TORCW.Get(Rand.Rand<size_t>()); DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size); } break; case 3: if (Options.UseMemmem) { - auto X = TPC.MMT.Get(Rand.Rand()); - DE = DictionaryEntry(X); + auto X = TPC.MMT.Get(Rand.Rand<size_t>()); + DE = DictionaryEntry(X); } break; default: assert(0); @@ -387,12 +405,12 @@ size_t ChangeBinaryInteger(uint8_t *Data, size_t Size, Random &Rand) { assert(Off + sizeof(T) <= Size); T Val; if (Off < 64 && !Rand(4)) { - Val = Size; + Val = static_cast<T>(Size); if (Rand.RandBool()) Val = Bswap(Val); } else { memcpy(&Val, Data + Off, sizeof(Val)); - T Add = Rand(21); + T Add = static_cast<T>(Rand(21)); Add -= 10; if (Rand.RandBool()) Val = Bswap(T(Bswap(Val) + Add)); // Add assuming different endiannes. @@ -462,12 +480,12 @@ void MutationDispatcher::RecordSuccessfulMutationSequence() { assert(DE->GetW().size()); // Linear search is fine here as this happens seldom. if (!PersistentAutoDictionary.ContainsWord(DE->GetW())) - PersistentAutoDictionary.push_back({DE->GetW(), 1}); + PersistentAutoDictionary.push_back(*DE); } } void MutationDispatcher::PrintRecommendedDictionary() { - Vector<DictionaryEntry> V; + std::vector<DictionaryEntry> V; for (auto &DE : PersistentAutoDictionary) if (!ManualDictionary.ContainsWord(DE.GetW())) V.push_back(DE); @@ -503,7 +521,7 @@ void MutationDispatcher::PrintMutationSequence(bool Verbose) { std::string MutationDispatcher::MutationSequence() { std::string MS; - for (auto M : CurrentMutatorSequence) { + for (const auto &M : CurrentMutatorSequence) { MS += M.Name; MS += "-"; } @@ -522,7 +540,7 @@ size_t MutationDispatcher::DefaultMutate(uint8_t *Data, size_t Size, // Mutates Data in place, returns new size. size_t MutationDispatcher::MutateImpl(uint8_t *Data, size_t Size, size_t MaxSize, - Vector<Mutator> &Mutators) { + std::vector<Mutator> &Mutators) { assert(MaxSize > 0); // Some mutations may fail (e.g. can't insert more bytes if Size == MaxSize), // in which case they will return 0. @@ -544,7 +562,7 @@ size_t MutationDispatcher::MutateImpl(uint8_t *Data, size_t Size, // Mask represents the set of Data bytes that are worth mutating. size_t MutationDispatcher::MutateWithMask(uint8_t *Data, size_t Size, size_t MaxSize, - const Vector<uint8_t> &Mask) { + const std::vector<uint8_t> &Mask) { size_t MaskedSize = std::min(Size, Mask.size()); // * Copy the worthy bytes into a temporary array T // * Mutate T diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerMutate.h b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerMutate.h index fd37191156d3..97704e2160aa 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerMutate.h +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerMutate.h @@ -77,7 +77,7 @@ public: /// that have '1' in Mask. /// Mask.size() should be >= Size. size_t MutateWithMask(uint8_t *Data, size_t Size, size_t MaxSize, - const Vector<uint8_t> &Mask); + const std::vector<uint8_t> &Mask); /// Applies one of the default mutations. Provided as a service /// to mutation authors. @@ -104,7 +104,7 @@ public: size_t AddWordFromDictionary(Dictionary &D, uint8_t *Data, size_t Size, size_t MaxSize); size_t MutateImpl(uint8_t *Data, size_t Size, size_t MaxSize, - Vector<Mutator> &Mutators); + std::vector<Mutator> &Mutators); size_t InsertPartOf(const uint8_t *From, size_t FromSize, uint8_t *To, size_t ToSize, size_t MaxToSize); @@ -133,22 +133,22 @@ public: // entries that led to successful discoveries in the past mutations. Dictionary PersistentAutoDictionary; - Vector<DictionaryEntry *> CurrentDictionaryEntrySequence; + std::vector<DictionaryEntry *> CurrentDictionaryEntrySequence; static const size_t kCmpDictionaryEntriesDequeSize = 16; DictionaryEntry CmpDictionaryEntriesDeque[kCmpDictionaryEntriesDequeSize]; size_t CmpDictionaryEntriesDequeIdx = 0; const Unit *CrossOverWith = nullptr; - Vector<uint8_t> MutateInPlaceHere; - Vector<uint8_t> MutateWithMaskTemp; + std::vector<uint8_t> MutateInPlaceHere; + std::vector<uint8_t> MutateWithMaskTemp; // CustomCrossOver needs its own buffer as a custom implementation may call // LLVMFuzzerMutate, which in turn may resize MutateInPlaceHere. - Vector<uint8_t> CustomCrossOverInPlaceHere; + std::vector<uint8_t> CustomCrossOverInPlaceHere; - Vector<Mutator> Mutators; - Vector<Mutator> DefaultMutators; - Vector<Mutator> CurrentMutatorSequence; + std::vector<Mutator> Mutators; + std::vector<Mutator> DefaultMutators; + std::vector<Mutator> CurrentMutatorSequence; }; } // namespace fuzzer diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerOptions.h b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerOptions.h index d0c285a6821d..72e256106194 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerOptions.h +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerOptions.h @@ -47,6 +47,7 @@ struct FuzzingOptions { int ReportSlowUnits = 10; bool OnlyASCII = false; bool Entropic = true; + bool ForkCorpusGroups = false; size_t EntropicFeatureFrequencyThreshold = 0xFF; size_t EntropicNumberOfRarestFeatures = 100; bool EntropicScalePerExecTime = false; diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerPlatform.h b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerPlatform.h index 1602e6789500..65e77ded8e82 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerPlatform.h +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerPlatform.h @@ -87,7 +87,7 @@ (LIBFUZZER_APPLE || LIBFUZZER_LINUX || LIBFUZZER_NETBSD || \ LIBFUZZER_FREEBSD || LIBFUZZER_EMSCRIPTEN) -#ifdef __x86_64 +#if defined(__x86_64) && defined(__POPCNT__) #if __has_attribute(target) #define ATTRIBUTE_TARGET_POPCNT __attribute__((target("popcnt"))) #else diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerRandom.h b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerRandom.h index 659283eee207..ad6c07eb5ef5 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerRandom.h +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerRandom.h @@ -18,18 +18,27 @@ class Random : public std::minstd_rand { public: Random(unsigned int seed) : std::minstd_rand(seed) {} result_type operator()() { return this->std::minstd_rand::operator()(); } - size_t Rand() { return this->operator()(); } - size_t RandBool() { return Rand() % 2; } + template <typename T> + typename std::enable_if<std::is_integral<T>::value, T>::type Rand() { + return static_cast<T>(this->operator()()); + } + size_t RandBool() { return this->operator()() % 2; } size_t SkewTowardsLast(size_t n) { size_t T = this->operator()(n * n); - size_t Res = sqrt(T); + size_t Res = static_cast<size_t>(sqrt(T)); return Res; } - size_t operator()(size_t n) { return n ? Rand() % n : 0; } - intptr_t operator()(intptr_t From, intptr_t To) { + template <typename T> + typename std::enable_if<std::is_integral<T>::value, T>::type operator()(T n) { + return n ? Rand<T>() % n : 0; + } + template <typename T> + typename std::enable_if<std::is_integral<T>::value, T>::type + operator()(T From, T To) { assert(From < To); - intptr_t RangeSize = To - From + 1; - return operator()(RangeSize) + From; + auto RangeSize = static_cast<unsigned long long>(To) - + static_cast<unsigned long long>(From) + 1; + return static_cast<T>(this->operator()(RangeSize) + From); } }; diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerSHA1.cpp b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerSHA1.cpp index 2005dc700305..b05655c8ef4d 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerSHA1.cpp +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerSHA1.cpp @@ -134,12 +134,13 @@ void sha1_hashBlock(sha1nfo *s) { s->state[4] += e; } -void sha1_addUncounted(sha1nfo *s, uint8_t data) { - uint8_t * const b = (uint8_t*) s->buffer; +// Adds the least significant byte of |data|. +void sha1_addUncounted(sha1nfo *s, uint32_t data) { + uint8_t *const b = (uint8_t *)s->buffer; #ifdef SHA_BIG_ENDIAN - b[s->bufferOffset] = data; + b[s->bufferOffset] = static_cast<uint8_t>(data); #else - b[s->bufferOffset ^ 3] = data; + b[s->bufferOffset ^ 3] = static_cast<uint8_t>(data); #endif s->bufferOffset++; if (s->bufferOffset == BLOCK_LENGTH) { diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp index 91e94d824002..7f4e8ef91c44 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp @@ -106,6 +106,15 @@ void TracePC::PrintModuleInfo() { } if (size_t NumExtraCounters = ExtraCountersEnd() - ExtraCountersBegin()) Printf("INFO: %zd Extra Counters\n", NumExtraCounters); + + size_t MaxFeatures = CollectFeatures([](uint32_t) {}); + if (MaxFeatures > std::numeric_limits<uint32_t>::max()) + Printf("WARNING: The coverage PC tables may produce up to %zu features.\n" + "This exceeds the maximum 32-bit value. Some features may be\n" + "ignored, and fuzzing may become less precise. If possible,\n" + "consider refactoring the fuzzer into several smaller fuzzers\n" + "linked against only a portion of the current target.\n", + MaxFeatures); } ATTRIBUTE_NO_SANITIZE_ALL @@ -124,13 +133,14 @@ inline ALWAYS_INLINE uintptr_t GetPreviousInstructionPc(uintptr_t PC) { // so we return (pc-2) in that case in order to be safe. // For A32 mode we return (pc-4) because all instructions are 32 bit long. return (PC - 3) & (~1); -#elif defined(__powerpc__) || defined(__powerpc64__) || defined(__aarch64__) - // PCs are always 4 byte aligned. - return PC - 4; #elif defined(__sparc__) || defined(__mips__) return PC - 8; -#else +#elif defined(__riscv__) + return PC - 2; +#elif defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64) return PC - 1; +#else + return PC - 4; #endif } @@ -139,8 +149,8 @@ inline ALWAYS_INLINE uintptr_t GetPreviousInstructionPc(uintptr_t PC) { ALWAYS_INLINE uintptr_t TracePC::GetNextInstructionPc(uintptr_t PC) { #if defined(__mips__) return PC + 8; -#elif defined(__powerpc__) || defined(__sparc__) || defined(__arm__) || \ - defined(__aarch64__) +#elif defined(__powerpc__) || defined(__sparc__) || defined(__arm__) || \ + defined(__aarch64__) || defined(__loongarch__) return PC + 4; #else return PC + 1; @@ -148,7 +158,7 @@ ALWAYS_INLINE uintptr_t TracePC::GetNextInstructionPc(uintptr_t PC) { } void TracePC::UpdateObservedPCs() { - Vector<uintptr_t> CoveredFuncs; + std::vector<uintptr_t> CoveredFuncs; auto ObservePC = [&](const PCTableEntry *TE) { if (ObservedPCs.insert(TE).second && DoPrintNewPCs) { PrintPC("\tNEW_PC: %p %F %L", "\tNEW_PC: %p", @@ -291,8 +301,8 @@ void TracePC::PrintCoverage(bool PrintAllCounters) { FunctionStr = FunctionStr.substr(3); std::string LineStr = DescribePC("%l", VisualizePC); size_t NumEdges = Last - First; - Vector<uintptr_t> UncoveredPCs; - Vector<uintptr_t> CoveredPCs; + std::vector<uintptr_t> UncoveredPCs; + std::vector<uintptr_t> CoveredPCs; for (auto TE = First; TE < Last; TE++) if (!ObservedPCs.count(TE)) UncoveredPCs.push_back(TE->PC); @@ -356,7 +366,7 @@ void TracePC::AddValueForMemcmp(void *caller_pc, const void *s1, const void *s2, uint8_t HammingDistance = 0; for (; I < Len; I++) { if (B1[I] != B2[I] || (StopAtZero && B1[I] == 0)) { - HammingDistance = Popcountll(B1[I] ^ B2[I]); + HammingDistance = static_cast<uint8_t>(Popcountll(B1[I] ^ B2[I])); break; } } @@ -382,6 +392,7 @@ void TracePC::HandleCmp(uintptr_t PC, T Arg1, T Arg2) { ValueProfileMap.AddValue(PC * 128 + 64 + AbsoluteDistance); } +ATTRIBUTE_NO_SANITIZE_MEMORY static size_t InternalStrnlen(const char *S, size_t MaxLen) { size_t Len = 0; for (; Len < MaxLen && S[Len]; Len++) {} @@ -389,7 +400,8 @@ static size_t InternalStrnlen(const char *S, size_t MaxLen) { } // Finds min of (strlen(S1), strlen(S2)). -// Needed bacause one of these strings may actually be non-zero terminated. +// Needed because one of these strings may actually be non-zero terminated. +ATTRIBUTE_NO_SANITIZE_MEMORY static size_t InternalStrnlen2(const char *S1, const char *S2) { size_t Len = 0; for (; S1[Len] && S2[Len]; Len++) {} diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerTracePC.h b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerTracePC.h index 00909230731d..af1f9d81e950 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerTracePC.h +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerTracePC.h @@ -54,7 +54,7 @@ struct MemMemTable { void Add(const uint8_t *Data, size_t Size) { if (Size <= 2) return; Size = std::min(Size, Word::GetMaxSize()); - size_t Idx = SimpleFastHash(Data, Size) % kSize; + auto Idx = SimpleFastHash(Data, Size) % kSize; MemMemWords[Idx].Set(Data, Size); } const Word &Get(size_t Idx) { @@ -79,7 +79,7 @@ class TracePC { void SetPrintNewPCs(bool P) { DoPrintNewPCs = P; } void SetPrintNewFuncs(size_t P) { NumPrintNewFuncs = P; } void UpdateObservedPCs(); - template <class Callback> void CollectFeatures(Callback CB) const; + template <class Callback> size_t CollectFeatures(Callback CB) const; void ResetMaps() { ValueProfileMap.Reset(); @@ -169,7 +169,7 @@ private: size_t NumPCTables; size_t NumPCsInPCTables; - Set<const PCTableEntry*> ObservedPCs; + std::set<const PCTableEntry *> ObservedPCs; std::unordered_map<uintptr_t, uintptr_t> ObservedFuncs; // PC => Counter. uint8_t *FocusFunctionCounterPtr = nullptr; @@ -193,7 +193,7 @@ size_t ForEachNonZeroByte(const uint8_t *Begin, const uint8_t *End, Handle8bitCounter(FirstFeature, P - Begin, V); // Iterate by Step bytes at a time. - for (; P < End; P += Step) + for (; P + Step <= End; P += Step) if (LargeType Bundle = *reinterpret_cast<const LargeType *>(P)) { Bundle = HostToLE(Bundle); for (size_t I = 0; I < Step; I++, Bundle >>= 8) @@ -234,16 +234,16 @@ unsigned CounterToFeature(T Counter) { return Bit; } -template <class Callback> // void Callback(size_t Feature) -ATTRIBUTE_NO_SANITIZE_ADDRESS -ATTRIBUTE_NOINLINE -void TracePC::CollectFeatures(Callback HandleFeature) const { +template <class Callback> // void Callback(uint32_t Feature) +ATTRIBUTE_NO_SANITIZE_ADDRESS ATTRIBUTE_NOINLINE size_t +TracePC::CollectFeatures(Callback HandleFeature) const { auto Handle8bitCounter = [&](size_t FirstFeature, size_t Idx, uint8_t Counter) { if (UseCounters) - HandleFeature(FirstFeature + Idx * 8 + CounterToFeature(Counter)); + HandleFeature(static_cast<uint32_t>(FirstFeature + Idx * 8 + + CounterToFeature(Counter))); else - HandleFeature(FirstFeature + Idx); + HandleFeature(static_cast<uint32_t>(FirstFeature + Idx)); }; size_t FirstFeature = 0; @@ -263,16 +263,18 @@ void TracePC::CollectFeatures(Callback HandleFeature) const { if (UseValueProfileMask) { ValueProfileMap.ForEach([&](size_t Idx) { - HandleFeature(FirstFeature + Idx); + HandleFeature(static_cast<uint32_t>(FirstFeature + Idx)); }); FirstFeature += ValueProfileMap.SizeInBits(); } // Step function, grows similar to 8 * Log_2(A). - auto StackDepthStepFunction = [](uint32_t A) -> uint32_t { - if (!A) return A; - uint32_t Log2 = Log(A); - if (Log2 < 3) return A; + auto StackDepthStepFunction = [](size_t A) -> size_t { + if (!A) + return A; + auto Log2 = Log(A); + if (Log2 < 3) + return A; Log2 -= 3; return (Log2 + 1) * 8 + ((A >> Log2) & 7); }; @@ -280,8 +282,13 @@ void TracePC::CollectFeatures(Callback HandleFeature) const { assert(StackDepthStepFunction(1024 * 4) == 80); assert(StackDepthStepFunction(1024 * 1024) == 144); - if (auto MaxStackOffset = GetMaxStackOffset()) - HandleFeature(FirstFeature + StackDepthStepFunction(MaxStackOffset / 8)); + if (auto MaxStackOffset = GetMaxStackOffset()) { + HandleFeature(static_cast<uint32_t>( + FirstFeature + StackDepthStepFunction(MaxStackOffset / 8))); + FirstFeature += StackDepthStepFunction(std::numeric_limits<size_t>::max()); + } + + return FirstFeature; } extern TracePC TPC; diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtil.cpp b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtil.cpp index 7eecb68d0729..aeab70f20c28 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtil.cpp +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtil.cpp @@ -43,7 +43,7 @@ void PrintASCIIByte(uint8_t Byte) { else if (Byte >= 32 && Byte < 127) Printf("%c", Byte); else - Printf("\\x%02x", Byte); + Printf("\\%03o", Byte); } void PrintASCII(const uint8_t *Data, size_t Size, const char *PrintAfter) { @@ -111,7 +111,7 @@ bool ParseOneDictionaryEntry(const std::string &Str, Unit *U) { char Hex[] = "0xAA"; Hex[2] = Str[Pos + 2]; Hex[3] = Str[Pos + 3]; - U->push_back(strtol(Hex, nullptr, 16)); + U->push_back(static_cast<uint8_t>(strtol(Hex, nullptr, 16))); Pos += 3; continue; } @@ -124,7 +124,7 @@ bool ParseOneDictionaryEntry(const std::string &Str, Unit *U) { return true; } -bool ParseDictionaryFile(const std::string &Text, Vector<Unit> *Units) { +bool ParseDictionaryFile(const std::string &Text, std::vector<Unit> *Units) { if (Text.empty()) { Printf("ParseDictionaryFile: file does not exist or is empty\n"); return false; @@ -226,10 +226,11 @@ unsigned NumberOfCpuCores() { return N; } -size_t SimpleFastHash(const uint8_t *Data, size_t Size) { - size_t Res = 0; +uint64_t SimpleFastHash(const void *Data, size_t Size, uint64_t Initial) { + uint64_t Res = Initial; + const uint8_t *Bytes = static_cast<const uint8_t *>(Data); for (size_t i = 0; i < Size; i++) - Res = Res * 11 + Data[i]; + Res = Res * 11 + Bytes[i]; return Res; } diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtil.h b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtil.h index e90be085008e..554567e1b8fc 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtil.h +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtil.h @@ -59,6 +59,8 @@ size_t GetPeakRSSMb(); int ExecuteCommand(const Command &Cmd); bool ExecuteCommand(const Command &Cmd, std::string *CmdOutput); +void SetThreadName(std::thread &thread, const std::string &name); + // Fuchsia does not have popen/pclose. FILE *OpenProcessPipe(const char *Command, const char *Mode); int CloseProcessPipe(FILE *F); @@ -66,10 +68,10 @@ int CloseProcessPipe(FILE *F); const void *SearchMemory(const void *haystack, size_t haystacklen, const void *needle, size_t needlelen); -std::string CloneArgsWithoutX(const Vector<std::string> &Args, +std::string CloneArgsWithoutX(const std::vector<std::string> &Args, const char *X1, const char *X2); -inline std::string CloneArgsWithoutX(const Vector<std::string> &Args, +inline std::string CloneArgsWithoutX(const std::vector<std::string> &Args, const char *X) { return CloneArgsWithoutX(Args, X, X); } @@ -88,11 +90,14 @@ std::string DisassembleCmd(const std::string &FileName); std::string SearchRegexCmd(const std::string &Regex); -size_t SimpleFastHash(const uint8_t *Data, size_t Size); +uint64_t SimpleFastHash(const void *Data, size_t Size, uint64_t Initial = 0); + +inline size_t Log(size_t X) { + return static_cast<size_t>((sizeof(unsigned long long) * 8) - Clzll(X) - 1); +} -inline uint32_t Log(uint32_t X) { return 32 - Clz(X) - 1; } +size_t PageSize(); -inline size_t PageSize() { return 4096; } inline uint8_t *RoundUpByPage(uint8_t *P) { uintptr_t X = reinterpret_cast<uintptr_t>(P); size_t Mask = PageSize() - 1; diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtilDarwin.cpp b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtilDarwin.cpp index a5bed658a446..6c3ece30f67b 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtilDarwin.cpp +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtilDarwin.cpp @@ -165,6 +165,11 @@ void DiscardOutput(int Fd) { fclose(Temp); } +void SetThreadName(std::thread &thread, const std::string &name) { + // TODO ? + // Darwin allows to set the name only on the current thread it seems +} + } // namespace fuzzer #endif // LIBFUZZER_APPLE diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp index af4394616776..cfb81cd3f780 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp @@ -52,6 +52,12 @@ void CrashTrampolineAsm() __asm__("CrashTrampolineAsm"); namespace { +// The signal handler thread uses Zircon exceptions to resume crashed threads +// into libFuzzer's POSIX signal handlers. The associated event is used to +// signal when the thread is running, and when it should stop. +std::thread SignalHandler; +zx_handle_t SignalHandlerEvent = ZX_HANDLE_INVALID; + // Helper function to handle Zircon syscall failures. void ExitOnErr(zx_status_t Status, const char *Syscall) { if (Status != ZX_OK) { @@ -68,23 +74,6 @@ void AlarmHandler(int Seconds) { } } -// CFAOffset is used to reference the stack pointer before entering the -// trampoline (Stack Pointer + CFAOffset = prev Stack Pointer). Before jumping -// to the trampoline we copy all the registers onto the stack. We need to make -// sure that the new stack has enough space to store all the registers. -// -// The trampoline holds CFI information regarding the registers stored in the -// stack, which is then used by the unwinder to restore them. -#if defined(__x86_64__) -// In x86_64 the crashing function might also be using the red zone (128 bytes -// on top of their rsp). -constexpr size_t CFAOffset = 128 + sizeof(zx_thread_state_general_regs_t); -#elif defined(__aarch64__) -// In aarch64 we need to always have the stack pointer aligned to 16 bytes, so we -// make sure that we are keeping that same alignment. -constexpr size_t CFAOffset = (sizeof(zx_thread_state_general_regs_t) + 15) & -(uintptr_t)16; -#endif - // For the crash handler, we need to call Fuzzer::StaticCrashSignalCallback // without POSIX signal handlers. To achieve this, we use an assembly function // to add the necessary CFI unwinding information and a C function to bridge @@ -98,6 +87,7 @@ constexpr size_t CFAOffset = (sizeof(zx_thread_state_general_regs_t) + 15) & -(u // Alternatively, Fuchsia may in future actually implement basic signal // handling for the machine trap signals. #if defined(__x86_64__) + #define FOREACH_REGISTER(OP_REG, OP_NUM) \ OP_REG(rax) \ OP_REG(rbx) \ @@ -118,6 +108,7 @@ constexpr size_t CFAOffset = (sizeof(zx_thread_state_general_regs_t) + 15) & -(u OP_REG(rip) #elif defined(__aarch64__) + #define FOREACH_REGISTER(OP_REG, OP_NUM) \ OP_NUM(0) \ OP_NUM(1) \ @@ -151,6 +142,41 @@ constexpr size_t CFAOffset = (sizeof(zx_thread_state_general_regs_t) + 15) & -(u OP_NUM(29) \ OP_REG(sp) +#elif defined(__riscv) + +#define FOREACH_REGISTER(OP_REG, OP_NUM) \ + OP_REG(ra) \ + OP_REG(sp) \ + OP_REG(gp) \ + OP_REG(tp) \ + OP_REG(t0) \ + OP_REG(t1) \ + OP_REG(t2) \ + OP_REG(s0) \ + OP_REG(s1) \ + OP_REG(a0) \ + OP_REG(a1) \ + OP_REG(a2) \ + OP_REG(a3) \ + OP_REG(a4) \ + OP_REG(a5) \ + OP_REG(a6) \ + OP_REG(a7) \ + OP_REG(s2) \ + OP_REG(s3) \ + OP_REG(s4) \ + OP_REG(s5) \ + OP_REG(s6) \ + OP_REG(s7) \ + OP_REG(s8) \ + OP_REG(s9) \ + OP_REG(s10) \ + OP_REG(s11) \ + OP_REG(t3) \ + OP_REG(t4) \ + OP_REG(t5) \ + OP_REG(t6) \ + #else #error "Unsupported architecture for fuzzing on Fuchsia" #endif @@ -163,10 +189,10 @@ constexpr size_t CFAOffset = (sizeof(zx_thread_state_general_regs_t) + 15) & -(u // Produces an assembler immediate operand for the named or numbered register. // This operand contains the offset of the register relative to the CFA. -#define ASM_OPERAND_REG(reg) \ - [reg] "i"(offsetof(zx_thread_state_general_regs_t, reg) - CFAOffset), -#define ASM_OPERAND_NUM(num) \ - [x##num] "i"(offsetof(zx_thread_state_general_regs_t, r[num]) - CFAOffset), +#define ASM_OPERAND_REG(reg) \ + [reg] "i"(offsetof(zx_thread_state_general_regs_t, reg)), +#define ASM_OPERAND_NUM(num) \ + [x##num] "i"(offsetof(zx_thread_state_general_regs_t, r[num])), // Trampoline to bridge from the assembly below to the static C++ crash // callback. @@ -178,62 +204,67 @@ static void StaticCrashHandler() { } } -// Creates the trampoline with the necessary CFI information to unwind through -// to the crashing call stack: -// * Defining the CFA so that it points to the stack pointer at the point -// of crash. -// * Storing all registers at the point of crash in the stack and refer to them -// via CFI information (relative to the CFA). -// * Setting the return column so the unwinder knows how to continue unwinding. -// * (x86_64) making sure rsp is aligned before calling StaticCrashHandler. -// * Calling StaticCrashHandler that will trigger the unwinder. +// This trampoline function has the necessary CFI information to unwind +// and get a backtrace: +// * The stack contains a copy of all the registers at the point of crash, +// the code has CFI directives specifying how to restore them. +// * A call to StaticCrashHandler, which will print the stacktrace and exit +// the fuzzer, generating a crash artifact. // // The __attribute__((used)) is necessary because the function // is never called; it's just a container around the assembly to allow it to // use operands for compile-time computed constants. __attribute__((used)) void MakeTrampoline() { - __asm__(".cfi_endproc\n" - ".pushsection .text.CrashTrampolineAsm\n" - ".type CrashTrampolineAsm,STT_FUNC\n" -"CrashTrampolineAsm:\n" - ".cfi_startproc simple\n" - ".cfi_signal_frame\n" + __asm__( + ".cfi_endproc\n" + ".pushsection .text.CrashTrampolineAsm\n" + ".type CrashTrampolineAsm,STT_FUNC\n" + "CrashTrampolineAsm:\n" + ".cfi_startproc simple\n" + ".cfi_signal_frame\n" #if defined(__x86_64__) - ".cfi_return_column rip\n" - ".cfi_def_cfa rsp, %c[CFAOffset]\n" - FOREACH_REGISTER(CFI_OFFSET_REG, CFI_OFFSET_NUM) - "mov %%rsp, %%rbp\n" - ".cfi_def_cfa_register rbp\n" - "andq $-16, %%rsp\n" - "call %c[StaticCrashHandler]\n" - "ud2\n" + ".cfi_return_column rip\n" + ".cfi_def_cfa rsp, 0\n" + FOREACH_REGISTER(CFI_OFFSET_REG, CFI_OFFSET_NUM) + "call %c[StaticCrashHandler]\n" + "ud2\n" #elif defined(__aarch64__) - ".cfi_return_column 33\n" - ".cfi_def_cfa sp, %c[CFAOffset]\n" - FOREACH_REGISTER(CFI_OFFSET_REG, CFI_OFFSET_NUM) - ".cfi_offset 33, %c[pc]\n" - ".cfi_offset 30, %c[lr]\n" - "bl %c[StaticCrashHandler]\n" - "brk 1\n" + ".cfi_return_column 33\n" + ".cfi_def_cfa sp, 0\n" + FOREACH_REGISTER(CFI_OFFSET_REG, CFI_OFFSET_NUM) + ".cfi_offset 33, %c[pc]\n" + ".cfi_offset 30, %c[lr]\n" + "bl %c[StaticCrashHandler]\n" + "brk 1\n" +#elif defined(__riscv) + ".cfi_return_column 64\n" + ".cfi_def_cfa sp, 0\n" + ".cfi_offset 64, %[pc]\n" + FOREACH_REGISTER(CFI_OFFSET_REG, CFI_OFFSET_NUM) + "call %c[StaticCrashHandler]\n" + "unimp\n" #else #error "Unsupported architecture for fuzzing on Fuchsia" #endif - ".cfi_endproc\n" - ".size CrashTrampolineAsm, . - CrashTrampolineAsm\n" - ".popsection\n" - ".cfi_startproc\n" - : // No outputs - : FOREACH_REGISTER(ASM_OPERAND_REG, ASM_OPERAND_NUM) + ".cfi_endproc\n" + ".size CrashTrampolineAsm, . - CrashTrampolineAsm\n" + ".popsection\n" + ".cfi_startproc\n" + : // No outputs + : FOREACH_REGISTER(ASM_OPERAND_REG, ASM_OPERAND_NUM) +#if defined(__aarch64__) || defined(__riscv) + ASM_OPERAND_REG(pc) +#endif #if defined(__aarch64__) - ASM_OPERAND_REG(pc) - ASM_OPERAND_REG(lr) + ASM_OPERAND_REG(lr) #endif - [StaticCrashHandler] "i" (StaticCrashHandler), - [CFAOffset] "i" (CFAOffset)); + [StaticCrashHandler] "i"(StaticCrashHandler)); } -void CrashHandler(zx_handle_t *Event) { +void CrashHandler() { + assert(SignalHandlerEvent != ZX_HANDLE_INVALID); + // This structure is used to ensure we close handles to objects we create in // this handler. struct ScopedHandle { @@ -251,16 +282,30 @@ void CrashHandler(zx_handle_t *Event) { Self, ZX_EXCEPTION_CHANNEL_DEBUGGER, &Channel.Handle), "_zx_task_create_exception_channel"); - ExitOnErr(_zx_object_signal(*Event, 0, ZX_USER_SIGNAL_0), + ExitOnErr(_zx_object_signal(SignalHandlerEvent, 0, ZX_USER_SIGNAL_0), "_zx_object_signal"); // This thread lives as long as the process in order to keep handling // crashes. In practice, the first crashed thread to reach the end of the // StaticCrashHandler will end the process. while (true) { - ExitOnErr(_zx_object_wait_one(Channel.Handle, ZX_CHANNEL_READABLE, - ZX_TIME_INFINITE, nullptr), - "_zx_object_wait_one"); + zx_wait_item_t WaitItems[] = { + { + .handle = SignalHandlerEvent, + .waitfor = ZX_SIGNAL_HANDLE_CLOSED, + .pending = 0, + }, + { + .handle = Channel.Handle, + .waitfor = ZX_CHANNEL_READABLE | ZX_CHANNEL_PEER_CLOSED, + .pending = 0, + }, + }; + auto Status = _zx_object_wait_many( + WaitItems, sizeof(WaitItems) / sizeof(WaitItems[0]), ZX_TIME_INFINITE); + if (Status != ZX_OK || (WaitItems[1].pending & ZX_CHANNEL_READABLE) == 0) { + break; + } zx_exception_info_t ExceptionInfo; ScopedHandle Exception; @@ -296,14 +341,19 @@ void CrashHandler(zx_handle_t *Event) { // onto the stack and jump into a trampoline with CFI instructions on how // to restore it. #if defined(__x86_64__) - uintptr_t StackPtr = GeneralRegisters.rsp - CFAOffset; + + uintptr_t StackPtr = + (GeneralRegisters.rsp - (128 + sizeof(GeneralRegisters))) & + -(uintptr_t)16; __unsanitized_memcpy(reinterpret_cast<void *>(StackPtr), &GeneralRegisters, sizeof(GeneralRegisters)); GeneralRegisters.rsp = StackPtr; GeneralRegisters.rip = reinterpret_cast<zx_vaddr_t>(CrashTrampolineAsm); -#elif defined(__aarch64__) - uintptr_t StackPtr = GeneralRegisters.sp - CFAOffset; +#elif defined(__aarch64__) || defined(__riscv) + + uintptr_t StackPtr = + (GeneralRegisters.sp - sizeof(GeneralRegisters)) & -(uintptr_t)16; __unsanitized_memcpy(reinterpret_cast<void *>(StackPtr), &GeneralRegisters, sizeof(GeneralRegisters)); GeneralRegisters.sp = StackPtr; @@ -327,6 +377,13 @@ void CrashHandler(zx_handle_t *Event) { } } +void StopSignalHandler() { + _zx_handle_close(SignalHandlerEvent); + if (SignalHandler.joinable()) { + SignalHandler.join(); + } +} + } // namespace // Platform specific functions. @@ -356,16 +413,14 @@ void SetSignalHandler(const FuzzingOptions &Options) { return; // Set up the crash handler and wait until it is ready before proceeding. - zx_handle_t Event; - ExitOnErr(_zx_event_create(0, &Event), "_zx_event_create"); + ExitOnErr(_zx_event_create(0, &SignalHandlerEvent), "_zx_event_create"); - std::thread T(CrashHandler, &Event); - zx_status_t Status = - _zx_object_wait_one(Event, ZX_USER_SIGNAL_0, ZX_TIME_INFINITE, nullptr); - _zx_handle_close(Event); + SignalHandler = std::thread(CrashHandler); + zx_status_t Status = _zx_object_wait_one(SignalHandlerEvent, ZX_USER_SIGNAL_0, + ZX_TIME_INFINITE, nullptr); ExitOnErr(Status, "_zx_object_wait_one"); - T.detach(); + std::atexit(StopSignalHandler); } void SleepSeconds(int Seconds) { @@ -515,7 +570,7 @@ int ExecuteCommand(const Command &Cmd) { return rc; } - return Info.return_code; + return static_cast<int>(Info.return_code); } bool ExecuteCommand(const Command &BaseCmd, std::string *CmdOutput) { @@ -545,6 +600,15 @@ void DiscardOutput(int Fd) { dup2(nullfd, Fd); } +size_t PageSize() { + static size_t PageSizeCached = _zx_system_get_page_size(); + return PageSizeCached; +} + +void SetThreadName(std::thread &thread, const std::string &name) { + // TODO ? +} + } // namespace fuzzer #endif // LIBFUZZER_FUCHSIA diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtilLinux.cpp b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtilLinux.cpp index 981f9a8b429f..5729448b0beb 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtilLinux.cpp +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtilLinux.cpp @@ -11,7 +11,9 @@ #if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FREEBSD || \ LIBFUZZER_EMSCRIPTEN #include "FuzzerCommand.h" +#include "FuzzerInternal.h" +#include <signal.h> #include <stdlib.h> #include <sys/types.h> #include <sys/wait.h> @@ -25,6 +27,8 @@ int ExecuteCommand(const Command &Cmd) { int exit_code = system(CmdLine.c_str()); if (WIFEXITED(exit_code)) return WEXITSTATUS(exit_code); + if (WIFSIGNALED(exit_code) && WTERMSIG(exit_code) == SIGINT) + return Fuzzer::InterruptExitCode(); return exit_code; } @@ -36,6 +40,14 @@ void DiscardOutput(int Fd) { fclose(Temp); } +void SetThreadName(std::thread &thread, const std::string &name) { +#if LIBFUZZER_LINUX || LIBFUZZER_FREEBSD + (void)pthread_setname_np(thread.native_handle(), name.c_str()); +#elif LIBFUZZER_NETBSD + (void)pthread_set_name_np(thread.native_handle(), "%s", name.c_str()); +#endif +} + } // namespace fuzzer #endif diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtilPosix.cpp b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtilPosix.cpp index afb733409ab5..392c1e5be4ee 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtilPosix.cpp +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtilPosix.cpp @@ -77,10 +77,13 @@ static void SetSigaction(int signum, return; } - sigact = {}; - sigact.sa_flags = SA_SIGINFO; - sigact.sa_sigaction = callback; - if (sigaction(signum, &sigact, 0)) { + struct sigaction new_sigact = {}; + // Address sanitizer needs SA_ONSTACK (causing the signal handler to run on a + // dedicated stack) in order to be able to detect stack overflows; keep the + // flag if it's set. + new_sigact.sa_flags = SA_SIGINFO | (sigact.sa_flags & SA_ONSTACK); + new_sigact.sa_sigaction = callback; + if (sigaction(signum, &new_sigact, nullptr)) { Printf("libFuzzer: sigaction failed with %d\n", errno); exit(1); } @@ -180,6 +183,11 @@ std::string SearchRegexCmd(const std::string &Regex) { return "grep '" + Regex + "'"; } +size_t PageSize() { + static size_t PageSizeCached = sysconf(_SC_PAGESIZE); + return PageSizeCached; +} + } // namespace fuzzer #endif // LIBFUZZER_POSIX diff --git a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp index 1a54bb569eca..71770166805f 100644 --- a/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp +++ b/contrib/llvm-project/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp @@ -204,7 +204,7 @@ const void *SearchMemory(const void *Data, size_t DataLen, const void *Patt, } std::string DisassembleCmd(const std::string &FileName) { - Vector<std::string> command_vector; + std::vector<std::string> command_vector; command_vector.push_back("dumpbin /summary > nul"); if (ExecuteCommand(Command(command_vector)) == 0) return "dumpbin /disasm " + FileName; @@ -224,6 +224,20 @@ void DiscardOutput(int Fd) { fclose(Temp); } +size_t PageSize() { + static size_t PageSizeCached = []() -> size_t { + SYSTEM_INFO si; + GetSystemInfo(&si); + return si.dwPageSize; + }(); + return PageSizeCached; +} + +void SetThreadName(std::thread &thread, const std::string &name) { + // TODO ? + // to UTF-8 then SetThreadDescription ? +} + } // namespace fuzzer #endif // LIBFUZZER_WINDOWS |