diff options
Diffstat (limited to 'clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h')
| -rw-r--r-- | clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h | 353 |
1 files changed, 254 insertions, 99 deletions
diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h index 7d0b8f2138f9..7c830d3f2733 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h @@ -6,13 +6,13 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_FILESYSTEM_H -#define LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_FILESYSTEM_H +#ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H +#define LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H #include "clang/Basic/LLVM.h" #include "clang/Lex/PreprocessorExcludedConditionalDirectiveSkipMapping.h" +#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/StringMap.h" -#include "llvm/ADT/StringSet.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/VirtualFileSystem.h" @@ -22,6 +22,26 @@ namespace clang { namespace tooling { namespace dependencies { +/// Original and minimized contents of a cached file entry. Single instance can +/// be shared between multiple entries. +struct CachedFileContents { + CachedFileContents(std::unique_ptr<llvm::MemoryBuffer> Original) + : Original(std::move(Original)), MinimizedAccess(nullptr) {} + + /// Owning storage for the minimized contents. + std::unique_ptr<llvm::MemoryBuffer> Original; + + /// The mutex that must be locked before mutating minimized contents. + std::mutex ValueLock; + /// Owning storage for the minimized contents. + std::unique_ptr<llvm::MemoryBuffer> MinimizedStorage; + /// Accessor to the minimized contents that's atomic to avoid data races. + std::atomic<llvm::MemoryBuffer *> MinimizedAccess; + /// Skipped range mapping of the minimized contents. + /// This is initialized iff `MinimizedAccess != nullptr`. + PreprocessorSkippedRangeMapping PPSkippedRangeMapping; +}; + /// An in-memory representation of a file system entity that is of interest to /// the dependency scanning filesystem. /// @@ -29,100 +49,99 @@ namespace dependencies { /// - opened file with original contents and a stat value, /// - opened file with original contents, minimized contents and a stat value, /// - directory entry with its stat value, -/// - filesystem error, -/// - uninitialized entry with unknown status. +/// - filesystem error. +/// +/// Single instance of this class can be shared across different filenames (e.g. +/// a regular file and a symlink). For this reason the status filename is empty +/// and is only materialized by \c EntryRef that knows the requested filename. class CachedFileSystemEntry { public: - /// Creates an uninitialized entry. - CachedFileSystemEntry() - : MaybeStat(llvm::vfs::Status()), MinimizedContentsAccess(nullptr) {} - - /// Initialize the cached file system entry. - void init(llvm::ErrorOr<llvm::vfs::Status> &&MaybeStatus, StringRef Filename, - llvm::vfs::FileSystem &FS); + /// Creates an entry without contents: either a filesystem error or + /// a directory with stat value. + CachedFileSystemEntry(llvm::ErrorOr<llvm::vfs::Status> Stat) + : MaybeStat(std::move(Stat)), Contents(nullptr) { + clearStatName(); + } - /// Initialize the entry as file with minimized or original contents. - /// - /// The filesystem opens the file even for `stat` calls open to avoid the - /// issues with stat + open of minimized files that might lead to a - /// mismatching size of the file. - llvm::ErrorOr<llvm::vfs::Status> initFile(StringRef Filename, - llvm::vfs::FileSystem &FS); - - /// Minimize contents of the file. - void minimizeFile(); - - /// \returns True if the entry is initialized. - bool isInitialized() const { - return !MaybeStat || MaybeStat->isStatusKnown(); + /// Creates an entry representing a file with contents. + CachedFileSystemEntry(llvm::ErrorOr<llvm::vfs::Status> Stat, + CachedFileContents *Contents) + : MaybeStat(std::move(Stat)), Contents(std::move(Contents)) { + clearStatName(); } - /// \returns True if the current entry points to a directory. - bool isDirectory() const { return MaybeStat && MaybeStat->isDirectory(); } + /// \returns True if the entry is a filesystem error. + bool isError() const { return !MaybeStat; } + + /// \returns True if the current entry represents a directory. + bool isDirectory() const { return !isError() && MaybeStat->isDirectory(); } - /// \returns The error or the file's original contents. - llvm::ErrorOr<StringRef> getOriginalContents() const { - if (!MaybeStat) - return MaybeStat.getError(); + /// \returns Original contents of the file. + StringRef getOriginalContents() const { + assert(!isError() && "error"); assert(!MaybeStat->isDirectory() && "not a file"); - assert(isInitialized() && "not initialized"); - assert(OriginalContents && "not read"); - return OriginalContents->getBuffer(); + assert(Contents && "contents not initialized"); + return Contents->Original->getBuffer(); } - /// \returns The error or the file's minimized contents. - llvm::ErrorOr<StringRef> getMinimizedContents() const { - if (!MaybeStat) - return MaybeStat.getError(); + /// \returns Minimized contents of the file. + StringRef getMinimizedContents() const { + assert(!isError() && "error"); assert(!MaybeStat->isDirectory() && "not a file"); - assert(isInitialized() && "not initialized"); - llvm::MemoryBuffer *Buffer = MinimizedContentsAccess.load(); + assert(Contents && "contents not initialized"); + llvm::MemoryBuffer *Buffer = Contents->MinimizedAccess.load(); assert(Buffer && "not minimized"); return Buffer->getBuffer(); } - /// \returns True if this entry represents a file that can be read. - bool isReadable() const { return MaybeStat && !MaybeStat->isDirectory(); } + /// \returns The error. + std::error_code getError() const { return MaybeStat.getError(); } - /// \returns True if this cached entry needs to be updated. - bool needsUpdate(bool ShouldBeMinimized) const { - return isReadable() && needsMinimization(ShouldBeMinimized); + /// \returns The entry status with empty filename. + llvm::vfs::Status getStatus() const { + assert(!isError() && "error"); + assert(MaybeStat->getName().empty() && "stat name must be empty"); + return *MaybeStat; } - /// \returns True if the contents of this entry need to be minimized. - bool needsMinimization(bool ShouldBeMinimized) const { - return ShouldBeMinimized && !MinimizedContentsAccess.load(); + /// \returns The unique ID of the entry. + llvm::sys::fs::UniqueID getUniqueID() const { + assert(!isError() && "error"); + return MaybeStat->getUniqueID(); } - /// \returns The error or the status of the entry. - llvm::ErrorOr<llvm::vfs::Status> getStatus() const { - assert(isInitialized() && "not initialized"); - return MaybeStat; + /// \returns The mapping between location -> distance that is used to speed up + /// the block skipping in the preprocessor. + const PreprocessorSkippedRangeMapping &getPPSkippedRangeMapping() const { + assert(!isError() && "error"); + assert(!isDirectory() && "not a file"); + assert(Contents && "contents not initialized"); + return Contents->PPSkippedRangeMapping; } - /// \returns the name of the file. - StringRef getName() const { - assert(isInitialized() && "not initialized"); - return MaybeStat->getName(); + /// \returns The data structure holding both original and minimized contents. + CachedFileContents *getContents() const { + assert(!isError() && "error"); + assert(!isDirectory() && "not a file"); + return Contents; } - /// Return the mapping between location -> distance that is used to speed up - /// the block skipping in the preprocessor. - const PreprocessorSkippedRangeMapping &getPPSkippedRangeMapping() const { - return PPSkippedRangeMapping; +private: + void clearStatName() { + if (MaybeStat) + MaybeStat = llvm::vfs::Status::copyWithNewName(*MaybeStat, ""); } -private: + /// Either the filesystem error or status of the entry. + /// The filename is empty and only materialized by \c EntryRef. llvm::ErrorOr<llvm::vfs::Status> MaybeStat; - std::unique_ptr<llvm::MemoryBuffer> OriginalContents; - - /// Owning storage for the minimized file contents. - std::unique_ptr<llvm::MemoryBuffer> MinimizedContentsStorage; - /// Atomic view of the minimized file contents. - /// This prevents data races when multiple threads call `needsMinimization`. - std::atomic<llvm::MemoryBuffer *> MinimizedContentsAccess; - PreprocessorSkippedRangeMapping PPSkippedRangeMapping; + /// Non-owning pointer to the file contents. + /// + /// We're using pointer here to keep the size of this class small. Instances + /// representing directories and filesystem errors don't hold any contents + /// anyway. + CachedFileContents *Contents; }; /// This class is a shared cache, that caches the 'stat' and 'open' calls to the @@ -133,24 +152,59 @@ private: /// the worker threads. class DependencyScanningFilesystemSharedCache { public: - struct SharedFileSystemEntry { - std::mutex ValueLock; - CachedFileSystemEntry Value; + struct CacheShard { + /// The mutex that needs to be locked before mutation of any member. + mutable std::mutex CacheLock; + + /// Map from filenames to cached entries. + llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator> + EntriesByFilename; + + /// Map from unique IDs to cached entries. + llvm::DenseMap<llvm::sys::fs::UniqueID, const CachedFileSystemEntry *> + EntriesByUID; + + /// The backing storage for cached entries. + llvm::SpecificBumpPtrAllocator<CachedFileSystemEntry> EntryStorage; + + /// The backing storage for cached contents. + llvm::SpecificBumpPtrAllocator<CachedFileContents> ContentsStorage; + + /// Returns entry associated with the filename or nullptr if none is found. + const CachedFileSystemEntry *findEntryByFilename(StringRef Filename) const; + + /// Returns entry associated with the unique ID or nullptr if none is found. + const CachedFileSystemEntry * + findEntryByUID(llvm::sys::fs::UniqueID UID) const; + + /// Returns entry associated with the filename if there is some. Otherwise, + /// constructs new one with the given status, associates it with the + /// filename and returns the result. + const CachedFileSystemEntry & + getOrEmplaceEntryForFilename(StringRef Filename, + llvm::ErrorOr<llvm::vfs::Status> Stat); + + /// Returns entry associated with the unique ID if there is some. Otherwise, + /// constructs new one with the given status and contents, associates it + /// with the unique ID and returns the result. + const CachedFileSystemEntry & + getOrEmplaceEntryForUID(llvm::sys::fs::UniqueID UID, llvm::vfs::Status Stat, + std::unique_ptr<llvm::MemoryBuffer> Contents); + + /// Returns entry associated with the filename if there is some. Otherwise, + /// associates the given entry with the filename and returns it. + const CachedFileSystemEntry & + getOrInsertEntryForFilename(StringRef Filename, + const CachedFileSystemEntry &Entry); }; DependencyScanningFilesystemSharedCache(); - /// Returns a cache entry for the corresponding key. - /// - /// A new cache entry is created if the key is not in the cache. This is a - /// thread safe call. - SharedFileSystemEntry &get(StringRef Key); + /// Returns shard for the given key. + CacheShard &getShardForFilename(StringRef Filename) const; + CacheShard &getShardForUID(llvm::sys::fs::UniqueID UID) const; private: - struct CacheShard { - std::mutex CacheLock; - llvm::StringMap<SharedFileSystemEntry, llvm::BumpPtrAllocator> Cache; - }; std::unique_ptr<CacheShard[]> CacheShards; unsigned NumShards; }; @@ -162,8 +216,20 @@ class DependencyScanningFilesystemLocalCache { llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator> Cache; public: - const CachedFileSystemEntry *getCachedEntry(StringRef Filename) { - return Cache[Filename]; + /// Returns entry associated with the filename or nullptr if none is found. + const CachedFileSystemEntry *findEntryByFilename(StringRef Filename) const { + auto It = Cache.find(Filename); + return It == Cache.end() ? nullptr : It->getValue(); + } + + /// Associates the given entry with the filename and returns the given entry + /// pointer (for convenience). + const CachedFileSystemEntry & + insertEntryForFilename(StringRef Filename, + const CachedFileSystemEntry &Entry) { + const auto *InsertedEntry = Cache.insert({Filename, &Entry}).first->second; + assert(InsertedEntry == &Entry && "entry already present"); + return *InsertedEntry; } }; @@ -176,26 +242,34 @@ class EntryRef { /// are minimized. bool Minimized; + /// The filename used to access this entry. + std::string Filename; + /// The underlying cached entry. const CachedFileSystemEntry &Entry; public: - EntryRef(bool Minimized, const CachedFileSystemEntry &Entry) - : Minimized(Minimized), Entry(Entry) {} - - llvm::ErrorOr<llvm::vfs::Status> getStatus() const { - auto MaybeStat = Entry.getStatus(); - if (!MaybeStat || MaybeStat->isDirectory()) - return MaybeStat; - return llvm::vfs::Status::copyWithNewSize(*MaybeStat, - getContents()->size()); + EntryRef(bool Minimized, StringRef Name, const CachedFileSystemEntry &Entry) + : Minimized(Minimized), Filename(Name), Entry(Entry) {} + + llvm::vfs::Status getStatus() const { + llvm::vfs::Status Stat = Entry.getStatus(); + if (!Stat.isDirectory()) + Stat = llvm::vfs::Status::copyWithNewSize(Stat, getContents().size()); + return llvm::vfs::Status::copyWithNewName(Stat, Filename); } + bool isError() const { return Entry.isError(); } bool isDirectory() const { return Entry.isDirectory(); } - StringRef getName() const { return Entry.getName(); } + /// If the cached entry represents an error, promotes it into `ErrorOr`. + llvm::ErrorOr<EntryRef> unwrapError() const { + if (isError()) + return Entry.getError(); + return *this; + } - llvm::ErrorOr<StringRef> getContents() const { + StringRef getContents() const { return Minimized ? Entry.getMinimizedContents() : Entry.getOriginalContents(); } @@ -234,9 +308,90 @@ public: private: /// Check whether the file should be minimized. - bool shouldMinimize(StringRef Filename); + bool shouldMinimize(StringRef Filename, llvm::sys::fs::UniqueID UID); + + /// Returns entry for the given filename. + /// + /// Attempts to use the local and shared caches first, then falls back to + /// using the underlying filesystem. + llvm::ErrorOr<EntryRef> + getOrCreateFileSystemEntry(StringRef Filename, + bool DisableMinimization = false); + + /// For a filename that's not yet associated with any entry in the caches, + /// uses the underlying filesystem to either look up the entry based in the + /// shared cache indexed by unique ID, or creates new entry from scratch. + llvm::ErrorOr<const CachedFileSystemEntry &> + computeAndStoreResult(StringRef Filename); + + /// Minimizes the given entry if necessary and returns a wrapper object with + /// reference semantics. + EntryRef minimizeIfNecessary(const CachedFileSystemEntry &Entry, + StringRef Filename, bool Disable); + + /// Represents a filesystem entry that has been stat-ed (and potentially read) + /// and that's about to be inserted into the cache as `CachedFileSystemEntry`. + struct TentativeEntry { + llvm::vfs::Status Status; + std::unique_ptr<llvm::MemoryBuffer> Contents; + + TentativeEntry(llvm::vfs::Status Status, + std::unique_ptr<llvm::MemoryBuffer> Contents = nullptr) + : Status(std::move(Status)), Contents(std::move(Contents)) {} + }; + + /// Reads file at the given path. Enforces consistency between the file size + /// in status and size of read contents. + llvm::ErrorOr<TentativeEntry> readFile(StringRef Filename); - llvm::ErrorOr<EntryRef> getOrCreateFileSystemEntry(StringRef Filename); + /// Returns entry associated with the unique ID of the given tentative entry + /// if there is some in the shared cache. Otherwise, constructs new one, + /// associates it with the unique ID and returns the result. + const CachedFileSystemEntry & + getOrEmplaceSharedEntryForUID(TentativeEntry TEntry); + + /// Returns entry associated with the filename or nullptr if none is found. + /// + /// Returns entry from local cache if there is some. Otherwise, if the entry + /// is found in the shared cache, writes it through the local cache and + /// returns it. Otherwise returns nullptr. + const CachedFileSystemEntry * + findEntryByFilenameWithWriteThrough(StringRef Filename); + + /// Returns entry associated with the unique ID in the shared cache or nullptr + /// if none is found. + const CachedFileSystemEntry * + findSharedEntryByUID(llvm::vfs::Status Stat) const { + return SharedCache.getShardForUID(Stat.getUniqueID()) + .findEntryByUID(Stat.getUniqueID()); + } + + /// Associates the given entry with the filename in the local cache and + /// returns it. + const CachedFileSystemEntry & + insertLocalEntryForFilename(StringRef Filename, + const CachedFileSystemEntry &Entry) { + return LocalCache.insertEntryForFilename(Filename, Entry); + } + + /// Returns entry associated with the filename in the shared cache if there is + /// some. Otherwise, constructs new one with the given error code, associates + /// it with the filename and returns the result. + const CachedFileSystemEntry & + getOrEmplaceSharedEntryForFilename(StringRef Filename, std::error_code EC) { + return SharedCache.getShardForFilename(Filename) + .getOrEmplaceEntryForFilename(Filename, EC); + } + + /// Returns entry associated with the filename in the shared cache if there is + /// some. Otherwise, associates the given entry with the filename and returns + /// it. + const CachedFileSystemEntry & + getOrInsertSharedEntryForFilename(StringRef Filename, + const CachedFileSystemEntry &Entry) { + return SharedCache.getShardForFilename(Filename) + .getOrInsertEntryForFilename(Filename, Entry); + } /// The global cache shared between worker threads. DependencyScanningFilesystemSharedCache &SharedCache; @@ -248,11 +403,11 @@ private: /// currently active preprocessor. ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings; /// The set of files that should not be minimized. - llvm::StringSet<> NotToBeMinimized; + llvm::DenseSet<llvm::sys::fs::UniqueID> NotToBeMinimized; }; } // end namespace dependencies } // end namespace tooling } // end namespace clang -#endif // LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_FILESYSTEM_H +#endif // LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H |