From 11b2b8c7e34c9456b8fe776f8b06b61b4b602de5 Mon Sep 17 00:00:00 2001 From: Jan Svoboda Date: Mon, 24 Mar 2025 10:14:15 -0700 Subject: [PATCH 1/2] Merge pull request #10286 from swiftlang/jan_svoboda/stable-no-file-locks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🍒 [clang][deps] Implement efficient in-process `ModuleCache` (cherry picked from commit 42e6d992f604daea7917028f949f6c460afcb5a5) --- clang/include/clang/Frontend/ASTUnit.h | 4 +- .../include/clang/Frontend/CompilerInstance.h | 13 +- clang/include/clang/Serialization/ASTReader.h | 11 +- clang/include/clang/Serialization/ASTWriter.h | 21 ++- .../include/clang/Serialization/ModuleCache.h | 50 ++++++ .../clang/Serialization/ModuleManager.h | 12 +- .../DependencyScanningService.h | 5 + .../DependencyScanning/InProcessModuleCache.h | 32 ++++ clang/lib/Frontend/ASTUnit.cpp | 21 ++- clang/lib/Frontend/CompilerInstance.cpp | 105 +++++++------ clang/lib/Frontend/PrecompiledPreamble.cpp | 5 +- clang/lib/Serialization/ASTReader.cpp | 48 +++--- clang/lib/Serialization/ASTWriter.cpp | 12 +- clang/lib/Serialization/CMakeLists.txt | 1 + clang/lib/Serialization/GeneratePCH.cpp | 8 +- clang/lib/Serialization/GlobalModuleIndex.cpp | 17 +- clang/lib/Serialization/ModuleCache.cpp | 44 ++++++ clang/lib/Serialization/ModuleManager.cpp | 23 +-- .../Tooling/DependencyScanning/CMakeLists.txt | 2 + .../DependencyScanningWorker.cpp | 5 +- .../InProcessModuleCache.cpp | 87 ++++++++++ .../unittests/Frontend/FrontendActionTest.cpp | 7 +- clang/unittests/Lex/HeaderSearchTest.cpp | 1 - llvm/include/llvm/Support/AdvisoryLock.h | 58 +++++++ llvm/include/llvm/Support/LockFileManager.h | 90 ++++------- llvm/lib/Support/LockFileManager.cpp | 148 +++++++----------- llvm/lib/Support/VirtualOutputBackends.cpp | 47 +++--- .../unittests/Support/LockFileManagerTest.cpp | 9 +- 28 files changed, 555 insertions(+), 331 deletions(-) create mode 100644 clang/include/clang/Serialization/ModuleCache.h create mode 100644 clang/include/clang/Tooling/DependencyScanning/InProcessModuleCache.h create mode 100644 clang/lib/Serialization/ModuleCache.cpp create mode 100644 clang/lib/Tooling/DependencyScanning/InProcessModuleCache.cpp create mode 100644 llvm/include/llvm/Support/AdvisoryLock.h diff --git a/clang/include/clang/Frontend/ASTUnit.h b/clang/include/clang/Frontend/ASTUnit.h index 080844893c13c..d35ef1a2883a7 100644 --- a/clang/include/clang/Frontend/ASTUnit.h +++ b/clang/include/clang/Frontend/ASTUnit.h @@ -70,7 +70,7 @@ class FileManager; class FrontendAction; class HeaderSearch; class InputKind; -class InMemoryModuleCache; +class ModuleCache; class PCHContainerOperations; class PCHContainerReader; class Preprocessor; @@ -110,7 +110,7 @@ class ASTUnit { IntrusiveRefCntPtr Diagnostics; IntrusiveRefCntPtr FileMgr; IntrusiveRefCntPtr SourceMgr; - IntrusiveRefCntPtr ModuleCache; + IntrusiveRefCntPtr ModCache; std::unique_ptr HeaderInfo; IntrusiveRefCntPtr Target; std::shared_ptr PP; diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h index 2c0d4dcb91873..25d798c378f66 100644 --- a/clang/include/clang/Frontend/CompilerInstance.h +++ b/clang/include/clang/Frontend/CompilerInstance.h @@ -54,8 +54,8 @@ class DiagnosticsEngine; class DiagnosticConsumer; class FileManager; class FrontendAction; -class InMemoryModuleCache; class Module; +class ModuleCache; class Preprocessor; class Sema; class SourceManager; @@ -115,7 +115,7 @@ class CompilerInstance : public ModuleLoader { IntrusiveRefCntPtr SourceMgr; /// The cache of PCM files. - IntrusiveRefCntPtr ModuleCache; + IntrusiveRefCntPtr ModCache; /// The preprocessor. std::shared_ptr PP; @@ -223,7 +223,7 @@ class CompilerInstance : public ModuleLoader { explicit CompilerInstance( std::shared_ptr PCHContainerOps = std::make_shared(), - InMemoryModuleCache *SharedModuleCache = nullptr); + ModuleCache *ModCache = nullptr); ~CompilerInstance() override; /// @name High-Level Operations @@ -793,9 +793,8 @@ class CompilerInstance : public ModuleLoader { static IntrusiveRefCntPtr createPCHExternalASTSource( StringRef Path, StringRef Sysroot, DisableValidationForModuleKind DisableValidation, - bool AllowPCHWithCompilerErrors, Preprocessor &PP, - InMemoryModuleCache &ModuleCache, ASTContext &Context, - const PCHContainerReader &PCHContainerRdr, + bool AllowPCHWithCompilerErrors, Preprocessor &PP, ModuleCache &ModCache, + ASTContext &Context, const PCHContainerReader &PCHContainerRdr, ArrayRef> Extensions, ArrayRef> DependencyCollectors, void *DeserializationListener, bool OwnDeserializationListener, @@ -966,7 +965,7 @@ class CompilerInstance : public ModuleLoader { bool addCachedModuleFile(StringRef Path, StringRef CacheKey, StringRef Provider); - InMemoryModuleCache &getModuleCache() const { return *ModuleCache; } + ModuleCache &getModuleCache() const { return *ModCache; } }; } // end namespace clang diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h index 16b63680ec6b6..e754236e832b0 100644 --- a/clang/include/clang/Serialization/ASTReader.h +++ b/clang/include/clang/Serialization/ASTReader.h @@ -88,7 +88,7 @@ struct HeaderFileInfo; class HeaderSearchOptions; class LangOptions; class MacroInfo; -class InMemoryModuleCache; +class ModuleCache; class NamedDecl; class NamespaceDecl; class ObjCCategoryDecl; @@ -1641,8 +1641,8 @@ class ASTReader /// /// \param ReadTimer If non-null, a timer used to track the time spent /// deserializing. - ASTReader(Preprocessor &PP, InMemoryModuleCache &ModuleCache, - ASTContext *Context, const PCHContainerReader &PCHContainerRdr, + ASTReader(Preprocessor &PP, ModuleCache &ModCache, ASTContext *Context, + const PCHContainerReader &PCHContainerRdr, ArrayRef> Extensions, StringRef isysroot = "", DisableValidationForModuleKind DisableValidationKind = @@ -1853,8 +1853,7 @@ class ASTReader /// /// \returns true if an error occurred, false otherwise. static bool readASTFileControlBlock( - StringRef Filename, FileManager &FileMgr, - const InMemoryModuleCache &ModuleCache, + StringRef Filename, FileManager &FileMgr, const ModuleCache &ModCache, const PCHContainerReader &PCHContainerRdr, bool FindModuleFileExtensions, ASTReaderListener &Listener, bool ValidateDiagnosticOptions, unsigned ClientLoadCapabilities = ARR_ConfigurationMismatch | @@ -1863,7 +1862,7 @@ class ASTReader /// Determine whether the given AST file is acceptable to load into a /// translation unit with the given language and target options. static bool isAcceptableASTFile(StringRef Filename, FileManager &FileMgr, - const InMemoryModuleCache &ModuleCache, + const ModuleCache &ModCache, const PCHContainerReader &PCHContainerRdr, const LangOptions &LangOpts, const TargetOptions &TargetOpts, diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h index db27904941098..f6a9c989e4907 100644 --- a/clang/include/clang/Serialization/ASTWriter.h +++ b/clang/include/clang/Serialization/ASTWriter.h @@ -60,7 +60,7 @@ class LangOptions; class MacroDefinitionRecord; class MacroInfo; class Module; -class InMemoryModuleCache; +class ModuleCache; class ModuleFileExtension; class ModuleFileExtensionWriter; class NamedDecl; @@ -117,7 +117,7 @@ class ASTWriter : public ASTDeserializationListener, const SmallVectorImpl &Buffer; /// The PCM manager which manages memory buffers for pcm files. - InMemoryModuleCache &ModuleCache; + ModuleCache &ModCache; /// The preprocessor we're writing. Preprocessor *PP = nullptr; @@ -638,7 +638,7 @@ class ASTWriter : public ASTDeserializationListener, /// Create a new precompiled header writer that outputs to /// the given bitstream. ASTWriter(llvm::BitstreamWriter &Stream, SmallVectorImpl &Buffer, - InMemoryModuleCache &ModuleCache, + ModuleCache &ModCache, ArrayRef> Extensions, bool IncludeTimestamps = true, bool BuildingImplicitModule = false, bool GeneratingReducedBMI = false); @@ -940,9 +940,8 @@ class PCHGenerator : public SemaConsumer { virtual Module *getEmittingModule(ASTContext &Ctx); public: - PCHGenerator(Preprocessor &PP, InMemoryModuleCache &ModuleCache, - StringRef OutputFile, StringRef isysroot, - std::shared_ptr Buffer, + PCHGenerator(Preprocessor &PP, ModuleCache &ModCache, StringRef OutputFile, + StringRef isysroot, std::shared_ptr Buffer, ArrayRef> Extensions, bool AllowASTWithErrors = false, bool IncludeTimestamps = true, bool BuildingImplicitModule = false, @@ -964,13 +963,13 @@ class CXX20ModulesGenerator : public PCHGenerator { protected: virtual Module *getEmittingModule(ASTContext &Ctx) override; - CXX20ModulesGenerator(Preprocessor &PP, InMemoryModuleCache &ModuleCache, + CXX20ModulesGenerator(Preprocessor &PP, ModuleCache &ModCache, StringRef OutputFile, bool GeneratingReducedBMI); public: - CXX20ModulesGenerator(Preprocessor &PP, InMemoryModuleCache &ModuleCache, + CXX20ModulesGenerator(Preprocessor &PP, ModuleCache &ModCache, StringRef OutputFile) - : CXX20ModulesGenerator(PP, ModuleCache, OutputFile, + : CXX20ModulesGenerator(PP, ModCache, OutputFile, /*GeneratingReducedBMI=*/false) {} void HandleTranslationUnit(ASTContext &Ctx) override; @@ -980,9 +979,9 @@ class ReducedBMIGenerator : public CXX20ModulesGenerator { void anchor() override; public: - ReducedBMIGenerator(Preprocessor &PP, InMemoryModuleCache &ModuleCache, + ReducedBMIGenerator(Preprocessor &PP, ModuleCache &ModCache, StringRef OutputFile) - : CXX20ModulesGenerator(PP, ModuleCache, OutputFile, + : CXX20ModulesGenerator(PP, ModCache, OutputFile, /*GeneratingReducedBMI=*/true) {} }; diff --git a/clang/include/clang/Serialization/ModuleCache.h b/clang/include/clang/Serialization/ModuleCache.h new file mode 100644 index 0000000000000..a7ba26bc4daae --- /dev/null +++ b/clang/include/clang/Serialization/ModuleCache.h @@ -0,0 +1,50 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_SERIALIZATION_MODULECACHE_H +#define LLVM_CLANG_SERIALIZATION_MODULECACHE_H + +#include "clang/Basic/LLVM.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" + +namespace llvm { +class AdvisoryLock; +} // namespace llvm + +namespace clang { +class InMemoryModuleCache; + +/// The module cache used for compiling modules implicitly. This centralizes the +/// operations the compiler might want to perform on the cache. +class ModuleCache : public RefCountedBase { +public: + /// May perform any work that only needs to be performed once for multiple + /// calls \c getLock() with the same module filename. + virtual void prepareForGetLock(StringRef ModuleFilename) = 0; + + /// Returns lock for the given module file. The lock is initially unlocked. + virtual std::unique_ptr + getLock(StringRef ModuleFilename) = 0; + + /// Returns this process's view of the module cache. + virtual InMemoryModuleCache &getInMemoryModuleCache() = 0; + virtual const InMemoryModuleCache &getInMemoryModuleCache() const = 0; + + // TODO: Virtualize writing/reading PCM files, timestamping, pruning, etc. + + virtual ~ModuleCache() = default; +}; + +/// Creates new \c ModuleCache backed by a file system directory that may be +/// operated on by multiple processes. This instance must be used across all +/// \c CompilerInstance instances participating in building modules for single +/// translation unit in order to share the same \c InMemoryModuleCache. +IntrusiveRefCntPtr createCrossProcessModuleCache(); +} // namespace clang + +#endif diff --git a/clang/include/clang/Serialization/ModuleManager.h b/clang/include/clang/Serialization/ModuleManager.h index f898dab39f06d..1eb74aee9787c 100644 --- a/clang/include/clang/Serialization/ModuleManager.h +++ b/clang/include/clang/Serialization/ModuleManager.h @@ -37,7 +37,7 @@ class FileEntry; class FileManager; class GlobalModuleIndex; class HeaderSearch; -class InMemoryModuleCache; +class ModuleCache; class PCHContainerReader; namespace serialization { @@ -65,7 +65,7 @@ class ModuleManager { FileManager &FileMgr; /// Cache of PCM files. - IntrusiveRefCntPtr ModuleCache; + IntrusiveRefCntPtr ModCache; /// Knows how to unwrap module containers. const PCHContainerReader &PCHContainerRdr; @@ -133,9 +133,9 @@ class ModuleManager { SmallVectorImpl>::reverse_iterator>; using ModuleOffset = std::pair; - explicit ModuleManager(FileManager &FileMgr, InMemoryModuleCache &ModuleCache, - const PCHContainerReader &PCHContainerRdr, - const HeaderSearch &HeaderSearchInfo); + ModuleManager(FileManager &FileMgr, ModuleCache &ModCache, + const PCHContainerReader &PCHContainerRdr, + const HeaderSearch &HeaderSearchInfo); /// Forward iterator to traverse all loaded modules. ModuleIterator begin() { return Chain.begin(); } @@ -306,7 +306,7 @@ class ModuleManager { /// View the graphviz representation of the module graph. void viewGraph(); - InMemoryModuleCache &getModuleCache() const { return *ModuleCache; } + ModuleCache &getModuleCache() const { return *ModCache; } }; } // namespace serialization diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h index 572a8a3666396..331509b7c435b 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h @@ -12,6 +12,7 @@ #include "clang/CAS/CASOptions.h" #include "clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h" #include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h" +#include "clang/Tooling/DependencyScanning/InProcessModuleCache.h" #include "llvm/ADT/BitmaskEnum.h" #include "llvm/CAS/ActionCache.h" @@ -123,6 +124,8 @@ class DependencyScanningService { bool useCASFS() const { return (bool)SharedFS; } + ModuleCacheMutexes &getModuleCacheMutexes() { return ModCacheMutexes; } + private: const ScanningMode Mode; const ScanningOutputFormat Format; @@ -140,6 +143,8 @@ class DependencyScanningService { IntrusiveRefCntPtr SharedFS; /// The global file system cache. std::optional SharedCache; + /// The global module cache mutexes. + ModuleCacheMutexes ModCacheMutexes; }; } // end namespace dependencies diff --git a/clang/include/clang/Tooling/DependencyScanning/InProcessModuleCache.h b/clang/include/clang/Tooling/DependencyScanning/InProcessModuleCache.h new file mode 100644 index 0000000000000..ba0454380b665 --- /dev/null +++ b/clang/include/clang/Tooling/DependencyScanning/InProcessModuleCache.h @@ -0,0 +1,32 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_INPROCESSMODULECACHE_H +#define LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_INPROCESSMODULECACHE_H + +#include "clang/Serialization/ModuleCache.h" +#include "llvm/ADT/StringMap.h" + +#include +#include + +namespace clang { +namespace tooling { +namespace dependencies { +struct ModuleCacheMutexes { + std::mutex Mutex; + llvm::StringMap> Map; +}; + +IntrusiveRefCntPtr +makeInProcessModuleCache(ModuleCacheMutexes &Mutexes); +} // namespace dependencies +} // namespace tooling +} // namespace clang + +#endif diff --git a/clang/lib/Frontend/ASTUnit.cpp b/clang/lib/Frontend/ASTUnit.cpp index ee7c36d3eba9f..95d41ef6ce93d 100644 --- a/clang/lib/Frontend/ASTUnit.cpp +++ b/clang/lib/Frontend/ASTUnit.cpp @@ -61,7 +61,7 @@ #include "clang/Serialization/ASTReader.h" #include "clang/Serialization/ASTWriter.h" #include "clang/Serialization/ContinuousRangeMap.h" -#include "clang/Serialization/InMemoryModuleCache.h" +#include "clang/Serialization/ModuleCache.h" #include "clang/Serialization/ModuleFile.h" #include "clang/Serialization/PCHContainerOperations.h" #include "llvm/ADT/ArrayRef.h" @@ -219,8 +219,8 @@ struct ASTUnit::ASTWriterData { llvm::BitstreamWriter Stream; ASTWriter Writer; - ASTWriterData(InMemoryModuleCache &ModuleCache) - : Stream(Buffer), Writer(Stream, Buffer, ModuleCache, {}) {} + ASTWriterData(ModuleCache &ModCache) + : Stream(Buffer), Writer(Stream, Buffer, ModCache, {}) {} }; void ASTUnit::clearFileLevelDecls() { @@ -825,7 +825,7 @@ std::unique_ptr ASTUnit::LoadFromASTFile( AST->SourceMgr = new SourceManager(AST->getDiagnostics(), AST->getFileManager(), UserFilesAreVolatile); - AST->ModuleCache = new InMemoryModuleCache; + AST->ModCache = createCrossProcessModuleCache(); AST->HSOpts = HSOpts ? HSOpts : std::make_shared(); AST->HSOpts->ModuleFormat = std::string(PCHContainerRdr.getFormats().front()); AST->HeaderInfo.reset(new HeaderSearch(AST->HSOpts, @@ -857,8 +857,7 @@ std::unique_ptr ASTUnit::LoadFromASTFile( if (::getenv("LIBCLANG_DISABLE_PCH_VALIDATION")) disableValid = DisableValidationForModuleKind::All; AST->Reader = new ASTReader( - PP, *AST->ModuleCache, AST->Ctx.get(), PCHContainerRdr, {}, - /*isysroot=*/"", + PP, *AST->ModCache, AST->Ctx.get(), PCHContainerRdr, {}, /*isysroot=*/"", /*DisableValidationKind=*/disableValid, AllowASTWithCompilerErrors); unsigned Counter = 0; @@ -1542,7 +1541,7 @@ ASTUnit::create(std::shared_ptr CI, AST->UserFilesAreVolatile = UserFilesAreVolatile; AST->SourceMgr = new SourceManager(AST->getDiagnostics(), *AST->FileMgr, UserFilesAreVolatile); - AST->ModuleCache = new InMemoryModuleCache; + AST->ModCache = createCrossProcessModuleCache(); return AST; } @@ -1829,7 +1828,7 @@ std::unique_ptr ASTUnit::LoadFromCommandLine( AST->FileMgr = new FileManager(AST->FileSystemOpts, VFS); AST->StorePreamblesInMemory = StorePreamblesInMemory; AST->PreambleStoragePath = PreambleStoragePath; - AST->ModuleCache = new InMemoryModuleCache; + AST->ModCache = createCrossProcessModuleCache(); AST->OnlyLocalDecls = OnlyLocalDecls; AST->CaptureDiagnostics = CaptureDiagnostics; AST->TUKind = TUKind; @@ -1840,7 +1839,7 @@ std::unique_ptr ASTUnit::LoadFromCommandLine( AST->Invocation = CI; AST->SkipFunctionBodies = SkipFunctionBodies; if (ForSerialization) - AST->WriterData.reset(new ASTWriterData(*AST->ModuleCache)); + AST->WriterData.reset(new ASTWriterData(*AST->ModCache)); // Zero out now to ease cleanup during crash recovery. CI = nullptr; Diags = nullptr; @@ -2373,8 +2372,8 @@ bool ASTUnit::serialize(raw_ostream &OS) { SmallString<128> Buffer; llvm::BitstreamWriter Stream(Buffer); - InMemoryModuleCache ModuleCache; - ASTWriter Writer(Stream, Buffer, ModuleCache, {}); + IntrusiveRefCntPtr ModCache = createCrossProcessModuleCache(); + ASTWriter Writer(Stream, Buffer, *ModCache, {}); return serializeUnit(Writer, Buffer, getSema(), OS); } diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp index 7c8fd631913b8..c6e754d87bc32 100644 --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -46,17 +46,18 @@ #include "clang/Serialization/ASTReader.h" #include "clang/Serialization/GlobalModuleIndex.h" #include "clang/Serialization/InMemoryModuleCache.h" +#include "clang/Serialization/ModuleCache.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/Statistic.h" #include "llvm/CAS/ActionCache.h" #include "llvm/CAS/ObjectStore.h" #include "llvm/Config/llvm-config.h" +#include "llvm/Support/AdvisoryLock.h" #include "llvm/Support/BuryPointer.h" #include "llvm/Support/CrashRecoveryContext.h" #include "llvm/Support/Errc.h" #include "llvm/Support/FileSystem.h" -#include "llvm/Support/LockFileManager.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/Program.h" @@ -75,11 +76,10 @@ using namespace clang; CompilerInstance::CompilerInstance( std::shared_ptr PCHContainerOps, - InMemoryModuleCache *SharedModuleCache) - : ModuleLoader(/* BuildingModule = */ SharedModuleCache), + ModuleCache *ModCache) + : ModuleLoader(/*BuildingModule=*/ModCache), Invocation(new CompilerInvocation()), - ModuleCache(SharedModuleCache ? SharedModuleCache - : new InMemoryModuleCache), + ModCache(ModCache ? ModCache : createCrossProcessModuleCache()), ThePCHContainerOperations(std::move(PCHContainerOps)) {} CompilerInstance::~CompilerInstance() { @@ -219,7 +219,7 @@ IntrusiveRefCntPtr CompilerInstance::getASTReader() const { return TheASTReader; } void CompilerInstance::setASTReader(IntrusiveRefCntPtr Reader) { - assert(ModuleCache.get() == &Reader->getModuleManager().getModuleCache() && + assert(ModCache.get() == &Reader->getModuleManager().getModuleCache() && "Expected ASTReader to use the same PCM cache"); TheASTReader = std::move(Reader); } @@ -633,9 +633,8 @@ struct ReadModuleNames : ASTReaderListener { class CompileCacheASTReaderHelper : public ASTReaderListener { public: CompileCacheASTReaderHelper(cas::ObjectStore &CAS, cas::ActionCache &Cache, - InMemoryModuleCache &ModuleCache, - DiagnosticsEngine &Diags) - : CAS(CAS), Cache(Cache), ModuleCache(ModuleCache), Diags(Diags) {} + ModuleCache &ModCache, DiagnosticsEngine &Diags) + : CAS(CAS), Cache(Cache), ModCache(ModCache), Diags(Diags) {} bool readCASFileSystemRootID(StringRef RootID, bool Complain) override; bool readIncludeTreeID(StringRef ID, bool Complain) override; @@ -648,7 +647,7 @@ class CompileCacheASTReaderHelper : public ASTReaderListener { cas::ObjectStore &CAS; cas::ActionCache &Cache; - InMemoryModuleCache &ModuleCache; + ModuleCache &ModCache; DiagnosticsEngine &Diags; }; } // namespace @@ -673,9 +672,8 @@ void CompilerInstance::createPCHExternalASTSource( IntrusiveRefCntPtr CompilerInstance::createPCHExternalASTSource( StringRef Path, StringRef Sysroot, DisableValidationForModuleKind DisableValidation, - bool AllowPCHWithCompilerErrors, Preprocessor &PP, - InMemoryModuleCache &ModuleCache, ASTContext &Context, - const PCHContainerReader &PCHContainerRdr, + bool AllowPCHWithCompilerErrors, Preprocessor &PP, ModuleCache &ModCache, + ASTContext &Context, const PCHContainerReader &PCHContainerRdr, ArrayRef> Extensions, ArrayRef> DependencyCollectors, void *DeserializationListener, bool OwnDeserializationListener, @@ -685,7 +683,7 @@ IntrusiveRefCntPtr CompilerInstance::createPCHExternalASTSource( HeaderSearchOptions &HSOpts = PP.getHeaderSearchInfo().getHeaderSearchOpts(); IntrusiveRefCntPtr Reader(new ASTReader( - PP, ModuleCache, &Context, PCHContainerRdr, Extensions, + PP, ModCache, &Context, PCHContainerRdr, Extensions, Sysroot.empty() ? "" : Sysroot.data(), DisableValidation, AllowPCHWithCompilerErrors, /*AllowConfigurationMismatch*/ false, HSOpts.ModulesValidateSystemHeaders, HSOpts.ValidateASTInputFilesContent, @@ -704,7 +702,7 @@ IntrusiveRefCntPtr CompilerInstance::createPCHExternalASTSource( if (!ignoreCAS) Reader->addListener(std::make_unique( - CAS, Cache, ModuleCache, PP.getDiagnostics())); + CAS, Cache, ModCache, PP.getDiagnostics())); auto Listener = std::make_unique(PP); auto &ListenerRef = *Listener; @@ -1302,7 +1300,8 @@ compileModuleImpl(CompilerInstance &ImportingInstance, SourceLocation ImportLoc, // Never compile a module that's already finalized - this would cause the // existing module to be freed, causing crashes if it is later referenced - if (ImportingInstance.getModuleCache().isPCMFinal(ModuleFileName)) { + if (ImportingInstance.getModuleCache().getInMemoryModuleCache().isPCMFinal( + ModuleFileName)) { ImportingInstance.getDiagnostics().Report( ImportLoc, diag::err_module_rebuild_finalized) << ModuleName; @@ -1557,7 +1556,7 @@ static bool readASTAfterCompileModule(CompilerInstance &ImportingInstance, SourceLocation ImportLoc, SourceLocation ModuleNameLoc, Module *Module, StringRef ModuleFileName, - bool *OutOfDate) { + bool *OutOfDate, bool *Missing) { DiagnosticsEngine &Diags = ImportingInstance.getDiagnostics(); unsigned ModuleLoadCapabilities = ASTReader::ARR_Missing; @@ -1578,6 +1577,12 @@ static bool readASTAfterCompileModule(CompilerInstance &ImportingInstance, return false; } + // The caller wants to handle missing module files. + if (Missing && ReadResult == ASTReader::Missing) { + *Missing = true; + return false; + } + // The ASTReader didn't diagnose the error, so conservatively report it. if (ReadResult == ASTReader::Missing || !Diags.hasErrorOccurred()) Diags.Report(ModuleNameLoc, diag::err_module_not_built) @@ -1603,7 +1608,7 @@ static bool compileModuleAndReadASTImpl(CompilerInstance &ImportingInstance, return readASTAfterCompileModule(ImportingInstance, ImportLoc, ModuleNameLoc, Module, ModuleFileName, - /*OutOfDate=*/nullptr); + /*OutOfDate=*/nullptr, /*Missing=*/nullptr); } /// Compile a module in a separate compiler instance and read the AST, @@ -1622,61 +1627,58 @@ static bool compileModuleAndReadASTBehindLock( Diags.Report(ModuleNameLoc, diag::remark_module_lock) << ModuleFileName << Module->Name; - // FIXME: have LockFileManager return an error_code so that we can - // avoid the mkdir when the directory already exists. - StringRef Dir = llvm::sys::path::parent_path(ModuleFileName); - llvm::sys::fs::create_directories(Dir); + auto &ModuleCache = ImportingInstance.getModuleCache(); + ModuleCache.prepareForGetLock(ModuleFileName); while (true) { - llvm::LockFileManager Locked(ModuleFileName); - switch (Locked) { - case llvm::LockFileManager::LFS_Error: + auto Lock = ModuleCache.getLock(ModuleFileName); + bool Owned; + if (llvm::Error Err = Lock->tryLock().moveInto(Owned)) { // ModuleCache takes care of correctness and locks are only necessary for // performance. Fallback to building the module in case of any lock // related errors. Diags.Report(ModuleNameLoc, diag::remark_module_lock_failure) - << Module->Name << Locked.getErrorMessage(); - // Clear out any potential leftover. - Locked.unsafeRemoveLockFile(); - [[fallthrough]]; - case llvm::LockFileManager::LFS_Owned: + << Module->Name << toString(std::move(Err)); + return compileModuleAndReadASTImpl(ImportingInstance, ImportLoc, + ModuleNameLoc, Module, ModuleFileName); + } + if (Owned) { // We're responsible for building the module ourselves. return compileModuleAndReadASTImpl(ImportingInstance, ImportLoc, ModuleNameLoc, Module, ModuleFileName); - - case llvm::LockFileManager::LFS_Shared: - break; // The interesting case. } // Someone else is responsible for building the module. Wait for them to // finish. - switch (Locked.waitForUnlock()) { - case llvm::LockFileManager::Res_Success: + switch (Lock->waitForUnlockFor(std::chrono::seconds(90))) { + case llvm::WaitForUnlockResult::Success: break; // The interesting case. - case llvm::LockFileManager::Res_OwnerDied: + case llvm::WaitForUnlockResult::OwnerDied: continue; // try again to get the lock. - case llvm::LockFileManager::Res_Timeout: - // Since ModuleCache takes care of correctness, we try waiting for - // another process to complete the build so clang does not do it done - // twice. If case of timeout, build it ourselves. + case llvm::WaitForUnlockResult::Timeout: + // Since the InMemoryModuleCache takes care of correctness, we try waiting + // for someone else to complete the build so that it does not happen + // twice. In case of timeout, build it ourselves. Diags.Report(ModuleNameLoc, diag::remark_module_lock_timeout) << Module->Name; // Clear the lock file so that future invocations can make progress. - Locked.unsafeRemoveLockFile(); + Lock->unsafeMaybeUnlock(); continue; } // Read the module that was just written by someone else. bool OutOfDate = false; + bool Missing = false; if (readASTAfterCompileModule(ImportingInstance, ImportLoc, ModuleNameLoc, - Module, ModuleFileName, &OutOfDate)) + Module, ModuleFileName, &OutOfDate, &Missing)) return true; - if (!OutOfDate) + if (!OutOfDate && !Missing) return false; - // The module may be out of date in the presence of file system races, - // or if one of its imports depends on header search paths that are not - // consistent with this ImportingInstance. Try again... + // The module may be missing or out of date in the presence of file system + // races. It may also be out of date if one of its imports depends on header + // search paths that are not consistent with this ImportingInstance. + // Try again... } } @@ -2526,10 +2528,10 @@ void CompilerInstance::setExternalSemaSource( static bool addCachedModuleFileToInMemoryCache( StringRef Path, StringRef CacheKey, StringRef Provider, - cas::ObjectStore &CAS, cas::ActionCache &Cache, - InMemoryModuleCache &ModuleCache, DiagnosticsEngine &Diags) { + cas::ObjectStore &CAS, cas::ActionCache &Cache, ModuleCache &ModCache, + DiagnosticsEngine &Diags) { - if (ModuleCache.lookupPCM(Path)) + if (ModCache.getInMemoryModuleCache().lookupPCM(Path)) return false; auto ID = CAS.parseID(CacheKey); @@ -2585,7 +2587,8 @@ static bool addCachedModuleFileToInMemoryCache( return true; } - ModuleCache.addPCM(Path, OutputProxy->getMemoryBuffer()); + ModCache.getInMemoryModuleCache().addPCM(Path, + OutputProxy->getMemoryBuffer()); return false; } @@ -2601,7 +2604,7 @@ bool CompileCacheASTReaderHelper::readModuleCacheKey(StringRef ModuleName, StringRef CacheKey) { // FIXME: add name/path of the importing module? return addCachedModuleFileToInMemoryCache( - Filename, CacheKey, "imported module", CAS, Cache, ModuleCache, Diags); + Filename, CacheKey, "imported module", CAS, Cache, ModCache, Diags); } /// Verify that ID is in the CAS. Otherwise the module cache probably was diff --git a/clang/lib/Frontend/PrecompiledPreamble.cpp b/clang/lib/Frontend/PrecompiledPreamble.cpp index cab5838fceb24..b4cb1f2500da2 100644 --- a/clang/lib/Frontend/PrecompiledPreamble.cpp +++ b/clang/lib/Frontend/PrecompiledPreamble.cpp @@ -292,10 +292,9 @@ class PrecompilePreambleAction : public ASTFrontendAction { class PrecompilePreambleConsumer : public PCHGenerator { public: PrecompilePreambleConsumer(PrecompilePreambleAction &Action, Preprocessor &PP, - InMemoryModuleCache &ModuleCache, - StringRef isysroot, + ModuleCache &ModCache, StringRef isysroot, std::shared_ptr Buffer) - : PCHGenerator(PP, ModuleCache, "", isysroot, std::move(Buffer), + : PCHGenerator(PP, ModCache, "", isysroot, std::move(Buffer), ArrayRef>(), /*AllowASTWithErrors=*/true), Action(Action) {} diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index d68cef0f7e5bb..6f7e35b5578f7 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -88,6 +88,7 @@ #include "clang/Serialization/ContinuousRangeMap.h" #include "clang/Serialization/GlobalModuleIndex.h" #include "clang/Serialization/InMemoryModuleCache.h" +#include "clang/Serialization/ModuleCache.h" #include "clang/Serialization/ModuleFile.h" #include "clang/Serialization/ModuleFileExtension.h" #include "clang/Serialization/ModuleManager.h" @@ -3150,9 +3151,12 @@ ASTReader::ReadControlBlock(ModuleFile &F, StoredSignature, Capabilities); // If we diagnosed a problem, produce a backtrace. - bool recompilingFinalized = - Result == OutOfDate && (Capabilities & ARR_OutOfDate) && - getModuleManager().getModuleCache().isPCMFinal(F.FileName); + bool recompilingFinalized = Result == OutOfDate && + (Capabilities & ARR_OutOfDate) && + getModuleManager() + .getModuleCache() + .getInMemoryModuleCache() + .isPCMFinal(F.FileName); if (isDiagnosedResult(Result, Capabilities) || recompilingFinalized) Diag(diag::note_module_file_imported_by) << F.FileName << !F.ModuleName.empty() << F.ModuleName; @@ -4772,7 +4776,7 @@ ASTReader::ReadASTCore(StringRef FileName, bool ShouldFinalizePCM = false; auto FinalizeOrDropPCM = llvm::make_scope_exit([&]() { - auto &MC = getModuleManager().getModuleCache(); + auto &MC = getModuleManager().getModuleCache().getInMemoryModuleCache(); if (ShouldFinalizePCM) MC.finalizePCM(FileName); else @@ -4913,7 +4917,8 @@ ASTReader::readUnhashedControlBlock(ModuleFile &F, bool WasImportedBy, // validation will fail during the as-system import since the PCM on disk // doesn't guarantee that -Werror was respected. However, the -Werror // flags were checked during the initial as-user import. - if (getModuleManager().getModuleCache().isPCMFinal(F.FileName)) { + if (getModuleManager().getModuleCache().getInMemoryModuleCache().isPCMFinal( + F.FileName)) { Diag(diag::warn_module_system_bit_conflict) << F.FileName; return Success; } @@ -5422,14 +5427,14 @@ namespace { } // namespace bool ASTReader::readASTFileControlBlock( - StringRef Filename, FileManager &FileMgr, - const InMemoryModuleCache &ModuleCache, + StringRef Filename, FileManager &FileMgr, const ModuleCache &ModCache, const PCHContainerReader &PCHContainerRdr, bool FindModuleFileExtensions, ASTReaderListener &Listener, bool ValidateDiagnosticOptions, unsigned ClientLoadCapabilities) { // Open the AST file. std::unique_ptr OwnedBuffer; - llvm::MemoryBuffer *Buffer = ModuleCache.lookupPCM(Filename); + llvm::MemoryBuffer *Buffer = + ModCache.getInMemoryModuleCache().lookupPCM(Filename); if (!Buffer) { // FIXME: We should add the pcm to the InMemoryModuleCache if it could be // read again later, but we do not have the context here to determine if it @@ -5719,19 +5724,15 @@ bool ASTReader::readASTFileControlBlock( return false; } -bool ASTReader::isAcceptableASTFile(StringRef Filename, FileManager &FileMgr, - const InMemoryModuleCache &ModuleCache, - const PCHContainerReader &PCHContainerRdr, - const LangOptions &LangOpts, - const TargetOptions &TargetOpts, - const PreprocessorOptions &PPOpts, - StringRef ExistingModuleCachePath, - bool RequireStrictOptionMatches) { +bool ASTReader::isAcceptableASTFile( + StringRef Filename, FileManager &FileMgr, const ModuleCache &ModCache, + const PCHContainerReader &PCHContainerRdr, const LangOptions &LangOpts, + const TargetOptions &TargetOpts, const PreprocessorOptions &PPOpts, + StringRef ExistingModuleCachePath, bool RequireStrictOptionMatches) { SimplePCHValidator validator(LangOpts, TargetOpts, PPOpts, ExistingModuleCachePath, FileMgr, RequireStrictOptionMatches); - return !readASTFileControlBlock(Filename, FileMgr, ModuleCache, - PCHContainerRdr, + return !readASTFileControlBlock(Filename, FileMgr, ModCache, PCHContainerRdr, /*FindModuleFileExtensions=*/false, validator, /*ValidateDiagnosticOptions=*/true); } @@ -6273,7 +6274,10 @@ ASTReader::getModulePreprocessedEntities(ModuleFile &Mod) const { bool ASTReader::canRecoverFromOutOfDate(StringRef ModuleFileName, unsigned int ClientLoadCapabilities) { return ClientLoadCapabilities & ARR_OutOfDate && - !getModuleManager().getModuleCache().isPCMFinal(ModuleFileName); + !getModuleManager() + .getModuleCache() + .getInMemoryModuleCache() + .isPCMFinal(ModuleFileName); } llvm::iterator_range @@ -10341,7 +10345,7 @@ void ASTReader::pushExternalDeclIntoScope(NamedDecl *D, DeclarationName Name) { } } -ASTReader::ASTReader(Preprocessor &PP, InMemoryModuleCache &ModuleCache, +ASTReader::ASTReader(Preprocessor &PP, ModuleCache &ModCache, ASTContext *Context, const PCHContainerReader &PCHContainerRdr, ArrayRef> Extensions, @@ -10351,12 +10355,12 @@ ASTReader::ASTReader(Preprocessor &PP, InMemoryModuleCache &ModuleCache, bool AllowConfigurationMismatch, bool ValidateSystemInputs, bool ValidateASTInputFilesContent, bool UseGlobalIndex, std::unique_ptr ReadTimer) - : Listener(bool(DisableValidationKind &DisableValidationForModuleKind::PCH) + : Listener(bool(DisableValidationKind & DisableValidationForModuleKind::PCH) ? cast(new SimpleASTReaderListener(PP)) : cast(new PCHValidator(PP, *this))), SourceMgr(PP.getSourceManager()), FileMgr(PP.getFileManager()), PCHContainerRdr(PCHContainerRdr), Diags(PP.getDiagnostics()), PP(PP), - ContextObj(Context), ModuleMgr(PP.getFileManager(), ModuleCache, + ContextObj(Context), ModuleMgr(PP.getFileManager(), ModCache, PCHContainerRdr, PP.getHeaderSearchInfo()), DummyIdResolver(PP), ReadTimer(std::move(ReadTimer)), isysroot(isysroot), DisableValidationKind(DisableValidationKind), diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index d693a767d998f..75bb57e662e19 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -72,6 +72,7 @@ #include "clang/Serialization/ASTReader.h" #include "clang/Serialization/ASTRecordWriter.h" #include "clang/Serialization/InMemoryModuleCache.h" +#include "clang/Serialization/ModuleCache.h" #include "clang/Serialization/ModuleFile.h" #include "clang/Serialization/ModuleFileExtension.h" #include "clang/Serialization/SerializationDiagnostic.h" @@ -4822,12 +4823,11 @@ void ASTWriter::SetSelectorOffset(Selector Sel, uint32_t Offset) { } ASTWriter::ASTWriter(llvm::BitstreamWriter &Stream, - SmallVectorImpl &Buffer, - InMemoryModuleCache &ModuleCache, + SmallVectorImpl &Buffer, ModuleCache &ModCache, ArrayRef> Extensions, bool IncludeTimestamps, bool BuildingImplicitModule, bool GeneratingReducedBMI) - : Stream(Stream), Buffer(Buffer), ModuleCache(ModuleCache), + : Stream(Stream), Buffer(Buffer), ModCache(ModCache), IncludeTimestamps(IncludeTimestamps), BuildingImplicitModule(BuildingImplicitModule), GeneratingReducedBMI(GeneratingReducedBMI) { @@ -4885,9 +4885,9 @@ ASTWriter::WriteAST(llvm::PointerUnion Subject, if (ShouldCacheASTInMemory) { // Construct MemoryBuffer and update buffer manager. - ModuleCache.addBuiltPCM(OutputFile, - llvm::MemoryBuffer::getMemBufferCopy( - StringRef(Buffer.begin(), Buffer.size()))); + ModCache.getInMemoryModuleCache().addBuiltPCM( + OutputFile, llvm::MemoryBuffer::getMemBufferCopy( + StringRef(Buffer.begin(), Buffer.size()))); } return Signature; } diff --git a/clang/lib/Serialization/CMakeLists.txt b/clang/lib/Serialization/CMakeLists.txt index 99c47c15a2f47..6b662d43fae86 100644 --- a/clang/lib/Serialization/CMakeLists.txt +++ b/clang/lib/Serialization/CMakeLists.txt @@ -18,6 +18,7 @@ add_clang_library(clangSerialization GeneratePCH.cpp GlobalModuleIndex.cpp InMemoryModuleCache.cpp + ModuleCache.cpp ModuleFile.cpp ModuleFileExtension.cpp ModuleManager.cpp diff --git a/clang/lib/Serialization/GeneratePCH.cpp b/clang/lib/Serialization/GeneratePCH.cpp index 7a8a951b34f25..601cb94a51e23 100644 --- a/clang/lib/Serialization/GeneratePCH.cpp +++ b/clang/lib/Serialization/GeneratePCH.cpp @@ -23,7 +23,7 @@ using namespace clang; PCHGenerator::PCHGenerator( - Preprocessor &PP, InMemoryModuleCache &ModuleCache, StringRef OutputFile, + Preprocessor &PP, ModuleCache &ModCache, StringRef OutputFile, StringRef isysroot, std::shared_ptr Buffer, ArrayRef> Extensions, bool AllowASTWithErrors, bool IncludeTimestamps, @@ -31,7 +31,7 @@ PCHGenerator::PCHGenerator( bool GeneratingReducedBMI) : PP(PP), Subject(&PP), OutputFile(OutputFile), isysroot(isysroot.str()), Buffer(std::move(Buffer)), Stream(this->Buffer->Data), - Writer(Stream, this->Buffer->Data, ModuleCache, Extensions, + Writer(Stream, this->Buffer->Data, ModCache, Extensions, IncludeTimestamps, BuildingImplicitModule, GeneratingReducedBMI), AllowASTWithErrors(AllowASTWithErrors), ShouldCacheASTInMemory(ShouldCacheASTInMemory) { @@ -100,11 +100,11 @@ ASTDeserializationListener *PCHGenerator::GetASTDeserializationListener() { void PCHGenerator::anchor() {} CXX20ModulesGenerator::CXX20ModulesGenerator(Preprocessor &PP, - InMemoryModuleCache &ModuleCache, + ModuleCache &ModCache, StringRef OutputFile, bool GeneratingReducedBMI) : PCHGenerator( - PP, ModuleCache, OutputFile, llvm::StringRef(), + PP, ModCache, OutputFile, llvm::StringRef(), std::make_shared(), /*Extensions=*/ArrayRef>(), /*AllowASTWithErrors*/ false, /*IncludeTimestamps=*/false, diff --git a/clang/lib/Serialization/GlobalModuleIndex.cpp b/clang/lib/Serialization/GlobalModuleIndex.cpp index 94b8f17b6a61c..21728ab0acbd8 100644 --- a/clang/lib/Serialization/GlobalModuleIndex.cpp +++ b/clang/lib/Serialization/GlobalModuleIndex.cpp @@ -857,22 +857,21 @@ GlobalModuleIndex::writeIndex(FileManager &FileMgr, // Coordinate building the global index file with other processes that might // try to do the same. - llvm::LockFileManager Locked(IndexPath); - switch (Locked) { - case llvm::LockFileManager::LFS_Error: + llvm::LockFileManager Lock(IndexPath); + bool Owned; + if (llvm::Error Err = Lock.tryLock().moveInto(Owned)) { + llvm::consumeError(std::move(Err)); return llvm::createStringError(std::errc::io_error, "LFS error"); - - case llvm::LockFileManager::LFS_Owned: - // We're responsible for building the index ourselves. Do so below. - break; - - case llvm::LockFileManager::LFS_Shared: + } + if (!Owned) { // Someone else is responsible for building the index. We don't care // when they finish, so we're done. return llvm::createStringError(std::errc::device_or_resource_busy, "someone else is building the index"); } + // We're responsible for building the index ourselves. + // The module index builder. GlobalModuleIndexBuilder Builder(FileMgr, PCHContainerRdr); diff --git a/clang/lib/Serialization/ModuleCache.cpp b/clang/lib/Serialization/ModuleCache.cpp new file mode 100644 index 0000000000000..955e5f322bcc3 --- /dev/null +++ b/clang/lib/Serialization/ModuleCache.cpp @@ -0,0 +1,44 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/Serialization/ModuleCache.h" + +#include "clang/Serialization/InMemoryModuleCache.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/LockFileManager.h" +#include "llvm/Support/Path.h" + +using namespace clang; + +namespace { +class CrossProcessModuleCache : public ModuleCache { + InMemoryModuleCache InMemory; + +public: + void prepareForGetLock(StringRef ModuleFilename) override { + // FIXME: Do this in LockFileManager and only if the directory doesn't + // exist. + StringRef Dir = llvm::sys::path::parent_path(ModuleFilename); + llvm::sys::fs::create_directories(Dir); + } + + std::unique_ptr + getLock(StringRef ModuleFilename) override { + return std::make_unique(ModuleFilename); + } + + InMemoryModuleCache &getInMemoryModuleCache() override { return InMemory; } + const InMemoryModuleCache &getInMemoryModuleCache() const override { + return InMemory; + } +}; +} // namespace + +IntrusiveRefCntPtr clang::createCrossProcessModuleCache() { + return llvm::makeIntrusiveRefCnt(); +} diff --git a/clang/lib/Serialization/ModuleManager.cpp b/clang/lib/Serialization/ModuleManager.cpp index 6c852130ad93b..672dfbae7c0f5 100644 --- a/clang/lib/Serialization/ModuleManager.cpp +++ b/clang/lib/Serialization/ModuleManager.cpp @@ -18,6 +18,7 @@ #include "clang/Lex/ModuleMap.h" #include "clang/Serialization/GlobalModuleIndex.h" #include "clang/Serialization/InMemoryModuleCache.h" +#include "clang/Serialization/ModuleCache.h" #include "clang/Serialization/ModuleFile.h" #include "clang/Serialization/PCHContainerOperations.h" #include "llvm/ADT/STLExtras.h" @@ -192,17 +193,20 @@ ModuleManager::addModule(StringRef FileName, ModuleKind Type, // Load the contents of the module if (std::unique_ptr Buffer = lookupBuffer(FileName)) { // The buffer was already provided for us. - NewModule->Buffer = &ModuleCache->addBuiltPCM(FileName, std::move(Buffer)); + NewModule->Buffer = &getModuleCache().getInMemoryModuleCache().addBuiltPCM( + FileName, std::move(Buffer)); // Since the cached buffer is reused, it is safe to close the file // descriptor that was opened while stat()ing the PCM in // lookupModuleFile() above, it won't be needed any longer. Entry->closeFile(); } else if (llvm::MemoryBuffer *Buffer = - getModuleCache().lookupPCM(FileName)) { + getModuleCache().getInMemoryModuleCache().lookupPCM( + FileName)) { NewModule->Buffer = Buffer; // As above, the file descriptor is no longer needed. Entry->closeFile(); - } else if (getModuleCache().shouldBuildPCM(FileName)) { + } else if (getModuleCache().getInMemoryModuleCache().shouldBuildPCM( + FileName)) { // Report that the module is out of date, since we tried (and failed) to // import it earlier. Entry->closeFile(); @@ -223,7 +227,8 @@ ModuleManager::addModule(StringRef FileName, ModuleKind Type, return Missing; } - NewModule->Buffer = &getModuleCache().addPCM(FileName, std::move(*Buf)); + NewModule->Buffer = &getModuleCache().getInMemoryModuleCache().addPCM( + FileName, std::move(*Buf)); } // Initialize the stream. @@ -334,12 +339,11 @@ void ModuleManager::moduleFileAccepted(ModuleFile *MF) { ModulesInCommonWithGlobalIndex.push_back(MF); } -ModuleManager::ModuleManager(FileManager &FileMgr, - InMemoryModuleCache &ModuleCache, +ModuleManager::ModuleManager(FileManager &FileMgr, ModuleCache &ModCache, const PCHContainerReader &PCHContainerRdr, const HeaderSearch &HeaderSearchInfo) - : FileMgr(FileMgr), ModuleCache(&ModuleCache), - PCHContainerRdr(PCHContainerRdr), HeaderSearchInfo(HeaderSearchInfo) {} + : FileMgr(FileMgr), ModCache(&ModCache), PCHContainerRdr(PCHContainerRdr), + HeaderSearchInfo(HeaderSearchInfo) {} void ModuleManager::visit(llvm::function_ref Visitor, llvm::SmallPtrSetImpl *ModuleFilesHit) { @@ -470,7 +474,8 @@ bool ModuleManager::lookupModuleFile(StringRef FileName, off_t ExpectedSize, // If the file is known to the module cache but not in the filesystem, it is // a memory buffer. Create a virtual file for it. if (!File) { - if (auto *KnownBuffer = getModuleCache().lookupPCM(FileName)) { + if (auto *KnownBuffer = + getModuleCache().getInMemoryModuleCache().lookupPCM(FileName)) { File = FileMgr.getVirtualFileRef(FileName, KnownBuffer->getBufferSize(), 0); } diff --git a/clang/lib/Tooling/DependencyScanning/CMakeLists.txt b/clang/lib/Tooling/DependencyScanning/CMakeLists.txt index 58fe01bfe0828..917b3f5810e96 100644 --- a/clang/lib/Tooling/DependencyScanning/CMakeLists.txt +++ b/clang/lib/Tooling/DependencyScanning/CMakeLists.txt @@ -14,6 +14,7 @@ add_clang_library(clangDependencyScanning DependencyScanningWorker.cpp DependencyScanningTool.cpp IncludeTreeActionController.cpp + InProcessModuleCache.cpp ModuleDepCollector.cpp ScanAndUpdateArgs.cpp @@ -29,4 +30,5 @@ add_clang_library(clangDependencyScanning clangLex clangSerialization clangTooling + ${LLVM_PTHREAD_LIB} ) diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index c355b416d3b9d..9f456107a1e8d 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -25,6 +25,7 @@ #include "clang/Lex/PreprocessorOptions.h" #include "clang/Serialization/ObjectFilePCHContainerReader.h" #include "clang/Tooling/DependencyScanning/DependencyScanningService.h" +#include "clang/Tooling/DependencyScanning/InProcessModuleCache.h" #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h" #include "clang/Tooling/DependencyScanning/ScanAndUpdateArgs.h" #include "clang/Tooling/Tooling.h" @@ -446,10 +447,12 @@ class DependencyScanningAction : public tooling::ToolAction { Scanned = true; // Create a compiler instance to handle the actual work. - ScanInstanceStorage.emplace(std::move(PCHContainerOps)); + auto ModCache = makeInProcessModuleCache(Service.getModuleCacheMutexes()); + ScanInstanceStorage.emplace(std::move(PCHContainerOps), ModCache.get()); CompilerInstance &ScanInstance = *ScanInstanceStorage; ScanInstance.setInvocation(std::move(Invocation)); ScanInstance.getInvocation().getCASOpts() = CASOpts; + ScanInstance.setBuildingModule(false); // Create the compiler's actual diagnostics engine. if (!DiagGenerationAsCompilation) diff --git a/clang/lib/Tooling/DependencyScanning/InProcessModuleCache.cpp b/clang/lib/Tooling/DependencyScanning/InProcessModuleCache.cpp new file mode 100644 index 0000000000000..71ce4d098932b --- /dev/null +++ b/clang/lib/Tooling/DependencyScanning/InProcessModuleCache.cpp @@ -0,0 +1,87 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/Tooling/DependencyScanning/InProcessModuleCache.h" + +#include "clang/Serialization/InMemoryModuleCache.h" +#include "llvm/Support/AdvisoryLock.h" + +#include + +using namespace clang; +using namespace tooling; +using namespace dependencies; + +namespace { +class ReaderWriterLock : public llvm::AdvisoryLock { + // TODO: Consider using std::atomic::{wait,notify_all} when we move to C++20. + std::unique_lock OwningLock; + +public: + ReaderWriterLock(std::shared_mutex &Mutex) + : OwningLock(Mutex, std::defer_lock) {} + + Expected tryLock() override { return OwningLock.try_lock(); } + + llvm::WaitForUnlockResult + waitForUnlockFor(std::chrono::seconds MaxSeconds) override { + assert(!OwningLock); + // We do not respect the timeout here. It's very generous for implicit + // modules, so we'd typically only reach it if the owner crashed (but so did + // we, since we run in the same process), or encountered deadlock. + (void)MaxSeconds; + std::shared_lock Lock(*OwningLock.mutex()); + return llvm::WaitForUnlockResult::Success; + } + + std::error_code unsafeMaybeUnlock() override { + // Unlocking the mutex here would trigger UB and we don't expect this to be + // actually called when compiling scanning modules due to the no-timeout + // guarantee above. + return {}; + } + + ~ReaderWriterLock() override = default; +}; + +class InProcessModuleCache : public ModuleCache { + ModuleCacheMutexes &Mutexes; + + // TODO: If we changed the InMemoryModuleCache API and relied on strict + // context hash, we could probably create more efficient thread-safe + // implementation of the InMemoryModuleCache such that it doesn't need to be + // recreated for each translation unit. + InMemoryModuleCache InMemory; + +public: + InProcessModuleCache(ModuleCacheMutexes &Mutexes) : Mutexes(Mutexes) {} + + void prepareForGetLock(StringRef Filename) override {} + + std::unique_ptr getLock(StringRef Filename) override { + auto &Mtx = [&]() -> std::shared_mutex & { + std::lock_guard Lock(Mutexes.Mutex); + auto &Mutex = Mutexes.Map[Filename]; + if (!Mutex) + Mutex = std::make_unique(); + return *Mutex; + }(); + return std::make_unique(Mtx); + } + + InMemoryModuleCache &getInMemoryModuleCache() override { return InMemory; } + const InMemoryModuleCache &getInMemoryModuleCache() const override { + return InMemory; + } +}; +} // namespace + +IntrusiveRefCntPtr +dependencies::makeInProcessModuleCache(ModuleCacheMutexes &Mutexes) { + return llvm::makeIntrusiveRefCnt(Mutexes); +} diff --git a/clang/unittests/Frontend/FrontendActionTest.cpp b/clang/unittests/Frontend/FrontendActionTest.cpp index 818e8cef27e51..a233ede311bc6 100644 --- a/clang/unittests/Frontend/FrontendActionTest.cpp +++ b/clang/unittests/Frontend/FrontendActionTest.cpp @@ -18,6 +18,7 @@ #include "clang/Lex/PreprocessorOptions.h" #include "clang/Sema/Sema.h" #include "clang/Serialization/InMemoryModuleCache.h" +#include "clang/Serialization/ModuleCache.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/ToolOutputFile.h" #include "llvm/TargetParser/Triple.h" @@ -285,10 +286,12 @@ TEST(GeneratePCHFrontendAction, CacheGeneratedPCH) { // Check whether the PCH was cached. if (ShouldCache) EXPECT_EQ(InMemoryModuleCache::Final, - Compiler.getModuleCache().getPCMState(PCHFilename)); + Compiler.getModuleCache().getInMemoryModuleCache().getPCMState( + PCHFilename)); else EXPECT_EQ(InMemoryModuleCache::Unknown, - Compiler.getModuleCache().getPCMState(PCHFilename)); + Compiler.getModuleCache().getInMemoryModuleCache().getPCMState( + PCHFilename)); } } diff --git a/clang/unittests/Lex/HeaderSearchTest.cpp b/clang/unittests/Lex/HeaderSearchTest.cpp index 4d07150c04e8d..89d096824e600 100644 --- a/clang/unittests/Lex/HeaderSearchTest.cpp +++ b/clang/unittests/Lex/HeaderSearchTest.cpp @@ -16,7 +16,6 @@ #include "clang/Basic/TargetInfo.h" #include "clang/Basic/TargetOptions.h" #include "clang/Lex/HeaderSearchOptions.h" -#include "clang/Serialization/InMemoryModuleCache.h" #include "llvm/Support/MemoryBuffer.h" #include "gtest/gtest.h" #include diff --git a/llvm/include/llvm/Support/AdvisoryLock.h b/llvm/include/llvm/Support/AdvisoryLock.h new file mode 100644 index 0000000000000..d1c3ccc187e64 --- /dev/null +++ b/llvm/include/llvm/Support/AdvisoryLock.h @@ -0,0 +1,58 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_ADVISORYLOCK_H +#define LLVM_SUPPORT_ADVISORYLOCK_H + +#include "llvm/Support/Error.h" + +#include + +namespace llvm { +/// Describes the result of waiting for the owner to release the lock. +enum class WaitForUnlockResult { + /// The lock was released successfully. + Success, + /// Owner died while holding the lock. + OwnerDied, + /// Reached timeout while waiting for the owner to release the lock. + Timeout, +}; + +/// A synchronization primitive with weak mutual exclusion guarantees. +/// Implementations of this interface may allow multiple threads/processes to +/// acquire the ownership of the lock simultaneously. +/// Typically, threads/processes waiting for the lock to be unlocked will +/// validate that the computation was performed by the expected thread/process +/// and re-run the computation if not. +class AdvisoryLock { +public: + /// Tries to acquire ownership of the lock without blocking. + /// + /// \returns true if ownership of the lock was acquired successfully, false if + /// the lock is already owned by someone else, or \c Error in case of an + /// unexpected failure. + virtual Expected tryLock() = 0; + + /// For a lock owned by someone else, wait until it is unlocked. + /// + /// \param MaxSeconds the maximum total wait time in seconds. + virtual WaitForUnlockResult + waitForUnlockFor(std::chrono::seconds MaxSeconds) = 0; + + /// For a lock owned by someone else, unlock it. A permitted side-effect is + /// that another thread/process may acquire ownership of the lock before the + /// existing owner unlocks it. This is an unsafe operation. + virtual std::error_code unsafeMaybeUnlock() = 0; + + /// Unlocks the lock if its ownership was previously acquired by \c tryLock(). + virtual ~AdvisoryLock() = default; +}; +} // end namespace llvm + +#endif diff --git a/llvm/include/llvm/Support/LockFileManager.h b/llvm/include/llvm/Support/LockFileManager.h index 92c7ceed6a929..a126fa3d6b529 100644 --- a/llvm/include/llvm/Support/LockFileManager.h +++ b/llvm/include/llvm/Support/LockFileManager.h @@ -9,92 +9,62 @@ #define LLVM_SUPPORT_LOCKFILEMANAGER_H #include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/AdvisoryLock.h" #include -#include -#include // for std::pair +#include +#include namespace llvm { -class StringRef; - -/// Class that manages the creation of a lock file to aid -/// implicit coordination between different processes. +/// Class that manages the creation of a lock file to aid implicit coordination +/// between different processes. /// -/// The implicit coordination works by creating a ".lock" file alongside -/// the file that we're coordinating for, using the atomicity of the file -/// system to ensure that only a single process can create that ".lock" file. -/// When the lock file is removed, the owning process has finished the -/// operation. -class LockFileManager { -public: - /// Describes the state of a lock file. - enum LockFileState { - /// The lock file has been created and is owned by this instance - /// of the object. - LFS_Owned, - /// The lock file already exists and is owned by some other - /// instance. - LFS_Shared, - /// An error occurred while trying to create or find the lock - /// file. - LFS_Error - }; - - /// Describes the result of waiting for the owner to release the lock. - enum WaitForUnlockResult { - /// The lock was released successfully. - Res_Success, - /// Owner died while holding the lock. - Res_OwnerDied, - /// Reached timeout while waiting for the owner to release the lock. - Res_Timeout - }; - -private: +/// The implicit coordination works by creating a ".lock" file, using the +/// atomicity of the file system to ensure that only a single process can create +/// that ".lock" file. When the lock file is removed, the owning process has +/// finished the operation. +class LockFileManager : public AdvisoryLock { SmallString<128> FileName; SmallString<128> LockFileName; SmallString<128> UniqueLockFileName; - std::optional> Owner; - std::error_code ErrorCode; - std::string ErrorDiagMsg; + struct OwnerUnknown {}; + struct OwnedByUs {}; + struct OwnedByAnother { + std::string OwnerHostName; + int OwnerPID; + }; + std::variant Owner; LockFileManager(const LockFileManager &) = delete; LockFileManager &operator=(const LockFileManager &) = delete; - static std::optional> - readLockFile(StringRef LockFileName); + static std::optional readLockFile(StringRef LockFileName); static bool processStillExecuting(StringRef Hostname, int PID); public: - + /// Does not try to acquire the lock. LockFileManager(StringRef FileName); - ~LockFileManager(); - - /// Determine the state of the lock file. - LockFileState getState() const; - operator LockFileState() const { return getState(); } + /// Tries to acquire the lock without blocking. + /// \returns true if the lock was successfully acquired, false if the lock is + /// already held by someone else, or \c Error in case of unexpected failure. + Expected tryLock() override; /// For a shared lock, wait until the owner releases the lock. - /// Total timeout for the file to appear is ~1.5 minutes. + /// /// \param MaxSeconds the maximum total wait time in seconds. - WaitForUnlockResult waitForUnlock(const unsigned MaxSeconds = 90); + WaitForUnlockResult + waitForUnlockFor(std::chrono::seconds MaxSeconds) override; /// Remove the lock file. This may delete a different lock file than /// the one previously read if there is a race. - std::error_code unsafeRemoveLockFile(); + std::error_code unsafeMaybeUnlock() override; - /// Get error message, or "" if there is no error. - std::string getErrorMessage() const; - - /// Set error and error message - void setError(const std::error_code &EC, StringRef ErrorMsg = "") { - ErrorCode = EC; - ErrorDiagMsg = ErrorMsg.str(); - } + /// Unlocks the lock if previously acquired by \c tryLock(). + ~LockFileManager() override; }; - } // end namespace llvm #endif // LLVM_SUPPORT_LOCKFILEMANAGER_H diff --git a/llvm/lib/Support/LockFileManager.cpp b/llvm/lib/Support/LockFileManager.cpp index ea040ccf22b99..ac2a043fffe4f 100644 --- a/llvm/lib/Support/LockFileManager.cpp +++ b/llvm/lib/Support/LockFileManager.cpp @@ -51,7 +51,7 @@ using namespace llvm; /// \param LockFileName The name of the lock file to read. /// /// \returns The process ID of the process that owns this lock file -std::optional> +std::optional LockFileManager::readLockFile(StringRef LockFileName) { // Read the owning host and PID out of the lock file. If it appears that the // owning process is dead, the lock file is invalid. @@ -69,8 +69,10 @@ LockFileManager::readLockFile(StringRef LockFileName) { PIDStr = PIDStr.substr(PIDStr.find_first_not_of(' ')); int PID; if (!PIDStr.getAsInteger(10, PID)) { - auto Owner = std::make_pair(std::string(Hostname), PID); - if (processStillExecuting(Owner.first, Owner.second)) + OwnedByAnother Owner; + Owner.OwnerHostName = Hostname; + Owner.OwnerPID = PID; + if (processStillExecuting(Owner.OwnerHostName, Owner.OwnerPID)) return Owner; } @@ -158,86 +160,77 @@ class RemoveUniqueLockFileOnSignal { } // end anonymous namespace LockFileManager::LockFileManager(StringRef FileName) -{ - this->FileName = FileName; - if (std::error_code EC = sys::fs::make_absolute(this->FileName)) { - std::string S("failed to obtain absolute path for "); - S.append(std::string(this->FileName)); - setError(EC, S); - return; - } - LockFileName = this->FileName; + : FileName(FileName), Owner(OwnerUnknown{}) {} + +Expected LockFileManager::tryLock() { + assert(std::holds_alternative(Owner) && + "lock has already been attempted"); + + SmallString<128> AbsoluteFileName(FileName); + if (std::error_code EC = sys::fs::make_absolute(AbsoluteFileName)) + return createStringError(EC, "failed to obtain absolute path for " + + AbsoluteFileName); + LockFileName = AbsoluteFileName; LockFileName += ".lock"; // If the lock file already exists, don't bother to try to create our own // lock file; it won't work anyway. Just figure out who owns this lock file. - if ((Owner = readLockFile(LockFileName))) - return; + if (auto LockFileOwner = readLockFile(LockFileName)) { + Owner = std::move(*LockFileOwner); + return false; + } // Create a lock file that is unique to this instance. UniqueLockFileName = LockFileName; UniqueLockFileName += "-%%%%%%%%"; int UniqueLockFileID; if (std::error_code EC = sys::fs::createUniqueFile( - UniqueLockFileName, UniqueLockFileID, UniqueLockFileName)) { - std::string S("failed to create unique file "); - S.append(std::string(UniqueLockFileName)); - setError(EC, S); - return; - } + UniqueLockFileName, UniqueLockFileID, UniqueLockFileName)) + return createStringError(EC, "failed to create unique file " + + UniqueLockFileName); + + // Clean up the unique file on signal or scope exit. + RemoveUniqueLockFileOnSignal RemoveUniqueFile(UniqueLockFileName); // Write our process ID to our unique lock file. { SmallString<256> HostID; - if (auto EC = getHostID(HostID)) { - setError(EC, "failed to get host id"); - return; - } + if (auto EC = getHostID(HostID)) + return createStringError(EC, "failed to get host id"); raw_fd_ostream Out(UniqueLockFileID, /*shouldClose=*/true); Out << HostID << ' ' << sys::Process::getProcessId(); Out.close(); if (Out.has_error()) { - // We failed to write out PID, so report the error, remove the - // unique lock file, and fail. - std::string S("failed to write to "); - S.append(std::string(UniqueLockFileName)); - setError(Out.error(), S); - sys::fs::remove(UniqueLockFileName); + // We failed to write out PID, so report the error and fail. + Error Err = createStringError(Out.error(), + "failed to write to " + UniqueLockFileName); // Don't call report_fatal_error. Out.clear_error(); - return; + return std::move(Err); } } - // Clean up the unique file on signal, which also releases the lock if it is - // held since the .lock symlink will point to a nonexistent file. - RemoveUniqueLockFileOnSignal RemoveUniqueFile(UniqueLockFileName); - while (true) { // Create a link from the lock file name. If this succeeds, we're done. std::error_code EC = sys::fs::create_link(UniqueLockFileName, LockFileName); if (!EC) { RemoveUniqueFile.lockAcquired(); - return; + Owner = OwnedByUs{}; + return true; } - if (EC != errc::file_exists) { - std::string S("failed to create link "); - raw_string_ostream OSS(S); - OSS << LockFileName.str() << " to " << UniqueLockFileName.str(); - setError(EC, S); - return; - } + if (EC != errc::file_exists) + return createStringError(EC, "failed to create link " + LockFileName + + " to " + UniqueLockFileName); // Someone else managed to create the lock file first. Read the process ID // from the lock file. - if ((Owner = readLockFile(LockFileName))) { - // Wipe out our unique lock file (it's useless now) - sys::fs::remove(UniqueLockFileName); - return; + if (auto LockFileOwner = readLockFile(LockFileName)) { + Owner = std::move(*LockFileOwner); + return false; } if (!sys::fs::exists(LockFileName)) { @@ -248,39 +241,14 @@ LockFileManager::LockFileManager(StringRef FileName) // There is a lock file that nobody owns; try to clean it up and get // ownership. - if ((EC = sys::fs::remove(LockFileName))) { - std::string S("failed to remove lockfile "); - S.append(std::string(UniqueLockFileName)); - setError(EC, S); - return; - } + if ((EC = sys::fs::remove(LockFileName))) + return createStringError(EC, "failed to remove lockfile " + + UniqueLockFileName); } } -LockFileManager::LockFileState LockFileManager::getState() const { - if (Owner) - return LFS_Shared; - - if (ErrorCode) - return LFS_Error; - - return LFS_Owned; -} - -std::string LockFileManager::getErrorMessage() const { - if (ErrorCode) { - std::string Str(ErrorDiagMsg); - std::string ErrCodeMsg = ErrorCode.message(); - raw_string_ostream OSS(Str); - if (!ErrCodeMsg.empty()) - OSS << ": " << ErrCodeMsg; - return Str; - } - return ""; -} - LockFileManager::~LockFileManager() { - if (getState() != LFS_Owned) + if (!std::holds_alternative(Owner)) return; // Since we own the lock, remove the lock file and our own unique lock file. @@ -291,38 +259,36 @@ LockFileManager::~LockFileManager() { sys::DontRemoveFileOnSignal(UniqueLockFileName); } -LockFileManager::WaitForUnlockResult -LockFileManager::waitForUnlock(const unsigned MaxSeconds) { - if (getState() != LFS_Shared) - return Res_Success; +WaitForUnlockResult +LockFileManager::waitForUnlockFor(std::chrono::seconds MaxSeconds) { + auto *LockFileOwner = std::get_if(&Owner); + assert(LockFileOwner && + "waiting for lock to be unlocked without knowing the owner"); // Since we don't yet have an event-based method to wait for the lock file, // use randomized exponential backoff, similar to Ethernet collision // algorithm. This improves performance on machines with high core counts // when the file lock is heavily contended by multiple clang processes using namespace std::chrono_literals; - ExponentialBackoff Backoff(std::chrono::seconds(MaxSeconds), 10ms, 500ms); + ExponentialBackoff Backoff(MaxSeconds, 10ms, 500ms); // Wait first as this is only called when the lock is known to be held. while (Backoff.waitForNextAttempt()) { // FIXME: implement event-based waiting if (sys::fs::access(LockFileName.c_str(), sys::fs::AccessMode::Exist) == - errc::no_such_file_or_directory) { - // If the original file wasn't created, somone thought the lock was dead. - if (!sys::fs::exists(FileName)) - return Res_OwnerDied; - return Res_Success; - } + errc::no_such_file_or_directory) + return WaitForUnlockResult::Success; // If the process owning the lock died without cleaning up, just bail out. - if (!processStillExecuting((*Owner).first, (*Owner).second)) - return Res_OwnerDied; + if (!processStillExecuting(LockFileOwner->OwnerHostName, + LockFileOwner->OwnerPID)) + return WaitForUnlockResult::OwnerDied; } // Give up. - return Res_Timeout; + return WaitForUnlockResult::Timeout; } -std::error_code LockFileManager::unsafeRemoveLockFile() { +std::error_code LockFileManager::unsafeMaybeUnlock() { return sys::fs::remove(LockFileName); } diff --git a/llvm/lib/Support/VirtualOutputBackends.cpp b/llvm/lib/Support/VirtualOutputBackends.cpp index b60c0e934af03..cff1de2072543 100644 --- a/llvm/lib/Support/VirtualOutputBackends.cpp +++ b/llvm/lib/Support/VirtualOutputBackends.cpp @@ -460,16 +460,16 @@ Error OnDiskOutputFile::keep() { while (1) { // Attempt to lock the output file. // Only one process is allowed to append to this file at a time. - llvm::LockFileManager Locked(OutputPath); - switch (Locked) { - case llvm::LockFileManager::LFS_Error: { + llvm::LockFileManager Lock(OutputPath); + bool Owned; + if (Error Err = Lock.tryLock().moveInto(Owned)) { // If we error acquiring a lock, we cannot ensure appends // to the trace file are atomic - cannot ensure output correctness. - Locked.unsafeRemoveLockFile(); + Lock.unsafeMaybeUnlock(); return convertToOutputError( OutputPath, std::make_error_code(std::errc::no_lock_available)); } - case llvm::LockFileManager::LFS_Owned: { + if (Owned) { // Lock acquired, perform the write and release the lock. std::error_code EC; llvm::raw_fd_ostream Out(OutputPath, EC, llvm::sys::fs::OF_Append); @@ -477,34 +477,31 @@ Error OnDiskOutputFile::keep() { return convertToOutputError(OutputPath, EC); Out << (*Content)->getBuffer(); Out.close(); - Locked.unsafeRemoveLockFile(); + Lock.unsafeMaybeUnlock(); if (Out.has_error()) return convertToOutputError(OutputPath, Out.error()); // Remove temp file and done. (void)sys::fs::remove(*TempPath); return Error::success(); } - case llvm::LockFileManager::LFS_Shared: { - // Someone else owns the lock on this file, wait. - switch (Locked.waitForUnlock(256)) { - case llvm::LockFileManager::Res_Success: - LLVM_FALLTHROUGH; - case llvm::LockFileManager::Res_OwnerDied: { - continue; // try again to get the lock. - } - case llvm::LockFileManager::Res_Timeout: { - // We could error on timeout to avoid potentially hanging forever, but - // it may be more likely that an interrupted process failed to clear - // the lock, causing other waiting processes to time-out. Let's clear - // the lock and try again right away. If we do start seeing compiler - // hangs in this location, we will need to re-consider. - Locked.unsafeRemoveLockFile(); - continue; - } - } - break; + // Someone else owns the lock on this file, wait. + switch (Lock.waitForUnlockFor(std::chrono::seconds(256))) { + case WaitForUnlockResult::Success: + LLVM_FALLTHROUGH; + case WaitForUnlockResult::OwnerDied: { + continue; // try again to get the lock. } + case WaitForUnlockResult::Timeout: { + // We could error on timeout to avoid potentially hanging forever, but + // it may be more likely that an interrupted process failed to clear + // the lock, causing other waiting processes to time-out. Let's clear + // the lock and try again right away. If we do start seeing compiler + // hangs in this location, we will need to re-consider. + Lock.unsafeMaybeUnlock(); + continue; } + } + break; } } diff --git a/llvm/unittests/Support/LockFileManagerTest.cpp b/llvm/unittests/Support/LockFileManagerTest.cpp index 552053d46e843..627b2daef650c 100644 --- a/llvm/unittests/Support/LockFileManagerTest.cpp +++ b/llvm/unittests/Support/LockFileManagerTest.cpp @@ -9,6 +9,7 @@ #include "llvm/Support/LockFileManager.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" +#include "llvm/Testing/Support/Error.h" #include "llvm/Testing/Support/SupportHelpers.h" #include "gtest/gtest.h" #include @@ -27,12 +28,12 @@ TEST(LockFileManagerTest, Basic) { { // The lock file should not exist, so we should successfully acquire it. LockFileManager Locked1(LockedFile); - EXPECT_EQ(LockFileManager::LFS_Owned, Locked1.getState()); + EXPECT_THAT_EXPECTED(Locked1.tryLock(), HasValue(true)); // Attempting to reacquire the lock should fail. Waiting on it would cause // deadlock, so don't try that. LockFileManager Locked2(LockedFile); - EXPECT_NE(LockFileManager::LFS_Owned, Locked2.getState()); + EXPECT_THAT_EXPECTED(Locked2.tryLock(), HasValue(false)); } // Now that the lock is out of scope, the file should be gone. @@ -68,7 +69,7 @@ TEST(LockFileManagerTest, LinkLockExists) { // The lock file doesn't point to a real file, so we should successfully // acquire it. LockFileManager Locked(LockedFile); - EXPECT_EQ(LockFileManager::LFS_Owned, Locked.getState()); + EXPECT_THAT_EXPECTED(Locked.tryLock(), HasValue(true)); } // Now that the lock is out of scope, the file should be gone. @@ -93,7 +94,7 @@ TEST(LockFileManagerTest, RelativePath) { { // The lock file should not exist, so we should successfully acquire it. LockFileManager Locked(LockedFile); - EXPECT_EQ(LockFileManager::LFS_Owned, Locked.getState()); + EXPECT_THAT_EXPECTED(Locked.tryLock(), HasValue(true)); EXPECT_TRUE(sys::fs::exists(FileLock.str())); } From 141d8f497e0218c85792c4b7391c8d03d3163aef Mon Sep 17 00:00:00 2001 From: Jan Svoboda Date: Wed, 5 Mar 2025 17:33:52 -0800 Subject: [PATCH 2/2] Merge pull request #10162 from swiftlang/jan_svoboda/stable-scanning-service MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🍒 [clang][deps] Propagate the entire service Shared state between dependency scanning workers is managed by the dependency scanning service. Right now, the members are individually threaded through the worker, action, and collector. This makes any change to the service and its members a very laborious process. Moreover, this situation causes frequent merge conflicts in our downstream repo where the service does have some extra members that need to be passed around. To ease the maintenance burden, this PR starts passing a reference to the entire service. (cherry picked from commit d2e66625bcdc09953c007cf1e9f80d38a18719f3) (cherry picked from commit de6bfa2f4bd7aacf75fe010f195f98002406c84b) --- .../DependencyScanningService.h | 2 +- .../DependencyScanningTool.h | 3 + .../DependencyScanningWorker.h | 15 +++-- .../DependencyScanning/ModuleDepCollector.h | 16 ++---- .../DependencyScanningService.cpp | 7 +++ .../DependencyScanningWorker.cpp | 57 +++++++------------ .../DependencyScanning/ModuleDepCollector.cpp | 40 ++++++------- 7 files changed, 65 insertions(+), 75 deletions(-) diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h index 331509b7c435b..31781989bb806 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h @@ -133,7 +133,7 @@ class DependencyScanningService { std::shared_ptr CAS; std::shared_ptr Cache; /// Whether to optimize the modules' command-line arguments. - const ScanningOptimizations OptimizeArgs; + ScanningOptimizations OptimizeArgs; /// Whether to set up command-lines to load PCM files eagerly. const bool EagerLoadModules; /// Whether to trace VFS accesses. diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h index c747bcc390764..b776a7a2c6158 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h @@ -96,6 +96,9 @@ struct P1689Rule { class DependencyScanningTool { public: /// Construct a dependency scanning tool. + /// + /// @param Service The parent service. Must outlive the tool. + /// @param FS The filesystem for the tool to use. Defaults to the physical FS. DependencyScanningTool(DependencyScanningService &Service, llvm::IntrusiveRefCntPtr FS = llvm::vfs::createPhysicalFileSystem()); diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h index 52c89e9626925..107391660d306 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h @@ -115,6 +115,10 @@ class DependencyActionController { /// using the regular processing run. class DependencyScanningWorker { public: + /// Construct a dependency scanning worker. + /// + /// @param Service The parent service. Must outlive the worker. + /// @param FS The filesystem for the worker to use. DependencyScanningWorker(DependencyScanningService &Service, llvm::IntrusiveRefCntPtr FS); @@ -149,7 +153,7 @@ class DependencyScanningWorker { DependencyActionController &Controller, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, bool DiagGenerationAsCompilation); - ScanningOutputFormat getScanningFormat() const { return Format; } + ScanningOutputFormat getScanningFormat() const { return Service.getFormat(); } CachingOnDiskFileSystemPtr getCASFS() { return CacheFS; } const CASOptions &getCASOpts() const { return CASOpts; } @@ -160,11 +164,11 @@ class DependencyScanningWorker { /// each invocation. llvm::IntrusiveRefCntPtr getOrCreateFileManager() const; - bool shouldEagerLoadModules() const { return EagerLoadModules; } - llvm::vfs::FileSystem &getVFS() const { return *BaseFS; } private: + /// The parent dependency scanning service. + DependencyScanningService &Service; std::shared_ptr PCHContainerOps; /// The file system to be used during the scan. /// This is either \c FS passed in the constructor (when performing canonical @@ -174,11 +178,6 @@ class DependencyScanningWorker { /// dependency-directives-extracting) filesystem overlaid on top of \c FS /// (passed in the constructor). llvm::IntrusiveRefCntPtr DepFS; - ScanningOutputFormat Format; - /// Whether to optimize the modules' command-line arguments. - ScanningOptimizations OptimizeArgs; - /// Whether to set up command-lines to load PCM files eagerly. - bool EagerLoadModules; /// The caching file system. CachingOnDiskFileSystemPtr CacheFS; diff --git a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h index dcd40193b0866..5207fb3f6f6cb 100644 --- a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h +++ b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h @@ -237,13 +237,12 @@ class ModuleDepCollectorPP final : public PPCallbacks { /// \c ModuleDepCollectorPP to the preprocessor. class ModuleDepCollector final : public DependencyCollector { public: - ModuleDepCollector(std::unique_ptr Opts, + ModuleDepCollector(DependencyScanningService &Service, + std::unique_ptr Opts, CompilerInstance &ScanInstance, DependencyConsumer &C, DependencyActionController &Controller, CompilerInvocation OriginalCI, - PrebuiltModuleVFSMapT PrebuiltModuleVFSMap, - ScanningOptimizations OptimizeArgs, bool EagerLoadModules, - bool IsStdModuleP1689Format); + PrebuiltModuleVFSMapT PrebuiltModuleVFSMap); void attachToPreprocessor(Preprocessor &PP) override; void attachToASTReader(ASTReader &R) override; @@ -255,6 +254,8 @@ class ModuleDepCollector final : public DependencyCollector { private: friend ModuleDepCollectorPP; + /// The parent dependency scanning service. + DependencyScanningService &Service; /// The compiler instance for scanning the current translation unit. CompilerInstance &ScanInstance; /// The consumer of collected dependency information. @@ -286,13 +287,6 @@ class ModuleDepCollector final : public DependencyCollector { /// a discovered modular dependency. Note that this still needs to be adjusted /// for each individual module. CowCompilerInvocation CommonInvocation; - /// Whether to optimize the modules' command-line arguments. - ScanningOptimizations OptimizeArgs; - /// Whether to set up command-lines to load PCM files eagerly. - bool EagerLoadModules; - /// If we're generating dependency output in P1689 format - /// for standard C++ modules. - bool IsStdModuleP1689Format; std::optional ProvidedStdCXXModule; std::vector RequiredStdCXXModules; diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp index 9e2509ec0e697..8d6e0fb0f9fc9 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "clang/Tooling/DependencyScanning/DependencyScanningService.h" +#include "clang/Basic/BitmaskEnum.h" #include "llvm/CAS/ActionCache.h" #include "llvm/CAS/CachingOnDiskFileSystem.h" #include "llvm/CAS/ObjectStore.h" @@ -26,4 +27,10 @@ DependencyScanningService::DependencyScanningService( SharedFS(std::move(SharedFS)) { if (!this->SharedFS) SharedCache.emplace(); + + // The FullIncludeTree output format completely subsumes header search and + // VFS optimizations due to how it works. Disable these optimizations so + // we're not doing unneeded work. + if (Format == ScanningOutputFormat::FullIncludeTree) + this->OptimizeArgs &= ~ScanningOptimizations::FullIncludeTreeIrrelevant; } diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index 9f456107a1e8d..1b26857c900d4 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -393,30 +393,22 @@ class WrapScanModuleBuildAction : public WrapperFrontendAction { class DependencyScanningAction : public tooling::ToolAction { public: DependencyScanningAction( - StringRef WorkingDirectory, DependencyConsumer &Consumer, - DependencyActionController &Controller, + DependencyScanningService &Service, StringRef WorkingDirectory, + DependencyConsumer &Consumer, DependencyActionController &Controller, llvm::IntrusiveRefCntPtr DepFS, llvm::IntrusiveRefCntPtr DepCASFS, llvm::IntrusiveRefCntPtr CacheFS, - ScanningOutputFormat Format, ScanningOptimizations OptimizeArgs, - bool EagerLoadModules, bool DisableFree, bool EmitDependencyFile, + bool DisableFree, bool EmitDependencyFile, bool DiagGenerationAsCompilation, const CASOptions &CASOpts, std::optional ModuleName = std::nullopt, raw_ostream *VerboseOS = nullptr) - : WorkingDirectory(WorkingDirectory), Consumer(Consumer), + : Service(Service), WorkingDirectory(WorkingDirectory), Consumer(Consumer), Controller(Controller), DepFS(std::move(DepFS)), DepCASFS(std::move(DepCASFS)), CacheFS(std::move(CacheFS)), - Format(Format), OptimizeArgs(OptimizeArgs), - EagerLoadModules(EagerLoadModules), DisableFree(DisableFree), + DisableFree(DisableFree), CASOpts(CASOpts), EmitDependencyFile(EmitDependencyFile), DiagGenerationAsCompilation(DiagGenerationAsCompilation), - ModuleName(ModuleName), VerboseOS(VerboseOS) { - // The FullIncludeTree output format completely subsumes header search and - // VFS optimizations due to how it works. Disable these optimizations so - // we're not doing unneeded work. - if (Format == ScanningOutputFormat::FullIncludeTree) - this->OptimizeArgs &= ~ScanningOptimizations::FullIncludeTreeIrrelevant; - } + ModuleName(ModuleName), VerboseOS(VerboseOS) {} bool runInvocation(std::shared_ptr Invocation, FileManager *DriverFileMgr, @@ -426,7 +418,7 @@ class DependencyScanningAction : public tooling::ToolAction { CompilerInvocation OriginalInvocation(*Invocation); // Restore the value of DisableFree, which may be modified by Tooling. OriginalInvocation.getFrontendOpts().DisableFree = DisableFree; - if (any(OptimizeArgs & ScanningOptimizations::Macros)) + if (any(Service.getOptimizeArgs() & ScanningOptimizations::Macros)) canonicalizeDefines(OriginalInvocation.getPreprocessorOpts()); if (Scanned) { @@ -479,7 +471,7 @@ class DependencyScanningAction : public tooling::ToolAction { ScanInstance.getFrontendOpts().ModulesShareFileManager = false; ScanInstance.getHeaderSearchOpts().ModuleFormat = "raw"; ScanInstance.getHeaderSearchOpts().ModulesIncludeVFSUsage = - any(OptimizeArgs & ScanningOptimizations::VFS); + any(Service.getOptimizeArgs() & ScanningOptimizations::VFS); // Support for virtual file system overlays. auto FS = createVFSFromCompilerInvocation( @@ -536,7 +528,7 @@ class DependencyScanningAction : public tooling::ToolAction { Opts->Targets = { deduceDepTarget(ScanInstance.getFrontendOpts().OutputFile, ScanInstance.getFrontendOpts().Inputs)}; - if (Format == ScanningOutputFormat::Make) { + if (Service.getFormat() == ScanningOutputFormat::Make) { // Only 'Make' scanning needs to force this because that mode depends on // getting the dependencies directly from \p DependencyFileGenerator. Opts->IncludeSystemHeaders = true; @@ -556,7 +548,7 @@ class DependencyScanningAction : public tooling::ToolAction { // \p DependencyScanningAction, and have the callers pass in a // “DependencyCollector factory” so the connection of collector<->consumer // is explicit in each \p DependencyScanningTool function. - switch (Format) { + switch (Service.getFormat()) { case ScanningOutputFormat::Make: case ScanningOutputFormat::Tree: ScanInstance.addDependencyCollector( @@ -577,9 +569,8 @@ class DependencyScanningAction : public tooling::ToolAction { } MDC = std::make_shared( - std::move(Opts), ScanInstance, Consumer, Controller, - OriginalInvocation, std::move(PrebuiltModuleVFSMap), OptimizeArgs, - EagerLoadModules, Format == ScanningOutputFormat::P1689); + Service, std::move(Opts), ScanInstance, Consumer, Controller, + OriginalInvocation, std::move(PrebuiltModuleVFSMap)); ScanInstance.addDependencyCollector(MDC); ScanInstance.setGenModuleActionWrapper( [&Controller = Controller](const FrontendOptions &Opts, @@ -681,16 +672,13 @@ class DependencyScanningAction : public tooling::ToolAction { return nullptr; } -private: + DependencyScanningService &Service; StringRef WorkingDirectory; DependencyConsumer &Consumer; DependencyActionController &Controller; llvm::IntrusiveRefCntPtr DepFS; llvm::IntrusiveRefCntPtr DepCASFS; llvm::IntrusiveRefCntPtr CacheFS; - ScanningOutputFormat Format; - ScanningOptimizations OptimizeArgs; - bool EagerLoadModules; bool DisableFree; const CASOptions &CASOpts; bool EmitDependencyFile = false; @@ -709,8 +697,7 @@ class DependencyScanningAction : public tooling::ToolAction { DependencyScanningWorker::DependencyScanningWorker( DependencyScanningService &Service, llvm::IntrusiveRefCntPtr FS) - : Format(Service.getFormat()), OptimizeArgs(Service.getOptimizeArgs()), - EagerLoadModules(Service.shouldEagerLoadModules()), + : Service(Service), CASOpts(Service.getCASOpts()), CAS(Service.getCAS()) { PCHContainerOps = std::make_shared(); // We need to read object files from PCH built outside the scanner. @@ -897,9 +884,8 @@ bool DependencyScanningWorker::computeDependencies( // in-process; preserve the original value, which is // always true for a driver invocation. bool DisableFree = true; - DependencyScanningAction Action(WorkingDirectory, Consumer, Controller, DepFS, + DependencyScanningAction Action(Service, WorkingDirectory, Consumer, Controller, DepFS, DepCASFS, CacheFS, - Format, OptimizeArgs, EagerLoadModules, DisableFree, /*EmitDependencyFile=*/false, /*DiagGenerationAsCompilation=*/false, getCASOpts(), @@ -973,13 +959,12 @@ void DependencyScanningWorker::computeDependenciesFromCompilerInvocation( // FIXME: EmitDependencyFile should only be set when it's for a real // compilation. - DependencyScanningAction Action( - WorkingDirectory, DepsConsumer, Controller, DepFS, DepCASFS, CacheFS, - Format, - ScanningOptimizations::Default, /*DisableFree=*/false, EagerLoadModules, - /*EmitDependencyFile=*/!DepFile.empty(), DiagGenerationAsCompilation, - getCASOpts(), - /*ModuleName=*/std::nullopt, VerboseOS); + DependencyScanningAction Action(Service, WorkingDirectory, DepsConsumer, + Controller, DepFS, DepCASFS, CacheFS, + /*DisableFree=*/false, + /*EmitDependencyFile=*/!DepFile.empty(), + DiagGenerationAsCompilation, getCASOpts(), + /*ModuleName=*/std::nullopt, VerboseOS); // Ignore result; we're just collecting dependencies. // diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp index d97ce61bd169c..a4a40822b67c4 100644 --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -279,7 +279,8 @@ ModuleDepCollector::getInvocationAdjustedForModuleBuildWithoutOutputs( // TODO: Verify this works fine when modulemap for module A is eagerly // loaded from A.pcm, and module map passed on the command line contains // definition of a submodule: "explicit module A.Private { ... }". - if (EagerLoadModules && DepModuleMapFiles.contains(*ModuleMapEntry)) + if (Service.shouldEagerLoadModules() && + DepModuleMapFiles.contains(*ModuleMapEntry)) continue; // Don't report module map file of the current module unless it also @@ -332,7 +333,7 @@ llvm::DenseSet ModuleDepCollector::collectModuleMapFiles( void ModuleDepCollector::addModuleMapFiles( CompilerInvocation &CI, ArrayRef ClangModuleDeps) const { - if (EagerLoadModules) + if (Service.shouldEagerLoadModules()) return; // Only pcm is needed for eager load. for (const ModuleID &MID : ClangModuleDeps) { @@ -354,7 +355,7 @@ void ModuleDepCollector::addModuleFiles( CI.getFrontendOpts().ModuleCacheKeys.emplace_back(PCMPath, *MD->ModuleCacheKey); - if (EagerLoadModules) + if (Service.shouldEagerLoadModules()) CI.getFrontendOpts().ModuleFiles.push_back(std::move(PCMPath)); else CI.getHeaderSearchOpts().PrebuiltModuleFiles.insert( @@ -374,7 +375,7 @@ void ModuleDepCollector::addModuleFiles( CI.getMutFrontendOpts().ModuleCacheKeys.emplace_back(PCMPath, *MD->ModuleCacheKey); - if (EagerLoadModules) + if (Service.shouldEagerLoadModules()) CI.getMutFrontendOpts().ModuleFiles.push_back(std::move(PCMPath)); else CI.getMutHeaderSearchOpts().PrebuiltModuleFiles.insert( @@ -511,8 +512,9 @@ static void checkCompileCacheKeyMatch(cas::ObjectStore &CAS, void ModuleDepCollector::associateWithContextHash( const CowCompilerInvocation &CI, ModuleDeps &Deps) { - Deps.ID.ContextHash = getModuleContextHash( - Deps, CI, EagerLoadModules, ScanInstance.getVirtualFileSystem()); + Deps.ID.ContextHash = + getModuleContextHash(Deps, CI, Service.shouldEagerLoadModules(), + ScanInstance.getVirtualFileSystem()); bool Inserted = ModuleDepsByID.insert({Deps.ID, &Deps}).second; (void)Inserted; assert(Inserted && "duplicate module mapping"); @@ -617,7 +619,7 @@ void ModuleDepCollectorPP::EndOfMainFile() { MDC.Consumer.handleDependencyOutputOpts(*MDC.Opts); - if (MDC.IsStdModuleP1689Format) + if (MDC.Service.getFormat() == ScanningOutputFormat::P1689) MDC.Consumer.handleProvidedAndRequiredStdCXXModules( MDC.ProvidedStdCXXModule, MDC.RequiredStdCXXModules); @@ -726,13 +728,15 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { CowCompilerInvocation CI = MDC.getInvocationAdjustedForModuleBuildWithoutOutputs( MD, [&](CowCompilerInvocation &BuildInvocation) { - if (any(MDC.OptimizeArgs & (ScanningOptimizations::HeaderSearch | - ScanningOptimizations::VFS))) + if (any(MDC.Service.getOptimizeArgs() & + (ScanningOptimizations::HeaderSearch | + ScanningOptimizations::VFS))) optimizeHeaderSearchOpts(BuildInvocation.getMutHeaderSearchOpts(), *MDC.ScanInstance.getASTReader(), *MF, MDC.PrebuiltModuleVFSMap, - MDC.OptimizeArgs); - if (any(MDC.OptimizeArgs & ScanningOptimizations::SystemWarnings)) + MDC.Service.getOptimizeArgs()); + if (any(MDC.Service.getOptimizeArgs() & + ScanningOptimizations::SystemWarnings)) optimizeDiagnosticOpts( BuildInvocation.getMutDiagnosticOpts(), BuildInvocation.getFrontendOpts().IsSystemModule); @@ -853,19 +857,17 @@ void ModuleDepCollectorPP::addAffectingClangModule( } ModuleDepCollector::ModuleDepCollector( + DependencyScanningService &Service, std::unique_ptr Opts, CompilerInstance &ScanInstance, DependencyConsumer &C, DependencyActionController &Controller, CompilerInvocation OriginalCI, - PrebuiltModuleVFSMapT PrebuiltModuleVFSMap, - ScanningOptimizations OptimizeArgs, bool EagerLoadModules, - bool IsStdModuleP1689Format) - : ScanInstance(ScanInstance), Consumer(C), Controller(Controller), + PrebuiltModuleVFSMapT PrebuiltModuleVFSMap) + : Service(Service), ScanInstance(ScanInstance), Consumer(C), + Controller(Controller), PrebuiltModuleVFSMap(std::move(PrebuiltModuleVFSMap)), Opts(std::move(Opts)), CommonInvocation( - makeCommonInvocationForModuleBuild(std::move(OriginalCI))), - OptimizeArgs(OptimizeArgs), EagerLoadModules(EagerLoadModules), - IsStdModuleP1689Format(IsStdModuleP1689Format) {} + makeCommonInvocationForModuleBuild(std::move(OriginalCI))) {} void ModuleDepCollector::attachToPreprocessor(Preprocessor &PP) { PP.addPPCallbacks(std::make_unique(*this)); @@ -897,7 +899,7 @@ static StringRef makeAbsoluteAndPreferred(CompilerInstance &CI, StringRef Path, } void ModuleDepCollector::addFileDep(StringRef Path) { - if (IsStdModuleP1689Format) { + if (Service.getFormat() == ScanningOutputFormat::P1689) { // Within P1689 format, we don't want all the paths to be absolute path // since it may violate the traditional make style dependencies info. FileDeps.emplace_back(Path);