mirrors/llvm-project
0
0
Fork 0
mirror of https://github.com/llvm/llvm-project.git synced 2025-04-17 18:26:37 +00:00
Code Issues Projects Releases Wiki Activity

[clang][modules] Introduce new ModuleCache interface (#131193)

Browse Source 
This PR adds new `ModuleCache` interface to Clang's implicitly-built
modules machinery. The main motivation for this change is to create a
second implementation that uses a more efficient kind of
`llvm::AdvisoryLock` during dependency scanning.

In addition to the lock abstraction, the `ModuleCache` interface also
manages the existing `InMemoryModuleCache` instance. I found that
compared to keeping these separate/independent, the code is a bit
simpler now, since these are two tightly coupled concepts. I can
envision a more efficient implementation of the `InMemoryModuleCache`
for the single-process case too, which will be much easier to implement
with the current setup.

This is not intended to be a functional change.
This commit is contained in:
Jan Svoboda 2025-03-14 11:32:39 -07:00 committed by GitHub
parent d0a0de50f7
commit c84d8e8f1c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 211 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
clang-tools-extra/clangd/ModulesBuilder.cpp
View File

@ -12,7 +12,7 @@
#include "clang/Frontend/FrontendAction.h"
#include "clang/Frontend/FrontendActions.h"
#include "clang/Serialization/ASTReader.h"
#include "clang/Serialization/InMemoryModuleCache.h"
#include "clang/Serialization/ModuleCache.h"
#include "llvm/ADT/ScopeExit.h"
#include <queue>
@ -206,9 +206,9 @@ bool IsModuleFileUpToDate(PathRef ModuleFilePath,
Preprocessor PP(std::make_shared<PreprocessorOptions>(), *Diags, LangOpts,
SourceMgr, HeaderInfo, ModuleLoader);
IntrusiveRefCntPtr<InMemoryModuleCache> ModuleCache = new InMemoryModuleCache;
IntrusiveRefCntPtr<ModuleCache> ModCache = createCrossProcessModuleCache();
PCHContainerOperations PCHOperations;
ASTReader Reader(PP, *ModuleCache, /*ASTContext=*/nullptr,
ASTReader Reader(PP, *ModCache, /*ASTContext=*/nullptr,
PCHOperations.getRawReader(), {});
// We don't need any listener here. By default it will use a validator

4
clang/include/clang/Frontend/ASTUnit.h
View File

@ -70,7 +70,7 @@ class FileManager;
class FrontendAction;
class HeaderSearch;
class InputKind;
class InMemoryModuleCache;
class ModuleCache;
class PCHContainerOperations;
class PCHContainerReader;
class Preprocessor;
@ -110,7 +110,7 @@ private:
IntrusiveRefCntPtr<DiagnosticsEngine> Diagnostics;
IntrusiveRefCntPtr<FileManager> FileMgr;
IntrusiveRefCntPtr<SourceManager> SourceMgr;
IntrusiveRefCntPtr<InMemoryModuleCache> ModuleCache;
IntrusiveRefCntPtr<ModuleCache> ModCache;
std::unique_ptr<HeaderSearch> HeaderInfo;
IntrusiveRefCntPtr<TargetInfo> Target;
std::shared_ptr<Preprocessor> PP;

13
clang/include/clang/Frontend/CompilerInstance.h
View File

@ -51,8 +51,8 @@ class DiagnosticsEngine;
class DiagnosticConsumer;
class FileManager;
class FrontendAction;
class InMemoryModuleCache;
class Module;
class ModuleCache;
class Preprocessor;
class Sema;
class SourceManager;
@ -97,7 +97,7 @@ class CompilerInstance : public ModuleLoader {
IntrusiveRefCntPtr<SourceManager> SourceMgr;
/// The cache of PCM files.
IntrusiveRefCntPtr<InMemoryModuleCache> ModuleCache;
IntrusiveRefCntPtr<ModuleCache> ModCache;
/// The preprocessor.
std::shared_ptr<Preprocessor> PP;
@ -209,7 +209,7 @@ public:
explicit CompilerInstance(
std::shared_ptr<PCHContainerOperations> PCHContainerOps =
std::make_shared<PCHContainerOperations>(),
InMemoryModuleCache *SharedModuleCache = nullptr);
ModuleCache *ModCache = nullptr);
~CompilerInstance() override;
/// @name High-Level Operations
@ -746,9 +746,8 @@ public:
static IntrusiveRefCntPtr<ASTReader> createPCHExternalASTSource(
StringRef Path, StringRef Sysroot,
DisableValidationForModuleKind DisableValidation,
bool AllowPCHWithCompilerErrors, Preprocessor &PP,
InMemoryModuleCache &ModuleCache, ASTContext &Context,
const PCHContainerReader &PCHContainerRdr,
bool AllowPCHWithCompilerErrors, Preprocessor &PP, ModuleCache &ModCache,
ASTContext &Context, const PCHContainerReader &PCHContainerRdr,
ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
ArrayRef<std::shared_ptr<DependencyCollector>> DependencyCollectors,
void *DeserializationListener, bool OwnDeserializationListener,
@ -896,7 +895,7 @@ public:
void setExternalSemaSource(IntrusiveRefCntPtr<ExternalSemaSource> ESS);
InMemoryModuleCache &getModuleCache() const { return *ModuleCache; }
ModuleCache &getModuleCache() const { return *ModCache; }
};
} // end namespace clang

11
clang/include/clang/Serialization/ASTReader.h
View File

@ -89,7 +89,7 @@ struct HeaderFileInfo;
class HeaderSearchOptions;
class LangOptions;
class MacroInfo;
class InMemoryModuleCache;
class ModuleCache;
class NamedDecl;
class NamespaceDecl;
class ObjCCategoryDecl;
@ -1742,8 +1742,8 @@ public:
///
/// \param ReadTimer If non-null, a timer used to track the time spent
/// deserializing.
ASTReader(Preprocessor &PP, InMemoryModuleCache &ModuleCache,
ASTContext *Context, const PCHContainerReader &PCHContainerRdr,
ASTReader(Preprocessor &PP, ModuleCache &ModCache, ASTContext *Context,
const PCHContainerReader &PCHContainerRdr,
ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
StringRef isysroot = "",
DisableValidationForModuleKind DisableValidationKind =
@ -1954,8 +1954,7 @@ public:
///
/// \returns true if an error occurred, false otherwise.
static bool readASTFileControlBlock(
StringRef Filename, FileManager &FileMgr,
const InMemoryModuleCache &ModuleCache,
StringRef Filename, FileManager &FileMgr, const ModuleCache &ModCache,
const PCHContainerReader &PCHContainerRdr, bool FindModuleFileExtensions,
ASTReaderListener &Listener, bool ValidateDiagnosticOptions,
unsigned ClientLoadCapabilities = ARR_ConfigurationMismatch |
@ -1964,7 +1963,7 @@ public:
/// Determine whether the given AST file is acceptable to load into a
/// translation unit with the given language and target options.
static bool isAcceptableASTFile(StringRef Filename, FileManager &FileMgr,
const InMemoryModuleCache &ModuleCache,
const ModuleCache &ModCache,
const PCHContainerReader &PCHContainerRdr,
const LangOptions &LangOpts,
const TargetOptions &TargetOpts,

21
clang/include/clang/Serialization/ASTWriter.h
View File

@ -60,7 +60,7 @@ class LangOptions;
class MacroDefinitionRecord;
class MacroInfo;
class Module;
class InMemoryModuleCache;
class ModuleCache;
class ModuleFileExtension;
class ModuleFileExtensionWriter;
class NamedDecl;
@ -117,7 +117,7 @@ private:
const SmallVectorImpl<char> &Buffer;
/// The PCM manager which manages memory buffers for pcm files.
InMemoryModuleCache &ModuleCache;
ModuleCache &ModCache;
/// The preprocessor we're writing.
Preprocessor *PP = nullptr;
@ -682,7 +682,7 @@ public:
/// Create a new precompiled header writer that outputs to
/// the given bitstream.
ASTWriter(llvm::BitstreamWriter &Stream, SmallVectorImpl<char> &Buffer,
InMemoryModuleCache &ModuleCache,
ModuleCache &ModCache,
ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
bool IncludeTimestamps = true, bool BuildingImplicitModule = false,
bool GeneratingReducedBMI = false);
@ -986,9 +986,8 @@ protected:
virtual Module *getEmittingModule(ASTContext &Ctx);
public:
PCHGenerator(Preprocessor &PP, InMemoryModuleCache &ModuleCache,
StringRef OutputFile, StringRef isysroot,
std::shared_ptr<PCHBuffer> Buffer,
PCHGenerator(Preprocessor &PP, ModuleCache &ModCache, StringRef OutputFile,
StringRef isysroot, std::shared_ptr<PCHBuffer> Buffer,
ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
bool AllowASTWithErrors = false, bool IncludeTimestamps = true,
bool BuildingImplicitModule = false,
@ -1010,14 +1009,14 @@ class CXX20ModulesGenerator : public PCHGenerator {
protected:
virtual Module *getEmittingModule(ASTContext &Ctx) override;
CXX20ModulesGenerator(Preprocessor &PP, InMemoryModuleCache &ModuleCache,
CXX20ModulesGenerator(Preprocessor &PP, ModuleCache &ModCache,
StringRef OutputFile, bool GeneratingReducedBMI,
bool AllowASTWithErrors);
public:
CXX20ModulesGenerator(Preprocessor &PP, InMemoryModuleCache &ModuleCache,
CXX20ModulesGenerator(Preprocessor &PP, ModuleCache &ModCache,
StringRef OutputFile, bool AllowASTWithErrors = false)
: CXX20ModulesGenerator(PP, ModuleCache, OutputFile,
: CXX20ModulesGenerator(PP, ModCache, OutputFile,
/*GeneratingReducedBMI=*/false,
AllowASTWithErrors) {}
@ -1028,9 +1027,9 @@ class ReducedBMIGenerator : public CXX20ModulesGenerator {
void anchor() override;
public:
ReducedBMIGenerator(Preprocessor &PP, InMemoryModuleCache &ModuleCache,
ReducedBMIGenerator(Preprocessor &PP, ModuleCache &ModCache,
StringRef OutputFile, bool AllowASTWithErrors = false)
: CXX20ModulesGenerator(PP, ModuleCache, OutputFile,
: CXX20ModulesGenerator(PP, ModCache, OutputFile,
/*GeneratingReducedBMI=*/true,
AllowASTWithErrors) {}
};

50
clang/include/clang/Serialization/ModuleCache.h Normal file
View File

@ -0,0 +1,50 @@
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef LLVM_CLANG_SERIALIZATION_MODULECACHE_H
#define LLVM_CLANG_SERIALIZATION_MODULECACHE_H
#include "clang/Basic/LLVM.h"
#include "llvm/ADT/IntrusiveRefCntPtr.h"
namespace llvm {
class AdvisoryLock;
} // namespace llvm
namespace clang {
class InMemoryModuleCache;
/// The module cache used for compiling modules implicitly. This centralizes the
/// operations the compiler might want to perform on the cache.
class ModuleCache : public RefCountedBase<ModuleCache> {
public:
/// May perform any work that only needs to be performed once for multiple
/// calls \c getLock() with the same module filename.
virtual void prepareForGetLock(StringRef ModuleFilename) = 0;
/// Returns lock for the given module file. The lock is initially unlocked.
virtual std::unique_ptr<llvm::AdvisoryLock>
getLock(StringRef ModuleFilename) = 0;
/// Returns this process's view of the module cache.
virtual InMemoryModuleCache &getInMemoryModuleCache() = 0;
virtual const InMemoryModuleCache &getInMemoryModuleCache() const = 0;
// TODO: Virtualize writing/reading PCM files, timestamping, pruning, etc.
virtual ~ModuleCache() = default;
};
/// Creates new \c ModuleCache backed by a file system directory that may be
/// operated on by multiple processes. This instance must be used across all
/// \c CompilerInstance instances participating in building modules for single
/// translation unit in order to share the same \c InMemoryModuleCache.
IntrusiveRefCntPtr<ModuleCache> createCrossProcessModuleCache();
} // namespace clang
#endif

12
clang/include/clang/Serialization/ModuleManager.h
View File

@ -37,7 +37,7 @@ class FileEntry;
class FileManager;
class GlobalModuleIndex;
class HeaderSearch;
class InMemoryModuleCache;
class ModuleCache;
class PCHContainerReader;
namespace serialization {
@ -65,7 +65,7 @@ class ModuleManager {
FileManager &FileMgr;
/// Cache of PCM files.
IntrusiveRefCntPtr<InMemoryModuleCache> ModuleCache;
IntrusiveRefCntPtr<ModuleCache> ModCache;
/// Knows how to unwrap module containers.
const PCHContainerReader &PCHContainerRdr;
@ -133,9 +133,9 @@ public:
SmallVectorImpl<std::unique_ptr<ModuleFile>>::reverse_iterator>;
using ModuleOffset = std::pair<uint32_t, StringRef>;
explicit ModuleManager(FileManager &FileMgr, InMemoryModuleCache &ModuleCache,
const PCHContainerReader &PCHContainerRdr,
const HeaderSearch &HeaderSearchInfo);
ModuleManager(FileManager &FileMgr, ModuleCache &ModCache,
const PCHContainerReader &PCHContainerRdr,
const HeaderSearch &HeaderSearchInfo);
/// Forward iterator to traverse all loaded modules.
ModuleIterator begin() { return Chain.begin(); }
@ -306,7 +306,7 @@ public:
/// View the graphviz representation of the module graph.
void viewGraph();
InMemoryModuleCache &getModuleCache() const { return *ModuleCache; }
ModuleCache &getModuleCache() const { return *ModCache; }
};
} // namespace serialization

21
clang/lib/Frontend/ASTUnit.cpp
View File

@ -61,7 +61,7 @@
#include "clang/Serialization/ASTReader.h"
#include "clang/Serialization/ASTWriter.h"
#include "clang/Serialization/ContinuousRangeMap.h"
#include "clang/Serialization/InMemoryModuleCache.h"
#include "clang/Serialization/ModuleCache.h"
#include "clang/Serialization/ModuleFile.h"
#include "clang/Serialization/PCHContainerOperations.h"
#include "llvm/ADT/ArrayRef.h"
@ -219,8 +219,8 @@ struct ASTUnit::ASTWriterData {
llvm::BitstreamWriter Stream;
ASTWriter Writer;
ASTWriterData(InMemoryModuleCache &ModuleCache)
: Stream(Buffer), Writer(Stream, Buffer, ModuleCache, {}) {}
ASTWriterData(ModuleCache &ModCache)
: Stream(Buffer), Writer(Stream, Buffer, ModCache, {}) {}
};
void ASTUnit::clearFileLevelDecls() {
@ -829,7 +829,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile(
AST->SourceMgr = new SourceManager(AST->getDiagnostics(),
AST->getFileManager(),
UserFilesAreVolatile);
AST->ModuleCache = new InMemoryModuleCache;
AST->ModCache = createCrossProcessModuleCache();
AST->HSOpts = HSOpts ? HSOpts : std::make_shared<HeaderSearchOptions>();
AST->HSOpts->ModuleFormat = std::string(PCHContainerRdr.getFormats().front());
AST->HeaderInfo.reset(new HeaderSearch(AST->HSOpts,
@ -861,8 +861,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile(
if (::getenv("LIBCLANG_DISABLE_PCH_VALIDATION"))
disableValid = DisableValidationForModuleKind::All;
AST->Reader = new ASTReader(
PP, *AST->ModuleCache, AST->Ctx.get(), PCHContainerRdr, {},
/*isysroot=*/"",
PP, *AST->ModCache, AST->Ctx.get(), PCHContainerRdr, {}, /*isysroot=*/"",
/*DisableValidationKind=*/disableValid, AllowASTWithCompilerErrors);
unsigned Counter = 0;
@ -1546,7 +1545,7 @@ ASTUnit::create(std::shared_ptr<CompilerInvocation> CI,
AST->UserFilesAreVolatile = UserFilesAreVolatile;
AST->SourceMgr = new SourceManager(AST->getDiagnostics(), *AST->FileMgr,
UserFilesAreVolatile);
AST->ModuleCache = new InMemoryModuleCache;
AST->ModCache = createCrossProcessModuleCache();
return AST;
}
@ -1833,7 +1832,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromCommandLine(
AST->FileMgr = new FileManager(AST->FileSystemOpts, VFS);
AST->StorePreamblesInMemory = StorePreamblesInMemory;
AST->PreambleStoragePath = PreambleStoragePath;
AST->ModuleCache = new InMemoryModuleCache;
AST->ModCache = createCrossProcessModuleCache();
AST->OnlyLocalDecls = OnlyLocalDecls;
AST->CaptureDiagnostics = CaptureDiagnostics;
AST->TUKind = TUKind;
@ -1844,7 +1843,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromCommandLine(
AST->Invocation = CI;
AST->SkipFunctionBodies = SkipFunctionBodies;
if (ForSerialization)
AST->WriterData.reset(new ASTWriterData(*AST->ModuleCache));
AST->WriterData.reset(new ASTWriterData(*AST->ModCache));
// Zero out now to ease cleanup during crash recovery.
CI = nullptr;
Diags = nullptr;
@ -2379,8 +2378,8 @@ bool ASTUnit::serialize(raw_ostream &OS) {
SmallString<128> Buffer;
llvm::BitstreamWriter Stream(Buffer);
InMemoryModuleCache ModuleCache;
ASTWriter Writer(Stream, Buffer, ModuleCache, {});
IntrusiveRefCntPtr<ModuleCache> ModCache = createCrossProcessModuleCache();
ASTWriter Writer(Stream, Buffer, *ModCache, {});
return serializeUnit(Writer, Buffer, getSema(), OS);
}

42
clang/lib/Frontend/CompilerInstance.cpp
View File

@ -39,16 +39,17 @@
#include "clang/Serialization/ASTReader.h"
#include "clang/Serialization/GlobalModuleIndex.h"
#include "clang/Serialization/InMemoryModuleCache.h"
#include "clang/Serialization/ModuleCache.h"
#include "llvm/ADT/IntrusiveRefCntPtr.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Config/llvm-config.h"
#include "llvm/Support/AdvisoryLock.h"
#include "llvm/Support/BuryPointer.h"
#include "llvm/Support/CrashRecoveryContext.h"
#include "llvm/Support/Errc.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/LockFileManager.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Program.h"
@ -66,11 +67,10 @@ using namespace clang;
CompilerInstance::CompilerInstance(
std::shared_ptr<PCHContainerOperations> PCHContainerOps,
InMemoryModuleCache *SharedModuleCache)
: ModuleLoader(/* BuildingModule = */ SharedModuleCache),
ModuleCache *ModCache)
: ModuleLoader(/*BuildingModule=*/ModCache),
Invocation(new CompilerInvocation()),
ModuleCache(SharedModuleCache ? SharedModuleCache
: new InMemoryModuleCache),
ModCache(ModCache ? ModCache : createCrossProcessModuleCache()),
ThePCHContainerOperations(std::move(PCHContainerOps)) {}
CompilerInstance::~CompilerInstance() {
@ -205,7 +205,7 @@ IntrusiveRefCntPtr<ASTReader> CompilerInstance::getASTReader() const {
return TheASTReader;
}
void CompilerInstance::setASTReader(IntrusiveRefCntPtr<ASTReader> Reader) {
assert(ModuleCache.get() == &Reader->getModuleManager().getModuleCache() &&
assert(ModCache.get() == &Reader->getModuleManager().getModuleCache() &&
"Expected ASTReader to use the same PCM cache");
TheASTReader = std::move(Reader);
}
@ -625,9 +625,8 @@ void CompilerInstance::createPCHExternalASTSource(
IntrusiveRefCntPtr<ASTReader> CompilerInstance::createPCHExternalASTSource(
StringRef Path, StringRef Sysroot,
DisableValidationForModuleKind DisableValidation,
bool AllowPCHWithCompilerErrors, Preprocessor &PP,
InMemoryModuleCache &ModuleCache, ASTContext &Context,
const PCHContainerReader &PCHContainerRdr,
bool AllowPCHWithCompilerErrors, Preprocessor &PP, ModuleCache &ModCache,
ASTContext &Context, const PCHContainerReader &PCHContainerRdr,
ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
ArrayRef<std::shared_ptr<DependencyCollector>> DependencyCollectors,
void *DeserializationListener, bool OwnDeserializationListener,
@ -635,7 +634,7 @@ IntrusiveRefCntPtr<ASTReader> CompilerInstance::createPCHExternalASTSource(
HeaderSearchOptions &HSOpts = PP.getHeaderSearchInfo().getHeaderSearchOpts();
IntrusiveRefCntPtr<ASTReader> Reader(new ASTReader(
PP, ModuleCache, &Context, PCHContainerRdr, Extensions,
PP, ModCache, &Context, PCHContainerRdr, Extensions,
Sysroot.empty() ? "" : Sysroot.data(), DisableValidation,
AllowPCHWithCompilerErrors, /*AllowConfigurationMismatch*/ false,
HSOpts.ModulesValidateSystemHeaders, HSOpts.ValidateASTInputFilesContent,
@ -1166,7 +1165,8 @@ compileModuleImpl(CompilerInstance &ImportingInstance, SourceLocation ImportLoc,
// Never compile a module that's already finalized - this would cause the
// existing module to be freed, causing crashes if it is later referenced
if (ImportingInstance.getModuleCache().isPCMFinal(ModuleFileName)) {
if (ImportingInstance.getModuleCache().getInMemoryModuleCache().isPCMFinal(
ModuleFileName)) {
ImportingInstance.getDiagnostics().Report(
ImportLoc, diag::err_module_rebuild_finalized)
<< ModuleName;
@ -1477,15 +1477,13 @@ static bool compileModuleAndReadASTBehindLock(
Diags.Report(ModuleNameLoc, diag::remark_module_lock)
<< ModuleFileName << Module->Name;
// FIXME: have LockFileManager return an error_code so that we can
// avoid the mkdir when the directory already exists.
StringRef Dir = llvm::sys::path::parent_path(ModuleFileName);
llvm::sys::fs::create_directories(Dir);
auto &ModuleCache = ImportingInstance.getModuleCache();
ModuleCache.prepareForGetLock(ModuleFileName);
while (true) {
llvm::LockFileManager Lock(ModuleFileName);
auto Lock = ModuleCache.getLock(ModuleFileName);
bool Owned;
if (llvm::Error Err = Lock.tryLock().moveInto(Owned)) {
if (llvm::Error Err = Lock->tryLock().moveInto(Owned)) {
// ModuleCache takes care of correctness and locks are only necessary for
// performance. Fallback to building the module in case of any lock
// related errors.
@ -1502,19 +1500,19 @@ static bool compileModuleAndReadASTBehindLock(
// Someone else is responsible for building the module. Wait for them to
// finish.
switch (Lock.waitForUnlockFor(std::chrono::seconds(90))) {
switch (Lock->waitForUnlockFor(std::chrono::seconds(90))) {
case llvm::WaitForUnlockResult::Success:
break; // The interesting case.
case llvm::WaitForUnlockResult::OwnerDied:
continue; // try again to get the lock.
case llvm::WaitForUnlockResult::Timeout:
// Since ModuleCache takes care of correctness, we try waiting for
// another process to complete the build so clang does not do it done
// twice. If case of timeout, build it ourselves.
// Since the InMemoryModuleCache takes care of correctness, we try waiting
// for someone else to complete the build so that it does not happen
// twice. In case of timeout, build it ourselves.
Diags.Report(ModuleNameLoc, diag::remark_module_lock_timeout)
<< Module->Name;
// Clear the lock file so that future invocations can make progress.
Lock.unsafeMaybeUnlock();
Lock->unsafeMaybeUnlock();
continue;
}

5
clang/lib/Frontend/PrecompiledPreamble.cpp
View File

@ -292,10 +292,9 @@ private:
class PrecompilePreambleConsumer : public PCHGenerator {
public:
PrecompilePreambleConsumer(PrecompilePreambleAction &Action, Preprocessor &PP,
InMemoryModuleCache &ModuleCache,
StringRef isysroot,
ModuleCache &ModCache, StringRef isysroot,
std::shared_ptr<PCHBuffer> Buffer)
: PCHGenerator(PP, ModuleCache, "", isysroot, std::move(Buffer),
: PCHGenerator(PP, ModCache, "", isysroot, std::move(Buffer),
ArrayRef<std::shared_ptr<ModuleFileExtension>>(),
/*AllowASTWithErrors=*/true),
Action(Action) {}

46
clang/lib/Serialization/ASTReader.cpp
View File

@ -90,6 +90,7 @@
#include "clang/Serialization/ContinuousRangeMap.h"
#include "clang/Serialization/GlobalModuleIndex.h"
#include "clang/Serialization/InMemoryModuleCache.h"
#include "clang/Serialization/ModuleCache.h"
#include "clang/Serialization/ModuleFile.h"
#include "clang/Serialization/ModuleFileExtension.h"
#include "clang/Serialization/ModuleManager.h"
@ -3318,9 +3319,12 @@ ASTReader::ReadControlBlock(ModuleFile &F,
StoredSignature, Capabilities);
// If we diagnosed a problem, produce a backtrace.
bool recompilingFinalized =
Result == OutOfDate && (Capabilities & ARR_OutOfDate) &&
getModuleManager().getModuleCache().isPCMFinal(F.FileName);
bool recompilingFinalized = Result == OutOfDate &&
(Capabilities & ARR_OutOfDate) &&
getModuleManager()
.getModuleCache()
.getInMemoryModuleCache()
.isPCMFinal(F.FileName);
if (isDiagnosedResult(Result, Capabilities) || recompilingFinalized)
Diag(diag::note_module_file_imported_by)
<< F.FileName << !F.ModuleName.empty() << F.ModuleName;
@ -5006,7 +5010,7 @@ ASTReader::ReadASTCore(StringRef FileName,
bool ShouldFinalizePCM = false;
auto FinalizeOrDropPCM = llvm::make_scope_exit([&]() {
auto &MC = getModuleManager().getModuleCache();
auto &MC = getModuleManager().getModuleCache().getInMemoryModuleCache();
if (ShouldFinalizePCM)
MC.finalizePCM(FileName);
else
@ -5143,7 +5147,8 @@ ASTReader::readUnhashedControlBlock(ModuleFile &F, bool WasImportedBy,
// validation will fail during the as-system import since the PCM on disk
// doesn't guarantee that -Werror was respected. However, the -Werror
// flags were checked during the initial as-user import.
if (getModuleManager().getModuleCache().isPCMFinal(F.FileName)) {
if (getModuleManager().getModuleCache().getInMemoryModuleCache().isPCMFinal(
F.FileName)) {
Diag(diag::warn_module_system_bit_conflict) << F.FileName;
return Success;
}
@ -5651,14 +5656,14 @@ namespace {
} // namespace
bool ASTReader::readASTFileControlBlock(
StringRef Filename, FileManager &FileMgr,
const InMemoryModuleCache &ModuleCache,
StringRef Filename, FileManager &FileMgr, const ModuleCache &ModCache,
const PCHContainerReader &PCHContainerRdr, bool FindModuleFileExtensions,
ASTReaderListener &Listener, bool ValidateDiagnosticOptions,
unsigned ClientLoadCapabilities) {
// Open the AST file.
std::unique_ptr<llvm::MemoryBuffer> OwnedBuffer;
llvm::MemoryBuffer *Buffer = ModuleCache.lookupPCM(Filename);
llvm::MemoryBuffer *Buffer =
ModCache.getInMemoryModuleCache().lookupPCM(Filename);
if (!Buffer) {
// FIXME: We should add the pcm to the InMemoryModuleCache if it could be
// read again later, but we do not have the context here to determine if it
@ -5947,19 +5952,15 @@ bool ASTReader::readASTFileControlBlock(
return false;
}
bool ASTReader::isAcceptableASTFile(StringRef Filename, FileManager &FileMgr,
const InMemoryModuleCache &ModuleCache,
const PCHContainerReader &PCHContainerRdr,
const LangOptions &LangOpts,
const TargetOptions &TargetOpts,
const PreprocessorOptions &PPOpts,
StringRef ExistingModuleCachePath,
bool RequireStrictOptionMatches) {
bool ASTReader::isAcceptableASTFile(
StringRef Filename, FileManager &FileMgr, const ModuleCache &ModCache,
const PCHContainerReader &PCHContainerRdr, const LangOptions &LangOpts,
const TargetOptions &TargetOpts, const PreprocessorOptions &PPOpts,
StringRef ExistingModuleCachePath, bool RequireStrictOptionMatches) {
SimplePCHValidator validator(LangOpts, TargetOpts, PPOpts,
ExistingModuleCachePath, FileMgr,
RequireStrictOptionMatches);
return !readASTFileControlBlock(Filename, FileMgr, ModuleCache,
PCHContainerRdr,
return !readASTFileControlBlock(Filename, FileMgr, ModCache, PCHContainerRdr,
/*FindModuleFileExtensions=*/false, validator,
/*ValidateDiagnosticOptions=*/true);
}
@ -6496,7 +6497,10 @@ ASTReader::getModulePreprocessedEntities(ModuleFile &Mod) const {
bool ASTReader::canRecoverFromOutOfDate(StringRef ModuleFileName,
unsigned int ClientLoadCapabilities) {
return ClientLoadCapabilities & ARR_OutOfDate &&
!getModuleManager().getModuleCache().isPCMFinal(ModuleFileName);
!getModuleManager()
.getModuleCache()
.getInMemoryModuleCache()
.isPCMFinal(ModuleFileName);
}
llvm::iterator_range<ASTReader::ModuleDeclIterator>
@ -10862,7 +10866,7 @@ void ASTReader::pushExternalDeclIntoScope(NamedDecl *D, DeclarationName Name) {
}
}
ASTReader::ASTReader(Preprocessor &PP, InMemoryModuleCache &ModuleCache,
ASTReader::ASTReader(Preprocessor &PP, ModuleCache &ModCache,
ASTContext *Context,
const PCHContainerReader &PCHContainerRdr,
ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
@ -10878,7 +10882,7 @@ ASTReader::ASTReader(Preprocessor &PP, InMemoryModuleCache &ModuleCache,
SourceMgr(PP.getSourceManager()), FileMgr(PP.getFileManager()),
PCHContainerRdr(PCHContainerRdr), Diags(PP.getDiagnostics()),
StackHandler(Diags), PP(PP), ContextObj(Context),
ModuleMgr(PP.getFileManager(), ModuleCache, PCHContainerRdr,
ModuleMgr(PP.getFileManager(), ModCache, PCHContainerRdr,
PP.getHeaderSearchInfo()),
DummyIdResolver(PP), ReadTimer(std::move(ReadTimer)), isysroot(isysroot),
DisableValidationKind(DisableValidationKind),

12
clang/lib/Serialization/ASTWriter.cpp
View File

@ -75,6 +75,7 @@
#include "clang/Serialization/ASTReader.h"
#include "clang/Serialization/ASTRecordWriter.h"
#include "clang/Serialization/InMemoryModuleCache.h"
#include "clang/Serialization/ModuleCache.h"
#include "clang/Serialization/ModuleFile.h"
#include "clang/Serialization/ModuleFileExtension.h"
#include "clang/Serialization/SerializationDiagnostic.h"
@ -5326,12 +5327,11 @@ void ASTWriter::SetSelectorOffset(Selector Sel, uint32_t Offset) {
}
ASTWriter::ASTWriter(llvm::BitstreamWriter &Stream,
SmallVectorImpl<char> &Buffer,
InMemoryModuleCache &ModuleCache,
SmallVectorImpl<char> &Buffer, ModuleCache &ModCache,
ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
bool IncludeTimestamps, bool BuildingImplicitModule,
bool GeneratingReducedBMI)
: Stream(Stream), Buffer(Buffer), ModuleCache(ModuleCache),
: Stream(Stream), Buffer(Buffer), ModCache(ModCache),
IncludeTimestamps(IncludeTimestamps),
BuildingImplicitModule(BuildingImplicitModule),
GeneratingReducedBMI(GeneratingReducedBMI) {
@ -5389,9 +5389,9 @@ ASTWriter::WriteAST(llvm::PointerUnion<Sema *, Preprocessor *> Subject,
if (ShouldCacheASTInMemory) {
// Construct MemoryBuffer and update buffer manager.
ModuleCache.addBuiltPCM(OutputFile,
llvm::MemoryBuffer::getMemBufferCopy(
StringRef(Buffer.begin(), Buffer.size())));
ModCache.getInMemoryModuleCache().addBuiltPCM(
OutputFile, llvm::MemoryBuffer::getMemBufferCopy(
StringRef(Buffer.begin(), Buffer.size())));
}
return Signature;
}

1
clang/lib/Serialization/CMakeLists.txt
View File

@ -18,6 +18,7 @@ add_clang_library(clangSerialization
GeneratePCH.cpp
GlobalModuleIndex.cpp
InMemoryModuleCache.cpp
ModuleCache.cpp
ModuleFile.cpp
ModuleFileExtension.cpp
ModuleManager.cpp

8
clang/lib/Serialization/GeneratePCH.cpp
View File

@ -23,7 +23,7 @@
using namespace clang;
PCHGenerator::PCHGenerator(
Preprocessor &PP, InMemoryModuleCache &ModuleCache, StringRef OutputFile,
Preprocessor &PP, ModuleCache &ModCache, StringRef OutputFile,
StringRef isysroot, std::shared_ptr<PCHBuffer> Buffer,
ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
bool AllowASTWithErrors, bool IncludeTimestamps,
@ -31,7 +31,7 @@ PCHGenerator::PCHGenerator(
bool GeneratingReducedBMI)
: PP(PP), Subject(&PP), OutputFile(OutputFile), isysroot(isysroot.str()),
Buffer(std::move(Buffer)), Stream(this->Buffer->Data),
Writer(Stream, this->Buffer->Data, ModuleCache, Extensions,
Writer(Stream, this->Buffer->Data, ModCache, Extensions,
IncludeTimestamps, BuildingImplicitModule, GeneratingReducedBMI),
AllowASTWithErrors(AllowASTWithErrors),
ShouldCacheASTInMemory(ShouldCacheASTInMemory) {
@ -100,12 +100,12 @@ ASTDeserializationListener *PCHGenerator::GetASTDeserializationListener() {
void PCHGenerator::anchor() {}
CXX20ModulesGenerator::CXX20ModulesGenerator(Preprocessor &PP,
InMemoryModuleCache &ModuleCache,
ModuleCache &ModCache,
StringRef OutputFile,
bool GeneratingReducedBMI,
bool AllowASTWithErrors)
: PCHGenerator(
PP, ModuleCache, OutputFile, llvm::StringRef(),
PP, ModCache, OutputFile, llvm::StringRef(),
std::make_shared<PCHBuffer>(),
/*Extensions=*/ArrayRef<std::shared_ptr<ModuleFileExtension>>(),
AllowASTWithErrors, /*IncludeTimestamps=*/false,

44
clang/lib/Serialization/ModuleCache.cpp Normal file
View File

@ -0,0 +1,44 @@
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "clang/Serialization/ModuleCache.h"
#include "clang/Serialization/InMemoryModuleCache.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/LockFileManager.h"
#include "llvm/Support/Path.h"
using namespace clang;
namespace {
class CrossProcessModuleCache : public ModuleCache {
InMemoryModuleCache InMemory;
public:
void prepareForGetLock(StringRef ModuleFilename) override {
// FIXME: Do this in LockFileManager and only if the directory doesn't
// exist.
StringRef Dir = llvm::sys::path::parent_path(ModuleFilename);
llvm::sys::fs::create_directories(Dir);
}
std::unique_ptr<llvm::AdvisoryLock>
getLock(StringRef ModuleFilename) override {
return std::make_unique<llvm::LockFileManager>(ModuleFilename);
}
InMemoryModuleCache &getInMemoryModuleCache() override { return InMemory; }
const InMemoryModuleCache &getInMemoryModuleCache() const override {
return InMemory;
}
};
} // namespace
IntrusiveRefCntPtr<ModuleCache> clang::createCrossProcessModuleCache() {
return llvm::makeIntrusiveRefCnt<CrossProcessModuleCache>();
}

20
clang/lib/Serialization/ModuleManager.cpp
View File

@ -18,6 +18,7 @@
#include "clang/Lex/ModuleMap.h"
#include "clang/Serialization/GlobalModuleIndex.h"
#include "clang/Serialization/InMemoryModuleCache.h"
#include "clang/Serialization/ModuleCache.h"
#include "clang/Serialization/ModuleFile.h"
#include "clang/Serialization/PCHContainerOperations.h"
#include "llvm/ADT/STLExtras.h"
@ -182,17 +183,20 @@ ModuleManager::addModule(StringRef FileName, ModuleKind Type,
// Load the contents of the module
if (std::unique_ptr<llvm::MemoryBuffer> Buffer = lookupBuffer(FileName)) {
// The buffer was already provided for us.
NewModule->Buffer = &ModuleCache->addBuiltPCM(FileName, std::move(Buffer));
NewModule->Buffer = &getModuleCache().getInMemoryModuleCache().addBuiltPCM(
FileName, std::move(Buffer));
// Since the cached buffer is reused, it is safe to close the file
// descriptor that was opened while stat()ing the PCM in
// lookupModuleFile() above, it won't be needed any longer.
Entry->closeFile();
} else if (llvm::MemoryBuffer *Buffer =
getModuleCache().lookupPCM(FileName)) {
getModuleCache().getInMemoryModuleCache().lookupPCM(
FileName)) {
NewModule->Buffer = Buffer;
// As above, the file descriptor is no longer needed.
Entry->closeFile();
} else if (getModuleCache().shouldBuildPCM(FileName)) {
} else if (getModuleCache().getInMemoryModuleCache().shouldBuildPCM(
FileName)) {
// Report that the module is out of date, since we tried (and failed) to
// import it earlier.
Entry->closeFile();
@ -213,7 +217,8 @@ ModuleManager::addModule(StringRef FileName, ModuleKind Type,
return Missing;
}
NewModule->Buffer = &getModuleCache().addPCM(FileName, std::move(*Buf));
NewModule->Buffer = &getModuleCache().getInMemoryModuleCache().addPCM(
FileName, std::move(*Buf));
}
// Initialize the stream.
@ -324,12 +329,11 @@ void ModuleManager::moduleFileAccepted(ModuleFile *MF) {
ModulesInCommonWithGlobalIndex.push_back(MF);
}
ModuleManager::ModuleManager(FileManager &FileMgr,
InMemoryModuleCache &ModuleCache,
ModuleManager::ModuleManager(FileManager &FileMgr, ModuleCache &ModCache,
const PCHContainerReader &PCHContainerRdr,
const HeaderSearch &HeaderSearchInfo)
: FileMgr(FileMgr), ModuleCache(&ModuleCache),
PCHContainerRdr(PCHContainerRdr), HeaderSearchInfo(HeaderSearchInfo) {}
: FileMgr(FileMgr), ModCache(&ModCache), PCHContainerRdr(PCHContainerRdr),
HeaderSearchInfo(HeaderSearchInfo) {}
void ModuleManager::visit(llvm::function_ref<bool(ModuleFile &M)> Visitor,
llvm::SmallPtrSetImpl<ModuleFile *> *ModuleFilesHit) {

7
clang/unittests/Frontend/FrontendActionTest.cpp
View File

@ -18,6 +18,7 @@
#include "clang/Lex/PreprocessorOptions.h"
#include "clang/Sema/Sema.h"
#include "clang/Serialization/InMemoryModuleCache.h"
#include "clang/Serialization/ModuleCache.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/ToolOutputFile.h"
#include "llvm/Support/VirtualFileSystem.h"
@ -287,10 +288,12 @@ TEST(GeneratePCHFrontendAction, CacheGeneratedPCH) {
// Check whether the PCH was cached.
if (ShouldCache)
EXPECT_EQ(InMemoryModuleCache::Final,
Compiler.getModuleCache().getPCMState(PCHFilename));
Compiler.getModuleCache().getInMemoryModuleCache().getPCMState(
PCHFilename));
else
EXPECT_EQ(InMemoryModuleCache::Unknown,
Compiler.getModuleCache().getPCMState(PCHFilename));
Compiler.getModuleCache().getInMemoryModuleCache().getPCMState(
PCHFilename));
}
}

1
clang/unittests/Lex/HeaderSearchTest.cpp
View File

@ -16,7 +16,6 @@
#include "clang/Basic/TargetInfo.h"
#include "clang/Basic/TargetOptions.h"
#include "clang/Lex/HeaderSearchOptions.h"
#include "clang/Serialization/InMemoryModuleCache.h"
#include "llvm/Support/MemoryBuffer.h"
#include "gtest/gtest.h"
#include <memory>