[CAS] gmodule support for caching build

clang/include/clang/Basic/ASTSourceDescriptor.h

@@ -30,21 +30,23 @@ class ASTSourceDescriptor {
   StringRef Path;
   StringRef ASTFile;
   ASTFileSignature Signature;
+  StringRef CASID;
   Module *ClangModule = nullptr;
 
 public:
   ASTSourceDescriptor() = default;
   ASTSourceDescriptor(StringRef Name, StringRef Path, StringRef ASTFile,
-                      ASTFileSignature Signature)
+                      ASTFileSignature Signature, StringRef CASID)
       : PCHModuleName(std::move(Name)), Path(std::move(Path)),
-        ASTFile(std::move(ASTFile)), Signature(Signature) {}
+        ASTFile(std::move(ASTFile)), Signature(Signature), CASID(CASID) {}
   ASTSourceDescriptor(Module &M);
 
   std::string getModuleName() const;
   StringRef getPath() const { return Path; }
   StringRef getASTFile() const { return ASTFile; }
   ASTFileSignature getSignature() const { return Signature; }
   Module *getModuleOrNull() const { return ClangModule; }
+  StringRef getCASID() const { return CASID; }
 };
 
 } // namespace clang

clang/include/clang/Basic/DiagnosticCASKinds.td

@@ -23,8 +23,10 @@ def err_cas_depscan_daemon_connection: Error<
 def err_cas_depscan_failed: Error<
   "CAS-based dependency scan failed: %0">, DefaultFatal;
 def err_cas_store: Error<"failed to store to CAS: %0">, DefaultFatal;
-def err_cas_unloadable_module : Error<
-  "module file '%0' not found: unloadable module cache key %1: %2">, DefaultFatal;
+def err_cas_unloadable_module
+    : Error<"module file '%0' not found: unloadable %select{casid|module cache "
+            "key}1 %2: %3">,
+      DefaultFatal;
 def err_cas_missing_module : Error<
   "module file '%0' not found: missing module cache key %1: %2">, DefaultFatal;
 def err_cas_missing_root_id : Error<

clang/include/clang/Basic/Module.h

@@ -274,6 +274,9 @@ class alignas(8) Module {
   /// The \c ActionCache key for this module, if any.
   std::optional<std::string> ModuleCacheKey;
 
+  /// The \c CASID for the loaded module, otherwise empty.
+  std::string CASID;
+
   /// The top-level headers associated with this module.
   llvm::SmallSetVector<FileEntryRef, 2> TopHeaders;
 
@@ -773,6 +776,13 @@ class alignas(8) Module {
     getTopLevelModule()->ModuleCacheKey = std::move(Key);
   }
 
+  StringRef getCASID() const { return getTopLevelModule()->CASID; }
+
+  void setCASID(std::string ID) {
+    assert(getCASID().empty() || getCASID() == ID);
+    getTopLevelModule()->CASID = std::move(ID);
+  }
+
   /// Retrieve the umbrella directory as written.
   std::optional<DirectoryName> getUmbrellaDirAsWritten() const {
     if (const auto *Dir = std::get_if<DirectoryEntryRef>(&Umbrella))

clang/include/clang/CAS/CASOptions.h

@@ -15,6 +15,7 @@
 #define LLVM_CLANG_CAS_CASOPTIONS_H
 
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/CAS/CASConfiguration.h"
 #include "llvm/Support/Error.h"
 #include <string>
 #include <vector>
@@ -30,52 +31,6 @@ namespace clang {
 
 class DiagnosticsEngine;
 
-/// Base class for options configuring which CAS to use. Separated for the
-/// fields where we don't need special move/copy logic.
-///
-/// TODO: Add appropriate options once we support plugins.
-class CASConfiguration {
-public:
-  enum CASKind {
-    UnknownCAS,
-    InMemoryCAS,
-    OnDiskCAS,
-  };
-
-  /// Kind of CAS to use.
-  CASKind getKind() const {
-    return IsFrozen ? UnknownCAS : CASPath.empty() ? InMemoryCAS : OnDiskCAS;
-  }
-
-  /// Path to a persistent backing store on-disk. This is optional, although \a
-  /// CASFileSystemRootID is unlikely to work without it.
-  ///
-  /// - "" means there is none; falls back to in-memory.
-  /// - "auto" is an alias for an automatically chosen location in the user's
-  ///   system cache.
-  std::string CASPath;
-
-  std::string PluginPath;
-  /// Each entry is a (<option-name>, <value>) pair.
-  std::vector<std::pair<std::string, std::string>> PluginOptions;
-
-  friend bool operator==(const CASConfiguration &LHS,
-                         const CASConfiguration &RHS) {
-    return LHS.CASPath == RHS.CASPath && LHS.PluginPath == RHS.PluginPath &&
-           LHS.PluginOptions == RHS.PluginOptions;
-  }
-  friend bool operator!=(const CASConfiguration &LHS,
-                         const CASConfiguration &RHS) {
-    return !(LHS == RHS);
-  }
-
-private:
-  /// Whether the configuration has been "frozen", in order to hide the kind of
-  /// CAS that's in use.
-  bool IsFrozen = false;
-  friend class CASOptions;
-};
-
 /// Options configuring which CAS to use. User-accessible fields should be
 /// defined in CASConfiguration to enable caching a CAS instance.
 ///
@@ -87,8 +42,18 @@ class CASConfiguration {
 /// clang::createVFSFromCompilerInvocation() uses the same CAS instance that
 /// the rest of the compiler job does, without updating all callers. Probably
 /// it would be better to update all callers and remove it from here.
-class CASOptions : public CASConfiguration {
+class CASOptions : public llvm::cas::CASConfiguration {
 public:
+  enum CASKind {
+    UnknownCAS,
+    InMemoryCAS,
+    OnDiskCAS,
+  };
+
+  /// Kind of CAS to use.
+  CASKind getKind() const {
+    return IsFrozen ? UnknownCAS : CASPath.empty() ? InMemoryCAS : OnDiskCAS;
+  }
   /// Get a CAS & ActionCache defined by the options above. Future calls will
   /// return the same instances... unless the configuration has changed, in
   /// which case new ones will be created.
@@ -117,8 +82,6 @@ class CASOptions : public CASConfiguration {
   /// default on-disk CAS, otherwise this is a noop.
   void ensurePersistentCAS();
 
-  void getResolvedCASPath(llvm::SmallVectorImpl<char> &Result) const;
-
 private:
   /// Initialize Cached CAS and ActionCache.
   llvm::Error initCache() const;
@@ -133,6 +96,10 @@ class CASOptions : public CASConfiguration {
     CASConfiguration Config;
   };
   mutable CachedCAS Cache;
+
+  /// Whether the configuration has been "frozen", in order to hide the kind of
+  /// CAS that's in use.
+  bool IsFrozen = false;
 };
 
 } // end namespace clang

clang/include/clang/Frontend/CompilerInstance.h

@@ -999,8 +999,8 @@ class CompilerInstance : public ModuleLoader {
   /// "-fmodule-file-cache-key", or an imported pcm file. Used in diagnostics.
   ///
   /// \returns true on failure.
-  bool addCachedModuleFile(StringRef Path, StringRef CacheKey,
-                           StringRef Provider);
+  bool addCachedModuleFile(StringRef Path, StringRef CASID, StringRef Provider,
+                           bool IsKey);
 
   ModuleCache &getModuleCache() const { return *ModCache; }
 

clang/include/clang/Serialization/ASTReader.h

@@ -286,6 +286,14 @@ class ASTReaderListener {
     return false;
   }
 
+  /// Called for each module CASID.
+  ///
+  /// \returns true to indicate the key cannot be loaded.
+  virtual bool readModuleCASID(StringRef ModuleName, StringRef Filename,
+                               StringRef CASID) {
+    return false;
+  }
+
   /// Indicates that a particular module file extension has been read.
   virtual void readModuleFileExtension(
                  const ModuleFileExtensionMetadata &Metadata) {}
@@ -342,6 +350,8 @@ class ChainedASTReaderListener : public ASTReaderListener {
   bool readIncludeTreeID(StringRef ID, bool Complain) override;
   bool readModuleCacheKey(StringRef ModuleName, StringRef Filename,
                           StringRef CacheKey) override;
+  bool readModuleCASID(StringRef ModuleName, StringRef Filename,
+                       StringRef CASID) override;
   void readModuleFileExtension(
          const ModuleFileExtensionMetadata &Metadata) override;
 };

clang/include/clang/Serialization/InMemoryModuleCache.h

@@ -30,6 +30,8 @@ class InMemoryModuleCache : public llvm::RefCountedBase<InMemoryModuleCache> {
   struct PCM {
     std::unique_ptr<llvm::MemoryBuffer> Buffer;
 
+    std::string CASID;
+
     /// Track whether this PCM is known to be good (either built or
     /// successfully imported by a CompilerInstance/ASTReader using this
     /// cache).
@@ -38,6 +40,9 @@ class InMemoryModuleCache : public llvm::RefCountedBase<InMemoryModuleCache> {
     PCM() = default;
     PCM(std::unique_ptr<llvm::MemoryBuffer> Buffer)
         : Buffer(std::move(Buffer)) {}
+
+    PCM(std::unique_ptr<llvm::MemoryBuffer> Buffer, llvm::StringRef CASID)
+        : Buffer(std::move(Buffer)), CASID(CASID.str()) {}
   };
 
   /// Cache of buffers.
@@ -64,7 +69,8 @@ class InMemoryModuleCache : public llvm::RefCountedBase<InMemoryModuleCache> {
   /// \post state is Tentative
   /// \return a reference to the buffer as a convenience.
   llvm::MemoryBuffer &addPCM(llvm::StringRef Filename,
-                             std::unique_ptr<llvm::MemoryBuffer> Buffer);
+                             std::unique_ptr<llvm::MemoryBuffer> Buffer,
+                             llvm::StringRef CASID = "");
 
   /// Store a just-built PCM under the Filename.
   ///
@@ -90,6 +96,9 @@ class InMemoryModuleCache : public llvm::RefCountedBase<InMemoryModuleCache> {
   /// Get a pointer to the pCM if it exists; else nullptr.
   llvm::MemoryBuffer *lookupPCM(llvm::StringRef Filename) const;
 
+  /// Get the PCM if it exits; else nullptr.
+  const PCM *lookup(llvm::StringRef Filename) const;
+
   /// Check whether the PCM is final and has been shown to work.
   ///
   /// \return true iff state is Final.

clang/include/clang/Serialization/ModuleFile.h

@@ -147,6 +147,9 @@ class ModuleFile {
   /// The \c ActionCache key for this module, or empty.
   std::string ModuleCacheKey;
 
+  /// The \c CASID for the module, or empty.
+  std::string CASID;
+
   /// The CAS filesystem root ID for implicit modules built with the dependency
   /// scanner, or empty.
   std::string CASFileSystemRootID;

clang/lib/Basic/ASTSourceDescriptor.cpp

@@ -21,6 +21,7 @@ ASTSourceDescriptor::ASTSourceDescriptor(Module &M)
     Path = M.Directory->getName();
   if (auto File = M.getASTFile())
     ASTFile = File->getName();
+  CASID = M.getCASID();
 }
 
 std::string ASTSourceDescriptor::getModuleName() const {

clang/lib/CAS/CASOptions.cpp

@@ -22,7 +22,7 @@ std::pair<std::shared_ptr<llvm::cas::ObjectStore>,
           std::shared_ptr<llvm::cas::ActionCache>>
 CASOptions::getOrCreateDatabases(DiagnosticsEngine &Diags,
                                  bool CreateEmptyDBsOnFailure) const {
-  if (Cache.Config.IsFrozen)
+  if (IsFrozen)
     return {Cache.CAS, Cache.AC};
 
   if (auto E = initCache())
@@ -44,7 +44,7 @@ CASOptions::getOrCreateDatabases() const {
 }
 
 void CASOptions::freezeConfig(DiagnosticsEngine &Diags) {
-  if (Cache.Config.IsFrozen)
+  if (IsFrozen)
     return;
 
   // Make sure the cache is initialized.
@@ -57,7 +57,7 @@ void CASOptions::freezeConfig(DiagnosticsEngine &Diags) {
   // scheduled/executed at a level that has access to the configuration.
   auto &CurrentConfig = static_cast<CASConfiguration &>(*this);
   CurrentConfig = CASConfiguration();
-  CurrentConfig.IsFrozen = Cache.Config.IsFrozen = true;
+  IsFrozen = true;
 
   if (Cache.CAS) {
     // Set the CASPath to the hash schema, since that leaks through CASContext's
@@ -90,41 +90,10 @@ llvm::Error CASOptions::initCache() const {
   Cache.Config = CurrentConfig;
   StringRef CASPath = Cache.Config.CASPath;
 
-  if (!PluginPath.empty()) {
-    std::pair<std::shared_ptr<ObjectStore>, std::shared_ptr<ActionCache>> DBs;
-    if (llvm::Error E =
-            createPluginCASDatabases(PluginPath, CASPath, PluginOptions)
-                .moveInto(DBs)) {
-      return E;
-    }
-    std::tie(Cache.CAS, Cache.AC) = std::move(DBs);
-    return llvm::Error::success();
-  }
-
-  if (CASPath.empty()) {
-    Cache.CAS = llvm::cas::createInMemoryCAS();
-    Cache.AC = llvm::cas::createInMemoryActionCache();
-    return llvm::Error::success();
-  }
-
-  SmallString<256> PathBuf;
-  getResolvedCASPath(PathBuf);
-  if (CASPath == "auto") {
-    getDefaultOnDiskCASPath(PathBuf);
-    CASPath = PathBuf;
-  }
-  std::pair<std::unique_ptr<ObjectStore>, std::unique_ptr<ActionCache>> DBs;
-  if (llvm::Error E = createOnDiskUnifiedCASDatabases(CASPath).moveInto(DBs))
-    return E;
+  auto DBs = Cache.Config.createDatabases();
+  if (!DBs)
+    return DBs.takeError();
 
-  std::tie(Cache.CAS, Cache.AC) = std::move(DBs);
+  std::tie(Cache.CAS, Cache.AC) = std::move(*DBs);
   return llvm::Error::success();
 }
-
-void CASOptions::getResolvedCASPath(SmallVectorImpl<char> &Result) const {
-  if (CASPath == "auto") {
-    getDefaultOnDiskCASPath(Result);
-  } else {
-    Result.assign(CASPath.begin(), CASPath.end());
-  }
-}

clang/lib/CodeGen/CGDebugInfo.cpp

@@ -3325,6 +3325,10 @@ llvm::DIModule *CGDebugInfo::getOrCreateModuleRef(ASTSourceDescriptor Mod,
     std::string Remapped = remapDIPath(Path);
     StringRef Relative(Remapped);
     StringRef CompDir = TheCU->getDirectory();
+    // If compilation is not empty, return the path as it is.
+    if (CompDir.empty())
+      return Remapped;
+
     if (Relative.consume_front(CompDir))
       Relative.consume_front(llvm::sys::path::get_separator());
 
@@ -3351,6 +3355,11 @@ llvm::DIModule *CGDebugInfo::getOrCreateModuleRef(ASTSourceDescriptor Mod,
         PCM = Mod.getPath();
     }
     llvm::sys::path::append(PCM, Mod.getASTFile());
+
+    // FIXME: Prefer CASID if exists.
+    if (!Mod.getCASID().empty())
+      PCM = Mod.getCASID();
+
     DIB.createCompileUnit(
         TheCU->getSourceLanguage(),
         // TODO: Support "Source" from external AST providers?

clang/lib/CodeGen/ObjectFilePCHContainerWriter.cpp

@@ -191,8 +191,8 @@ class PCHContainerGenerator : public ASTConsumer {
     // Prepare CGDebugInfo to emit debug info for a clang module.
     auto *DI = Builder->getModuleDebugInfo();
     StringRef ModuleName = llvm::sys::path::filename(MainFileName);
-    DI->setPCHDescriptor(
-        {ModuleName, "", OutputFileName, ASTFileSignature::createDISentinel()});
+    DI->setPCHDescriptor({ModuleName, "", OutputFileName,
+                          ASTFileSignature::createDISentinel(), /*CASID=*/""});
     DI->setModuleMap(MMap);
   }