[Dependency Scanning] Bridge Clang dependency scanner results on-demand

include/swift/AST/ModuleDependencies.h

@@ -26,6 +26,7 @@
 #include "swift/Serialization/Validation.h"
 #include "clang/CAS/CASOptions.h"
 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
+#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseSet.h"
@@ -163,6 +164,12 @@ using ModuleDependencyIDSetVector =
     llvm::SetVector<ModuleDependencyID, std::vector<ModuleDependencyID>,
                     std::set<ModuleDependencyID>>;
 
+/// A callback to lookup module outputs for "-fmodule-file=", "-o" etc.
+using LookupModuleOutputCallback = llvm::function_ref<std::string(
+    const clang::tooling::dependencies::ModuleDeps &,
+    clang::tooling::dependencies::ModuleOutputKind)>;
+using RemapPathCallback = llvm::function_ref<std::string(StringRef)>;
+
 namespace dependencies {
 std::string createEncodedModuleKindAndName(ModuleDependencyID id);
 bool checkImportNotTautological(const ImportPath::Module, const SourceLoc,
@@ -1023,6 +1030,8 @@ using ModuleNameToDependencyMap = llvm::StringMap<ModuleDependencyInfo>;
 using ModuleDependenciesKindMap =
     std::unordered_map<ModuleDependencyKind, ModuleNameToDependencyMap,
                        ModuleDependencyKindHash>;
+using BridgeClangDependencyCallback = llvm::function_ref<ModuleDependencyInfo(
+    const clang::tooling::dependencies::ModuleDeps &clangModuleDep)>;
 
 // MARK: SwiftDependencyScanningService
 /// A carrier of state shared among possibly multiple invocations of the
@@ -1173,8 +1182,10 @@ class ModuleDependenciesCache {
                         ModuleDependencyInfo dependencies);
 
   /// Record dependencies for the given collection of Clang modules.
-  void recordClangDependencies(ModuleDependencyVector moduleDependencies,
-                               DiagnosticEngine &diags);
+  void recordClangDependencies(
+      const clang::tooling::dependencies::ModuleDepsGraph &dependencies,
+      DiagnosticEngine &diags,
+      BridgeClangDependencyCallback bridgeClangModule);
 
   /// Update stored dependencies for the given module.
   void updateDependency(ModuleDependencyID moduleID,

include/swift/ClangImporter/ClangImporter.h

@@ -492,19 +492,6 @@ class ClangImporter final : public ClangModuleLoader {
 
   void verifyAllModules() override;
 
-  using RemapPathCallback = llvm::function_ref<std::string(StringRef)>;
-  using LookupModuleOutputCallback =
-      llvm::function_ref<std::string(const clang::tooling::dependencies::ModuleDeps &,
-                                     clang::tooling::dependencies::ModuleOutputKind)>;
-
-  static llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
-  bridgeClangModuleDependencies(
-      const ASTContext &ctx,
-      clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
-      clang::tooling::dependencies::ModuleDepsGraph &clangDependencies,
-      LookupModuleOutputCallback LookupModuleOutput,
-      RemapPathCallback remapPath = nullptr);
-
   static void getBridgingHeaderOptions(
       const ASTContext &ctx,
       const clang::tooling::dependencies::TranslationUnitDeps &deps,

include/swift/DependencyScan/ModuleDependencyScanner.h

@@ -40,41 +40,26 @@ class ModuleDependencyScanningWorker {
 
 private:
   /// Retrieve the module dependencies for the Clang module with the given name.
-  ClangModuleScannerQueryResult scanFilesystemForClangModuleDependency(
+  std::optional<clang::tooling::dependencies::TranslationUnitDeps>
+  scanFilesystemForClangModuleDependency(
       Identifier moduleName,
+      LookupModuleOutputCallback lookupModuleCallback,
       const llvm::DenseSet<clang::tooling::dependencies::ModuleID>
           &alreadySeenModules);
 
-  /// Retrieve the module dependencies for the Swift module with the given name.
-  SwiftModuleScannerQueryResult scanFilesystemForSwiftModuleDependency(
-      Identifier moduleName, bool isTestableImport = false);
-
   /// Query dependency information for header dependencies
-  /// of a binary Swift module.
-  ///
-  /// \param moduleID the name of the Swift module whose dependency
-  /// information will be augmented with information about the given
-  /// textual header inputs.
-  ///
-  /// \param headerPath the path to the header to be scanned.
-  ///
-  /// \param clangScanningTool The clang dependency scanner.
-  ///
-  /// \param cache The module dependencies cache to update, with information
-  /// about new Clang modules discovered along the way.
-  ///
-  /// \returns \c true if an error occurred, \c false otherwise
-  bool scanHeaderDependenciesOfSwiftModule(
-      const ASTContext &ctx,
+  /// of a Swift module.
+  std::optional<clang::tooling::dependencies::TranslationUnitDeps>
+  scanHeaderDependenciesOfSwiftModule(
       ModuleDependencyID moduleID, std::optional<StringRef> headerPath,
       std::optional<llvm::MemoryBufferRef> sourceBuffer,
-      ModuleDependenciesCache &cache,
-      ModuleDependencyIDSetVector &headerClangModuleDependencies,
-      std::vector<std::string> &headerFileInputs,
-      std::vector<std::string> &bridgingHeaderCommandLine,
-      std::vector<std::string> &visibleClangModules,
-      std::optional<std::string> &includeTreeID);
+      LookupModuleOutputCallback lookupModuleCallback,
+      const llvm::DenseSet<clang::tooling::dependencies::ModuleID>
+         &alreadySeenModules);
 
+  /// Retrieve the module dependencies for the Swift module with the given name.
+  SwiftModuleScannerQueryResult scanFilesystemForSwiftModuleDependency(
+      Identifier moduleName, bool isTestableImport = false);
 
   /// Store cache entry for include tree.
   llvm::Error
@@ -92,16 +77,9 @@ class ModuleDependencyScanningWorker {
   // Swift and Clang module loaders acting as scanners.
   std::unique_ptr<SwiftModuleScanner> swiftModuleScannerLoader;
 
-  /// The location of where the explicitly-built modules will be output to
-  std::string moduleOutputPath;
-  /// The location of where the explicitly-built SDK modules will be output to
-  std::string sdkModuleOutputPath;
-
   // CAS instance.
   std::shared_ptr<llvm::cas::ObjectStore> CAS;
   std::shared_ptr<llvm::cas::ActionCache> ActionCache;
-  /// File prefix mapper.
-  llvm::PrefixMapper *PrefixMapper;
 
   // Base command line invocation for clang scanner queries (both module and header)
   std::vector<std::string> clangScanningBaseCommandLineArgs;
@@ -222,11 +200,7 @@ class ModuleDependencyScanner {
     return PrefixMapper && !PrefixMapper->getMappings().empty();
   }
   llvm::PrefixMapper *getPrefixMapper() const { return PrefixMapper.get(); }
-  std::string remapPath(StringRef Path) const {
-    if (!PrefixMapper)
-      return Path.str();
-    return PrefixMapper->mapToString(Path);
-  }
+  std::string remapPath(StringRef Path) const;
 
   /// CAS options.
   llvm::cas::ObjectStore &getCAS() const {
@@ -293,11 +267,22 @@ class ModuleDependencyScanner {
                                 ModuleDependenciesCache &cache,
                                 ModuleDependencyIDSetVector &allModules);
 
+  /// Bridge Clang dependency scanner's dependency node
+  /// to the Swift scanner's `ModuleDependencyInfo`.
+  ModuleDependencyInfo
+  bridgeClangModuleDependency(
+      const clang::tooling::dependencies::ModuleDeps &clangDependency);
+
   /// Perform an operation utilizing one of the Scanning workers
   /// available to this scanner.
   template <typename Function, typename... Args>
   auto withDependencyScanningWorker(Function &&F, Args &&...ArgList);
 
+  /// Determine cache-relative output path for a given Clang module
+  std::string clangModuleOutputPathLookup(
+      const clang::tooling::dependencies::ModuleDeps &clangDep,
+      clang::tooling::dependencies::ModuleOutputKind moduleOutputKind) const;
+
   /// Use the scanner's ASTContext to construct an `Identifier`
   /// for a given module name.
   Identifier getModuleImportIdentifier(StringRef moduleName);
@@ -317,6 +302,11 @@ class ModuleDependencyScanner {
   ASTContext &ScanASTContext;
   ModuleDependencyIssueReporter IssueReporter;
 
+  /// The location of where the explicitly-built modules will be output to
+  std::string ModuleOutputPath;
+  /// The location of where the explicitly-built SDK modules will be output to
+  std::string SDKModuleOutputPath;
+
   /// The available pool of workers for filesystem module search
   unsigned NumThreads;
   std::list<std::unique_ptr<ModuleDependencyScanningWorker>> Workers;

include/swift/Serialization/ScanningLoaders.h

@@ -40,18 +40,6 @@ struct SwiftModuleScannerQueryResult {
   std::vector<IncompatibleCandidate> incompatibleCandidates;
 };
 
-/// Result of looking up a Clang module on the current filesystem
-/// search paths.
-struct ClangModuleScannerQueryResult {
-  ClangModuleScannerQueryResult(const ModuleDependencyVector &dependencyModuleGraph,
-                                const std::vector<std::string> &visibleModuleIdentifiers)
-      : foundDependencyModuleGraph(dependencyModuleGraph),
-        visibleModuleIdentifiers(visibleModuleIdentifiers) {}
-
-  ModuleDependencyVector foundDependencyModuleGraph;
-  std::vector<std::string> visibleModuleIdentifiers;
-};
-
 /// A module "loader" that looks for .swiftinterface and .swiftmodule files
 /// for the purpose of determining dependencies, but does not attempt to
 /// load the module files.
@@ -89,10 +77,6 @@ class SwiftModuleScanner : public SerializedModuleLoaderBase {
 
   /// AST delegate to be used for textual interface scanning
   InterfaceSubContextDelegate &astDelegate;
-  /// Location where pre-built modules are to be built into.
-  std::string moduleOutputPath;
-  /// Location where pre-built SDK modules are to be built into.
-  std::string sdkModuleOutputPath;
   /// Clang-specific (-Xcc) command-line flags to include on
   /// Swift module compilation commands
   std::vector<std::string> swiftModuleClangCC1CommandLineArgs;
@@ -108,14 +92,12 @@ class SwiftModuleScanner : public SerializedModuleLoaderBase {
 public:
   SwiftModuleScanner(
       ASTContext &ctx, ModuleLoadingMode LoadMode,
-      InterfaceSubContextDelegate &astDelegate, StringRef moduleOutputPath,
-      StringRef sdkModuleOutputPath,
+      InterfaceSubContextDelegate &astDelegate,
       std::vector<std::string> swiftModuleClangCC1CommandLineArgs,
       llvm::StringMap<std::string> &explicitSwiftModuleInputs)
       : SerializedModuleLoaderBase(ctx, nullptr, LoadMode,
                                    /*IgnoreSwiftSourceInfoFile=*/true),
-        astDelegate(astDelegate), moduleOutputPath(moduleOutputPath),
-        sdkModuleOutputPath(sdkModuleOutputPath),
+        astDelegate(astDelegate),
         swiftModuleClangCC1CommandLineArgs(swiftModuleClangCC1CommandLineArgs),
         explicitSwiftModuleInputs(explicitSwiftModuleInputs) {
   }

lib/AST/ModuleDependencies.cpp

@@ -23,6 +23,7 @@
 #include "swift/AST/SourceFile.h"
 #include "swift/Frontend/Frontend.h"
 #include "swift/Strings.h"
+#include "clang/Lex/HeaderSearchOptions.h"
 #include "llvm/Config/config.h"
 #include "llvm/Support/Path.h"
 using namespace swift;
@@ -779,16 +780,18 @@ void ModuleDependenciesCache::recordDependency(
 }
 
 void ModuleDependenciesCache::recordClangDependencies(
-    ModuleDependencyVector dependencies, DiagnosticEngine &diags) {
+    const clang::tooling::dependencies::ModuleDepsGraph &dependencies,
+    DiagnosticEngine &diags,
+    BridgeClangDependencyCallback bridgeClangModule) {
   for (const auto &dep : dependencies) {
-    ASSERT(dep.first.Kind == ModuleDependencyKind::Clang);
-    auto newClangModuleDetails = dep.second.getAsClangModule();
-    if (hasDependency(dep.first)) {
+    auto depID =
+        ModuleDependencyID{dep.ID.ModuleName, ModuleDependencyKind::Clang};
+    if (hasDependency(depID)) {
       auto priorClangModuleDetails =
-          findKnownDependency(dep.first).getAsClangModule();
-      DEBUG_ASSERT(priorClangModuleDetails && newClangModuleDetails);
+          findKnownDependency(depID).getAsClangModule();
+      DEBUG_ASSERT(priorClangModuleDetails);
       auto priorContextHash = priorClangModuleDetails->contextHash;
-      auto newContextHash = newClangModuleDetails->contextHash;
+      auto newContextHash = dep.ID.ContextHash;
       if (priorContextHash != newContextHash) {
         // This situation means that within the same scanning action, Clang
         // Dependency Scanner has produced two different variants of the same
@@ -799,22 +802,23 @@ void ModuleDependenciesCache::recordClangDependencies(
         //
         // Emit a failure diagnostic here that is hopefully more actionable
         // for the time being.
-        diags.diagnose(SourceLoc(), diag::dependency_scan_unexpected_variant,
-                       dep.first.ModuleName);
+        diags.diagnose(SourceLoc(),
+                           diag::dependency_scan_unexpected_variant,
+                           dep.ID.ModuleName);
         diags.diagnose(
             SourceLoc(),
             diag::dependency_scan_unexpected_variant_context_hash_note,
             priorContextHash, newContextHash);
         diags.diagnose(
             SourceLoc(),
             diag::dependency_scan_unexpected_variant_module_map_note,
-            priorClangModuleDetails->moduleMapFile,
-            newClangModuleDetails->moduleMapFile);
+            priorClangModuleDetails->moduleMapFile, dep.ClangModuleMapFile);
 
+        auto newClangModuleDetails = bridgeClangModule(dep).getAsClangModule();
         auto diagnoseExtraCommandLineFlags =
             [&diags](const ClangModuleDependencyStorage *checkModuleDetails,
-                     const ClangModuleDependencyStorage *baseModuleDetails,
-                     bool isNewlyDiscovered) -> void {
+                   const ClangModuleDependencyStorage *baseModuleDetails,
+                   bool isNewlyDiscovered) -> void {
           std::unordered_set<std::string> baseCommandLineSet(
               baseModuleDetails->buildCommandLine.begin(),
               baseModuleDetails->buildCommandLine.end());
@@ -831,9 +835,8 @@ void ModuleDependenciesCache::recordClangDependencies(
                                       priorClangModuleDetails, false);
       }
     } else {
-      recordDependency(dep.first.ModuleName, dep.second);
-      addSeenClangModule(clang::tooling::dependencies::ModuleID{
-        dep.first.ModuleName, newClangModuleDetails->contextHash});
+      recordDependency(dep.ID.ModuleName, bridgeClangModule(dep));
+      addSeenClangModule(dep.ID);
     }
   }
 }