diff --git a/include/swift/AST/ModuleDependencies.h b/include/swift/AST/ModuleDependencies.h index bbb6235a95004..6356c937e597f 100644 --- a/include/swift/AST/ModuleDependencies.h +++ b/include/swift/AST/ModuleDependencies.h @@ -26,6 +26,7 @@ #include "swift/Serialization/Validation.h" #include "clang/CAS/CASOptions.h" #include "clang/Tooling/DependencyScanning/DependencyScanningService.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseSet.h" @@ -163,6 +164,12 @@ using ModuleDependencyIDSetVector = llvm::SetVector, std::set>; +/// A callback to lookup module outputs for "-fmodule-file=", "-o" etc. +using LookupModuleOutputCallback = llvm::function_ref; +using RemapPathCallback = llvm::function_ref; + namespace dependencies { std::string createEncodedModuleKindAndName(ModuleDependencyID id); bool checkImportNotTautological(const ImportPath::Module, const SourceLoc, @@ -1023,6 +1030,8 @@ using ModuleNameToDependencyMap = llvm::StringMap; using ModuleDependenciesKindMap = std::unordered_map; +using BridgeClangDependencyCallback = llvm::function_ref; // MARK: SwiftDependencyScanningService /// A carrier of state shared among possibly multiple invocations of the @@ -1173,8 +1182,10 @@ class ModuleDependenciesCache { ModuleDependencyInfo dependencies); /// Record dependencies for the given collection of Clang modules. - void recordClangDependencies(ModuleDependencyVector moduleDependencies, - DiagnosticEngine &diags); + void recordClangDependencies( + const clang::tooling::dependencies::ModuleDepsGraph &dependencies, + DiagnosticEngine &diags, + BridgeClangDependencyCallback bridgeClangModule); /// Update stored dependencies for the given module. void updateDependency(ModuleDependencyID moduleID, diff --git a/include/swift/ClangImporter/ClangImporter.h b/include/swift/ClangImporter/ClangImporter.h index a20c0505c1dc0..a23e05124fd7f 100644 --- a/include/swift/ClangImporter/ClangImporter.h +++ b/include/swift/ClangImporter/ClangImporter.h @@ -492,19 +492,6 @@ class ClangImporter final : public ClangModuleLoader { void verifyAllModules() override; - using RemapPathCallback = llvm::function_ref; - using LookupModuleOutputCallback = - llvm::function_ref; - - static llvm::SmallVector, 1> - bridgeClangModuleDependencies( - const ASTContext &ctx, - clang::tooling::dependencies::DependencyScanningTool &clangScanningTool, - clang::tooling::dependencies::ModuleDepsGraph &clangDependencies, - LookupModuleOutputCallback LookupModuleOutput, - RemapPathCallback remapPath = nullptr); - static void getBridgingHeaderOptions( const ASTContext &ctx, const clang::tooling::dependencies::TranslationUnitDeps &deps, diff --git a/include/swift/DependencyScan/ModuleDependencyScanner.h b/include/swift/DependencyScan/ModuleDependencyScanner.h index 9fbfd48b51e4e..7bd2c9ff6c9e5 100644 --- a/include/swift/DependencyScan/ModuleDependencyScanner.h +++ b/include/swift/DependencyScan/ModuleDependencyScanner.h @@ -40,41 +40,26 @@ class ModuleDependencyScanningWorker { private: /// Retrieve the module dependencies for the Clang module with the given name. - ClangModuleScannerQueryResult scanFilesystemForClangModuleDependency( + std::optional + scanFilesystemForClangModuleDependency( Identifier moduleName, + LookupModuleOutputCallback lookupModuleCallback, const llvm::DenseSet &alreadySeenModules); - /// Retrieve the module dependencies for the Swift module with the given name. - SwiftModuleScannerQueryResult scanFilesystemForSwiftModuleDependency( - Identifier moduleName, bool isTestableImport = false); - /// Query dependency information for header dependencies - /// of a binary Swift module. - /// - /// \param moduleID the name of the Swift module whose dependency - /// information will be augmented with information about the given - /// textual header inputs. - /// - /// \param headerPath the path to the header to be scanned. - /// - /// \param clangScanningTool The clang dependency scanner. - /// - /// \param cache The module dependencies cache to update, with information - /// about new Clang modules discovered along the way. - /// - /// \returns \c true if an error occurred, \c false otherwise - bool scanHeaderDependenciesOfSwiftModule( - const ASTContext &ctx, + /// of a Swift module. + std::optional + scanHeaderDependenciesOfSwiftModule( ModuleDependencyID moduleID, std::optional headerPath, std::optional sourceBuffer, - ModuleDependenciesCache &cache, - ModuleDependencyIDSetVector &headerClangModuleDependencies, - std::vector &headerFileInputs, - std::vector &bridgingHeaderCommandLine, - std::vector &visibleClangModules, - std::optional &includeTreeID); + LookupModuleOutputCallback lookupModuleCallback, + const llvm::DenseSet + &alreadySeenModules); + /// Retrieve the module dependencies for the Swift module with the given name. + SwiftModuleScannerQueryResult scanFilesystemForSwiftModuleDependency( + Identifier moduleName, bool isTestableImport = false); /// Store cache entry for include tree. llvm::Error @@ -92,16 +77,9 @@ class ModuleDependencyScanningWorker { // Swift and Clang module loaders acting as scanners. std::unique_ptr swiftModuleScannerLoader; - /// The location of where the explicitly-built modules will be output to - std::string moduleOutputPath; - /// The location of where the explicitly-built SDK modules will be output to - std::string sdkModuleOutputPath; - // CAS instance. std::shared_ptr CAS; std::shared_ptr ActionCache; - /// File prefix mapper. - llvm::PrefixMapper *PrefixMapper; // Base command line invocation for clang scanner queries (both module and header) std::vector clangScanningBaseCommandLineArgs; @@ -222,11 +200,7 @@ class ModuleDependencyScanner { return PrefixMapper && !PrefixMapper->getMappings().empty(); } llvm::PrefixMapper *getPrefixMapper() const { return PrefixMapper.get(); } - std::string remapPath(StringRef Path) const { - if (!PrefixMapper) - return Path.str(); - return PrefixMapper->mapToString(Path); - } + std::string remapPath(StringRef Path) const; /// CAS options. llvm::cas::ObjectStore &getCAS() const { @@ -293,11 +267,22 @@ class ModuleDependencyScanner { ModuleDependenciesCache &cache, ModuleDependencyIDSetVector &allModules); + /// Bridge Clang dependency scanner's dependency node + /// to the Swift scanner's `ModuleDependencyInfo`. + ModuleDependencyInfo + bridgeClangModuleDependency( + const clang::tooling::dependencies::ModuleDeps &clangDependency); + /// Perform an operation utilizing one of the Scanning workers /// available to this scanner. template auto withDependencyScanningWorker(Function &&F, Args &&...ArgList); + /// Determine cache-relative output path for a given Clang module + std::string clangModuleOutputPathLookup( + const clang::tooling::dependencies::ModuleDeps &clangDep, + clang::tooling::dependencies::ModuleOutputKind moduleOutputKind) const; + /// Use the scanner's ASTContext to construct an `Identifier` /// for a given module name. Identifier getModuleImportIdentifier(StringRef moduleName); @@ -317,6 +302,11 @@ class ModuleDependencyScanner { ASTContext &ScanASTContext; ModuleDependencyIssueReporter IssueReporter; + /// The location of where the explicitly-built modules will be output to + std::string ModuleOutputPath; + /// The location of where the explicitly-built SDK modules will be output to + std::string SDKModuleOutputPath; + /// The available pool of workers for filesystem module search unsigned NumThreads; std::list> Workers; diff --git a/include/swift/Serialization/ScanningLoaders.h b/include/swift/Serialization/ScanningLoaders.h index f80c80c0fe4cf..1799d7262f803 100644 --- a/include/swift/Serialization/ScanningLoaders.h +++ b/include/swift/Serialization/ScanningLoaders.h @@ -40,18 +40,6 @@ struct SwiftModuleScannerQueryResult { std::vector incompatibleCandidates; }; -/// Result of looking up a Clang module on the current filesystem -/// search paths. -struct ClangModuleScannerQueryResult { - ClangModuleScannerQueryResult(const ModuleDependencyVector &dependencyModuleGraph, - const std::vector &visibleModuleIdentifiers) - : foundDependencyModuleGraph(dependencyModuleGraph), - visibleModuleIdentifiers(visibleModuleIdentifiers) {} - - ModuleDependencyVector foundDependencyModuleGraph; - std::vector visibleModuleIdentifiers; -}; - /// A module "loader" that looks for .swiftinterface and .swiftmodule files /// for the purpose of determining dependencies, but does not attempt to /// load the module files. @@ -89,10 +77,6 @@ class SwiftModuleScanner : public SerializedModuleLoaderBase { /// AST delegate to be used for textual interface scanning InterfaceSubContextDelegate &astDelegate; - /// Location where pre-built modules are to be built into. - std::string moduleOutputPath; - /// Location where pre-built SDK modules are to be built into. - std::string sdkModuleOutputPath; /// Clang-specific (-Xcc) command-line flags to include on /// Swift module compilation commands std::vector swiftModuleClangCC1CommandLineArgs; @@ -108,14 +92,12 @@ class SwiftModuleScanner : public SerializedModuleLoaderBase { public: SwiftModuleScanner( ASTContext &ctx, ModuleLoadingMode LoadMode, - InterfaceSubContextDelegate &astDelegate, StringRef moduleOutputPath, - StringRef sdkModuleOutputPath, + InterfaceSubContextDelegate &astDelegate, std::vector swiftModuleClangCC1CommandLineArgs, llvm::StringMap &explicitSwiftModuleInputs) : SerializedModuleLoaderBase(ctx, nullptr, LoadMode, /*IgnoreSwiftSourceInfoFile=*/true), - astDelegate(astDelegate), moduleOutputPath(moduleOutputPath), - sdkModuleOutputPath(sdkModuleOutputPath), + astDelegate(astDelegate), swiftModuleClangCC1CommandLineArgs(swiftModuleClangCC1CommandLineArgs), explicitSwiftModuleInputs(explicitSwiftModuleInputs) { } diff --git a/lib/AST/ModuleDependencies.cpp b/lib/AST/ModuleDependencies.cpp index 1ac5bd6543267..4df96ab8eac5a 100644 --- a/lib/AST/ModuleDependencies.cpp +++ b/lib/AST/ModuleDependencies.cpp @@ -23,6 +23,7 @@ #include "swift/AST/SourceFile.h" #include "swift/Frontend/Frontend.h" #include "swift/Strings.h" +#include "clang/Lex/HeaderSearchOptions.h" #include "llvm/Config/config.h" #include "llvm/Support/Path.h" using namespace swift; @@ -779,16 +780,18 @@ void ModuleDependenciesCache::recordDependency( } void ModuleDependenciesCache::recordClangDependencies( - ModuleDependencyVector dependencies, DiagnosticEngine &diags) { + const clang::tooling::dependencies::ModuleDepsGraph &dependencies, + DiagnosticEngine &diags, + BridgeClangDependencyCallback bridgeClangModule) { for (const auto &dep : dependencies) { - ASSERT(dep.first.Kind == ModuleDependencyKind::Clang); - auto newClangModuleDetails = dep.second.getAsClangModule(); - if (hasDependency(dep.first)) { + auto depID = + ModuleDependencyID{dep.ID.ModuleName, ModuleDependencyKind::Clang}; + if (hasDependency(depID)) { auto priorClangModuleDetails = - findKnownDependency(dep.first).getAsClangModule(); - DEBUG_ASSERT(priorClangModuleDetails && newClangModuleDetails); + findKnownDependency(depID).getAsClangModule(); + DEBUG_ASSERT(priorClangModuleDetails); auto priorContextHash = priorClangModuleDetails->contextHash; - auto newContextHash = newClangModuleDetails->contextHash; + auto newContextHash = dep.ID.ContextHash; if (priorContextHash != newContextHash) { // This situation means that within the same scanning action, Clang // Dependency Scanner has produced two different variants of the same @@ -799,8 +802,9 @@ void ModuleDependenciesCache::recordClangDependencies( // // Emit a failure diagnostic here that is hopefully more actionable // for the time being. - diags.diagnose(SourceLoc(), diag::dependency_scan_unexpected_variant, - dep.first.ModuleName); + diags.diagnose(SourceLoc(), + diag::dependency_scan_unexpected_variant, + dep.ID.ModuleName); diags.diagnose( SourceLoc(), diag::dependency_scan_unexpected_variant_context_hash_note, @@ -808,13 +812,13 @@ void ModuleDependenciesCache::recordClangDependencies( diags.diagnose( SourceLoc(), diag::dependency_scan_unexpected_variant_module_map_note, - priorClangModuleDetails->moduleMapFile, - newClangModuleDetails->moduleMapFile); + priorClangModuleDetails->moduleMapFile, dep.ClangModuleMapFile); + auto newClangModuleDetails = bridgeClangModule(dep).getAsClangModule(); auto diagnoseExtraCommandLineFlags = [&diags](const ClangModuleDependencyStorage *checkModuleDetails, - const ClangModuleDependencyStorage *baseModuleDetails, - bool isNewlyDiscovered) -> void { + const ClangModuleDependencyStorage *baseModuleDetails, + bool isNewlyDiscovered) -> void { std::unordered_set baseCommandLineSet( baseModuleDetails->buildCommandLine.begin(), baseModuleDetails->buildCommandLine.end()); @@ -831,9 +835,8 @@ void ModuleDependenciesCache::recordClangDependencies( priorClangModuleDetails, false); } } else { - recordDependency(dep.first.ModuleName, dep.second); - addSeenClangModule(clang::tooling::dependencies::ModuleID{ - dep.first.ModuleName, newClangModuleDetails->contextHash}); + recordDependency(dep.ID.ModuleName, bridgeClangModule(dep)); + addSeenClangModule(dep.ID); } } } diff --git a/lib/ClangImporter/ClangModuleDependencyScanner.cpp b/lib/ClangImporter/ClangModuleDependencyScanner.cpp index e10edaf691364..6eb7e99d9ac76 100644 --- a/lib/ClangImporter/ClangModuleDependencyScanner.cpp +++ b/lib/ClangImporter/ClangModuleDependencyScanner.cpp @@ -82,126 +82,6 @@ std::vector ClangImporter::getClangDepScanningInvocationArguments( return commandLineArgs; } -ModuleDependencyVector ClangImporter::bridgeClangModuleDependencies( - const ASTContext &ctx, - clang::tooling::dependencies::DependencyScanningTool &clangScanningTool, - clang::tooling::dependencies::ModuleDepsGraph &clangDependencies, - LookupModuleOutputCallback lookupModuleOutput, - RemapPathCallback callback) { - ModuleDependencyVector result; - - auto remapPath = [&](StringRef path) { - if (callback) - return callback(path); - return path.str(); - }; - - for (auto &clangModuleDep : clangDependencies) { - // File dependencies for this module. - std::vector fileDeps; - clangModuleDep.forEachFileDep( - [&fileDeps](StringRef fileDep) { fileDeps.emplace_back(fileDep); }); - - std::vector swiftArgs; - auto addClangArg = [&](Twine arg) { - swiftArgs.push_back("-Xcc"); - swiftArgs.push_back(arg.str()); - }; - - // We are using Swift frontend mode. - swiftArgs.push_back("-frontend"); - - // Swift frontend action: -emit-pcm - swiftArgs.push_back("-emit-pcm"); - swiftArgs.push_back("-module-name"); - swiftArgs.push_back(clangModuleDep.ID.ModuleName); - - auto pcmPath = lookupModuleOutput(clangModuleDep, - ModuleOutputKind::ModuleFile); - swiftArgs.push_back("-o"); - swiftArgs.push_back(pcmPath); - - // Ensure that the resulting PCM build invocation uses Clang frontend - // directly - swiftArgs.push_back("-direct-clang-cc1-module-build"); - - // Swift frontend option for input file path (Foo.modulemap). - swiftArgs.push_back(remapPath(clangModuleDep.ClangModuleMapFile)); - - auto invocation = clangModuleDep.getUnderlyingCompilerInvocation(); - // Clear some options from clang scanner. - invocation.getMutFrontendOpts().ModuleCacheKeys.clear(); - invocation.getMutFrontendOpts().PathPrefixMappings.clear(); - invocation.getMutFrontendOpts().OutputFile.clear(); - - // Reset CASOptions since that should be coming from swift. - invocation.getMutCASOpts() = clang::CASOptions(); - invocation.getMutFrontendOpts().CASIncludeTreeID.clear(); - - // FIXME: workaround for rdar://105684525: find the -ivfsoverlay option - // from clang scanner and pass to swift. - if (!ctx.CASOpts.EnableCaching) { - auto &overlayFiles = invocation.getMutHeaderSearchOpts().VFSOverlayFiles; - for (auto overlay : overlayFiles) { - swiftArgs.push_back("-vfsoverlay"); - swiftArgs.push_back(overlay); - } - } - - // Add args reported by the scanner. - auto clangArgs = invocation.getCC1CommandLine(); - llvm::for_each(clangArgs, addClangArg); - - // CASFileSystemRootID. - std::string RootID = clangModuleDep.CASFileSystemRootID - ? clangModuleDep.CASFileSystemRootID->toString() - : ""; - - std::string IncludeTree = - clangModuleDep.IncludeTreeID ? *clangModuleDep.IncludeTreeID : ""; - - ctx.CASOpts.enumerateCASConfigurationFlags( - [&](StringRef Arg) { swiftArgs.push_back(Arg.str()); }); - - if (!IncludeTree.empty()) { - swiftArgs.push_back("-clang-include-tree-root"); - swiftArgs.push_back(IncludeTree); - } - std::string mappedPCMPath = remapPath(pcmPath); - - std::vector LinkLibraries; - for (const auto &ll : clangModuleDep.LinkLibraries) - LinkLibraries.emplace_back( - ll.Library, - ll.IsFramework ? LibraryKind::Framework : LibraryKind::Library, - /*static=*/false); - - // Module-level dependencies. - llvm::StringSet<> alreadyAddedModules; - auto dependencies = ModuleDependencyInfo::forClangModule( - pcmPath, mappedPCMPath, clangModuleDep.ClangModuleMapFile, - clangModuleDep.ID.ContextHash, swiftArgs, fileDeps, - LinkLibraries, RootID, IncludeTree, /*module-cache-key*/ "", - clangModuleDep.IsSystem); - - std::vector directDependencyIDs; - for (const auto &moduleName : clangModuleDep.ClangModuleDeps) { - // FIXME: This assumes, conservatively, that all Clang module imports - // are exported. We need to fix this once the clang scanner gains the appropriate - // API to query this. - dependencies.addModuleImport(moduleName.ModuleName, /* isExported */ true, - AccessLevel::Public, &alreadyAddedModules); - // It is safe to assume that all dependencies of a Clang module are Clang modules. - directDependencyIDs.push_back({moduleName.ModuleName, ModuleDependencyKind::Clang}); - } - dependencies.setImportedClangDependencies(directDependencyIDs); - result.push_back(std::make_pair(ModuleDependencyID{clangModuleDep.ID.ModuleName, - ModuleDependencyKind::Clang}, - dependencies)); - } - return result; -} - void ClangImporter::getBridgingHeaderOptions( const ASTContext &ctx, const clang::tooling::dependencies::TranslationUnitDeps &deps, diff --git a/lib/DependencyScan/ModuleDependencyScanner.cpp b/lib/DependencyScan/ModuleDependencyScanner.cpp index 5bc5166c1cd33..0007d1f17ad2b 100644 --- a/lib/DependencyScan/ModuleDependencyScanner.cpp +++ b/lib/DependencyScan/ModuleDependencyScanner.cpp @@ -137,28 +137,28 @@ computeClangWorkingDirectory(const std::vector &commandLineArgs, return workingDir; } -static std::string moduleCacheRelativeLookupModuleOutput( - const clang::tooling::dependencies::ModuleDeps &MD, - clang::tooling::dependencies::ModuleOutputKind MOK, - const StringRef moduleCachePath, const StringRef stableModuleCachePath, - const StringRef runtimeResourcePath) { - llvm::SmallString<128> outputPath(moduleCachePath); - if (MD.IsInStableDirectories) - outputPath = stableModuleCachePath; +std::string ModuleDependencyScanner::clangModuleOutputPathLookup( + const clang::tooling::dependencies::ModuleDeps &clangDeps, + clang::tooling::dependencies::ModuleOutputKind moduleOutputKind) const { + llvm::SmallString<128> outputPath(ModuleOutputPath); + if (clangDeps.IsInStableDirectories) + outputPath = SDKModuleOutputPath; + + auto runtimeResourcePath = ScanASTContext.SearchPathOpts.RuntimeResourcePath; // FIXME: This is a hack to treat Clang modules defined in the compiler's // own resource directory as stable, when they are not reported as such // by the Clang scanner. if (!runtimeResourcePath.empty() && - hasPrefix(llvm::sys::path::begin(MD.ClangModuleMapFile), - llvm::sys::path::end(MD.ClangModuleMapFile), + hasPrefix(llvm::sys::path::begin(clangDeps.ClangModuleMapFile), + llvm::sys::path::end(clangDeps.ClangModuleMapFile), llvm::sys::path::begin(runtimeResourcePath), llvm::sys::path::end(runtimeResourcePath))) - outputPath = stableModuleCachePath; + outputPath = SDKModuleOutputPath; - llvm::sys::path::append(outputPath, - MD.ID.ModuleName + "-" + MD.ID.ContextHash); - switch (MOK) { + llvm::sys::path::append(outputPath, clangDeps.ID.ModuleName + "-" + + clangDeps.ID.ContextHash); + switch (moduleOutputKind) { case clang::tooling::dependencies::ModuleOutputKind::ModuleFile: llvm::sys::path::replace_extension( outputPath, getExtension(swift::file_types::TY_ClangModuleFile)); @@ -168,7 +168,7 @@ static std::string moduleCacheRelativeLookupModuleOutput( outputPath, getExtension(swift::file_types::TY_Dependencies)); break; case clang::tooling::dependencies::ModuleOutputKind::DependencyTargets: - return MD.ID.ModuleName + "-" + MD.ID.ContextHash; + return clangDeps.ID.ModuleName + "-" + clangDeps.ID.ContextHash; case clang::tooling::dependencies::ModuleOutputKind:: DiagnosticSerializationFile: llvm::sys::path::replace_extension( @@ -193,12 +193,6 @@ static std::vector inputSpecificClangScannerCommand( return result; } -static std::string remapPath(llvm::PrefixMapper *PrefixMapper, StringRef Path) { - if (!PrefixMapper) - return Path.str(); - return PrefixMapper->mapToString(Path); -} - static llvm::IntrusiveRefCntPtr getClangScanningFS(std::shared_ptr cas, ASTContext &ctx) { @@ -251,11 +245,7 @@ ModuleDependencyScanningWorker::ModuleDependencyScanningWorker( RequireOSSAModules_t(SILOptions))), clangScanningTool(*globalScanningService.ClangScanningService, getClangScanningFS(CAS, ScanASTContext)), - moduleOutputPath(workerCompilerInvocation->getFrontendOptions() - .ExplicitModulesOutputPath), - sdkModuleOutputPath(workerCompilerInvocation->getFrontendOptions() - .ExplicitSDKModulesOutputPath), - CAS(CAS), ActionCache(ActionCache), PrefixMapper(Mapper) { + CAS(CAS), ActionCache(ActionCache) { auto loader = std::make_unique( *workerASTContext, /*DepTracker=*/nullptr, workerCompilerInvocation->getFrontendOptions().CacheReplayPrefixMap, @@ -293,8 +283,7 @@ ModuleDependencyScanningWorker::ModuleDependencyScanningWorker( swiftModuleScannerLoader = std::make_unique( *workerASTContext, workerCompilerInvocation->getSearchPathOptions().ModuleLoadMode, - *scanningASTDelegate, moduleOutputPath, sdkModuleOutputPath, - swiftModuleClangCC1CommandLineArgs, + *scanningASTDelegate, swiftModuleClangCC1CommandLineArgs, workerCompilerInvocation->getSearchPathOptions().ExplicitSwiftModuleInputs); } @@ -305,20 +294,12 @@ ModuleDependencyScanningWorker::scanFilesystemForSwiftModuleDependency( isTestableImport); } -ClangModuleScannerQueryResult +std::optional ModuleDependencyScanningWorker::scanFilesystemForClangModuleDependency( Identifier moduleName, + LookupModuleOutputCallback lookupModuleOutput, const llvm::DenseSet &alreadySeenModules) { - auto lookupModuleOutput = - [this](const clang::tooling::dependencies::ModuleDeps &MD, - const clang::tooling::dependencies::ModuleOutputKind MOK) - -> std::string { - return moduleCacheRelativeLookupModuleOutput( - MD, MOK, moduleOutputPath, sdkModuleOutputPath, - workerASTContext->SearchPathOpts.RuntimeResourcePath); - }; - auto clangModuleDependencies = clangScanningTool.getModuleDependencies( moduleName.str(), clangScanningModuleCommandLineArgs, clangScanningWorkingDirectoryPath, alreadySeenModules, @@ -332,62 +313,30 @@ ModuleDependencyScanningWorker::scanFilesystemForClangModuleDependency( "' not found") == std::string::npos) workerASTContext->Diags.diagnose( SourceLoc(), diag::clang_dependency_scan_error, errorStr); - return ClangModuleScannerQueryResult({}, {}); + return std::nullopt; } - return ClangModuleScannerQueryResult( - ClangImporter::bridgeClangModuleDependencies( - *workerASTContext, clangScanningTool, - clangModuleDependencies->ModuleGraph, lookupModuleOutput, - [&](StringRef path) { return remapPath(PrefixMapper, path); }), - clangModuleDependencies->VisibleModules); + return clangModuleDependencies.get(); } -bool ModuleDependencyScanningWorker::scanHeaderDependenciesOfSwiftModule( - const ASTContext &ctx, ModuleDependencyID moduleID, +std::optional +ModuleDependencyScanningWorker::scanHeaderDependenciesOfSwiftModule( + ModuleDependencyID moduleID, std::optional headerPath, std::optional sourceBuffer, - ModuleDependenciesCache &cache, - ModuleDependencyIDSetVector &headerClangModuleDependencies, - std::vector &headerFileInputs, - std::vector &bridgingHeaderCommandLine, - std::vector &visibleClangModules, - std::optional &includeTreeID) { + LookupModuleOutputCallback lookupModuleOutput, + const llvm::DenseSet + &alreadySeenModules) { // Scan the specified textual header file and collect its dependencies auto scanHeaderDependencies = [&]() -> llvm::Expected { - auto lookupModuleOutput = - [this, &ctx](const clang::tooling::dependencies::ModuleDeps &MD, - const clang::tooling::dependencies::ModuleOutputKind MOK) - -> std::string { - return moduleCacheRelativeLookupModuleOutput( - MD, MOK, moduleOutputPath, sdkModuleOutputPath, - ctx.SearchPathOpts.RuntimeResourcePath); - }; - auto dependencies = clangScanningTool.getTranslationUnitDependencies( inputSpecificClangScannerCommand(clangScanningBaseCommandLineArgs, headerPath), - clangScanningWorkingDirectoryPath, cache.getAlreadySeenClangModules(), + clangScanningWorkingDirectoryPath, alreadySeenModules, lookupModuleOutput, sourceBuffer); if (!dependencies) return dependencies.takeError(); - - // Record module dependencies for each new module we found. - auto bridgedDeps = ClangImporter::bridgeClangModuleDependencies( - ctx, clangScanningTool, dependencies->ModuleGraph, lookupModuleOutput, - [this](StringRef path) { return remapPath(PrefixMapper, path); }); - cache.recordClangDependencies(bridgedDeps, ctx.Diags); - visibleClangModules = dependencies->VisibleModules; - - llvm::copy(dependencies->FileDeps, std::back_inserter(headerFileInputs)); - auto bridgedDependencyIDs = - llvm::map_range(dependencies->ClangModuleDeps, [](auto &input) { - return ModuleDependencyID{input.ModuleName, - ModuleDependencyKind::Clang}; - }); - headerClangModuleDependencies.insert(bridgedDependencyIDs.begin(), - bridgedDependencyIDs.end()); return dependencies; }; @@ -399,18 +348,10 @@ bool ModuleDependencyScanningWorker::scanHeaderDependenciesOfSwiftModule( auto errorStr = toString(clangModuleDependencies.takeError()); workerASTContext->Diags.diagnose( SourceLoc(), diag::clang_header_dependency_scan_error, errorStr); - return true; + return std::nullopt; } - auto targetModuleInfo = cache.findKnownDependency(moduleID); - if (!targetModuleInfo.isTextualSwiftModule()) - return false; - - if (auto TreeID = clangModuleDependencies->IncludeTreeID) - includeTreeID = TreeID; - ClangImporter::getBridgingHeaderOptions(ctx, *clangModuleDependencies, - bridgingHeaderCommandLine); - return false; + return *clangModuleDependencies; } template @@ -582,6 +523,10 @@ ModuleDependencyScanner::ModuleDependencyScanner( DiagnosticEngine &Diagnostics, bool ParallelScan) : ScanCompilerInvocation(ScanCompilerInvocation), ScanASTContext(ScanASTContext), IssueReporter(Diagnostics), + ModuleOutputPath(ScanCompilerInvocation.getFrontendOptions() + .ExplicitModulesOutputPath), + SDKModuleOutputPath(ScanCompilerInvocation.getFrontendOptions() + .ExplicitSDKModulesOutputPath), NumThreads(ParallelScan ? llvm::hardware_concurrency().compute_thread_count() : 1), @@ -612,11 +557,10 @@ ModuleDependencyScanner::ModuleDependencyScanner( /// Find all of the imported Clang modules starting with the given module name. static void findAllImportedClangModules(StringRef moduleName, const ModuleDependenciesCache &cache, - std::vector &allModules, - llvm::StringSet<> &knownModules) { - if (!knownModules.insert(moduleName).second) + llvm::StringSet<> &allModules) { + if (!allModules.insert(moduleName).second) return; - allModules.push_back(moduleName.str()); + auto moduleID = ModuleDependencyID{moduleName.str(), ModuleDependencyKind::Clang}; auto optionalDependencies = cache.findDependency(moduleID); @@ -624,8 +568,7 @@ static void findAllImportedClangModules(StringRef moduleName, return; for (const auto &dep : cache.getAllClangDependencies(moduleID)) - findAllImportedClangModules(dep.ModuleName, cache, allModules, - knownModules); + findAllImportedClangModules(dep.ModuleName, cache, allModules); } static std::set @@ -1117,18 +1060,21 @@ void ModuleDependencyScanner::resolveAllClangModuleDependencies( } // Module lookup result collection - llvm::StringMap moduleLookupResult; - const llvm::DenseSet - seenClangModules = cache.getAlreadySeenClangModules(); + llvm::StringMap< + std::optional> + moduleLookupResult; + auto seenClangModules = cache.getAlreadySeenClangModules(); std::mutex resultAccessLock; auto scanForClangModuleDependency = [this, &moduleLookupResult, &resultAccessLock, &seenClangModules]( Identifier moduleIdentifier) { auto scanResult = withDependencyScanningWorker( - [&seenClangModules, - moduleIdentifier](ModuleDependencyScanningWorker *ScanningWorker) { + [&](ModuleDependencyScanningWorker *ScanningWorker) { + auto lookupModuleOutput = [this](const auto &cd, auto mok) -> auto { + return clangModuleOutputPathLookup(cd, mok); + }; return ScanningWorker->scanFilesystemForClangModuleDependency( - moduleIdentifier, seenClangModules); + moduleIdentifier, lookupModuleOutput, seenClangModules); }); { std::lock_guard guard(resultAccessLock); @@ -1159,16 +1105,17 @@ void ModuleDependencyScanner::resolveAllClangModuleDependencies( ASSERT(moduleLookupResult.contains(moduleImport.importIdentifier)); const auto &lookupResult = moduleLookupResult.at(moduleImport.importIdentifier); - // Cache discovered module dependencies. - if (!lookupResult.foundDependencyModuleGraph.empty() || - !lookupResult.visibleModuleIdentifiers.empty()) { - if (!lookupResult.foundDependencyModuleGraph.empty()) { - cache.recordClangDependencies(lookupResult.foundDependencyModuleGraph, - IssueReporter.Diagnostics); - // Add the full transitive dependency set - for (const auto &dep : lookupResult.foundDependencyModuleGraph) - allDiscoveredClangModules.insert(dep.first); - } + if (lookupResult.has_value()) { + cache.recordClangDependencies( + lookupResult->ModuleGraph, ScanASTContext.Diags, + [this](auto &clangDep) { + return bridgeClangModuleDependency(clangDep); + }); + + // Add the full transitive dependency set + for (const auto &dep : lookupResult->ModuleGraph) + allDiscoveredClangModules.insert( + {dep.ID.ModuleName, ModuleDependencyKind::Clang}); importedClangDependencies.insert( {moduleImport.importIdentifier, ModuleDependencyKind::Clang}); @@ -1176,7 +1123,7 @@ void ModuleDependencyScanner::resolveAllClangModuleDependencies( // Add visible Clang modules for this query to the depending // Swift module cache.addVisibleClangModules(moduleID, - lookupResult.visibleModuleIdentifiers); + lookupResult->VisibleModules); } else if (!optionalImport) { // Otherwise, we failed to resolve this dependency. We will try // again using the cache after all other imports have been @@ -1450,12 +1397,39 @@ void ModuleDependencyScanner::resolveHeaderDependenciesForModule( std::optional includeTreeID; std::vector bridgingHeaderCommandLine; std::vector visibleClangModules; - auto headerScan = ScanningWorker->scanHeaderDependenciesOfSwiftModule( - *ScanningWorker->workerASTContext, moduleID, headerPath, - sourceBufferRef, cache, headerClangModuleDependencies, - headerFileInputs, bridgingHeaderCommandLine, visibleClangModules, - includeTreeID); - if (!headerScan) { + auto lookupModuleOutput = [this](const auto &cd, auto mok) -> auto { + return clangModuleOutputPathLookup(cd, mok); + }; + auto headerScanResult = + ScanningWorker->scanHeaderDependenciesOfSwiftModule( + moduleID, headerPath, sourceBufferRef, lookupModuleOutput, + cache.getAlreadySeenClangModules()); + + if (headerScanResult) { + // Record module dependencies for each new module we found. + cache.recordClangDependencies( + headerScanResult->ModuleGraph, ScanASTContext.Diags, + [this](auto &clangDep) { + return bridgeClangModuleDependency(clangDep); + }); + llvm::copy(headerScanResult->FileDeps, + std::back_inserter(headerFileInputs)); + auto bridgedDependencyIDs = llvm::map_range( + headerScanResult->ClangModuleDeps, [](auto &input) { + return ModuleDependencyID{input.ModuleName, + ModuleDependencyKind::Clang}; + }); + headerClangModuleDependencies.insert(bridgedDependencyIDs.begin(), + bridgedDependencyIDs.end()); + + auto targetModuleInfo = cache.findKnownDependency(moduleID); + if (targetModuleInfo.isTextualSwiftModule()) { + if (auto TreeID = headerScanResult->IncludeTreeID) + includeTreeID = TreeID; + ClangImporter::getBridgingHeaderOptions( + ScanASTContext, *headerScanResult, bridgingHeaderCommandLine); + } + // Record direct header Clang dependencies moduleDependencyInfo.setHeaderClangDependencies( headerClangModuleDependencies.getArrayRef()); @@ -1468,7 +1442,8 @@ void ModuleDependencyScanner::resolveHeaderDependenciesForModule( bridgingHeaderCommandLine); moduleDependencyInfo.setHeaderSourceFiles(headerFileInputs); // Update the set of visible Clang modules - moduleDependencyInfo.addVisibleClangModules(visibleClangModules); + moduleDependencyInfo.addVisibleClangModules( + headerScanResult->VisibleModules); // Update the dependency in the cache cache.updateDependency(moduleID, moduleDependencyInfo); } else { @@ -1483,13 +1458,11 @@ void ModuleDependencyScanner::resolveSwiftOverlayDependenciesForModule( ModuleDependencyIDSetVector &swiftOverlayDependencies) { PrettyStackTraceStringAction trace( "Resolving Swift Overlay dependencies of module", moduleID.ModuleName); - std::vector allClangDependencies; - llvm::StringSet<> knownModules; + llvm::StringSet<> allClangDependencies; // Find all of the discovered Clang modules that this module depends on. for (const auto &dep : cache.getAllClangDependencies(moduleID)) - findAllImportedClangModules(dep.ModuleName, cache, allClangDependencies, - knownModules); + findAllImportedClangModules(dep.ModuleName, cache, allClangDependencies); llvm::StringMap swiftOverlayLookupResult; std::mutex lookupResultLock; @@ -1522,7 +1495,7 @@ void ModuleDependencyScanner::resolveSwiftOverlayDependenciesForModule( // Enque asynchronous lookup tasks for (const auto &clangDep : allClangDependencies) ScanningThreadPool.async(scanForSwiftDependency, - getModuleImportIdentifier(clangDep)); + getModuleImportIdentifier(clangDep.getKey().str())); ScanningThreadPool.wait(); // Aggregate both previously-cached and freshly-scanned module results @@ -1550,7 +1523,7 @@ void ModuleDependencyScanner::resolveSwiftOverlayDependenciesForModule( } }; for (const auto &clangDep : allClangDependencies) - recordResult(clangDep); + recordResult(clangDep.getKey().str()); // C++ Interop requires additional handling bool lookupCxxStdLibOverlay = ScanCompilerInvocation.getLangOptions().EnableCXXInterop; @@ -1572,7 +1545,8 @@ void ModuleDependencyScanner::resolveSwiftOverlayDependenciesForModule( if (lookupCxxStdLibOverlay) { for (const auto &clangDepNameEntry : allClangDependencies) { - auto clangDepName = clangDepNameEntry; + auto clangDepName = clangDepNameEntry.getKey().str(); + // If this Clang module is a part of the C++ stdlib, and we haven't // loaded the overlay for it so far, it is a split libc++ module (e.g. // std_vector). Load the CxxStdlib overlay explicitly. @@ -1750,14 +1724,41 @@ llvm::Error ModuleDependencyScanner::performBridgingHeaderChaining( std::vector headerFileInputs; std::vector bridgingHeaderCommandLine; std::vector visibleClangModules; - if (ScanningWorker->scanHeaderDependenciesOfSwiftModule( - *ScanningWorker->workerASTContext, rootModuleID, - /*headerPath=*/std::nullopt, sourceBuffer->getMemBufferRef(), - cache, headerClangModuleDependencies, headerFileInputs, - bridgingHeaderCommandLine, visibleClangModules, includeTreeID)) + auto lookupModuleOutput = [this](const auto &cd, auto mok) -> auto { + return clangModuleOutputPathLookup(cd, mok); + }; + auto headerScanResult = + ScanningWorker->scanHeaderDependenciesOfSwiftModule( + rootModuleID, /*headerPath=*/std::nullopt, + sourceBuffer->getMemBufferRef(), lookupModuleOutput, + cache.getAlreadySeenClangModules()); + + if (!headerScanResult) return llvm::createStringError( "failed to scan generated bridging header " + outputPath); + // Record module dependencies for each new module we found. + cache.recordClangDependencies( + headerScanResult->ModuleGraph, ScanASTContext.Diags, + [this](auto &clangDep) { + return bridgeClangModuleDependency(clangDep); + }); + llvm::copy(headerScanResult->FileDeps, + std::back_inserter(headerFileInputs)); + auto bridgedDependencyIDs = + llvm::map_range(headerScanResult->ClangModuleDeps, [](auto &input) { + return ModuleDependencyID{input.ModuleName, + ModuleDependencyKind::Clang}; + }); + headerClangModuleDependencies.insert(bridgedDependencyIDs.begin(), + bridgedDependencyIDs.end()); + + // Update visible module set + if (auto TreeID = headerScanResult->IncludeTreeID) + includeTreeID = TreeID; + ClangImporter::getBridgingHeaderOptions( + ScanASTContext, *headerScanResult, bridgingHeaderCommandLine); + cache.setHeaderClangDependencies( rootModuleID, headerClangModuleDependencies.getArrayRef()); // Record include Tree ID @@ -1778,6 +1779,9 @@ llvm::Error ModuleDependencyScanner::performBridgingHeaderChaining( mainModuleDeps.setHeaderSourceFiles(headerFileInputs); mainModuleDeps.setChainedBridgingHeaderBuffer( outputPath, sourceBuffer->getBuffer()); + // Update the set of visible Clang modules + mainModuleDeps.addVisibleClangModules(headerScanResult->VisibleModules); + // Update the dependency in the cache cache.updateDependency(rootModuleID, mainModuleDeps); return llvm::Error::success(); @@ -1796,6 +1800,118 @@ llvm::Error ModuleDependencyScanner::performBridgingHeaderChaining( return llvm::Error::success(); } +std::string ModuleDependencyScanner::remapPath(StringRef Path) const { + if (!PrefixMapper) + return Path.str(); + return PrefixMapper->mapToString(Path); +} + +ModuleDependencyInfo ModuleDependencyScanner::bridgeClangModuleDependency( + const clang::tooling::dependencies::ModuleDeps &clangModuleDep) { + // File dependencies for this module. + std::vector fileDeps; + clangModuleDep.forEachFileDep( + [&fileDeps](StringRef fileDep) { fileDeps.emplace_back(fileDep); }); + + std::vector swiftArgs; + auto addClangArg = [&](Twine arg) { + swiftArgs.push_back("-Xcc"); + swiftArgs.push_back(arg.str()); + }; + + // We are using Swift frontend mode. + swiftArgs.push_back("-frontend"); + + // Swift frontend action: -emit-pcm + swiftArgs.push_back("-emit-pcm"); + swiftArgs.push_back("-module-name"); + swiftArgs.push_back(clangModuleDep.ID.ModuleName); + + auto pcmPath = clangModuleOutputPathLookup( + clangModuleDep, + clang::tooling::dependencies::ModuleOutputKind::ModuleFile); + swiftArgs.push_back("-o"); + swiftArgs.push_back(pcmPath); + + // Ensure that the resulting PCM build invocation uses Clang frontend + // directly + swiftArgs.push_back("-direct-clang-cc1-module-build"); + + // Swift frontend option for input file path (Foo.modulemap). + swiftArgs.push_back(remapPath(clangModuleDep.ClangModuleMapFile)); + + auto invocation = clangModuleDep.getUnderlyingCompilerInvocation(); + // Clear some options from clang scanner. + invocation.getMutFrontendOpts().ModuleCacheKeys.clear(); + invocation.getMutFrontendOpts().PathPrefixMappings.clear(); + invocation.getMutFrontendOpts().OutputFile.clear(); + + // Reset CASOptions since that should be coming from swift. + invocation.getMutCASOpts() = clang::CASOptions(); + invocation.getMutFrontendOpts().CASIncludeTreeID.clear(); + + // FIXME: workaround for rdar://105684525: find the -ivfsoverlay option + // from clang scanner and pass to swift. + if (!ScanASTContext.CASOpts.EnableCaching) { + auto &overlayFiles = invocation.getMutHeaderSearchOpts().VFSOverlayFiles; + for (auto overlay : overlayFiles) { + swiftArgs.push_back("-vfsoverlay"); + swiftArgs.push_back(overlay); + } + } + + // Add args reported by the scanner. + auto clangArgs = invocation.getCC1CommandLine(); + llvm::for_each(clangArgs, addClangArg); + + // CASFileSystemRootID. + std::string RootID = clangModuleDep.CASFileSystemRootID + ? clangModuleDep.CASFileSystemRootID->toString() + : ""; + + std::string IncludeTree = + clangModuleDep.IncludeTreeID ? *clangModuleDep.IncludeTreeID : ""; + + ScanASTContext.CASOpts.enumerateCASConfigurationFlags( + [&](StringRef Arg) { swiftArgs.push_back(Arg.str()); }); + + if (!IncludeTree.empty()) { + swiftArgs.push_back("-clang-include-tree-root"); + swiftArgs.push_back(IncludeTree); + } + std::string mappedPCMPath = remapPath(pcmPath); + + std::vector LinkLibraries; + for (const auto &ll : clangModuleDep.LinkLibraries) + LinkLibraries.emplace_back(ll.Library, + ll.IsFramework ? LibraryKind::Framework + : LibraryKind::Library, + /*static=*/false); + + // Module-level dependencies. + llvm::StringSet<> alreadyAddedModules; + auto bridgedDependencyInfo = ModuleDependencyInfo::forClangModule( + pcmPath, mappedPCMPath, clangModuleDep.ClangModuleMapFile, + clangModuleDep.ID.ContextHash, swiftArgs, fileDeps, LinkLibraries, RootID, + IncludeTree, /*module-cache-key*/ "", clangModuleDep.IsSystem); + + std::vector directDependencyIDs; + for (const auto &moduleName : clangModuleDep.ClangModuleDeps) { + // FIXME: This assumes, conservatively, that all Clang module imports + // are exported. We need to fix this once the clang scanner gains the + // appropriate API to query this. + bridgedDependencyInfo.addModuleImport( + moduleName.ModuleName, /* isExported */ true, AccessLevel::Public, + &alreadyAddedModules); + // It is safe to assume that all dependencies of a Clang module are Clang + // modules. + directDependencyIDs.push_back( + {moduleName.ModuleName, ModuleDependencyKind::Clang}); + } + bridgedDependencyInfo.setImportedClangDependencies(directDependencyIDs); + return bridgedDependencyInfo; +} + void ModuleDependencyIssueReporter::diagnoseModuleNotFoundFailure( const ScannerImportStatementInfo &moduleImport, const ModuleDependenciesCache &cache, @@ -1954,8 +2070,7 @@ ModuleDependencyScanner::attemptToFindResolvingSerializedSearchPath( // paths. That's fine because we are diagnosing a scan failure here, but // worth being aware of. result = withDependencyScanningWorker( - [this, &binaryModInfo, &moduleImport, - &binaryDepID](ModuleDependencyScanningWorker *ScanningWorker) + [&](ModuleDependencyScanningWorker *ScanningWorker) -> std::optional> { for (const auto &sp : binaryModInfo->serializedSearchPaths) ScanningWorker->workerASTContext->addSearchPath( @@ -1971,13 +2086,15 @@ ModuleDependencyScanner::attemptToFindResolvingSerializedSearchPath( binaryDepID, swiftResult.foundDependencyInfo->getModuleDefiningPath()); - ClangModuleScannerQueryResult clangResult = + auto clangResult = ScanningWorker->scanFilesystemForClangModuleDependency( - importIdentifier, {}); - if (!clangResult.foundDependencyModuleGraph.empty()) - return std::make_pair(binaryDepID, - clangResult.foundDependencyModuleGraph[0] - .second.getModuleDefiningPath()); + importIdentifier, + [this](const auto &cd, auto mok) -> std::string { + return clangModuleOutputPathLookup(cd, mok); + }, {}); + if (clangResult) + return std::make_pair( + binaryDepID, clangResult->ModuleGraph[0].ClangModuleMapFile); return std::nullopt; }); if (result)