Run Command: Add SwiftBuild repl support (#8971)

The `swift run --repl` was explicitly using the Native build system.
This change modified the `swift run --repl` command to respect the
`--build-system <arg>` command line option. The change introduces a new
build output request `replArguments`. When provided, the build system is
responsible for providing the arguments required for the REPL in the
BuildResult.
    
The Swift Run Command will inspect the build result for this property.
If it's unavailable, the command provides an error indicating repl
support is unavailable.
    
A caveat, more work is required to get proper REPL integration with the
Package. At the moment, the System Library paths are not provided by the
SwiftBuild System when a repl session is requested via the `run`
command.
    
Relates to: #8846
issue: rdar://153822861

Depends on : #8942

Author: bkhouri

Sources/Build/BuildOperation.swift

@@ -37,8 +37,8 @@ import SwiftDriver
 #endif
 
 package struct LLBuildSystemConfiguration {
-    let toolsBuildParameters: BuildParameters
-    let destinationBuildParameters: BuildParameters
+    fileprivate let toolsBuildParameters: BuildParameters
+    fileprivate let destinationBuildParameters: BuildParameters
 
     let scratchDirectory: AbsolutePath
 
@@ -397,7 +397,10 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
 
     /// Perform a build using the given build description and subset.
     public func build(subset: BuildSubset, buildOutputs: [BuildOutput]) async throws -> BuildResult {
-        var result = BuildResult(serializedDiagnosticPathsByTargetName: .failure(StringError("Building was skipped")))
+        var result = BuildResult(
+            serializedDiagnosticPathsByTargetName: .failure(StringError("Building was skipped")),
+            replArguments: nil,
+        )
 
         guard !self.config.shouldSkipBuilding(for: .target) else {
             return result
@@ -455,10 +458,15 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         )
         guard success else { throw Diagnostics.fatalError }
 
-        if buildOutputs.contains(.buildPlan) {
-            result.buildPlan = try buildPlan
-        }
+        let buildResultBuildPlan = buildOutputs.contains(.buildPlan) ? try buildPlan : nil
+        let buildResultReplArgs = buildOutputs.contains(.replArguments) ? try buildPlan.createREPLArguments() : nil
 
+        result = BuildResult(
+            serializedDiagnosticPathsByTargetName: result.serializedDiagnosticPathsByTargetName,
+            symbolGraph: result.symbolGraph,
+            buildPlan: buildResultBuildPlan,
+            replArguments: buildResultReplArgs,
+        )
         var serializedDiagnosticPaths: [String: [AbsolutePath]] = [:]
         do {
             for module in try buildPlan.buildModules {
@@ -701,7 +709,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
 
         // Create the build plan based on the modules graph and any information from plugins.
         return try await BuildPlan(
-            destinationBuildParameters: self.config.destinationBuildParameters,
+            destinationBuildParameters: self.config.buildParameters(for: .target),
             toolsBuildParameters: self.config.buildParameters(for: .host),
             graph: graph,
             pluginConfiguration: self.pluginConfiguration,

Sources/Build/BuildPlan/BuildPlan.swift

@@ -637,13 +637,12 @@ public class BuildPlan: SPMBuildCore.BuildPlan {
 
     /// Creates arguments required to launch the Swift REPL that will allow
     /// importing the modules in the package graph.
-    public func createREPLArguments() throws -> [String] {
+    public func createREPLArguments() throws -> CLIArguments {
         let buildPath = self.toolsBuildParameters.buildPath.pathString
         var arguments = ["repl", "-I" + buildPath, "-L" + buildPath]
 
         // Link the special REPL product that contains all of the library targets.
-        let replProductName = self.graph.rootPackages[self.graph.rootPackages.startIndex].identity.description +
-            Product.replProductSuffix
+        let replProductName = try self.graph.getReplProductName()
         arguments.append("-l" + replProductName)
 
         // The graph should have the REPL product.

Sources/Commands/SwiftRunCommand.swift

@@ -134,23 +134,24 @@ public struct SwiftRunCommand: AsyncSwiftCommand {
             // Construct the build operation.
             // FIXME: We need to implement the build tool invocation closure here so that build tool plugins work with the REPL. rdar://86112934
             let buildSystem = try await swiftCommandState.createBuildSystem(
-                explicitBuildSystem: .native,
                 cacheBuildManifest: false,
                 packageGraphLoader: asyncUnsafeGraphLoader
             )
 
             // Perform build.
-            let buildResult = try await buildSystem.build(subset: .allExcludingTests, buildOutputs: [.buildPlan])
-            guard let buildPlan = buildResult.buildPlan else {
+            let buildResult = try await buildSystem.build(subset: .allExcludingTests, buildOutputs: [.replArguments])
+            guard let arguments = buildResult.replArguments else {
+                swiftCommandState.observabilityScope.emit(error: "\(globalOptions.build.buildSystem) build system does not support this command")
                 throw ExitCode.failure
             }
 
             // Execute the REPL.
-            let arguments = try buildPlan.createREPLArguments()
-            print("Launching Swift REPL with arguments: \(arguments.joined(separator: " "))")
+            let interpreterPath = try swiftCommandState.getTargetToolchain().swiftInterpreterPath
+            swiftCommandState.outputStream.send("Launching Swift (interpreter at \(interpreterPath)) REPL with arguments: \(arguments.joined(separator: " "))\n")
+            swiftCommandState.outputStream.flush()
             try self.run(
                 fileSystem: swiftCommandState.fileSystem,
-                executablePath: swiftCommandState.getTargetToolchain().swiftInterpreterPath,
+                executablePath: interpreterPath,
                 originalWorkingDirectory: swiftCommandState.originalWorkingDirectory,
                 arguments: arguments
             )
@@ -197,7 +198,7 @@ public struct SwiftRunCommand: AsyncSwiftCommand {
         case .run:
             // Detect deprecated uses of swift run to interpret scripts.
             if let executable = options.executable, try isValidSwiftFilePath(fileSystem: swiftCommandState.fileSystem, path: executable) {
-                swiftCommandState.observabilityScope.emit(.runFileDeprecation)
+                swiftCommandState.observabilityScope.emit(.runFileDeprecation(filePath: executable))
                 // Redirect execution to the toolchain's swift executable.
                 let swiftInterpreterPath = try swiftCommandState.getTargetToolchain().swiftInterpreterPath
                 // Prepend the script to interpret to the arguments.
@@ -364,8 +365,8 @@ public struct SwiftRunCommand: AsyncSwiftCommand {
 }
 
 private extension Basics.Diagnostic {
-    static var runFileDeprecation: Self {
-        .warning("'swift run file.swift' command to interpret swift files is deprecated; use 'swift file.swift' instead")
+    static func runFileDeprecation(filePath: String) -> Self {
+        .warning("'swift run \(filePath)' command to interpret swift files is deprecated; use 'swift \(filePath)' instead")
     }
 }
 

Sources/PackageGraph/ModulesGraph.swift

@@ -285,6 +285,14 @@ public struct ModulesGraph {
 
         return result
     }
+
+    public func getReplProductName() throws -> String {
+        if self.rootPackages.isEmpty {
+            throw StringError("Root package does not exist.")
+        }
+        return self.rootPackages[self.rootPackages.startIndex].identity.description +
+            Product.replProductSuffix
+    }
 }
 
 extension PackageGraphError: CustomStringConvertible {

Sources/SPMBuildCore/BuildSystem/BuildSystem.swift

@@ -47,6 +47,7 @@ public enum BuildOutput {
     // "-emit-extension-block-symbols"
     // "-emit-synthesized-members"
     case buildPlan
+    case replArguments
 }
 
 /// A protocol that represents a build system used by SwiftPM for all build operations. This allows factoring out the
@@ -91,15 +92,25 @@ public struct SymbolGraphResult {
     public let outputLocationForTarget: (String, BuildParameters) -> [String]
 }
 
+public typealias CLIArguments = [String]
+
 public struct BuildResult {
-    package init(serializedDiagnosticPathsByTargetName: Result<[String: [AbsolutePath]], Error>, symbolGraph: SymbolGraphResult? = nil, buildPlan: BuildPlan? = nil) {
+    package init(
+        serializedDiagnosticPathsByTargetName: Result<[String: [AbsolutePath]], Error>,
+        symbolGraph: SymbolGraphResult? = nil,
+        buildPlan: BuildPlan? = nil,
+        replArguments: CLIArguments?
+    ) {
         self.serializedDiagnosticPathsByTargetName = serializedDiagnosticPathsByTargetName
         self.symbolGraph = symbolGraph
         self.buildPlan = buildPlan
+        self.replArguments = replArguments
     }
 
-    public var symbolGraph: SymbolGraphResult?
-    public var buildPlan: BuildPlan?
+    public let replArguments: CLIArguments?
+    public let symbolGraph: SymbolGraphResult?
+    public let buildPlan: BuildPlan?
+
     public var serializedDiagnosticPathsByTargetName: Result<[String: [AbsolutePath]], Error>
 }
 

Sources/SwiftBuildSupport/SwiftBuildSystem.swift

@@ -276,24 +276,95 @@ public final class SwiftBuildSystem: SPMBuildCore.BuildSystem {
         self.delegate = delegate
     }
 
+    private func createREPLArguments(
+        session: SWBBuildServiceSession,
+        request: SWBBuildRequest
+    ) async throws -> CLIArguments {
+        self.outputStream.send("Gathering repl arguments...")
+        self.outputStream.flush()
+
+        func getUniqueBuildSettingsIncludingDependencies(of targetGuid: [SWBConfiguredTarget], buildSettings: [String]) async throws -> Set<String> {
+            let dependencyGraph = try await session.computeDependencyGraph(
+                targetGUIDs: request.configuredTargets.map { SWBTargetGUID(rawValue: $0.guid)},
+                buildParameters: request.parameters,
+                includeImplicitDependencies: true,
+            )
+            var uniquePaths = Set<String>()
+            for setting in buildSettings {
+                self.outputStream.send(".")
+                self.outputStream.flush()
+                for (target, targetDependencies) in dependencyGraph {
+                    for t in [target] + targetDependencies {
+                        try await session.evaluateMacroAsStringList(
+                            setting,
+                            level: .target(t.rawValue),
+                            buildParameters: request.parameters,
+                            overrides: nil,
+                        ).forEach({
+                            uniquePaths.insert($0)
+                        })
+                    }
+                }
+
+            }
+            return uniquePaths
+        }
+
+        // TODO: Need to determine how to get the inlude path of package system library dependencies
+        let includePaths = try await getUniqueBuildSettingsIncludingDependencies(
+            of: request.configuredTargets,
+            buildSettings: [
+                "BUILT_PRODUCTS_DIR",
+                "HEADER_SEARCH_PATHS",
+                "USER_HEADER_SEARCH_PATHS",
+                "FRAMEWORK_SEARCH_PATHS",
+            ]
+        )
+
+        let graph = try await self.getPackageGraph()
+        // Link the special REPL product that contains all of the library targets.
+        let replProductName: String = try graph.getReplProductName()
+
+        // The graph should have the REPL product.
+        assert(graph.product(for: replProductName) != nil)
+
+        let arguments = ["repl", "-l\(replProductName)"] + includePaths.map {
+            "-I\($0)"
+        }
+
+        self.outputStream.send("Done.\n")
+        return arguments
+    }
+
     private func supportedSwiftVersions() throws -> [SwiftLanguageVersion] {
         // Swift Build should support any of the supported language versions of SwiftPM and the rest of the toolchain
         SwiftLanguageVersion.supportedSwiftLanguageVersions
     }
 
     public func build(subset: BuildSubset, buildOutputs: [BuildOutput]) async throws -> BuildResult {
         guard !buildParameters.shouldSkipBuilding else {
-            return BuildResult(serializedDiagnosticPathsByTargetName: .failure(StringError("Building was skipped")))
+            return BuildResult(
+                serializedDiagnosticPathsByTargetName: .failure(StringError("Building was skipped")),
+                replArguments: nil,
+            )
         }
 
         try await writePIF(buildParameters: buildParameters)
 
-        return try await startSWBuildOperation(pifTargetName: subset.pifTargetName, genSymbolGraph: buildOutputs.contains(.symbolGraph))
+        return try await startSWBuildOperation(
+            pifTargetName: subset.pifTargetName,
+            genSymbolGraph: buildOutputs.contains(.symbolGraph),
+            generateReplArguments: buildOutputs.contains(.replArguments),
+        )
     }
 
-    private func startSWBuildOperation(pifTargetName: String, genSymbolGraph: Bool) async throws -> BuildResult {
+    private func startSWBuildOperation(
+        pifTargetName: String,
+        genSymbolGraph: Bool,
+        generateReplArguments: Bool
+    ) async throws -> BuildResult {
         let buildStartTime = ContinuousClock.Instant.now
-
+        var replArguments: CLIArguments?
         return try await withService(connectionMode: .inProcessStatic(swiftbuildServiceEntryPoint)) { service in
             let derivedDataPath = self.buildParameters.dataPath
 
@@ -418,7 +489,7 @@ public final class SwiftBuildSystem: SPMBuildCore.BuildSystem {
                                 case .note: .info
                                 case .remark: .debug
                                 }
-                                self.observabilityScope.emit(severity: severity, message: message)
+                                self.observabilityScope.emit(severity: severity, message: "\(message)\n")
 
                                 for childDiagnostic in info.childDiagnostics {
                                     emitInfoAsDiagnostic(info: childDiagnostic)
@@ -502,6 +573,8 @@ public final class SwiftBuildSystem: SPMBuildCore.BuildSystem {
                         self.observabilityScope.emit(error: "Unexpected build state")
                         throw Diagnostics.fatalError
                     }
+
+                    replArguments = generateReplArguments ? try await self.createREPLArguments(session: session, request: request) : nil
                 }
             } catch let sessError as SessionFailedError {
                 for diagnostic in sessError.diagnostics {
@@ -512,9 +585,15 @@ public final class SwiftBuildSystem: SPMBuildCore.BuildSystem {
                 throw error
             }
 
-            return BuildResult(serializedDiagnosticPathsByTargetName: .success(serializedDiagnosticPathsByTargetName), symbolGraph: SymbolGraphResult(outputLocationForTarget: { target, buildParameters in
-                return ["\(buildParameters.triple.archName)", "\(target).symbolgraphs"]
-            }))
+            return BuildResult(
+                serializedDiagnosticPathsByTargetName: .success(serializedDiagnosticPathsByTargetName),
+                symbolGraph: SymbolGraphResult(
+                    outputLocationForTarget: { target, buildParameters in
+                        return ["\(buildParameters.triple.archName)", "\(target).symbolgraphs"]
+                    }
+                ),
+                replArguments: replArguments,
+            )
         }
     }
 

Sources/XCBuildSupport/XcodeBuildSystem.swift

@@ -161,7 +161,10 @@ public final class XcodeBuildSystem: SPMBuildCore.BuildSystem {
     }
 
     public func build(subset: BuildSubset, buildOutputs: [BuildOutput]) async throws -> BuildResult {
-        let buildResult = BuildResult(serializedDiagnosticPathsByTargetName: .failure(StringError("XCBuild does not support reporting serialized diagnostics.")))
+        let buildResult = BuildResult(
+            serializedDiagnosticPathsByTargetName: .failure(StringError("XCBuild does not support reporting serialized diagnostics.")),
+            replArguments: nil,
+        )
 
         guard !buildParameters.shouldSkipBuilding else {
             return buildResult