From 984df4295621715809d0c66022b30202c508b34f Mon Sep 17 00:00:00 2001 From: Jake Petroules Date: Fri, 14 Feb 2025 15:30:25 +0900 Subject: [PATCH 1/5] Add FreeBSD support This allows building Swift Build for FreeBSD hosts, as well as building for a FreeBSD target from a FreeBSD host. Also adds some speculative support for targeting OpenBSD on OpenBSD hosts, since SwiftPM has minimal support. --- Package.swift | 4 +-- Sources/SWBCore/Settings/Settings.swift | 4 +++ Sources/SWBGenericUnixPlatform/Plugin.swift | 12 ++++++--- .../Specs/FreeBSDLibtool.xcspec | 27 +++++++++++++++++++ .../RunDestinationTestSupport.swift | 20 ++++++++++++++ .../SWBTestSupport/SkippedTestSupport.swift | 12 +++++++-- Sources/SWBUtil/Architecture.swift | 13 ++++++++- Sources/SWBUtil/FSProxy.swift | 8 ++++++ Sources/SWBUtil/Lock.swift | 2 +- Sources/SWBUtil/Process.swift | 2 ++ Sources/SWBUtil/ProcessInfo.swift | 8 +++++- .../BuildTaskBehaviorTests.swift | 2 +- ...mmandLineToolSpecDiscoveredInfoTests.swift | 2 +- .../SWBCoreTests/FileTextEncodingTests.swift | 3 +-- Tests/SWBCoreTests/SettingsTests.swift | 2 +- Tests/SWBTaskExecutionTests/PBXCpTests.swift | 2 +- Tests/SWBUtilTests/ElapsedTimerTests.swift | 3 ++- Tests/SWBUtilTests/FSProxyTests.swift | 4 +-- Tests/SWBUtilTests/FileHandleTests.swift | 2 +- Tests/SWBUtilTests/HeavyCacheTests.swift | 5 ++-- Tests/SWBUtilTests/MiscTests.swift | 2 +- Tests/SWBUtilTests/RateLimiterTests.swift | 4 ++- 22 files changed, 118 insertions(+), 25 deletions(-) create mode 100644 Sources/SWBGenericUnixPlatform/Specs/FreeBSDLibtool.xcspec diff --git a/Package.swift b/Package.swift index 8fdc5af7..5ca44540 100644 --- a/Package.swift +++ b/Package.swift @@ -110,7 +110,7 @@ let package = Package( "SWBBuildSystem", "SWBServiceCore", "SWBTaskExecution", - .product(name: "SystemPackage", package: "swift-system", condition: .when(platforms: [.linux, .android, .windows])), + .product(name: "SystemPackage", package: "swift-system", condition: .when(platforms: [.linux, .openbsd, .android, .windows, .custom("freebsd")])), ], exclude: ["CMakeLists.txt"], swiftSettings: swiftSettings(languageMode: .v5)), @@ -201,7 +201,7 @@ let package = Package( "SWBCSupport", "SWBLibc", .product(name: "ArgumentParser", package: "swift-argument-parser"), - .product(name: "SystemPackage", package: "swift-system", condition: .when(platforms: [.linux, .android, .windows])), + .product(name: "SystemPackage", package: "swift-system", condition: .when(platforms: [.linux, .openbsd, .android, .windows, .custom("freebsd")])), ], exclude: ["CMakeLists.txt"], swiftSettings: swiftSettings(languageMode: .v5)), diff --git a/Sources/SWBCore/Settings/Settings.swift b/Sources/SWBCore/Settings/Settings.swift index 42fcbbe9..7e4e6f07 100644 --- a/Sources/SWBCore/Settings/Settings.swift +++ b/Sources/SWBCore/Settings/Settings.swift @@ -5331,6 +5331,10 @@ extension OperatingSystem { return "windows" case .linux: return "linux" + case .freebsd: + return "freebsd" + case .openbsd: + return "openbsd" case .android: return "android" case .unknown: diff --git a/Sources/SWBGenericUnixPlatform/Plugin.swift b/Sources/SWBGenericUnixPlatform/Plugin.swift index 128fde75..a3839068 100644 --- a/Sources/SWBGenericUnixPlatform/Plugin.swift +++ b/Sources/SWBGenericUnixPlatform/Plugin.swift @@ -39,7 +39,11 @@ struct GenericUnixPlatformSpecsExtension: SpecificationsExtension { } func specificationDomains() -> [String: [String]] { - ["linux": ["generic-unix"]] + [ + "linux": ["generic-unix"], + "freebsd": ["generic-unix"], + "openbsd": ["generic-unix"], + ] } } @@ -73,9 +77,9 @@ struct GenericUnixSDKRegistryExtension: SDKRegistryExtension { let defaultProperties: [String: PropertyListItem] switch operatingSystem { - case .linux: + case .linux, .freebsd: defaultProperties = [ - // Workaround to avoid `-dependency_info` on Linux. + // Workaround to avoid `-dependency_info`. "LD_DEPENDENCY_INFO_FILE": .plString(""), "GENERATE_TEXT_BASED_STUBS": "NO", @@ -167,6 +171,6 @@ struct GenericUnixToolchainRegistryExtension: ToolchainRegistryExtension { extension OperatingSystem { /// Whether the Core is allowed to create a fallback toolchain, SDK, and platform for this operating system in cases where no others have been provided. var createFallbackSystemToolchain: Bool { - return self == .linux + return self == .linux || self == .freebsd || self == .openbsd } } diff --git a/Sources/SWBGenericUnixPlatform/Specs/FreeBSDLibtool.xcspec b/Sources/SWBGenericUnixPlatform/Specs/FreeBSDLibtool.xcspec new file mode 100644 index 00000000..8d247c89 --- /dev/null +++ b/Sources/SWBGenericUnixPlatform/Specs/FreeBSDLibtool.xcspec @@ -0,0 +1,27 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +( + { + Domain = freebsd; + Identifier = com.apple.pbx.linkers.libtool; + BasedOn = generic-unix:com.apple.pbx.linkers.libtool; + Type = Linker; + Options = ( + { + Name = "LIBTOOL_USE_RESPONSE_FILE"; + Type = Boolean; + DefaultValue = NO; + }, + ); + }, +) diff --git a/Sources/SWBTestSupport/RunDestinationTestSupport.swift b/Sources/SWBTestSupport/RunDestinationTestSupport.swift index ecdff852..69b83184 100644 --- a/Sources/SWBTestSupport/RunDestinationTestSupport.swift +++ b/Sources/SWBTestSupport/RunDestinationTestSupport.swift @@ -98,6 +98,10 @@ extension _RunDestinationInfo { windows case .linux: linux + case .freebsd: + freebsd + case .openbsd: + openbsd case .android: android case .unknown: @@ -259,6 +263,22 @@ extension _RunDestinationInfo { return .init(platform: "linux", sdk: "linux", sdkVariant: "linux", targetArchitecture: arch, supportedArchitectures: ["x86_64", "aarch64"], disableOnlyActiveArch: false) } + /// A run destination targeting FreeBSD generic device, using the public SDK. + package static var freebsd: Self { + guard let arch = Architecture.hostStringValue else { + preconditionFailure("Unknown architecture \(Architecture.host.stringValue ?? "")") + } + return .init(platform: "freebsd", sdk: "freebsd", sdkVariant: "freebsd", targetArchitecture: arch, supportedArchitectures: ["x86_64", "aarch64"], disableOnlyActiveArch: false) + } + + /// A run destination targeting OpenBSD generic device, using the public SDK. + package static var openbsd: Self { + guard let arch = Architecture.hostStringValue else { + preconditionFailure("Unknown architecture \(Architecture.host.stringValue ?? "")") + } + return .init(platform: "openbsd", sdk: "openbsd", sdkVariant: "openbsd", targetArchitecture: arch, supportedArchitectures: ["x86_64", "aarch64"], disableOnlyActiveArch: false) + } + /// A run destination targeting Android generic device, using the public SDK. package static var android: Self { return .init(platform: "android", sdk: "android", sdkVariant: "android", targetArchitecture: "undefined_arch", supportedArchitectures: ["armv7", "aarch64", "riscv64", "i686", "x86_64"], disableOnlyActiveArch: true) diff --git a/Sources/SWBTestSupport/SkippedTestSupport.swift b/Sources/SWBTestSupport/SkippedTestSupport.swift index 4d82e95f..1db40901 100644 --- a/Sources/SWBTestSupport/SkippedTestSupport.swift +++ b/Sources/SWBTestSupport/SkippedTestSupport.swift @@ -49,6 +49,10 @@ extension KnownSDK { return windows case .success(.linux): return linux + case .success(.freebsd): + return freebsd + case .success(.openbsd): + return openbsd case .success(.android): return android case .success(.unknown), .failure: @@ -69,6 +73,8 @@ extension KnownSDK { extension KnownSDK { package static let windows: Self = "windows" package static let linux: Self = "linux" + package static let freebsd: Self = "freebsd" + package static let openbsd: Self = "openbsd" package static let android: Self = "android" package static let qnx: Self = "qnx" package static let wasi: Self = "wasi" @@ -196,7 +202,7 @@ extension Trait where Self == Testing.ConditionTrait { } } - package static func requireSystemPackages(apt: String..., yum: String..., sourceLocation: SourceLocation = #_sourceLocation) -> Self { + package static func requireSystemPackages(apt: String..., yum: String..., freebsd: String..., sourceLocation: SourceLocation = #_sourceLocation) -> Self { enabled("required system packages are not installed") { func checkInstalled(hostOS: OperatingSystem, packageManagerPath: Path, args: [String], packages: [String], regex: Regex<(Substring, name: Substring)>) async throws -> Bool { if try ProcessInfo.processInfo.hostOperatingSystem() == hostOS && localFS.exists(packageManagerPath) { @@ -222,7 +228,9 @@ extension Trait where Self == Testing.ConditionTrait { // spelled `--installed` in newer versions of yum, but Amazon Linux 2 is on older versions let yum = try await checkInstalled(hostOS: .linux, packageManagerPath: Path("/usr/bin/yum"), args: ["list", "installed", "yum"], packages: yum, regex: #/(?.+)\./#) - return apt && yum + let freebsd = try await checkInstalled(hostOS: .freebsd, packageManagerPath: Path("/usr/sbin/pkg"), args: ["info"], packages: freebsd, regex: #/^Name(?:[ ]+): (?.+)$/#) + + return apt && yum && freebsd } } diff --git a/Sources/SWBUtil/Architecture.swift b/Sources/SWBUtil/Architecture.swift index 340491bd..d7518ec8 100644 --- a/Sources/SWBUtil/Architecture.swift +++ b/Sources/SWBUtil/Architecture.swift @@ -98,7 +98,18 @@ public struct Architecture: Sendable { if uname(&buf) == 0 { return withUnsafeBytes(of: &buf.machine) { buf in let data = Data(buf) - return String(decoding: data[0...(data.lastIndex(where: { $0 != 0 }) ?? 0)], as: UTF8.self) + let value = String(decoding: data[0...(data.lastIndex(where: { $0 != 0 }) ?? 0)], as: UTF8.self) + #if os(FreeBSD) + switch value { + case "amd64": + return "x86_64" + case "arm64": + return "aarch64" + default: + break + } + #endif + return value } } return nil diff --git a/Sources/SWBUtil/FSProxy.swift b/Sources/SWBUtil/FSProxy.swift index adf69e3a..9f0eb4c2 100644 --- a/Sources/SWBUtil/FSProxy.swift +++ b/Sources/SWBUtil/FSProxy.swift @@ -718,6 +718,9 @@ class LocalFS: FSProxy, @unchecked Sendable { #if os(Windows) // Implement ADS on Windows? See also https://github.com/swiftlang/swift-foundation/issues/1166 return [] + #elseif os(FreeBSD) + // FreeBSD blocked on https://github.com/swiftlang/swift/pull/77836 + return [] #elseif os(OpenBSD) // OpenBSD no longer supports extended attributes return [] @@ -758,6 +761,8 @@ class LocalFS: FSProxy, @unchecked Sendable { func setExtendedAttribute(_ path: Path, key: String, value: ByteString) throws { #if os(Windows) // Implement ADS on Windows? See also https://github.com/swiftlang/swift-foundation/issues/1166 + #elseif os(FreeBSD) + // FreeBSD blocked on https://github.com/swiftlang/swift/pull/77836 #elseif os(OpenBSD) // OpenBSD no longer supports extended attributes #else @@ -778,6 +783,9 @@ class LocalFS: FSProxy, @unchecked Sendable { #if os(Windows) // Implement ADS on Windows? See also https://github.com/swiftlang/swift-foundation/issues/1166 return nil + #elseif os(FreeBSD) + // FreeBSD blocked on https://github.com/swiftlang/swift/pull/77836 + return nil #elseif os(OpenBSD) // OpenBSD no longer supports extended attributes return nil diff --git a/Sources/SWBUtil/Lock.swift b/Sources/SWBUtil/Lock.swift index abc1664a..b45625c6 100644 --- a/Sources/SWBUtil/Lock.swift +++ b/Sources/SWBUtil/Lock.swift @@ -28,7 +28,7 @@ public final class Lock: @unchecked Sendable { #if os(Windows) @usableFromInline let mutex: UnsafeMutablePointer = UnsafeMutablePointer.allocate(capacity: 1) - #elseif os(OpenBSD) + #elseif os(FreeBSD) || os(OpenBSD) @usableFromInline let mutex: UnsafeMutablePointer = UnsafeMutablePointer.allocate(capacity: 1) #else diff --git a/Sources/SWBUtil/Process.swift b/Sources/SWBUtil/Process.swift index d07f8e9f..d832eb5a 100644 --- a/Sources/SWBUtil/Process.swift +++ b/Sources/SWBUtil/Process.swift @@ -70,6 +70,8 @@ extension Process { case .linux: // Amazon Linux 2 has glibc 2.26, and glibc 2.29 is needed for posix_spawn_file_actions_addchdir_np support FileManager.default.contents(atPath: "/etc/system-release").map { String(decoding: $0, as: UTF8.self) == "Amazon Linux release 2 (Karoo)\n" } ?? false + case .openbsd: + true default: false } diff --git a/Sources/SWBUtil/ProcessInfo.swift b/Sources/SWBUtil/ProcessInfo.swift index 85b4ef2c..e96f8631 100644 --- a/Sources/SWBUtil/ProcessInfo.swift +++ b/Sources/SWBUtil/ProcessInfo.swift @@ -99,6 +99,10 @@ extension ProcessInfo { return .windows #elseif os(Linux) return .linux + #elseif os(FreeBSD) + return .freebsd + #elseif os(OpenBSD) + return .openbsd #else if try FileManager.default.isReadableFile(atPath: systemVersionPlistURL.filePath.str) { switch try systemVersion().productName { @@ -129,6 +133,8 @@ public enum OperatingSystem: Hashable, Sendable { case visionOS(simulator: Bool) case windows case linux + case freebsd + case openbsd case android case unknown @@ -157,7 +163,7 @@ public enum OperatingSystem: Hashable, Sendable { return .macho case .windows: return .pe - case .linux, .android, .unknown: + case .linux, .freebsd, .openbsd, .android, .unknown: return .elf } } diff --git a/Tests/SWBBuildSystemTests/BuildTaskBehaviorTests.swift b/Tests/SWBBuildSystemTests/BuildTaskBehaviorTests.swift index edc2d359..a101fdee 100644 --- a/Tests/SWBBuildSystemTests/BuildTaskBehaviorTests.swift +++ b/Tests/SWBBuildSystemTests/BuildTaskBehaviorTests.swift @@ -307,7 +307,7 @@ fileprivate struct BuildTaskBehaviorTests: CoreBasedTests { } /// Check that we honor specs which are unsafe to interrupt. - @Test(.requireSDKs(.host), .skipHostOS(.windows, "no bash shell")) + @Test(.requireSDKs(.host), .skipHostOS(.windows, "no bash shell"), .skipHostOS(.freebsd, "Currently hangs on FreeBSD")) func unsafeToInterrupt() async throws { let fs = localFS let output = MakePlannedVirtualNode("") diff --git a/Tests/SWBCoreTests/CommandLineToolSpecDiscoveredInfoTests.swift b/Tests/SWBCoreTests/CommandLineToolSpecDiscoveredInfoTests.swift index 169c89c7..1b541d75 100644 --- a/Tests/SWBCoreTests/CommandLineToolSpecDiscoveredInfoTests.swift +++ b/Tests/SWBCoreTests/CommandLineToolSpecDiscoveredInfoTests.swift @@ -242,7 +242,7 @@ import SWBMacro } } - @Test(.skipHostOS(.windows), .requireSystemPackages(apt: "libtool", yum: "libtool")) + @Test(.skipHostOS(.windows), .requireSystemPackages(apt: "libtool", yum: "libtool", freebsd: "libtool")) func discoveredLibtoolSpecInfo() async throws { try await withSpec(LibtoolLinkerSpec.self, .deferred) { (info: DiscoveredLibtoolLinkerToolSpecInfo) in #expect(info.toolPath.basename == "libtool") diff --git a/Tests/SWBCoreTests/FileTextEncodingTests.swift b/Tests/SWBCoreTests/FileTextEncodingTests.swift index 190dca66..ee898097 100644 --- a/Tests/SWBCoreTests/FileTextEncodingTests.swift +++ b/Tests/SWBCoreTests/FileTextEncodingTests.swift @@ -26,8 +26,7 @@ import SWBTestSupport #expect(FileTextEncoding("utf8") != FileTextEncoding.utf8) } - @Test(.skipHostOS(.windows, "feature not available on Windows due to missing CF APIs"), - .skipHostOS(.linux, "feature not available on Linux due to missing CF APIs")) + @Test(.requireHostOS(.macOS)) // requires CoreFoundation which is macOS-only func encoding() throws { #expect(FileTextEncoding.utf8.stringEncoding == String.Encoding.utf8) #expect(FileTextEncoding.utf16.stringEncoding == String.Encoding.utf16) diff --git a/Tests/SWBCoreTests/SettingsTests.swift b/Tests/SWBCoreTests/SettingsTests.swift index fc9f0ef0..be9080a1 100644 --- a/Tests/SWBCoreTests/SettingsTests.swift +++ b/Tests/SWBCoreTests/SettingsTests.swift @@ -1772,7 +1772,7 @@ import SWBMacro #expect(!core.platformRegistry.platforms.isEmpty) for developmentTeam in ["ABCDWXYZ", ""] { for platform in core.platformRegistry.platforms { - if ["android", "linux", "qnx", "windows"].contains(platform.name) { + if ["android", "freebsd", "linux", "qnx", "windows"].contains(platform.name) { continue } for sdk in platform.sdks { diff --git a/Tests/SWBTaskExecutionTests/PBXCpTests.swift b/Tests/SWBTaskExecutionTests/PBXCpTests.swift index 109bba4d..cfb37a75 100644 --- a/Tests/SWBTaskExecutionTests/PBXCpTests.swift +++ b/Tests/SWBTaskExecutionTests/PBXCpTests.swift @@ -553,7 +553,7 @@ fileprivate struct PBXCpTests: CoreBasedTests { } } - @Test + @Test(.skipHostOS(.freebsd, "Currently hangs on FreeBSD")) func skipCopyIfContentsEqual() async throws { try await withTemporaryDirectory { tmp in let src = tmp.join("src") diff --git a/Tests/SWBUtilTests/ElapsedTimerTests.swift b/Tests/SWBUtilTests/ElapsedTimerTests.swift index e9f6dab6..3c364148 100644 --- a/Tests/SWBUtilTests/ElapsedTimerTests.swift +++ b/Tests/SWBUtilTests/ElapsedTimerTests.swift @@ -13,9 +13,10 @@ import Foundation import SWBUtil import Testing +import SWBTestSupport @Suite fileprivate struct ElapsedTimerTests { - @Test + @Test(.skipHostOS(.freebsd, "Currently hangs on FreeBSD")) func time() async throws { do { let delta = try await ElapsedTimer.measure { diff --git a/Tests/SWBUtilTests/FSProxyTests.swift b/Tests/SWBUtilTests/FSProxyTests.swift index 557a33d1..1f56717d 100644 --- a/Tests/SWBUtilTests/FSProxyTests.swift +++ b/Tests/SWBUtilTests/FSProxyTests.swift @@ -488,7 +488,7 @@ import SWBTestSupport case .android, .linux: // This will _usually_ be correct on Linux-derived OSes (see above), but not always. #expect(current_gid == ownership.group) - case .macOS, .iOS, .tvOS, .watchOS, .visionOS: + case .macOS, .iOS, .tvOS, .watchOS, .visionOS, .freebsd, .openbsd: #expect(parentDirOwnership.group == ownership.group) case .windows: // POSIX permissions don't exist, so everything is hardcoded to zero. @@ -566,7 +566,7 @@ import SWBTestSupport } } - @Test(.skipHostOS(.windows)) + @Test(.skipHostOS(.windows), .skipHostOS(.freebsd, "Blocked on https://github.com/swiftlang/swift/pull/77836")) func extendedAttributesSupport() throws { try withTemporaryDirectory { (tmpDir: Path) in // Many filesystems on other platforms (e.g. various non-ext4 temporary filesystems on Linux) don't support xattrs and will return ENOTSUP. diff --git a/Tests/SWBUtilTests/FileHandleTests.swift b/Tests/SWBUtilTests/FileHandleTests.swift index e118f507..51b504b3 100644 --- a/Tests/SWBUtilTests/FileHandleTests.swift +++ b/Tests/SWBUtilTests/FileHandleTests.swift @@ -22,7 +22,7 @@ import SystemPackage #endif @Suite fileprivate struct FileHandleTests { - @Test + @Test(.skipHostOS(.freebsd, "Currently crashes on FreeBSD")) func asyncReadFileDescriptor() async throws { let fs = localFS try await withTemporaryDirectory(fs: fs) { testDataPath in diff --git a/Tests/SWBUtilTests/HeavyCacheTests.swift b/Tests/SWBUtilTests/HeavyCacheTests.swift index 49c496b4..e4f0c9fb 100644 --- a/Tests/SWBUtilTests/HeavyCacheTests.swift +++ b/Tests/SWBUtilTests/HeavyCacheTests.swift @@ -14,6 +14,7 @@ import Foundation import Testing @_spi(Testing) import SWBUtil import Synchronization +import SWBTestSupport @Suite fileprivate struct HeavyCacheTests { @@ -105,7 +106,7 @@ fileprivate struct HeavyCacheTests { } /// Check initial TTL. - @Test + @Test(.skipHostOS(.freebsd, "Currently hangs on FreeBSD")) func TTL_initial() async throws { let fudgeFactor = 10.0 let ttl = Duration.seconds(0.01) @@ -124,7 +125,7 @@ fileprivate struct HeavyCacheTests { } /// Check TTL set after the fact. - @Test + @Test(.skipHostOS(.freebsd, "Currently hangs on FreeBSD")) func TTL_after() async throws { let fudgeFactor = 10.0 let ttl = Duration.seconds(0.01) diff --git a/Tests/SWBUtilTests/MiscTests.swift b/Tests/SWBUtilTests/MiscTests.swift index 36240ad9..f93094a4 100644 --- a/Tests/SWBUtilTests/MiscTests.swift +++ b/Tests/SWBUtilTests/MiscTests.swift @@ -25,7 +25,7 @@ import SWBUtil #expect(SWBUtil.userCacheDir().str.hasPrefix("/var/folders")) case .android: #expect(SWBUtil.userCacheDir().str.hasPrefix("/data/local/tmp")) - case .linux, .unknown: + case .linux, .freebsd, .openbsd, .unknown: #expect(SWBUtil.userCacheDir().str.hasPrefix("/tmp")) } } diff --git a/Tests/SWBUtilTests/RateLimiterTests.swift b/Tests/SWBUtilTests/RateLimiterTests.swift index 932b94fb..3affb9b4 100644 --- a/Tests/SWBUtilTests/RateLimiterTests.swift +++ b/Tests/SWBUtilTests/RateLimiterTests.swift @@ -13,8 +13,10 @@ import Foundation import Testing import SWBUtil +import SWBTestSupport -@Suite fileprivate struct RateLimiterTests { +@Suite(.skipHostOS(.freebsd, "Currently hangs on FreeBSD")) +fileprivate struct RateLimiterTests { @Test func rateLimiterSeconds() async throws { let timer = ElapsedTimer() From e072c6d3c639d7a84b65bed2c31f8ae7e74ba774 Mon Sep 17 00:00:00 2001 From: Jake Petroules Date: Tue, 15 Jul 2025 08:39:57 -0700 Subject: [PATCH 2/5] Make Android NDK discovery more robust, and add tests Add support for Android NDK versions 23 through 26, and emit an explicit error for older versions. Add test coverage to verify parsing of the metadata. I chose 23 as the cutoff for now simply because that's the next last time that the abis.json schema changed. That version was released in August 2021. We can add older versions if anyone really wants them. --- Sources/SWBAndroidPlatform/AndroidSDK.swift | 231 +++++++---- Sources/SWBAndroidPlatform/Plugin.swift | 11 +- .../AndroidSDKTests.swift | 361 ++++++++++++++++++ 3 files changed, 523 insertions(+), 80 deletions(-) create mode 100644 Tests/SWBAndroidPlatformTests/AndroidSDKTests.swift diff --git a/Sources/SWBAndroidPlatform/AndroidSDK.swift b/Sources/SWBAndroidPlatform/AndroidSDK.swift index 44b93114..cb66619d 100644 --- a/Sources/SWBAndroidPlatform/AndroidSDK.swift +++ b/Sources/SWBAndroidPlatform/AndroidSDK.swift @@ -10,30 +10,51 @@ // //===----------------------------------------------------------------------===// -import SWBUtil -import Foundation +public import SWBUtil +public import Foundation -struct AndroidSDK: Sendable { +@_spi(Testing) public struct AndroidSDK: Sendable { public let host: OperatingSystem public let path: Path - public let ndkVersion: Version? + + /// List of NDKs available in this SDK installation, sorted by version number from oldest to newest. + @_spi(Testing) public let ndks: [NDK] + + public var latestNDK: NDK? { + ndks.last + } init(host: OperatingSystem, path: Path, fs: any FSProxy) throws { self.host = host self.path = path + self.ndks = try NDK.findInstallations(host: host, sdkPath: path, fs: fs) + } - let ndkBasePath = path.join("ndk") - if fs.exists(ndkBasePath) { - self.ndkVersion = try fs.listdir(ndkBasePath).map { try Version($0) }.max() - } else { - self.ndkVersion = nil - } + @_spi(Testing) public struct NDK: Equatable, Sendable { + public static let minimumNDKVersion = Version(23) + + public let host: OperatingSystem + public let path: Path + public let version: Version + public let abis: [String: ABI] + public let deploymentTargetRange: DeploymentTargetRange + + init(host: OperatingSystem, path ndkPath: Path, version: Version, fs: any FSProxy) throws { + self.host = host + self.path = ndkPath + self.version = version - if let ndkVersion { - let ndkPath = ndkBasePath.join(ndkVersion.description) let metaPath = ndkPath.join("meta") - self.abis = try JSONDecoder().decode([String: ABI].self, from: Data(fs.read(metaPath.join("abis.json")))) + guard #available(macOS 14, *) else { + throw StubError.error("Unsupported macOS version") + } + + if version < Self.minimumNDKVersion { + throw StubError.error("Android NDK version at path '\(ndkPath.str)' is not supported (r\(Self.minimumNDKVersion.description) or later required)") + } + + self.abis = try JSONDecoder().decode(ABIs.self, from: Data(fs.read(metaPath.join("abis.json"))), configuration: version).abis struct PlatformsInfo: Codable { let min: Int @@ -41,87 +62,145 @@ struct AndroidSDK: Sendable { } let platformsInfo = try JSONDecoder().decode(PlatformsInfo.self, from: Data(fs.read(metaPath.join("platforms.json")))) - self.ndkPath = ndkPath - deploymentTargetRange = (platformsInfo.min, platformsInfo.max) - } else { - ndkPath = nil - deploymentTargetRange = nil - abis = nil + deploymentTargetRange = DeploymentTargetRange(min: platformsInfo.min, max: platformsInfo.max) } - } - struct ABI: Codable { - enum Bitness: Int, Codable { - case bits32 = 32 - case bits64 = 64 + struct ABIs: DecodableWithConfiguration { + let abis: [String: ABI] + + init(from decoder: any Decoder, configuration: Version) throws { + struct DynamicCodingKey: CodingKey { + var stringValue: String + + init?(stringValue: String) { + self.stringValue = stringValue + } + + let intValue: Int? = nil + + init?(intValue: Int) { + nil + } + } + let container = try decoder.container(keyedBy: DynamicCodingKey.self) + abis = try Dictionary(uniqueKeysWithValues: container.allKeys.map { try ($0.stringValue, container.decode(ABI.self, forKey: $0, configuration: configuration)) }) + } } - struct LLVMTriple: Codable { - var arch: String - var vendor: String - var system: String - var environment: String - - var description: String { - "\(arch)-\(vendor)-\(system)-\(environment)" + @_spi(Testing) public struct ABI: DecodableWithConfiguration, Equatable, Sendable { + @_spi(Testing) public enum Bitness: Int, Codable, Equatable, Sendable { + case bits32 = 32 + case bits64 = 64 } - init(from decoder: any Decoder) throws { - let container = try decoder.singleValueContainer() - let triple = try container.decode(String.self) - if let match = try #/(?.+)-(?.+)-(?.+)-(?.+)/#.wholeMatch(in: triple) { - self.arch = String(match.output.arch) - self.vendor = String(match.output.vendor) - self.system = String(match.output.system) - self.environment = String(match.output.environment) - } else { - throw DecodingError.dataCorruptedError(in: container, debugDescription: "Invalid triple string: \(triple)") + @_spi(Testing) public struct LLVMTriple: Codable, Equatable, Sendable { + public var arch: String + public var vendor: String + public var system: String + public var environment: String + + var description: String { + "\(arch)-\(vendor)-\(system)-\(environment)" + } + + public init(from decoder: any Decoder) throws { + let container = try decoder.singleValueContainer() + let triple = try container.decode(String.self) + if let match = try #/(?.+)-(?.+)-(?.+)-(?.+)/#.wholeMatch(in: triple) { + self.arch = String(match.output.arch) + self.vendor = String(match.output.vendor) + self.system = String(match.output.system) + self.environment = String(match.output.environment) + } else { + throw DecodingError.dataCorruptedError(in: container, debugDescription: "Invalid triple string: \(triple)") + } } } - func encode(to encoder: any Encoder) throws { - var container = encoder.singleValueContainer() - try container.encode(description) + public let bitness: Bitness + public let `default`: Bool + public let deprecated: Bool + public let proc: String + public let arch: String + public let triple: String + public let llvm_triple: LLVMTriple + public let min_os_version: Int + + enum CodingKeys: String, CodingKey { + case bitness + case `default` = "default" + case deprecated + case proc + case arch + case triple + case llvm_triple = "llvm_triple" + case min_os_version = "min_os_version" + } + + public init(from decoder: any Decoder, configuration ndkVersion: Version) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.bitness = try container.decode(Bitness.self, forKey: .bitness) + self.default = try container.decode(Bool.self, forKey: .default) + self.deprecated = try container.decode(Bool.self, forKey: .deprecated) + self.proc = try container.decode(String.self, forKey: .proc) + self.arch = try container.decode(String.self, forKey: .arch) + self.triple = try container.decode(String.self, forKey: .triple) + self.llvm_triple = try container.decode(LLVMTriple.self, forKey: .llvm_triple) + self.min_os_version = try container.decodeIfPresent(Int.self, forKey: .min_os_version) ?? { + if ndkVersion < Version(27) { + return 21 // min_os_version wasn't present prior to NDKr27, fill it in with 21, which is the appropriate value + } else { + throw DecodingError.valueNotFound(Int.self, .init(codingPath: container.codingPath, debugDescription: "No value associated with key \(CodingKeys.min_os_version) (\"\(CodingKeys.min_os_version.rawValue)\").")) + } + }() } } - let bitness: Bitness - let `default`: Bool - let deprecated: Bool - let proc: String - let arch: String - let triple: String - let llvm_triple: LLVMTriple - let min_os_version: Int - } + @_spi(Testing) public struct DeploymentTargetRange: Equatable, Sendable { + public let min: Int + public let max: Int + } - public let abis: [String: ABI]? + public var toolchainPath: Path { + path.join("toolchains").join("llvm").join("prebuilt").join(hostTag) + } - public let deploymentTargetRange: (min: Int, max: Int)? + public var sysroot: Path { + toolchainPath.join("sysroot") + } - public let ndkPath: Path? + private var hostTag: String? { + switch host { + case .windows: + // Also works on Windows on ARM via Prism binary translation. + "windows-x86_64" + case .macOS: + // Despite the x86_64 tag in the Darwin name, these are universal binaries including arm64. + "darwin-x86_64" + case .linux: + // Also works on non-x86 archs via binfmt support and qemu (or Rosetta on Apple-hosted VMs). + "linux-x86_64" + default: + nil // unsupported host + } + } - public var toolchainPath: Path? { - ndkPath?.join("toolchains").join("llvm").join("prebuilt").join(hostTag) - } + public static func findInstallations(host: OperatingSystem, sdkPath: Path, fs: any FSProxy) throws -> [NDK] { + let ndkBasePath = sdkPath.join("ndk") + guard fs.exists(ndkBasePath) else { + return [] + } - public var sysroot: Path? { - toolchainPath?.join("sysroot") - } + let ndks = try fs.listdir(ndkBasePath).map({ try Version($0) }).sorted() + let supportedNdks = ndks.filter { $0 >= minimumNDKVersion } - private var hostTag: String? { - switch host { - case .windows: - // Also works on Windows on ARM via Prism binary translation. - "windows-x86_64" - case .macOS: - // Despite the x86_64 tag in the Darwin name, these are universal binaries including arm64. - "darwin-x86_64" - case .linux: - // Also works on non-x86 archs via binfmt support and qemu (or Rosetta on Apple-hosted VMs). - "linux-x86_64" - default: - nil // unsupported host + // If we have some NDKs but all of them are unsupported, try parsing them so that parsing fails and provides a more useful error. Otherwise, simply filter out and ignore the unsupported versions. + let discoveredNdks = supportedNdks.isEmpty && !ndks.isEmpty ? ndks : supportedNdks + + return try discoveredNdks.map { ndkVersion in + let ndkPath = ndkBasePath.join(ndkVersion.description) + return try NDK(host: host, path: ndkPath, version: ndkVersion, fs: fs) + } } } diff --git a/Sources/SWBAndroidPlatform/Plugin.swift b/Sources/SWBAndroidPlatform/Plugin.swift index 8b6c8d5a..88fa5f76 100644 --- a/Sources/SWBAndroidPlatform/Plugin.swift +++ b/Sources/SWBAndroidPlatform/Plugin.swift @@ -54,7 +54,7 @@ struct AndroidEnvironmentExtension: EnvironmentExtension { if let latest = try? await plugin.cachedAndroidSDKInstallations(host: context.hostOperatingSystem).first { return [ "ANDROID_SDK_ROOT": latest.path.str, - "ANDROID_NDK_ROOT": latest.ndkPath?.str, + "ANDROID_NDK_ROOT": latest.ndks.last?.path.str, ].compactMapValues { $0 } } default: @@ -112,10 +112,13 @@ struct AndroidSDKRegistryExtension: SDKRegistryExtension { return [] } - guard let abis = androidSdk.abis, let deploymentTargetRange = androidSdk.deploymentTargetRange else { + guard let androidNdk = androidSdk.latestNDK else { return [] } + let abis = androidNdk.abis + let deploymentTargetRange = androidNdk.deploymentTargetRange + let allPossibleTriples = abis.values.flatMap { abi in (max(deploymentTargetRange.min, abi.min_os_version)...deploymentTargetRange.max).map { deploymentTarget in var triple = abi.llvm_triple @@ -147,7 +150,7 @@ struct AndroidSDKRegistryExtension: SDKRegistryExtension { swiftSettings = [:] } - return [(androidSdk.sysroot ?? .root, androidPlatform, [ + return [(androidNdk.sysroot, androidPlatform, [ "Type": .plString("SDK"), "Version": .plString("0.0.0"), "CanonicalName": .plString("android"), @@ -184,7 +187,7 @@ struct AndroidToolchainRegistryExtension: ToolchainRegistryExtension { let plugin: AndroidPlugin func additionalToolchains(context: any ToolchainRegistryExtensionAdditionalToolchainsContext) async throws -> [Toolchain] { - guard let toolchainPath = try? await plugin.cachedAndroidSDKInstallations(host: context.hostOperatingSystem).first?.toolchainPath else { + guard let toolchainPath = try? await plugin.cachedAndroidSDKInstallations(host: context.hostOperatingSystem).first?.latestNDK?.toolchainPath else { return [] } diff --git a/Tests/SWBAndroidPlatformTests/AndroidSDKTests.swift b/Tests/SWBAndroidPlatformTests/AndroidSDKTests.swift new file mode 100644 index 00000000..37329d94 --- /dev/null +++ b/Tests/SWBAndroidPlatformTests/AndroidSDKTests.swift @@ -0,0 +1,361 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation +@_spi(Testing) import SWBAndroidPlatform +import SWBTestSupport +import SWBUtil +import Testing + +@Suite +fileprivate struct AndroidSDKTests { + @Test func findInstallations() async throws { + let host = try ProcessInfo.processInfo.hostOperatingSystem() + let installations = try await AndroidSDK.findInstallations(host: host, fs: localFS) + // It's OK if `installations` is an empty set, the host system might have no Android SDK/NDK installed + for installation in installations { + #expect(installation.host == host) + #expect(installation.latestNDK == installation.ndks.last) + } + } + + @Test func abis_r22() async throws { + try await withNDKVersion(version: Version("22.1.7171670")) { host, fs, ndkVersionPath in + let error = #expect(throws: StubError.self) { + try AndroidSDK.NDK.findInstallations(host: host, sdkPath: ndkVersionPath.dirname.dirname, fs: fs) + } + #expect(error?.description == "Android NDK version at path '\(ndkVersionPath.str)' is not supported (r23 or later required)") + } + } + + @Test func abis_r26_3() async throws { + try await withNDKVersion(version: Version("26.3.11579264")) { host, fs, ndkVersionPath in + try await fs.writeFileContents(ndkVersionPath.join("meta").join("abis.json")) { contents in + contents <<< + """ + { + "armeabi-v7a": { + "bitness": 32, + "default": true, + "deprecated": false, + "proc": "armv7-a", + "arch": "arm", + "triple": "arm-linux-androideabi", + "llvm_triple": "armv7-none-linux-androideabi" + }, + "arm64-v8a": { + "bitness": 64, + "default": true, + "deprecated": false, + "proc": "aarch64", + "arch": "arm64", + "triple": "aarch64-linux-android", + "llvm_triple": "aarch64-none-linux-android" + }, + "x86": { + "bitness": 32, + "default": true, + "deprecated": false, + "proc": "i686", + "arch": "x86", + "triple": "i686-linux-android", + "llvm_triple": "i686-none-linux-android" + }, + "x86_64": { + "bitness": 64, + "default": true, + "deprecated": false, + "proc": "x86_64", + "arch": "x86_64", + "triple": "x86_64-linux-android", + "llvm_triple": "x86_64-none-linux-android" + } + } + """ + } + + try await fs.writeFileContents(ndkVersionPath.join("meta").join("platforms.json")) { contents in + contents <<< + """ + { + "min": 21, + "max": 34, + "aliases": { + "20": 19, + "25": 24, + "J": 16, + "J-MR1": 17, + "J-MR2": 18, + "K": 19, + "L": 21, + "L-MR1": 22, + "M": 23, + "N": 24, + "N-MR1": 24, + "O": 26, + "O-MR1": 27, + "P": 28, + "Q": 29, + "R": 30, + "S": 31, + "Sv2": 32, + "Tiramisu": 33, + "UpsideDownCake": 34 + } + } + """ + } + + let installations = try AndroidSDK.NDK.findInstallations(host: host, sdkPath: ndkVersionPath.dirname.dirname, fs: fs) + let installation = try #require(installations.only) + #expect(installation.host == host) + #expect(installation.path == ndkVersionPath) + #expect(try installation.version == Version("26.3.11579264")) + #expect(installation.deploymentTargetRange.min == 21) + #expect(installation.deploymentTargetRange.max == 34) + + #expect(installation.abis.count == 4) + + let armv7 = try #require(installation.abis["armeabi-v7a"]) + #expect(armv7.bitness == .bits32) + #expect(armv7.default == true) + #expect(armv7.deprecated == false) + #expect(armv7.proc == "armv7-a") + #expect(armv7.arch == "arm") + #expect(armv7.triple == "arm-linux-androideabi") + #expect(armv7.llvm_triple.arch == "armv7") + #expect(armv7.llvm_triple.vendor == "none") + #expect(armv7.llvm_triple.system == "linux") + #expect(armv7.llvm_triple.environment == "androideabi") + #expect(armv7.min_os_version == 21) + + let arm64 = try #require(installation.abis["arm64-v8a"]) + #expect(arm64.bitness == .bits64) + #expect(arm64.default == true) + #expect(arm64.deprecated == false) + #expect(arm64.proc == "aarch64") + #expect(arm64.arch == "arm64") + #expect(arm64.triple == "aarch64-linux-android") + #expect(arm64.llvm_triple.arch == "aarch64") + #expect(arm64.llvm_triple.vendor == "none") + #expect(arm64.llvm_triple.system == "linux") + #expect(arm64.llvm_triple.environment == "android") + #expect(arm64.min_os_version == 21) + + let x86 = try #require(installation.abis["x86"]) + #expect(x86.bitness == .bits32) + #expect(x86.default == true) + #expect(x86.deprecated == false) + #expect(x86.proc == "i686") + #expect(x86.arch == "x86") + #expect(x86.triple == "i686-linux-android") + #expect(x86.llvm_triple.arch == "i686") + #expect(x86.llvm_triple.vendor == "none") + #expect(x86.llvm_triple.system == "linux") + #expect(x86.llvm_triple.environment == "android") + #expect(x86.min_os_version == 21) + + let x86_64 = try #require(installation.abis["x86_64"]) + #expect(x86_64.bitness == .bits64) + #expect(x86_64.default == true) + #expect(x86_64.deprecated == false) + #expect(x86_64.proc == "x86_64") + #expect(x86_64.arch == "x86_64") + #expect(x86_64.triple == "x86_64-linux-android") + #expect(x86_64.llvm_triple.arch == "x86_64") + #expect(x86_64.llvm_triple.vendor == "none") + #expect(x86_64.llvm_triple.system == "linux") + #expect(x86_64.llvm_triple.environment == "android") + #expect(x86_64.min_os_version == 21) + } + } + + @Test func abis_r27() async throws { + try await withNDKVersion(version: Version("27.0.11718014")) { host, fs, ndkVersionPath in + try await fs.writeFileContents(ndkVersionPath.join("meta").join("abis.json")) { contents in + contents <<< + """ + { + "armeabi-v7a": { + "bitness": 32, + "default": true, + "deprecated": false, + "proc": "armv7-a", + "arch": "arm", + "triple": "arm-linux-androideabi", + "llvm_triple": "armv7-none-linux-androideabi", + "min_os_version": 21 + }, + "arm64-v8a": { + "bitness": 64, + "default": true, + "deprecated": false, + "proc": "aarch64", + "arch": "arm64", + "triple": "aarch64-linux-android", + "llvm_triple": "aarch64-none-linux-android", + "min_os_version": 21 + }, + "riscv64": { + "bitness": 64, + "default": false, + "deprecated": false, + "proc": "riscv64", + "arch": "riscv64", + "triple": "riscv64-linux-android", + "llvm_triple": "riscv64-none-linux-android", + "min_os_version": 35 + }, + "x86": { + "bitness": 32, + "default": true, + "deprecated": false, + "proc": "i686", + "arch": "x86", + "triple": "i686-linux-android", + "llvm_triple": "i686-none-linux-android", + "min_os_version": 21 + }, + "x86_64": { + "bitness": 64, + "default": true, + "deprecated": false, + "proc": "x86_64", + "arch": "x86_64", + "triple": "x86_64-linux-android", + "llvm_triple": "x86_64-none-linux-android", + "min_os_version": 21 + } + } + """ + } + + try await fs.writeFileContents(ndkVersionPath.join("meta").join("platforms.json")) { contents in + contents <<< + """ + { + "min": 21, + "max": 35, + "aliases": { + "20": 19, + "25": 24, + "J": 16, + "J-MR1": 17, + "J-MR2": 18, + "K": 19, + "L": 21, + "L-MR1": 22, + "M": 23, + "N": 24, + "N-MR1": 24, + "O": 26, + "O-MR1": 27, + "P": 28, + "Q": 29, + "R": 30, + "S": 31, + "Sv2": 32, + "Tiramisu": 33, + "UpsideDownCake": 34, + "VanillaIceCream": 35 + } + } + """ + } + + let installations = try AndroidSDK.NDK.findInstallations(host: host, sdkPath: ndkVersionPath.dirname.dirname, fs: fs) + let installation = try #require(installations.only) + #expect(installation.host == host) + #expect(installation.path == ndkVersionPath) + #expect(try installation.version == Version("27.0.11718014")) + #expect(installation.deploymentTargetRange.min == 21) + #expect(installation.deploymentTargetRange.max == 35) + + #expect(installation.abis.count == 5) + + let armv7 = try #require(installation.abis["armeabi-v7a"]) + #expect(armv7.bitness == .bits32) + #expect(armv7.default == true) + #expect(armv7.deprecated == false) + #expect(armv7.proc == "armv7-a") + #expect(armv7.arch == "arm") + #expect(armv7.triple == "arm-linux-androideabi") + #expect(armv7.llvm_triple.arch == "armv7") + #expect(armv7.llvm_triple.vendor == "none") + #expect(armv7.llvm_triple.system == "linux") + #expect(armv7.llvm_triple.environment == "androideabi") + #expect(armv7.min_os_version == 21) + + let arm64 = try #require(installation.abis["arm64-v8a"]) + #expect(arm64.bitness == .bits64) + #expect(arm64.default == true) + #expect(arm64.deprecated == false) + #expect(arm64.proc == "aarch64") + #expect(arm64.arch == "arm64") + #expect(arm64.triple == "aarch64-linux-android") + #expect(arm64.llvm_triple.arch == "aarch64") + #expect(arm64.llvm_triple.vendor == "none") + #expect(arm64.llvm_triple.system == "linux") + #expect(arm64.llvm_triple.environment == "android") + #expect(arm64.min_os_version == 21) + + let riscv64 = try #require(installation.abis["riscv64"]) + #expect(riscv64.bitness == .bits64) + #expect(riscv64.default == false) + #expect(riscv64.deprecated == false) + #expect(riscv64.proc == "riscv64") + #expect(riscv64.arch == "riscv64") + #expect(riscv64.triple == "riscv64-linux-android") + #expect(riscv64.llvm_triple.arch == "riscv64") + #expect(riscv64.llvm_triple.vendor == "none") + #expect(riscv64.llvm_triple.system == "linux") + #expect(riscv64.llvm_triple.environment == "android") + #expect(riscv64.min_os_version == 35) + + let x86 = try #require(installation.abis["x86"]) + #expect(x86.bitness == .bits32) + #expect(x86.default == true) + #expect(x86.deprecated == false) + #expect(x86.proc == "i686") + #expect(x86.arch == "x86") + #expect(x86.triple == "i686-linux-android") + #expect(x86.llvm_triple.arch == "i686") + #expect(x86.llvm_triple.vendor == "none") + #expect(x86.llvm_triple.system == "linux") + #expect(x86.llvm_triple.environment == "android") + #expect(x86.min_os_version == 21) + + let x86_64 = try #require(installation.abis["x86_64"]) + #expect(x86_64.bitness == .bits64) + #expect(x86_64.default == true) + #expect(x86_64.deprecated == false) + #expect(x86_64.proc == "x86_64") + #expect(x86_64.arch == "x86_64") + #expect(x86_64.triple == "x86_64-linux-android") + #expect(x86_64.llvm_triple.arch == "x86_64") + #expect(x86_64.llvm_triple.vendor == "none") + #expect(x86_64.llvm_triple.system == "linux") + #expect(x86_64.llvm_triple.environment == "android") + #expect(x86_64.min_os_version == 21) + } + } + + private func withNDKVersion(version: Version, _ block: (OperatingSystem, any FSProxy, Path) async throws -> ()) async throws { + let fs = PseudoFS() + let ndkPath = Path.root.join("ndk") + let ndkVersionPath = ndkPath.join(version.description) + try fs.createDirectory(ndkPath, recursive: true) + try fs.createDirectory(ndkVersionPath.join("meta"), recursive: true) + let host = try ProcessInfo.processInfo.hostOperatingSystem() + try await block(host, fs, ndkVersionPath) + } +} From f0df6bf1654e396fbd5e74c9b7f2e13c1842bb8e Mon Sep 17 00:00:00 2001 From: Jake Petroules Date: Fri, 1 Aug 2025 18:26:46 -0700 Subject: [PATCH 3/5] Improve Android SDK/NDK discovery Now respects the environment variable overrides and looks for the Debian/Ubuntu package location. Closes #495 --- .github/scripts/linux_pre_build.sh | 66 +++++++ .github/workflows/pull_request.yml | 36 +--- Sources/SWBAndroidPlatform/AndroidSDK.swift | 187 ++++++++++++++---- Sources/SWBAndroidPlatform/CMakeLists.txt | 1 + .../SWBAndroidPlatform/JavaProperties.swift | 32 +++ Sources/SWBAndroidPlatform/Plugin.swift | 44 +++-- Sources/SWBUtil/FSProxy.swift | 10 +- Sources/SWBUtil/Path.swift | 4 + Sources/SWBUtil/URL.swift | 12 +- .../AndroidSDKTests.swift | 111 ++++++++--- .../SWBAndroidPlatformTests.swift | 7 + 11 files changed, 392 insertions(+), 118 deletions(-) create mode 100755 .github/scripts/linux_pre_build.sh create mode 100644 Sources/SWBAndroidPlatform/JavaProperties.swift diff --git a/.github/scripts/linux_pre_build.sh b/.github/scripts/linux_pre_build.sh new file mode 100755 index 00000000..8adf2410 --- /dev/null +++ b/.github/scripts/linux_pre_build.sh @@ -0,0 +1,66 @@ +#!/bin/bash +##===----------------------------------------------------------------------===## +## +## This source file is part of the Swift open source project +## +## Copyright (c) 2025 Apple Inc. and the Swift project authors +## Licensed under Apache License v2.0 with Runtime Library Exception +## +## See http://swift.org/LICENSE.txt for license information +## See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +## +##===----------------------------------------------------------------------===## + +set -e + +if command -v apt-get >/dev/null 2>&1 ; then # bookworm, noble, jammy + export DEBIAN_FRONTEND=noninteractive + + apt-get update -y + + # Build dependencies + apt-get install -y libsqlite3-dev libncurses-dev + + # Debug symbols + apt-get install -y libc6-dbg + + if [[ "$INSTALL_CMAKE" == "1" ]] ; then + apt-get install -y cmake ninja-build + fi + + # Android NDK + dpkg_architecture="$(dpkg --print-architecture)" + if [[ "$SKIP_ANDROID" != "1" ]] && [[ "$dpkg_architecture" == amd64 ]] ; then + eval "$(cat /etc/lsb-release)" + case "$DISTRIB_CODENAME" in + bookworm|jammy) + : # Not available + ;; + noble) + apt-get install -y google-android-ndk-r26c-installer + ;; + *) + echo "Unknown distribution: $DISTRIB_CODENAME" >&2 + exit 1 + esac + else + echo "Skipping Android NDK installation on $dpkg_architecture" >&2 + fi +elif command -v dnf >/dev/null 2>&1 ; then # rhel-ubi9 + dnf update -y + + # Build dependencies + dnf install -y sqlite-devel ncurses-devel + + # Debug symbols + dnf debuginfo-install -y glibc +elif command -v yum >/dev/null 2>&1 ; then # amazonlinux2 + yum update -y + + # Build dependencies + yum install -y sqlite-devel ncurses-devel + + # Debug symbols + yum install -y yum-utils + debuginfo-install -y glibc +fi diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index c39112b4..1f1e70d1 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -14,33 +14,7 @@ jobs: uses: swiftlang/github-workflows/.github/workflows/swift_package_test.yml@main with: linux_os_versions: '["noble", "jammy", "rhel-ubi9"]' - linux_pre_build_command: | - if command -v apt-get >/dev/null 2>&1 ; then # bookworm, noble, jammy - apt-get update -y - - # Build dependencies - apt-get install -y libsqlite3-dev libncurses-dev - - # Debug symbols - apt-get install -y libc6-dbg - elif command -v dnf >/dev/null 2>&1 ; then # rhel-ubi9 - dnf update -y - - # Build dependencies - dnf install -y sqlite-devel ncurses-devel - - # Debug symbols - dnf debuginfo-install -y glibc - elif command -v yum >/dev/null 2>&1 ; then # amazonlinux2 - yum update -y - - # Build dependencies - yum install -y sqlite-devel ncurses-devel - - # Debug symbols - yum install -y yum-utils - debuginfo-install -y glibc - fi + linux_pre_build_command: ./.github/scripts/linux_pre_build.sh linux_build_command: 'swift test --no-parallel' linux_swift_versions: '["nightly-main", "nightly-6.2"]' windows_swift_versions: '["nightly-main"]' @@ -50,13 +24,7 @@ jobs: uses: swiftlang/github-workflows/.github/workflows/swift_package_test.yml@main with: linux_os_versions: '["noble"]' - linux_pre_build_command: | - apt-get update -y - - # Build dependencies - apt-get install -y libsqlite3-dev libncurses-dev - - apt-get install -y cmake ninja-build + linux_pre_build_command: SKIP_ANDROID=1 INSTALL_CMAKE=1 ./.github/scripts/linux_pre_build.sh linux_build_command: 'swift package -Xbuild-tools-swiftc -DUSE_PROCESS_SPAWNING_WORKAROUND cmake-smoke-test --disable-sandbox --cmake-path `which cmake` --ninja-path `which ninja` --extra-cmake-arg -DCMAKE_C_COMPILER=`which clang` --extra-cmake-arg -DCMAKE_CXX_COMPILER=`which clang++` --extra-cmake-arg -DCMAKE_Swift_COMPILER=`which swiftc`' linux_swift_versions: '["nightly-main"]' windows_swift_versions: '[]' diff --git a/Sources/SWBAndroidPlatform/AndroidSDK.swift b/Sources/SWBAndroidPlatform/AndroidSDK.swift index cb66619d..3b934838 100644 --- a/Sources/SWBAndroidPlatform/AndroidSDK.swift +++ b/Sources/SWBAndroidPlatform/AndroidSDK.swift @@ -15,43 +15,54 @@ public import Foundation @_spi(Testing) public struct AndroidSDK: Sendable { public let host: OperatingSystem - public let path: Path + public let path: AbsolutePath + private let ndkInstallations: NDK.Installations /// List of NDKs available in this SDK installation, sorted by version number from oldest to newest. - @_spi(Testing) public let ndks: [NDK] + @_spi(Testing) public var ndks: [NDK] { + ndkInstallations.ndks + } - public var latestNDK: NDK? { - ndks.last + public var preferredNDK: NDK? { + ndkInstallations.preferredNDK ?? ndks.last } - init(host: OperatingSystem, path: Path, fs: any FSProxy) throws { + init(host: OperatingSystem, path: AbsolutePath, fs: any FSProxy) throws { self.host = host self.path = path - self.ndks = try NDK.findInstallations(host: host, sdkPath: path, fs: fs) + self.ndkInstallations = try NDK.findInstallations(host: host, sdkPath: path, fs: fs) } @_spi(Testing) public struct NDK: Equatable, Sendable { public static let minimumNDKVersion = Version(23) public let host: OperatingSystem - public let path: Path + public let path: AbsolutePath public let version: Version public let abis: [String: ABI] public let deploymentTargetRange: DeploymentTargetRange - init(host: OperatingSystem, path ndkPath: Path, version: Version, fs: any FSProxy) throws { + @_spi(Testing) public init(host: OperatingSystem, path ndkPath: AbsolutePath, fs: any FSProxy) throws { self.host = host self.path = ndkPath - self.version = version + self.toolchainPath = try AbsolutePath(validating: path.path.join("toolchains").join("llvm").join("prebuilt").join(Self.hostTag(host))) + self.sysroot = try AbsolutePath(validating: toolchainPath.path.join("sysroot")) + + let propertiesFile = ndkPath.path.join("source.properties") + guard fs.exists(propertiesFile) else { + throw Error.notAnNDK(ndkPath) + } - let metaPath = ndkPath.join("meta") + self.version = try NDK.Properties(data: Data(fs.read(propertiesFile))).revision + + let metaPath = ndkPath.path.join("meta") guard #available(macOS 14, *) else { throw StubError.error("Unsupported macOS version") } if version < Self.minimumNDKVersion { - throw StubError.error("Android NDK version at path '\(ndkPath.str)' is not supported (r\(Self.minimumNDKVersion.description) or later required)") + throw Error.unsupportedVersion(path: ndkPath, minimumVersion: Self.minimumNDKVersion) } self.abis = try JSONDecoder().decode(ABIs.self, from: Data(fs.read(metaPath.join("abis.json"))), configuration: version).abis @@ -65,6 +76,36 @@ public import Foundation deploymentTargetRange = DeploymentTargetRange(min: platformsInfo.min, max: platformsInfo.max) } + public enum Error: Swift.Error, CustomStringConvertible, Sendable { + case notAnNDK(AbsolutePath) + case unsupportedVersion(path: AbsolutePath, minimumVersion: Version) + case noSupportedVersions(minimumVersion: Version) + + public var description: String { + switch self { + case let .notAnNDK(path): + "Package at path '\(path.path.str)' is not an Android NDK (no source.properties file)" + case let .unsupportedVersion(path, minimumVersion): + "Android NDK version at path '\(path.path.str)' is not supported (r\(minimumVersion.description) or later required)" + case let .noSupportedVersions(minimumVersion): + "All installed NDK versions are not supported (r\(minimumVersion.description) or later required)" + } + } + } + + struct Properties { + let properties: JavaProperties + let revision: Version + + init(data: Data) throws { + properties = try .init(data: data) + guard properties["Pkg.Desc"] == "Android NDK" else { + throw StubError.error("Package is not an Android NDK") + } + revision = try Version(properties["Pkg.BaseRevision"] ?? properties["Pkg.Revision"] ?? "") + } + } + struct ABIs: DecodableWithConfiguration { let abis: [String: ABI] @@ -161,15 +202,10 @@ public import Foundation public let max: Int } - public var toolchainPath: Path { - path.join("toolchains").join("llvm").join("prebuilt").join(hostTag) - } - - public var sysroot: Path { - toolchainPath.join("sysroot") - } + public let toolchainPath: AbsolutePath + public let sysroot: AbsolutePath - private var hostTag: String? { + private static func hostTag(_ host: OperatingSystem) -> String? { switch host { case .windows: // Also works on Windows on ARM via Prism binary translation. @@ -185,44 +221,119 @@ public import Foundation } } - public static func findInstallations(host: OperatingSystem, sdkPath: Path, fs: any FSProxy) throws -> [NDK] { - let ndkBasePath = sdkPath.join("ndk") + public struct Installations: Sendable { + private let preferredIndex: Int? + public let ndks: [NDK] + + init(preferredIndex: Int? = nil, ndks: [NDK]) { + self.preferredIndex = preferredIndex + self.ndks = ndks + } + + public var preferredNDK: NDK? { + preferredIndex.map { ndks[$0] } ?? ndks.only + } + } + + public static func findInstallations(host: OperatingSystem, sdkPath: AbsolutePath, fs: any FSProxy) throws -> Installations { + if let overridePath = NDK.environmentOverrideLocation { + return try Installations(ndks: [NDK(host: host, path: overridePath, fs: fs)]) + } + + let ndkBasePath = sdkPath.path.join("ndk") guard fs.exists(ndkBasePath) else { - return [] + return Installations(ndks: []) } - let ndks = try fs.listdir(ndkBasePath).map({ try Version($0) }).sorted() - let supportedNdks = ndks.filter { $0 >= minimumNDKVersion } + var hadUnsupportedVersions: Bool = false + let ndks = try fs.listdir(ndkBasePath).compactMap({ subdir in + do { + return try NDK(host: host, path: AbsolutePath(validating: ndkBasePath.join(subdir)), fs: fs) + } catch Error.notAnNDK(_) { + return nil + } catch Error.unsupportedVersion(_, _) { + hadUnsupportedVersions = true + return nil + } + }).sorted(by: \.version) - // If we have some NDKs but all of them are unsupported, try parsing them so that parsing fails and provides a more useful error. Otherwise, simply filter out and ignore the unsupported versions. - let discoveredNdks = supportedNdks.isEmpty && !ndks.isEmpty ? ndks : supportedNdks + // If we have some NDKs but all of them are unsupported, provide a more useful error. Otherwise, simply filter out and ignore the unsupported versions. + if ndks.isEmpty && hadUnsupportedVersions { + throw Error.noSupportedVersions(minimumVersion: Self.minimumNDKVersion) + } - return try discoveredNdks.map { ndkVersion in - let ndkPath = ndkBasePath.join(ndkVersion.description) - return try NDK(host: host, path: ndkPath, version: ndkVersion, fs: fs) + // Respect Debian alternatives + let preferredIndex: Int? + if sdkPath == AndroidSDK.defaultDebianLocation, let ndkLinkPath = AndroidSDK.NDK.defaultDebianLocation { + preferredIndex = try ndks.firstIndex(where: { try $0.path.path == fs.realpath(ndkLinkPath.path) }) + } else { + preferredIndex = nil } + + return Installations(preferredIndex: preferredIndex, ndks: ndks) } } public static func findInstallations(host: OperatingSystem, fs: any FSProxy) async throws -> [AndroidSDK] { - let defaultLocation: Path? = switch host { + var paths: [AbsolutePath] = [] + if let path = AndroidSDK.environmentOverrideLocation { + paths.append(path) + } + if let path = try AndroidSDK.defaultAndroidStudioLocation(host: host) { + paths.append(path) + } + if let path = AndroidSDK.defaultDebianLocation, host == .linux { + paths.append(path) + } + return try paths.compactMap { path in + guard fs.exists(path.path) else { + return nil + } + return try AndroidSDK(host: host, path: path, fs: fs) + } + } +} + +fileprivate extension AndroidSDK.NDK { + /// The location of the Android NDK based on the `ANDROID_NDK_ROOT` environment variable (falling back to the deprecated but well known `ANDROID_NDK_HOME`). + /// - seealso: [Configuring NDK Path](https://github.com/android/ndk-samples/wiki/Configure-NDK-Path#terminologies) + static var environmentOverrideLocation: AbsolutePath? { + (getEnvironmentVariable("ANDROID_NDK_ROOT") ?? getEnvironmentVariable("ANDROID_NDK_HOME"))?.nilIfEmpty.map { AbsolutePath($0) } ?? nil + } + + /// Location of the Android NDK installed by the `google-android-ndk-*-installer` family of packages available in Debian 13 "Trixie" and Ubuntu 24.04 "Noble". + /// These packages are available in non-free / multiverse and multiple versions can be installed simultaneously. + static var defaultDebianLocation: AbsolutePath? { + AbsolutePath("/usr/lib/android-ndk") + } +} + +fileprivate extension AndroidSDK { + /// The location of the Android SDK based on the `ANDROID_HOME` environment variable (falling back to the deprecated but well known `ANDROID_SDK_ROOT`). + /// - seealso: [Android environment variables](https://developer.android.com/tools/variables) + static var environmentOverrideLocation: AbsolutePath? { + (getEnvironmentVariable("ANDROID_HOME") ?? getEnvironmentVariable("ANDROID_SDK_ROOT"))?.nilIfEmpty.map { AbsolutePath($0) } ?? nil + } + + static func defaultAndroidStudioLocation(host: OperatingSystem) throws -> AbsolutePath? { + switch host { case .windows: // %LOCALAPPDATA%\Android\Sdk - try FileManager.default.url(for: .applicationSupportDirectory, in: .userDomainMask, appropriateFor: nil, create: false).appendingPathComponent("Android").appendingPathComponent("Sdk").filePath + try FileManager.default.url(for: .applicationSupportDirectory, in: .userDomainMask, appropriateFor: nil, create: false).appendingPathComponent("Android").appendingPathComponent("Sdk").absoluteFilePath case .macOS: // ~/Library/Android/sdk - try FileManager.default.url(for: .libraryDirectory, in: .userDomainMask, appropriateFor: nil, create: false).appendingPathComponent("Android").appendingPathComponent("sdk").filePath + try FileManager.default.url(for: .libraryDirectory, in: .userDomainMask, appropriateFor: nil, create: false).appendingPathComponent("Android").appendingPathComponent("sdk").absoluteFilePath case .linux: // ~/Android/Sdk - Path.homeDirectory.join("Android").join("Sdk") + try AbsolutePath(validating: Path.homeDirectory.join("Android").join("Sdk")) default: nil } + } - if let path = defaultLocation, fs.exists(path) { - return try [AndroidSDK(host: host, path: path, fs: fs)] - } - - return [] + /// Location of the Android SDK installed by the `google-*` family of packages available in Debian 13 "Trixie" and Ubuntu 24.04 "Noble". + /// These packages are available in non-free / multiverse and multiple versions can be installed simultaneously. + static var defaultDebianLocation: AbsolutePath? { + AbsolutePath("/usr/lib/android-sdk") } } diff --git a/Sources/SWBAndroidPlatform/CMakeLists.txt b/Sources/SWBAndroidPlatform/CMakeLists.txt index fdb0bfa7..72d8cd9b 100644 --- a/Sources/SWBAndroidPlatform/CMakeLists.txt +++ b/Sources/SWBAndroidPlatform/CMakeLists.txt @@ -10,6 +10,7 @@ See http://swift.org/CONTRIBUTORS.txt for Swift project authors add_library(SWBAndroidPlatform AndroidSDK.swift + JavaProperties.swift Plugin.swift) SwiftBuild_Bundle(MODULE SWBAndroidPlatform FILES Specs/Android.xcspec) diff --git a/Sources/SWBAndroidPlatform/JavaProperties.swift b/Sources/SWBAndroidPlatform/JavaProperties.swift new file mode 100644 index 00000000..0a6059d7 --- /dev/null +++ b/Sources/SWBAndroidPlatform/JavaProperties.swift @@ -0,0 +1,32 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation +internal import SWBUtil + +/// A simple representation of a Java properties file. +/// +/// See `java.util.Properties` for a description of the file format. This parser is a simplified version that doesn't handle line continuations, etc., because our use case is narrow. +struct JavaProperties { + private let properties: [String: String] + + init(data: Data) throws { + properties = Dictionary(uniqueKeysWithValues: String(decoding: data, as: UTF8.self).split(whereSeparator: { $0.isNewline }).map(String.init).map { + let (key, value) = $0.split("=") + return (key.trimmingCharacters(in: .whitespaces), value.trimmingCharacters(in: .whitespaces)) + }) + } + + subscript(_ propertyName: String) -> String? { + properties[propertyName] + } +} diff --git a/Sources/SWBAndroidPlatform/Plugin.swift b/Sources/SWBAndroidPlatform/Plugin.swift index 88fa5f76..3eb0ca1d 100644 --- a/Sources/SWBAndroidPlatform/Plugin.swift +++ b/Sources/SWBAndroidPlatform/Plugin.swift @@ -11,7 +11,7 @@ //===----------------------------------------------------------------------===// public import SWBUtil -import SWBCore +public import SWBCore import SWBMacro import Foundation @@ -24,7 +24,7 @@ import Foundation manager.register(AndroidToolchainRegistryExtension(plugin: plugin), type: ToolchainRegistryExtensionPoint.self) } -final class AndroidPlugin: Sendable { +@_spi(Testing) public final class AndroidPlugin: Sendable { private let androidSDKInstallations = AsyncCache() func cachedAndroidSDKInstallations(host: OperatingSystem) async throws -> [AndroidSDK] { @@ -33,6 +33,18 @@ final class AndroidPlugin: Sendable { try await AndroidSDK.findInstallations(host: host, fs: localFS) } } + + @_spi(Testing) public func effectiveInstallation(host: OperatingSystem) async throws -> (sdk: AndroidSDK, ndk: AndroidSDK.NDK)? { + guard let androidSdk = try? await cachedAndroidSDKInstallations(host: host).first else { + return nil + } + + guard let androidNdk = androidSdk.preferredNDK else { + return nil + } + + return (androidSdk, androidNdk) + } } struct AndroidPlatformSpecsExtension: SpecificationsExtension { @@ -52,9 +64,13 @@ struct AndroidEnvironmentExtension: EnvironmentExtension { switch context.hostOperatingSystem { case .windows, .macOS, .linux: if let latest = try? await plugin.cachedAndroidSDKInstallations(host: context.hostOperatingSystem).first { + let sdkPath = latest.path.path.str + let ndkPath = latest.preferredNDK?.path.path.str return [ - "ANDROID_SDK_ROOT": latest.path.str, - "ANDROID_NDK_ROOT": latest.ndks.last?.path.str, + "ANDROID_HOME": sdkPath, + "ANDROID_SDK_ROOT": sdkPath, + "ANDROID_NDK_ROOT": ndkPath, + "ANDROID_NDK_HOME": ndkPath, ].compactMapValues { $0 } } default: @@ -80,10 +96,10 @@ struct AndroidPlatformExtension: PlatformInfoExtension { } } -struct AndroidSDKRegistryExtension: SDKRegistryExtension { - let plugin: AndroidPlugin +@_spi(Testing) public struct AndroidSDKRegistryExtension: SDKRegistryExtension { + @_spi(Testing) public let plugin: AndroidPlugin - func additionalSDKs(context: any SDKRegistryExtensionAdditionalSDKsContext) async throws -> [(path: Path, platform: SWBCore.Platform?, data: [String: PropertyListItem])] { + public func additionalSDKs(context: any SDKRegistryExtensionAdditionalSDKsContext) async throws -> [(path: Path, platform: SWBCore.Platform?, data: [String: PropertyListItem])] { let host = context.hostOperatingSystem guard let androidPlatform = context.platformRegistry.lookup(name: "android") else { return [] @@ -108,11 +124,7 @@ struct AndroidSDKRegistryExtension: SDKRegistryExtension { "AR": .plString(host.imageFormat.executableName(basename: "llvm-ar")), ] - guard let androidSdk = try? await plugin.cachedAndroidSDKInstallations(host: host).first else { - return [] - } - - guard let androidNdk = androidSdk.latestNDK else { + guard let (_, androidNdk) = try await plugin.effectiveInstallation(host: host) else { return [] } @@ -150,7 +162,7 @@ struct AndroidSDKRegistryExtension: SDKRegistryExtension { swiftSettings = [:] } - return [(androidNdk.sysroot, androidPlatform, [ + return [(androidNdk.sysroot.path, androidPlatform, [ "Type": .plString("SDK"), "Version": .plString("0.0.0"), "CanonicalName": .plString("android"), @@ -187,7 +199,7 @@ struct AndroidToolchainRegistryExtension: ToolchainRegistryExtension { let plugin: AndroidPlugin func additionalToolchains(context: any ToolchainRegistryExtensionAdditionalToolchainsContext) async throws -> [Toolchain] { - guard let toolchainPath = try? await plugin.cachedAndroidSDKInstallations(host: context.hostOperatingSystem).first?.latestNDK?.toolchainPath else { + guard let toolchainPath = try? await plugin.cachedAndroidSDKInstallations(host: context.hostOperatingSystem).first?.preferredNDK?.toolchainPath else { return [] } @@ -197,13 +209,13 @@ struct AndroidToolchainRegistryExtension: ToolchainRegistryExtension { displayName: "Android", version: Version(0, 0, 0), aliases: [], - path: toolchainPath, + path: toolchainPath.path, frameworkPaths: [], libraryPaths: [], defaultSettings: [:], overrideSettings: [:], defaultSettingsWhenPrimary: [:], - executableSearchPaths: [toolchainPath.join("bin")], + executableSearchPaths: [toolchainPath.path.join("bin")], testingLibraryPlatformNames: [], fs: context.fs) ] diff --git a/Sources/SWBUtil/FSProxy.swift b/Sources/SWBUtil/FSProxy.swift index 9f0eb4c2..2bc1e10e 100644 --- a/Sources/SWBUtil/FSProxy.swift +++ b/Sources/SWBUtil/FSProxy.swift @@ -961,9 +961,13 @@ public class PseudoFS: FSProxy, @unchecked Sendable { public func realpath(_ path: Path) throws -> Path { // TODO: Update this to actually return the link target when we support - // symlinks; for now it just returns the input, which seems reasonably - // correct. - return path + // symlinks; for now it just returns the input (or the link target if it's a symlink), + // which seems reasonably correct for simple cases. + do { + return try readlink(path) + } catch { + return path + } } public func readlink(_ path: Path) throws -> Path { diff --git a/Sources/SWBUtil/Path.swift b/Sources/SWBUtil/Path.swift index 1edf33b7..28294f29 100644 --- a/Sources/SWBUtil/Path.swift +++ b/Sources/SWBUtil/Path.swift @@ -1006,6 +1006,10 @@ public struct RelativePath: Hashable, Equatable, Serializable, Sendable { } extension AbsolutePath { + public static var root: AbsolutePath { + AbsolutePath(.root)! + } + public var dirname: AbsolutePath { AbsolutePath(path.dirname)! } diff --git a/Sources/SWBUtil/URL.swift b/Sources/SWBUtil/URL.swift index 725f7ac5..9195baa6 100644 --- a/Sources/SWBUtil/URL.swift +++ b/Sources/SWBUtil/URL.swift @@ -18,7 +18,7 @@ extension URL { /// This should always be used whenever the file path equivalent of a URL is needed. DO NOT use ``path`` or ``path(percentEncoded:)``, as these deal in terms of the path portion of the URL representation per RFC8089, which on Windows would include a leading slash. /// /// - throws: ``FileURLError`` if the URL does not represent a file or its path is otherwise not representable. - public var filePath: Path { + public var absoluteFilePath: AbsolutePath { get throws { guard isFileURL else { throw FileURLError.notRepresentable(self) @@ -27,12 +27,16 @@ extension URL { guard let cString else { throw FileURLError.notRepresentable(self) } - let fp = Path(String(cString: cString)) - precondition(fp.isAbsolute, "path '\(fp.str)' is not absolute") - return fp + return try AbsolutePath(validating: String(cString: cString)) } } } + + public var filePath: Path { + get throws { + try absoluteFilePath.path + } + } } fileprivate enum FileURLError: Error, CustomStringConvertible { diff --git a/Tests/SWBAndroidPlatformTests/AndroidSDKTests.swift b/Tests/SWBAndroidPlatformTests/AndroidSDKTests.swift index 37329d94..b567c294 100644 --- a/Tests/SWBAndroidPlatformTests/AndroidSDKTests.swift +++ b/Tests/SWBAndroidPlatformTests/AndroidSDKTests.swift @@ -24,22 +24,74 @@ fileprivate struct AndroidSDKTests { // It's OK if `installations` is an empty set, the host system might have no Android SDK/NDK installed for installation in installations { #expect(installation.host == host) - #expect(installation.latestNDK == installation.ndks.last) } } - @Test func abis_r22() async throws { - try await withNDKVersion(version: Version("22.1.7171670")) { host, fs, ndkVersionPath in - let error = #expect(throws: StubError.self) { - try AndroidSDK.NDK.findInstallations(host: host, sdkPath: ndkVersionPath.dirname.dirname, fs: fs) + @Test(.skipHostOS(.windows, "This test inherently relies on Unix-style paths")) + func debian() async throws { + let fs = PseudoFS() + let sdkPath = try AbsolutePath(validating: "/usr/lib/android-sdk") + try fs.createDirectory(sdkPath.path, recursive: true) + try await withNDKVersions(fs: fs, sdkPath: sdkPath, versions: Version("24"), Version("25"), Version("26")) { host, fs, sdkPath, ndkVersionPaths in + for ndkVersionPath in ndkVersionPaths { + try await fs.writeFileContents(ndkVersionPath.path.join("meta").join("abis.json")) { contents in + contents <<< + """ + { + } + """ + } + + try await fs.writeFileContents(ndkVersionPath.path.join("meta").join("platforms.json")) { contents in + contents <<< + """ + { + "min": 21, + "max": 35, + "aliases": { + } + } + """ + } } - #expect(error?.description == "Android NDK version at path '\(ndkVersionPath.str)' is not supported (r23 or later required)") + + try fs.symlink(Path("/usr/lib/android-ndk"), target: ndkVersionPaths[1].path) + + let installations = try AndroidSDK.NDK.findInstallations(host: host, sdkPath: sdkPath, fs: fs) + let installation = try #require(installations.preferredNDK) + #expect(installations.ndks.count == 3) + #expect(installation != installations.ndks.first) + #expect(installation == installations.ndks[1]) + #expect(installation != installations.ndks.last) + #expect(installation.host == host) + #expect(installation.path == ndkVersionPaths[1]) + #expect(try installation.version == Version("25")) + #expect(installation.deploymentTargetRange.min == 21) + #expect(installation.deploymentTargetRange.max == 35) + + #expect(installation.abis.isEmpty) + } + } + + @Test func unsupportedVersions() async throws { + try await withNDKVersion(version: Version("22.1.7171670")) { host, fs, sdkPath, ndkVersionPath in + let error = try #require(throws: AndroidSDK.NDK.Error.self) { + try AndroidSDK.NDK.findInstallations(host: host, sdkPath: sdkPath, fs: fs) + } + #expect(error.description == "All installed NDK versions are not supported (r23 or later required)") + } + + try await withNDKVersion(version: Version("22.1.7171670")) { host, fs, sdkPath, ndkVersionPath in + let error = try #require(throws: AndroidSDK.NDK.Error.self) { + try AndroidSDK.NDK(host: host, path: ndkVersionPath, fs: fs) + } + #expect(error.description == "Android NDK version at path '\(ndkVersionPath.path.str)' is not supported (r23 or later required)") } } @Test func abis_r26_3() async throws { - try await withNDKVersion(version: Version("26.3.11579264")) { host, fs, ndkVersionPath in - try await fs.writeFileContents(ndkVersionPath.join("meta").join("abis.json")) { contents in + try await withNDKVersion(version: Version("26.3.11579264")) { host, fs, sdkPath, ndkVersionPath in + try await fs.writeFileContents(ndkVersionPath.path.join("meta").join("abis.json")) { contents in contents <<< """ { @@ -83,7 +135,7 @@ fileprivate struct AndroidSDKTests { """ } - try await fs.writeFileContents(ndkVersionPath.join("meta").join("platforms.json")) { contents in + try await fs.writeFileContents(ndkVersionPath.path.join("meta").join("platforms.json")) { contents in contents <<< """ { @@ -115,8 +167,8 @@ fileprivate struct AndroidSDKTests { """ } - let installations = try AndroidSDK.NDK.findInstallations(host: host, sdkPath: ndkVersionPath.dirname.dirname, fs: fs) - let installation = try #require(installations.only) + let installations = try AndroidSDK.NDK.findInstallations(host: host, sdkPath: sdkPath, fs: fs) + let installation = try #require(installations.ndks.only) #expect(installation.host == host) #expect(installation.path == ndkVersionPath) #expect(try installation.version == Version("26.3.11579264")) @@ -180,8 +232,8 @@ fileprivate struct AndroidSDKTests { } @Test func abis_r27() async throws { - try await withNDKVersion(version: Version("27.0.11718014")) { host, fs, ndkVersionPath in - try await fs.writeFileContents(ndkVersionPath.join("meta").join("abis.json")) { contents in + try await withNDKVersion(version: Version("27.0.11718014")) { host, fs, sdkPath, ndkVersionPath in + try await fs.writeFileContents(ndkVersionPath.path.join("meta").join("abis.json")) { contents in contents <<< """ { @@ -239,7 +291,7 @@ fileprivate struct AndroidSDKTests { """ } - try await fs.writeFileContents(ndkVersionPath.join("meta").join("platforms.json")) { contents in + try await fs.writeFileContents(ndkVersionPath.path.join("meta").join("platforms.json")) { contents in contents <<< """ { @@ -272,8 +324,8 @@ fileprivate struct AndroidSDKTests { """ } - let installations = try AndroidSDK.NDK.findInstallations(host: host, sdkPath: ndkVersionPath.dirname.dirname, fs: fs) - let installation = try #require(installations.only) + let installations = try AndroidSDK.NDK.findInstallations(host: host, sdkPath: sdkPath, fs: fs) + let installation = try #require(installations.ndks.only) #expect(installation.host == host) #expect(installation.path == ndkVersionPath) #expect(try installation.version == Version("27.0.11718014")) @@ -349,13 +401,26 @@ fileprivate struct AndroidSDKTests { } } - private func withNDKVersion(version: Version, _ block: (OperatingSystem, any FSProxy, Path) async throws -> ()) async throws { - let fs = PseudoFS() - let ndkPath = Path.root.join("ndk") - let ndkVersionPath = ndkPath.join(version.description) - try fs.createDirectory(ndkPath, recursive: true) - try fs.createDirectory(ndkVersionPath.join("meta"), recursive: true) + private func withNDKVersions(fs: PseudoFS = PseudoFS(), sdkPath: AbsolutePath = .root, versions: Version..., block: (OperatingSystem, any FSProxy, AbsolutePath, [AbsolutePath]) async throws -> ()) async throws { + let ndkPath = sdkPath.path.join("ndk") + let ndkVersionPaths = try await versions.asyncMap { version in + let ndkVersionPath = ndkPath.join(version.description) + try fs.createDirectory(ndkPath, recursive: true) + try fs.createDirectory(ndkVersionPath.join("meta"), recursive: true) + try await fs.writeFileContents(ndkVersionPath.join("source.properties")) { + $0 <<< "Pkg.Desc = Android NDK\n" + $0 <<< "Pkg.Revision = \(version.description)-beta1\n" + $0 <<< "Pkg.BaseRevision = \(version.description)\n" + } + return ndkVersionPath + } let host = try ProcessInfo.processInfo.hostOperatingSystem() - try await block(host, fs, ndkVersionPath) + try await block(host, fs, sdkPath, ndkVersionPaths.map { try AbsolutePath(validating: $0) }) + } + + private func withNDKVersion(fs: PseudoFS = PseudoFS(), sdkPath: AbsolutePath = .root, version: Version, _ block: (OperatingSystem, any FSProxy, AbsolutePath, AbsolutePath) async throws -> ()) async throws { + try await withNDKVersions(fs: fs, sdkPath: sdkPath, versions: version) { host, fs, sdkPath, ndkVersionPaths in + try await block(host, fs, sdkPath, ndkVersionPaths[0]) + } } } diff --git a/Tests/SWBAndroidPlatformTests/SWBAndroidPlatformTests.swift b/Tests/SWBAndroidPlatformTests/SWBAndroidPlatformTests.swift index 74ef48e4..117588f9 100644 --- a/Tests/SWBAndroidPlatformTests/SWBAndroidPlatformTests.swift +++ b/Tests/SWBAndroidPlatformTests/SWBAndroidPlatformTests.swift @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// import Testing +@_spi(Testing) import SWBAndroidPlatform import SWBProtocol import SWBTestSupport import SWBTaskExecution @@ -94,6 +95,12 @@ fileprivate struct AndroidBuildOperationTests: CoreBasedTests { ), ]) let core = try await getCore() + let androidExtension = try await #require(core.pluginManager.extensions(of: SDKRegistryExtensionPoint.self).compactMap { $0 as? AndroidSDKRegistryExtension }.only) + let (_, androidNdk) = try #require(await androidExtension.plugin.effectiveInstallation(host: core.hostOperatingSystem)) + if androidNdk.version < Version(27) && arch == "riscv64" { + return // riscv64 support was introduced in NDK r27 + } + let tester = try await BuildOperationTester(core, testProject, simulated: false) let projectDir = tester.workspace.projects[0].sourceRoot From d4d156e0661da755ad465153f208845a137219d6 Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Sun, 15 Jun 2025 19:34:04 -0700 Subject: [PATCH 4/5] Lookup unix fallback toolchain relative to SWIFT_EXEC when set --- Sources/SWBGenericUnixPlatform/Plugin.swift | 69 +++++++++++---------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/Sources/SWBGenericUnixPlatform/Plugin.swift b/Sources/SWBGenericUnixPlatform/Plugin.swift index a3839068..0d6cd88c 100644 --- a/Sources/SWBGenericUnixPlatform/Plugin.swift +++ b/Sources/SWBGenericUnixPlatform/Plugin.swift @@ -129,39 +129,44 @@ struct GenericUnixToolchainRegistryExtension: ToolchainRegistryExtension { let fs = context.fs - if let swift = StackedSearchPath(environment: .current, fs: fs).lookup(Path("swift")), fs.exists(swift) { - let realSwiftPath = try fs.realpath(swift).dirname.normalize() - let hasUsrBin = realSwiftPath.str.hasSuffix("/usr/bin") - let hasUsrLocalBin = realSwiftPath.str.hasSuffix("/usr/local/bin") - let path: Path - switch (hasUsrBin, hasUsrLocalBin) { - case (true, false): - path = realSwiftPath.dirname.dirname - case (false, true): - path = realSwiftPath.dirname.dirname.dirname - case (false, false): - throw StubError.error("Unexpected toolchain layout for Swift installation path: \(realSwiftPath)") - case (true, true): - preconditionFailure() + for swift in [ + Environment.current["SWIFT_EXEC"].map(Path.init), + StackedSearchPath(environment: .current, fs: fs).lookup(Path("swift")) + ].compactMap(\.self) { + if fs.exists(swift) { + let realSwiftPath = try fs.realpath(swift).dirname.normalize() + let hasUsrBin = realSwiftPath.str.hasSuffix("/usr/bin") + let hasUsrLocalBin = realSwiftPath.str.hasSuffix("/usr/local/bin") + let path: Path + switch (hasUsrBin, hasUsrLocalBin) { + case (true, false): + path = realSwiftPath.dirname.dirname + case (false, true): + path = realSwiftPath.dirname.dirname.dirname + case (false, false): + throw StubError.error("Unexpected toolchain layout for Swift installation path: \(realSwiftPath)") + case (true, true): + preconditionFailure() + } + let llvmDirectories = try Array(fs.listdir(Path("/usr/lib")).filter { $0.hasPrefix("llvm-") }.sorted().reversed()) + let llvmDirectoriesLocal = try Array(fs.listdir(Path("/usr/local")).filter { $0.hasPrefix("llvm") }.sorted().reversed()) + return [ + Toolchain( + identifier: ToolchainRegistry.defaultToolchainIdentifier, + displayName: "Default", + version: Version(), + aliases: ["default"], + path: path, + frameworkPaths: [], + libraryPaths: llvmDirectories.map { "/usr/lib/\($0)/lib" } + llvmDirectoriesLocal.map { "/usr/local/\($0)/lib" } + ["/usr/lib64"], + defaultSettings: [:], + overrideSettings: [:], + defaultSettingsWhenPrimary: [:], + executableSearchPaths: realSwiftPath.dirname.relativeSubpath(from: path).map { [path.join($0).join("bin")] } ?? [], + testingLibraryPlatformNames: [], + fs: fs) + ] } - let llvmDirectories = try Array(fs.listdir(Path("/usr/lib")).filter { $0.hasPrefix("llvm-") }.sorted().reversed()) - let llvmDirectoriesLocal = try Array(fs.listdir(Path("/usr/local")).filter { $0.hasPrefix("llvm") }.sorted().reversed()) - return [ - Toolchain( - identifier: ToolchainRegistry.defaultToolchainIdentifier, - displayName: "Default", - version: Version(), - aliases: ["default"], - path: path, - frameworkPaths: [], - libraryPaths: llvmDirectories.map { "/usr/lib/\($0)/lib" } + llvmDirectoriesLocal.map { "/usr/local/\($0)/lib" } + ["/usr/lib64"], - defaultSettings: [:], - overrideSettings: [:], - defaultSettingsWhenPrimary: [:], - executableSearchPaths: realSwiftPath.dirname.relativeSubpath(from: path).map { [path.join($0).join("bin")] } ?? [], - testingLibraryPlatformNames: [], - fs: fs) - ] } return [] From 344f8802a5772224d144f4dfa645387d86ce7ec1 Mon Sep 17 00:00:00 2001 From: Jake Petroules Date: Mon, 4 Aug 2025 10:58:45 -0700 Subject: [PATCH 5/5] Handle the OS version in the triple for FreeBSD We're going to keep the versioned triple, so it needs to be passed through after all. --- Sources/SWBAndroidPlatform/AndroidSDK.swift | 24 --- Sources/SWBAndroidPlatform/Plugin.swift | 13 +- Sources/SWBGenericUnixPlatform/Plugin.swift | 138 ++++++++++++------ Sources/SWBTestSupport/CoreTestSupport.swift | 2 +- Sources/SWBUtil/CMakeLists.txt | 1 + Sources/SWBUtil/LLVMTriple.swift | 50 +++++++ .../SWBWindowsPlatform/VSInstallation.swift | 4 +- 7 files changed, 153 insertions(+), 79 deletions(-) create mode 100644 Sources/SWBUtil/LLVMTriple.swift diff --git a/Sources/SWBAndroidPlatform/AndroidSDK.swift b/Sources/SWBAndroidPlatform/AndroidSDK.swift index 3b934838..9a2c00a0 100644 --- a/Sources/SWBAndroidPlatform/AndroidSDK.swift +++ b/Sources/SWBAndroidPlatform/AndroidSDK.swift @@ -134,30 +134,6 @@ public import Foundation case bits64 = 64 } - @_spi(Testing) public struct LLVMTriple: Codable, Equatable, Sendable { - public var arch: String - public var vendor: String - public var system: String - public var environment: String - - var description: String { - "\(arch)-\(vendor)-\(system)-\(environment)" - } - - public init(from decoder: any Decoder) throws { - let container = try decoder.singleValueContainer() - let triple = try container.decode(String.self) - if let match = try #/(?.+)-(?.+)-(?.+)-(?.+)/#.wholeMatch(in: triple) { - self.arch = String(match.output.arch) - self.vendor = String(match.output.vendor) - self.system = String(match.output.system) - self.environment = String(match.output.environment) - } else { - throw DecodingError.dataCorruptedError(in: container, debugDescription: "Invalid triple string: \(triple)") - } - } - } - public let bitness: Bitness public let `default`: Bool public let deprecated: Bool diff --git a/Sources/SWBAndroidPlatform/Plugin.swift b/Sources/SWBAndroidPlatform/Plugin.swift index 3eb0ca1d..117e7c6e 100644 --- a/Sources/SWBAndroidPlatform/Plugin.swift +++ b/Sources/SWBAndroidPlatform/Plugin.swift @@ -131,11 +131,14 @@ struct AndroidPlatformExtension: PlatformInfoExtension { let abis = androidNdk.abis let deploymentTargetRange = androidNdk.deploymentTargetRange - let allPossibleTriples = abis.values.flatMap { abi in - (max(deploymentTargetRange.min, abi.min_os_version)...deploymentTargetRange.max).map { deploymentTarget in + let allPossibleTriples = try abis.values.flatMap { abi in + try (max(deploymentTargetRange.min, abi.min_os_version)...deploymentTargetRange.max).map { deploymentTarget in var triple = abi.llvm_triple triple.vendor = "unknown" // Android NDK uses "none", Swift SDKs use "unknown" - triple.environment += "\(deploymentTarget)" + guard let env = triple.environment else { + throw StubError.error("Android triples must have an environment") + } + triple.environment = "\(env)\(deploymentTarget)" return triple } }.map(\.description) @@ -173,8 +176,8 @@ struct AndroidPlatformExtension: PlatformInfoExtension { "CustomProperties": .plDict([ // Unlike most platforms, the Android version goes on the environment field rather than the system field // FIXME: Make this configurable in a better way so we don't need to push build settings at the SDK definition level - "LLVM_TARGET_TRIPLE_OS_VERSION": .plString("linux"), - "LLVM_TARGET_TRIPLE_SUFFIX": .plString("-android$(ANDROID_DEPLOYMENT_TARGET)"), + "LLVM_TARGET_TRIPLE_OS_VERSION": .plString("$(SWIFT_PLATFORM_TARGET_PREFIX)"), + "LLVM_TARGET_TRIPLE_SUFFIX": .plString("-android$($(DEPLOYMENT_TARGET_SETTING_NAME))"), ].merging(swiftSettings, uniquingKeysWith: { _, new in new })), "SupportedTargets": .plDict([ "android": .plDict([ diff --git a/Sources/SWBGenericUnixPlatform/Plugin.swift b/Sources/SWBGenericUnixPlatform/Plugin.swift index 0d6cd88c..632a4cae 100644 --- a/Sources/SWBGenericUnixPlatform/Plugin.swift +++ b/Sources/SWBGenericUnixPlatform/Plugin.swift @@ -15,11 +15,44 @@ import SWBCore import Foundation @PluginExtensionSystemActor public func initializePlugin(_ manager: PluginManager) { + let plugin = GenericUnixPlugin() manager.register(GenericUnixDeveloperDirectoryExtension(), type: DeveloperDirectoryExtensionPoint.self) manager.register(GenericUnixPlatformSpecsExtension(), type: SpecificationsExtensionPoint.self) manager.register(GenericUnixPlatformInfoExtension(), type: PlatformInfoExtensionPoint.self) - manager.register(GenericUnixSDKRegistryExtension(), type: SDKRegistryExtensionPoint.self) - manager.register(GenericUnixToolchainRegistryExtension(), type: ToolchainRegistryExtensionPoint.self) + manager.register(GenericUnixSDKRegistryExtension(plugin: plugin), type: SDKRegistryExtensionPoint.self) + manager.register(GenericUnixToolchainRegistryExtension(plugin: plugin), type: ToolchainRegistryExtensionPoint.self) +} + +final class GenericUnixPlugin: Sendable { + func swiftExecutablePath(fs: any FSProxy) -> Path? { + [ + Environment.current["SWIFT_EXEC"].map(Path.init), + StackedSearchPath(environment: .current, fs: fs).lookup(Path("swift")) + ].compactMap { $0 }.first(where: fs.exists) + } + + func swiftTargetInfo(swiftExecutablePath: Path) async throws -> SwiftTargetInfo { + let args = ["-print-target-info"] + let executionResult = try await Process.getOutput(url: URL(fileURLWithPath: swiftExecutablePath.str), arguments: args) + guard executionResult.exitStatus.isSuccess else { + throw RunProcessNonZeroExitError(args: [swiftExecutablePath.str] + args, workingDirectory: nil, environment: [:], status: executionResult.exitStatus, stdout: ByteString(executionResult.stdout), stderr: ByteString(executionResult.stderr)) + } + return try JSONDecoder().decode(SwiftTargetInfo.self, from: executionResult.stdout) + } +} + +struct SwiftTargetInfo: Decodable { + struct TargetInfo: Decodable { + let triple: LLVMTriple + let unversionedTriple: LLVMTriple + } + let target: TargetInfo +} + +extension SwiftTargetInfo.TargetInfo { + var tripleVersion: String? { + triple != unversionedTriple && triple.system.hasPrefix(unversionedTriple.system) ? String(triple.system.dropFirst(unversionedTriple.system.count)).nilIfEmpty : nil + } } struct GenericUnixDeveloperDirectoryExtension: DeveloperDirectoryExtension { @@ -69,9 +102,11 @@ struct GenericUnixPlatformInfoExtension: PlatformInfoExtension { } struct GenericUnixSDKRegistryExtension: SDKRegistryExtension { + let plugin: GenericUnixPlugin + func additionalSDKs(context: any SDKRegistryExtensionAdditionalSDKsContext) async throws -> [(path: Path, platform: SWBCore.Platform?, data: [String: PropertyListItem])] { let operatingSystem = context.hostOperatingSystem - guard operatingSystem.createFallbackSystemToolchain, let platform = try context.platformRegistry.lookup(name: operatingSystem.xcodePlatformName) else { + guard operatingSystem.createFallbackSystemToolchain, let platform = try context.platformRegistry.lookup(name: operatingSystem.xcodePlatformName), let swift = plugin.swiftExecutablePath(fs: context.fs) else { return [] } @@ -100,6 +135,23 @@ struct GenericUnixSDKRegistryExtension: SDKRegistryExtension { tripleEnvironment = "" } + let swiftTargetInfo = try await plugin.swiftTargetInfo(swiftExecutablePath: swift) + + let deploymentTargetSettings: [String: PropertyListItem] + if operatingSystem == .freebsd { + guard let tripleVersion = swiftTargetInfo.target.tripleVersion else { + throw StubError.error("Unknown FreeBSD triple version") + } + deploymentTargetSettings = [ + "DeploymentTargetSettingName": .plString("FREEBSD_DEPLOYMENT_TARGET"), + "DefaultDeploymentTarget": .plString(tripleVersion), + "MinimumDeploymentTarget": .plString(tripleVersion), + "MaximumDeploymentTarget": .plString(tripleVersion), + ] + } else { + deploymentTargetSettings = [:] + } + return try [(.root, platform, [ "Type": .plString("SDK"), "Version": .plString(Version(ProcessInfo.processInfo.operatingSystemVersion).zeroTrimmed.description), @@ -114,62 +166,54 @@ struct GenericUnixSDKRegistryExtension: SDKRegistryExtension { "LLVMTargetTripleEnvironment": .plString(tripleEnvironment), "LLVMTargetTripleSys": .plString(operatingSystem.xcodePlatformName), "LLVMTargetTripleVendor": .plString("unknown"), - ]) + ].merging(deploymentTargetSettings, uniquingKeysWith: { _, new in new })) ]), ])] } } struct GenericUnixToolchainRegistryExtension: ToolchainRegistryExtension { + let plugin: GenericUnixPlugin + func additionalToolchains(context: any ToolchainRegistryExtensionAdditionalToolchainsContext) async throws -> [Toolchain] { let operatingSystem = context.hostOperatingSystem - guard operatingSystem.createFallbackSystemToolchain else { + let fs = context.fs + guard operatingSystem.createFallbackSystemToolchain, let swift = plugin.swiftExecutablePath(fs: fs) else { return [] } - let fs = context.fs - - for swift in [ - Environment.current["SWIFT_EXEC"].map(Path.init), - StackedSearchPath(environment: .current, fs: fs).lookup(Path("swift")) - ].compactMap(\.self) { - if fs.exists(swift) { - let realSwiftPath = try fs.realpath(swift).dirname.normalize() - let hasUsrBin = realSwiftPath.str.hasSuffix("/usr/bin") - let hasUsrLocalBin = realSwiftPath.str.hasSuffix("/usr/local/bin") - let path: Path - switch (hasUsrBin, hasUsrLocalBin) { - case (true, false): - path = realSwiftPath.dirname.dirname - case (false, true): - path = realSwiftPath.dirname.dirname.dirname - case (false, false): - throw StubError.error("Unexpected toolchain layout for Swift installation path: \(realSwiftPath)") - case (true, true): - preconditionFailure() - } - let llvmDirectories = try Array(fs.listdir(Path("/usr/lib")).filter { $0.hasPrefix("llvm-") }.sorted().reversed()) - let llvmDirectoriesLocal = try Array(fs.listdir(Path("/usr/local")).filter { $0.hasPrefix("llvm") }.sorted().reversed()) - return [ - Toolchain( - identifier: ToolchainRegistry.defaultToolchainIdentifier, - displayName: "Default", - version: Version(), - aliases: ["default"], - path: path, - frameworkPaths: [], - libraryPaths: llvmDirectories.map { "/usr/lib/\($0)/lib" } + llvmDirectoriesLocal.map { "/usr/local/\($0)/lib" } + ["/usr/lib64"], - defaultSettings: [:], - overrideSettings: [:], - defaultSettingsWhenPrimary: [:], - executableSearchPaths: realSwiftPath.dirname.relativeSubpath(from: path).map { [path.join($0).join("bin")] } ?? [], - testingLibraryPlatformNames: [], - fs: fs) - ] - } + let realSwiftPath = try fs.realpath(swift).dirname.normalize() + let hasUsrBin = realSwiftPath.str.hasSuffix("/usr/bin") + let hasUsrLocalBin = realSwiftPath.str.hasSuffix("/usr/local/bin") + let path: Path + switch (hasUsrBin, hasUsrLocalBin) { + case (true, false): + path = realSwiftPath.dirname.dirname + case (false, true): + path = realSwiftPath.dirname.dirname.dirname + case (false, false): + throw StubError.error("Unexpected toolchain layout for Swift installation path: \(realSwiftPath)") + case (true, true): + preconditionFailure() } - - return [] + let llvmDirectories = try Array(fs.listdir(Path("/usr/lib")).filter { $0.hasPrefix("llvm-") }.sorted().reversed()) + let llvmDirectoriesLocal = try Array(fs.listdir(Path("/usr/local")).filter { $0.hasPrefix("llvm") }.sorted().reversed()) + return [ + Toolchain( + identifier: ToolchainRegistry.defaultToolchainIdentifier, + displayName: "Default", + version: Version(), + aliases: ["default"], + path: path, + frameworkPaths: [], + libraryPaths: llvmDirectories.map { "/usr/lib/\($0)/lib" } + llvmDirectoriesLocal.map { "/usr/local/\($0)/lib" } + ["/usr/lib64"], + defaultSettings: [:], + overrideSettings: [:], + defaultSettingsWhenPrimary: [:], + executableSearchPaths: realSwiftPath.dirname.relativeSubpath(from: path).map { [path.join($0).join("bin")] } ?? [], + testingLibraryPlatformNames: [], + fs: fs) + ] } } diff --git a/Sources/SWBTestSupport/CoreTestSupport.swift b/Sources/SWBTestSupport/CoreTestSupport.swift index 1bf280ea..5ce4bedf 100644 --- a/Sources/SWBTestSupport/CoreTestSupport.swift +++ b/Sources/SWBTestSupport/CoreTestSupport.swift @@ -56,7 +56,7 @@ extension Core { developerPath = .xcode(xcodeDeveloperDirPath) } else { // In the context of auto-generated package schemes, try to infer the active Xcode. - let potentialDeveloperPath = getEnvironmentVariable("PATH")?.components(separatedBy: String(Path.pathEnvironmentSeparator)).first.map(Path.init)?.dirname.dirname + let potentialDeveloperPath = getEnvironmentVariable(.path)?.components(separatedBy: String(Path.pathEnvironmentSeparator)).first.map(Path.init)?.dirname.dirname let versionInfo = potentialDeveloperPath?.dirname.join("version.plist") if let versionInfo = versionInfo, (try? PropertyList.fromPath(versionInfo, fs: localFS))?.dictValue?["ProjectName"] == "IDEApplication" { developerPath = potentialDeveloperPath.map { .xcode($0) } diff --git a/Sources/SWBUtil/CMakeLists.txt b/Sources/SWBUtil/CMakeLists.txt index 91348fd9..b366c370 100644 --- a/Sources/SWBUtil/CMakeLists.txt +++ b/Sources/SWBUtil/CMakeLists.txt @@ -54,6 +54,7 @@ add_library(SWBUtil LazyCache.swift Library.swift LineReader.swift + LLVMTriple.swift Lock.swift MachO.swift Math.swift diff --git a/Sources/SWBUtil/LLVMTriple.swift b/Sources/SWBUtil/LLVMTriple.swift new file mode 100644 index 00000000..2ec27fa5 --- /dev/null +++ b/Sources/SWBUtil/LLVMTriple.swift @@ -0,0 +1,50 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +public struct LLVMTriple: Decodable, Equatable, Sendable, CustomStringConvertible { + public var arch: String + public var vendor: String + public var system: String + public var environment: String? + + public var description: String { + if let environment { + return "\(arch)-\(vendor)-\(system)-\(environment)" + } + return "\(arch)-\(vendor)-\(system)" + } + + public init(_ string: String) throws { + guard let match = try #/(?[^-]+)-(?[^-]+)-(?[^-]+)(-(?[^-]+))?/#.wholeMatch(in: string) else { + throw LLVMTripleError.invalidTripleStringFormat(string) + } + self.arch = String(match.output.arch) + self.vendor = String(match.output.vendor) + self.system = String(match.output.system) + self.environment = match.output.environment.map { String($0) } + } + + public init(from decoder: any Swift.Decoder) throws { + self = try Self(decoder.singleValueContainer().decode(String.self)) + } +} + +enum LLVMTripleError: Error, CustomStringConvertible { + case invalidTripleStringFormat(String) + + var description: String { + switch self { + case let .invalidTripleStringFormat(tripleString): + "Invalid triple string format: \(tripleString)" + } + } +} diff --git a/Sources/SWBWindowsPlatform/VSInstallation.swift b/Sources/SWBWindowsPlatform/VSInstallation.swift index 23ab705c..68c35abb 100644 --- a/Sources/SWBWindowsPlatform/VSInstallation.swift +++ b/Sources/SWBWindowsPlatform/VSInstallation.swift @@ -57,14 +57,14 @@ public struct VSInstallation: Decodable, Sendable { ] let executionResult = try await Process.getOutput(url: URL(fileURLWithPath: vswhere.str), arguments: args) guard executionResult.exitStatus.isSuccess else { - throw RunProcessNonZeroExitError(args: args, workingDirectory: nil, environment: [:], status: executionResult.exitStatus, stdout: ByteString(executionResult.stdout), stderr: ByteString(executionResult.stderr)) + throw RunProcessNonZeroExitError(args: [vswhere.str] + args, workingDirectory: nil, environment: [:], status: executionResult.exitStatus, stdout: ByteString(executionResult.stdout), stderr: ByteString(executionResult.stderr)) } return try JSONDecoder().decode([VSInstallation].self, from: executionResult.stdout) } private static func vswherePath(fs: any FSProxy) throws -> Path? { var paths: [Path] = [] - if let path = try POSIX.getenv("PATH") { + if let path = getEnvironmentVariable(.path) { paths.append(contentsOf: path.split(separator: Path.pathEnvironmentSeparator).map(Path.init).filter { // PATH may contain unexpanded shell variable references $0.isAbsolute