From 5f1dbaca121a5b86a42a2b5330c243814735661f Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Mon, 31 Mar 2025 14:28:15 -0400 Subject: [PATCH 01/44] Add Android workflow --- .github/workflows/pull_request.yml | 22 ++++++++++++++++++++++ .gitignore | 2 ++ 2 files changed, 24 insertions(+) create mode 100644 .gitignore diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 1ed0a799..ab7601c9 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -21,3 +21,25 @@ jobs: name: docker-logs path: | *.log + + static-linux-build: + name: Build Static Linux image + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Build Docker images + working-directory: swift-ci/sdks/static-linux + run: ./build + + android-build: + name: Build Android image + runs-on: ubuntu-latest + if: false + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Build Docker images + working-directory: swift-ci/sdks/android + run: ./build + diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..98bcc3db --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.*.swp +static-swift-linux-sources From 65650529c889d988eccab044dc15c2bcb3292fc4 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Tue, 22 Apr 2025 18:00:06 -0400 Subject: [PATCH 02/44] Build Android image (#1) * Build Android image * Checkout without ssh * Retry build if it fails * Swift 6.1 Release Dockerfiles (#456) * Change binutils-gold package dependency on Debian 12 to binutils (#457) * Update installed packages after nightly platform expansion (#458) * update nightly-6.1 dependencies * update nightly-main dependencies * fix ubuntu images * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Fedora 41 Dockerfile (#464) * Build Android image * Build Android image * Swift 6.1 Release Dockerfiles (#456) * Change binutils-gold package dependency on Debian 12 to binutils (#457) * Build Android image * Build Android image * Build Android image * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Android SDK build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Androd build * Swift Android build * Swift Android build * Swift Android build * Swift Android build --------- Co-authored-by: Mishal Shah Co-authored-by: Chris McGee <87777443+cmcgee1024@users.noreply.github.com> Co-authored-by: Justice Adams <107649528+justice-adams-apple@users.noreply.github.com> Co-authored-by: Andrew Sukach <134116196+sookach@users.noreply.github.com> --- .github/workflows/pull_request.yml | 126 +++- swift-ci/sdks/android/README.md | 43 ++ swift-ci/sdks/android/build | 91 +++ swift-ci/sdks/android/scripts/build.sh | 536 ++++++++++++++++++ swift-ci/sdks/android/scripts/fetch-source.sh | 183 ++++++ 5 files changed, 974 insertions(+), 5 deletions(-) create mode 100644 swift-ci/sdks/android/README.md create mode 100755 swift-ci/sdks/android/build create mode 100755 swift-ci/sdks/android/scripts/build.sh create mode 100755 swift-ci/sdks/android/scripts/fetch-source.sh diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index ab7601c9..ed578bc6 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -33,13 +33,129 @@ jobs: run: ./build android-build: - name: Build Android image - runs-on: ubuntu-latest - if: false + name: Build Android ${{ matrix.arch }} SDK + strategy: + fail-fast: false + matrix: + # blank arch builds all (aarch64,x86_64,armv7) + #arch: [''] + # builds only x86_64 to speed up the validation + #arch: ['x86_64'] + # build both the quick (x86_64) and complete (aarch64,x86_64,armv7) SDKs + arch: ['x86_64', ''] + runs-on: ubuntu-24.04 steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Build Docker images + - name: Install Dependencies + run: | + sudo apt install -q ninja-build patchelf + - name: Build Android SDK working-directory: swift-ci/sdks/android - run: ./build + run: | + TARGET_ARCHS=${{ matrix.arch }} ./build + - name: Get artifact info + id: info + shell: bash + run: | + set -ex + SWIFT_ROOT=$(dirname ${{ runner.temp }}/swift-android-sdk/host-toolchain/*/usr) + echo "swift-root=${SWIFT_ROOT}" >> $GITHUB_OUTPUT + echo "swift-path=${SWIFT_ROOT}/usr/bin/swift" >> $GITHUB_OUTPUT + + ARTIFACT_BUILD=$(realpath ${{ runner.temp }}/swift-android-sdk/build/*.artifactbundle) + ARTIFACT_PATH=$(realpath ${{ runner.temp }}/swift-android-sdk/products/*.artifactbundle.tar.gz) + echo "artifact-path=${ARTIFACT_PATH}" >> $GITHUB_OUTPUT + echo "sdk-id=x86_64-unknown-linux-android28" >> $GITHUB_OUTPUT + + ARTIFACT_EXT=".artifactbundle.tar.gz" + ARTIFACT_NAME="$(basename ${ARTIFACT_PATH} ${ARTIFACT_EXT})" + # artifacts need a unique name so we suffix with the matrix arch(s) + if [[ ! -z "${{ matrix.arch }}" ]]; then + ARTIFACT_NAME="${ARTIFACT_NAME}-$(echo ${{ matrix.arch }} | tr ',' '-')" + fi + ARTIFACT_NAME="${ARTIFACT_NAME}${ARTIFACT_EXT}" + + # There is no way to prevent even a single-file artifact from being zipped: + # https://github.com/actions/upload-artifact?tab=readme-ov-file#zip-archives + # so the actual artifact download will look like: + # swift-6.1-RELEASE_android-0.1-x86_64.artifactbundle.tar.gz.zip + echo "artifact-name=${ARTIFACT_NAME}" >> $GITHUB_OUTPUT + + # validate some required paths in the artifactbundle + pushd ${ARTIFACT_BUILD}/*/*/usr + ls lib/swift/android + ls lib/swift/android/* + ls lib/swift/android/*/swiftrt.o + ls lib/*-linux-android/*/crtbegin_dynamic.o + + ls lib/swift_static-* + ls lib/swift_static-*/android + ls lib/swift_static-*/android/libFoundationEssentials.a + + ls lib/*-linux-android/libFoundationEssentials.so + ls lib/*-linux-android/libFoundationNetworking.so + ls lib/*-linux-android/libFoundationInternationalization.so + ls lib/*-linux-android/lib_FoundationICU.so + ls lib/*-linux-android/libFoundationXML.so + ls lib/*-linux-android/libTesting.so + + ls lib/swift/clang/lib + ls lib/swift/clang/lib/linux + ls lib/swift/clang/lib/linux/*/libunwind.a + popd + - name: Upload SDK artifactbundle + uses: actions/upload-artifact@v4 + with: + compression-level: 0 + name: ${{ steps.info.outputs.artifact-name }} + path: ${{ steps.info.outputs.artifact-path }} + - name: Cleanup + run: | + # need to free up some space or else when installing we get: No space left on device + rm -rf ${{ runner.temp }}/swift-android-sdk/{build,src} + - name: Install artifactbundle + shell: bash + run: | + set -ex + ${{ steps.info.outputs.swift-path }} sdk install ${{ steps.info.outputs.artifact-path }} + ${{ steps.info.outputs.swift-path }} sdk configure --show-configuration $(${{ steps.info.outputs.swift-path }} sdk list | head -n 1) ${{ steps.info.outputs.sdk-id }} + # recent releases require that ANDROID_NDK_ROOT *not* be set + # see https://github.com/finagolfin/swift-android-sdk/issues/207 + echo "ANDROID_NDK_ROOT=" >> $GITHUB_ENV + + - name: Create Demo Project + run: | + cd ${{ runner.temp }} + mkdir DemoProject + cd DemoProject + ${{ steps.info.outputs.swift-path }} --version + ${{ steps.info.outputs.swift-path }} package init + echo 'import Foundation' >> Sources/DemoProject/DemoProject.swift + echo 'import FoundationEssentials' >> Sources/DemoProject/DemoProject.swift + echo 'import FoundationXML' >> Sources/DemoProject/DemoProject.swift + echo 'import FoundationNetworking' >> Sources/DemoProject/DemoProject.swift + echo 'import Dispatch' >> Sources/DemoProject/DemoProject.swift + echo 'import Android' >> Sources/DemoProject/DemoProject.swift + - name: Test Demo Project on Android + uses: skiptools/swift-android-action@v2 + with: + # only test for the complete arch SDK build to speed up CI + run-tests: ${{ matrix.arch == '' }} + package-path: ${{ runner.temp }}/DemoProject + installed-sdk: ${{ steps.info.outputs.sdk-id }} + installed-swift: ${{ steps.info.outputs.swift-root }} + + - name: Checkout swift-algorithms + uses: actions/checkout@v4 + with: + repository: apple/swift-algorithms + path: swift-algorithms + - name: Test swift-algorithms + uses: skiptools/swift-android-action@v2 + with: + run-tests: ${{ matrix.arch == '' }} + package-path: swift-algorithms + installed-sdk: ${{ steps.info.outputs.sdk-id }} + installed-swift: ${{ steps.info.outputs.swift-root }} diff --git a/swift-ci/sdks/android/README.md b/swift-ci/sdks/android/README.md new file mode 100644 index 00000000..531aa88a --- /dev/null +++ b/swift-ci/sdks/android/README.md @@ -0,0 +1,43 @@ +# Build scripts for Swift Android SDK + +This folder contains scripts to build a Swift Android SDK +in the form of an artifactbundle. + +## Running + +The top-level `./build` script installs a host toolchain and the +Android NDK, and then invokes `scripts/fetch-source.sh` which will +fetch tagged sources for libxml2, curl, boringssl, and swift. + +It then applies some patches and invokes `scripts/build.sh`, +which will build the sources for each of the specified +architectures. Finally, it combines the NDK and the newly built +SDKs into a single artifactbundle. + +## Specifying Architectures + +By default all the supported Android architectures +(`aarch64`, `x86_64`, `aarmv7`) +will be built, but this can be reduced in order to speed +up the build. This can be useful, e.g., as part of a CI that +validates a pull request, as building a single architecture +takes around 30 minutes on a standard ubuntu-24.04 GitHub runner, +whereas building for all the architectures takes over an hour. + +To build an artifactbundle for just the `x86_64` architecture, run: + +``` +TARGET_ARCHS=x86_64 ./build +``` + +## Installing and validating the SDK + +The `.github/workflows/pull_request.yml` workflow +will create and upload an installable SDK named something like: +`swift-6.1-RELEASE_android-0.1.artifactbundle.tar.gz` + +The workflow will also install the SDK locally and use +[swift-android-action](https://github.com/marketplace/actions/swift-android-action) +to build and test various Swift packages in an Android emulator. + + diff --git a/swift-ci/sdks/android/build b/swift-ci/sdks/android/build new file mode 100755 index 00000000..e35bd431 --- /dev/null +++ b/swift-ci/sdks/android/build @@ -0,0 +1,91 @@ +#!/bin/bash -e +# Swift Android SDK: Top-level Build Script + +# default architectures to build for +TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} + +ANDROID_NDK_VERSION=android-ndk-r27c +ANDROID_API=28 + +SDKROOT=${RUNNER_TEMP:-${TMPDIR:-'/tmp'}}/swift-android-sdk +mkdir -p ${SDKROOT} + +# Install a Swift host toolchain if it isn't already present +SWIFT_ROOT=${SDKROOT}/host-toolchain +HOST_OS=ubuntu$(lsb_release -sr) +SWIFT_VERSION=6.1 +SWIFT_TAG="swift-${SWIFT_VERSION}-RELEASE" +SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-release" +SWIFT_BASE=$SWIFT_TAG-$HOST_OS + +if [[ ! -d $SWIFT_ROOT ]]; then + mkdir -p $SWIFT_ROOT + pushd $SWIFT_ROOT + + SWIFT_URL="https://download.swift.org/$SWIFT_BRANCH/$(echo $HOST_OS | tr -d '.')/$SWIFT_TAG/$SWIFT_BASE.tar.gz" + wget -q $SWIFT_URL + tar xzf $SWIFT_BASE.tar.gz + + popd +fi + +#HOST_TOOLCHAIN=${HOST_TOOLCHAIN:-$(dirname $(dirname $(which swiftc)))} +HOST_TOOLCHAIN=$SWIFT_ROOT/$SWIFT_BASE/usr +$HOST_TOOLCHAIN/bin/swift --version + +# ensure the correct Swift is first in the PATH +export PATH=$HOST_TOOLCHAIN/bin:$PATH + +export ANDROID_NDK_HOME=${SDKROOT}/ndk/${ANDROID_NDK_VERSION} +export ANDROID_NDK=${ANDROID_NDK_HOME} + +if [[ ! -d ${ANDROID_NDK_HOME} ]]; then + mkdir -p $(dirname ${ANDROID_NDK_HOME}) + pushd $(dirname ${ANDROID_NDK_HOME}) + NDKFILE=$(basename $ANDROID_NDK_HOME)-linux.zip + wget -q https://dl.google.com/android/repository/${NDKFILE} + unzip -q ${NDKFILE} + popd +fi + +mkdir -p ${SDKROOT}/products + +# Check-out the sources +if [[ ! -d ${SDKROOT}/src ]]; then + scripts/fetch-source.sh --source-dir ${SDKROOT}/src --swift-tag ${SWIFT_TAG} +fi + +# fetch and apply the patches +PATCHDIR=${SDKROOT}/patches +if [[ ! -d ${PATCHDIR} ]]; then + git clone https://github.com/finagolfin/swift-android-sdk.git ${PATCHDIR} + + # TODO: need to selectively apply patches based on release or not release + pushd ${SDKROOT}/src/swift-project + echo "Applying patches" + + # patch the patch, which seems to only be needed for an API less than 28 + # https://github.com/finagolfin/swift-android-sdk/blob/main/swift-android.patch#L110 + perl -pi -e 's/#if os\(Windows\)/#if os\(Android\)/g' $PATCHDIR/swift-android.patch + + # remove the need to link in android-execinfo + perl -pi -e 's/dispatch android-execinfo/dispatch/g' $PATCHDIR/swift-android.patch + + git apply -v $PATCHDIR/swift-android.patch + # swift-android-ci.patch is not needed, since it is only used for llbuild, etc. + #git apply -C1 $PATCHDIR/swift-android-ci.patch + #git apply -v $PATCHDIR/swift-android-ci-release.patch + git apply -v $PATCHDIR/swift-android-testing-release.patch + + perl -pi -e 's%String\(cString: getpass%\"fake\" //%' swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift + # disable backtrace() for Android (needs either API33+ or libandroid-execinfo, or to manually add in backtrace backport) + perl -pi -e 's/os\(Android\)/os\(AndroidDISABLED\)/g' swift-testing/Sources/Testing/SourceAttribution/Backtrace.swift + + # need to un-apply libandroid-spawn since we don't need it for API28+ + perl -pi -e 's/MATCHES "Android"/MATCHES "AndroidDISABLED"/g' llbuild/lib/llvm/Support/CMakeLists.txt + perl -pi -e 's/ STREQUAL Android\)/ STREQUAL AndroidDISABLED\)/g' swift-corelibs-foundation/Sources/Foundation/CMakeLists.txt + popd +fi + +./scripts/build.sh --products-dir ${SDKROOT}/products --source-dir ${SDKROOT}/src --build-dir ${SDKROOT}/build --ndk-home ${ANDROID_NDK_HOME} --android-api ${ANDROID_API} --host-toolchain ${HOST_TOOLCHAIN} --archs ${TARGET_ARCHS} + diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh new file mode 100755 index 00000000..843ff2db --- /dev/null +++ b/swift-ci/sdks/android/scripts/build.sh @@ -0,0 +1,536 @@ +#!/bin/bash +# Swift SDK for Android: Build Script +set -e + +# Docker sets TERM to xterm if using a pty; we probably want +# xterm-256color, otherwise we only get eight colors +if [ -t 1 ]; then + if [[ "$TERM" == "xterm" ]]; then + export TERM=xterm-256color + fi +fi + +if [[ -n "$TERM" ]]; then + bold="" + white="" + grey="" + reset="" +else + bold=$(tput bold) + white=$(tput setaf 15) + grey=$(tput setaf 8) + reset=$(tput sgr0) +fi + +function cleanup { + echo "${reset}" +} +trap cleanup EXIT + +function header { + local text="$1" + echo "" + echo "${white}${bold}*** ${text} ***${reset}${grey}" + echo "" +} + +function groupstart { + local text="$1" + if [[ ! -z "$CI" ]]; then + echo "::group::${text}" + fi + header $text +} + +function groupend { + if [[ ! -z "$CI" ]]; then + echo "::endgroup::" + fi +} + +function usage { + cat < --products-dir --ndk-home --host-toolchain + [--name ] [--version ] [--build-dir ] + [--archs [, ...]] + +Build the Swift Android SDK. + +Options: + + --name Specify the name of the SDK bundle. + --version Specify the version of the Android SDK. + --source-dir Specify the path in which the sources can be found. + --ndk-home Specify the path to the Android NDK + --host-toolchain Specify the path to the host Swift toolchain + --products-dir Specify the path in which the products should be written. + --build-dir Specify the path in which intermediates should be stored. + --android-api Specify the Android API level + (Default is ${android_api}). + --archs [, ...] + Specify the architectures for which we should build + the SDK. + (Default is ${archs}). + --build Specify the CMake build type to use (Release, Debug, + RelWithDebInfo). + (Default is ${build_type}). + -j + --jobs Specify the number of parallel jobs to run at a time. + (Default is ${parallel_jobs}.) +EOF +} + +# Declare all the packages we depend on +declare -a packages + +function declare_package +{ + local name=$1 + local userVisibleName=$2 + local license=$3 + local url=$4 + + local snake=$(echo ${name} | tr '_' '-') + + declare -g ${name}_snake="$snake" + declare -g ${name}_name="$userVisibleName" + declare -g ${name}_license="$license" + declare -g ${name}_url="$url" + + packages+=(${name}) +} + +declare_package android_sdk \ + "Swift SDK for Android" \ + "Apache-2.0" "https://swift.org/install" +declare_package swift "swift" "Apache-2.0" "https://swift.org" +declare_package libxml2 "libxml2" "MIT" \ + "https://github.com/GNOME/libxml2" +declare_package curl "curl" "MIT" "https://curl.se" +declare_package boringssl "boringssl" "OpenSSL AND ISC AND MIT" \ + "https://boringssl.googlesource.com/boringssl/" +declare_package zlib "zlib" "Zlib" "https://zlib.net" + +# Parse command line arguments +android_sdk_version=0.1 +sdk_name= +archs=aarch64,armv7,x86_64 +android_api=28 +build_type=Release +parallel_jobs=$(($(nproc --all) + 2)) +source_dir= +ndk_home=${ANDROID_NDK} +build_dir=$(pwd)/build +products_dir= + +while [ "$#" -gt 0 ]; do + case "$1" in + --source-dir) + source_dir="$2"; shift ;; + --ndk-home) + ndk_home="$2"; shift ;; + --host-toolchain) + host_toolchain="$2"; shift ;; + --build-dir) + build_dir="$2"; shift ;; + --android-api) + android_api="$2"; shift ;; + --products-dir) + products_dir="$2"; shift ;; + --name) + sdk_name="$2"; shift ;; + --archs) + archs="$2"; shift ;; + --version) + android_sdk_version="$2"; shift ;; + -j|--jobs) + parallel_jobs=$2; shift ;; + *) + echo "Unknown argument '$1'"; usage; exit 0 ;; + esac + shift +done + +# Change the commas for spaces +archs="${archs//,/ }" + +if [[ -z "$source_dir" || -z "$products_dir" || -z "$ndk_home" || -z "$host_toolchain" ]]; then + usage + exit 1 +fi + +if ! swiftc=$(which swiftc); then + echo "build.sh: Unable to find Swift compiler. You must have a Swift toolchain installed to build the Android SDK." + exit 1 +fi + +script_dir=$(dirname -- "${BASH_SOURCE[0]}") +resource_dir="${script_dir}/../resources" + +# Find the version numbers of the various dependencies +function describe { + pushd $1 >/dev/null 2>&1 + git describe --tags + popd >/dev/null 2>&1 +} +function versionFromTag { + desc=$(describe $1) + if [[ $desc == v* ]]; then + echo "${desc#v}" + else + echo "${desc}" + fi +} + +swift_version=$(describe ${source_dir}/swift-project/swift) +if [[ $swift_version == swift-* ]]; then + swift_version=${swift_version#swift-} +fi + +if [[ -z "$sdk_name" ]]; then + sdk_name=swift-${swift_version}_android-${android_sdk_version} +fi + +libxml2_version=$(versionFromTag ${source_dir}/libxml2) + +curl_desc=$(describe ${source_dir}/curl | tr '_' '.') +curl_version=${curl_desc#curl-} + +boringssl_version=$(describe ${source_dir}/boringssl) + +zlib_version=$(versionFromTag ${source_dir}/zlib) + +function quiet_pushd { + pushd "$1" >/dev/null 2>&1 +} +function quiet_popd { + popd >/dev/null 2>&1 +} + +header "Swift Android SDK build script" + +swift_dir=$(realpath $(dirname "$swiftc")/..) +HOST=linux-x86_64 +#HOST=$(uname -s -m | tr '[:upper:]' '[:lower:]' | tr ' ' '-') +ndk_toolchain=$ndk_home/toolchains/llvm/prebuilt/$HOST + + +echo "Swift found at ${swift_dir}" +echo "Host toolchain found at ${host_toolchain}" +${host_toolchain}/bin/swift --version +echo "Android NDK found at ${ndk_home}" +${ndk_toolchain}/bin/clang --version +echo "Building for ${archs}" +echo "Sources are in ${source_dir}" +echo "Build will happen in ${build_dir}" +echo "Products will be placed in ${products_dir}" +echo +echo "Building from:" +echo " - Swift ${swift_version}" +echo " - libxml2 ${libxml2_version}" +echo " - curl ${curl_version}" +echo " - BoringSSL ${boringssl_version}" +echo " - zlib ${zlib_version}" + +function run() { + echo "$@" + "$@" +} + +for arch in $archs; do + # enable short-circuiting the individual builds + if [[ ! -z "$SWIFT_ANDROID_ARCHIVEONLY" ]]; then + continue + fi + + case $arch in + armv7) target_host="arm-linux-androideabi"; compiler_target_host="armv7a-linux-androideabi$android_api"; android_abi="armeabi-v7a" ;; + aarch64) target_host="aarch64-linux-android"; compiler_target_host="$target_host$android_api"; android_abi="arm64-v8a" ;; + x86_64) target_host="x86_64-linux-android"; compiler_target_host="$target_host$android_api"; android_abi="x86_64" ;; + x86) target_host="x86-linux-android"; compiler_target_host="$target_host$android_api"; android_abi="x86" ;; + *) echo "Unknown architecture '$1'"; usage; exit 0 ;; + esac + + sdk_root=${build_dir}/sdk_root/${arch} + mkdir -p "$sdk_root" + + groupstart "Building libxml2 for $arch" + quiet_pushd ${source_dir}/libxml2 + run cmake \ + -G Ninja \ + -S ${source_dir}/libxml2 \ + -B ${build_dir}/$arch/libxml2 \ + -DANDROID_ABI=$android_abi \ + -DANDROID_PLATFORM=android-$android_api \ + -DCMAKE_TOOLCHAIN_FILE=$ndk_home/build/cmake/android.toolchain.cmake \ + -DCMAKE_BUILD_TYPE=$build_type \ + -DCMAKE_EXTRA_LINK_FLAGS="-rtlib=compiler-rt -unwindlib=libunwind -stdlib=libc++ -fuse-ld=lld -lc++ -lc++abi" \ + -DCMAKE_BUILD_TYPE=$build_type \ + -DCMAKE_INSTALL_PREFIX=$sdk_root/usr \ + -DLIBXML2_WITH_PYTHON=NO \ + -DLIBXML2_WITH_ICU=NO \ + -DLIBXML2_WITH_ICONV=NO \ + -DLIBXML2_WITH_LZMA=NO \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_STATIC_LIBS=ON + + quiet_pushd ${build_dir}/$arch/libxml2 + run ninja -j$parallel_jobs + quiet_popd + + header "Installing libxml2 for $arch" + quiet_pushd ${build_dir}/$arch/libxml2 + run ninja -j$parallel_jobs install + quiet_popd + quiet_popd + groupend + + groupstart "Building boringssl for ${compiler_target_host}" + quiet_pushd ${source_dir}/boringssl + run cmake \ + -GNinja \ + -B ${build_dir}/$arch/boringssl \ + -DANDROID_ABI=$android_abi \ + -DANDROID_PLATFORM=android-$android_api \ + -DCMAKE_TOOLCHAIN_FILE=$ndk_home/build/cmake/android.toolchain.cmake \ + -DCMAKE_BUILD_TYPE=$build_type \ + -DCMAKE_INSTALL_PREFIX=$sdk_root/usr \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_STATIC_LIBS=ON \ + -DBUILD_TESTING=OFF + + quiet_pushd ${build_dir}/$arch/boringssl + run ninja -j$parallel_jobs + quiet_popd + + header "Installing BoringSSL for $arch" + quiet_pushd ${build_dir}/$arch/boringssl + run ninja -j$parallel_jobs install + quiet_popd + quiet_popd + groupend + + groupstart "Building libcurl for ${compiler_target_host}" + quiet_pushd ${source_dir}/curl + run cmake \ + -G Ninja \ + -S ${source_dir}/curl \ + -B ${build_dir}/$arch/curl \ + -DANDROID_ABI=$android_abi \ + -DANDROID_PLATFORM=android-$android_api \ + -DCMAKE_TOOLCHAIN_FILE=$ndk_home/build/cmake/android.toolchain.cmake \ + -DCMAKE_BUILD_TYPE=$build_type \ + -DCMAKE_INSTALL_PREFIX=$sdk_root/usr \ + -DOPENSSL_ROOT_DIR=$sdk_root/usr \ + -DOPENSSL_INCLUDE_DIR=$sdk_root/usr/include \ + -DOPENSSL_SSL_LIBRARY=$sdk_root/usr/lib/libssl.a \ + -DOPENSSL_CRYPTO_LIBRARY=$sdk_root/usr/lib/libcrypto.a \ + -DCURL_USE_OPENSSL=ON \ + -DCURLSSLOPT_NATIVE_CA=ON \ + -DTHREADS_PREFER_PTHREAD_FLAG=OFF \ + -DCMAKE_THREAD_PREFER_PTHREAD=OFF \ + -DCMAKE_THREADS_PREFER_PTHREAD_FLAG=OFF \ + -DCMAKE_HAVE_LIBC_PTHREAD=YES \ + -DBUILD_CURL_EXE=NO \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_STATIC_LIBS=ON \ + -DCURL_BUILD_TESTS=OFF + + quiet_pushd ${build_dir}/$arch/curl + run ninja -j$parallel_jobs + quiet_popd + + header "Installing libcurl for $arch" + quiet_pushd ${build_dir}/$arch/curl + run ninja -j$parallel_jobs install + quiet_popd + quiet_popd + groupend + + groupstart "Building Android SDK for ${compiler_target_host}" + quiet_pushd ${source_dir}/swift-project + build_type_flag="--debug" + case $build_type in + Debug) build_type_flag="--debug" ;; + Release) build_type_flag="--release" ;; + RelWithDebInfo) build_type_flag="--release-debuginfo" ;; + esac + + # use an out-of-tree build folder, otherwise subsequent arch builds have conflicts + export SWIFT_BUILD_ROOT=${build_dir}/$arch/swift-project + + ./swift/utils/build-script \ + $build_type_flag \ + --reconfigure \ + --no-assertions \ + --android \ + --android-ndk=$ndk_home \ + --android-arch=$arch \ + --android-api-level=$android_api \ + --native-swift-tools-path=$host_toolchain/bin \ + --native-clang-tools-path=$host_toolchain/bin \ + --cross-compile-hosts=android-$arch \ + --cross-compile-deps-path=$sdk_root \ + --install-destdir=$sdk_root \ + --build-llvm=0 \ + --build-swift-tools=0 \ + --skip-build-cmark \ + --skip-local-build \ + --build-swift-static-stdlib \ + --install-swift \ + --install-libdispatch \ + --install-foundation \ + --xctest --install-xctest \ + --swift-testing --install-swift-testing \ + --cross-compile-append-host-target-to-destdir=False + + # need to remove symlink that gets created in the NDK to the previous arch's build + # or else we get errors like: + # error: could not find module '_Builtin_float' for target 'x86_64-unknown-linux-android'; found: aarch64-unknown-linux-android, at: /home/runner/work/_temp/swift-android-sdk/ndk/android-ndk-r27c/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/swift/android/_Builtin_float.swiftmodule + rm -f $ndk_toolchain/sysroot/usr/lib/swift + quiet_popd + groupend +done + +# Now generate the bundle +groupstart "Bundling SDK" + +sdk_base=swift-android +sdk_staging="sdk_staging" + +bundle="${sdk_name}.artifactbundle" + +rm -rf ${build_dir}/$bundle +mkdir -p ${build_dir}/$bundle +quiet_pushd ${build_dir}/$bundle + +# First the info.json, for SwiftPM +cat > info.json < $sysroot_path/SDKSettings.json < swift-sdk.json <> swift-sdk.json <> swift-sdk.json <> swift-sdk.json < swift-toolset.json <|--swift-tag + |--swift-version ] + [--curl-version ] + [--boringssl-version ] + [--clone-with-ssh] + [--source-dir ] + +Fetch all the sources required to build the fully statically linked Linux +SDK for Swift. Options are: + + --clone-with-ssh Use git-over-SSH rather than HTTPS where possible. + --source-dir Specify the path in which the sources should be checked + out. This directory will be created it if does not exist. + --swift-scheme + --swift-tag + --swift-version + Select the version of Swift to check out sources for. + If starts with "scheme:" or "tag:", it will + select a scheme or tag; otherwise it will be treated as + a version number. + --libxml2-version + --curl-version + --boringssl-version +EOF +} + +# Defaults +if [[ -z "${SWIFT_VERSION}" ]]; then + SWIFT_VERSION=scheme:release/6.1 +fi +if [[ -z "${LIBXML2_VERSION}" ]]; then + LIBXML2_VERSION=2.12.7 + #LIBXML2_VERSION=2.14.2 +fi +if [[ -z "${CURL_VERSION}" ]]; then + CURL_VERSION=8.7.1 + #CURL_VERSION=8.13.0 +fi +if [[ -z "${BORINGSSL_VERSION}" ]]; then + BORINGSSL_VERSION=fips-20220613 +fi + +clone_with_ssh=false +while [ "$#" -gt 0 ]; do + case "$1" in + --swift-scheme) + SWIFT_VERSION="scheme:$2"; shift ;; + --swift-tag) + SWIFT_VERSION="tag:$2"; shift ;; + --swift-version) + SWIFT_VERSION="$2"; shift ;; + --libxml2-version) + LIBXML2_VERSION="$2"; shift ;; + --curl-version) + CURL_VERSION="$2"; shift ;; + --boringssl-version) + BORINGSSL_VERSION="$2"; shift ;; + --clone-with-ssh) + clone_with_ssh=true ;; + --source-dir) + source_dir="$2"; shift ;; + *) + usage; exit 0 ;; + esac + shift +done + +if [[ ! -z "$source_dir" ]]; then + mkdir -p "$source_dir" +else + source_dir=. +fi + +if [[ "$clone_with_ssh" == "true" ]]; then + github=git@github.com: + clone_arg=--clone-with-ssh +else + github=https://github.com/ + clone_arg=--clone +fi + +cd "$source_dir" + +# Fetch Swift +mkdir -p swift-project + +groupstart "Fetching Swift" +pushd swift-project >/dev/null + +[[ -d swift ]] || git clone ${github}apple/swift.git +cd swift + +# Get its dependencies +header "Fetching Swift Dependencies" + +extra_args="--skip-history --all-repositories" +if [[ $SWIFT_VERSION == scheme:* ]]; then + utils/update-checkout ${clone_arg} --scheme ${SWIFT_VERSION#scheme:} ${extra_args} +elif [[ $SWIFT_VERSION == tag:* ]]; then + utils/update-checkout ${clone_arg} --tag ${SWIFT_VERSION#tag:} ${extra_args} +else + utils/update-checkout ${clone_arg} --tag swift-${SWIFT_VERSION}-RELEASE ${extra_args} +fi + +popd >/dev/null +groupend + +# Fetch libxml2 +groupstart "Fetching libxml2" +[[ -d libxml2 ]] || git clone ${github}GNOME/libxml2.git +pushd libxml2 >/dev/null 2>&1 +git checkout v${LIBXML2_VERSION} +popd >/dev/null 2>&1 +groupend + +# Fetch curl +groupstart "Fetching curl" +[[ -d curl ]] || git clone ${github}curl/curl.git +pushd curl >/dev/null 2>&1 +git checkout curl-$(echo ${CURL_VERSION} | tr '.' '_') +popd >/dev/null 2>&1 +groupend + +# Fetch BoringSSL +groupstart "Fetching BoringSSL" +[[ -d boringssl ]] || git clone https://boringssl.googlesource.com/boringssl +pushd boringssl >/dev/null 2>&1 +git checkout ${BORINGSSL_VERSION} +popd >/dev/null 2>&1 +groupend + From 40733f718438f1553fcbd359c10b53f8b5d150b7 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Wed, 30 Apr 2025 20:25:12 -0400 Subject: [PATCH 03/44] Swift Android build 6.2 (#2) * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 * Start splitting NDK out from the rest of the SDK * Start splitting NDK out from the rest of the SDK * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 * Swift Android build 6.2 --- .github/workflows/pull_request.yml | 50 ++++++------ swift-ci/sdks/android/build | 30 +++++-- swift-ci/sdks/android/scripts/build.sh | 103 ++++++++++++++++++------- 3 files changed, 123 insertions(+), 60 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index ed578bc6..582b2b9b 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -8,6 +8,8 @@ env: jobs: build: name: Build Docker images + # disabled for CI testing + if: false runs-on: ubuntu-latest steps: - name: Checkout repository @@ -24,6 +26,8 @@ jobs: static-linux-build: name: Build Static Linux image + # disabled for CI testing + if: false runs-on: ubuntu-latest steps: - name: Checkout repository @@ -33,7 +37,7 @@ jobs: run: ./build android-build: - name: Build Android ${{ matrix.arch }} SDK + name: Build Android ${{ matrix.swift-version }} ${{ matrix.arch }} SDK strategy: fail-fast: false matrix: @@ -43,8 +47,18 @@ jobs: #arch: ['x86_64'] # build both the quick (x86_64) and complete (aarch64,x86_64,armv7) SDKs arch: ['x86_64', ''] + swift-version: ['release', 'devel', 'trunk'] runs-on: ubuntu-24.04 steps: + - name: Free Disk Space + run: | + df -h + # brings available space from 25G to 32G + # otherwise we sometimes run out of space during the build + sudo rm -rf /usr/share/miniconda /usr/share/az* /usr/share/glade* /usr/local/share/chromium /usr/local/share/powershell /usr/share/dotnet /opt/ghc /opt/hostedtoolcache/CodeQL + sudo docker image prune --all --force + sudo docker builder prune -a + df -h - name: Checkout repository uses: actions/checkout@v4 - name: Install Dependencies @@ -53,7 +67,7 @@ jobs: - name: Build Android SDK working-directory: swift-ci/sdks/android run: | - TARGET_ARCHS=${{ matrix.arch }} ./build + BUILD_VERSION=${{ matrix.swift-version }} TARGET_ARCHS=${{ matrix.arch }} ./build - name: Get artifact info id: info shell: bash @@ -82,28 +96,8 @@ jobs: # swift-6.1-RELEASE_android-0.1-x86_64.artifactbundle.tar.gz.zip echo "artifact-name=${ARTIFACT_NAME}" >> $GITHUB_OUTPUT - # validate some required paths in the artifactbundle - pushd ${ARTIFACT_BUILD}/*/*/usr - ls lib/swift/android - ls lib/swift/android/* - ls lib/swift/android/*/swiftrt.o - ls lib/*-linux-android/*/crtbegin_dynamic.o - - ls lib/swift_static-* - ls lib/swift_static-*/android - ls lib/swift_static-*/android/libFoundationEssentials.a - - ls lib/*-linux-android/libFoundationEssentials.so - ls lib/*-linux-android/libFoundationNetworking.so - ls lib/*-linux-android/libFoundationInternationalization.so - ls lib/*-linux-android/lib_FoundationICU.so - ls lib/*-linux-android/libFoundationXML.so - ls lib/*-linux-android/libTesting.so - - ls lib/swift/clang/lib - ls lib/swift/clang/lib/linux - ls lib/swift/clang/lib/linux/*/libunwind.a - popd + # show an abridged tree + tree ${ARTIFACT_BUILD} --filesfirst --prune -P 'Android.swiftmodule' -P 'libswiftAndroid.*' -P 'libFoundation.*' -P 'swiftrt.o' -P 'swift*.json' -P 'info.json' -P 'api-level.h' -P 'android.modulemap' -P 'SwiftAndroidNDK.h' -P 'bridging.modulemap' -P 'linux' -P 'libclang*.a' -P 'libunwind.a' -P 'libclang_rt.builtins-*-android.a' - name: Upload SDK artifactbundle uses: actions/upload-artifact@v4 with: @@ -113,7 +107,9 @@ jobs: - name: Cleanup run: | # need to free up some space or else when installing we get: No space left on device + df -h rm -rf ${{ runner.temp }}/swift-android-sdk/{build,src} + df -h - name: Install artifactbundle shell: bash run: | @@ -138,10 +134,10 @@ jobs: echo 'import Dispatch' >> Sources/DemoProject/DemoProject.swift echo 'import Android' >> Sources/DemoProject/DemoProject.swift - name: Test Demo Project on Android - uses: skiptools/swift-android-action@v2 + uses: skiptools/swift-android-action@main with: # only test for the complete arch SDK build to speed up CI - run-tests: ${{ matrix.arch == '' }} + #run-tests: ${{ matrix.arch == '' }} package-path: ${{ runner.temp }}/DemoProject installed-sdk: ${{ steps.info.outputs.sdk-id }} installed-swift: ${{ steps.info.outputs.swift-root }} @@ -152,7 +148,7 @@ jobs: repository: apple/swift-algorithms path: swift-algorithms - name: Test swift-algorithms - uses: skiptools/swift-android-action@v2 + uses: skiptools/swift-android-action@main with: run-tests: ${{ matrix.arch == '' }} package-path: swift-algorithms diff --git a/swift-ci/sdks/android/build b/swift-ci/sdks/android/build index e35bd431..20b9fce0 100755 --- a/swift-ci/sdks/android/build +++ b/swift-ci/sdks/android/build @@ -13,12 +13,28 @@ mkdir -p ${SDKROOT} # Install a Swift host toolchain if it isn't already present SWIFT_ROOT=${SDKROOT}/host-toolchain HOST_OS=ubuntu$(lsb_release -sr) -SWIFT_VERSION=6.1 -SWIFT_TAG="swift-${SWIFT_VERSION}-RELEASE" -SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-release" + +#BUILD_VERSION='release' +#BUILD_VERSION='trunk' +BUILD_VERSION=${BUILD_VERSION:-'devel'} + +if [ "${BUILD_VERSION}" = 'release' ]; then + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/releases | grep -m1 swift-6.1 | cut -d- -f2) + SWIFT_TAG="swift-${LATEST_TOOLCHAIN_VERSION}-RELEASE" + SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-release" +elif [ "${BUILD_VERSION}" = 'devel' ]; then + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-6.2-DEV | cut -d- -f8-10) + SWIFT_TAG="swift-6.2-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" + SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-branch" +else + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-DEV | cut -d- -f7-9) + SWIFT_TAG="swift-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" + SWIFT_BRANCH="development" +fi + SWIFT_BASE=$SWIFT_TAG-$HOST_OS -if [[ ! -d $SWIFT_ROOT ]]; then +if [[ ! -d "$SWIFT_ROOT/$SWIFT_BASE" ]]; then mkdir -p $SWIFT_ROOT pushd $SWIFT_ROOT @@ -75,7 +91,11 @@ if [[ ! -d ${PATCHDIR} ]]; then # swift-android-ci.patch is not needed, since it is only used for llbuild, etc. #git apply -C1 $PATCHDIR/swift-android-ci.patch #git apply -v $PATCHDIR/swift-android-ci-release.patch - git apply -v $PATCHDIR/swift-android-testing-release.patch + if [ "${BUILD_VERSION}" = 'release' ]; then + git apply -v $PATCHDIR/swift-android-testing-release.patch + else + git apply -v $PATCHDIR/swift-android-testing-except-release.patch + fi perl -pi -e 's%String\(cString: getpass%\"fake\" //%' swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift # disable backtrace() for Android (needs either API33+ or libandroid-execinfo, or to manually add in backtrace backport) diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 843ff2db..a592b2b4 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -1,6 +1,10 @@ #!/bin/bash # Swift SDK for Android: Build Script -set -e +set -ex + +# temporary for splitting out NDK installation from the rest of the SDK +#NDK_LOCATION=${NDK_LOCATION:-"merged"} +NDK_LOCATION=${NDK_LOCATION:-"external"} # Docker sets TERM to xterm if using a pty; we probably want # xterm-256color, otherwise we only get eight colors @@ -188,7 +192,7 @@ if [[ $swift_version == swift-* ]]; then fi if [[ -z "$sdk_name" ]]; then - sdk_name=swift-${swift_version}_android-${android_sdk_version} + sdk_name=swift-${swift_version}-android-${android_sdk_version} fi libxml2_version=$(versionFromTag ${source_dir}/libxml2) @@ -212,14 +216,13 @@ header "Swift Android SDK build script" swift_dir=$(realpath $(dirname "$swiftc")/..) HOST=linux-x86_64 #HOST=$(uname -s -m | tr '[:upper:]' '[:lower:]' | tr ' ' '-') -ndk_toolchain=$ndk_home/toolchains/llvm/prebuilt/$HOST - +ndk_installation=$ndk_home/toolchains/llvm/prebuilt/$HOST echo "Swift found at ${swift_dir}" echo "Host toolchain found at ${host_toolchain}" ${host_toolchain}/bin/swift --version echo "Android NDK found at ${ndk_home}" -${ndk_toolchain}/bin/clang --version +${ndk_installation}/bin/clang --version echo "Building for ${archs}" echo "Sources are in ${source_dir}" echo "Build will happen in ${build_dir}" @@ -387,7 +390,7 @@ for arch in $archs; do # need to remove symlink that gets created in the NDK to the previous arch's build # or else we get errors like: # error: could not find module '_Builtin_float' for target 'x86_64-unknown-linux-android'; found: aarch64-unknown-linux-android, at: /home/runner/work/_temp/swift-android-sdk/ndk/android-ndk-r27c/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/swift/android/_Builtin_float.swiftmodule - rm -f $ndk_toolchain/sysroot/usr/lib/swift + rm -f $ndk_installation/sysroot/usr/lib/swift quiet_popd groupend done @@ -425,12 +428,22 @@ EOF mkdir -p $sdk_base quiet_pushd $sdk_base -#sysroot_path="ndk-sysroot" -#sysroot_path="android-27c-sysroot" -sysroot_path="sysroot" -cp -a ${ndk_toolchain}/sysroot ${sysroot_path} +cp -a ${build_dir}/sdk_root ${sdk_staging} + +if [ "${NDK_LOCATION}" = "external" ]; then + swift_res_root="swift-resources" + ndk_sysroot="ndk-sysroot" + cp -a ${ndk_installation}/sysroot ${ndk_sysroot} +else + merged_sysroot_path="sysroot" + swift_res_root=${merged_sysroot_path} + ndk_sysroot=${merged_sysroot_path} + cp -a ${ndk_installation}/sysroot ${ndk_sysroot} +fi -cat > $sysroot_path/SDKSettings.json < $swift_res_root/SDKSettings.json < $sysroot_path/SDKSettings.json < swift-sdk.json <> swift-sdk.json < Date: Thu, 8 May 2025 12:15:51 -0400 Subject: [PATCH 04/44] Swift Android build 6.2 (#3) * Swift Android build 6.2 * Swift Android build 6.2 * Add post-install script to SDK bundle * Add post-install script to SDK bundle * Add post-install script to SDK bundle * Add post-install script to SDK bundle --- swift-ci/sdks/android/build | 2 +- swift-ci/sdks/android/scripts/build.sh | 113 +++++++++++++++++-------- 2 files changed, 79 insertions(+), 36 deletions(-) diff --git a/swift-ci/sdks/android/build b/swift-ci/sdks/android/build index 20b9fce0..a2834f71 100755 --- a/swift-ci/sdks/android/build +++ b/swift-ci/sdks/android/build @@ -107,5 +107,5 @@ if [[ ! -d ${PATCHDIR} ]]; then popd fi -./scripts/build.sh --products-dir ${SDKROOT}/products --source-dir ${SDKROOT}/src --build-dir ${SDKROOT}/build --ndk-home ${ANDROID_NDK_HOME} --android-api ${ANDROID_API} --host-toolchain ${HOST_TOOLCHAIN} --archs ${TARGET_ARCHS} +./scripts/build.sh --products-dir ${SDKROOT}/products --source-dir ${SDKROOT}/src --build-dir ${SDKROOT}/build --ndk-home ${ANDROID_NDK_HOME} --android-api ${ANDROID_API} --host-toolchain ${HOST_TOOLCHAIN} --archs ${TARGET_ARCHS} ${@} diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index a592b2b4..879aa1e4 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -1,10 +1,6 @@ #!/bin/bash # Swift SDK for Android: Build Script -set -ex - -# temporary for splitting out NDK installation from the rest of the SDK -#NDK_LOCATION=${NDK_LOCATION:-"merged"} -NDK_LOCATION=${NDK_LOCATION:-"external"} +set -e # Docker sets TERM to xterm if using a pty; we probably want # xterm-256color, otherwise we only get eight colors @@ -145,6 +141,8 @@ while [ "$#" -gt 0 ]; do sdk_name="$2"; shift ;; --archs) archs="$2"; shift ;; + --build) + build_type="$2"; shift ;; --version) android_sdk_version="$2"; shift ;; -j|--jobs) @@ -430,17 +428,7 @@ quiet_pushd $sdk_base cp -a ${build_dir}/sdk_root ${sdk_staging} -if [ "${NDK_LOCATION}" = "external" ]; then - swift_res_root="swift-resources" - ndk_sysroot="ndk-sysroot" - cp -a ${ndk_installation}/sysroot ${ndk_sysroot} -else - merged_sysroot_path="sysroot" - swift_res_root=${merged_sysroot_path} - ndk_sysroot=${merged_sysroot_path} - cp -a ${ndk_installation}/sysroot ${ndk_sysroot} -fi - +swift_res_root="swift-resources" mkdir -p ${swift_res_root} cat > $swift_res_root/SDKSettings.json < scripts/setup-android-sdk.sh <<'EOF' +#/bin/sh +# this script will setup the ndk-sysroot with links to the +# local installation indicated by ANDROID_NDK_HOME +set -e +if [ -z "${ANDROID_NDK_HOME}" ]; then + echo "$(basename $0): error: missing environment variable ANDROID_NDK_HOME" + exit 1 +fi +PREBUILT="${ANDROID_NDK_HOME}/toolchains/llvm/prebuilt" +if [ ! -d "${PREBUILT}" ]; then + echo "$(basename $0): error: ANDROID_NDK_HOME not found: ${PREBUILT}" + exit 1 +fi +DESTINATION=$(dirname $(dirname $(realpath $0)))/ndk-sysroot +# clear out any previous NDK setup +rm -rf ${DESTINATION} + +# copy vs. link the NDK files +SWIFT_ANDROID_NDK_LINK=${SWIFT_ANDROID_NDK_LINK:-1} +if [[ "${SWIFT_ANDROID_NDK_LINK}" != 1 ]]; then + ANDROID_NDK_DESC="copied" + cp -a ${PREBUILT}/*/sysroot ${DESTINATION} +else + ANDROID_NDK_DESC="linked" + mkdir -p ${DESTINATION}/usr/lib + ln -s $(realpath ${PREBUILT}/*/sysroot/usr/include) ${DESTINATION}/usr/include + for triplePath in ${PREBUILT}/*/sysroot/usr/lib/*; do + triple=$(basename ${triplePath}) + ln -s $(realpath ${triplePath}) ${DESTINATION}/usr/lib/${triple} + done +fi + +# copy each architecture's swiftrt.o into the sysroot, +# working around https://github.com/swiftlang/swift/pull/79621 +for swiftrt in ${DESTINATION}/../swift-resources/usr/lib/swift-*/android/*/swiftrt.o; do + arch=$(basename $(dirname ${swiftrt})) + mkdir -p ${DESTINATION}/usr/lib/swift/android/${arch} + cp -a ${swiftrt} ${DESTINATION}/usr/lib/swift/android/${arch} +done + +echo "$(basename $0): success: ndk-sysroot ${ANDROID_NDK_DESC} to Android SDK" +EOF + chmod +x scripts/setup-android-sdk.sh +else + COPY_NDK_SYSROOT=${COPY_NDK_SYSROOT:-1} + if [[ ${COPY_NDK_SYSROOT} == 1 ]]; then + cp -a ${ndk_installation}/sysroot ${ndk_sysroot} + else + # rather than copying the sysroot, we can instead make links to + # the various sub-folders this won't work for the distribution, + # since the NDK is going to be located in different places + # for different machines + mkdir -p ${ndk_sysroot}/usr/lib + ln -sv $(realpath ${ndk_installation}/sysroot/usr/include) ${ndk_sysroot}/usr/include + for triplePath in ${ndk_installation}/sysroot/usr/lib/*; do + triple=$(basename ${triplePath}) + ln -sv $(realpath ${triplePath}) ${ndk_sysroot}/usr/lib/${triple} + ls ${ndk_sysroot}/usr/lib/${triple}/ + done + fi + # need to manually copy over swiftrt.o or else: # error: link command failed with exit code 1 (use -v to see invocation) # clang: error: no such file or directory: '${HOME}/.swiftpm/swift-sdks/swift-6.2-DEVELOPMENT-SNAPSHOT-2025-04-24-a-android-0.1.artifactbundle/swift-android/ndk-sysroot/usr/lib/swift/android/x86_64/swiftrt.o' # see: https://github.com/swiftlang/swift-driver/pull/1822#issuecomment-2762811807 + # should be fixed by: https://github.com/swiftlang/swift/pull/79621 for arch in $archs; do mkdir -p ${ndk_sysroot}/usr/lib/swift/android/${arch} ln -srv ${swift_res_root}/usr/lib/swift-${arch}/android/${arch}/swiftrt.o ${ndk_sysroot}/usr/lib/swift/android/${arch}/swiftrt.o done -else - rm -r ${swift_res_root}/usr/{include,lib}/{i686,riscv64}-linux-android - rm -r ${swift_res_root}/usr/lib/swift/clang/lib/linux/*{i[36]86,riscv64}* fi rm -r ${swift_res_root}/usr/share/{doc,man} @@ -535,11 +582,7 @@ for api in $(eval echo "{$android_api..36}"); do EOF fi - if [ "${NDK_LOCATION}" = "external" ]; then - SWIFT_RES_DIR="swift-${arch}" - else - SWIFT_RES_DIR="swift" - fi + SWIFT_RES_DIR="swift-${arch}" SWIFT_STATIC_RES_DIR="swift_static-${arch}" cat >> swift-sdk.json < Date: Sat, 17 May 2025 20:03:09 -0400 Subject: [PATCH 05/44] Build SDK in Docker container (#4) * Add static libraries to post-install script * Add post-install script to SDK bundle * Add post-install script to SDK bundle * Update submodules --- .github/workflows/pull_request.yml | 62 +++++-- .gitmodules | 3 + swift-ci/sdks/android/Dockerfile | 89 +++++++++ swift-ci/sdks/android/README.md | 43 ++++- swift-ci/sdks/android/build | 111 ------------ swift-ci/sdks/android/build-docker | 67 +++++++ swift-ci/sdks/android/build-local | 64 +++++++ swift-ci/sdks/android/resources/patches | 1 + swift-ci/sdks/android/scripts/build.sh | 169 +++++++++--------- swift-ci/sdks/android/scripts/install-ndk.sh | 35 ++++ .../sdks/android/scripts/install-swift.sh | 54 ++++++ .../sdks/android/scripts/patch-sources.sh | 87 +++++++++ .../sdks/android/scripts/toolchain-vars.sh | 44 +++++ 13 files changed, 611 insertions(+), 218 deletions(-) create mode 100644 .gitmodules create mode 100644 swift-ci/sdks/android/Dockerfile delete mode 100755 swift-ci/sdks/android/build create mode 100755 swift-ci/sdks/android/build-docker create mode 100755 swift-ci/sdks/android/build-local create mode 160000 swift-ci/sdks/android/resources/patches create mode 100755 swift-ci/sdks/android/scripts/install-ndk.sh create mode 100755 swift-ci/sdks/android/scripts/install-swift.sh create mode 100755 swift-ci/sdks/android/scripts/patch-sources.sh create mode 100644 swift-ci/sdks/android/scripts/toolchain-vars.sh diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 582b2b9b..13c57b39 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -8,8 +8,6 @@ env: jobs: build: name: Build Docker images - # disabled for CI testing - if: false runs-on: ubuntu-latest steps: - name: Checkout repository @@ -26,8 +24,6 @@ jobs: static-linux-build: name: Build Static Linux image - # disabled for CI testing - if: false runs-on: ubuntu-latest steps: - name: Checkout repository @@ -37,16 +33,18 @@ jobs: run: ./build android-build: - name: Build Android ${{ matrix.swift-version }} ${{ matrix.arch }} SDK + name: Build Android ${{ matrix.build-type }} ${{ matrix.swift-version }} ${{ matrix.arch }} SDK strategy: fail-fast: false matrix: + build-type: ['docker'] + #build-type: ['docker', 'local'] # blank arch builds all (aarch64,x86_64,armv7) - #arch: [''] + arch: [''] # builds only x86_64 to speed up the validation #arch: ['x86_64'] # build both the quick (x86_64) and complete (aarch64,x86_64,armv7) SDKs - arch: ['x86_64', ''] + #arch: ['x86_64', ''] swift-version: ['release', 'devel', 'trunk'] runs-on: ubuntu-24.04 steps: @@ -59,31 +57,60 @@ jobs: sudo docker image prune --all --force sudo docker builder prune -a df -h + - name: Setup + id: config + run: | + # these variabes are used by build-docker and build-local + # to determine which Swift version to build for + echo "BUILD_VERSION=${{ matrix.swift-version }}" >> $GITHUB_ENV + echo "TARGET_ARCHS=${{ matrix.arch }}" >> $GITHUB_ENV + echo "WORKDIR=${{ runner.temp }}/swift-android-sdk" >> $GITHUB_ENV - name: Checkout repository uses: actions/checkout@v4 - - name: Install Dependencies + with: + submodules: 'true' + - name: Build Android SDK (Local) + if: ${{ matrix.build-type == 'local' }} + working-directory: swift-ci/sdks/android run: | sudo apt install -q ninja-build patchelf - - name: Build Android SDK + ./build-local ${BUILD_VERSION} ${WORKDIR} + - name: Build Android SDK (Docker) + if: ${{ matrix.build-type == 'docker' }} + working-directory: swift-ci/sdks/android + run: | + ./build-docker ${BUILD_VERSION} ${WORKDIR} + - name: Install Host Toolchain + if: ${{ matrix.build-type == 'docker' }} working-directory: swift-ci/sdks/android run: | - BUILD_VERSION=${{ matrix.swift-version }} TARGET_ARCHS=${{ matrix.arch }} ./build + # when building in a Docker container, we don't have a local host toolchain, + # but we need one in order to run the SDK validation tests, so we install it now + HOST_OS=ubuntu$(lsb_release -sr) + source ./scripts/toolchain-vars.sh + mkdir -p ${WORKDIR}/host-toolchain + ./scripts/install-swift.sh ${WORKDIR}/host-toolchain/$SWIFT_BASE/usr + ls ${WORKDIR}/host-toolchain + ${WORKDIR}/host-toolchain/*/usr/bin/swift --version - name: Get artifact info id: info shell: bash run: | set -ex - SWIFT_ROOT=$(dirname ${{ runner.temp }}/swift-android-sdk/host-toolchain/*/usr) + SWIFT_ROOT=$(dirname ${WORKDIR}/host-toolchain/*/usr) echo "swift-root=${SWIFT_ROOT}" >> $GITHUB_OUTPUT echo "swift-path=${SWIFT_ROOT}/usr/bin/swift" >> $GITHUB_OUTPUT - ARTIFACT_BUILD=$(realpath ${{ runner.temp }}/swift-android-sdk/build/*.artifactbundle) - ARTIFACT_PATH=$(realpath ${{ runner.temp }}/swift-android-sdk/products/*.artifactbundle.tar.gz) + ARTIFACT_PATH=$(realpath ${WORKDIR}/products/*.artifactbundle.tar.gz) echo "artifact-path=${ARTIFACT_PATH}" >> $GITHUB_OUTPUT echo "sdk-id=x86_64-unknown-linux-android28" >> $GITHUB_OUTPUT ARTIFACT_EXT=".artifactbundle.tar.gz" ARTIFACT_NAME="$(basename ${ARTIFACT_PATH} ${ARTIFACT_EXT})" + # depending on whether we are building locally or in a container, add a maker to the name + if [[ "${{ matrix.build-type }}" == 'local' ]]; then + ARTIFACT_NAME="${ARTIFACT_NAME}-local" + fi # artifacts need a unique name so we suffix with the matrix arch(s) if [[ ! -z "${{ matrix.arch }}" ]]; then ARTIFACT_NAME="${ARTIFACT_NAME}-$(echo ${{ matrix.arch }} | tr ',' '-')" @@ -95,9 +122,6 @@ jobs: # so the actual artifact download will look like: # swift-6.1-RELEASE_android-0.1-x86_64.artifactbundle.tar.gz.zip echo "artifact-name=${ARTIFACT_NAME}" >> $GITHUB_OUTPUT - - # show an abridged tree - tree ${ARTIFACT_BUILD} --filesfirst --prune -P 'Android.swiftmodule' -P 'libswiftAndroid.*' -P 'libFoundation.*' -P 'swiftrt.o' -P 'swift*.json' -P 'info.json' -P 'api-level.h' -P 'android.modulemap' -P 'SwiftAndroidNDK.h' -P 'bridging.modulemap' -P 'linux' -P 'libclang*.a' -P 'libunwind.a' -P 'libclang_rt.builtins-*-android.a' - name: Upload SDK artifactbundle uses: actions/upload-artifact@v4 with: @@ -108,7 +132,9 @@ jobs: run: | # need to free up some space or else when installing we get: No space left on device df -h - rm -rf ${{ runner.temp }}/swift-android-sdk/{build,src} + rm -rf ${WORKDIR}/{build,source} + sudo docker image prune --all --force + sudo docker builder prune -a df -h - name: Install artifactbundle shell: bash @@ -117,7 +143,7 @@ jobs: ${{ steps.info.outputs.swift-path }} sdk install ${{ steps.info.outputs.artifact-path }} ${{ steps.info.outputs.swift-path }} sdk configure --show-configuration $(${{ steps.info.outputs.swift-path }} sdk list | head -n 1) ${{ steps.info.outputs.sdk-id }} # recent releases require that ANDROID_NDK_ROOT *not* be set - # see https://github.com/finagolfin/swift-android-sdk/issues/207 + # see https://github.com/swiftlang/swift-driver/pull/1879 echo "ANDROID_NDK_ROOT=" >> $GITHUB_ENV - name: Create Demo Project diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..f3724176 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "swift-ci/sdks/android/resources/patches"] + path = swift-ci/sdks/android/resources/patches + url = https://github.com/swift-android-sdk/swift-android-sdk diff --git a/swift-ci/sdks/android/Dockerfile b/swift-ci/sdks/android/Dockerfile new file mode 100644 index 00000000..2be3fc37 --- /dev/null +++ b/swift-ci/sdks/android/Dockerfile @@ -0,0 +1,89 @@ +# ===----------------------------------------------------------------------=== +# +# Swift Android SDK: Docker-based build +# +# ===----------------------------------------------------------------------=== + +FROM ubuntu:24.04 + +# Architecture to build on (empty means x86-64) +ARG OS_ARCH_SUFFIX= + +# the Swift toolchain URL to download +ARG SWIFT_TOOLCHAIN_URL= + +# ............................................................................ + +# Install development tools +RUN apt-get -q update \ + && DEBIAN_FRONTEND=noninteractive apt-get -q install -y \ + build-essential \ + cmake \ + ninja-build \ + python3 \ + golang \ + git \ + gnupg2 \ + libsqlite3-dev \ + libcurl4-openssl-dev \ + libedit-dev \ + libicu-dev \ + libncurses5-dev \ + libpython3-dev \ + libsqlite3-dev \ + libxml2-dev \ + rsync \ + uuid-dev \ + uuid-runtime \ + tzdata \ + curl \ + unzip \ + && rm -rf /var/lib/apt-lists/* + +# Install Swift +ARG SWIFT_SIGNING_KEY=E813C892820A6FA13755B268F167DF1ACF9CE069 +ARG SWIFT_PLATFORM=ubuntu +ARG OS_MAJOR_VER=24 +ARG OS_MINOR_VER=04 + +ENV SWIFT_SIGNING_KEY=$SWIFT_SIGNING_KEY \ + SWIFT_PLATFORM=$SWIFT_PLATFORM \ + OS_MAJOR_VER=$OS_MAJOR_VER \ + OS_MINOR_VER=$OS_MINOR_VER \ + OS_VER=$SWIFT_PLATFORM$OS_MAJOR_VER.$OS_MINOR_VER + +COPY scripts/install-swift.sh /scripts/install-swift.sh +RUN chmod ugo+x /scripts/install-swift.sh +RUN /scripts/install-swift.sh /usr/local/swift +ENV PATH="/usr/local/swift/bin:${PATH}" + +ARG ANDROID_NDK_VERSION= + +ENV ANDROID_NDK_VERSION=$ANDROID_NDK_VERSION + +COPY scripts/install-ndk.sh /scripts/install-ndk.sh +RUN chmod ugo+x /scripts/install-ndk.sh +RUN /scripts/install-ndk.sh +ENV ANDROID_NDK_HOME="/usr/local/ndk/${ANDROID_NDK_VERSION}" + +ENV SWIFT_VERSION=$SWIFT_VERSION \ + LIBXML2_VERSION=$LIBXML2_VERSION \ + CURL_VERSION=$CURL_VERSION \ + BORINGSSL_VERSION=$BORINGSSL_VERSION \ + ICU_VERSION=$ICU_VERSION \ + ZLIB_VERSION=$ZLIB_VERSION + +ENV SWIFT_BUILD_DOCKER="1" + +COPY scripts /scripts +RUN chmod ugo+x /scripts/* + +COPY resources /resources + +# Create a user +RUN groupadd -g 998 build-user && \ + useradd -m -r -u 998 -g build-user build-user + +USER build-user + +WORKDIR /home/build-user diff --git a/swift-ci/sdks/android/README.md b/swift-ci/sdks/android/README.md index 531aa88a..10d4db3e 100644 --- a/swift-ci/sdks/android/README.md +++ b/swift-ci/sdks/android/README.md @@ -1,11 +1,39 @@ -# Build scripts for Swift Android SDK +# Dockerfile-based build for Swift Android SDK -This folder contains scripts to build a Swift Android SDK -in the form of an artifactbundle. +This is a Dockerfile-based build set-up for the Swift Android SDK. + +The top-level `./build-docker` script will create a +Docker container and install a host toolchain and the +Android NDK, and then invoke `scripts/fetch-source.sh` which will +fetch tagged sources for libxml2, curl, boringssl, and swift. + +It can be run with: + +``` +$ ./build-docker +``` + +for example: + +``` +$ ./build-docker release /tmp/android-sdk +``` + +This will create an Ubuntu 24.04 container with the necessary dependencies +to build the Android SDK, including a Swift host toolchain and the +Android NDK that will be used for cross-compilation. + +The `version` argument can be one of the following values: + +| version | Swift version | +| --- | --- | +| `release` | swift-6.1-RELEASE | +| `devel` | swift-6.2-DEVELOPMENT-SNAPSHOT-yyyy-mm-dd | +| `trunk` | swift-DEVELOPMENT-SNAPSHOT-yyyy-mm-dd | ## Running -The top-level `./build` script installs a host toolchain and the +The top-level `./build-docker` script installs a host toolchain and the Android NDK, and then invokes `scripts/fetch-source.sh` which will fetch tagged sources for libxml2, curl, boringssl, and swift. @@ -27,7 +55,7 @@ whereas building for all the architectures takes over an hour. To build an artifactbundle for just the `x86_64` architecture, run: ``` -TARGET_ARCHS=x86_64 ./build +TARGET_ARCHS=aarch64 ./build-docker release /tmp/android-sdk ``` ## Installing and validating the SDK @@ -38,6 +66,5 @@ will create and upload an installable SDK named something like: The workflow will also install the SDK locally and use [swift-android-action](https://github.com/marketplace/actions/swift-android-action) -to build and test various Swift packages in an Android emulator. - - +to build and test various Swift packages in an Android emulator using the +freshly-created SDK bundle. diff --git a/swift-ci/sdks/android/build b/swift-ci/sdks/android/build deleted file mode 100755 index a2834f71..00000000 --- a/swift-ci/sdks/android/build +++ /dev/null @@ -1,111 +0,0 @@ -#!/bin/bash -e -# Swift Android SDK: Top-level Build Script - -# default architectures to build for -TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} - -ANDROID_NDK_VERSION=android-ndk-r27c -ANDROID_API=28 - -SDKROOT=${RUNNER_TEMP:-${TMPDIR:-'/tmp'}}/swift-android-sdk -mkdir -p ${SDKROOT} - -# Install a Swift host toolchain if it isn't already present -SWIFT_ROOT=${SDKROOT}/host-toolchain -HOST_OS=ubuntu$(lsb_release -sr) - -#BUILD_VERSION='release' -#BUILD_VERSION='trunk' -BUILD_VERSION=${BUILD_VERSION:-'devel'} - -if [ "${BUILD_VERSION}" = 'release' ]; then - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/releases | grep -m1 swift-6.1 | cut -d- -f2) - SWIFT_TAG="swift-${LATEST_TOOLCHAIN_VERSION}-RELEASE" - SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-release" -elif [ "${BUILD_VERSION}" = 'devel' ]; then - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-6.2-DEV | cut -d- -f8-10) - SWIFT_TAG="swift-6.2-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" - SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-branch" -else - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-DEV | cut -d- -f7-9) - SWIFT_TAG="swift-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" - SWIFT_BRANCH="development" -fi - -SWIFT_BASE=$SWIFT_TAG-$HOST_OS - -if [[ ! -d "$SWIFT_ROOT/$SWIFT_BASE" ]]; then - mkdir -p $SWIFT_ROOT - pushd $SWIFT_ROOT - - SWIFT_URL="https://download.swift.org/$SWIFT_BRANCH/$(echo $HOST_OS | tr -d '.')/$SWIFT_TAG/$SWIFT_BASE.tar.gz" - wget -q $SWIFT_URL - tar xzf $SWIFT_BASE.tar.gz - - popd -fi - -#HOST_TOOLCHAIN=${HOST_TOOLCHAIN:-$(dirname $(dirname $(which swiftc)))} -HOST_TOOLCHAIN=$SWIFT_ROOT/$SWIFT_BASE/usr -$HOST_TOOLCHAIN/bin/swift --version - -# ensure the correct Swift is first in the PATH -export PATH=$HOST_TOOLCHAIN/bin:$PATH - -export ANDROID_NDK_HOME=${SDKROOT}/ndk/${ANDROID_NDK_VERSION} -export ANDROID_NDK=${ANDROID_NDK_HOME} - -if [[ ! -d ${ANDROID_NDK_HOME} ]]; then - mkdir -p $(dirname ${ANDROID_NDK_HOME}) - pushd $(dirname ${ANDROID_NDK_HOME}) - NDKFILE=$(basename $ANDROID_NDK_HOME)-linux.zip - wget -q https://dl.google.com/android/repository/${NDKFILE} - unzip -q ${NDKFILE} - popd -fi - -mkdir -p ${SDKROOT}/products - -# Check-out the sources -if [[ ! -d ${SDKROOT}/src ]]; then - scripts/fetch-source.sh --source-dir ${SDKROOT}/src --swift-tag ${SWIFT_TAG} -fi - -# fetch and apply the patches -PATCHDIR=${SDKROOT}/patches -if [[ ! -d ${PATCHDIR} ]]; then - git clone https://github.com/finagolfin/swift-android-sdk.git ${PATCHDIR} - - # TODO: need to selectively apply patches based on release or not release - pushd ${SDKROOT}/src/swift-project - echo "Applying patches" - - # patch the patch, which seems to only be needed for an API less than 28 - # https://github.com/finagolfin/swift-android-sdk/blob/main/swift-android.patch#L110 - perl -pi -e 's/#if os\(Windows\)/#if os\(Android\)/g' $PATCHDIR/swift-android.patch - - # remove the need to link in android-execinfo - perl -pi -e 's/dispatch android-execinfo/dispatch/g' $PATCHDIR/swift-android.patch - - git apply -v $PATCHDIR/swift-android.patch - # swift-android-ci.patch is not needed, since it is only used for llbuild, etc. - #git apply -C1 $PATCHDIR/swift-android-ci.patch - #git apply -v $PATCHDIR/swift-android-ci-release.patch - if [ "${BUILD_VERSION}" = 'release' ]; then - git apply -v $PATCHDIR/swift-android-testing-release.patch - else - git apply -v $PATCHDIR/swift-android-testing-except-release.patch - fi - - perl -pi -e 's%String\(cString: getpass%\"fake\" //%' swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift - # disable backtrace() for Android (needs either API33+ or libandroid-execinfo, or to manually add in backtrace backport) - perl -pi -e 's/os\(Android\)/os\(AndroidDISABLED\)/g' swift-testing/Sources/Testing/SourceAttribution/Backtrace.swift - - # need to un-apply libandroid-spawn since we don't need it for API28+ - perl -pi -e 's/MATCHES "Android"/MATCHES "AndroidDISABLED"/g' llbuild/lib/llvm/Support/CMakeLists.txt - perl -pi -e 's/ STREQUAL Android\)/ STREQUAL AndroidDISABLED\)/g' swift-corelibs-foundation/Sources/Foundation/CMakeLists.txt - popd -fi - -./scripts/build.sh --products-dir ${SDKROOT}/products --source-dir ${SDKROOT}/src --build-dir ${SDKROOT}/build --ndk-home ${ANDROID_NDK_HOME} --android-api ${ANDROID_API} --host-toolchain ${HOST_TOOLCHAIN} --archs ${TARGET_ARCHS} ${@} - diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker new file mode 100755 index 00000000..fb3b377a --- /dev/null +++ b/swift-ci/sdks/android/build-docker @@ -0,0 +1,67 @@ +#!/bin/bash -e +# +# ===----------------------------------------------------------------------=== +# +# Swift Android SDK: Docker Container Build Script +# +# ===----------------------------------------------------------------------=== + +# default architectures to build for +TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} + +ANDROID_NDK_VERSION=android-ndk-r27c +ANDROID_API=28 + +export BUILD_VERSION=${1} +# note that WORKDIR must not be under the current checkout or the patches will fail to apply +WORKDIR=${2} +if [[ "${WORKDIR}" == '' ]]; then + echo "Usage: $(basename $0) " + exit 1 +fi +mkdir -p ${WORKDIR} +WORKDIR=$(realpath ${WORKDIR}) + +HOST_OS=ubuntu24.04 +source ./scripts/toolchain-vars.sh + +# Check-out and patch the sources +./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-tag ${SWIFT_TAG} +./scripts/patch-sources.sh ${WORKDIR}/source + +mkdir -p ${WORKDIR}/products +chmod ugo+rwx ${WORKDIR}/products + +if [[ "$DOCKER" == "" ]]; then + DOCKER=docker +fi + +case $(arch) in + arm64|aarch64) + OS_ARCH_SUFFIX=-aarch64 + ;; + amd64|x86_64) + OS_ARCH_SUFFIX= + ;; + *) + echo "Unknown architecture $(arch)" + exit 1 + ;; +esac + +CONTAINER_NAME="swift-android" + +# Build the Docker image +$DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX --build-arg SWIFT_TOOLCHAIN_URL=$SWIFT_TOOLCHAIN_URL --build-arg ANDROID_NDK_VERSION=$ANDROID_NDK_VERSION -t ${CONTAINER_NAME} . + +$DOCKER run -i --rm \ + -v ${WORKDIR}/source:/source \ + -v ${WORKDIR}/products:/products:rw \ + ${CONTAINER_NAME} \ + /scripts/build.sh \ + --source-dir /source \ + --products-dir /products \ + --host-toolchain /usr/local/swift \ + --android-api ${ANDROID_API} \ + --ndk-home /usr/local/ndk/${ANDROID_NDK_VERSION} \ + --archs ${TARGET_ARCHS} diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local new file mode 100755 index 00000000..30bd4f92 --- /dev/null +++ b/swift-ci/sdks/android/build-local @@ -0,0 +1,64 @@ +#!/bin/bash -e +# +# ===----------------------------------------------------------------------=== +# +# Swift Android SDK: Local (non-Docker-containerized) Build Script +# +# ===----------------------------------------------------------------------=== + +# default architectures to build for +TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} + +ANDROID_NDK_VERSION=android-ndk-r27c +ANDROID_API=28 + +export BUILD_VERSION=${1} +# note that WORKDIR must not be under the current checkout or the patches will fail to apply +WORKDIR=${2} +if [[ "${WORKDIR}" == '' ]]; then + echo "Usage: $(basename $0) " + exit 1 +fi +mkdir -p ${WORKDIR} +WORKDIR=$(realpath ${WORKDIR}) + +HOST_OS=ubuntu$(lsb_release -sr) +source ./scripts/toolchain-vars.sh + +SWIFT_ROOT=${WORKDIR}/host-toolchain +HOST_TOOLCHAIN=$SWIFT_ROOT/$SWIFT_BASE/usr +if [[ ! -d "$HOST_TOOLCHAIN" ]]; then + ./scripts/install-swift.sh ${HOST_TOOLCHAIN} +fi + +$HOST_TOOLCHAIN/bin/swift --version + +# ensure the correct Swift is first in the PATH +export PATH=$HOST_TOOLCHAIN/bin:$PATH + +export ANDROID_NDK_HOME=${WORKDIR}/ndk/${ANDROID_NDK_VERSION} + +if [[ ! -d ${ANDROID_NDK_HOME} ]]; then + mkdir -p $(dirname ${ANDROID_NDK_HOME}) + pushd $(dirname ${ANDROID_NDK_HOME}) + NDKFILE=$(basename $ANDROID_NDK_HOME)-linux.zip + wget -q https://dl.google.com/android/repository/${NDKFILE} + unzip -q ${NDKFILE} + popd +fi + + +# Check-out and patch the sources +./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-tag ${SWIFT_TAG} +./scripts/patch-sources.sh ${WORKDIR}/source + +mkdir -p ${WORKDIR}/products + +./scripts/build.sh \ + --source-dir ${WORKDIR}/source \ + --products-dir ${WORKDIR}/products \ + --build-dir ${WORKDIR}/build \ + --host-toolchain ${HOST_TOOLCHAIN} \ + --android-api ${ANDROID_API} \ + --ndk-home ${ANDROID_NDK_HOME} \ + --archs ${TARGET_ARCHS} diff --git a/swift-ci/sdks/android/resources/patches b/swift-ci/sdks/android/resources/patches new file mode 160000 index 00000000..d0828093 --- /dev/null +++ b/swift-ci/sdks/android/resources/patches @@ -0,0 +1 @@ +Subproject commit d08280938ff70b790bbd0f559c0b420b518877ca diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 879aa1e4..20877272 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -109,7 +109,6 @@ declare_package libxml2 "libxml2" "MIT" \ declare_package curl "curl" "MIT" "https://curl.se" declare_package boringssl "boringssl" "OpenSSL AND ISC AND MIT" \ "https://boringssl.googlesource.com/boringssl/" -declare_package zlib "zlib" "Zlib" "https://zlib.net" # Parse command line arguments android_sdk_version=0.1 @@ -166,12 +165,14 @@ if ! swiftc=$(which swiftc); then exit 1 fi -script_dir=$(dirname -- "${BASH_SOURCE[0]}") -resource_dir="${script_dir}/../resources" - # Find the version numbers of the various dependencies function describe { pushd $1 >/dev/null 2>&1 + # this is needed for docker containers or else we get the error: + # fatal: detected dubious ownership in repository at '/source/curl' + if [[ "${SWIFT_BUILD_DOCKER}" == "1" ]]; then + git config --global --add safe.directory $(pwd) + fi git describe --tags popd >/dev/null 2>&1 } @@ -200,8 +201,6 @@ curl_version=${curl_desc#curl-} boringssl_version=$(describe ${source_dir}/boringssl) -zlib_version=$(versionFromTag ${source_dir}/zlib) - function quiet_pushd { pushd "$1" >/dev/null 2>&1 } @@ -214,8 +213,24 @@ header "Swift Android SDK build script" swift_dir=$(realpath $(dirname "$swiftc")/..) HOST=linux-x86_64 #HOST=$(uname -s -m | tr '[:upper:]' '[:lower:]' | tr ' ' '-') + +# in a Docker container, the pre-installed NDK is read-only, +# but the build script needs to write to it to work around +# https://github.com/swiftlang/swift-driver/pull/1822 +# so we copy it to a read-write location for the purposes of the build +# this can all be removed once that PR lands +mkdir -p ${build_dir}/ndk/ +ndk_home_tmp=${build_dir}/ndk/$(basename $ndk_home) +cp -a $ndk_home $ndk_home_tmp +ndk_home=$ndk_home_tmp + ndk_installation=$ndk_home/toolchains/llvm/prebuilt/$HOST +# ANDROID_NDK env needed by the swift-android.patch for: +# call ln -sf "${SWIFT_BUILD_PATH}/lib/swift" "${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib" +export ANDROID_NDK_HOME=$ndk_home +export ANDROID_NDK=$ndk_home + echo "Swift found at ${swift_dir}" echo "Host toolchain found at ${host_toolchain}" ${host_toolchain}/bin/swift --version @@ -231,7 +246,12 @@ echo " - Swift ${swift_version}" echo " - libxml2 ${libxml2_version}" echo " - curl ${curl_version}" echo " - BoringSSL ${boringssl_version}" -echo " - zlib ${zlib_version}" + +# make sure the products_dir is writeable +ls -lad $products_dir +touch $products_dir/products_dir_write_test.tmp +rm $products_dir/products_dir_write_test.tmp +#chown -R $(id -u):$(id -g) $products_dir function run() { echo "$@" @@ -239,11 +259,6 @@ function run() { } for arch in $archs; do - # enable short-circuiting the individual builds - if [[ ! -z "$SWIFT_ANDROID_ARCHIVEONLY" ]]; then - continue - fi - case $arch in armv7) target_host="arm-linux-androideabi"; compiler_target_host="armv7a-linux-androideabi$android_api"; android_abi="armeabi-v7a" ;; aarch64) target_host="aarch64-linux-android"; compiler_target_host="$target_host$android_api"; android_abi="arm64-v8a" ;; @@ -326,8 +341,9 @@ for arch in $archs; do -DOPENSSL_INCLUDE_DIR=$sdk_root/usr/include \ -DOPENSSL_SSL_LIBRARY=$sdk_root/usr/lib/libssl.a \ -DOPENSSL_CRYPTO_LIBRARY=$sdk_root/usr/lib/libcrypto.a \ - -DCURL_USE_OPENSSL=ON \ -DCURLSSLOPT_NATIVE_CA=ON \ + -DCURL_USE_OPENSSL=ON \ + -DCURL_USE_LIBSSH2=OFF \ -DTHREADS_PREFER_PTHREAD_FLAG=OFF \ -DCMAKE_THREAD_PREFER_PTHREAD=OFF \ -DCMAKE_THREADS_PREFER_PTHREAD_FLAG=OFF \ @@ -477,16 +493,17 @@ for arch in $archs; do rsync -a ${sdk_staging}/${arch}/usr ${swift_res_root} done +rm -r ${swift_res_root}/usr/share/{doc,man} +rm -r ${sdk_staging} + +# create an install script to set up the NDK links +#ANDROID_NDK_HOME="/opt/homebrew/share/android-ndk" +mkdir scripts/ + ndk_sysroot="ndk-sysroot" -# whether to include the ndk-sysroot in the SDK bundle -INCLUDE_NDK_SYSROOT=${INCLUDE_NDK_SYSROOT:-0} -if [[ ${INCLUDE_NDK_SYSROOT} != 1 ]]; then - # if we do not include the NDK, then create an install script - #ANDROID_NDK_HOME="/opt/homebrew/share/android-ndk" - mkdir scripts/ - cat > scripts/setup-android-sdk.sh <<'EOF' -#/bin/sh +cat > scripts/setup-android-sdk.sh <<'EOF' +#/bin/bash # this script will setup the ndk-sysroot with links to the # local installation indicated by ANDROID_NDK_HOME set -e @@ -494,72 +511,64 @@ if [ -z "${ANDROID_NDK_HOME}" ]; then echo "$(basename $0): error: missing environment variable ANDROID_NDK_HOME" exit 1 fi -PREBUILT="${ANDROID_NDK_HOME}/toolchains/llvm/prebuilt" -if [ ! -d "${PREBUILT}" ]; then - echo "$(basename $0): error: ANDROID_NDK_HOME not found: ${PREBUILT}" + +ndk_prebuilt="${ANDROID_NDK_HOME}/toolchains/llvm/prebuilt" +if [ ! -d "${ndk_prebuilt}" ]; then + echo "$(basename $0): error: ANDROID_NDK_HOME not found: ${ndk_prebuilt}" exit 1 fi -DESTINATION=$(dirname $(dirname $(realpath $0)))/ndk-sysroot -# clear out any previous NDK setup -rm -rf ${DESTINATION} -# copy vs. link the NDK files +#Pkg.Revision = 27.0.12077973 +#Pkg.Revision = 28.1.13356709 +ndk_version=$(grep '^Pkg.Revision = ' "${ANDROID_NDK_HOME}/source.properties" | cut -f3- -d' ' | cut -f 1 -d '.') +if [[ "${ndk_version}" -lt 27 ]]; then + echo "$(basename $0): error: minimum NDK version 27 required; found ${ndk_version} in ${ANDROID_NDK_HOME}/source.properties" + exit 1 +fi + +cd $(dirname $(dirname $(realpath -- "${BASH_SOURCE[0]}"))) +swift_resources=swift-resources +ndk_sysroot=ndk-sysroot + +if [[ -d "${ndk_sysroot}" ]]; then + # clear out any previous NDK setup + rm -rf ${ndk_sysroot} + ndk_re="re-" +fi + +# link vs. copy the NDK files SWIFT_ANDROID_NDK_LINK=${SWIFT_ANDROID_NDK_LINK:-1} -if [[ "${SWIFT_ANDROID_NDK_LINK}" != 1 ]]; then - ANDROID_NDK_DESC="copied" - cp -a ${PREBUILT}/*/sysroot ${DESTINATION} -else - ANDROID_NDK_DESC="linked" - mkdir -p ${DESTINATION}/usr/lib - ln -s $(realpath ${PREBUILT}/*/sysroot/usr/include) ${DESTINATION}/usr/include - for triplePath in ${PREBUILT}/*/sysroot/usr/lib/*; do +if [[ "${SWIFT_ANDROID_NDK_LINK}" == 1 ]]; then + ndk_action="${ndk_re}linked" + mkdir -p ${ndk_sysroot}/usr/lib + ln -s ${ndk_prebuilt}/*/sysroot/usr/include ${ndk_sysroot}/usr/include + for triplePath in ${ndk_prebuilt}/*/sysroot/usr/lib/*; do triple=$(basename ${triplePath}) - ln -s $(realpath ${triplePath}) ${DESTINATION}/usr/lib/${triple} + ln -s ${triplePath} ${ndk_sysroot}/usr/lib/${triple} done +else + ndk_action="${ndk_re}copied" + cp -a ${ndk_prebuilt}/*/sysroot ${ndk_sysroot} fi # copy each architecture's swiftrt.o into the sysroot, # working around https://github.com/swiftlang/swift/pull/79621 -for swiftrt in ${DESTINATION}/../swift-resources/usr/lib/swift-*/android/*/swiftrt.o; do - arch=$(basename $(dirname ${swiftrt})) - mkdir -p ${DESTINATION}/usr/lib/swift/android/${arch} - cp -a ${swiftrt} ${DESTINATION}/usr/lib/swift/android/${arch} +for folder in swift swift_static; do + for swiftrt in ${swift_resources}/usr/lib/${folder}-*/android/*/swiftrt.o; do + arch=$(basename $(dirname ${swiftrt})) + mkdir -p ${ndk_sysroot}/usr/lib/${folder}/android/${arch} + if [[ "${SWIFT_ANDROID_NDK_LINK}" == 1 ]]; then + ln -s ../../../../../../${swiftrt} ${ndk_sysroot}/usr/lib/${folder}/android/${arch}/ + else + cp -a ${swiftrt} ${ndk_sysroot}/usr/lib/${folder}/android/${arch}/ + fi + done done -echo "$(basename $0): success: ndk-sysroot ${ANDROID_NDK_DESC} to Android SDK" +echo "$(basename $0): success: ndk-sysroot ${ndk_action} to Android NDK at ${ndk_prebuilt}" EOF - chmod +x scripts/setup-android-sdk.sh -else - COPY_NDK_SYSROOT=${COPY_NDK_SYSROOT:-1} - if [[ ${COPY_NDK_SYSROOT} == 1 ]]; then - cp -a ${ndk_installation}/sysroot ${ndk_sysroot} - else - # rather than copying the sysroot, we can instead make links to - # the various sub-folders this won't work for the distribution, - # since the NDK is going to be located in different places - # for different machines - mkdir -p ${ndk_sysroot}/usr/lib - ln -sv $(realpath ${ndk_installation}/sysroot/usr/include) ${ndk_sysroot}/usr/include - for triplePath in ${ndk_installation}/sysroot/usr/lib/*; do - triple=$(basename ${triplePath}) - ln -sv $(realpath ${triplePath}) ${ndk_sysroot}/usr/lib/${triple} - ls ${ndk_sysroot}/usr/lib/${triple}/ - done - fi - # need to manually copy over swiftrt.o or else: - # error: link command failed with exit code 1 (use -v to see invocation) - # clang: error: no such file or directory: '${HOME}/.swiftpm/swift-sdks/swift-6.2-DEVELOPMENT-SNAPSHOT-2025-04-24-a-android-0.1.artifactbundle/swift-android/ndk-sysroot/usr/lib/swift/android/x86_64/swiftrt.o' - # see: https://github.com/swiftlang/swift-driver/pull/1822#issuecomment-2762811807 - # should be fixed by: https://github.com/swiftlang/swift/pull/79621 - for arch in $archs; do - mkdir -p ${ndk_sysroot}/usr/lib/swift/android/${arch} - ln -srv ${swift_res_root}/usr/lib/swift-${arch}/android/${arch}/swiftrt.o ${ndk_sysroot}/usr/lib/swift/android/${arch}/swiftrt.o - done -fi - -rm -r ${swift_res_root}/usr/share/{doc,man} -rm -r ${sdk_staging} +chmod +x scripts/setup-android-sdk.sh cat > swift-sdk.json </dev/null + +if [[ "${ANDROID_NDK_VERSION}" == "" ]]; then + echo "$0: Missing ANDROID_NDK_VERSION environment" + exit 1 +fi + + +NDKFILE=${ANDROID_NDK_VERSION}-linux.zip + +NDKURL="https://dl.google.com/android/repository/${NDKFILE}" +echo "Going to fetch ${NDKURL}" + +curl -fsSL "${NDKURL}" -o ${NDKFILE} + +echo "Extracting NDK" +unzip -q ${NDKFILE} + +rm ${NDKFILE} + +popd >/dev/null + diff --git a/swift-ci/sdks/android/scripts/install-swift.sh b/swift-ci/sdks/android/scripts/install-swift.sh new file mode 100755 index 00000000..0acd9e02 --- /dev/null +++ b/swift-ci/sdks/android/scripts/install-swift.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# +# ===----------------------------------------------------------------------=== +# +# Swift Android SDK: Install Swift +# +# ===----------------------------------------------------------------------=== + +set -e + +if [[ "${SWIFT_TOOLCHAIN_URL}" == "" ]]; then + echo "$0: Missing SWIFT_TOOLCHAIN_URL environment" + exit 1 +fi + +destination=$1 +if [[ "${destination}" == "" ]]; then + echo "$0: Usage: $(basename $0) " + exit 1 +fi + +echo "Installing Swift from: ${SWIFT_TOOLCHAIN_URL} into: ${destination}" + +# Make a temporary directory +tmpdir=$(mktemp -d) +function cleanup { + rm -rf "$tmpdir" +} +trap cleanup EXIT + +pushd "$tmpdir" >/dev/null +export GNUPGHOME="$tmpdir" + +# Fetch the toolchain and signature +echo "Going to fetch ${SWIFT_TOOLCHAIN_URL}" +curl -fsSL "${SWIFT_TOOLCHAIN_URL}" -o toolchain.tar.gz + +echo "Going to fetch ${SWIFT_TOOLCHAIN_URL}.sig" +curl -fsSL "${SWIFT_TOOLCHAIN_URL}.sig" -o toolchain.sig + +echo "Fetching keys" +curl -fsSL --compressed https://swift.org/keys/all-keys.asc | gpg --import - + +echo "Verifying signature" +gpg --batch --verify toolchain.sig toolchain.tar.gz + +# Extract and install the toolchain +echo "Extracting Swift" +mkdir -p ${destination} +tar -xzf toolchain.tar.gz --directory ${destination} --strip-components=2 +chmod -R o+r ${destination}/lib/swift + +popd >/dev/null + diff --git a/swift-ci/sdks/android/scripts/patch-sources.sh b/swift-ci/sdks/android/scripts/patch-sources.sh new file mode 100755 index 00000000..ee425c6a --- /dev/null +++ b/swift-ci/sdks/android/scripts/patch-sources.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# Swift Android SDK: Patch Sources +set -e + +source_dir=$1 + +if [[ ! -d "${source_dir}" ]]; then + echo "$0: source_dir ${source_dir} does not exist" + exit 1 +fi + +script_dir=$(dirname -- "${BASH_SOURCE[0]}") +resource_dir=$(realpath "${script_dir}/../resources") +patches_dir="${resource_dir}/patches" + +if [[ ! -d "${patches_dir}" ]]; then + echo "$0: patches_dir ${patches_dir} does not exist" + exit 1 +fi + +cd ${source_dir}/swift-project +swift_android_patch="${patches_dir}/swift-android.patch" + +ls -la swift/utils/build-script-impl +ls -la ${swift_android_patch} +ls -la swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift + +# patch the patch, which seems to only be needed for an API less than 28 +# https://github.com/finagolfin/swift-android-sdk/blob/main/swift-android.patch#L110 +perl -pi -e 's/#if os\(Windows\)/#if os\(Android\)/g' $swift_android_patch + +# remove the need to link in android-execinfo +perl -pi -e 's;dispatch android-execinfo;dispatch;g' $swift_android_patch + +# debug symbolic link setup +perl -pi -e 's;call ln -sf;call ln -svf;g' $swift_android_patch +perl -pi -e 's%linux-x86_64/sysroot/usr/lib"%linux-x86_64/sysroot/usr/lib"; echo "VALIDATING SYMBOLIC LINK"; ls -la "\${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib"; ls -la "\${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/swift"; %g' $swift_android_patch + +case "${BUILD_VERSION}" in + release) + testing_patch="${patches_dir}/swift-android-testing-release.patch" + ;; + devel) + testing_patch="${patches_dir}/swift-android-testing-except-release.patch" + ;; + trunk) + testing_patch="${patches_dir}/swift-android-testing-except-release.patch" + dispatch_patch="${patches_dir}/swift-android-trunk-libdispatch.patch" + ;; + *) + echo "$0: invalid BUILD_VERSION=${BUILD_VERSION}" + exit 1 + ;; +esac + +for patch in "$swift_android_patch" "$testing_patch" "$dispatch_patch"; do + if [[ "${patch}" == "" ]]; then + continue + fi + + echo "applying patch $patch in $PWD…" + # first check to make sure the patches can apply and fail if not + git apply -v --check -C1 "$patch" + git apply --no-index -v -C1 "$patch" + + #if git apply -C1 --reverse --check "$patch" >/dev/null 2>&1 ; then + # echo "already patched" + #elif git apply -C1 "$patch" ; then + # echo "done" + #else + # echo "failed to apply patch $patch in $PWD" + # exit 1 + #fi +done + +perl -pi -e 's%String\(cString: getpass%\"fake\" //%' swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift +# disable backtrace() for Android (needs either API33+ or libandroid-execinfo, or to manually add in backtrace backport) +perl -pi -e 's;os\(Android\);os\(AndroidDISABLED\);g' swift-testing/Sources/Testing/SourceAttribution/Backtrace.swift + +# need to un-apply libandroid-spawn since we don't need it for API28+ +perl -pi -e 's;MATCHES "Android";MATCHES "AndroidDISABLED";g' llbuild/lib/llvm/Support/CMakeLists.txt +perl -pi -e 's; STREQUAL Android\); STREQUAL AndroidDISABLED\);g' swift-corelibs-foundation/Sources/Foundation/CMakeLists.txt + +# validate the patches +ls -la swift/utils/build-script-impl +grep 'VALIDATING SYMBOLIC LINK' swift/utils/build-script-impl + diff --git a/swift-ci/sdks/android/scripts/toolchain-vars.sh b/swift-ci/sdks/android/scripts/toolchain-vars.sh new file mode 100644 index 00000000..c1001931 --- /dev/null +++ b/swift-ci/sdks/android/scripts/toolchain-vars.sh @@ -0,0 +1,44 @@ +#!/bin/bash -e +# +# ===----------------------------------------------------------------------=== +# +# Swift Android SDK: Toolchain source variables +# +# ===----------------------------------------------------------------------=== + +# This script is meant to be sourced from another script that sets the +# BUILD_VERSION environment variable to one of "release", "devel", or "trunk" +# and will set check the latest builds for each build type in order +# to provide information about the Swift tag name in use and where to +# obtain the latest toolchain for building. + +# TODO: we could instead use the latest-build.yml files for this, like: +# https://download.swift.org/swift-6.2-branch/ubuntu2404/latest-build.yml +# https://download.swift.org/development/ubuntu2404/latest-build.yml +# but there doesn't seem to be one for the current release build. + +case "${BUILD_VERSION}" in + release) + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/swiftlang/swift/releases | grep -m1 swift-6.1 | cut -d- -f2) + SWIFT_TAG="swift-${LATEST_TOOLCHAIN_VERSION}-RELEASE" + SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-release" + ;; + devel) + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/swiftlang/swift/tags | grep -m1 swift-6.2-DEV | cut -d- -f8-10) + SWIFT_TAG="swift-6.2-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" + SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-branch" + ;; + trunk) + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/swiftlang/swift/tags | grep -m1 swift-DEV | cut -d- -f7-9) + SWIFT_TAG="swift-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" + SWIFT_BRANCH="development" + ;; + *) + echo "$0: invalid BUILD_VERSION=${BUILD_VERSION}" + exit 1 + ;; +esac + +SWIFT_BASE=$SWIFT_TAG-$HOST_OS +export SWIFT_TOOLCHAIN_URL="https://download.swift.org/$SWIFT_BRANCH/$(echo $HOST_OS | tr -d '.')/$SWIFT_TAG/$SWIFT_BASE.tar.gz" + From 4987bcc7e57590e81bd5594434310d379fb2510f Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sun, 18 May 2025 18:19:00 -0400 Subject: [PATCH 06/44] Checkout patches repo instead of using a git submodule --- .github/workflows/pull_request.yml | 2 -- .gitmodules | 3 --- swift-ci/sdks/android/resources/patches | 1 - swift-ci/sdks/android/scripts/fetch-source.sh | 12 +++++++++++- swift-ci/sdks/android/scripts/patch-sources.sh | 10 +--------- 5 files changed, 12 insertions(+), 16 deletions(-) delete mode 160000 swift-ci/sdks/android/resources/patches diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 13c57b39..d5610edb 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -67,8 +67,6 @@ jobs: echo "WORKDIR=${{ runner.temp }}/swift-android-sdk" >> $GITHUB_ENV - name: Checkout repository uses: actions/checkout@v4 - with: - submodules: 'true' - name: Build Android SDK (Local) if: ${{ matrix.build-type == 'local' }} working-directory: swift-ci/sdks/android diff --git a/.gitmodules b/.gitmodules index f3724176..e69de29b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "swift-ci/sdks/android/resources/patches"] - path = swift-ci/sdks/android/resources/patches - url = https://github.com/swift-android-sdk/swift-android-sdk diff --git a/swift-ci/sdks/android/resources/patches b/swift-ci/sdks/android/resources/patches deleted file mode 160000 index d0828093..00000000 --- a/swift-ci/sdks/android/resources/patches +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d08280938ff70b790bbd0f559c0b420b518877ca diff --git a/swift-ci/sdks/android/scripts/fetch-source.sh b/swift-ci/sdks/android/scripts/fetch-source.sh index 26c88d7a..3b7b7b36 100755 --- a/swift-ci/sdks/android/scripts/fetch-source.sh +++ b/swift-ci/sdks/android/scripts/fetch-source.sh @@ -91,6 +91,9 @@ fi if [[ -z "${BORINGSSL_VERSION}" ]]; then BORINGSSL_VERSION=fips-20220613 fi +if [[ -z "${SWIFT_ANDROID_PATCHES_VERSION}" ]]; then + SWIFT_ANDROID_PATCHES_VERSION=main +fi clone_with_ssh=false while [ "$#" -gt 0 ]; do @@ -139,7 +142,7 @@ mkdir -p swift-project groupstart "Fetching Swift" pushd swift-project >/dev/null -[[ -d swift ]] || git clone ${github}apple/swift.git +[[ -d swift ]] || git clone ${github}swiftlang/swift.git cd swift # Get its dependencies @@ -181,3 +184,10 @@ git checkout ${BORINGSSL_VERSION} popd >/dev/null 2>&1 groupend +# Fetch BoringSSL +groupstart "Fetching Patches" +[[ -d swift-android-patches ]] || git clone https://github.com/swift-android-sdk/swift-android-sdk swift-android-patches +pushd swift-android-patches >/dev/null 2>&1 +git checkout ${SWIFT_ANDROID_PATCHES_VERSION} +popd >/dev/null 2>&1 +groupend diff --git a/swift-ci/sdks/android/scripts/patch-sources.sh b/swift-ci/sdks/android/scripts/patch-sources.sh index ee425c6a..1282c706 100755 --- a/swift-ci/sdks/android/scripts/patch-sources.sh +++ b/swift-ci/sdks/android/scripts/patch-sources.sh @@ -3,16 +3,12 @@ set -e source_dir=$1 - if [[ ! -d "${source_dir}" ]]; then echo "$0: source_dir ${source_dir} does not exist" exit 1 fi -script_dir=$(dirname -- "${BASH_SOURCE[0]}") -resource_dir=$(realpath "${script_dir}/../resources") -patches_dir="${resource_dir}/patches" - +patches_dir="${source_dir}/swift-android-patches" if [[ ! -d "${patches_dir}" ]]; then echo "$0: patches_dir ${patches_dir} does not exist" exit 1 @@ -21,10 +17,6 @@ fi cd ${source_dir}/swift-project swift_android_patch="${patches_dir}/swift-android.patch" -ls -la swift/utils/build-script-impl -ls -la ${swift_android_patch} -ls -la swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift - # patch the patch, which seems to only be needed for an API less than 28 # https://github.com/finagolfin/swift-android-sdk/blob/main/swift-android.patch#L110 perl -pi -e 's/#if os\(Windows\)/#if os\(Android\)/g' $swift_android_patch From ff3f274cca53d02f5b3f331ba64dde177fb27333 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sun, 18 May 2025 18:28:34 -0400 Subject: [PATCH 07/44] Update libcurl to 8.13.0 --- swift-ci/sdks/android/scripts/fetch-source.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swift-ci/sdks/android/scripts/fetch-source.sh b/swift-ci/sdks/android/scripts/fetch-source.sh index 3b7b7b36..93309130 100755 --- a/swift-ci/sdks/android/scripts/fetch-source.sh +++ b/swift-ci/sdks/android/scripts/fetch-source.sh @@ -85,8 +85,8 @@ if [[ -z "${LIBXML2_VERSION}" ]]; then #LIBXML2_VERSION=2.14.2 fi if [[ -z "${CURL_VERSION}" ]]; then - CURL_VERSION=8.7.1 - #CURL_VERSION=8.13.0 + #CURL_VERSION=8.7.1 + CURL_VERSION=8.13.0 fi if [[ -z "${BORINGSSL_VERSION}" ]]; then BORINGSSL_VERSION=fips-20220613 From 595efc35bdd9e2c96add429309be5397ca615b97 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sun, 18 May 2025 18:33:49 -0400 Subject: [PATCH 08/44] Remove resources that we no longer use --- swift-ci/sdks/android/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/swift-ci/sdks/android/Dockerfile b/swift-ci/sdks/android/Dockerfile index 2be3fc37..69522bb2 100644 --- a/swift-ci/sdks/android/Dockerfile +++ b/swift-ci/sdks/android/Dockerfile @@ -78,8 +78,6 @@ ENV SWIFT_BUILD_DOCKER="1" COPY scripts /scripts RUN chmod ugo+x /scripts/* -COPY resources /resources - # Create a user RUN groupadd -g 998 build-user && \ useradd -m -r -u 998 -g build-user build-user From 27b1bf444d1f07abfcf4b61aae58d7d259e35d36 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sun, 18 May 2025 18:53:47 -0400 Subject: [PATCH 09/44] Update libcurl to 8.13.0 --- swift-ci/sdks/android/scripts/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 20877272..75085411 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -344,6 +344,7 @@ for arch in $archs; do -DCURLSSLOPT_NATIVE_CA=ON \ -DCURL_USE_OPENSSL=ON \ -DCURL_USE_LIBSSH2=OFF \ + -DCURL_USE_LIBPSL=OFF \ -DTHREADS_PREFER_PTHREAD_FLAG=OFF \ -DCMAKE_THREAD_PREFER_PTHREAD=OFF \ -DCMAKE_THREADS_PREFER_PTHREAD_FLAG=OFF \ From b898129337799dd43c98859a9a540e790a0a23f1 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sun, 18 May 2025 18:54:26 -0400 Subject: [PATCH 10/44] Update libxml2 to 2.14.2 --- swift-ci/sdks/android/scripts/fetch-source.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/swift-ci/sdks/android/scripts/fetch-source.sh b/swift-ci/sdks/android/scripts/fetch-source.sh index 93309130..b42abd07 100755 --- a/swift-ci/sdks/android/scripts/fetch-source.sh +++ b/swift-ci/sdks/android/scripts/fetch-source.sh @@ -81,11 +81,9 @@ if [[ -z "${SWIFT_VERSION}" ]]; then SWIFT_VERSION=scheme:release/6.1 fi if [[ -z "${LIBXML2_VERSION}" ]]; then - LIBXML2_VERSION=2.12.7 - #LIBXML2_VERSION=2.14.2 + LIBXML2_VERSION=2.14.2 fi if [[ -z "${CURL_VERSION}" ]]; then - #CURL_VERSION=8.7.1 CURL_VERSION=8.13.0 fi if [[ -z "${BORINGSSL_VERSION}" ]]; then From f566b239b8e608d77ef5938a59259c3b639764d6 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sun, 18 May 2025 19:36:29 -0400 Subject: [PATCH 11/44] Build libxml2, libcurl, and boringssl with support for Android 16kb page sizes --- swift-ci/sdks/android/Dockerfile | 1 - swift-ci/sdks/android/scripts/build.sh | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/swift-ci/sdks/android/Dockerfile b/swift-ci/sdks/android/Dockerfile index 69522bb2..c1d8c6db 100644 --- a/swift-ci/sdks/android/Dockerfile +++ b/swift-ci/sdks/android/Dockerfile @@ -24,7 +24,6 @@ RUN apt-get -q update \ golang \ git \ gnupg2 \ - libsqlite3-dev \ libcurl4-openssl-dev \ libedit-dev \ libicu-dev \ diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 75085411..ad59c311 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -283,6 +283,7 @@ for arch in $archs; do -DCMAKE_EXTRA_LINK_FLAGS="-rtlib=compiler-rt -unwindlib=libunwind -stdlib=libc++ -fuse-ld=lld -lc++ -lc++abi" \ -DCMAKE_BUILD_TYPE=$build_type \ -DCMAKE_INSTALL_PREFIX=$sdk_root/usr \ + -DCMAKE_SHARED_LINKER_FLAGS="-Wl,-z,max-page-size=16384" \ -DLIBXML2_WITH_PYTHON=NO \ -DLIBXML2_WITH_ICU=NO \ -DLIBXML2_WITH_ICONV=NO \ @@ -311,6 +312,7 @@ for arch in $archs; do -DCMAKE_TOOLCHAIN_FILE=$ndk_home/build/cmake/android.toolchain.cmake \ -DCMAKE_BUILD_TYPE=$build_type \ -DCMAKE_INSTALL_PREFIX=$sdk_root/usr \ + -DCMAKE_SHARED_LINKER_FLAGS="-Wl,-z,max-page-size=16384" \ -DBUILD_SHARED_LIBS=OFF \ -DBUILD_STATIC_LIBS=ON \ -DBUILD_TESTING=OFF @@ -337,6 +339,7 @@ for arch in $archs; do -DCMAKE_TOOLCHAIN_FILE=$ndk_home/build/cmake/android.toolchain.cmake \ -DCMAKE_BUILD_TYPE=$build_type \ -DCMAKE_INSTALL_PREFIX=$sdk_root/usr \ + -DCMAKE_SHARED_LINKER_FLAGS="-Wl,-z,max-page-size=16384" \ -DOPENSSL_ROOT_DIR=$sdk_root/usr \ -DOPENSSL_INCLUDE_DIR=$sdk_root/usr/include \ -DOPENSSL_SSL_LIBRARY=$sdk_root/usr/lib/libssl.a \ From 529e3f19d1f927b2671c4cc0717ec342f7525d7b Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sun, 18 May 2025 19:50:32 -0400 Subject: [PATCH 12/44] Add build-script --extra-cmake-options=-DCMAKE_EXTRA_LINK_FLAGS=-Wl,-z,max-page-size=16384 --- swift-ci/sdks/android/scripts/build.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index ad59c311..1d9504e0 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -280,10 +280,9 @@ for arch in $archs; do -DANDROID_PLATFORM=android-$android_api \ -DCMAKE_TOOLCHAIN_FILE=$ndk_home/build/cmake/android.toolchain.cmake \ -DCMAKE_BUILD_TYPE=$build_type \ - -DCMAKE_EXTRA_LINK_FLAGS="-rtlib=compiler-rt -unwindlib=libunwind -stdlib=libc++ -fuse-ld=lld -lc++ -lc++abi" \ + -DCMAKE_EXTRA_LINK_FLAGS="-rtlib=compiler-rt -unwindlib=libunwind -stdlib=libc++ -fuse-ld=lld -lc++ -lc++abi -Wl,-z,max-page-size=16384" \ -DCMAKE_BUILD_TYPE=$build_type \ -DCMAKE_INSTALL_PREFIX=$sdk_root/usr \ - -DCMAKE_SHARED_LINKER_FLAGS="-Wl,-z,max-page-size=16384" \ -DLIBXML2_WITH_PYTHON=NO \ -DLIBXML2_WITH_ICU=NO \ -DLIBXML2_WITH_ICONV=NO \ @@ -312,7 +311,7 @@ for arch in $archs; do -DCMAKE_TOOLCHAIN_FILE=$ndk_home/build/cmake/android.toolchain.cmake \ -DCMAKE_BUILD_TYPE=$build_type \ -DCMAKE_INSTALL_PREFIX=$sdk_root/usr \ - -DCMAKE_SHARED_LINKER_FLAGS="-Wl,-z,max-page-size=16384" \ + -DCMAKE_EXTRA_LINK_FLAGS="-Wl,-z,max-page-size=16384" \ -DBUILD_SHARED_LIBS=OFF \ -DBUILD_STATIC_LIBS=ON \ -DBUILD_TESTING=OFF @@ -339,7 +338,7 @@ for arch in $archs; do -DCMAKE_TOOLCHAIN_FILE=$ndk_home/build/cmake/android.toolchain.cmake \ -DCMAKE_BUILD_TYPE=$build_type \ -DCMAKE_INSTALL_PREFIX=$sdk_root/usr \ - -DCMAKE_SHARED_LINKER_FLAGS="-Wl,-z,max-page-size=16384" \ + -DCMAKE_EXTRA_LINK_FLAGS="-Wl,-z,max-page-size=16384" \ -DOPENSSL_ROOT_DIR=$sdk_root/usr \ -DOPENSSL_INCLUDE_DIR=$sdk_root/usr/include \ -DOPENSSL_SSL_LIBRARY=$sdk_root/usr/lib/libssl.a \ @@ -403,7 +402,8 @@ for arch in $archs; do --install-foundation \ --xctest --install-xctest \ --swift-testing --install-swift-testing \ - --cross-compile-append-host-target-to-destdir=False + --cross-compile-append-host-target-to-destdir=False \ + --extra-cmake-options='-DCMAKE_EXTRA_LINK_FLAGS="-Wl,-z,max-page-size=16384"' # need to remove symlink that gets created in the NDK to the previous arch's build # or else we get errors like: From c82587dbb084b774ade6564ecb0f233e70ae9981 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sun, 18 May 2025 19:56:15 -0400 Subject: [PATCH 13/44] Add 16KB page size linker flags to linker flags in swift-toolset.json --- swift-ci/sdks/android/scripts/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 1d9504e0..02931862 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -620,6 +620,7 @@ cat > swift-toolset.json < Date: Sun, 18 May 2025 22:01:29 -0400 Subject: [PATCH 14/44] Add 16KB page size linker flags to linker flags in swift-toolset.json --- swift-ci/sdks/android/scripts/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 02931862..b82c36ad 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -620,7 +620,7 @@ cat > swift-toolset.json < Date: Mon, 19 May 2025 08:32:20 -0400 Subject: [PATCH 15/44] Build with ndk-r28b --- swift-ci/sdks/android/build-docker | 2 +- swift-ci/sdks/android/build-local | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index fb3b377a..4dfa3c96 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -9,7 +9,7 @@ # default architectures to build for TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} -ANDROID_NDK_VERSION=android-ndk-r27c +ANDROID_NDK_VERSION=android-ndk-r28b ANDROID_API=28 export BUILD_VERSION=${1} diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index 30bd4f92..97b7aa11 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -9,7 +9,7 @@ # default architectures to build for TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} -ANDROID_NDK_VERSION=android-ndk-r27c +ANDROID_NDK_VERSION=android-ndk-r28b ANDROID_API=28 export BUILD_VERSION=${1} From 8b18b5fafa51e9aa6fc4e2204c86193959dac8a6 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Mon, 19 May 2025 09:05:28 -0400 Subject: [PATCH 16/44] Revert to building with ndk-r27c --- swift-ci/sdks/android/build-docker | 2 +- swift-ci/sdks/android/build-local | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index 4dfa3c96..fb3b377a 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -9,7 +9,7 @@ # default architectures to build for TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} -ANDROID_NDK_VERSION=android-ndk-r28b +ANDROID_NDK_VERSION=android-ndk-r27c ANDROID_API=28 export BUILD_VERSION=${1} diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index 97b7aa11..30bd4f92 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -9,7 +9,7 @@ # default architectures to build for TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} -ANDROID_NDK_VERSION=android-ndk-r28b +ANDROID_NDK_VERSION=android-ndk-r27c ANDROID_API=28 export BUILD_VERSION=${1} From f67f9bf939247eba99dbf0a7b93a64a5a922f3aa Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Mon, 19 May 2025 14:59:18 -0400 Subject: [PATCH 17/44] Use official endpoints for discovering latest Swift release/devel/trunk tags --- .../sdks/android/scripts/toolchain-vars.sh | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/swift-ci/sdks/android/scripts/toolchain-vars.sh b/swift-ci/sdks/android/scripts/toolchain-vars.sh index c1001931..162689a9 100644 --- a/swift-ci/sdks/android/scripts/toolchain-vars.sh +++ b/swift-ci/sdks/android/scripts/toolchain-vars.sh @@ -12,25 +12,23 @@ # to provide information about the Swift tag name in use and where to # obtain the latest toolchain for building. -# TODO: we could instead use the latest-build.yml files for this, like: -# https://download.swift.org/swift-6.2-branch/ubuntu2404/latest-build.yml -# https://download.swift.org/development/ubuntu2404/latest-build.yml -# but there doesn't seem to be one for the current release build. +OS=$(echo $HOST_OS | tr -d '.') case "${BUILD_VERSION}" in release) - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/swiftlang/swift/releases | grep -m1 swift-6.1 | cut -d- -f2) - SWIFT_TAG="swift-${LATEST_TOOLCHAIN_VERSION}-RELEASE" - SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-release" + # e.g., "swift-6.1-RELEASE" + SWIFT_TAG=$(curl -fsSL https://www.swift.org/api/v1/install/releases.json | jq -r '.[-1].tag') + # e.g., "swift-6.1-release" + SWIFT_BRANCH=$(echo "${SWIFT_TAG}" | tr '[A-Z]' '[a-z]') ;; devel) - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/swiftlang/swift/tags | grep -m1 swift-6.2-DEV | cut -d- -f8-10) - SWIFT_TAG="swift-6.2-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" + # e.g., swift-6.2-DEVELOPMENT-SNAPSHOT-2025-05-15-a + SWIFT_TAG=$(curl -fsSL https://download.swift.org/swift-6.2-branch/$OS/latest-build.yml | grep '^dir: ' | cut -f 2 -d ' ' SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-branch" ;; trunk) - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/swiftlang/swift/tags | grep -m1 swift-DEV | cut -d- -f7-9) - SWIFT_TAG="swift-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" + # e.g., swift-DEVELOPMENT-SNAPSHOT-2025-05-14-a + SWIFT_TAG=$(curl -fsSL https://download.swift.org/development/$OS/latest-build.yml | grep '^dir: ' | cut -f 2 -d ' ') SWIFT_BRANCH="development" ;; *) @@ -40,5 +38,5 @@ case "${BUILD_VERSION}" in esac SWIFT_BASE=$SWIFT_TAG-$HOST_OS -export SWIFT_TOOLCHAIN_URL="https://download.swift.org/$SWIFT_BRANCH/$(echo $HOST_OS | tr -d '.')/$SWIFT_TAG/$SWIFT_BASE.tar.gz" +export SWIFT_TOOLCHAIN_URL="https://download.swift.org/$SWIFT_BRANCH/$OS/$SWIFT_TAG/$SWIFT_BASE.tar.gz" From 7011a45c5dee830a184226c71c22ccde402a0baf Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Mon, 19 May 2025 15:21:38 -0400 Subject: [PATCH 18/44] Typo fix in version script --- swift-ci/sdks/android/scripts/toolchain-vars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift-ci/sdks/android/scripts/toolchain-vars.sh b/swift-ci/sdks/android/scripts/toolchain-vars.sh index 162689a9..5f8c5e8a 100644 --- a/swift-ci/sdks/android/scripts/toolchain-vars.sh +++ b/swift-ci/sdks/android/scripts/toolchain-vars.sh @@ -23,7 +23,7 @@ case "${BUILD_VERSION}" in ;; devel) # e.g., swift-6.2-DEVELOPMENT-SNAPSHOT-2025-05-15-a - SWIFT_TAG=$(curl -fsSL https://download.swift.org/swift-6.2-branch/$OS/latest-build.yml | grep '^dir: ' | cut -f 2 -d ' ' + SWIFT_TAG=$(curl -fsSL https://download.swift.org/swift-6.2-branch/$OS/latest-build.yml | grep '^dir: ' | cut -f 2 -d ' ') SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-branch" ;; trunk) From 4c65a9385415d748f775bdd44751b585e9fb53cc Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Mon, 19 May 2025 18:08:57 -0400 Subject: [PATCH 19/44] Cleanup for PR --- .github/workflows/pull_request.yml | 10 ---------- .gitignore | 1 - .gitmodules | 0 swift-ci/sdks/android/README.md | 8 +++++--- 4 files changed, 5 insertions(+), 14 deletions(-) delete mode 100644 .gitmodules diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index d5610edb..95503da5 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -22,16 +22,6 @@ jobs: path: | *.log - static-linux-build: - name: Build Static Linux image - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Build Docker images - working-directory: swift-ci/sdks/static-linux - run: ./build - android-build: name: Build Android ${{ matrix.build-type }} ${{ matrix.swift-version }} ${{ matrix.arch }} SDK strategy: diff --git a/.gitignore b/.gitignore index 98bcc3db..a01ee289 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1 @@ .*.swp -static-swift-linux-sources diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index e69de29b..00000000 diff --git a/swift-ci/sdks/android/README.md b/swift-ci/sdks/android/README.md index 10d4db3e..c3c91010 100644 --- a/swift-ci/sdks/android/README.md +++ b/swift-ci/sdks/android/README.md @@ -39,18 +39,20 @@ fetch tagged sources for libxml2, curl, boringssl, and swift. It then applies some patches and invokes `scripts/build.sh`, which will build the sources for each of the specified -architectures. Finally, it combines the NDK and the newly built -SDKs into a single artifactbundle. +architectures and then combines the SDKs into a single +artifactbundle with targetTriples for each of the supported +architectures (`aarch64`, `x86_64`, `aarmv7`) +and Android API levels (28-35). ## Specifying Architectures By default all the supported Android architectures -(`aarch64`, `x86_64`, `aarmv7`) will be built, but this can be reduced in order to speed up the build. This can be useful, e.g., as part of a CI that validates a pull request, as building a single architecture takes around 30 minutes on a standard ubuntu-24.04 GitHub runner, whereas building for all the architectures takes over an hour. +Building within a docker container increases this by about 50%. To build an artifactbundle for just the `x86_64` architecture, run: From d80c80b9a2d8ff54ecd6d3dfe10286dbbd595730 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Mon, 19 May 2025 18:17:25 -0400 Subject: [PATCH 20/44] Change BUILD_VERSION to BUILD_SCHEME and have it match release, swift-*-branch, or development --- .github/workflows/pull_request.yml | 8 ++++---- swift-ci/sdks/android/build-docker | 2 +- swift-ci/sdks/android/build-local | 2 +- swift-ci/sdks/android/scripts/patch-sources.sh | 8 ++++---- swift-ci/sdks/android/scripts/toolchain-vars.sh | 14 +++++++------- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 95503da5..1a4b5833 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -35,7 +35,7 @@ jobs: #arch: ['x86_64'] # build both the quick (x86_64) and complete (aarch64,x86_64,armv7) SDKs #arch: ['x86_64', ''] - swift-version: ['release', 'devel', 'trunk'] + swift-version: ['release', 'swift-6.2-branch', 'development'] runs-on: ubuntu-24.04 steps: - name: Free Disk Space @@ -52,7 +52,7 @@ jobs: run: | # these variabes are used by build-docker and build-local # to determine which Swift version to build for - echo "BUILD_VERSION=${{ matrix.swift-version }}" >> $GITHUB_ENV + echo "BUILD_SCHEME=${{ matrix.swift-version }}" >> $GITHUB_ENV echo "TARGET_ARCHS=${{ matrix.arch }}" >> $GITHUB_ENV echo "WORKDIR=${{ runner.temp }}/swift-android-sdk" >> $GITHUB_ENV - name: Checkout repository @@ -62,12 +62,12 @@ jobs: working-directory: swift-ci/sdks/android run: | sudo apt install -q ninja-build patchelf - ./build-local ${BUILD_VERSION} ${WORKDIR} + ./build-local ${BUILD_SCHEME} ${WORKDIR} - name: Build Android SDK (Docker) if: ${{ matrix.build-type == 'docker' }} working-directory: swift-ci/sdks/android run: | - ./build-docker ${BUILD_VERSION} ${WORKDIR} + ./build-docker ${BUILD_SCHEME} ${WORKDIR} - name: Install Host Toolchain if: ${{ matrix.build-type == 'docker' }} working-directory: swift-ci/sdks/android diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index fb3b377a..ef283152 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -12,7 +12,7 @@ TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} ANDROID_NDK_VERSION=android-ndk-r27c ANDROID_API=28 -export BUILD_VERSION=${1} +export BUILD_SCHEME=${1} # note that WORKDIR must not be under the current checkout or the patches will fail to apply WORKDIR=${2} if [[ "${WORKDIR}" == '' ]]; then diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index 30bd4f92..3a5e448d 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -12,7 +12,7 @@ TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} ANDROID_NDK_VERSION=android-ndk-r27c ANDROID_API=28 -export BUILD_VERSION=${1} +export BUILD_SCHEME=${1} # note that WORKDIR must not be under the current checkout or the patches will fail to apply WORKDIR=${2} if [[ "${WORKDIR}" == '' ]]; then diff --git a/swift-ci/sdks/android/scripts/patch-sources.sh b/swift-ci/sdks/android/scripts/patch-sources.sh index 1282c706..efced711 100755 --- a/swift-ci/sdks/android/scripts/patch-sources.sh +++ b/swift-ci/sdks/android/scripts/patch-sources.sh @@ -28,19 +28,19 @@ perl -pi -e 's;dispatch android-execinfo;dispatch;g' $swift_android_patch perl -pi -e 's;call ln -sf;call ln -svf;g' $swift_android_patch perl -pi -e 's%linux-x86_64/sysroot/usr/lib"%linux-x86_64/sysroot/usr/lib"; echo "VALIDATING SYMBOLIC LINK"; ls -la "\${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib"; ls -la "\${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/swift"; %g' $swift_android_patch -case "${BUILD_VERSION}" in +case "${BUILD_SCHEME}" in release) testing_patch="${patches_dir}/swift-android-testing-release.patch" ;; - devel) + swift-*-branch) testing_patch="${patches_dir}/swift-android-testing-except-release.patch" ;; - trunk) + development) testing_patch="${patches_dir}/swift-android-testing-except-release.patch" dispatch_patch="${patches_dir}/swift-android-trunk-libdispatch.patch" ;; *) - echo "$0: invalid BUILD_VERSION=${BUILD_VERSION}" + echo "$0: invalid BUILD_SCHEME=${BUILD_SCHEME}" exit 1 ;; esac diff --git a/swift-ci/sdks/android/scripts/toolchain-vars.sh b/swift-ci/sdks/android/scripts/toolchain-vars.sh index 5f8c5e8a..010f7dd2 100644 --- a/swift-ci/sdks/android/scripts/toolchain-vars.sh +++ b/swift-ci/sdks/android/scripts/toolchain-vars.sh @@ -7,32 +7,32 @@ # ===----------------------------------------------------------------------=== # This script is meant to be sourced from another script that sets the -# BUILD_VERSION environment variable to one of "release", "devel", or "trunk" +# BUILD_SCHEME environment variable to one of "release", "swift-VERSION-branch", or "development" # and will set check the latest builds for each build type in order # to provide information about the Swift tag name in use and where to # obtain the latest toolchain for building. OS=$(echo $HOST_OS | tr -d '.') -case "${BUILD_VERSION}" in +case "${BUILD_SCHEME}" in release) # e.g., "swift-6.1-RELEASE" SWIFT_TAG=$(curl -fsSL https://www.swift.org/api/v1/install/releases.json | jq -r '.[-1].tag') # e.g., "swift-6.1-release" SWIFT_BRANCH=$(echo "${SWIFT_TAG}" | tr '[A-Z]' '[a-z]') ;; - devel) + swift-*-branch) # e.g., swift-6.2-DEVELOPMENT-SNAPSHOT-2025-05-15-a - SWIFT_TAG=$(curl -fsSL https://download.swift.org/swift-6.2-branch/$OS/latest-build.yml | grep '^dir: ' | cut -f 2 -d ' ') - SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-branch" + SWIFT_TAG=$(curl -fsSL https://download.swift.org/$BUILD_SCHEME/$OS/latest-build.yml | grep '^dir: ' | cut -f 2 -d ' ') + SWIFT_BRANCH=$BUILD_SCHEME ;; - trunk) + development) # e.g., swift-DEVELOPMENT-SNAPSHOT-2025-05-14-a SWIFT_TAG=$(curl -fsSL https://download.swift.org/development/$OS/latest-build.yml | grep '^dir: ' | cut -f 2 -d ' ') SWIFT_BRANCH="development" ;; *) - echo "$0: invalid BUILD_VERSION=${BUILD_VERSION}" + echo "$0: invalid BUILD_SCHEME=${BUILD_SCHEME}" exit 1 ;; esac From 2c07eef666993487e9eb70ccc9f28095abfc385b Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Mon, 19 May 2025 18:36:07 -0400 Subject: [PATCH 21/44] Update Android README --- swift-ci/sdks/android/README.md | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/swift-ci/sdks/android/README.md b/swift-ci/sdks/android/README.md index c3c91010..1e57edd1 100644 --- a/swift-ci/sdks/android/README.md +++ b/swift-ci/sdks/android/README.md @@ -25,11 +25,15 @@ Android NDK that will be used for cross-compilation. The `version` argument can be one of the following values: -| version | Swift version | +| version | Swift version example | | --- | --- | | `release` | swift-6.1-RELEASE | -| `devel` | swift-6.2-DEVELOPMENT-SNAPSHOT-yyyy-mm-dd | -| `trunk` | swift-DEVELOPMENT-SNAPSHOT-yyyy-mm-dd | +| `swift-6.2-branch` | swift-6.2-DEVELOPMENT-SNAPSHOT-yyyy-mm-dd | +| `development` | swift-DEVELOPMENT-SNAPSHOT-yyyy-mm-dd | + +> [!WARNING] +> The workdir argument must not be located in a git repository (e.g., it cannot be the +> current directory) ## Running @@ -52,12 +56,11 @@ up the build. This can be useful, e.g., as part of a CI that validates a pull request, as building a single architecture takes around 30 minutes on a standard ubuntu-24.04 GitHub runner, whereas building for all the architectures takes over an hour. -Building within a docker container increases this by about 50%. To build an artifactbundle for just the `x86_64` architecture, run: ``` -TARGET_ARCHS=aarch64 ./build-docker release /tmp/android-sdk +TARGET_ARCHS=x86_64 ./build-docker release /tmp/android-sdk ``` ## Installing and validating the SDK @@ -66,7 +69,17 @@ The `.github/workflows/pull_request.yml` workflow will create and upload an installable SDK named something like: `swift-6.1-RELEASE_android-0.1.artifactbundle.tar.gz` -The workflow will also install the SDK locally and use +The GitHub workflow will also install the SDK locally and use [swift-android-action](https://github.com/marketplace/actions/swift-android-action) to build and test various Swift packages in an Android emulator using the freshly-created SDK bundle. + +## Building locally + +Instead of building within a Docker container, the script can also +perform the build locally on an Ubuntu 24.04 machine with all the +build prerequisites already installed. This will generate +the same artifacts in approximately half the time, and +may be suitable to an already containerized envrionment (such as +a GitHub runner). + From 51d93f81fe5156b190616d0ed9e18a6a8d7cd3ab Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Wed, 21 May 2025 16:53:57 -0400 Subject: [PATCH 22/44] Update how patches are applied --- swift-ci/sdks/android/build-docker | 2 +- swift-ci/sdks/android/build-local | 2 +- .../sdks/android/scripts/patch-sources.sh | 79 ------------------- 3 files changed, 2 insertions(+), 81 deletions(-) delete mode 100755 swift-ci/sdks/android/scripts/patch-sources.sh diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index ef283152..47d067dd 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -27,7 +27,7 @@ source ./scripts/toolchain-vars.sh # Check-out and patch the sources ./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-tag ${SWIFT_TAG} -./scripts/patch-sources.sh ${WORKDIR}/source +${WORKDIR}/source/swift-android-patches/apply-patches.sh ${WORKDIR}/source mkdir -p ${WORKDIR}/products chmod ugo+rwx ${WORKDIR}/products diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index 3a5e448d..e44b896a 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -50,7 +50,7 @@ fi # Check-out and patch the sources ./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-tag ${SWIFT_TAG} -./scripts/patch-sources.sh ${WORKDIR}/source +${WORKDIR}/source/swift-android-patches/apply-patches.sh ${WORKDIR}/source mkdir -p ${WORKDIR}/products diff --git a/swift-ci/sdks/android/scripts/patch-sources.sh b/swift-ci/sdks/android/scripts/patch-sources.sh deleted file mode 100755 index efced711..00000000 --- a/swift-ci/sdks/android/scripts/patch-sources.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash -# Swift Android SDK: Patch Sources -set -e - -source_dir=$1 -if [[ ! -d "${source_dir}" ]]; then - echo "$0: source_dir ${source_dir} does not exist" - exit 1 -fi - -patches_dir="${source_dir}/swift-android-patches" -if [[ ! -d "${patches_dir}" ]]; then - echo "$0: patches_dir ${patches_dir} does not exist" - exit 1 -fi - -cd ${source_dir}/swift-project -swift_android_patch="${patches_dir}/swift-android.patch" - -# patch the patch, which seems to only be needed for an API less than 28 -# https://github.com/finagolfin/swift-android-sdk/blob/main/swift-android.patch#L110 -perl -pi -e 's/#if os\(Windows\)/#if os\(Android\)/g' $swift_android_patch - -# remove the need to link in android-execinfo -perl -pi -e 's;dispatch android-execinfo;dispatch;g' $swift_android_patch - -# debug symbolic link setup -perl -pi -e 's;call ln -sf;call ln -svf;g' $swift_android_patch -perl -pi -e 's%linux-x86_64/sysroot/usr/lib"%linux-x86_64/sysroot/usr/lib"; echo "VALIDATING SYMBOLIC LINK"; ls -la "\${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib"; ls -la "\${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/swift"; %g' $swift_android_patch - -case "${BUILD_SCHEME}" in - release) - testing_patch="${patches_dir}/swift-android-testing-release.patch" - ;; - swift-*-branch) - testing_patch="${patches_dir}/swift-android-testing-except-release.patch" - ;; - development) - testing_patch="${patches_dir}/swift-android-testing-except-release.patch" - dispatch_patch="${patches_dir}/swift-android-trunk-libdispatch.patch" - ;; - *) - echo "$0: invalid BUILD_SCHEME=${BUILD_SCHEME}" - exit 1 - ;; -esac - -for patch in "$swift_android_patch" "$testing_patch" "$dispatch_patch"; do - if [[ "${patch}" == "" ]]; then - continue - fi - - echo "applying patch $patch in $PWD…" - # first check to make sure the patches can apply and fail if not - git apply -v --check -C1 "$patch" - git apply --no-index -v -C1 "$patch" - - #if git apply -C1 --reverse --check "$patch" >/dev/null 2>&1 ; then - # echo "already patched" - #elif git apply -C1 "$patch" ; then - # echo "done" - #else - # echo "failed to apply patch $patch in $PWD" - # exit 1 - #fi -done - -perl -pi -e 's%String\(cString: getpass%\"fake\" //%' swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift -# disable backtrace() for Android (needs either API33+ or libandroid-execinfo, or to manually add in backtrace backport) -perl -pi -e 's;os\(Android\);os\(AndroidDISABLED\);g' swift-testing/Sources/Testing/SourceAttribution/Backtrace.swift - -# need to un-apply libandroid-spawn since we don't need it for API28+ -perl -pi -e 's;MATCHES "Android";MATCHES "AndroidDISABLED";g' llbuild/lib/llvm/Support/CMakeLists.txt -perl -pi -e 's; STREQUAL Android\); STREQUAL AndroidDISABLED\);g' swift-corelibs-foundation/Sources/Foundation/CMakeLists.txt - -# validate the patches -ls -la swift/utils/build-script-impl -grep 'VALIDATING SYMBOLIC LINK' swift/utils/build-script-impl - From 46a06df6d05f843b560fce8cc2cde1d742d79b2e Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Wed, 21 May 2025 17:40:18 -0400 Subject: [PATCH 23/44] Fix source directory for patch target --- swift-ci/sdks/android/build-docker | 2 +- swift-ci/sdks/android/build-local | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index 47d067dd..17a18820 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -27,7 +27,7 @@ source ./scripts/toolchain-vars.sh # Check-out and patch the sources ./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-tag ${SWIFT_TAG} -${WORKDIR}/source/swift-android-patches/apply-patches.sh ${WORKDIR}/source +${WORKDIR}/source/swift-android-patches/apply-patches.sh ${WORKDIR}/source/swift-project mkdir -p ${WORKDIR}/products chmod ugo+rwx ${WORKDIR}/products diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index e44b896a..8311ecb1 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -50,7 +50,7 @@ fi # Check-out and patch the sources ./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-tag ${SWIFT_TAG} -${WORKDIR}/source/swift-android-patches/apply-patches.sh ${WORKDIR}/source +${WORKDIR}/source/swift-android-patches/apply-patches.sh ${WORKDIR}/source/swift-project mkdir -p ${WORKDIR}/products From afb291884f4180054d0d2910f5973d7e9ec6035e Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Wed, 21 May 2025 18:50:10 -0400 Subject: [PATCH 24/44] Harmonize timestamps in artifactbundle with the swift source tag date for build reproducibility --- .github/workflows/pull_request.yml | 8 ++++---- swift-ci/sdks/android/scripts/build.sh | 11 +++++++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 1a4b5833..88a19bdb 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -27,14 +27,14 @@ jobs: strategy: fail-fast: false matrix: - build-type: ['docker'] - #build-type: ['docker', 'local'] + #build-type: ['docker'] + build-type: ['docker', 'local'] # blank arch builds all (aarch64,x86_64,armv7) - arch: [''] + #arch: [''] # builds only x86_64 to speed up the validation #arch: ['x86_64'] # build both the quick (x86_64) and complete (aarch64,x86_64,armv7) SDKs - #arch: ['x86_64', ''] + arch: ['x86_64', ''] swift-version: ['release', 'swift-6.2-branch', 'development'] runs-on: ubuntu-24.04 steps: diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index b82c36ad..6e7f1714 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -186,6 +186,8 @@ function versionFromTag { } swift_version=$(describe ${source_dir}/swift-project/swift) +swift_tag_date=$(git -C ${source_dir}/swift-project/swift log -1 --format=%ct 2>/dev/null) + if [[ $swift_version == swift-* ]]; then swift_version=${swift_version#swift-} fi @@ -631,8 +633,13 @@ header "Outputting compressed bundle" quiet_pushd "${build_dir}" mkdir -p "${products_dir}" - tar czf "${bundle}.tar.gz" "${bundle}" - mv "${bundle}.tar.gz" "${products_dir}" + # set the timestamps of every file in the artifact to the tag date for the swift repo for build reproducibility + touch_date=$(date -d "@$swift_tag_date" "+%Y%m%d%H%M.%S") + find "${bundle}" -exec touch -t "$touch_date" {} + + + bundle_archive="${products_dir}/${bundle}.tar.gz" + tar czf "${bundle_archive}" "${bundle}" + shasum -a 256 "${bundle_archive}" quiet_popd groupend From a469e852b25b029b9efe0cd909c62de1a752f05e Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Wed, 21 May 2025 21:21:26 -0400 Subject: [PATCH 25/44] Simplify toolchain-vars.sh --- swift-ci/sdks/android/scripts/toolchain-vars.sh | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/swift-ci/sdks/android/scripts/toolchain-vars.sh b/swift-ci/sdks/android/scripts/toolchain-vars.sh index 010f7dd2..39868951 100644 --- a/swift-ci/sdks/android/scripts/toolchain-vars.sh +++ b/swift-ci/sdks/android/scripts/toolchain-vars.sh @@ -17,19 +17,16 @@ OS=$(echo $HOST_OS | tr -d '.') case "${BUILD_SCHEME}" in release) # e.g., "swift-6.1-RELEASE" - SWIFT_TAG=$(curl -fsSL https://www.swift.org/api/v1/install/releases.json | jq -r '.[-1].tag') + # there is no latest-build.yml for releases, so we need to get it from the API + export SWIFT_TAG=$(curl -fsSL https://www.swift.org/api/v1/install/releases.json | jq -r '.[-1].tag') # e.g., "swift-6.1-release" - SWIFT_BRANCH=$(echo "${SWIFT_TAG}" | tr '[A-Z]' '[a-z]') + export SWIFT_BRANCH=$(echo "${SWIFT_TAG}" | tr '[A-Z]' '[a-z]') ;; - swift-*-branch) + development|swift-*-branch) # e.g., swift-6.2-DEVELOPMENT-SNAPSHOT-2025-05-15-a - SWIFT_TAG=$(curl -fsSL https://download.swift.org/$BUILD_SCHEME/$OS/latest-build.yml | grep '^dir: ' | cut -f 2 -d ' ') - SWIFT_BRANCH=$BUILD_SCHEME - ;; - development) # e.g., swift-DEVELOPMENT-SNAPSHOT-2025-05-14-a - SWIFT_TAG=$(curl -fsSL https://download.swift.org/development/$OS/latest-build.yml | grep '^dir: ' | cut -f 2 -d ' ') - SWIFT_BRANCH="development" + export SWIFT_TAG=$(curl -fsSL https://download.swift.org/$BUILD_SCHEME/$OS/latest-build.yml | grep '^dir: ' | cut -f 2 -d ' ') + export SWIFT_BRANCH=$BUILD_SCHEME ;; *) echo "$0: invalid BUILD_SCHEME=${BUILD_SCHEME}" From 50ba1cdde2280f61d8b506a0cf0dff538be159fa Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sun, 29 Jun 2025 15:41:29 -0400 Subject: [PATCH 26/44] Run the compiler validation suite for Android (#8) * Run the compiler validation suite for Android * Add --build-compiler option * Add --cross-compile-build-swift-tools=0 from https://github.com/swiftlang/swift/pull/38441 * Build with --build-llvm=0 * Check out Yams for Swift 6.1.1 build * Add --llvm-ninja-targets-for-cross-compile-hosts=help * Install pre-requisites * Install build prerequisites * Only setup local toolchain if build-compiler is not 0 * Fix Yams version checkout * Quote arguments to build scripts * Permit empty host-toolchain argument in build.sh * Fix check for BUILD_COMPILER * Re-order Docker PATH to system clang is used before Swift toolchain clang * Use --host-test to skip attempt to test on connected device/emulator * Install clang in Dockerfile * Add --skip-test-linux flag to build * Remove more folders to free up space * Update patches * Add docker-specific CI variants that run the compiler validation tests * meaningless edit * Update pull_request.yml to stop the 6.1 release builds * Update pull_request.yml to really only build the full docker compiler with tests * Update README.md with meaningless edit to bump build * Update build.sh to skip testing XCTest for linux, as a handful of the linux tests fail for some reason * Update build.sh to disable building libTesting and for 16K memory pages, as both don't work yet * Update build.sh to skip testing Foundation for linux, which requires building SwiftPM from source first * Update build.sh to not build each arch in a separate build directory * Update Dockerfile to use clang 19 instead * Add self-hosted runner CI variant * Fix CI workflow syntax * Update CI runner config * Update CI runner config * Fix name of self-hosted CI runner * Re-order arch run sequence * Update CI for self-hosted runner * Build for self-hosted runner * Add swift-6.2-branch to self-hosted run matrix * Update build.sh to disable aarch64 temporarily, as we know it built fine * Update pull_request.yml to disable non-compiler builds, as they all work, and comment out self-hosted till we know armv7 works * Run compiler validation on self-hosted with increased timeout * Update build-docker to only build for armv7 * Update pull_request.yml to disable self-hosted builds temporarily * Update pull_request.yml to build the full compiler in github runners with the armv7 stdlib and tests * Update build-docker to build for aarch64 first * Update toolchain-vars.sh to use older trunk snapahot toolchain that didn't crash * Update toolchain-vars.sh to only download the latest release compiler if building the Swift compiler from source * Update build-docker to only build for armv7 again * Update toolchain-vars.sh to fix setting branch variable * Update build-docker to build for all three supported arches * Update pull_request.yml to try self-hosted runs again * Update build.sh to only install SDK components and remove linux stdlib * Tolerate missing linux folder when attempting to clean up unnecessary build artifacts * Fix extraCLIOptions in swift-toolset.json * gcpdw * Fix patch application * Update apply-patches.sh to remove unused changes * Update swift-android-testing-except-release.patch to add Testing fix * Update swift-android.patch to remove unneeded patches * Update build.sh to try and fix Testing and clean up the bundle more * Fix README.md --------- Co-authored-by: finagolfin --- .github/workflows/pull_request.yml | 49 ++++-- swift-ci/sdks/android/Dockerfile | 3 +- swift-ci/sdks/android/build-docker | 15 +- swift-ci/sdks/android/build-local | 35 ++-- .../sdks/android/patches/apply-patches.sh | 23 +++ ...swift-android-testing-except-release.patch | 156 ++++++++++++++++++ .../swift-android-testing-release.patch | 61 +++++++ .../swift-android-trunk-libdispatch.patch | 63 +++++++ .../sdks/android/patches/swift-android.patch | 103 ++++++++++++ swift-ci/sdks/android/scripts/build.sh | 98 ++++++++--- swift-ci/sdks/android/scripts/fetch-source.sh | 21 ++- .../sdks/android/scripts/toolchain-vars.sh | 22 ++- 12 files changed, 573 insertions(+), 76 deletions(-) create mode 100755 swift-ci/sdks/android/patches/apply-patches.sh create mode 100644 swift-ci/sdks/android/patches/swift-android-testing-except-release.patch create mode 100644 swift-ci/sdks/android/patches/swift-android-testing-release.patch create mode 100644 swift-ci/sdks/android/patches/swift-android-trunk-libdispatch.patch create mode 100644 swift-ci/sdks/android/patches/swift-android.patch diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 88a19bdb..ad586a0d 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -23,27 +23,38 @@ jobs: *.log android-build: - name: Build Android ${{ matrix.build-type }} ${{ matrix.swift-version }} ${{ matrix.arch }} SDK + name: Android ${{ matrix.build-type }} ${{ matrix.swift-version }} ${{ matrix.arch }} ${{ matrix.runner }} (compiler=${{ matrix.build-compiler }}) strategy: fail-fast: false matrix: - #build-type: ['docker'] - build-type: ['docker', 'local'] - # blank arch builds all (aarch64,x86_64,armv7) - #arch: [''] - # builds only x86_64 to speed up the validation - #arch: ['x86_64'] - # build both the quick (x86_64) and complete (aarch64,x86_64,armv7) SDKs - arch: ['x86_64', ''] - swift-version: ['release', 'swift-6.2-branch', 'development'] - runs-on: ubuntu-24.04 + include: + - swift-version: 'swift-6.2-branch' + build-type: 'docker' + build-compiler: '1' + runner: 'self-hosted' + - swift-version: 'development' + build-type: 'docker' + build-compiler: '1' + runner: 'self-hosted' + - swift-version: 'swift-6.2-branch' + build-type: 'docker' + build-compiler: '0' + runner: 'ubuntu-24.04' + - swift-version: 'development' + build-type: 'docker' + build-compiler: '0' + runner: 'ubuntu-24.04' + runs-on: ${{ matrix.runner }} + # 15 hour timeout + timeout-minutes: 900 steps: - name: Free Disk Space + if: ${{ matrix.runner != 'self-hosted' }} run: | df -h # brings available space from 25G to 32G # otherwise we sometimes run out of space during the build - sudo rm -rf /usr/share/miniconda /usr/share/az* /usr/share/glade* /usr/local/share/chromium /usr/local/share/powershell /usr/share/dotnet /opt/ghc /opt/hostedtoolcache/CodeQL + sudo rm -rf /usr/share/miniconda /usr/share/az* /usr/share/glade* /usr/local/share/chromium /usr/local/share/powershell /usr/share/dotnet /opt/ghc /opt/hostedtoolcache /usr/local/graalvm/ /usr/local/.ghcup/ /usr/local/lib/node_modules /usr/local/share/boost sudo docker image prune --all --force sudo docker builder prune -a df -h @@ -53,6 +64,8 @@ jobs: # these variabes are used by build-docker and build-local # to determine which Swift version to build for echo "BUILD_SCHEME=${{ matrix.swift-version }}" >> $GITHUB_ENV + # pass the build-compiler matrix through to the build script + echo "BUILD_COMPILER=${{ matrix.build-compiler }}" >> $GITHUB_ENV echo "TARGET_ARCHS=${{ matrix.arch }}" >> $GITHUB_ENV echo "WORKDIR=${{ runner.temp }}/swift-android-sdk" >> $GITHUB_ENV - name: Checkout repository @@ -61,7 +74,7 @@ jobs: if: ${{ matrix.build-type == 'local' }} working-directory: swift-ci/sdks/android run: | - sudo apt install -q ninja-build patchelf + sudo apt install -q patchelf build-essential cmake ninja-build python3 golang git gnupg2 libcurl4-openssl-dev libedit-dev libicu-dev libncurses5-dev libpython3-dev libsqlite3-dev libxml2-dev rsync uuid-dev uuid-runtime tzdata curl unzip ./build-local ${BUILD_SCHEME} ${WORKDIR} - name: Build Android SDK (Docker) if: ${{ matrix.build-type == 'docker' }} @@ -99,6 +112,9 @@ jobs: if [[ "${{ matrix.build-type }}" == 'local' ]]; then ARTIFACT_NAME="${ARTIFACT_NAME}-local" fi + if [[ "${{ matrix.build-compiler }}" == '1' ]]; then + ARTIFACT_NAME="${ARTIFACT_NAME}-hostbuild" + fi # artifacts need a unique name so we suffix with the matrix arch(s) if [[ ! -z "${{ matrix.arch }}" ]]; then ARTIFACT_NAME="${ARTIFACT_NAME}-$(echo ${{ matrix.arch }} | tr ',' '-')" @@ -117,6 +133,7 @@ jobs: name: ${{ steps.info.outputs.artifact-name }} path: ${{ steps.info.outputs.artifact-path }} - name: Cleanup + if: ${{ matrix.runner != 'self-hosted' }} run: | # need to free up some space or else when installing we get: No space left on device df -h @@ -125,6 +142,7 @@ jobs: sudo docker builder prune -a df -h - name: Install artifactbundle + if: ${{ matrix.runner != 'self-hosted' }} shell: bash run: | set -ex @@ -135,6 +153,7 @@ jobs: echo "ANDROID_NDK_ROOT=" >> $GITHUB_ENV - name: Create Demo Project + if: ${{ matrix.runner != 'self-hosted' }} run: | cd ${{ runner.temp }} mkdir DemoProject @@ -149,6 +168,7 @@ jobs: echo 'import Android' >> Sources/DemoProject/DemoProject.swift - name: Test Demo Project on Android uses: skiptools/swift-android-action@main + if: ${{ matrix.runner != 'self-hosted' }} with: # only test for the complete arch SDK build to speed up CI #run-tests: ${{ matrix.arch == '' }} @@ -157,14 +177,15 @@ jobs: installed-swift: ${{ steps.info.outputs.swift-root }} - name: Checkout swift-algorithms + if: ${{ matrix.runner != 'self-hosted' }} uses: actions/checkout@v4 with: repository: apple/swift-algorithms path: swift-algorithms - name: Test swift-algorithms + if: ${{ matrix.runner != 'self-hosted' }} uses: skiptools/swift-android-action@main with: - run-tests: ${{ matrix.arch == '' }} package-path: swift-algorithms installed-sdk: ${{ steps.info.outputs.sdk-id }} installed-swift: ${{ steps.info.outputs.swift-root }} diff --git a/swift-ci/sdks/android/Dockerfile b/swift-ci/sdks/android/Dockerfile index c1d8c6db..8708586a 100644 --- a/swift-ci/sdks/android/Dockerfile +++ b/swift-ci/sdks/android/Dockerfile @@ -18,6 +18,7 @@ ARG SWIFT_TOOLCHAIN_URL= RUN apt-get -q update \ && DEBIAN_FRONTEND=noninteractive apt-get -q install -y \ build-essential \ + clang-19 \ cmake \ ninja-build \ python3 \ @@ -54,7 +55,7 @@ ENV SWIFT_SIGNING_KEY=$SWIFT_SIGNING_KEY \ COPY scripts/install-swift.sh /scripts/install-swift.sh RUN chmod ugo+x /scripts/install-swift.sh RUN /scripts/install-swift.sh /usr/local/swift -ENV PATH="/usr/local/swift/bin:${PATH}" +ENV PATH="/usr/lib/llvm-19/bin:/usr/local/swift/bin:${PATH}" ARG ANDROID_NDK_VERSION= diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index 17a18820..e8f1f3f1 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -27,7 +27,7 @@ source ./scripts/toolchain-vars.sh # Check-out and patch the sources ./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-tag ${SWIFT_TAG} -${WORKDIR}/source/swift-android-patches/apply-patches.sh ${WORKDIR}/source/swift-project +./patches/apply-patches.sh ${WORKDIR}/source/swift-project mkdir -p ${WORKDIR}/products chmod ugo+rwx ${WORKDIR}/products @@ -59,9 +59,10 @@ $DOCKER run -i --rm \ -v ${WORKDIR}/products:/products:rw \ ${CONTAINER_NAME} \ /scripts/build.sh \ - --source-dir /source \ - --products-dir /products \ - --host-toolchain /usr/local/swift \ - --android-api ${ANDROID_API} \ - --ndk-home /usr/local/ndk/${ANDROID_NDK_VERSION} \ - --archs ${TARGET_ARCHS} + --source-dir "/source" \ + --products-dir "/products" \ + --host-toolchain "/usr/local/swift" \ + --build-compiler "${BUILD_COMPILER}" \ + --android-api "${ANDROID_API}" \ + --ndk-home "/usr/local/ndk/${ANDROID_NDK_VERSION}" \ + --archs "${TARGET_ARCHS}" diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index 8311ecb1..31eaf1e3 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -25,16 +25,18 @@ WORKDIR=$(realpath ${WORKDIR}) HOST_OS=ubuntu$(lsb_release -sr) source ./scripts/toolchain-vars.sh -SWIFT_ROOT=${WORKDIR}/host-toolchain -HOST_TOOLCHAIN=$SWIFT_ROOT/$SWIFT_BASE/usr -if [[ ! -d "$HOST_TOOLCHAIN" ]]; then - ./scripts/install-swift.sh ${HOST_TOOLCHAIN} -fi +if [[ "${BUILD_COMPILER}" != "1" ]]; then + SWIFT_ROOT=${WORKDIR}/host-toolchain + HOST_TOOLCHAIN=$SWIFT_ROOT/$SWIFT_BASE/usr + if [[ ! -d "$HOST_TOOLCHAIN" ]]; then + ./scripts/install-swift.sh ${HOST_TOOLCHAIN} + fi -$HOST_TOOLCHAIN/bin/swift --version + $HOST_TOOLCHAIN/bin/swift --version -# ensure the correct Swift is first in the PATH -export PATH=$HOST_TOOLCHAIN/bin:$PATH + # ensure the correct Swift is first in the PATH + export PATH=$HOST_TOOLCHAIN/bin:$PATH +fi export ANDROID_NDK_HOME=${WORKDIR}/ndk/${ANDROID_NDK_VERSION} @@ -50,15 +52,16 @@ fi # Check-out and patch the sources ./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-tag ${SWIFT_TAG} -${WORKDIR}/source/swift-android-patches/apply-patches.sh ${WORKDIR}/source/swift-project +./patches/apply-patches.sh ${WORKDIR}/source/swift-project mkdir -p ${WORKDIR}/products ./scripts/build.sh \ - --source-dir ${WORKDIR}/source \ - --products-dir ${WORKDIR}/products \ - --build-dir ${WORKDIR}/build \ - --host-toolchain ${HOST_TOOLCHAIN} \ - --android-api ${ANDROID_API} \ - --ndk-home ${ANDROID_NDK_HOME} \ - --archs ${TARGET_ARCHS} + --source-dir "${WORKDIR}/source" \ + --products-dir "${WORKDIR}/products" \ + --build-dir "${WORKDIR}/build" \ + --build-compiler "${BUILD_COMPILER}" \ + --host-toolchain "${HOST_TOOLCHAIN}" \ + --android-api "${ANDROID_API}" \ + --ndk-home "${ANDROID_NDK_HOME}" \ + --archs "${TARGET_ARCHS}" diff --git a/swift-ci/sdks/android/patches/apply-patches.sh b/swift-ci/sdks/android/patches/apply-patches.sh new file mode 100755 index 00000000..36f234f1 --- /dev/null +++ b/swift-ci/sdks/android/patches/apply-patches.sh @@ -0,0 +1,23 @@ +#!/bin/bash -ex + +patches_dir=$(dirname $(realpath -- "${BASH_SOURCE[0]}")) +cd ${1} + +case "${BUILD_SCHEME}" in + swift-*-branch) + git apply -v -C1 ${patches_dir}/swift-android.patch + git apply -v -C1 ${patches_dir}/swift-android-testing-except-release.patch + ;; + development) + git apply -v -C1 ${patches_dir}/swift-android.patch + git apply -v -C1 ${patches_dir}/swift-android-trunk-libdispatch.patch + git apply -v -C1 ${patches_dir}/swift-android-testing-except-release.patch + ;; + *) + echo "$0: invalid BUILD_SCHEME=${BUILD_SCHEME}" + exit 1 + ;; +esac + +# disable backtrace() for Android (needs either API33+ or libandroid-execinfo, or to manually add in backtrace backport) +perl -pi -e 's;os\(Android\);os\(AndroidDISABLED\);g' swift-testing/Sources/Testing/SourceAttribution/Backtrace.swift diff --git a/swift-ci/sdks/android/patches/swift-android-testing-except-release.patch b/swift-ci/sdks/android/patches/swift-android-testing-except-release.patch new file mode 100644 index 00000000..4d6fea0d --- /dev/null +++ b/swift-ci/sdks/android/patches/swift-android-testing-except-release.patch @@ -0,0 +1,156 @@ +diff --git a/swift/test/ClangImporter/pcm-emit-direct-cc1-mode.swift b/swift/test/ClangImporter/pcm-emit-direct-cc1-mode.swift +index f5048b9e3d5..c227ef7b77d 100644 +--- a/swift/test/ClangImporter/pcm-emit-direct-cc1-mode.swift ++++ b/swift/test/ClangImporter/pcm-emit-direct-cc1-mode.swift +@@ -9,7 +9,7 @@ + // CHECK-DUMP: Module map file: {{.*[/\\]}}Inputs{{/|\\}}custom-modules{{/|\\}}module.modulemap + + // Verify that the clang command-line used is cc1 +-// RUN: %FileCheck -check-prefix CHECK-CLANG -DTRIPLE=%target-triple %s < %t.diags.txt ++// RUN: %FileCheck -check-prefix CHECK-CLANG -DTRIPLE=%module-target-triple %s < %t.diags.txt + // CHECK-CLANG: clang importer cc1 args + // CHECK-CLANG-SAME: '-o' '{{.*[/\\]}}script.pcm' '-fsyntax-only' '-x' 'objective-c' '{{.*[/\\]}}module.modulemap' '-triple' '[[TRIPLE]]' + // CHECK-CLANG-SAME: '-fmodules' +diff --git a/swift/test/IRGen/abitypes_arm.swift b/swift/test/IRGen/abitypes_arm.swift +index 9428294483d..52bd8b10145 100644 +--- a/swift/test/IRGen/abitypes_arm.swift ++++ b/swift/test/IRGen/abitypes_arm.swift +@@ -11,6 +11,6 @@ class Foo { + } + } + +-// armv7: define internal void @makeOne(ptr noalias sret({{.*}}) align 4 %agg.result, float{{( noundef)?}} %f, float{{( noundef)?}} %s) ++// armv7: define internal void @makeOne(ptr{{( dead_on_unwind)?}} noalias{{( writable)?}} sret({{.*}}) align 4 %agg.result, float{{( noundef)?}} %f, float{{( noundef)?}} %s) + // armv7s: define internal void @makeOne(ptr noalias sret({{.*}}) align 4 %agg.result, float %f, float %s) + // armv7k: define internal %struct.One @makeOne(float {{.*}}%f, float {{.*}}%s) +diff --git a/swift/test/Interop/Cxx/class/constructors-copy-module-interface.swift b/swift/test/Interop/Cxx/class/constructors-copy-module-interface.swift +index 042c10e7d1d..254f6543b58 100644 +--- a/swift/test/Interop/Cxx/class/constructors-copy-module-interface.swift ++++ b/swift/test/Interop/Cxx/class/constructors-copy-module-interface.swift +@@ -1,4 +1,5 @@ + // RUN: %target-swift-ide-test -print-module -module-to-print=Constructors -I %S/Inputs -source-filename=x -enable-experimental-cxx-interop | %FileCheck %s ++// XFAIL: OS=linux-androideabi + + // CHECK: struct TemplatedCopyConstructor + // CHECK: struct TemplatedCopyConstructorWithExtraArg +diff --git a/swift/test/Interop/Cxx/class/constructors-diagnostics.swift b/swift/test/Interop/Cxx/class/constructors-diagnostics.swift +index aac01480660..8fa99fae7fb 100644 +--- a/swift/test/Interop/Cxx/class/constructors-diagnostics.swift ++++ b/swift/test/Interop/Cxx/class/constructors-diagnostics.swift +@@ -2,6 +2,7 @@ + + // This test uses -verify-additional-file, which do not work well on Windows. + // UNSUPPORTED: OS=windows-msvc ++// XFAIL: OS=linux-androideabi + + import Constructors + +diff --git a/swift/test/Interop/Cxx/class/constructors-module-interface.swift b/swift/test/Interop/Cxx/class/constructors-module-interface.swift +index bf189b6e302..d2c3d65f03e 100644 +--- a/swift/test/Interop/Cxx/class/constructors-module-interface.swift ++++ b/swift/test/Interop/Cxx/class/constructors-module-interface.swift +@@ -1,4 +1,5 @@ + // RUN: %target-swift-ide-test -print-module -module-to-print=Constructors -I %S/Inputs/ -source-filename=x -enable-experimental-cxx-interop | %FileCheck %s ++// XFAIL: OS=linux-androideabi + + // CHECK: struct ExplicitDefaultConstructor { + // CHECK-NEXT: init() +diff --git a/swift/test/Interop/Cxx/class/constructors-silgen.swift b/swift/test/Interop/Cxx/class/constructors-silgen.swift +index b1624ccccfe..79cbdf170de 100644 +--- a/swift/test/Interop/Cxx/class/constructors-silgen.swift ++++ b/swift/test/Interop/Cxx/class/constructors-silgen.swift +@@ -1,4 +1,5 @@ + // RUN: %target-swiftxx-frontend -I %S/Inputs -Xllvm -sil-print-types -emit-silgen %s | %FileCheck %s ++// XFAIL: OS=linux-androideabi + + import Constructors + +diff --git a/swift/test/Interop/Cxx/class/constructors-typechecker.swift b/swift/test/Interop/Cxx/class/constructors-typechecker.swift +index 8ce10b1ca75..52ff4104b83 100644 +--- a/swift/test/Interop/Cxx/class/constructors-typechecker.swift ++++ b/swift/test/Interop/Cxx/class/constructors-typechecker.swift +@@ -1,4 +1,5 @@ + // RUN: %target-typecheck-verify-swift -verify-ignore-unknown -I %S/Inputs -enable-experimental-cxx-interop ++// XFAIL: OS=linux-androideabi + + import Constructors + +diff --git a/swift/test/Interop/Cxx/class/custom-new-operator-irgen.swift b/swift/test/Interop/Cxx/class/custom-new-operator-irgen.swift +index 74e692c9db2..0423493afac 100644 +--- a/swift/test/Interop/Cxx/class/custom-new-operator-irgen.swift ++++ b/swift/test/Interop/Cxx/class/custom-new-operator-irgen.swift +@@ -5,4 +5,4 @@ import CustomNewOperator + var x = callsCustomNew() + + // Make sure the definition of `operator new` is emitted. +-// CHECK: define {{.*}} @{{_ZnwmPv15container_new_t|"\?\?2@YAPEAX_KPEAXUcontainer_new_t@@@Z"}} ++// CHECK: define {{.*}} @{{_Znw(j|m)Pv15container_new_t|"\?\?2@YAPEAX_KPEAXUcontainer_new_t@@@Z"}} +diff --git a/swift/test/SILOptimizer/package-cmo-serialize-tables.swift b/swift/test/SILOptimizer/package-cmo-serialize-tables.swift +index 382ff2e8080..f061374b2a1 100644 +--- a/swift/test/SILOptimizer/package-cmo-serialize-tables.swift ++++ b/swift/test/SILOptimizer/package-cmo-serialize-tables.swift +@@ -19,7 +19,7 @@ + + // Temporarily disabling on watchOS (both arm64_32 & armv7k): + // rdar://140330692 (🟠 OSS Swift CI: oss-swift_tools-RA_stdlib-DA_test-device-non_executable failed... +-// UNSUPPORTED: OS=watchos ++// UNSUPPORTED: OS=watchos, OS=linux-androideabi + + //--- main.swift + +diff --git a/swift/test/SILOptimizer/throws_prediction.swift b/swift/test/SILOptimizer/throws_prediction.swift +index f7f2664b2de..01d36e127c6 100644 +--- a/swift/test/SILOptimizer/throws_prediction.swift ++++ b/swift/test/SILOptimizer/throws_prediction.swift +@@ -16,7 +16,7 @@ + // RUN: -sil-verify-all -module-name=test -emit-sil \ + // RUN: | %FileCheck --check-prefix CHECK-DISABLED %s + +-// UNSUPPORTED: CPU=armv7k || CPU=arm64_32 ++// UNSUPPORTED: CPU=armv7k, CPU=arm64_32, CPU=armv7 + + // CHECK-DISABLED-NOT: normal_count + +diff --git a/swift/utils/swift_build_support/swift_build_support/products/product.py b/swift/utils/swift_build_support/swift_build_support/products/product.py +index 274b6037305..bf0f7fcf671 100644 +--- a/swift/utils/swift_build_support/swift_build_support/products/product.py ++++ b/swift/utils/swift_build_support/swift_build_support/products/product.py +@@ -212,7 +212,7 @@ class Product(object): + if self.args.cross_compile_hosts: + if self.is_darwin_host(host_target): + install_destdir = self.host_install_destdir(host_target) +- else: ++ elif self.args.cross_compile_append_host_target_to_destdir: + install_destdir = os.path.join(install_destdir, self.args.host_target) + return targets.toolchain_path(install_destdir, + self.args.install_prefix) +diff --git a/swift-corelibs-xctest/cmake/modules/PlatformInfo.cmake b/swift-corelibs-xctest/cmake/modules/PlatformInfo.cmake +index 468dc61..a90ae3d 100644 +--- a/swift-corelibs-xctest/cmake/modules/PlatformInfo.cmake ++++ b/swift-corelibs-xctest/cmake/modules/PlatformInfo.cmake +@@ -2,6 +2,10 @@ + set(print_target_info_invocation "${CMAKE_Swift_COMPILER}" -print-target-info) + if(CMAKE_Swift_COMPILER_TARGET) + list(APPEND print_target_info_invocation -target ${CMAKE_Swift_COMPILER_TARGET}) ++else() ++ set(arg_list ${CMAKE_Swift_FLAGS}) ++ separate_arguments(arg_list) ++ list(APPEND print_target_info_invocation ${arg_list}) + endif() + execute_process(COMMAND ${print_target_info_invocation} OUTPUT_VARIABLE target_info_json) + message(CONFIGURE_LOG "Swift Target Info: ${print_target_info_invocation}\n" +diff --git a/swift-testing/cmake/modules/PlatformInfo.cmake b/swift-testing/cmake/modules/PlatformInfo.cmake +index 94c60ef..ab8a491 100644 +--- a/swift-testing/cmake/modules/PlatformInfo.cmake ++++ b/swift-testing/cmake/modules/PlatformInfo.cmake +@@ -9,6 +9,10 @@ + set(print_target_info_invocation "${CMAKE_Swift_COMPILER}" -print-target-info) + if(CMAKE_Swift_COMPILER_TARGET) + list(APPEND print_target_info_invocation -target ${CMAKE_Swift_COMPILER_TARGET}) ++else() ++ set(arg_list ${CMAKE_Swift_FLAGS}) ++ separate_arguments(arg_list) ++ list(APPEND print_target_info_invocation ${arg_list}) + endif() + execute_process(COMMAND ${print_target_info_invocation} OUTPUT_VARIABLE target_info_json) + message(CONFIGURE_LOG "Swift Target Info: ${print_target_info_invocation}\n" diff --git a/swift-ci/sdks/android/patches/swift-android-testing-release.patch b/swift-ci/sdks/android/patches/swift-android-testing-release.patch new file mode 100644 index 00000000..923d1194 --- /dev/null +++ b/swift-ci/sdks/android/patches/swift-android-testing-release.patch @@ -0,0 +1,61 @@ +diff --git a/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py b/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py +index b6bfa5760ef..324d1a77eea 100644 +--- a/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py ++++ b/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py +@@ -51,7 +51,7 @@ class SwiftTesting(product.Product): + return False + + def should_install(self, host_target): +- return self.args.install_swift_testing_macros ++ return self.args.install_swift_testing + + def _cmake_product(self, host_target): + build_root = os.path.dirname(self.build_dir) +diff --git a/swift-corelibs-foundation/Sources/CoreFoundation/CFPlatform.c b/swift-corelibs-foundation/Sources/CoreFoundation/CFPlatform.c +index 90f4aa78..0429425b 100644 +--- a/swift-corelibs-foundation/Sources/CoreFoundation/CFPlatform.c ++++ b/swift-corelibs-foundation/Sources/CoreFoundation/CFPlatform.c +@@ -1948,7 +1948,7 @@ CF_CROSS_PLATFORM_EXPORT void *_CFReallocf(void *ptr, size_t size) { + #endif + } + +-#if TARGET_OS_ANDROID ++#if TARGET_OS_ANDROID && __ANDROID_API__ < 28 + + #include + #include +@@ -2277,6 +2277,10 @@ CF_EXPORT int _CFPosixSpawnFileActionsAddClose(_CFPosixSpawnFileActionsRef file_ + return _CFPosixSpawnFileActionsAddCloseImpl(file_actions, filedes); + } + ++CF_EXPORT int _CFPosixSpawnFileActionsChdir(_CFPosixSpawnFileActionsRef file_actions, const char *path) { ++ return ENOSYS; ++} ++ + CF_EXPORT int _CFPosixSpawn(pid_t *_CF_RESTRICT pid, const char *_CF_RESTRICT path, _CFPosixSpawnFileActionsRef file_actions, _CFPosixSpawnAttrRef _Nullable _CF_RESTRICT attrp, char *_Nullable const argv[_Nullable _CF_RESTRICT], char *_Nullable const envp[_Nullable _CF_RESTRICT]) { + _CFPosixSpawnInitialize(); + return _CFPosixSpawnImpl(pid, path, file_actions, attrp, argv, envp); +@@ -2279,7 +2279,7 @@ CF_EXPORT int _CFPosixSpawnFileActionsAddClose(_CFPosixSpawnFileActionsRef file_ + } + + CF_EXPORT int _CFPosixSpawnFileActionsChdir(_CFPosixSpawnFileActionsRef file_actions, const char *path) { +- #if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 29) ++ #if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 29) || (defined(__ANDROID__) && __ANDROID_API__ < 34) + // Glibc versions prior to 2.29 don't support posix_spawn_file_actions_addchdir_np, impacting: + // - Amazon Linux 2 (EoL mid-2025) + return ENOSYS; +diff --git a/swift-testing/cmake/modules/SwiftModuleInstallation.cmake b/swift-testing/cmake/modules/SwiftModuleInstallation.cmake +index 1553725..d9f9e5b 100644 +--- a/swift-testing/cmake/modules/SwiftModuleInstallation.cmake ++++ b/swift-testing/cmake/modules/SwiftModuleInstallation.cmake +@@ -75,6 +75,10 @@ function(_swift_testing_install_target module) + set(module_triple_command "${CMAKE_Swift_COMPILER}" -print-target-info) + if(CMAKE_Swift_COMPILER_TARGET) + list(APPEND module_triple_command -target ${CMAKE_Swift_COMPILER_TARGET}) ++ else() ++ set(arg_list ${CMAKE_Swift_FLAGS}) ++ separate_arguments(arg_list) ++ list(APPEND module_triple_command ${arg_list}) + endif() + execute_process(COMMAND ${module_triple_command} OUTPUT_VARIABLE target_info_json) + string(JSON module_triple GET "${target_info_json}" "target" "moduleTriple") diff --git a/swift-ci/sdks/android/patches/swift-android-trunk-libdispatch.patch b/swift-ci/sdks/android/patches/swift-android-trunk-libdispatch.patch new file mode 100644 index 00000000..4e824fad --- /dev/null +++ b/swift-ci/sdks/android/patches/swift-android-trunk-libdispatch.patch @@ -0,0 +1,63 @@ +diff --git a/swift/test/Misc/verify-swift-feature-testing.test-sh b/swift/test/Misc/verify-swift-feature-testing.test-sh +index 3f36fc979cb..8713ae8fea8 100755 +--- a/swift/test/Misc/verify-swift-feature-testing.test-sh ++++ b/swift/test/Misc/verify-swift-feature-testing.test-sh +@@ -98,7 +98,6 @@ def find_matches(swift_src_root): + # `-enable-upcoming-feature` in the test directories. + output = subprocess.check_output( + [ +- "git", + "grep", + "--extended-regexp", + "--recursive", +diff --git a/swift/test/Reflection/conformance_descriptors.swift b/swift/test/Reflection/conformance_descriptors.swift +index b114f764552..e1bd98c08f4 100644 +--- a/swift/test/Reflection/conformance_descriptors.swift ++++ b/swift/test/Reflection/conformance_descriptors.swift +@@ -5,6 +5,7 @@ + // + // Temporarily disable on AArch64 Linux (rdar://88451721) + // UNSUPPORTED: OS=linux-gnu && CPU=aarch64 ++// XFAIL: OS=linux-android + + // rdar://100558042 + // UNSUPPORTED: CPU=arm64e +diff --git a/swift/test/Reflection/typeref_decoding.swift b/swift/test/Reflection/typeref_decoding.swift +index e0a3b4465a3..c01af893eda 100644 +--- a/swift/test/Reflection/typeref_decoding.swift ++++ b/swift/test/Reflection/typeref_decoding.swift +@@ -10,6 +10,7 @@ + + // FIXME: rdar://127796117 + // UNSUPPORTED: OS=linux-gnu && CPU=aarch64 ++// XFAIL: OS=linux-android + + // RUN: %target-build-swift -target %target-swift-5.2-abi-triple -Xfrontend -enable-anonymous-context-mangled-names %S/Inputs/ConcreteTypes.swift %S/Inputs/GenericTypes.swift %S/Inputs/Protocols.swift %S/Inputs/Extensions.swift %S/Inputs/Closures.swift -parse-as-library -emit-module -emit-library %no-fixup-chains -module-name TypesToReflect -o %t/%target-library-name(TypesToReflect) + // RUN: %target-build-swift -target %target-swift-5.2-abi-triple -Xfrontend -enable-anonymous-context-mangled-names %S/Inputs/ConcreteTypes.swift %S/Inputs/GenericTypes.swift %S/Inputs/Protocols.swift %S/Inputs/Extensions.swift %S/Inputs/Closures.swift %S/Inputs/main.swift -emit-module -emit-executable %no-fixup-chains -module-name TypesToReflect -o %t/TypesToReflect +diff --git a/swift-corelibs-libdispatch/cmake/modules/SwiftSupport.cmake b/swift-corelibs-libdispatch/cmake/modules/SwiftSupport.cmake +index 3da519e..e2dd8ed 100644 +--- a/swift-corelibs-libdispatch/cmake/modules/SwiftSupport.cmake ++++ b/swift-corelibs-libdispatch/cmake/modules/SwiftSupport.cmake +@@ -4,6 +4,10 @@ if(NOT dispatch_MODULE_TRIPLE) + set(module_triple_command "${CMAKE_Swift_COMPILER}" -print-target-info) + if(CMAKE_Swift_COMPILER_TARGET) + list(APPEND module_triple_command -target ${CMAKE_Swift_COMPILER_TARGET}) ++ else() ++ set(arg_list ${CMAKE_Swift_FLAGS}) ++ separate_arguments(arg_list) ++ list(APPEND module_triple_command ${arg_list}) + endif() + execute_process(COMMAND ${module_triple_command} OUTPUT_VARIABLE target_info_json) + +diff --git a/swift-corelibs-libdispatch/src/swift/CMakeLists.txt b/swift-corelibs-libdispatch/src/swift/CMakeLists.txt +index 38bef37..d0ddf98 100644 +--- a/swift-corelibs-libdispatch/src/swift/CMakeLists.txt ++++ b/swift-corelibs-libdispatch/src/swift/CMakeLists.txt +@@ -15,6 +15,7 @@ add_library(swiftDispatch + Source.swift + Time.swift + Wrapper.swift) ++set(CMAKE_SHARED_LINKER_FLAGS "") + target_compile_options(swiftDispatch PRIVATE + "SHELL:-Xcc -fblocks" + "SHELL:-Xcc -fmodule-map-file=${PROJECT_SOURCE_DIR}/dispatch/module.modulemap" diff --git a/swift-ci/sdks/android/patches/swift-android.patch b/swift-ci/sdks/android/patches/swift-android.patch new file mode 100644 index 00000000..2d465ea2 --- /dev/null +++ b/swift-ci/sdks/android/patches/swift-android.patch @@ -0,0 +1,103 @@ +diff --git a/swift/utils/build-script-impl b/swift/utils/build-script-impl +index 16e05052609..7ab8cebfab8 100755 +--- a/swift/utils/build-script-impl ++++ b/swift/utils/build-script-impl +@@ -2622,6 +2622,7 @@ for host in "${ALL_HOSTS[@]}"; do + echo "Cleaning the libdispatch build directory" + call rm -rf "${LIBDISPATCH_BUILD_DIR}" + fi ++ call ln -sf "${SWIFT_BUILD_PATH}/lib/swift" "${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib" + + cmake_options=( + -DENABLE_SWIFT=YES +diff --git a/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py b/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py +index 324d1a77eea..e88601a8701 100644 +--- a/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py ++++ b/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py +@@ -13,6 +13,9 @@ + import os + + from build_swift.build_swift.versions import Version ++from ..host_specific_configuration \ ++ import HostSpecificConfiguration ++from ..targets import StdlibDeploymentTarget + + from . import cmake_product + from . import product +@@ -115,6 +117,22 @@ class SwiftTestingCMakeShim(cmake_product.CMakeProduct): + # FIXME: If we build macros for the builder, specify the path. + self.cmake_options.define('SwiftTesting_MACRO', 'NO') + ++ if host_target.startswith('android') and self.is_cross_compile_target(host_target): ++ host_config = HostSpecificConfiguration(host_target, self.args) ++ self.cmake_options.extend(host_config.cmake_options) ++ flags = '-target %s-unknown-linux-android%s ' % (self.args.android_arch, ++ self.args.android_api_level) ++ ++ flags += '-resource-dir %s/lib/swift ' % ( ++ self.host_install_destdir(host_target) + self.args.install_prefix) ++ ++ ndk_path = StdlibDeploymentTarget.get_target_for_name(host_target).platform.ndk_toolchain_path(self.args) ++ flags += '-sdk %s/sysroot ' % (ndk_path) ++ flags += '-tools-directory %s/bin' % (ndk_path) ++ self.cmake_options.define('CMAKE_Swift_FLAGS', flags) ++ self.cmake_options.define('CMAKE_CXX_COMPILER_WORKS', 'True') ++ self.cmake_options.define('CMAKE_FIND_ROOT_PATH', self.args.cross_compile_deps_path) ++ + self.generate_toolchain_file_for_darwin_or_linux( + host_target, override_macos_deployment_version=override_deployment_version) + self.build_with_cmake([], self.args.build_variant, [], +diff --git a/swift-corelibs-foundation/CMakeLists.txt b/swift-corelibs-foundation/CMakeLists.txt +index 7f290d16..95366592 100644 +--- a/swift-corelibs-foundation/CMakeLists.txt ++++ b/swift-corelibs-foundation/CMakeLists.txt +@@ -51,6 +51,7 @@ if(NOT CMAKE_SYSTEM_NAME STREQUAL Windows) + endif() + endif() + ++set(CMAKE_SHARED_LINKER_FLAGS "") + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) +diff --git a/swift-foundation/cmake/modules/SwiftFoundationSwiftSupport.cmake b/swift-foundation/cmake/modules/SwiftFoundationSwiftSupport.cmake +index cbdfc2a..bb4121d 100644 +--- a/swift-foundation/cmake/modules/SwiftFoundationSwiftSupport.cmake ++++ b/swift-foundation/cmake/modules/SwiftFoundationSwiftSupport.cmake +@@ -37,6 +37,10 @@ function(_swift_foundation_install_target module) + set(module_triple_command "${CMAKE_Swift_COMPILER}" -print-target-info) + if(CMAKE_Swift_COMPILER_TARGET) + list(APPEND module_triple_command -target ${CMAKE_Swift_COMPILER_TARGET}) ++ else() ++ set(arg_list ${CMAKE_Swift_FLAGS}) ++ separate_arguments(arg_list) ++ list(APPEND module_triple_command ${arg_list}) + endif() + execute_process(COMMAND ${module_triple_command} OUTPUT_VARIABLE target_info_json) + string(JSON module_triple GET "${target_info_json}" "target" "moduleTriple") +diff --git a/swift-testing/CMakeLists.txt b/swift-testing/CMakeLists.txt +index 1be9a4b..bd7b1bd 100644 +--- a/swift-testing/CMakeLists.txt ++++ b/swift-testing/CMakeLists.txt +@@ -28,6 +28,7 @@ list(APPEND CMAKE_MODULE_PATH + ${PROJECT_SOURCE_DIR}/cmake/modules + ${PROJECT_SOURCE_DIR}/cmake/modules/shared) + ++set(CMAKE_SHARED_LINKER_FLAGS "") + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) +diff --git a/swift-testing/cmake/modules/TargetTriple.cmake b/swift-testing/cmake/modules/TargetTriple.cmake +index e087cc4..02f3a95 100644 +--- a/swift-testing/cmake/modules/TargetTriple.cmake ++++ b/swift-testing/cmake/modules/TargetTriple.cmake +@@ -10,6 +10,10 @@ + set(SWT_TARGET_INFO_COMMAND "${CMAKE_Swift_COMPILER}" -print-target-info) + if(CMAKE_Swift_COMPILER_TARGET) + list(APPEND SWT_TARGET_INFO_COMMAND -target ${CMAKE_Swift_COMPILER_TARGET}) ++else() ++ set(arg_list ${CMAKE_Swift_FLAGS}) ++ separate_arguments(arg_list) ++ list(APPEND SWT_TARGET_INFO_COMMAND ${arg_list}) + endif() + execute_process(COMMAND ${SWT_TARGET_INFO_COMMAND} OUTPUT_VARIABLE SWT_TARGET_INFO_JSON) + string(JSON SWT_TARGET_TRIPLE GET "${SWT_TARGET_INFO_JSON}" "target" "unversionedTriple") diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 6e7f1714..b388f025 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -50,7 +50,7 @@ function groupend { function usage { cat < --products-dir --ndk-home --host-toolchain +usage: build.sh --source-dir --products-dir --ndk-home [--name ] [--version ] [--build-dir ] [--archs [, ...]] @@ -63,6 +63,7 @@ Options: --source-dir Specify the path in which the sources can be found. --ndk-home Specify the path to the Android NDK --host-toolchain Specify the path to the host Swift toolchain + --build-compiler Whether to build and validate the host compiler --products-dir Specify the path in which the products should be written. --build-dir Specify the path in which intermediates should be stored. --android-api Specify the Android API level @@ -130,6 +131,8 @@ while [ "$#" -gt 0 ]; do ndk_home="$2"; shift ;; --host-toolchain) host_toolchain="$2"; shift ;; + --build-compiler) + build_compiler="$2"; shift ;; --build-dir) build_dir="$2"; shift ;; --android-api) @@ -155,7 +158,7 @@ done # Change the commas for spaces archs="${archs//,/ }" -if [[ -z "$source_dir" || -z "$products_dir" || -z "$ndk_home" || -z "$host_toolchain" ]]; then +if [[ -z "$source_dir" || -z "$products_dir" || -z "$ndk_home" ]]; then usage exit 1 fi @@ -234,8 +237,10 @@ export ANDROID_NDK_HOME=$ndk_home export ANDROID_NDK=$ndk_home echo "Swift found at ${swift_dir}" -echo "Host toolchain found at ${host_toolchain}" -${host_toolchain}/bin/swift --version +if [[ ! -z "${host_toolchain}" ]]; then + echo "Host toolchain found at ${host_toolchain}" + ${host_toolchain}/bin/swift --version +fi echo "Android NDK found at ${ndk_home}" ${ndk_installation}/bin/clang --version echo "Building for ${archs}" @@ -262,11 +267,31 @@ function run() { for arch in $archs; do case $arch in - armv7) target_host="arm-linux-androideabi"; compiler_target_host="armv7a-linux-androideabi$android_api"; android_abi="armeabi-v7a" ;; - aarch64) target_host="aarch64-linux-android"; compiler_target_host="$target_host$android_api"; android_abi="arm64-v8a" ;; - x86_64) target_host="x86_64-linux-android"; compiler_target_host="$target_host$android_api"; android_abi="x86_64" ;; - x86) target_host="x86-linux-android"; compiler_target_host="$target_host$android_api"; android_abi="x86" ;; - *) echo "Unknown architecture '$1'"; usage; exit 0 ;; + armv7) + target_host="arm-linux-androideabi" + compiler_target_host="armv7a-linux-androideabi$android_api" + android_abi="armeabi-v7a" + ;; + aarch64) + target_host="aarch64-linux-android" + compiler_target_host="$target_host$android_api" + android_abi="arm64-v8a" + ;; + x86_64) + target_host="x86_64-linux-android" + compiler_target_host="$target_host$android_api" + android_abi="x86_64" + ;; + x86) + target_host="x86-linux-android" + compiler_target_host="$target_host$android_api" + android_abi="x86" + ;; + *) + echo "Unknown architecture '$1'" + usage + exit 0 + ;; esac sdk_root=${build_dir}/sdk_root/${arch} @@ -378,35 +403,62 @@ for arch in $archs; do RelWithDebInfo) build_type_flag="--release-debuginfo" ;; esac - # use an out-of-tree build folder, otherwise subsequent arch builds have conflicts - export SWIFT_BUILD_ROOT=${build_dir}/$arch/swift-project + case $build_compiler in + 1|true|yes|YES) + build_cmark="" + local_build="" + build_llvm="1" + build_swift_tools="1" + validation_test="1" + native_swift_tools_path="" + native_clang_tools_path="" + ;; + *) + build_cmark="--skip-build-cmark" + local_build="--skip-local-build" + build_llvm="0" + build_swift_tools="0" + validation_test="0" + native_swift_tools_path="--native-swift-tools-path=$host_toolchain/bin" + native_clang_tools_path="--native-clang-tools-path=$host_toolchain/bin" + ;; + esac + + # use an out-of-tree build folder + export SWIFT_BUILD_ROOT=${build_dir}/swift-project ./swift/utils/build-script \ $build_type_flag \ --reconfigure \ --no-assertions \ + --validation-test=$validation_test \ --android \ --android-ndk=$ndk_home \ --android-arch=$arch \ --android-api-level=$android_api \ - --native-swift-tools-path=$host_toolchain/bin \ - --native-clang-tools-path=$host_toolchain/bin \ --cross-compile-hosts=android-$arch \ --cross-compile-deps-path=$sdk_root \ --install-destdir=$sdk_root \ - --build-llvm=0 \ - --build-swift-tools=0 \ - --skip-build-cmark \ - --skip-local-build \ + --build-llvm=$build_llvm \ + --build-swift-tools=$build_swift_tools \ + ${native_swift_tools_path} \ + ${native_clang_tools_path} \ + ${build_cmark} \ + ${local_build} \ + --host-test \ + --skip-test-linux \ + --skip-test-xctest --skip-test-foundation \ --build-swift-static-stdlib \ + --swift-install-components='compiler;clang-resource-dir-symlink;license;stdlib;sdk-overlay' \ --install-swift \ --install-libdispatch \ --install-foundation \ --xctest --install-xctest \ --swift-testing --install-swift-testing \ - --cross-compile-append-host-target-to-destdir=False \ - --extra-cmake-options='-DCMAKE_EXTRA_LINK_FLAGS="-Wl,-z,max-page-size=16384"' - + --cross-compile-build-swift-tools=0 \ + --llvm-ninja-targets-for-cross-compile-hosts=help \ + --cross-compile-append-host-target-to-destdir=False + # --extra-cmake-options='-DCMAKE_EXTRA_LINK_FLAGS="-Wl,-z,max-page-size=16384"' # need to remove symlink that gets created in the NDK to the previous arch's build # or else we get errors like: # error: could not find module '_Builtin_float' for target 'x86_64-unknown-linux-android'; found: aarch64-unknown-linux-android, at: /home/runner/work/_temp/swift-android-sdk/ndk/android-ndk-r27c/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/swift/android/_Builtin_float.swiftmodule @@ -477,7 +529,9 @@ for arch in $archs; do arch_triple="arm-linux-androideabi" fi - rm -r lib/swift{,_static}/clang + # need force rm in case linux is not present (when not running tests) + rm -rf lib/swift{,_static}/{FrameworkABIBaseline,_InternalSwiftScan,_InternalSwiftStaticMirror,clang,embedded,host,linux,migrator} + rm -rf lib/lib*.so mv lib/swift lib/swift-$arch ln -s ../swift/clang lib/swift-$arch/clang @@ -622,7 +676,7 @@ cat > swift-toolset.json </dev/null groupend +# Fetch yams (needed for Swift 6.1.x) +groupstart "Fetching yams" +pushd swift-project >/dev/null +[[ -d yams ]] || git clone ${github}jpsim/Yams.git yams +pushd yams >/dev/null 2>&1 +git checkout ${YAMS_VERSION} +popd >/dev/null 2>&1 +popd >/dev/null +groupend + # Fetch libxml2 groupstart "Fetching libxml2" [[ -d libxml2 ]] || git clone ${github}GNOME/libxml2.git @@ -182,10 +192,3 @@ git checkout ${BORINGSSL_VERSION} popd >/dev/null 2>&1 groupend -# Fetch BoringSSL -groupstart "Fetching Patches" -[[ -d swift-android-patches ]] || git clone https://github.com/swift-android-sdk/swift-android-sdk swift-android-patches -pushd swift-android-patches >/dev/null 2>&1 -git checkout ${SWIFT_ANDROID_PATCHES_VERSION} -popd >/dev/null 2>&1 -groupend diff --git a/swift-ci/sdks/android/scripts/toolchain-vars.sh b/swift-ci/sdks/android/scripts/toolchain-vars.sh index 39868951..4bf5b5ed 100644 --- a/swift-ci/sdks/android/scripts/toolchain-vars.sh +++ b/swift-ci/sdks/android/scripts/toolchain-vars.sh @@ -13,14 +13,16 @@ # obtain the latest toolchain for building. OS=$(echo $HOST_OS | tr -d '.') +# e.g., "swift-6.1-RELEASE" +# there is no latest-build.yml for releases, so we need to get it from the API +RELEASE_TAG=$(curl -fsSL https://www.swift.org/api/v1/install/releases.json | jq -r '.[-1].tag') +# e.g., "swift-6.1-release" +RELEASE_BRANCH=$(echo "${RELEASE_TAG}" | tr '[A-Z]' '[a-z]') case "${BUILD_SCHEME}" in release) - # e.g., "swift-6.1-RELEASE" - # there is no latest-build.yml for releases, so we need to get it from the API - export SWIFT_TAG=$(curl -fsSL https://www.swift.org/api/v1/install/releases.json | jq -r '.[-1].tag') - # e.g., "swift-6.1-release" - export SWIFT_BRANCH=$(echo "${SWIFT_TAG}" | tr '[A-Z]' '[a-z]') + export SWIFT_TAG=$RELEASE_TAG + export SWIFT_BRANCH=$RELEASE_BRANCH ;; development|swift-*-branch) # e.g., swift-6.2-DEVELOPMENT-SNAPSHOT-2025-05-15-a @@ -35,5 +37,11 @@ case "${BUILD_SCHEME}" in esac SWIFT_BASE=$SWIFT_TAG-$HOST_OS -export SWIFT_TOOLCHAIN_URL="https://download.swift.org/$SWIFT_BRANCH/$OS/$SWIFT_TAG/$SWIFT_BASE.tar.gz" - +case $BUILD_COMPILER in + 1|true|yes|YES) + export SWIFT_TOOLCHAIN_URL="https://download.swift.org/$RELEASE_BRANCH/$OS/$RELEASE_TAG/$RELEASE_TAG-$HOST_OS.tar.gz" + ;; + *) + export SWIFT_TOOLCHAIN_URL="https://download.swift.org/$SWIFT_BRANCH/$OS/$SWIFT_TAG/$SWIFT_BASE.tar.gz" + ;; +esac From 71b7131dc55a92b03750b84bfb59746f80bcb6e2 Mon Sep 17 00:00:00 2001 From: finagolfin Date: Mon, 30 Jun 2025 11:50:35 +0530 Subject: [PATCH 27/44] Build compiler-validated bundles from latest branch commits, not older tags Also update patches, particularly to disable failing tests from the compiler validation suite --- swift-ci/sdks/android/build-docker | 8 +++ .../sdks/android/patches/apply-patches.sh | 3 +- ...elease.patch => swift-android-devel.patch} | 72 +++++++++---------- .../swift-android-testing-release.patch | 61 ---------------- .../swift-android-trunk-libdispatch.patch | 38 +++++----- .../sdks/android/patches/swift-android.patch | 54 ++++++++++++++ 6 files changed, 112 insertions(+), 124 deletions(-) rename swift-ci/sdks/android/patches/{swift-android-testing-except-release.patch => swift-android-devel.patch} (74%) delete mode 100644 swift-ci/sdks/android/patches/swift-android-testing-release.patch diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index e8f1f3f1..8d1a8c4c 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -26,7 +26,15 @@ HOST_OS=ubuntu24.04 source ./scripts/toolchain-vars.sh # Check-out and patch the sources +if [[ "${BUILD_COMPILER}" != "1" ]]; then ./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-tag ${SWIFT_TAG} +else +if [[ "${BUILD_SCHEME}" == "development" ]]; then +./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-scheme main +else +./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-scheme release/6.2 +fi +fi ./patches/apply-patches.sh ${WORKDIR}/source/swift-project mkdir -p ${WORKDIR}/products diff --git a/swift-ci/sdks/android/patches/apply-patches.sh b/swift-ci/sdks/android/patches/apply-patches.sh index 36f234f1..dabb8e93 100755 --- a/swift-ci/sdks/android/patches/apply-patches.sh +++ b/swift-ci/sdks/android/patches/apply-patches.sh @@ -6,12 +6,11 @@ cd ${1} case "${BUILD_SCHEME}" in swift-*-branch) git apply -v -C1 ${patches_dir}/swift-android.patch - git apply -v -C1 ${patches_dir}/swift-android-testing-except-release.patch + git apply -v -C1 ${patches_dir}/swift-android-devel.patch ;; development) git apply -v -C1 ${patches_dir}/swift-android.patch git apply -v -C1 ${patches_dir}/swift-android-trunk-libdispatch.patch - git apply -v -C1 ${patches_dir}/swift-android-testing-except-release.patch ;; *) echo "$0: invalid BUILD_SCHEME=${BUILD_SCHEME}" diff --git a/swift-ci/sdks/android/patches/swift-android-testing-except-release.patch b/swift-ci/sdks/android/patches/swift-android-devel.patch similarity index 74% rename from swift-ci/sdks/android/patches/swift-android-testing-except-release.patch rename to swift-ci/sdks/android/patches/swift-android-devel.patch index 4d6fea0d..60dd9615 100644 --- a/swift-ci/sdks/android/patches/swift-android-testing-except-release.patch +++ b/swift-ci/sdks/android/patches/swift-android-devel.patch @@ -1,16 +1,23 @@ +commit 2e87eb24f85f3e84bff78051fba4870a5c061ec0 +Author: finagolfin +Date: Mon Jun 30 10:39:26 2025 + + [test] Fix or disable tests for 32-bit platforms (#82501) + + Fix two IRGen tests that are failing on Android armv7 and disable eight ClangImporter, C++ Interop, and SILOptimizer tests, two of which that were already failing on other 32-bit platforms. + diff --git a/swift/test/ClangImporter/pcm-emit-direct-cc1-mode.swift b/swift/test/ClangImporter/pcm-emit-direct-cc1-mode.swift -index f5048b9e3d5..c227ef7b77d 100644 +index f5048b9e3d5..6bc26ca3177 100644 --- a/swift/test/ClangImporter/pcm-emit-direct-cc1-mode.swift +++ b/swift/test/ClangImporter/pcm-emit-direct-cc1-mode.swift -@@ -9,7 +9,7 @@ - // CHECK-DUMP: Module map file: {{.*[/\\]}}Inputs{{/|\\}}custom-modules{{/|\\}}module.modulemap - - // Verify that the clang command-line used is cc1 --// RUN: %FileCheck -check-prefix CHECK-CLANG -DTRIPLE=%target-triple %s < %t.diags.txt -+// RUN: %FileCheck -check-prefix CHECK-CLANG -DTRIPLE=%module-target-triple %s < %t.diags.txt - // CHECK-CLANG: clang importer cc1 args - // CHECK-CLANG-SAME: '-o' '{{.*[/\\]}}script.pcm' '-fsyntax-only' '-x' 'objective-c' '{{.*[/\\]}}module.modulemap' '-triple' '[[TRIPLE]]' +@@ -15,5 +15,7 @@ // CHECK-CLANG-SAME: '-fmodules' + // CHECK-CLANG-NOT: clang importer driver args + ++// XFAIL: OS=linux-androideabi ++ + import script + var _ : ScriptTy diff --git a/swift/test/IRGen/abitypes_arm.swift b/swift/test/IRGen/abitypes_arm.swift index 9428294483d..52bd8b10145 100644 --- a/swift/test/IRGen/abitypes_arm.swift @@ -111,6 +118,23 @@ index f7f2664b2de..01d36e127c6 100644 // CHECK-DISABLED-NOT: normal_count +diff --git a/swift/utils/swift_build_support/swift_build_support/cmake.py b/swift/utils/swift_build_support/swift_build_support/cmake.py +index bfb69965890..b5e9f5349c2 100644 +--- a/swift/utils/swift_build_support/swift_build_support/cmake.py ++++ b/swift/utils/swift_build_support/swift_build_support/cmake.py +@@ -153,10 +153,8 @@ class CMake(object): + toolchain_path = product.native_toolchain_path(args.host_target) + cmake_swiftc_path = os.getenv('CMAKE_Swift_COMPILER', + os.path.join(toolchain_path, 'bin', 'swiftc')) +- define("CMAKE_C_COMPILER:PATH", os.path.join(toolchain_path, +- 'bin', 'clang')) +- define("CMAKE_CXX_COMPILER:PATH", os.path.join(toolchain_path, +- 'bin', 'clang++')) ++ define("CMAKE_C_COMPILER:PATH", toolchain.cc) ++ define("CMAKE_CXX_COMPILER:PATH", toolchain.cxx) + define("CMAKE_Swift_COMPILER:PATH", cmake_swiftc_path) + else: + cmake_swiftc_path = os.getenv('CMAKE_Swift_COMPILER', toolchain.swiftc) diff --git a/swift/utils/swift_build_support/swift_build_support/products/product.py b/swift/utils/swift_build_support/swift_build_support/products/product.py index 274b6037305..bf0f7fcf671 100644 --- a/swift/utils/swift_build_support/swift_build_support/products/product.py @@ -124,33 +148,3 @@ index 274b6037305..bf0f7fcf671 100644 install_destdir = os.path.join(install_destdir, self.args.host_target) return targets.toolchain_path(install_destdir, self.args.install_prefix) -diff --git a/swift-corelibs-xctest/cmake/modules/PlatformInfo.cmake b/swift-corelibs-xctest/cmake/modules/PlatformInfo.cmake -index 468dc61..a90ae3d 100644 ---- a/swift-corelibs-xctest/cmake/modules/PlatformInfo.cmake -+++ b/swift-corelibs-xctest/cmake/modules/PlatformInfo.cmake -@@ -2,6 +2,10 @@ - set(print_target_info_invocation "${CMAKE_Swift_COMPILER}" -print-target-info) - if(CMAKE_Swift_COMPILER_TARGET) - list(APPEND print_target_info_invocation -target ${CMAKE_Swift_COMPILER_TARGET}) -+else() -+ set(arg_list ${CMAKE_Swift_FLAGS}) -+ separate_arguments(arg_list) -+ list(APPEND print_target_info_invocation ${arg_list}) - endif() - execute_process(COMMAND ${print_target_info_invocation} OUTPUT_VARIABLE target_info_json) - message(CONFIGURE_LOG "Swift Target Info: ${print_target_info_invocation}\n" -diff --git a/swift-testing/cmake/modules/PlatformInfo.cmake b/swift-testing/cmake/modules/PlatformInfo.cmake -index 94c60ef..ab8a491 100644 ---- a/swift-testing/cmake/modules/PlatformInfo.cmake -+++ b/swift-testing/cmake/modules/PlatformInfo.cmake -@@ -9,6 +9,10 @@ - set(print_target_info_invocation "${CMAKE_Swift_COMPILER}" -print-target-info) - if(CMAKE_Swift_COMPILER_TARGET) - list(APPEND print_target_info_invocation -target ${CMAKE_Swift_COMPILER_TARGET}) -+else() -+ set(arg_list ${CMAKE_Swift_FLAGS}) -+ separate_arguments(arg_list) -+ list(APPEND print_target_info_invocation ${arg_list}) - endif() - execute_process(COMMAND ${print_target_info_invocation} OUTPUT_VARIABLE target_info_json) - message(CONFIGURE_LOG "Swift Target Info: ${print_target_info_invocation}\n" diff --git a/swift-ci/sdks/android/patches/swift-android-testing-release.patch b/swift-ci/sdks/android/patches/swift-android-testing-release.patch deleted file mode 100644 index 923d1194..00000000 --- a/swift-ci/sdks/android/patches/swift-android-testing-release.patch +++ /dev/null @@ -1,61 +0,0 @@ -diff --git a/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py b/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py -index b6bfa5760ef..324d1a77eea 100644 ---- a/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py -+++ b/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py -@@ -51,7 +51,7 @@ class SwiftTesting(product.Product): - return False - - def should_install(self, host_target): -- return self.args.install_swift_testing_macros -+ return self.args.install_swift_testing - - def _cmake_product(self, host_target): - build_root = os.path.dirname(self.build_dir) -diff --git a/swift-corelibs-foundation/Sources/CoreFoundation/CFPlatform.c b/swift-corelibs-foundation/Sources/CoreFoundation/CFPlatform.c -index 90f4aa78..0429425b 100644 ---- a/swift-corelibs-foundation/Sources/CoreFoundation/CFPlatform.c -+++ b/swift-corelibs-foundation/Sources/CoreFoundation/CFPlatform.c -@@ -1948,7 +1948,7 @@ CF_CROSS_PLATFORM_EXPORT void *_CFReallocf(void *ptr, size_t size) { - #endif - } - --#if TARGET_OS_ANDROID -+#if TARGET_OS_ANDROID && __ANDROID_API__ < 28 - - #include - #include -@@ -2277,6 +2277,10 @@ CF_EXPORT int _CFPosixSpawnFileActionsAddClose(_CFPosixSpawnFileActionsRef file_ - return _CFPosixSpawnFileActionsAddCloseImpl(file_actions, filedes); - } - -+CF_EXPORT int _CFPosixSpawnFileActionsChdir(_CFPosixSpawnFileActionsRef file_actions, const char *path) { -+ return ENOSYS; -+} -+ - CF_EXPORT int _CFPosixSpawn(pid_t *_CF_RESTRICT pid, const char *_CF_RESTRICT path, _CFPosixSpawnFileActionsRef file_actions, _CFPosixSpawnAttrRef _Nullable _CF_RESTRICT attrp, char *_Nullable const argv[_Nullable _CF_RESTRICT], char *_Nullable const envp[_Nullable _CF_RESTRICT]) { - _CFPosixSpawnInitialize(); - return _CFPosixSpawnImpl(pid, path, file_actions, attrp, argv, envp); -@@ -2279,7 +2279,7 @@ CF_EXPORT int _CFPosixSpawnFileActionsAddClose(_CFPosixSpawnFileActionsRef file_ - } - - CF_EXPORT int _CFPosixSpawnFileActionsChdir(_CFPosixSpawnFileActionsRef file_actions, const char *path) { -- #if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 29) -+ #if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 29) || (defined(__ANDROID__) && __ANDROID_API__ < 34) - // Glibc versions prior to 2.29 don't support posix_spawn_file_actions_addchdir_np, impacting: - // - Amazon Linux 2 (EoL mid-2025) - return ENOSYS; -diff --git a/swift-testing/cmake/modules/SwiftModuleInstallation.cmake b/swift-testing/cmake/modules/SwiftModuleInstallation.cmake -index 1553725..d9f9e5b 100644 ---- a/swift-testing/cmake/modules/SwiftModuleInstallation.cmake -+++ b/swift-testing/cmake/modules/SwiftModuleInstallation.cmake -@@ -75,6 +75,10 @@ function(_swift_testing_install_target module) - set(module_triple_command "${CMAKE_Swift_COMPILER}" -print-target-info) - if(CMAKE_Swift_COMPILER_TARGET) - list(APPEND module_triple_command -target ${CMAKE_Swift_COMPILER_TARGET}) -+ else() -+ set(arg_list ${CMAKE_Swift_FLAGS}) -+ separate_arguments(arg_list) -+ list(APPEND module_triple_command ${arg_list}) - endif() - execute_process(COMMAND ${module_triple_command} OUTPUT_VARIABLE target_info_json) - string(JSON module_triple GET "${target_info_json}" "target" "moduleTriple") diff --git a/swift-ci/sdks/android/patches/swift-android-trunk-libdispatch.patch b/swift-ci/sdks/android/patches/swift-android-trunk-libdispatch.patch index 4e824fad..ffec4ff6 100644 --- a/swift-ci/sdks/android/patches/swift-android-trunk-libdispatch.patch +++ b/swift-ci/sdks/android/patches/swift-android-trunk-libdispatch.patch @@ -10,30 +10,24 @@ index 3f36fc979cb..8713ae8fea8 100755 "grep", "--extended-regexp", "--recursive", -diff --git a/swift/test/Reflection/conformance_descriptors.swift b/swift/test/Reflection/conformance_descriptors.swift -index b114f764552..e1bd98c08f4 100644 ---- a/swift/test/Reflection/conformance_descriptors.swift -+++ b/swift/test/Reflection/conformance_descriptors.swift -@@ -5,6 +5,7 @@ - // - // Temporarily disable on AArch64 Linux (rdar://88451721) - // UNSUPPORTED: OS=linux-gnu && CPU=aarch64 -+// XFAIL: OS=linux-android +diff --git a/swift/utils/swift_build_support/swift_build_support/cmake.py b/swift/utils/swift_build_support/swift_build_support/cmake.py +index 9a0f5b43e2d..5c7d1320a2d 100644 +--- a/swift/utils/swift_build_support/swift_build_support/cmake.py ++++ b/swift/utils/swift_build_support/swift_build_support/cmake.py +@@ -150,11 +150,8 @@ class CMake(object): + define("CMAKE_CXX_COMPILER_LAUNCHER:PATH", args.cmake_cxx_launcher) - // rdar://100558042 - // UNSUPPORTED: CPU=arm64e -diff --git a/swift/test/Reflection/typeref_decoding.swift b/swift/test/Reflection/typeref_decoding.swift -index e0a3b4465a3..c01af893eda 100644 ---- a/swift/test/Reflection/typeref_decoding.swift -+++ b/swift/test/Reflection/typeref_decoding.swift -@@ -10,6 +10,7 @@ + if self.prefer_native_toolchain and product: +- clang_tools_path = product.native_clang_tools_path(args.host_target) +- define("CMAKE_C_COMPILER:PATH", os.path.join(clang_tools_path, +- 'bin', 'clang')) +- define("CMAKE_CXX_COMPILER:PATH", os.path.join(clang_tools_path, +- 'bin', 'clang++')) ++ define("CMAKE_C_COMPILER:PATH", toolchain.cc) ++ define("CMAKE_CXX_COMPILER:PATH", toolchain.cxx) - // FIXME: rdar://127796117 - // UNSUPPORTED: OS=linux-gnu && CPU=aarch64 -+// XFAIL: OS=linux-android - - // RUN: %target-build-swift -target %target-swift-5.2-abi-triple -Xfrontend -enable-anonymous-context-mangled-names %S/Inputs/ConcreteTypes.swift %S/Inputs/GenericTypes.swift %S/Inputs/Protocols.swift %S/Inputs/Extensions.swift %S/Inputs/Closures.swift -parse-as-library -emit-module -emit-library %no-fixup-chains -module-name TypesToReflect -o %t/%target-library-name(TypesToReflect) - // RUN: %target-build-swift -target %target-swift-5.2-abi-triple -Xfrontend -enable-anonymous-context-mangled-names %S/Inputs/ConcreteTypes.swift %S/Inputs/GenericTypes.swift %S/Inputs/Protocols.swift %S/Inputs/Extensions.swift %S/Inputs/Closures.swift %S/Inputs/main.swift -emit-module -emit-executable %no-fixup-chains -module-name TypesToReflect -o %t/TypesToReflect + toolchain_path = product.native_toolchain_path(args.host_target) + cmake_swiftc_path = os.getenv('CMAKE_Swift_COMPILER', diff --git a/swift-corelibs-libdispatch/cmake/modules/SwiftSupport.cmake b/swift-corelibs-libdispatch/cmake/modules/SwiftSupport.cmake index 3da519e..e2dd8ed 100644 --- a/swift-corelibs-libdispatch/cmake/modules/SwiftSupport.cmake diff --git a/swift-ci/sdks/android/patches/swift-android.patch b/swift-ci/sdks/android/patches/swift-android.patch index 2d465ea2..eeb84df8 100644 --- a/swift-ci/sdks/android/patches/swift-android.patch +++ b/swift-ci/sdks/android/patches/swift-android.patch @@ -1,3 +1,27 @@ +diff --git a/swift/test/SILOptimizer/concat_string_literals.32.swift b/swift/test/SILOptimizer/concat_string_literals.32.swift +index 01390761c8c..d84c142397c 100644 +--- a/swift/test/SILOptimizer/concat_string_literals.32.swift ++++ b/swift/test/SILOptimizer/concat_string_literals.32.swift +@@ -4,6 +4,7 @@ + + // We have a separate test for 64-bit architectures. + // REQUIRES: PTRSIZE=32 ++// XFAIL: OS=linux-androideabi + + // NOTE: 25185.byteSwapped = 0x62 'a', 0x61 'b' + // CHECK-LABEL: test_ascii_scalar_scalar2 +diff --git a/swift/test/SILOptimizer/semantic-arc-opts-redundantcopyopts.sil b/swift/test/SILOptimizer/semantic-arc-opts-redundantcopyopts.sil +index 98765e0e4a3..4dc68bf522b 100644 +--- a/swift/test/SILOptimizer/semantic-arc-opts-redundantcopyopts.sil ++++ b/swift/test/SILOptimizer/semantic-arc-opts-redundantcopyopts.sil +@@ -1,5 +1,7 @@ + // RUN: %target-sil-opt -module-name Swift -enable-sil-verify-all -semantic-arc-opts -sil-semantic-arc-peepholes-redundant-copyvalue-elim %s | %FileCheck %s + ++// XFAIL: OS=linux-android, OS=linux-androideabi ++ + // NOTE: Some of our tests here depend on borrow elimination /not/ running! + // Please do not add it to clean up the IR like we did in + // semanticarcopts-loadcopy-to-loadborrow! diff --git a/swift/utils/build-script-impl b/swift/utils/build-script-impl index 16e05052609..7ab8cebfab8 100755 --- a/swift/utils/build-script-impl @@ -59,6 +83,21 @@ index 7f290d16..95366592 100644 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) +diff --git a/swift-corelibs-xctest/cmake/modules/PlatformInfo.cmake b/swift-corelibs-xctest/cmake/modules/PlatformInfo.cmake +index 468dc61..a90ae3d 100644 +--- a/swift-corelibs-xctest/cmake/modules/PlatformInfo.cmake ++++ b/swift-corelibs-xctest/cmake/modules/PlatformInfo.cmake +@@ -2,6 +2,10 @@ + set(print_target_info_invocation "${CMAKE_Swift_COMPILER}" -print-target-info) + if(CMAKE_Swift_COMPILER_TARGET) + list(APPEND print_target_info_invocation -target ${CMAKE_Swift_COMPILER_TARGET}) ++else() ++ set(arg_list ${CMAKE_Swift_FLAGS}) ++ separate_arguments(arg_list) ++ list(APPEND print_target_info_invocation ${arg_list}) + endif() + execute_process(COMMAND ${print_target_info_invocation} OUTPUT_VARIABLE target_info_json) + message(CONFIGURE_LOG "Swift Target Info: ${print_target_info_invocation}\n" diff --git a/swift-foundation/cmake/modules/SwiftFoundationSwiftSupport.cmake b/swift-foundation/cmake/modules/SwiftFoundationSwiftSupport.cmake index cbdfc2a..bb4121d 100644 --- a/swift-foundation/cmake/modules/SwiftFoundationSwiftSupport.cmake @@ -86,6 +125,21 @@ index 1be9a4b..bd7b1bd 100644 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) +diff --git a/swift-testing/cmake/modules/PlatformInfo.cmake b/swift-testing/cmake/modules/PlatformInfo.cmake +index 94c60ef..ab8a491 100644 +--- a/swift-testing/cmake/modules/PlatformInfo.cmake ++++ b/swift-testing/cmake/modules/PlatformInfo.cmake +@@ -9,6 +9,10 @@ + set(print_target_info_invocation "${CMAKE_Swift_COMPILER}" -print-target-info) + if(CMAKE_Swift_COMPILER_TARGET) + list(APPEND print_target_info_invocation -target ${CMAKE_Swift_COMPILER_TARGET}) ++else() ++ set(arg_list ${CMAKE_Swift_FLAGS}) ++ separate_arguments(arg_list) ++ list(APPEND print_target_info_invocation ${arg_list}) + endif() + execute_process(COMMAND ${print_target_info_invocation} OUTPUT_VARIABLE target_info_json) + message(CONFIGURE_LOG "Swift Target Info: ${print_target_info_invocation}\n" diff --git a/swift-testing/cmake/modules/TargetTriple.cmake b/swift-testing/cmake/modules/TargetTriple.cmake index e087cc4..02f3a95 100644 --- a/swift-testing/cmake/modules/TargetTriple.cmake From 3f4cd4a77c253f26c8ccbb7e96b7ed313b24e29b Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Wed, 16 Jul 2025 17:45:24 -0400 Subject: [PATCH 28/44] gcpd 'Update patches' --- .../android/patches/swift-android-devel.patch | 14 ++++ .../swift-android-trunk-libdispatch.patch | 15 ---- .../sdks/android/patches/swift-android.patch | 80 ++----------------- 3 files changed, 19 insertions(+), 90 deletions(-) diff --git a/swift-ci/sdks/android/patches/swift-android-devel.patch b/swift-ci/sdks/android/patches/swift-android-devel.patch index 60dd9615..2b76b66e 100644 --- a/swift-ci/sdks/android/patches/swift-android-devel.patch +++ b/swift-ci/sdks/android/patches/swift-android-devel.patch @@ -118,6 +118,20 @@ index f7f2664b2de..01d36e127c6 100644 // CHECK-DISABLED-NOT: normal_count +diff --git a/swift/utils/build-script-impl b/swift/utils/build-script-impl +index 16e05052609..7ab8cebfab8 100755 +--- a/swift/utils/build-script-impl ++++ b/swift/utils/build-script-impl +@@ -828,6 +828,9 @@ function set_build_options_for_host() { + # and it must be the same for both LLVM and Swift + + if [[ "${SWIFT_HOST_TRIPLE}" ]] ; then ++ SWIFT_TARGET_CMAKE_OPTIONS+=( ++ -DCMAKE_Swift_COMPILER_TARGET:STRING="${SWIFT_HOST_TRIPLE}" ++ ) + llvm_cmake_options+=( + -DLLVM_HOST_TRIPLE:STRING="${SWIFT_HOST_TRIPLE}" + ) diff --git a/swift/utils/swift_build_support/swift_build_support/cmake.py b/swift/utils/swift_build_support/swift_build_support/cmake.py index bfb69965890..b5e9f5349c2 100644 --- a/swift/utils/swift_build_support/swift_build_support/cmake.py diff --git a/swift-ci/sdks/android/patches/swift-android-trunk-libdispatch.patch b/swift-ci/sdks/android/patches/swift-android-trunk-libdispatch.patch index ffec4ff6..f09bd748 100644 --- a/swift-ci/sdks/android/patches/swift-android-trunk-libdispatch.patch +++ b/swift-ci/sdks/android/patches/swift-android-trunk-libdispatch.patch @@ -28,21 +28,6 @@ index 9a0f5b43e2d..5c7d1320a2d 100644 toolchain_path = product.native_toolchain_path(args.host_target) cmake_swiftc_path = os.getenv('CMAKE_Swift_COMPILER', -diff --git a/swift-corelibs-libdispatch/cmake/modules/SwiftSupport.cmake b/swift-corelibs-libdispatch/cmake/modules/SwiftSupport.cmake -index 3da519e..e2dd8ed 100644 ---- a/swift-corelibs-libdispatch/cmake/modules/SwiftSupport.cmake -+++ b/swift-corelibs-libdispatch/cmake/modules/SwiftSupport.cmake -@@ -4,6 +4,10 @@ if(NOT dispatch_MODULE_TRIPLE) - set(module_triple_command "${CMAKE_Swift_COMPILER}" -print-target-info) - if(CMAKE_Swift_COMPILER_TARGET) - list(APPEND module_triple_command -target ${CMAKE_Swift_COMPILER_TARGET}) -+ else() -+ set(arg_list ${CMAKE_Swift_FLAGS}) -+ separate_arguments(arg_list) -+ list(APPEND module_triple_command ${arg_list}) - endif() - execute_process(COMMAND ${module_triple_command} OUTPUT_VARIABLE target_info_json) - diff --git a/swift-corelibs-libdispatch/src/swift/CMakeLists.txt b/swift-corelibs-libdispatch/src/swift/CMakeLists.txt index 38bef37..d0ddf98 100644 --- a/swift-corelibs-libdispatch/src/swift/CMakeLists.txt diff --git a/swift-ci/sdks/android/patches/swift-android.patch b/swift-ci/sdks/android/patches/swift-android.patch index eeb84df8..b0010510 100644 --- a/swift-ci/sdks/android/patches/swift-android.patch +++ b/swift-ci/sdks/android/patches/swift-android.patch @@ -10,18 +10,6 @@ index 01390761c8c..d84c142397c 100644 // NOTE: 25185.byteSwapped = 0x62 'a', 0x61 'b' // CHECK-LABEL: test_ascii_scalar_scalar2 -diff --git a/swift/test/SILOptimizer/semantic-arc-opts-redundantcopyopts.sil b/swift/test/SILOptimizer/semantic-arc-opts-redundantcopyopts.sil -index 98765e0e4a3..4dc68bf522b 100644 ---- a/swift/test/SILOptimizer/semantic-arc-opts-redundantcopyopts.sil -+++ b/swift/test/SILOptimizer/semantic-arc-opts-redundantcopyopts.sil -@@ -1,5 +1,7 @@ - // RUN: %target-sil-opt -module-name Swift -enable-sil-verify-all -semantic-arc-opts -sil-semantic-arc-peepholes-redundant-copyvalue-elim %s | %FileCheck %s - -+// XFAIL: OS=linux-android, OS=linux-androideabi -+ - // NOTE: Some of our tests here depend on borrow elimination /not/ running! - // Please do not add it to clean up the IR like we did in - // semanticarcopts-loadcopy-to-loadborrow! diff --git a/swift/utils/build-script-impl b/swift/utils/build-script-impl index 16e05052609..7ab8cebfab8 100755 --- a/swift/utils/build-script-impl @@ -48,15 +36,16 @@ index 324d1a77eea..e88601a8701 100644 from . import cmake_product from . import product -@@ -115,6 +117,22 @@ class SwiftTestingCMakeShim(cmake_product.CMakeProduct): +@@ -115,6 +117,24 @@ class SwiftTestingCMakeShim(cmake_product.CMakeProduct): # FIXME: If we build macros for the builder, specify the path. self.cmake_options.define('SwiftTesting_MACRO', 'NO') + if host_target.startswith('android') and self.is_cross_compile_target(host_target): + host_config = HostSpecificConfiguration(host_target, self.args) + self.cmake_options.extend(host_config.cmake_options) -+ flags = '-target %s-unknown-linux-android%s ' % (self.args.android_arch, -+ self.args.android_api_level) ++ triple = '%s-unknown-linux-android%s' % (self.args.android_arch, ++ self.args.android_api_level) ++ flags = '-target %s ' % (triple) + + flags += '-resource-dir %s/lib/swift ' % ( + self.host_install_destdir(host_target) + self.args.install_prefix) @@ -65,6 +54,7 @@ index 324d1a77eea..e88601a8701 100644 + flags += '-sdk %s/sysroot ' % (ndk_path) + flags += '-tools-directory %s/bin' % (ndk_path) + self.cmake_options.define('CMAKE_Swift_FLAGS', flags) ++ self.cmake_options.define('CMAKE_Swift_COMPILER_TARGET', triple) + self.cmake_options.define('CMAKE_CXX_COMPILER_WORKS', 'True') + self.cmake_options.define('CMAKE_FIND_ROOT_PATH', self.args.cross_compile_deps_path) + @@ -83,36 +73,6 @@ index 7f290d16..95366592 100644 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) -diff --git a/swift-corelibs-xctest/cmake/modules/PlatformInfo.cmake b/swift-corelibs-xctest/cmake/modules/PlatformInfo.cmake -index 468dc61..a90ae3d 100644 ---- a/swift-corelibs-xctest/cmake/modules/PlatformInfo.cmake -+++ b/swift-corelibs-xctest/cmake/modules/PlatformInfo.cmake -@@ -2,6 +2,10 @@ - set(print_target_info_invocation "${CMAKE_Swift_COMPILER}" -print-target-info) - if(CMAKE_Swift_COMPILER_TARGET) - list(APPEND print_target_info_invocation -target ${CMAKE_Swift_COMPILER_TARGET}) -+else() -+ set(arg_list ${CMAKE_Swift_FLAGS}) -+ separate_arguments(arg_list) -+ list(APPEND print_target_info_invocation ${arg_list}) - endif() - execute_process(COMMAND ${print_target_info_invocation} OUTPUT_VARIABLE target_info_json) - message(CONFIGURE_LOG "Swift Target Info: ${print_target_info_invocation}\n" -diff --git a/swift-foundation/cmake/modules/SwiftFoundationSwiftSupport.cmake b/swift-foundation/cmake/modules/SwiftFoundationSwiftSupport.cmake -index cbdfc2a..bb4121d 100644 ---- a/swift-foundation/cmake/modules/SwiftFoundationSwiftSupport.cmake -+++ b/swift-foundation/cmake/modules/SwiftFoundationSwiftSupport.cmake -@@ -37,6 +37,10 @@ function(_swift_foundation_install_target module) - set(module_triple_command "${CMAKE_Swift_COMPILER}" -print-target-info) - if(CMAKE_Swift_COMPILER_TARGET) - list(APPEND module_triple_command -target ${CMAKE_Swift_COMPILER_TARGET}) -+ else() -+ set(arg_list ${CMAKE_Swift_FLAGS}) -+ separate_arguments(arg_list) -+ list(APPEND module_triple_command ${arg_list}) - endif() - execute_process(COMMAND ${module_triple_command} OUTPUT_VARIABLE target_info_json) - string(JSON module_triple GET "${target_info_json}" "target" "moduleTriple") diff --git a/swift-testing/CMakeLists.txt b/swift-testing/CMakeLists.txt index 1be9a4b..bd7b1bd 100644 --- a/swift-testing/CMakeLists.txt @@ -125,33 +85,3 @@ index 1be9a4b..bd7b1bd 100644 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) -diff --git a/swift-testing/cmake/modules/PlatformInfo.cmake b/swift-testing/cmake/modules/PlatformInfo.cmake -index 94c60ef..ab8a491 100644 ---- a/swift-testing/cmake/modules/PlatformInfo.cmake -+++ b/swift-testing/cmake/modules/PlatformInfo.cmake -@@ -9,6 +9,10 @@ - set(print_target_info_invocation "${CMAKE_Swift_COMPILER}" -print-target-info) - if(CMAKE_Swift_COMPILER_TARGET) - list(APPEND print_target_info_invocation -target ${CMAKE_Swift_COMPILER_TARGET}) -+else() -+ set(arg_list ${CMAKE_Swift_FLAGS}) -+ separate_arguments(arg_list) -+ list(APPEND print_target_info_invocation ${arg_list}) - endif() - execute_process(COMMAND ${print_target_info_invocation} OUTPUT_VARIABLE target_info_json) - message(CONFIGURE_LOG "Swift Target Info: ${print_target_info_invocation}\n" -diff --git a/swift-testing/cmake/modules/TargetTriple.cmake b/swift-testing/cmake/modules/TargetTriple.cmake -index e087cc4..02f3a95 100644 ---- a/swift-testing/cmake/modules/TargetTriple.cmake -+++ b/swift-testing/cmake/modules/TargetTriple.cmake -@@ -10,6 +10,10 @@ - set(SWT_TARGET_INFO_COMMAND "${CMAKE_Swift_COMPILER}" -print-target-info) - if(CMAKE_Swift_COMPILER_TARGET) - list(APPEND SWT_TARGET_INFO_COMMAND -target ${CMAKE_Swift_COMPILER_TARGET}) -+else() -+ set(arg_list ${CMAKE_Swift_FLAGS}) -+ separate_arguments(arg_list) -+ list(APPEND SWT_TARGET_INFO_COMMAND ${arg_list}) - endif() - execute_process(COMMAND ${SWT_TARGET_INFO_COMMAND} OUTPUT_VARIABLE SWT_TARGET_INFO_JSON) - string(JSON SWT_TARGET_TRIPLE GET "${SWT_TARGET_INFO_JSON}" "target" "unversionedTriple") From bd0df119db6f505232d95619305a7b0d55c4fd6e Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Wed, 16 Jul 2025 20:50:00 -0400 Subject: [PATCH 29/44] Update patches and build matrix --- .../swift-android-trunk-libdispatch.patch | 30 ------------------- 1 file changed, 30 deletions(-) diff --git a/swift-ci/sdks/android/patches/swift-android-trunk-libdispatch.patch b/swift-ci/sdks/android/patches/swift-android-trunk-libdispatch.patch index f09bd748..d568fa59 100644 --- a/swift-ci/sdks/android/patches/swift-android-trunk-libdispatch.patch +++ b/swift-ci/sdks/android/patches/swift-android-trunk-libdispatch.patch @@ -1,33 +1,3 @@ -diff --git a/swift/test/Misc/verify-swift-feature-testing.test-sh b/swift/test/Misc/verify-swift-feature-testing.test-sh -index 3f36fc979cb..8713ae8fea8 100755 ---- a/swift/test/Misc/verify-swift-feature-testing.test-sh -+++ b/swift/test/Misc/verify-swift-feature-testing.test-sh -@@ -98,7 +98,6 @@ def find_matches(swift_src_root): - # `-enable-upcoming-feature` in the test directories. - output = subprocess.check_output( - [ -- "git", - "grep", - "--extended-regexp", - "--recursive", -diff --git a/swift/utils/swift_build_support/swift_build_support/cmake.py b/swift/utils/swift_build_support/swift_build_support/cmake.py -index 9a0f5b43e2d..5c7d1320a2d 100644 ---- a/swift/utils/swift_build_support/swift_build_support/cmake.py -+++ b/swift/utils/swift_build_support/swift_build_support/cmake.py -@@ -150,11 +150,8 @@ class CMake(object): - define("CMAKE_CXX_COMPILER_LAUNCHER:PATH", args.cmake_cxx_launcher) - - if self.prefer_native_toolchain and product: -- clang_tools_path = product.native_clang_tools_path(args.host_target) -- define("CMAKE_C_COMPILER:PATH", os.path.join(clang_tools_path, -- 'bin', 'clang')) -- define("CMAKE_CXX_COMPILER:PATH", os.path.join(clang_tools_path, -- 'bin', 'clang++')) -+ define("CMAKE_C_COMPILER:PATH", toolchain.cc) -+ define("CMAKE_CXX_COMPILER:PATH", toolchain.cxx) - - toolchain_path = product.native_toolchain_path(args.host_target) - cmake_swiftc_path = os.getenv('CMAKE_Swift_COMPILER', diff --git a/swift-corelibs-libdispatch/src/swift/CMakeLists.txt b/swift-corelibs-libdispatch/src/swift/CMakeLists.txt index 38bef37..d0ddf98 100644 --- a/swift-corelibs-libdispatch/src/swift/CMakeLists.txt From 80291aa9d0eadf2ef4ad46a08a6785dbda435681 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Wed, 16 Jul 2025 20:55:29 -0400 Subject: [PATCH 30/44] Disable compiler validated builds on self-hosted --- .github/workflows/pull_request.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index ad586a0d..82aaff59 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -28,14 +28,22 @@ jobs: fail-fast: false matrix: include: + #- swift-version: 'swift-6.2-branch' + # build-type: 'docker' + # build-compiler: '1' + # runner: 'self-hosted' + #- swift-version: 'development' + # build-type: 'docker' + # build-compiler: '1' + # runner: 'self-hosted' - swift-version: 'swift-6.2-branch' build-type: 'docker' - build-compiler: '1' - runner: 'self-hosted' + build-compiler: '0' + runner: 'ubuntu-24.04' - swift-version: 'development' build-type: 'docker' - build-compiler: '1' - runner: 'self-hosted' + build-compiler: '0' + runner: 'ubuntu-24.04' - swift-version: 'swift-6.2-branch' build-type: 'docker' build-compiler: '0' From 18c563ffb2fb8a9556c9dc79314997ce7800a2bd Mon Sep 17 00:00:00 2001 From: Finagolfin Date: Sun, 6 Jul 2025 16:33:59 +0530 Subject: [PATCH 31/44] Centralize cmake variable for 6.2 in patches, as done for trunk upstream --- .../android/patches/swift-android-devel.patch | 14 ++++ .../swift-android-trunk-libdispatch.patch | 15 ---- .../sdks/android/patches/swift-android.patch | 68 ++----------------- 3 files changed, 19 insertions(+), 78 deletions(-) diff --git a/swift-ci/sdks/android/patches/swift-android-devel.patch b/swift-ci/sdks/android/patches/swift-android-devel.patch index 60dd9615..2b76b66e 100644 --- a/swift-ci/sdks/android/patches/swift-android-devel.patch +++ b/swift-ci/sdks/android/patches/swift-android-devel.patch @@ -118,6 +118,20 @@ index f7f2664b2de..01d36e127c6 100644 // CHECK-DISABLED-NOT: normal_count +diff --git a/swift/utils/build-script-impl b/swift/utils/build-script-impl +index 16e05052609..7ab8cebfab8 100755 +--- a/swift/utils/build-script-impl ++++ b/swift/utils/build-script-impl +@@ -828,6 +828,9 @@ function set_build_options_for_host() { + # and it must be the same for both LLVM and Swift + + if [[ "${SWIFT_HOST_TRIPLE}" ]] ; then ++ SWIFT_TARGET_CMAKE_OPTIONS+=( ++ -DCMAKE_Swift_COMPILER_TARGET:STRING="${SWIFT_HOST_TRIPLE}" ++ ) + llvm_cmake_options+=( + -DLLVM_HOST_TRIPLE:STRING="${SWIFT_HOST_TRIPLE}" + ) diff --git a/swift/utils/swift_build_support/swift_build_support/cmake.py b/swift/utils/swift_build_support/swift_build_support/cmake.py index bfb69965890..b5e9f5349c2 100644 --- a/swift/utils/swift_build_support/swift_build_support/cmake.py diff --git a/swift-ci/sdks/android/patches/swift-android-trunk-libdispatch.patch b/swift-ci/sdks/android/patches/swift-android-trunk-libdispatch.patch index ffec4ff6..f09bd748 100644 --- a/swift-ci/sdks/android/patches/swift-android-trunk-libdispatch.patch +++ b/swift-ci/sdks/android/patches/swift-android-trunk-libdispatch.patch @@ -28,21 +28,6 @@ index 9a0f5b43e2d..5c7d1320a2d 100644 toolchain_path = product.native_toolchain_path(args.host_target) cmake_swiftc_path = os.getenv('CMAKE_Swift_COMPILER', -diff --git a/swift-corelibs-libdispatch/cmake/modules/SwiftSupport.cmake b/swift-corelibs-libdispatch/cmake/modules/SwiftSupport.cmake -index 3da519e..e2dd8ed 100644 ---- a/swift-corelibs-libdispatch/cmake/modules/SwiftSupport.cmake -+++ b/swift-corelibs-libdispatch/cmake/modules/SwiftSupport.cmake -@@ -4,6 +4,10 @@ if(NOT dispatch_MODULE_TRIPLE) - set(module_triple_command "${CMAKE_Swift_COMPILER}" -print-target-info) - if(CMAKE_Swift_COMPILER_TARGET) - list(APPEND module_triple_command -target ${CMAKE_Swift_COMPILER_TARGET}) -+ else() -+ set(arg_list ${CMAKE_Swift_FLAGS}) -+ separate_arguments(arg_list) -+ list(APPEND module_triple_command ${arg_list}) - endif() - execute_process(COMMAND ${module_triple_command} OUTPUT_VARIABLE target_info_json) - diff --git a/swift-corelibs-libdispatch/src/swift/CMakeLists.txt b/swift-corelibs-libdispatch/src/swift/CMakeLists.txt index 38bef37..d0ddf98 100644 --- a/swift-corelibs-libdispatch/src/swift/CMakeLists.txt diff --git a/swift-ci/sdks/android/patches/swift-android.patch b/swift-ci/sdks/android/patches/swift-android.patch index eeb84df8..7be06e86 100644 --- a/swift-ci/sdks/android/patches/swift-android.patch +++ b/swift-ci/sdks/android/patches/swift-android.patch @@ -48,15 +48,16 @@ index 324d1a77eea..e88601a8701 100644 from . import cmake_product from . import product -@@ -115,6 +117,22 @@ class SwiftTestingCMakeShim(cmake_product.CMakeProduct): +@@ -115,6 +117,24 @@ class SwiftTestingCMakeShim(cmake_product.CMakeProduct): # FIXME: If we build macros for the builder, specify the path. self.cmake_options.define('SwiftTesting_MACRO', 'NO') + if host_target.startswith('android') and self.is_cross_compile_target(host_target): + host_config = HostSpecificConfiguration(host_target, self.args) + self.cmake_options.extend(host_config.cmake_options) -+ flags = '-target %s-unknown-linux-android%s ' % (self.args.android_arch, -+ self.args.android_api_level) ++ triple = '%s-unknown-linux-android%s' % (self.args.android_arch, ++ self.args.android_api_level) ++ flags = '-target %s ' % (triple) + + flags += '-resource-dir %s/lib/swift ' % ( + self.host_install_destdir(host_target) + self.args.install_prefix) @@ -65,6 +66,7 @@ index 324d1a77eea..e88601a8701 100644 + flags += '-sdk %s/sysroot ' % (ndk_path) + flags += '-tools-directory %s/bin' % (ndk_path) + self.cmake_options.define('CMAKE_Swift_FLAGS', flags) ++ self.cmake_options.define('CMAKE_Swift_COMPILER_TARGET', triple) + self.cmake_options.define('CMAKE_CXX_COMPILER_WORKS', 'True') + self.cmake_options.define('CMAKE_FIND_ROOT_PATH', self.args.cross_compile_deps_path) + @@ -83,36 +85,6 @@ index 7f290d16..95366592 100644 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) -diff --git a/swift-corelibs-xctest/cmake/modules/PlatformInfo.cmake b/swift-corelibs-xctest/cmake/modules/PlatformInfo.cmake -index 468dc61..a90ae3d 100644 ---- a/swift-corelibs-xctest/cmake/modules/PlatformInfo.cmake -+++ b/swift-corelibs-xctest/cmake/modules/PlatformInfo.cmake -@@ -2,6 +2,10 @@ - set(print_target_info_invocation "${CMAKE_Swift_COMPILER}" -print-target-info) - if(CMAKE_Swift_COMPILER_TARGET) - list(APPEND print_target_info_invocation -target ${CMAKE_Swift_COMPILER_TARGET}) -+else() -+ set(arg_list ${CMAKE_Swift_FLAGS}) -+ separate_arguments(arg_list) -+ list(APPEND print_target_info_invocation ${arg_list}) - endif() - execute_process(COMMAND ${print_target_info_invocation} OUTPUT_VARIABLE target_info_json) - message(CONFIGURE_LOG "Swift Target Info: ${print_target_info_invocation}\n" -diff --git a/swift-foundation/cmake/modules/SwiftFoundationSwiftSupport.cmake b/swift-foundation/cmake/modules/SwiftFoundationSwiftSupport.cmake -index cbdfc2a..bb4121d 100644 ---- a/swift-foundation/cmake/modules/SwiftFoundationSwiftSupport.cmake -+++ b/swift-foundation/cmake/modules/SwiftFoundationSwiftSupport.cmake -@@ -37,6 +37,10 @@ function(_swift_foundation_install_target module) - set(module_triple_command "${CMAKE_Swift_COMPILER}" -print-target-info) - if(CMAKE_Swift_COMPILER_TARGET) - list(APPEND module_triple_command -target ${CMAKE_Swift_COMPILER_TARGET}) -+ else() -+ set(arg_list ${CMAKE_Swift_FLAGS}) -+ separate_arguments(arg_list) -+ list(APPEND module_triple_command ${arg_list}) - endif() - execute_process(COMMAND ${module_triple_command} OUTPUT_VARIABLE target_info_json) - string(JSON module_triple GET "${target_info_json}" "target" "moduleTriple") diff --git a/swift-testing/CMakeLists.txt b/swift-testing/CMakeLists.txt index 1be9a4b..bd7b1bd 100644 --- a/swift-testing/CMakeLists.txt @@ -125,33 +97,3 @@ index 1be9a4b..bd7b1bd 100644 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) -diff --git a/swift-testing/cmake/modules/PlatformInfo.cmake b/swift-testing/cmake/modules/PlatformInfo.cmake -index 94c60ef..ab8a491 100644 ---- a/swift-testing/cmake/modules/PlatformInfo.cmake -+++ b/swift-testing/cmake/modules/PlatformInfo.cmake -@@ -9,6 +9,10 @@ - set(print_target_info_invocation "${CMAKE_Swift_COMPILER}" -print-target-info) - if(CMAKE_Swift_COMPILER_TARGET) - list(APPEND print_target_info_invocation -target ${CMAKE_Swift_COMPILER_TARGET}) -+else() -+ set(arg_list ${CMAKE_Swift_FLAGS}) -+ separate_arguments(arg_list) -+ list(APPEND print_target_info_invocation ${arg_list}) - endif() - execute_process(COMMAND ${print_target_info_invocation} OUTPUT_VARIABLE target_info_json) - message(CONFIGURE_LOG "Swift Target Info: ${print_target_info_invocation}\n" -diff --git a/swift-testing/cmake/modules/TargetTriple.cmake b/swift-testing/cmake/modules/TargetTriple.cmake -index e087cc4..02f3a95 100644 ---- a/swift-testing/cmake/modules/TargetTriple.cmake -+++ b/swift-testing/cmake/modules/TargetTriple.cmake -@@ -10,6 +10,10 @@ - set(SWT_TARGET_INFO_COMMAND "${CMAKE_Swift_COMPILER}" -print-target-info) - if(CMAKE_Swift_COMPILER_TARGET) - list(APPEND SWT_TARGET_INFO_COMMAND -target ${CMAKE_Swift_COMPILER_TARGET}) -+else() -+ set(arg_list ${CMAKE_Swift_FLAGS}) -+ separate_arguments(arg_list) -+ list(APPEND SWT_TARGET_INFO_COMMAND ${arg_list}) - endif() - execute_process(COMMAND ${SWT_TARGET_INFO_COMMAND} OUTPUT_VARIABLE SWT_TARGET_INFO_JSON) - string(JSON SWT_TARGET_TRIPLE GET "${SWT_TARGET_INFO_JSON}" "target" "unversionedTriple") From ed09034cd0d61b32cbacd56b26a38fa8b5e823fe Mon Sep 17 00:00:00 2001 From: Finagolfin Date: Sat, 12 Jul 2025 15:26:31 +0530 Subject: [PATCH 32/44] Update disabled tests --- swift-ci/sdks/android/patches/swift-android.patch | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/swift-ci/sdks/android/patches/swift-android.patch b/swift-ci/sdks/android/patches/swift-android.patch index 7be06e86..b0010510 100644 --- a/swift-ci/sdks/android/patches/swift-android.patch +++ b/swift-ci/sdks/android/patches/swift-android.patch @@ -10,18 +10,6 @@ index 01390761c8c..d84c142397c 100644 // NOTE: 25185.byteSwapped = 0x62 'a', 0x61 'b' // CHECK-LABEL: test_ascii_scalar_scalar2 -diff --git a/swift/test/SILOptimizer/semantic-arc-opts-redundantcopyopts.sil b/swift/test/SILOptimizer/semantic-arc-opts-redundantcopyopts.sil -index 98765e0e4a3..4dc68bf522b 100644 ---- a/swift/test/SILOptimizer/semantic-arc-opts-redundantcopyopts.sil -+++ b/swift/test/SILOptimizer/semantic-arc-opts-redundantcopyopts.sil -@@ -1,5 +1,7 @@ - // RUN: %target-sil-opt -module-name Swift -enable-sil-verify-all -semantic-arc-opts -sil-semantic-arc-peepholes-redundant-copyvalue-elim %s | %FileCheck %s - -+// XFAIL: OS=linux-android, OS=linux-androideabi -+ - // NOTE: Some of our tests here depend on borrow elimination /not/ running! - // Please do not add it to clean up the IR like we did in - // semanticarcopts-loadcopy-to-loadborrow! diff --git a/swift/utils/build-script-impl b/swift/utils/build-script-impl index 16e05052609..7ab8cebfab8 100755 --- a/swift/utils/build-script-impl From f472e9eb94415d6fcc4663d62dffabee3301b6ce Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 17 Jul 2025 15:49:44 -0400 Subject: [PATCH 33/44] Try running Docker build on macOS host --- .github/workflows/pull_request.yml | 10 +++++++ swift-ci/sdks/android/Dockerfile | 1 + swift-ci/sdks/android/build-docker | 28 ++++++------------- swift-ci/sdks/android/build-local | 3 ++ swift-ci/sdks/android/scripts/build.sh | 1 + .../sdks/android/scripts/toolchain-vars.sh | 19 +++++++++++-- 6 files changed, 41 insertions(+), 21 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 82aaff59..5648736c 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -36,6 +36,16 @@ jobs: # build-type: 'docker' # build-compiler: '1' # runner: 'self-hosted' + - swift-version: 'swift-6.2-branch' + build-type: 'docker' + build-compiler: '0' + # Intel macOS + runner: 'macos-13' + - swift-version: 'swift-6.2-branch' + build-type: 'docker' + build-compiler: '0' + # ARM macOS + runner: 'macos-15' - swift-version: 'swift-6.2-branch' build-type: 'docker' build-compiler: '0' diff --git a/swift-ci/sdks/android/Dockerfile b/swift-ci/sdks/android/Dockerfile index 8708586a..deec9fd6 100644 --- a/swift-ci/sdks/android/Dockerfile +++ b/swift-ci/sdks/android/Dockerfile @@ -85,3 +85,4 @@ RUN groupadd -g 998 build-user && \ USER build-user WORKDIR /home/build-user + diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index 8d1a8c4c..802adf27 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -12,6 +12,9 @@ TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} ANDROID_NDK_VERSION=android-ndk-r27c ANDROID_API=28 +BASEPATH=$(dirname $(realpath $0)) +cd ${BASEPATH} + export BUILD_SCHEME=${1} # note that WORKDIR must not be under the current checkout or the patches will fail to apply WORKDIR=${2} @@ -27,13 +30,13 @@ source ./scripts/toolchain-vars.sh # Check-out and patch the sources if [[ "${BUILD_COMPILER}" != "1" ]]; then -./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-tag ${SWIFT_TAG} -else -if [[ "${BUILD_SCHEME}" == "development" ]]; then -./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-scheme main + ./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-tag ${SWIFT_TAG} else -./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-scheme release/6.2 -fi + if [[ "${BUILD_SCHEME}" == "development" ]]; then + ./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-scheme main + else + ./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-scheme release/6.2 + fi fi ./patches/apply-patches.sh ${WORKDIR}/source/swift-project @@ -44,19 +47,6 @@ if [[ "$DOCKER" == "" ]]; then DOCKER=docker fi -case $(arch) in - arm64|aarch64) - OS_ARCH_SUFFIX=-aarch64 - ;; - amd64|x86_64) - OS_ARCH_SUFFIX= - ;; - *) - echo "Unknown architecture $(arch)" - exit 1 - ;; -esac - CONTAINER_NAME="swift-android" # Build the Docker image diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index 31eaf1e3..2888ba6c 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -12,6 +12,9 @@ TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} ANDROID_NDK_VERSION=android-ndk-r27c ANDROID_API=28 +BASEPATH=$(dirname $(realpath $0)) +cd ${BASEPATH} + export BUILD_SCHEME=${1} # note that WORKDIR must not be under the current checkout or the patches will fail to apply WORKDIR=${2} diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index b388f025..a1d3b9c2 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -217,6 +217,7 @@ header "Swift Android SDK build script" swift_dir=$(realpath $(dirname "$swiftc")/..) HOST=linux-x86_64 +# The Linux NDK only supports x86 #HOST=$(uname -s -m | tr '[:upper:]' '[:lower:]' | tr ' ' '-') # in a Docker container, the pre-installed NDK is read-only, diff --git a/swift-ci/sdks/android/scripts/toolchain-vars.sh b/swift-ci/sdks/android/scripts/toolchain-vars.sh index 4bf5b5ed..d804dfe7 100644 --- a/swift-ci/sdks/android/scripts/toolchain-vars.sh +++ b/swift-ci/sdks/android/scripts/toolchain-vars.sh @@ -37,11 +37,26 @@ case "${BUILD_SCHEME}" in esac SWIFT_BASE=$SWIFT_TAG-$HOST_OS + +case $(arch) in + arm64|aarch64) + export OS_ARCH_SUFFIX=-aarch64 + ;; + amd64|x86_64) + export OS_ARCH_SUFFIX= + ;; + *) + echo "Unknown architecture $(arch)" + exit 1 + ;; +esac + + case $BUILD_COMPILER in 1|true|yes|YES) - export SWIFT_TOOLCHAIN_URL="https://download.swift.org/$RELEASE_BRANCH/$OS/$RELEASE_TAG/$RELEASE_TAG-$HOST_OS.tar.gz" + export SWIFT_TOOLCHAIN_URL="https://download.swift.org/$RELEASE_BRANCH/$OS$OS_ARCH_SUFFIX/$RELEASE_TAG/$RELEASE_TAG-$HOST_OS$OS_ARCH_SUFFIX.tar.gz" ;; *) - export SWIFT_TOOLCHAIN_URL="https://download.swift.org/$SWIFT_BRANCH/$OS/$SWIFT_TAG/$SWIFT_BASE.tar.gz" + export SWIFT_TOOLCHAIN_URL="https://download.swift.org/$SWIFT_BRANCH/$OS$OS_ARCH_SUFFIX/$SWIFT_TAG/$SWIFT_BASE$OS_ARCH_SUFFIX.tar.gz" ;; esac From 22a4320e016fd3735a8586c07ac2294a5ae17c83 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 17 Jul 2025 21:07:32 -0400 Subject: [PATCH 34/44] Remove separate checks of libxml2, curl, and yams --- .github/workflows/pull_request.yml | 10 ----- swift-ci/sdks/android/scripts/build.sh | 22 +++++----- swift-ci/sdks/android/scripts/fetch-source.sh | 42 ------------------- 3 files changed, 12 insertions(+), 62 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 5648736c..82aaff59 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -36,16 +36,6 @@ jobs: # build-type: 'docker' # build-compiler: '1' # runner: 'self-hosted' - - swift-version: 'swift-6.2-branch' - build-type: 'docker' - build-compiler: '0' - # Intel macOS - runner: 'macos-13' - - swift-version: 'swift-6.2-branch' - build-type: 'docker' - build-compiler: '0' - # ARM macOS - runner: 'macos-15' - swift-version: 'swift-6.2-branch' build-type: 'docker' build-compiler: '0' diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index a1d3b9c2..c28fa68f 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -188,8 +188,10 @@ function versionFromTag { fi } -swift_version=$(describe ${source_dir}/swift-project/swift) -swift_tag_date=$(git -C ${source_dir}/swift-project/swift log -1 --format=%ct 2>/dev/null) +swift_source_dir=${source_dir}/swift-project + +swift_version=$(describe ${swift_source_dir}/swift) +swift_tag_date=$(git -C ${swift_source_dir}/swift log -1 --format=%ct 2>/dev/null) if [[ $swift_version == swift-* ]]; then swift_version=${swift_version#swift-} @@ -199,9 +201,9 @@ if [[ -z "$sdk_name" ]]; then sdk_name=swift-${swift_version}-android-${android_sdk_version} fi -libxml2_version=$(versionFromTag ${source_dir}/libxml2) +libxml2_version=$(versionFromTag ${swift_source_dir}/libxml2) -curl_desc=$(describe ${source_dir}/curl | tr '_' '.') +curl_desc=$(describe ${swift_source_dir}/curl | tr '_' '.') curl_version=${curl_desc#curl-} boringssl_version=$(describe ${source_dir}/boringssl) @@ -299,10 +301,10 @@ for arch in $archs; do mkdir -p "$sdk_root" groupstart "Building libxml2 for $arch" - quiet_pushd ${source_dir}/libxml2 + quiet_pushd ${swift_source_dir}/libxml2 run cmake \ -G Ninja \ - -S ${source_dir}/libxml2 \ + -S ${swift_source_dir}/libxml2 \ -B ${build_dir}/$arch/libxml2 \ -DANDROID_ABI=$android_abi \ -DANDROID_PLATFORM=android-$android_api \ @@ -356,10 +358,10 @@ for arch in $archs; do groupend groupstart "Building libcurl for ${compiler_target_host}" - quiet_pushd ${source_dir}/curl + quiet_pushd ${swift_source_dir}/curl run cmake \ -G Ninja \ - -S ${source_dir}/curl \ + -S ${swift_source_dir}/curl \ -B ${build_dir}/$arch/curl \ -DANDROID_ABI=$android_abi \ -DANDROID_PLATFORM=android-$android_api \ @@ -396,7 +398,7 @@ for arch in $archs; do groupend groupstart "Building Android SDK for ${compiler_target_host}" - quiet_pushd ${source_dir}/swift-project + quiet_pushd ${swift_source_dir} build_type_flag="--debug" case $build_type in Debug) build_type_flag="--debug" ;; @@ -523,7 +525,7 @@ for arch in $archs; do quiet_pushd ${sdk_staging}/${arch}/usr rm -r bin rm -r include/* - cp -r ${source_dir}/swift-project/swift/lib/ClangImporter/SwiftBridging/{module.modulemap,swift} include/ + cp -r ${swift_source_dir}/swift/lib/ClangImporter/SwiftBridging/{module.modulemap,swift} include/ arch_triple="$arch-linux-android" if [[ $arch == 'armv7' ]]; then diff --git a/swift-ci/sdks/android/scripts/fetch-source.sh b/swift-ci/sdks/android/scripts/fetch-source.sh index 29f63fce..d07cb68b 100755 --- a/swift-ci/sdks/android/scripts/fetch-source.sh +++ b/swift-ci/sdks/android/scripts/fetch-source.sh @@ -52,7 +52,6 @@ function usage { cat <|--swift-tag |--swift-version ] - [--curl-version ] [--boringssl-version ] [--clone-with-ssh] [--source-dir ] @@ -70,8 +69,6 @@ SDK for Swift. Options are: If starts with "scheme:" or "tag:", it will select a scheme or tag; otherwise it will be treated as a version number. - --libxml2-version - --curl-version --boringssl-version EOF } @@ -80,18 +77,9 @@ EOF if [[ -z "${SWIFT_VERSION}" ]]; then SWIFT_VERSION=scheme:release/6.1 fi -if [[ -z "${LIBXML2_VERSION}" ]]; then - LIBXML2_VERSION=2.14.2 -fi -if [[ -z "${CURL_VERSION}" ]]; then - CURL_VERSION=8.13.0 -fi if [[ -z "${BORINGSSL_VERSION}" ]]; then BORINGSSL_VERSION=fips-20220613 fi -if [[ -z "${YAMS_VERSION}" ]]; then - YAMS_VERSION=5.0.6 -fi clone_with_ssh=false while [ "$#" -gt 0 ]; do @@ -102,10 +90,6 @@ while [ "$#" -gt 0 ]; do SWIFT_VERSION="tag:$2"; shift ;; --swift-version) SWIFT_VERSION="$2"; shift ;; - --libxml2-version) - LIBXML2_VERSION="$2"; shift ;; - --curl-version) - CURL_VERSION="$2"; shift ;; --boringssl-version) BORINGSSL_VERSION="$2"; shift ;; --clone-with-ssh) @@ -158,32 +142,6 @@ fi popd >/dev/null groupend -# Fetch yams (needed for Swift 6.1.x) -groupstart "Fetching yams" -pushd swift-project >/dev/null -[[ -d yams ]] || git clone ${github}jpsim/Yams.git yams -pushd yams >/dev/null 2>&1 -git checkout ${YAMS_VERSION} -popd >/dev/null 2>&1 -popd >/dev/null -groupend - -# Fetch libxml2 -groupstart "Fetching libxml2" -[[ -d libxml2 ]] || git clone ${github}GNOME/libxml2.git -pushd libxml2 >/dev/null 2>&1 -git checkout v${LIBXML2_VERSION} -popd >/dev/null 2>&1 -groupend - -# Fetch curl -groupstart "Fetching curl" -[[ -d curl ]] || git clone ${github}curl/curl.git -pushd curl >/dev/null 2>&1 -git checkout curl-$(echo ${CURL_VERSION} | tr '.' '_') -popd >/dev/null 2>&1 -groupend - # Fetch BoringSSL groupstart "Fetching BoringSSL" [[ -d boringssl ]] || git clone https://boringssl.googlesource.com/boringssl From a7b303afb22631ebb588b7a0a5d946ba8a54a5ed Mon Sep 17 00:00:00 2001 From: Finagolfin Date: Fri, 18 Jul 2025 15:36:54 +0530 Subject: [PATCH 35/44] Remove upstreamed 6.2 branch patches --- .../android/patches/swift-android-devel.patch | 147 ------------------ 1 file changed, 147 deletions(-) diff --git a/swift-ci/sdks/android/patches/swift-android-devel.patch b/swift-ci/sdks/android/patches/swift-android-devel.patch index 2b76b66e..c94eff10 100644 --- a/swift-ci/sdks/android/patches/swift-android-devel.patch +++ b/swift-ci/sdks/android/patches/swift-android-devel.patch @@ -1,137 +1,3 @@ -commit 2e87eb24f85f3e84bff78051fba4870a5c061ec0 -Author: finagolfin -Date: Mon Jun 30 10:39:26 2025 - - [test] Fix or disable tests for 32-bit platforms (#82501) - - Fix two IRGen tests that are failing on Android armv7 and disable eight ClangImporter, C++ Interop, and SILOptimizer tests, two of which that were already failing on other 32-bit platforms. - -diff --git a/swift/test/ClangImporter/pcm-emit-direct-cc1-mode.swift b/swift/test/ClangImporter/pcm-emit-direct-cc1-mode.swift -index f5048b9e3d5..6bc26ca3177 100644 ---- a/swift/test/ClangImporter/pcm-emit-direct-cc1-mode.swift -+++ b/swift/test/ClangImporter/pcm-emit-direct-cc1-mode.swift -@@ -15,5 +15,7 @@ - // CHECK-CLANG-SAME: '-fmodules' - // CHECK-CLANG-NOT: clang importer driver args - -+// XFAIL: OS=linux-androideabi -+ - import script - var _ : ScriptTy -diff --git a/swift/test/IRGen/abitypes_arm.swift b/swift/test/IRGen/abitypes_arm.swift -index 9428294483d..52bd8b10145 100644 ---- a/swift/test/IRGen/abitypes_arm.swift -+++ b/swift/test/IRGen/abitypes_arm.swift -@@ -11,6 +11,6 @@ class Foo { - } - } - --// armv7: define internal void @makeOne(ptr noalias sret({{.*}}) align 4 %agg.result, float{{( noundef)?}} %f, float{{( noundef)?}} %s) -+// armv7: define internal void @makeOne(ptr{{( dead_on_unwind)?}} noalias{{( writable)?}} sret({{.*}}) align 4 %agg.result, float{{( noundef)?}} %f, float{{( noundef)?}} %s) - // armv7s: define internal void @makeOne(ptr noalias sret({{.*}}) align 4 %agg.result, float %f, float %s) - // armv7k: define internal %struct.One @makeOne(float {{.*}}%f, float {{.*}}%s) -diff --git a/swift/test/Interop/Cxx/class/constructors-copy-module-interface.swift b/swift/test/Interop/Cxx/class/constructors-copy-module-interface.swift -index 042c10e7d1d..254f6543b58 100644 ---- a/swift/test/Interop/Cxx/class/constructors-copy-module-interface.swift -+++ b/swift/test/Interop/Cxx/class/constructors-copy-module-interface.swift -@@ -1,4 +1,5 @@ - // RUN: %target-swift-ide-test -print-module -module-to-print=Constructors -I %S/Inputs -source-filename=x -enable-experimental-cxx-interop | %FileCheck %s -+// XFAIL: OS=linux-androideabi - - // CHECK: struct TemplatedCopyConstructor - // CHECK: struct TemplatedCopyConstructorWithExtraArg -diff --git a/swift/test/Interop/Cxx/class/constructors-diagnostics.swift b/swift/test/Interop/Cxx/class/constructors-diagnostics.swift -index aac01480660..8fa99fae7fb 100644 ---- a/swift/test/Interop/Cxx/class/constructors-diagnostics.swift -+++ b/swift/test/Interop/Cxx/class/constructors-diagnostics.swift -@@ -2,6 +2,7 @@ - - // This test uses -verify-additional-file, which do not work well on Windows. - // UNSUPPORTED: OS=windows-msvc -+// XFAIL: OS=linux-androideabi - - import Constructors - -diff --git a/swift/test/Interop/Cxx/class/constructors-module-interface.swift b/swift/test/Interop/Cxx/class/constructors-module-interface.swift -index bf189b6e302..d2c3d65f03e 100644 ---- a/swift/test/Interop/Cxx/class/constructors-module-interface.swift -+++ b/swift/test/Interop/Cxx/class/constructors-module-interface.swift -@@ -1,4 +1,5 @@ - // RUN: %target-swift-ide-test -print-module -module-to-print=Constructors -I %S/Inputs/ -source-filename=x -enable-experimental-cxx-interop | %FileCheck %s -+// XFAIL: OS=linux-androideabi - - // CHECK: struct ExplicitDefaultConstructor { - // CHECK-NEXT: init() -diff --git a/swift/test/Interop/Cxx/class/constructors-silgen.swift b/swift/test/Interop/Cxx/class/constructors-silgen.swift -index b1624ccccfe..79cbdf170de 100644 ---- a/swift/test/Interop/Cxx/class/constructors-silgen.swift -+++ b/swift/test/Interop/Cxx/class/constructors-silgen.swift -@@ -1,4 +1,5 @@ - // RUN: %target-swiftxx-frontend -I %S/Inputs -Xllvm -sil-print-types -emit-silgen %s | %FileCheck %s -+// XFAIL: OS=linux-androideabi - - import Constructors - -diff --git a/swift/test/Interop/Cxx/class/constructors-typechecker.swift b/swift/test/Interop/Cxx/class/constructors-typechecker.swift -index 8ce10b1ca75..52ff4104b83 100644 ---- a/swift/test/Interop/Cxx/class/constructors-typechecker.swift -+++ b/swift/test/Interop/Cxx/class/constructors-typechecker.swift -@@ -1,4 +1,5 @@ - // RUN: %target-typecheck-verify-swift -verify-ignore-unknown -I %S/Inputs -enable-experimental-cxx-interop -+// XFAIL: OS=linux-androideabi - - import Constructors - -diff --git a/swift/test/Interop/Cxx/class/custom-new-operator-irgen.swift b/swift/test/Interop/Cxx/class/custom-new-operator-irgen.swift -index 74e692c9db2..0423493afac 100644 ---- a/swift/test/Interop/Cxx/class/custom-new-operator-irgen.swift -+++ b/swift/test/Interop/Cxx/class/custom-new-operator-irgen.swift -@@ -5,4 +5,4 @@ import CustomNewOperator - var x = callsCustomNew() - - // Make sure the definition of `operator new` is emitted. --// CHECK: define {{.*}} @{{_ZnwmPv15container_new_t|"\?\?2@YAPEAX_KPEAXUcontainer_new_t@@@Z"}} -+// CHECK: define {{.*}} @{{_Znw(j|m)Pv15container_new_t|"\?\?2@YAPEAX_KPEAXUcontainer_new_t@@@Z"}} -diff --git a/swift/test/SILOptimizer/package-cmo-serialize-tables.swift b/swift/test/SILOptimizer/package-cmo-serialize-tables.swift -index 382ff2e8080..f061374b2a1 100644 ---- a/swift/test/SILOptimizer/package-cmo-serialize-tables.swift -+++ b/swift/test/SILOptimizer/package-cmo-serialize-tables.swift -@@ -19,7 +19,7 @@ - - // Temporarily disabling on watchOS (both arm64_32 & armv7k): - // rdar://140330692 (🟠 OSS Swift CI: oss-swift_tools-RA_stdlib-DA_test-device-non_executable failed... --// UNSUPPORTED: OS=watchos -+// UNSUPPORTED: OS=watchos, OS=linux-androideabi - - //--- main.swift - -diff --git a/swift/test/SILOptimizer/throws_prediction.swift b/swift/test/SILOptimizer/throws_prediction.swift -index f7f2664b2de..01d36e127c6 100644 ---- a/swift/test/SILOptimizer/throws_prediction.swift -+++ b/swift/test/SILOptimizer/throws_prediction.swift -@@ -16,7 +16,7 @@ - // RUN: -sil-verify-all -module-name=test -emit-sil \ - // RUN: | %FileCheck --check-prefix CHECK-DISABLED %s - --// UNSUPPORTED: CPU=armv7k || CPU=arm64_32 -+// UNSUPPORTED: CPU=armv7k, CPU=arm64_32, CPU=armv7 - - // CHECK-DISABLED-NOT: normal_count - -diff --git a/swift/utils/build-script-impl b/swift/utils/build-script-impl -index 16e05052609..7ab8cebfab8 100755 ---- a/swift/utils/build-script-impl -+++ b/swift/utils/build-script-impl -@@ -828,6 +828,9 @@ function set_build_options_for_host() { - # and it must be the same for both LLVM and Swift - - if [[ "${SWIFT_HOST_TRIPLE}" ]] ; then -+ SWIFT_TARGET_CMAKE_OPTIONS+=( -+ -DCMAKE_Swift_COMPILER_TARGET:STRING="${SWIFT_HOST_TRIPLE}" -+ ) - llvm_cmake_options+=( - -DLLVM_HOST_TRIPLE:STRING="${SWIFT_HOST_TRIPLE}" - ) diff --git a/swift/utils/swift_build_support/swift_build_support/cmake.py b/swift/utils/swift_build_support/swift_build_support/cmake.py index bfb69965890..b5e9f5349c2 100644 --- a/swift/utils/swift_build_support/swift_build_support/cmake.py @@ -149,16 +15,3 @@ index bfb69965890..b5e9f5349c2 100644 define("CMAKE_Swift_COMPILER:PATH", cmake_swiftc_path) else: cmake_swiftc_path = os.getenv('CMAKE_Swift_COMPILER', toolchain.swiftc) -diff --git a/swift/utils/swift_build_support/swift_build_support/products/product.py b/swift/utils/swift_build_support/swift_build_support/products/product.py -index 274b6037305..bf0f7fcf671 100644 ---- a/swift/utils/swift_build_support/swift_build_support/products/product.py -+++ b/swift/utils/swift_build_support/swift_build_support/products/product.py -@@ -212,7 +212,7 @@ class Product(object): - if self.args.cross_compile_hosts: - if self.is_darwin_host(host_target): - install_destdir = self.host_install_destdir(host_target) -- else: -+ elif self.args.cross_compile_append_host_target_to_destdir: - install_destdir = os.path.join(install_destdir, self.args.host_target) - return targets.toolchain_path(install_destdir, - self.args.install_prefix) From 3f36409bb82ac1342e6daec6b237ca864d77f64c Mon Sep 17 00:00:00 2001 From: Finagolfin Date: Mon, 21 Jul 2025 09:53:20 +0530 Subject: [PATCH 36/44] Use command-line flags to work around CMake 3.30+ linker flag bug, rather than patching files --- .github/workflows/pull_request.yml | 8 ------ swift-ci/sdks/android/build-docker | 2 +- swift-ci/sdks/android/build-local | 2 +- .../sdks/android/patches/apply-patches.sh | 2 +- ...spatch.patch => swift-android-trunk.patch} | 12 --------- .../sdks/android/patches/swift-android.patch | 27 ++----------------- swift-ci/sdks/android/scripts/build.sh | 2 ++ 7 files changed, 7 insertions(+), 48 deletions(-) rename swift-ci/sdks/android/patches/{swift-android-trunk-libdispatch.patch => swift-android-trunk.patch} (75%) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 82aaff59..cc762b1c 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -44,14 +44,6 @@ jobs: build-type: 'docker' build-compiler: '0' runner: 'ubuntu-24.04' - - swift-version: 'swift-6.2-branch' - build-type: 'docker' - build-compiler: '0' - runner: 'ubuntu-24.04' - - swift-version: 'development' - build-type: 'docker' - build-compiler: '0' - runner: 'ubuntu-24.04' runs-on: ${{ matrix.runner }} # 15 hour timeout timeout-minutes: 900 diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index 802adf27..65d90507 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -9,7 +9,7 @@ # default architectures to build for TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} -ANDROID_NDK_VERSION=android-ndk-r27c +ANDROID_NDK_VERSION=android-ndk-r27d ANDROID_API=28 BASEPATH=$(dirname $(realpath $0)) diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index 2888ba6c..8e887ff2 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -9,7 +9,7 @@ # default architectures to build for TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} -ANDROID_NDK_VERSION=android-ndk-r27c +ANDROID_NDK_VERSION=android-ndk-r27d ANDROID_API=28 BASEPATH=$(dirname $(realpath $0)) diff --git a/swift-ci/sdks/android/patches/apply-patches.sh b/swift-ci/sdks/android/patches/apply-patches.sh index dabb8e93..38f5fa89 100755 --- a/swift-ci/sdks/android/patches/apply-patches.sh +++ b/swift-ci/sdks/android/patches/apply-patches.sh @@ -10,7 +10,7 @@ case "${BUILD_SCHEME}" in ;; development) git apply -v -C1 ${patches_dir}/swift-android.patch - git apply -v -C1 ${patches_dir}/swift-android-trunk-libdispatch.patch + git apply -v -C1 ${patches_dir}/swift-android-trunk.patch ;; *) echo "$0: invalid BUILD_SCHEME=${BUILD_SCHEME}" diff --git a/swift-ci/sdks/android/patches/swift-android-trunk-libdispatch.patch b/swift-ci/sdks/android/patches/swift-android-trunk.patch similarity index 75% rename from swift-ci/sdks/android/patches/swift-android-trunk-libdispatch.patch rename to swift-ci/sdks/android/patches/swift-android-trunk.patch index f09bd748..8ab486e1 100644 --- a/swift-ci/sdks/android/patches/swift-android-trunk-libdispatch.patch +++ b/swift-ci/sdks/android/patches/swift-android-trunk.patch @@ -28,15 +28,3 @@ index 9a0f5b43e2d..5c7d1320a2d 100644 toolchain_path = product.native_toolchain_path(args.host_target) cmake_swiftc_path = os.getenv('CMAKE_Swift_COMPILER', -diff --git a/swift-corelibs-libdispatch/src/swift/CMakeLists.txt b/swift-corelibs-libdispatch/src/swift/CMakeLists.txt -index 38bef37..d0ddf98 100644 ---- a/swift-corelibs-libdispatch/src/swift/CMakeLists.txt -+++ b/swift-corelibs-libdispatch/src/swift/CMakeLists.txt -@@ -15,6 +15,7 @@ add_library(swiftDispatch - Source.swift - Time.swift - Wrapper.swift) -+set(CMAKE_SHARED_LINKER_FLAGS "") - target_compile_options(swiftDispatch PRIVATE - "SHELL:-Xcc -fblocks" - "SHELL:-Xcc -fmodule-map-file=${PROJECT_SOURCE_DIR}/dispatch/module.modulemap" diff --git a/swift-ci/sdks/android/patches/swift-android.patch b/swift-ci/sdks/android/patches/swift-android.patch index b0010510..d4f5c353 100644 --- a/swift-ci/sdks/android/patches/swift-android.patch +++ b/swift-ci/sdks/android/patches/swift-android.patch @@ -36,7 +36,7 @@ index 324d1a77eea..e88601a8701 100644 from . import cmake_product from . import product -@@ -115,6 +117,24 @@ class SwiftTestingCMakeShim(cmake_product.CMakeProduct): +@@ -115,6 +117,25 @@ class SwiftTestingCMakeShim(cmake_product.CMakeProduct): # FIXME: If we build macros for the builder, specify the path. self.cmake_options.define('SwiftTesting_MACRO', 'NO') @@ -56,32 +56,9 @@ index 324d1a77eea..e88601a8701 100644 + self.cmake_options.define('CMAKE_Swift_FLAGS', flags) + self.cmake_options.define('CMAKE_Swift_COMPILER_TARGET', triple) + self.cmake_options.define('CMAKE_CXX_COMPILER_WORKS', 'True') ++ self.cmake_options.define('CMAKE_SHARED_LINKER_FLAGS', '') + self.cmake_options.define('CMAKE_FIND_ROOT_PATH', self.args.cross_compile_deps_path) + self.generate_toolchain_file_for_darwin_or_linux( host_target, override_macos_deployment_version=override_deployment_version) self.build_with_cmake([], self.args.build_variant, [], -diff --git a/swift-corelibs-foundation/CMakeLists.txt b/swift-corelibs-foundation/CMakeLists.txt -index 7f290d16..95366592 100644 ---- a/swift-corelibs-foundation/CMakeLists.txt -+++ b/swift-corelibs-foundation/CMakeLists.txt -@@ -51,6 +51,7 @@ if(NOT CMAKE_SYSTEM_NAME STREQUAL Windows) - endif() - endif() - -+set(CMAKE_SHARED_LINKER_FLAGS "") - set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) - set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) - set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) -diff --git a/swift-testing/CMakeLists.txt b/swift-testing/CMakeLists.txt -index 1be9a4b..bd7b1bd 100644 ---- a/swift-testing/CMakeLists.txt -+++ b/swift-testing/CMakeLists.txt -@@ -28,6 +28,7 @@ list(APPEND CMAKE_MODULE_PATH - ${PROJECT_SOURCE_DIR}/cmake/modules - ${PROJECT_SOURCE_DIR}/cmake/modules/shared) - -+set(CMAKE_SHARED_LINKER_FLAGS "") - set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) - set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) - set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index c28fa68f..06606a7f 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -460,6 +460,8 @@ for arch in $archs; do --swift-testing --install-swift-testing \ --cross-compile-build-swift-tools=0 \ --llvm-ninja-targets-for-cross-compile-hosts=help \ + --libdispatch-cmake-options=-DCMAKE_SHARED_LINKER_FLAGS= \ + --foundation-cmake-options=-DCMAKE_SHARED_LINKER_FLAGS= \ --cross-compile-append-host-target-to-destdir=False # --extra-cmake-options='-DCMAKE_EXTRA_LINK_FLAGS="-Wl,-z,max-page-size=16384"' # need to remove symlink that gets created in the NDK to the previous arch's build From f658604a4e2387bc06f80f969e372dbf97f98303 Mon Sep 17 00:00:00 2001 From: Finagolfin Date: Wed, 23 Jul 2025 13:23:13 +0530 Subject: [PATCH 37/44] Create an Android CMake toolchain file instead to cross-compile Testing and add 16 KB page linker flag to 6.2 branch --- .../sdks/android/patches/apply-patches.sh | 2 +- .../android/patches/swift-android-devel.patch | 168 ++++++++++++++++++ .../sdks/android/patches/swift-android.patch | 52 ------ 3 files changed, 169 insertions(+), 53 deletions(-) diff --git a/swift-ci/sdks/android/patches/apply-patches.sh b/swift-ci/sdks/android/patches/apply-patches.sh index 38f5fa89..17f07715 100755 --- a/swift-ci/sdks/android/patches/apply-patches.sh +++ b/swift-ci/sdks/android/patches/apply-patches.sh @@ -5,8 +5,8 @@ cd ${1} case "${BUILD_SCHEME}" in swift-*-branch) - git apply -v -C1 ${patches_dir}/swift-android.patch git apply -v -C1 ${patches_dir}/swift-android-devel.patch + git apply -v -C1 ${patches_dir}/swift-android.patch ;; development) git apply -v -C1 ${patches_dir}/swift-android.patch diff --git a/swift-ci/sdks/android/patches/swift-android-devel.patch b/swift-ci/sdks/android/patches/swift-android-devel.patch index c94eff10..7f9716b2 100644 --- a/swift-ci/sdks/android/patches/swift-android-devel.patch +++ b/swift-ci/sdks/android/patches/swift-android-devel.patch @@ -1,3 +1,171 @@ +diff --git a/swift/stdlib/cmake/modules/AddSwiftStdlib.cmake b/swift/stdlib/cmake/modules/AddSwiftStdlib.cmake +index 0b18957eadb..5021409ba06 100644 +--- a/swift/stdlib/cmake/modules/AddSwiftStdlib.cmake ++++ b/swift/stdlib/cmake/modules/AddSwiftStdlib.cmake +@@ -2504,6 +2504,8 @@ function(add_swift_target_library name) + list(APPEND swiftlib_link_flags_all "-shared") + # TODO: Instead of `lib${name}.so` find variable or target property which already have this value. + list(APPEND swiftlib_link_flags_all "-Wl,-soname,lib${name}.so") ++ # Ensure compatibility with Android 15+ devices using 16KB memory pages. ++ list(APPEND swiftlib_link_flags_all "-Wl,-z,max-page-size=16384") + endif() + + if (SWIFTLIB_BACK_DEPLOYMENT_LIBRARY) +diff --git a/swift/test/SILOptimizer/concat_string_literals.32.swift b/swift/test/SILOptimizer/concat_string_literals.32.swift +index 01390761c8c..d84c142397c 100644 +--- a/swift/test/SILOptimizer/concat_string_literals.32.swift ++++ b/swift/test/SILOptimizer/concat_string_literals.32.swift +@@ -4,6 +4,7 @@ + + // We have a separate test for 64-bit architectures. + // REQUIRES: PTRSIZE=32 ++// XFAIL: OS=linux-androideabi + + // NOTE: 25185.byteSwapped = 0x62 'a', 0x61 'b' + // CHECK-LABEL: test_ascii_scalar_scalar2 +diff --git a/swift/utils/swift_build_support/swift_build_support/products/product.py b/swift/utils/swift_build_support/swift_build_support/products/product.py +index 47e7ab79905..6bd94c3cad8 100644 +--- a/swift/utils/swift_build_support/swift_build_support/products/product.py ++++ b/swift/utils/swift_build_support/swift_build_support/products/product.py +@@ -389,7 +389,7 @@ class Product(object): + sysroot_arch, vendor, abi = self.get_linux_target_components(arch) + return '{}-{}-linux-{}'.format(sysroot_arch, vendor, abi) + +- def generate_linux_toolchain_file(self, platform, arch): ++ def generate_linux_toolchain_file(self, platform, arch, crosscompiling=True): + """ + Generates a new CMake tolchain file that specifies Linux as a target + platform. +@@ -402,18 +402,34 @@ class Product(object): + + toolchain_args = {} + +- toolchain_args['CMAKE_SYSTEM_NAME'] = 'Linux' +- toolchain_args['CMAKE_SYSTEM_PROCESSOR'] = arch ++ if crosscompiling: ++ if platform == "linux": ++ toolchain_args['CMAKE_SYSTEM_NAME'] = 'Linux' ++ toolchain_args['CMAKE_SYSTEM_PROCESSOR'] = arch ++ elif platform == "android": ++ toolchain_args['CMAKE_SYSTEM_NAME'] = 'Android' ++ toolchain_args['CMAKE_SYSTEM_VERSION'] = self.args.android_api_level ++ toolchain_args['CMAKE_SYSTEM_PROCESSOR'] = arch if not arch == 'armv7' \ ++ else 'armv7-a' ++ toolchain_args['CMAKE_ANDROID_NDK'] = self.args.android_ndk ++ toolchain_args['CMAKE_FIND_ROOT_PATH'] = self.args.cross_compile_deps_path ++ # This is a workaround for a CMake 3.30+ bug, ++ # https://gitlab.kitware.com/cmake/cmake/-/issues/26154, and can ++ # be removed once that is fixed. ++ toolchain_args['CMAKE_SHARED_LINKER_FLAGS'] = '\"\"' + + # We only set the actual sysroot if we are actually cross + # compiling. This is important since otherwise cmake seems to change the + # RUNPATH to be a relative rather than an absolute path, breaking + # certain cmark tests (and maybe others). +- maybe_sysroot = self.get_linux_sysroot(platform, arch) +- if maybe_sysroot is not None: +- toolchain_args['CMAKE_SYSROOT'] = maybe_sysroot +- +- target = self.get_linux_target(platform, arch) ++ if platform == "linux": ++ maybe_sysroot = self.get_linux_sysroot(platform, arch) ++ if maybe_sysroot is not None: ++ toolchain_args['CMAKE_SYSROOT'] = maybe_sysroot ++ ++ target = self.get_linux_target(platform, arch) ++ elif platform == "android": ++ target = '%s-unknown-linux-android%s' % (arch, self.args.android_api_level) + if self.toolchain.cc.endswith('clang'): + toolchain_args['CMAKE_C_COMPILER_TARGET'] = target + if self.toolchain.cxx.endswith('clang++'): +@@ -459,10 +475,30 @@ class Product(object): + platform, arch, + macos_deployment_version=override_macos_deployment_version) + self.cmake_options.define('CMAKE_TOOLCHAIN_FILE:PATH', toolchain_file) +- elif platform == "linux": +- toolchain_file = self.generate_linux_toolchain_file(platform, arch) ++ elif platform == "linux" or platform == "android": ++ # Always cross-compile for linux, but not on Android, as a native ++ # compile on Android does not use the NDK and its CMake config. ++ cross_compile = platform == "linux" or \ ++ self.is_cross_compile_target(host_target) ++ toolchain_file = self.generate_linux_toolchain_file(platform, arch, ++ cross_compile) + self.cmake_options.define('CMAKE_TOOLCHAIN_FILE:PATH', toolchain_file) + ++ if cross_compile and platform == "android": ++ resource_dir = None ++ # build-script-impl products build before the install and use ++ # the Swift stdlib from the compiler build directory instead, ++ # while products built even before that currently do not support ++ # cross-compiling Swift. ++ if not self.is_before_build_script_impl_product() and \ ++ not self.is_build_script_impl_product(): ++ install_path = self.host_install_destdir(host_target) + \ ++ self.args.install_prefix ++ resource_dir = '%s/lib/swift' % install_path ++ flags = targets.StdlibDeploymentTarget.get_target_for_name( ++ host_target).platform.swift_flags(self.args, resource_dir) ++ self.cmake_options.define('CMAKE_Swift_FLAGS', flags) ++ + return toolchain_file + + def get_openbsd_toolchain_file(self): +diff --git a/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py b/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py +index 417056efdd0..177ea9f0623 100644 +--- a/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py ++++ b/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py +@@ -127,3 +127,11 @@ class SwiftTestingCMakeShim(cmake_product.CMakeProduct): + install_prefix = install_destdir + self.args.install_prefix + + self.install_with_cmake(['install'], install_prefix) ++ ++ @classmethod ++ def is_build_script_impl_product(cls): ++ return False ++ ++ @classmethod ++ def is_before_build_script_impl_product(cls): ++ return False +diff --git a/swift/utils/swift_build_support/swift_build_support/targets.py b/swift/utils/swift_build_support/swift_build_support/targets.py +index 508d6204eaf..64a862eaf0e 100644 +--- a/swift/utils/swift_build_support/swift_build_support/targets.py ++++ b/swift/utils/swift_build_support/swift_build_support/targets.py +@@ -72,7 +72,7 @@ class Platform(object): + return True + return False + +- def swift_flags(self, args): ++ def swift_flags(self, args, resource_path=None): + """ + Swift compiler flags for a platform, useful for cross-compiling + """ +@@ -154,17 +154,21 @@ class AndroidPlatform(Platform): + """ + return True + +- def swift_flags(self, args): ++ def swift_flags(self, args, resource_path=None): + flags = '-target %s-unknown-linux-android%s ' % (args.android_arch, + args.android_api_level) + +- flags += '-resource-dir %s/swift-%s-%s/lib/swift ' % ( +- args.build_root, self.name, args.android_arch) ++ if resource_path is not None: ++ flags += '-resource-dir %s ' % (resource_path) ++ else: ++ flags += '-resource-dir %s/swift-%s-%s/lib/swift ' % ( ++ args.build_root, self.name, args.android_arch) + + android_toolchain_path = self.ndk_toolchain_path(args) + + flags += '-sdk %s/sysroot ' % (android_toolchain_path) +- flags += '-tools-directory %s/bin' % (android_toolchain_path) ++ flags += '-tools-directory %s/bin ' % (android_toolchain_path) ++ flags += '-Xclang-linker -Wl,-z,max-page-size=16384' + return flags + + def cmake_options(self, args): diff --git a/swift/utils/swift_build_support/swift_build_support/cmake.py b/swift/utils/swift_build_support/swift_build_support/cmake.py index bfb69965890..b5e9f5349c2 100644 --- a/swift/utils/swift_build_support/swift_build_support/cmake.py diff --git a/swift-ci/sdks/android/patches/swift-android.patch b/swift-ci/sdks/android/patches/swift-android.patch index d4f5c353..3808c02e 100644 --- a/swift-ci/sdks/android/patches/swift-android.patch +++ b/swift-ci/sdks/android/patches/swift-android.patch @@ -1,15 +1,3 @@ -diff --git a/swift/test/SILOptimizer/concat_string_literals.32.swift b/swift/test/SILOptimizer/concat_string_literals.32.swift -index 01390761c8c..d84c142397c 100644 ---- a/swift/test/SILOptimizer/concat_string_literals.32.swift -+++ b/swift/test/SILOptimizer/concat_string_literals.32.swift -@@ -4,6 +4,7 @@ - - // We have a separate test for 64-bit architectures. - // REQUIRES: PTRSIZE=32 -+// XFAIL: OS=linux-androideabi - - // NOTE: 25185.byteSwapped = 0x62 'a', 0x61 'b' - // CHECK-LABEL: test_ascii_scalar_scalar2 diff --git a/swift/utils/build-script-impl b/swift/utils/build-script-impl index 16e05052609..7ab8cebfab8 100755 --- a/swift/utils/build-script-impl @@ -22,43 +10,3 @@ index 16e05052609..7ab8cebfab8 100755 cmake_options=( -DENABLE_SWIFT=YES -diff --git a/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py b/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py -index 324d1a77eea..e88601a8701 100644 ---- a/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py -+++ b/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py -@@ -13,6 +13,9 @@ - import os - - from build_swift.build_swift.versions import Version -+from ..host_specific_configuration \ -+ import HostSpecificConfiguration -+from ..targets import StdlibDeploymentTarget - - from . import cmake_product - from . import product -@@ -115,6 +117,25 @@ class SwiftTestingCMakeShim(cmake_product.CMakeProduct): - # FIXME: If we build macros for the builder, specify the path. - self.cmake_options.define('SwiftTesting_MACRO', 'NO') - -+ if host_target.startswith('android') and self.is_cross_compile_target(host_target): -+ host_config = HostSpecificConfiguration(host_target, self.args) -+ self.cmake_options.extend(host_config.cmake_options) -+ triple = '%s-unknown-linux-android%s' % (self.args.android_arch, -+ self.args.android_api_level) -+ flags = '-target %s ' % (triple) -+ -+ flags += '-resource-dir %s/lib/swift ' % ( -+ self.host_install_destdir(host_target) + self.args.install_prefix) -+ -+ ndk_path = StdlibDeploymentTarget.get_target_for_name(host_target).platform.ndk_toolchain_path(self.args) -+ flags += '-sdk %s/sysroot ' % (ndk_path) -+ flags += '-tools-directory %s/bin' % (ndk_path) -+ self.cmake_options.define('CMAKE_Swift_FLAGS', flags) -+ self.cmake_options.define('CMAKE_Swift_COMPILER_TARGET', triple) -+ self.cmake_options.define('CMAKE_CXX_COMPILER_WORKS', 'True') -+ self.cmake_options.define('CMAKE_SHARED_LINKER_FLAGS', '') -+ self.cmake_options.define('CMAKE_FIND_ROOT_PATH', self.args.cross_compile_deps_path) -+ - self.generate_toolchain_file_for_darwin_or_linux( - host_target, override_macos_deployment_version=override_deployment_version) - self.build_with_cmake([], self.args.build_variant, [], From c9fc3d8b8733653f28a6a1d8c512f9b8f65c02ec Mon Sep 17 00:00:00 2001 From: Finagolfin Date: Thu, 24 Jul 2025 00:12:40 +0530 Subject: [PATCH 38/44] Install the native host LLVM tools for full compiler builds --- .../android/patches/swift-android-devel.patch | 17 ----------------- .../android/patches/swift-android-trunk.patch | 18 ------------------ swift-ci/sdks/android/scripts/build.sh | 10 ++++++---- 3 files changed, 6 insertions(+), 39 deletions(-) diff --git a/swift-ci/sdks/android/patches/swift-android-devel.patch b/swift-ci/sdks/android/patches/swift-android-devel.patch index 7f9716b2..71dc3af2 100644 --- a/swift-ci/sdks/android/patches/swift-android-devel.patch +++ b/swift-ci/sdks/android/patches/swift-android-devel.patch @@ -166,20 +166,3 @@ index 508d6204eaf..64a862eaf0e 100644 return flags def cmake_options(self, args): -diff --git a/swift/utils/swift_build_support/swift_build_support/cmake.py b/swift/utils/swift_build_support/swift_build_support/cmake.py -index bfb69965890..b5e9f5349c2 100644 ---- a/swift/utils/swift_build_support/swift_build_support/cmake.py -+++ b/swift/utils/swift_build_support/swift_build_support/cmake.py -@@ -153,10 +153,8 @@ class CMake(object): - toolchain_path = product.native_toolchain_path(args.host_target) - cmake_swiftc_path = os.getenv('CMAKE_Swift_COMPILER', - os.path.join(toolchain_path, 'bin', 'swiftc')) -- define("CMAKE_C_COMPILER:PATH", os.path.join(toolchain_path, -- 'bin', 'clang')) -- define("CMAKE_CXX_COMPILER:PATH", os.path.join(toolchain_path, -- 'bin', 'clang++')) -+ define("CMAKE_C_COMPILER:PATH", toolchain.cc) -+ define("CMAKE_CXX_COMPILER:PATH", toolchain.cxx) - define("CMAKE_Swift_COMPILER:PATH", cmake_swiftc_path) - else: - cmake_swiftc_path = os.getenv('CMAKE_Swift_COMPILER', toolchain.swiftc) diff --git a/swift-ci/sdks/android/patches/swift-android-trunk.patch b/swift-ci/sdks/android/patches/swift-android-trunk.patch index 8ab486e1..4f10577e 100644 --- a/swift-ci/sdks/android/patches/swift-android-trunk.patch +++ b/swift-ci/sdks/android/patches/swift-android-trunk.patch @@ -10,21 +10,3 @@ index 3f36fc979cb..8713ae8fea8 100755 "grep", "--extended-regexp", "--recursive", -diff --git a/swift/utils/swift_build_support/swift_build_support/cmake.py b/swift/utils/swift_build_support/swift_build_support/cmake.py -index 9a0f5b43e2d..5c7d1320a2d 100644 ---- a/swift/utils/swift_build_support/swift_build_support/cmake.py -+++ b/swift/utils/swift_build_support/swift_build_support/cmake.py -@@ -150,11 +150,8 @@ class CMake(object): - define("CMAKE_CXX_COMPILER_LAUNCHER:PATH", args.cmake_cxx_launcher) - - if self.prefer_native_toolchain and product: -- clang_tools_path = product.native_clang_tools_path(args.host_target) -- define("CMAKE_C_COMPILER:PATH", os.path.join(clang_tools_path, -- 'bin', 'clang')) -- define("CMAKE_CXX_COMPILER:PATH", os.path.join(clang_tools_path, -- 'bin', 'clang++')) -+ define("CMAKE_C_COMPILER:PATH", toolchain.cc) -+ define("CMAKE_CXX_COMPILER:PATH", toolchain.cxx) - - toolchain_path = product.native_toolchain_path(args.host_target) - cmake_swiftc_path = os.getenv('CMAKE_Swift_COMPILER', diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 06606a7f..4517d971 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -411,6 +411,7 @@ for arch in $archs; do build_cmark="" local_build="" build_llvm="1" + install_llvm="--install-llvm" build_swift_tools="1" validation_test="1" native_swift_tools_path="" @@ -420,6 +421,7 @@ for arch in $archs; do build_cmark="--skip-build-cmark" local_build="--skip-local-build" build_llvm="0" + install_llvm="" build_swift_tools="0" validation_test="0" native_swift_tools_path="--native-swift-tools-path=$host_toolchain/bin" @@ -442,7 +444,7 @@ for arch in $archs; do --cross-compile-hosts=android-$arch \ --cross-compile-deps-path=$sdk_root \ --install-destdir=$sdk_root \ - --build-llvm=$build_llvm \ + --build-llvm=$build_llvm ${install_llvm} \ --build-swift-tools=$build_swift_tools \ ${native_swift_tools_path} \ ${native_clang_tools_path} \ @@ -525,7 +527,7 @@ cp -r $host_toolchain/lib/clang/*/include $swift_res_root/usr/lib/swift/clang for arch in $archs; do quiet_pushd ${sdk_staging}/${arch}/usr - rm -r bin + rm -rf bin lib/clang local rm -r include/* cp -r ${swift_source_dir}/swift/lib/ClangImporter/SwiftBridging/{module.modulemap,swift} include/ @@ -536,7 +538,7 @@ for arch in $archs; do # need force rm in case linux is not present (when not running tests) rm -rf lib/swift{,_static}/{FrameworkABIBaseline,_InternalSwiftScan,_InternalSwiftStaticMirror,clang,embedded,host,linux,migrator} - rm -rf lib/lib*.so + rm -rf lib/lib*.so* mv lib/swift lib/swift-$arch ln -s ../swift/clang lib/swift-$arch/clang @@ -558,7 +560,7 @@ for arch in $archs; do rsync -a ${sdk_staging}/${arch}/usr ${swift_res_root} done -rm -r ${swift_res_root}/usr/share/{doc,man} +rm -rf ${swift_res_root}/usr/share/{aclocal,doc,man} rm -r ${sdk_staging} # create an install script to set up the NDK links From b881e14d80614f32dc26f60329ccd73af6a401a1 Mon Sep 17 00:00:00 2001 From: Finagolfin Date: Sat, 26 Jul 2025 20:39:49 +0530 Subject: [PATCH 39/44] Extend `--cross-compile-build-swift-tools=False` to disable building and installing LLVM and the Testing macros for cross-compiled hosts, then enable building and using the Testing macros for the native host. --- .../android/patches/swift-android-devel.patch | 13 ++ .../android/patches/swift-android-trunk.patch | 13 ++ .../sdks/android/patches/swift-android.patch | 148 ++++++++++++++++++ swift-ci/sdks/android/scripts/build.sh | 4 +- 4 files changed, 176 insertions(+), 2 deletions(-) diff --git a/swift-ci/sdks/android/patches/swift-android-devel.patch b/swift-ci/sdks/android/patches/swift-android-devel.patch index 71dc3af2..43b9df77 100644 --- a/swift-ci/sdks/android/patches/swift-android-devel.patch +++ b/swift-ci/sdks/android/patches/swift-android-devel.patch @@ -23,6 +23,19 @@ index 01390761c8c..d84c142397c 100644 // NOTE: 25185.byteSwapped = 0x62 'a', 0x61 'b' // CHECK-LABEL: test_ascii_scalar_scalar2 +diff --git a/swift/utils/swift_build_support/swift_build_support/products/cmake_product.py b/swift/utils/swift_build_support/swift_build_support/products/cmake_product.py +index dc338334f28..f1a9f4d28bf 100644 +--- a/swift/utils/swift_build_support/swift_build_support/products/cmake_product.py ++++ b/swift/utils/swift_build_support/swift_build_support/products/cmake_product.py +@@ -24,7 +24,7 @@ class CMakeProduct(product.Product): + return self.args.verbose_build + + def build_with_cmake(self, build_targets, build_type, build_args, +- prefer_native_toolchain=False): ++ prefer_native_toolchain=False, build_llvm=True): + assert self.toolchain.cmake is not None + cmake_build = [] + _cmake = cmake.CMake(self.args, self.toolchain, diff --git a/swift/utils/swift_build_support/swift_build_support/products/product.py b/swift/utils/swift_build_support/swift_build_support/products/product.py index 47e7ab79905..6bd94c3cad8 100644 --- a/swift/utils/swift_build_support/swift_build_support/products/product.py diff --git a/swift-ci/sdks/android/patches/swift-android-trunk.patch b/swift-ci/sdks/android/patches/swift-android-trunk.patch index 4f10577e..471e6bda 100644 --- a/swift-ci/sdks/android/patches/swift-android-trunk.patch +++ b/swift-ci/sdks/android/patches/swift-android-trunk.patch @@ -10,3 +10,16 @@ index 3f36fc979cb..8713ae8fea8 100755 "grep", "--extended-regexp", "--recursive", +diff --git a/swift/utils/swift_build_support/swift_build_support/products/cmake_product.py b/swift/utils/swift_build_support/swift_build_support/products/cmake_product.py +index 6d0aae32835..3713086fec3 100644 +--- a/swift/utils/swift_build_support/swift_build_support/products/cmake_product.py ++++ b/swift/utils/swift_build_support/swift_build_support/products/cmake_product.py +@@ -25,7 +25,7 @@ class CMakeProduct(product.Product): + + def build_with_cmake(self, build_targets, build_type, build_args, + prefer_native_toolchain=False, +- ignore_extra_cmake_options=False): ++ ignore_extra_cmake_options=False, build_llvm=True): + assert self.toolchain.cmake is not None + cmake_build = [] + _cmake = cmake.CMake(self.args, self.toolchain, diff --git a/swift-ci/sdks/android/patches/swift-android.patch b/swift-ci/sdks/android/patches/swift-android.patch index 3808c02e..edf2de04 100644 --- a/swift-ci/sdks/android/patches/swift-android.patch +++ b/swift-ci/sdks/android/patches/swift-android.patch @@ -10,3 +10,151 @@ index 16e05052609..7ab8cebfab8 100755 cmake_options=( -DENABLE_SWIFT=YES +diff --git a/swift/utils/build_swift/build_swift/driver_arguments.py b/swift/utils/build_swift/build_swift/driver_arguments.py +index 2b7d6d07799..e0d04d22825 100644 +--- a/swift/utils/build_swift/build_swift/driver_arguments.py ++++ b/swift/utils/build_swift/build_swift/driver_arguments.py +@@ -677,6 +677,12 @@ def create_argument_parser(): + "for each cross-compiled toolchain's destdir, useful when building " + "multiple toolchains and can be disabled if only cross-compiling one.") + ++ option('--cross-compile-build-swift-tools', toggle_true, ++ default=True, ++ help="Cross-compile the Swift compiler, other host tools from the " ++ "compiler repository, and various macros for each listed " ++ "--cross-compile-hosts platform.") ++ + option('--stdlib-deployment-targets', store, + type=argparse.ShellSplitType(), + default=None, +diff --git a/swift/utils/build_swift/tests/expected_options.py b/swift/utils/build_swift/tests/expected_options.py +index f8d1a3526ed..b655d610923 100644 +--- a/swift/utils/build_swift/tests/expected_options.py ++++ b/swift/utils/build_swift/tests/expected_options.py +@@ -155,6 +155,7 @@ EXPECTED_DEFAULTS = { + 'compiler_vendor': defaults.COMPILER_VENDOR, + 'coverage_db': None, + 'cross_compile_append_host_target_to_destdir': True, ++ 'cross_compile_build_swift_tools': True, + 'cross_compile_deps_path': None, + 'cross_compile_hosts': [], + 'infer_cross_compile_hosts_on_darwin': False, +@@ -622,6 +623,7 @@ EXPECTED_OPTIONS = [ + EnableOption('--build-swift-clang-overlays'), + EnableOption('--build-swift-remote-mirror'), + EnableOption('--cross-compile-append-host-target-to-destdir'), ++ EnableOption('--cross-compile-build-swift-tools'), + EnableOption('--color-in-tests'), + EnableOption('--distcc'), + EnableOption('--sccache'), +diff --git a/swift/utils/swift_build_support/swift_build_support/build_script_invocation.py b/swift/utils/swift_build_support/swift_build_support/build_script_invocation.py +index a0289515fd0..355d72fe21f 100644 +--- a/swift/utils/swift_build_support/swift_build_support/build_script_invocation.py ++++ b/swift/utils/swift_build_support/swift_build_support/build_script_invocation.py +@@ -119,6 +119,8 @@ class BuildScriptInvocation(object): + "--cmake-generator", args.cmake_generator, + "--cross-compile-append-host-target-to-destdir", str( + args.cross_compile_append_host_target_to_destdir).lower(), ++ "--cross-compile-build-swift-tools", str( ++ args.cross_compile_build_swift_tools).lower(), + "--build-jobs", str(args.build_jobs), + "--lit-jobs", str(args.lit_jobs), + "--common-cmake-options=%s" % ' '.join( +diff --git a/swift/utils/swift_build_support/swift_build_support/products/cmake_product.py b/swift/utils/swift_build_support/swift_build_support/products/cmake_product.py +index dc338334f28..f1a9f4d28bf 100644 +--- a/swift/utils/swift_build_support/swift_build_support/products/cmake_product.py ++++ b/swift/utils/swift_build_support/swift_build_support/products/cmake_product.py +@@ -71,9 +71,7 @@ class CMakeProduct(product.Product): + env=env) + + is_llvm = self.product_name() == "llvm" +- if (not is_llvm and not self.args.skip_build) or ( +- is_llvm and self.args._build_llvm +- ): ++ if (not is_llvm and not self.args.skip_build) or (is_llvm and build_llvm): + cmake_opts = [self.build_dir, "--config", build_type] + + shell.call( +diff --git a/swift/utils/swift_build_support/swift_build_support/products/llvm.py b/swift/utils/swift_build_support/swift_build_support/products/llvm.py +index ffae1d66702..72e400c55be 100644 +--- a/swift/utils/swift_build_support/swift_build_support/products/llvm.py ++++ b/swift/utils/swift_build_support/swift_build_support/products/llvm.py +@@ -249,10 +249,13 @@ class LLVM(cmake_product.CMakeProduct): + # space/time efficient than -g on that platform. + llvm_cmake_options.define('LLVM_USE_SPLIT_DWARF:BOOL', 'YES') + +- if not self.args._build_llvm: ++ build = True ++ if not self.args._build_llvm or (not self.args.cross_compile_build_swift_tools ++ and self.is_cross_compile_target(host_target)): + # Indicating we don't want to build LLVM at all should + # override everything. + build_targets = [] ++ build = False + elif self.args.skip_build or not self.args.build_llvm: + # We can't skip the build completely because the standalone + # build of Swift depends on these. +@@ -399,7 +402,8 @@ class LLVM(cmake_product.CMakeProduct): + + self._handle_cxx_headers(host_target, platform) + +- self.build_with_cmake(build_targets, self.args.llvm_build_variant, []) ++ self.build_with_cmake(build_targets, self.args.llvm_build_variant, [], ++ build_llvm=build) + + # copy over the compiler-rt builtins for iOS/tvOS/watchOS to ensure + # that Swift's stdlib can use compiler-rt builtins when targeting +@@ -484,7 +488,9 @@ class LLVM(cmake_product.CMakeProduct): + Whether or not this product should be installed with the given + arguments. + """ +- return self.args.install_llvm ++ return self.args.install_llvm and ( ++ self.args.cross_compile_build_swift_tools or ++ not self.is_cross_compile_target(host_target)) + + def install(self, host_target): + """ +diff --git a/swift/utils/swift_build_support/swift_build_support/products/swift_testing_macros.py b/swift/utils/swift_build_support/swift_build_support/products/swift_testing_macros.py +index ddb3b553de7..d127424709c 100644 +--- a/swift/utils/swift_build_support/swift_build_support/products/swift_testing_macros.py ++++ b/swift/utils/swift_build_support/swift_build_support/products/swift_testing_macros.py +@@ -42,13 +42,24 @@ class SwiftTestingMacros(product.Product): + return True + + def should_build(self, host_target): +- return True ++ build_macros = not self.is_cross_compile_target(host_target) or \ ++ self.args.cross_compile_build_swift_tools ++ if not build_macros: ++ print("Skipping building Testing Macros for %s, because the host tools " ++ "are not being built" % host_target) ++ return build_macros + + def should_test(self, host_target): + return False + + def should_install(self, host_target): +- return self.args.install_swift_testing_macros ++ install_macros = self.args.install_swift_testing_macros and \ ++ (not self.is_cross_compile_target(host_target) or ++ self.args.cross_compile_build_swift_tools) ++ if self.args.install_swift_testing_macros and not install_macros: ++ print("Skipping installing Testing Macros for %s, because the host tools " ++ "are not being built" % host_target) ++ return install_macros + + def _cmake_product(self, host_target): + build_root = os.path.dirname(self.build_dir) +@@ -121,3 +132,11 @@ class SwiftTestingMacrosCMakeShim(cmake_product.CMakeProduct): + install_prefix = install_destdir + self.args.install_prefix + + self.install_with_cmake(['install'], install_prefix) ++ ++ @classmethod ++ def is_build_script_impl_product(cls): ++ return False ++ ++ @classmethod ++ def is_before_build_script_impl_product(cls): ++ return False diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 4517d971..4c94179d 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -460,8 +460,8 @@ for arch in $archs; do --install-foundation \ --xctest --install-xctest \ --swift-testing --install-swift-testing \ - --cross-compile-build-swift-tools=0 \ - --llvm-ninja-targets-for-cross-compile-hosts=help \ + --swift-testing-macros --install-swift-testing-macros \ + --cross-compile-build-swift-tools=False \ --libdispatch-cmake-options=-DCMAKE_SHARED_LINKER_FLAGS= \ --foundation-cmake-options=-DCMAKE_SHARED_LINKER_FLAGS= \ --cross-compile-append-host-target-to-destdir=False From 9af1d901755fa2c57de07c8ec653963a74661d9a Mon Sep 17 00:00:00 2001 From: Finagolfin Date: Fri, 8 Aug 2025 12:36:27 +0530 Subject: [PATCH 40/44] Don't copy any libraries from the NDK: link the NDK's clang resource directory in the post-install script instead --- swift-ci/sdks/android/scripts/build.sh | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 4c94179d..a0df5721 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -233,6 +233,7 @@ cp -a $ndk_home $ndk_home_tmp ndk_home=$ndk_home_tmp ndk_installation=$ndk_home/toolchains/llvm/prebuilt/$HOST +ndk_clang_version=18 # ANDROID_NDK env needed by the swift-android.patch for: # call ln -sf "${SWIFT_BUILD_PATH}/lib/swift" "${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib" @@ -521,10 +522,6 @@ cat > $swift_res_root/SDKSettings.json < Date: Mon, 18 Aug 2025 11:26:49 -0400 Subject: [PATCH 41/44] Wildcard the clang version link in the post-install script in order to accomodate NDK 27 and 28 --- swift-ci/sdks/android/scripts/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index a0df5721..676a8b77 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -611,7 +611,8 @@ else fi # link the NDK's clang resource directory -ln -sf ${ndk_prebuilt}/*/lib/clang/18 ${swift_resources}/usr/lib/swift/clang +# e.g., ~/Library/Android/sdk/ndk/27.0.12077973/toolchains/llvm/prebuilt/darwin-x86_64/lib/clang/18 or /opt/homebrew/share/android-ndk/toolchains/llvm/prebuilt/darwin-x86_64/lib/clang/19 +ln -sf ${ndk_prebuilt}/*/lib/clang/* ${swift_resources}/usr/lib/swift/clang # copy each architecture's swiftrt.o into the sysroot, # working around https://github.com/swiftlang/swift/pull/79621 From 1221544e8ed170bdb53588efbbc69aeb1b3ebd5c Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 21 Aug 2025 20:20:32 -0400 Subject: [PATCH 42/44] Switch generated shell header to use #!/usr/bin/env bash --- swift-ci/sdks/android/scripts/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 676a8b77..db92f02c 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -562,7 +562,7 @@ mkdir scripts/ ndk_sysroot="ndk-sysroot" cat > scripts/setup-android-sdk.sh <<'EOF' -#/bin/bash +#!/usr/bin/env bash # this script will setup the ndk-sysroot with links to the # local installation indicated by ANDROID_NDK_HOME set -e From f555f48acebb744ce64be2040d490ee912c78290 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 21 Aug 2025 22:52:27 -0400 Subject: [PATCH 43/44] SBOM and nits (#14) * Address nits * Eliminate unnecessaryt pushds and use run cmake install instead of ninja * Generate SBOM with SDK dependencies * Add example of command for building locally --- swift-ci/sdks/android/README.md | 6 +- swift-ci/sdks/android/scripts/build.sh | 96 ++++++++++++++++++- swift-ci/sdks/android/scripts/fetch-source.sh | 1 - .../sdks/android/scripts/install-swift.sh | 1 - 4 files changed, 97 insertions(+), 7 deletions(-) diff --git a/swift-ci/sdks/android/README.md b/swift-ci/sdks/android/README.md index 1e57edd1..e7358192 100644 --- a/swift-ci/sdks/android/README.md +++ b/swift-ci/sdks/android/README.md @@ -81,5 +81,9 @@ perform the build locally on an Ubuntu 24.04 machine with all the build prerequisites already installed. This will generate the same artifacts in approximately half the time, and may be suitable to an already containerized envrionment (such as -a GitHub runner). +a GitHub runner). A local build can be run with the +`build-local` script, such as: +``` +./build-local swift-6.2-branch /tmp/android-sdk-devel +``` diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index db92f02c..eeb4e0fc 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -101,7 +101,7 @@ function declare_package packages+=(${name}) } -declare_package android_sdk \ +declare_package swift_android_sdk \ "Swift SDK for Android" \ "Apache-2.0" "https://swift.org/install" declare_package swift "swift" "Apache-2.0" "https://swift.org" @@ -359,7 +359,6 @@ for arch in $archs; do groupend groupstart "Building libcurl for ${compiler_target_host}" - quiet_pushd ${swift_source_dir}/curl run cmake \ -G Ninja \ -S ${swift_source_dir}/curl \ @@ -393,9 +392,8 @@ for arch in $archs; do header "Installing libcurl for $arch" quiet_pushd ${build_dir}/$arch/curl - run ninja -j$parallel_jobs install + run cmake --install ${build_dir}/${arch}/curl quiet_popd - quiet_popd groupend groupstart "Building Android SDK for ${compiler_target_host}" @@ -505,6 +503,96 @@ cat > info.json < sbom.spdx.json <> sbom.spdx.json <> sbom.spdx.json <> sbom.spdx.json <> sbom.spdx.json <> sbom.spdx.json <> sbom.spdx.json </dev/null 2>&1 git checkout ${BORINGSSL_VERSION} popd >/dev/null 2>&1 groupend - diff --git a/swift-ci/sdks/android/scripts/install-swift.sh b/swift-ci/sdks/android/scripts/install-swift.sh index 0acd9e02..bfe96b02 100755 --- a/swift-ci/sdks/android/scripts/install-swift.sh +++ b/swift-ci/sdks/android/scripts/install-swift.sh @@ -51,4 +51,3 @@ tar -xzf toolchain.tar.gz --directory ${destination} --strip-components=2 chmod -R o+r ${destination}/lib/swift popd >/dev/null - From 7696a446e8060d6954190d29ea07c81d0a3d6ec9 Mon Sep 17 00:00:00 2001 From: Finagolfin Date: Sat, 9 Aug 2025 17:45:27 +0530 Subject: [PATCH 44/44] Remove upstreamed patches and turn the last modifications into perl substitutions instead. --- swift-ci/sdks/android/build-docker | 11 +- swift-ci/sdks/android/build-local | 10 +- .../sdks/android/patches/apply-patches.sh | 22 --- .../android/patches/swift-android-devel.patch | 181 ------------------ .../android/patches/swift-android-trunk.patch | 25 --- .../sdks/android/patches/swift-android.patch | 160 ---------------- swift-ci/sdks/android/scripts/build.sh | 5 - 7 files changed, 19 insertions(+), 395 deletions(-) delete mode 100755 swift-ci/sdks/android/patches/apply-patches.sh delete mode 100644 swift-ci/sdks/android/patches/swift-android-devel.patch delete mode 100644 swift-ci/sdks/android/patches/swift-android-trunk.patch delete mode 100644 swift-ci/sdks/android/patches/swift-android.patch diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index 65d90507..ec6af53d 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -38,7 +38,16 @@ else ./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-scheme release/6.2 fi fi -./patches/apply-patches.sh ${WORKDIR}/source/swift-project + +# This `git grep` invocation in a trunk test fails in our Docker for some +# reason, so just turn it into a plain `grep` again. +perl -pi -e 's:"git",:#:' ${WORKDIR}/source/swift-project/swift/test/Misc/verify-swift-feature-testing.test-sh + +# Work around swiftlang/swift-driver#1822 for now +perl -pi -g -we "s#(call rm ... \".\{LIBDISPATCH_BUILD_DIR\}\"\n(\s+)fi\n)#\1\2if [[ -d \"\\\${ANDROID_NDK}\" ]]; then call ln -sf \"\\\${SWIFT_BUILD_PATH}/lib/swift\" \"\\\${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib\"; fi#" ${WORKDIR}/source/swift-project/swift/utils/build-script-impl + +# disable backtrace() for Android (needs either API33+ or libandroid-execinfo, or to manually add in backtrace backport) +perl -pi -e 's;os\(Android\);os\(AndroidDISABLED\);g' ${WORKDIR}/source/swift-project/swift-testing/Sources/Testing/SourceAttribution/Backtrace.swift mkdir -p ${WORKDIR}/products chmod ugo+rwx ${WORKDIR}/products diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index 8e887ff2..1ef20e9d 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -55,7 +55,15 @@ fi # Check-out and patch the sources ./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-tag ${SWIFT_TAG} -./patches/apply-patches.sh ${WORKDIR}/source/swift-project +# This `git grep` invocation in a trunk test fails in our Docker for some +# reason, so just turn it into a plain `grep` again. +perl -pi -e 's:"git",:#:' ${WORKDIR}/source/swift-project/swift/test/Misc/verify-swift-feature-testing.test-sh + +# Work around swiftlang/swift-driver#1822 for now +perl -pi -g -we "s#(call rm ... \".\{LIBDISPATCH_BUILD_DIR\}\"\n(\s+)fi\n)#\1\2if [[ -d \"\\\${ANDROID_NDK}\" ]]; then call ln -sf \"\\\${SWIFT_BUILD_PATH}/lib/swift\" \"\\\${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib\"; fi#" ${WORKDIR}/source/swift-project/swift/utils/build-script-impl + +# disable backtrace() for Android (needs either API33+ or libandroid-execinfo, or to manually add in backtrace backport) +perl -pi -e 's;os\(Android\);os\(AndroidDISABLED\);g' ${WORKDIR}/source/swift-project/swift-testing/Sources/Testing/SourceAttribution/Backtrace.swift mkdir -p ${WORKDIR}/products diff --git a/swift-ci/sdks/android/patches/apply-patches.sh b/swift-ci/sdks/android/patches/apply-patches.sh deleted file mode 100755 index 17f07715..00000000 --- a/swift-ci/sdks/android/patches/apply-patches.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -ex - -patches_dir=$(dirname $(realpath -- "${BASH_SOURCE[0]}")) -cd ${1} - -case "${BUILD_SCHEME}" in - swift-*-branch) - git apply -v -C1 ${patches_dir}/swift-android-devel.patch - git apply -v -C1 ${patches_dir}/swift-android.patch - ;; - development) - git apply -v -C1 ${patches_dir}/swift-android.patch - git apply -v -C1 ${patches_dir}/swift-android-trunk.patch - ;; - *) - echo "$0: invalid BUILD_SCHEME=${BUILD_SCHEME}" - exit 1 - ;; -esac - -# disable backtrace() for Android (needs either API33+ or libandroid-execinfo, or to manually add in backtrace backport) -perl -pi -e 's;os\(Android\);os\(AndroidDISABLED\);g' swift-testing/Sources/Testing/SourceAttribution/Backtrace.swift diff --git a/swift-ci/sdks/android/patches/swift-android-devel.patch b/swift-ci/sdks/android/patches/swift-android-devel.patch deleted file mode 100644 index 43b9df77..00000000 --- a/swift-ci/sdks/android/patches/swift-android-devel.patch +++ /dev/null @@ -1,181 +0,0 @@ -diff --git a/swift/stdlib/cmake/modules/AddSwiftStdlib.cmake b/swift/stdlib/cmake/modules/AddSwiftStdlib.cmake -index 0b18957eadb..5021409ba06 100644 ---- a/swift/stdlib/cmake/modules/AddSwiftStdlib.cmake -+++ b/swift/stdlib/cmake/modules/AddSwiftStdlib.cmake -@@ -2504,6 +2504,8 @@ function(add_swift_target_library name) - list(APPEND swiftlib_link_flags_all "-shared") - # TODO: Instead of `lib${name}.so` find variable or target property which already have this value. - list(APPEND swiftlib_link_flags_all "-Wl,-soname,lib${name}.so") -+ # Ensure compatibility with Android 15+ devices using 16KB memory pages. -+ list(APPEND swiftlib_link_flags_all "-Wl,-z,max-page-size=16384") - endif() - - if (SWIFTLIB_BACK_DEPLOYMENT_LIBRARY) -diff --git a/swift/test/SILOptimizer/concat_string_literals.32.swift b/swift/test/SILOptimizer/concat_string_literals.32.swift -index 01390761c8c..d84c142397c 100644 ---- a/swift/test/SILOptimizer/concat_string_literals.32.swift -+++ b/swift/test/SILOptimizer/concat_string_literals.32.swift -@@ -4,6 +4,7 @@ - - // We have a separate test for 64-bit architectures. - // REQUIRES: PTRSIZE=32 -+// XFAIL: OS=linux-androideabi - - // NOTE: 25185.byteSwapped = 0x62 'a', 0x61 'b' - // CHECK-LABEL: test_ascii_scalar_scalar2 -diff --git a/swift/utils/swift_build_support/swift_build_support/products/cmake_product.py b/swift/utils/swift_build_support/swift_build_support/products/cmake_product.py -index dc338334f28..f1a9f4d28bf 100644 ---- a/swift/utils/swift_build_support/swift_build_support/products/cmake_product.py -+++ b/swift/utils/swift_build_support/swift_build_support/products/cmake_product.py -@@ -24,7 +24,7 @@ class CMakeProduct(product.Product): - return self.args.verbose_build - - def build_with_cmake(self, build_targets, build_type, build_args, -- prefer_native_toolchain=False): -+ prefer_native_toolchain=False, build_llvm=True): - assert self.toolchain.cmake is not None - cmake_build = [] - _cmake = cmake.CMake(self.args, self.toolchain, -diff --git a/swift/utils/swift_build_support/swift_build_support/products/product.py b/swift/utils/swift_build_support/swift_build_support/products/product.py -index 47e7ab79905..6bd94c3cad8 100644 ---- a/swift/utils/swift_build_support/swift_build_support/products/product.py -+++ b/swift/utils/swift_build_support/swift_build_support/products/product.py -@@ -389,7 +389,7 @@ class Product(object): - sysroot_arch, vendor, abi = self.get_linux_target_components(arch) - return '{}-{}-linux-{}'.format(sysroot_arch, vendor, abi) - -- def generate_linux_toolchain_file(self, platform, arch): -+ def generate_linux_toolchain_file(self, platform, arch, crosscompiling=True): - """ - Generates a new CMake tolchain file that specifies Linux as a target - platform. -@@ -402,18 +402,34 @@ class Product(object): - - toolchain_args = {} - -- toolchain_args['CMAKE_SYSTEM_NAME'] = 'Linux' -- toolchain_args['CMAKE_SYSTEM_PROCESSOR'] = arch -+ if crosscompiling: -+ if platform == "linux": -+ toolchain_args['CMAKE_SYSTEM_NAME'] = 'Linux' -+ toolchain_args['CMAKE_SYSTEM_PROCESSOR'] = arch -+ elif platform == "android": -+ toolchain_args['CMAKE_SYSTEM_NAME'] = 'Android' -+ toolchain_args['CMAKE_SYSTEM_VERSION'] = self.args.android_api_level -+ toolchain_args['CMAKE_SYSTEM_PROCESSOR'] = arch if not arch == 'armv7' \ -+ else 'armv7-a' -+ toolchain_args['CMAKE_ANDROID_NDK'] = self.args.android_ndk -+ toolchain_args['CMAKE_FIND_ROOT_PATH'] = self.args.cross_compile_deps_path -+ # This is a workaround for a CMake 3.30+ bug, -+ # https://gitlab.kitware.com/cmake/cmake/-/issues/26154, and can -+ # be removed once that is fixed. -+ toolchain_args['CMAKE_SHARED_LINKER_FLAGS'] = '\"\"' - - # We only set the actual sysroot if we are actually cross - # compiling. This is important since otherwise cmake seems to change the - # RUNPATH to be a relative rather than an absolute path, breaking - # certain cmark tests (and maybe others). -- maybe_sysroot = self.get_linux_sysroot(platform, arch) -- if maybe_sysroot is not None: -- toolchain_args['CMAKE_SYSROOT'] = maybe_sysroot -- -- target = self.get_linux_target(platform, arch) -+ if platform == "linux": -+ maybe_sysroot = self.get_linux_sysroot(platform, arch) -+ if maybe_sysroot is not None: -+ toolchain_args['CMAKE_SYSROOT'] = maybe_sysroot -+ -+ target = self.get_linux_target(platform, arch) -+ elif platform == "android": -+ target = '%s-unknown-linux-android%s' % (arch, self.args.android_api_level) - if self.toolchain.cc.endswith('clang'): - toolchain_args['CMAKE_C_COMPILER_TARGET'] = target - if self.toolchain.cxx.endswith('clang++'): -@@ -459,10 +475,30 @@ class Product(object): - platform, arch, - macos_deployment_version=override_macos_deployment_version) - self.cmake_options.define('CMAKE_TOOLCHAIN_FILE:PATH', toolchain_file) -- elif platform == "linux": -- toolchain_file = self.generate_linux_toolchain_file(platform, arch) -+ elif platform == "linux" or platform == "android": -+ # Always cross-compile for linux, but not on Android, as a native -+ # compile on Android does not use the NDK and its CMake config. -+ cross_compile = platform == "linux" or \ -+ self.is_cross_compile_target(host_target) -+ toolchain_file = self.generate_linux_toolchain_file(platform, arch, -+ cross_compile) - self.cmake_options.define('CMAKE_TOOLCHAIN_FILE:PATH', toolchain_file) - -+ if cross_compile and platform == "android": -+ resource_dir = None -+ # build-script-impl products build before the install and use -+ # the Swift stdlib from the compiler build directory instead, -+ # while products built even before that currently do not support -+ # cross-compiling Swift. -+ if not self.is_before_build_script_impl_product() and \ -+ not self.is_build_script_impl_product(): -+ install_path = self.host_install_destdir(host_target) + \ -+ self.args.install_prefix -+ resource_dir = '%s/lib/swift' % install_path -+ flags = targets.StdlibDeploymentTarget.get_target_for_name( -+ host_target).platform.swift_flags(self.args, resource_dir) -+ self.cmake_options.define('CMAKE_Swift_FLAGS', flags) -+ - return toolchain_file - - def get_openbsd_toolchain_file(self): -diff --git a/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py b/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py -index 417056efdd0..177ea9f0623 100644 ---- a/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py -+++ b/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py -@@ -127,3 +127,11 @@ class SwiftTestingCMakeShim(cmake_product.CMakeProduct): - install_prefix = install_destdir + self.args.install_prefix - - self.install_with_cmake(['install'], install_prefix) -+ -+ @classmethod -+ def is_build_script_impl_product(cls): -+ return False -+ -+ @classmethod -+ def is_before_build_script_impl_product(cls): -+ return False -diff --git a/swift/utils/swift_build_support/swift_build_support/targets.py b/swift/utils/swift_build_support/swift_build_support/targets.py -index 508d6204eaf..64a862eaf0e 100644 ---- a/swift/utils/swift_build_support/swift_build_support/targets.py -+++ b/swift/utils/swift_build_support/swift_build_support/targets.py -@@ -72,7 +72,7 @@ class Platform(object): - return True - return False - -- def swift_flags(self, args): -+ def swift_flags(self, args, resource_path=None): - """ - Swift compiler flags for a platform, useful for cross-compiling - """ -@@ -154,17 +154,21 @@ class AndroidPlatform(Platform): - """ - return True - -- def swift_flags(self, args): -+ def swift_flags(self, args, resource_path=None): - flags = '-target %s-unknown-linux-android%s ' % (args.android_arch, - args.android_api_level) - -- flags += '-resource-dir %s/swift-%s-%s/lib/swift ' % ( -- args.build_root, self.name, args.android_arch) -+ if resource_path is not None: -+ flags += '-resource-dir %s ' % (resource_path) -+ else: -+ flags += '-resource-dir %s/swift-%s-%s/lib/swift ' % ( -+ args.build_root, self.name, args.android_arch) - - android_toolchain_path = self.ndk_toolchain_path(args) - - flags += '-sdk %s/sysroot ' % (android_toolchain_path) -- flags += '-tools-directory %s/bin' % (android_toolchain_path) -+ flags += '-tools-directory %s/bin ' % (android_toolchain_path) -+ flags += '-Xclang-linker -Wl,-z,max-page-size=16384' - return flags - - def cmake_options(self, args): diff --git a/swift-ci/sdks/android/patches/swift-android-trunk.patch b/swift-ci/sdks/android/patches/swift-android-trunk.patch deleted file mode 100644 index 471e6bda..00000000 --- a/swift-ci/sdks/android/patches/swift-android-trunk.patch +++ /dev/null @@ -1,25 +0,0 @@ -diff --git a/swift/test/Misc/verify-swift-feature-testing.test-sh b/swift/test/Misc/verify-swift-feature-testing.test-sh -index 3f36fc979cb..8713ae8fea8 100755 ---- a/swift/test/Misc/verify-swift-feature-testing.test-sh -+++ b/swift/test/Misc/verify-swift-feature-testing.test-sh -@@ -98,7 +98,6 @@ def find_matches(swift_src_root): - # `-enable-upcoming-feature` in the test directories. - output = subprocess.check_output( - [ -- "git", - "grep", - "--extended-regexp", - "--recursive", -diff --git a/swift/utils/swift_build_support/swift_build_support/products/cmake_product.py b/swift/utils/swift_build_support/swift_build_support/products/cmake_product.py -index 6d0aae32835..3713086fec3 100644 ---- a/swift/utils/swift_build_support/swift_build_support/products/cmake_product.py -+++ b/swift/utils/swift_build_support/swift_build_support/products/cmake_product.py -@@ -25,7 +25,7 @@ class CMakeProduct(product.Product): - - def build_with_cmake(self, build_targets, build_type, build_args, - prefer_native_toolchain=False, -- ignore_extra_cmake_options=False): -+ ignore_extra_cmake_options=False, build_llvm=True): - assert self.toolchain.cmake is not None - cmake_build = [] - _cmake = cmake.CMake(self.args, self.toolchain, diff --git a/swift-ci/sdks/android/patches/swift-android.patch b/swift-ci/sdks/android/patches/swift-android.patch deleted file mode 100644 index edf2de04..00000000 --- a/swift-ci/sdks/android/patches/swift-android.patch +++ /dev/null @@ -1,160 +0,0 @@ -diff --git a/swift/utils/build-script-impl b/swift/utils/build-script-impl -index 16e05052609..7ab8cebfab8 100755 ---- a/swift/utils/build-script-impl -+++ b/swift/utils/build-script-impl -@@ -2622,6 +2622,7 @@ for host in "${ALL_HOSTS[@]}"; do - echo "Cleaning the libdispatch build directory" - call rm -rf "${LIBDISPATCH_BUILD_DIR}" - fi -+ call ln -sf "${SWIFT_BUILD_PATH}/lib/swift" "${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib" - - cmake_options=( - -DENABLE_SWIFT=YES -diff --git a/swift/utils/build_swift/build_swift/driver_arguments.py b/swift/utils/build_swift/build_swift/driver_arguments.py -index 2b7d6d07799..e0d04d22825 100644 ---- a/swift/utils/build_swift/build_swift/driver_arguments.py -+++ b/swift/utils/build_swift/build_swift/driver_arguments.py -@@ -677,6 +677,12 @@ def create_argument_parser(): - "for each cross-compiled toolchain's destdir, useful when building " - "multiple toolchains and can be disabled if only cross-compiling one.") - -+ option('--cross-compile-build-swift-tools', toggle_true, -+ default=True, -+ help="Cross-compile the Swift compiler, other host tools from the " -+ "compiler repository, and various macros for each listed " -+ "--cross-compile-hosts platform.") -+ - option('--stdlib-deployment-targets', store, - type=argparse.ShellSplitType(), - default=None, -diff --git a/swift/utils/build_swift/tests/expected_options.py b/swift/utils/build_swift/tests/expected_options.py -index f8d1a3526ed..b655d610923 100644 ---- a/swift/utils/build_swift/tests/expected_options.py -+++ b/swift/utils/build_swift/tests/expected_options.py -@@ -155,6 +155,7 @@ EXPECTED_DEFAULTS = { - 'compiler_vendor': defaults.COMPILER_VENDOR, - 'coverage_db': None, - 'cross_compile_append_host_target_to_destdir': True, -+ 'cross_compile_build_swift_tools': True, - 'cross_compile_deps_path': None, - 'cross_compile_hosts': [], - 'infer_cross_compile_hosts_on_darwin': False, -@@ -622,6 +623,7 @@ EXPECTED_OPTIONS = [ - EnableOption('--build-swift-clang-overlays'), - EnableOption('--build-swift-remote-mirror'), - EnableOption('--cross-compile-append-host-target-to-destdir'), -+ EnableOption('--cross-compile-build-swift-tools'), - EnableOption('--color-in-tests'), - EnableOption('--distcc'), - EnableOption('--sccache'), -diff --git a/swift/utils/swift_build_support/swift_build_support/build_script_invocation.py b/swift/utils/swift_build_support/swift_build_support/build_script_invocation.py -index a0289515fd0..355d72fe21f 100644 ---- a/swift/utils/swift_build_support/swift_build_support/build_script_invocation.py -+++ b/swift/utils/swift_build_support/swift_build_support/build_script_invocation.py -@@ -119,6 +119,8 @@ class BuildScriptInvocation(object): - "--cmake-generator", args.cmake_generator, - "--cross-compile-append-host-target-to-destdir", str( - args.cross_compile_append_host_target_to_destdir).lower(), -+ "--cross-compile-build-swift-tools", str( -+ args.cross_compile_build_swift_tools).lower(), - "--build-jobs", str(args.build_jobs), - "--lit-jobs", str(args.lit_jobs), - "--common-cmake-options=%s" % ' '.join( -diff --git a/swift/utils/swift_build_support/swift_build_support/products/cmake_product.py b/swift/utils/swift_build_support/swift_build_support/products/cmake_product.py -index dc338334f28..f1a9f4d28bf 100644 ---- a/swift/utils/swift_build_support/swift_build_support/products/cmake_product.py -+++ b/swift/utils/swift_build_support/swift_build_support/products/cmake_product.py -@@ -71,9 +71,7 @@ class CMakeProduct(product.Product): - env=env) - - is_llvm = self.product_name() == "llvm" -- if (not is_llvm and not self.args.skip_build) or ( -- is_llvm and self.args._build_llvm -- ): -+ if (not is_llvm and not self.args.skip_build) or (is_llvm and build_llvm): - cmake_opts = [self.build_dir, "--config", build_type] - - shell.call( -diff --git a/swift/utils/swift_build_support/swift_build_support/products/llvm.py b/swift/utils/swift_build_support/swift_build_support/products/llvm.py -index ffae1d66702..72e400c55be 100644 ---- a/swift/utils/swift_build_support/swift_build_support/products/llvm.py -+++ b/swift/utils/swift_build_support/swift_build_support/products/llvm.py -@@ -249,10 +249,13 @@ class LLVM(cmake_product.CMakeProduct): - # space/time efficient than -g on that platform. - llvm_cmake_options.define('LLVM_USE_SPLIT_DWARF:BOOL', 'YES') - -- if not self.args._build_llvm: -+ build = True -+ if not self.args._build_llvm or (not self.args.cross_compile_build_swift_tools -+ and self.is_cross_compile_target(host_target)): - # Indicating we don't want to build LLVM at all should - # override everything. - build_targets = [] -+ build = False - elif self.args.skip_build or not self.args.build_llvm: - # We can't skip the build completely because the standalone - # build of Swift depends on these. -@@ -399,7 +402,8 @@ class LLVM(cmake_product.CMakeProduct): - - self._handle_cxx_headers(host_target, platform) - -- self.build_with_cmake(build_targets, self.args.llvm_build_variant, []) -+ self.build_with_cmake(build_targets, self.args.llvm_build_variant, [], -+ build_llvm=build) - - # copy over the compiler-rt builtins for iOS/tvOS/watchOS to ensure - # that Swift's stdlib can use compiler-rt builtins when targeting -@@ -484,7 +488,9 @@ class LLVM(cmake_product.CMakeProduct): - Whether or not this product should be installed with the given - arguments. - """ -- return self.args.install_llvm -+ return self.args.install_llvm and ( -+ self.args.cross_compile_build_swift_tools or -+ not self.is_cross_compile_target(host_target)) - - def install(self, host_target): - """ -diff --git a/swift/utils/swift_build_support/swift_build_support/products/swift_testing_macros.py b/swift/utils/swift_build_support/swift_build_support/products/swift_testing_macros.py -index ddb3b553de7..d127424709c 100644 ---- a/swift/utils/swift_build_support/swift_build_support/products/swift_testing_macros.py -+++ b/swift/utils/swift_build_support/swift_build_support/products/swift_testing_macros.py -@@ -42,13 +42,24 @@ class SwiftTestingMacros(product.Product): - return True - - def should_build(self, host_target): -- return True -+ build_macros = not self.is_cross_compile_target(host_target) or \ -+ self.args.cross_compile_build_swift_tools -+ if not build_macros: -+ print("Skipping building Testing Macros for %s, because the host tools " -+ "are not being built" % host_target) -+ return build_macros - - def should_test(self, host_target): - return False - - def should_install(self, host_target): -- return self.args.install_swift_testing_macros -+ install_macros = self.args.install_swift_testing_macros and \ -+ (not self.is_cross_compile_target(host_target) or -+ self.args.cross_compile_build_swift_tools) -+ if self.args.install_swift_testing_macros and not install_macros: -+ print("Skipping installing Testing Macros for %s, because the host tools " -+ "are not being built" % host_target) -+ return install_macros - - def _cmake_product(self, host_target): - build_root = os.path.dirname(self.build_dir) -@@ -121,3 +132,11 @@ class SwiftTestingMacrosCMakeShim(cmake_product.CMakeProduct): - install_prefix = install_destdir + self.args.install_prefix - - self.install_with_cmake(['install'], install_prefix) -+ -+ @classmethod -+ def is_build_script_impl_product(cls): -+ return False -+ -+ @classmethod -+ def is_before_build_script_impl_product(cls): -+ return False diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index eeb4e0fc..62845be0 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -235,11 +235,6 @@ ndk_home=$ndk_home_tmp ndk_installation=$ndk_home/toolchains/llvm/prebuilt/$HOST ndk_clang_version=18 -# ANDROID_NDK env needed by the swift-android.patch for: -# call ln -sf "${SWIFT_BUILD_PATH}/lib/swift" "${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib" -export ANDROID_NDK_HOME=$ndk_home -export ANDROID_NDK=$ndk_home - echo "Swift found at ${swift_dir}" if [[ ! -z "${host_toolchain}" ]]; then echo "Host toolchain found at ${host_toolchain}"