From be0e242506925d464f90d2dfa647fb06e9e56628 Mon Sep 17 00:00:00 2001 From: Itay Brenner Date: Wed, 29 Oct 2025 18:22:04 -0300 Subject: [PATCH 01/19] Add test runner on Cirrus Runners --- .github/workflows/test_cirrus_labs.yml | 541 +++++++++++++++++++++++++ 1 file changed, 541 insertions(+) create mode 100644 .github/workflows/test_cirrus_labs.yml diff --git a/.github/workflows/test_cirrus_labs.yml b/.github/workflows/test_cirrus_labs.yml new file mode 100644 index 00000000000..0129f8004b7 --- /dev/null +++ b/.github/workflows/test_cirrus_labs.yml @@ -0,0 +1,541 @@ +name: Test +on: + push: + branches: + - main + - release/** + + pull_request: + +# Concurrency configuration: +# - We use workflow-specific concurrency groups to allow independent test runs across different workflows +# while preventing multiple runs of the same test suite on the same branch/commit. +# - For pull requests, we cancel in-progress runs when new commits are pushed to save CI resources +# and provide faster feedback on the latest changes. +# - For main branch pushes and scheduled runs, we never cancel in-progress runs to ensure the complete +# test suite always finishes, maintaining the integrity of our main branch quality gates. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + # This job detects if the PR contains changes that require running unit tests. + # If yes, the job will output a flag that will be used by the next job to run the unit tests. + # If no, the job will output a flag that will be used by the next job to skip running the unit tests. + # At the end of this workflow, we run a check that validates that either all unit tests passed or were + # called unit-tests-required-check. + files-changed: + name: Detect File Changes + runs-on: ubuntu-latest + # Map a step output to a job output + outputs: + run_unit_tests_for_prs: ${{ steps.changes.outputs.run_unit_tests_for_prs }} + steps: + - uses: actions/checkout@v5 + - name: Get changed files + id: changes + uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + with: + token: ${{ github.token }} + filters: .github/file-filters.yml + build-test-server: + name: Build test server + if: needs.files-changed.outputs.run_unit_tests_for_prs == 'true' + needs: files-changed + runs-on: ["ghcr.io/cirruslabs/macos-runner:sequoia", "runner_group_id:10"] + steps: + - uses: actions/checkout@v5 + - name: Cache for Test Server + id: cache_test_server + uses: actions/cache@v4 + with: + path: ./test-server/.build + key: test-server-${{ hashFiles('./test-server') }}-universal + restore-keys: | + test-server-${{ hashFiles('./test-server') }}-universal + test-server- + + - name: Build Test Server for Intel CPU + if: steps.cache_test_server.outputs.cache-hit != 'true' + working-directory: test-server + run: >- + swift build -c release --triple x86_64-apple-macosx 2>&1 | tee test-server-build-intel.log + + - name: Build Test Server for M1 CPU + if: steps.cache_test_server.outputs.cache-hit != 'true' + working-directory: test-server + run: >- + swift build -c release --arch arm64 2>&1 | tee test-server-build-arm64.log + + - name: Combine Test Server + working-directory: test-server + run: >- + lipo -create -output test-server-exec $(swift build --show-bin-path -c release --triple arm64-apple-macosx)/Run $(swift build --show-bin-path -c release --triple x86_64-apple-macosx)/Run + + - name: Archiving DerivedData + uses: actions/upload-artifact@v5 + with: + name: test-server + path: | + ./test-server/test-server-exec + + - name: Archiving Raw Test Logs + uses: actions/upload-artifact@v5 + if: ${{ failure() || cancelled() }} + with: + name: test-server-build-log + path: | + test-server-build-intel.log + test-server-build-arm64.log + + - name: Run CI Diagnostics + if: failure() + run: ./scripts/ci-diagnostics.sh + + distribution-tests: + name: Distribution Tests + runs-on: ["ghcr.io/cirruslabs/macos-runner:sequoia", "runner_group_id:10"] + needs: files-changed + steps: + - uses: actions/checkout@v5 + - name: Prepare Package.swift + uses: ./.github/actions/prepare-package.swift + with: + is-pr: ${{ github.event_name == 'pull_request' }} + change-path: false + remove-duplicate: true + - run: rm -r Sentry.xcodeproj && rm -r Sentry.xcworkspace + - run: set -o pipefail && NSUnbufferedIO=YES SKIP_BINARIES=1 xcodebuild test -scheme Sentry-Package -sdk iphonesimulator -destination 'platform=iOS Simulator,OS=18.4,name=iPhone 16 Pro' | tee raw-test-output-distribution.log | xcbeautify --preserve-unbeautified + shell: sh + - name: Upload Distribution Test Logs + uses: actions/upload-artifact@v5 + if: ${{ failure() || cancelled() }} + with: + name: raw-test-output-distribution + path: | + raw-test-output-distribution.log + + # This matrix runs only the unit tests requiring the test server. + # We do this to speed up the other unit test jobs and to avoid running the + # test server with potential side effects in GH actions for all unit tests. + # We don't run this matrix for all different OS versions, because the chances + # of a bug solely on a specific OS version is minimal. + unit-tests-with-test-server: + name: Unit with Test Server ${{matrix.name}} + runs-on: ["ghcr.io/cirruslabs/macos-runner:sequoia", "runner_group_id:10"] + timeout-minutes: 20 + needs: build-test-server + + strategy: + fail-fast: false + matrix: + include: + # Running the tests with simulators is incredibly flaky. Our assumption is that simulators might have difficulties + # with communicating with the test server in CI. + # We are going to add these back in https://github.com/getsentry/sentry-cocoa/issues/6361 + - name: macOS 15 + platform: "macOS" + xcode: "16.4" + test-destination-os: "15.0" + + steps: + - uses: actions/checkout@v5 + - uses: actions/download-artifact@v6 + with: + name: test-server + + - name: Start Test Server + run: ./scripts/start-test-server.sh + - run: ./scripts/ci-select-xcode.sh ${{matrix.xcode}} + + # Note: We don't install Slather or gather code coverage for this job because it only runs + # the SentryTestServerTests target, which contains minimal tests that don't significantly + # contribute to overall code coverage metrics. The main unit-tests job handles comprehensive + # code coverage reporting. + + # We split building and running tests in two steps so we know how long running the tests takes. + - name: Build tests + id: build_tests + run: | + ./scripts/sentry-xcodebuild.sh \ + --platform ${{matrix.platform}} \ + --os ${{matrix.test-destination-os}} \ + --ref ${{ github.ref_name }} \ + --command build-for-testing \ + --configuration TestCI \ + --scheme Sentry \ + --test-plan Sentry_TestServer + + - name: Run tests + run: | + ./scripts/sentry-xcodebuild.sh \ + --platform ${{matrix.platform}} \ + --os ${{matrix.test-destination-os}} \ + --ref ${{ github.ref_name }} \ + --command test-without-building \ + --configuration TestCI \ + --scheme Sentry \ + --test-plan Sentry_TestServer + + - name: Archiving DerivedData Logs + uses: actions/upload-artifact@v5 + if: steps.build_tests.outcome == 'failure' + with: + name: derived-data-test-server-${{matrix.platform}}-xcode-${{matrix.xcode}} + path: | + /Users/runner/Library/Developer/Xcode/DerivedData/**/Logs/** + + - name: Archiving Raw Logs + uses: actions/upload-artifact@v5 + if: ${{ failure() || cancelled() }} + with: + name: raw-output-test-server-${{matrix.platform}}-xcode-${{matrix.xcode}} + path: | + raw-build-output.log + raw-build-for-testing-output.log + raw-test-output.log + + - name: Archiving Crash Logs + uses: actions/upload-artifact@v5 + if: ${{ failure() || cancelled() }} + with: + name: crash-logs-test-server-${{matrix.platform}}-xcode-${{matrix.xcode}} + path: | + ~/Library/Logs/DiagnosticReports/** + + - name: Run CI Diagnostics + if: ${{ failure() || cancelled() }} + run: ./scripts/ci-diagnostics.sh + + - name: Store screenshot + uses: ./.github/actions/capture-screenshot + if: ${{ failure() || cancelled() }} + with: + suffix: ${{ matrix.platform }}-xcode-${{ matrix.xcode }} + + unit-tests: + name: Unit ${{matrix.name}} + runs-on: ${{fromJSON(format('["ghcr.io/cirruslabs/macos-runner:{0}", "runner_group_id:10"]', matrix.runs-on)) }} + timeout-minutes: ${{ matrix.timeout || 20 }} + needs: files-changed + if: needs.files-changed.outputs.run_unit_tests_for_prs == 'true' + + strategy: + fail-fast: false + matrix: + # Can't run tests on watchOS because XCTest is not available + include: + # We are running tests on iOS 17 and later, as there were OS-internal changes introduced in succeeding versions. + + # iOS 17 + - name: iOS 17 Sentry + runs-on: sequoia + xcode: "16.0" + test-destination-os: "17.5" + platform: "iOS" + create_device: true + device: "iPhone 15 Pro" + scheme: "Sentry" + + # iOS 18 - Use pre-installed iOS 18.4 runtime on macOS-15 + - name: iOS 18 Sentry + runs-on: sequoia + xcode: "16.4" + test-destination-os: "18.6" + platform: "iOS" + device: "iPhone 16 Pro" + scheme: "Sentry" + + # iOS 26 - Download iOS 26.1 beta runtime on macOS-26 + - name: iOS 26 Sentry + runs-on: tahoe + xcode: "26.1" + test-destination-os: "26.1" + install_platforms: true + platform: "iOS" + create_device: true + device: "iPhone 17 Pro" + scheme: "Sentry" + # This runner seems to be slower than the others, so we give it more time because otherwise it frequently times out. The build step often alone takes 10 minutes and also booting the simulator takes a while. + timeout: 30 + + # We don't run the unit tests on macOS 13 cause we run them on all on GH actions available iOS versions. + # The chance of missing a bug solely on tvOS 16 that doesn't occur on iOS, macOS 12 or macOS 14 is minimal. + # We are running tests on macOS 14 and later, as there were OS-internal changes introduced in succeeding versions. + + # macOS 14 + - name: macOS 14 Sentry + runs-on: sonoma + xcode: "16.1" + test-destination-os: "latest" + platform: "macOS" + scheme: "Sentry" + + # macOS 15 + - name: macOS 15 Sentry + runs-on: sequoia + xcode: "16.4" + test-destination-os: "latest" + platform: "macOS" + scheme: "Sentry" + + # macOS 26 + - name: macOS 26 Sentry + runs-on: tahoe + xcode: "26.1" + test-destination-os: "26.1" + platform: "macOS" + scheme: "Sentry" + + # Catalyst. We test the latest version, as the risk something breaking on Catalyst and not + # on an older iOS or macOS version is low. + # In addition we are running tests on macOS 14, as there were OS-internal changes introduced in succeeding versions. + # TODO: Itay check + - name: Catalyst 14 Sentry + runs-on: sonoma + xcode: "16.1" + test-destination-os: "latest" + platform: "Catalyst" + scheme: "Sentry" + + - name: Catalyst 15 Sentry + runs-on: sequoia + xcode: "16.4" + test-destination-os: "latest" + platform: "Catalyst" + scheme: "Sentry" + + ## We don't run unit tests on macCatalyst 26 yet because of https://github.com/getsentry/sentry-cocoa/issues/6165. + + # We don't run the unit tests on tvOS 16 cause we run them on all on GH actions available iOS versions. + # The chance of missing a bug solely on tvOS 16 that doesn't occur on iOS, tvOS 15 or tvOS 16 is minimal. + # We are running tests on tvOS 17 and latest, as there were OS-internal changes introduced in succeeding versions. + + # tvOS 17 + # TODO: Itay install old platform + # - name: tvOS 17 Sentry + # runs-on: sonoma + # xcode: "15.4" + # test-destination-os: "17.5" + # platform: "tvOS" + # scheme: "Sentry" + + # iOS 17 - Use pre-installed iOS 17.5 + - name: iOS 17 SentrySwiftUI + runs-on: sonoma + xcode: "16.1" + test-destination-os: "17.5" + platform: "iOS" + create_device: true + device: "iPhone 15 Pro" + scheme: "SentrySwiftUI" + + # tvOS 18 + - name: tvOS 18 Sentry + runs-on: sequoia + xcode: "16.4" + test-destination-os: "18.5" + platform: "tvOS" + scheme: "Sentry" + + # tvOS 26 + - name: tvOS 26 Sentry + runs-on: tahoe + xcode: "26.1" + test-destination-os: "26.1" + install_platforms: true + platform: "tvOS" + create_device: true + device: "Apple TV" + scheme: "Sentry" + + steps: + - uses: actions/checkout@v5 + - run: ./scripts/ci-select-xcode.sh ${{matrix.xcode}} + + - name: Install Slather + run: gem install slather + + # Install platform runtimes that are not included by default + # For iOS 16.4: Not included with Xcode 15.4 on macOS-14 (macOS-13 sunset) + # For iOS/tvOS 26.1: Beta platforms not included by default + - name: Install required platforms + if: ${{ matrix.install_platforms }} + run: ./scripts/ci-install-platforms.sh --platforms "${{ matrix.platform }}" --os-version "${{ matrix.test-destination-os }}" + + # Create simulator devices for non-preinstalled simulators + # Required for iOS 16.4, iOS 17.5 (on Xcode 15.4), and iOS/tvOS 26.1 + - name: Create simulator device + if: ${{ matrix.create_device }} + run: ./scripts/ci-create-simulator.sh --platform "${{ matrix.platform }}" --os-version "${{ matrix.test-destination-os }}" --device-name "${{ matrix.device }}" + + # Boot created simulators to ensure they're ready before tests run + # Based on CircleCI forum comment, booting is especially important for Xcode 26: https://discuss.circleci.com/t/xcode-26-rc/54066/18 + - name: Boot simulator + if: ${{ matrix.create_device && matrix.platform == 'iOS' }} + run: ./scripts/ci-boot-simulator.sh --xcode ${{ matrix.xcode }} --device "${{ matrix.device }}" --os-version "${{ matrix.test-destination-os }}" + + # We split building and running tests in two steps so we know how long running the tests takes. + - name: Build Tests + id: build_tests + run: | + ./scripts/sentry-xcodebuild.sh \ + --platform ${{matrix.platform}} \ + --os ${{matrix.test-destination-os}} \ + --ref ${{ github.ref_name }} \ + --command build-for-testing \ + --device "${{matrix.device}}" \ + --configuration TestCI \ + --scheme ${{matrix.scheme}} + + # Run Flaky Tests TestPlan which has a retry mechanism on failure. + # We intentionally run these before the other test plan to fail early. + # Use a separate result bundle name to avoid conflicts with the regular test run. + # xcodebuild fails if a result bundle already exists at the target path. + - name: Run Flaky Tests + # Only the Sentry Scheme has the Flaky TestPlan. + if: ${{ matrix.scheme == 'Sentry' }} + run: | + ./scripts/sentry-xcodebuild.sh \ + --platform ${{matrix.platform}} \ + --os ${{matrix.test-destination-os}} \ + --ref ${{ github.ref_name }} \ + --command test-without-building \ + --device "${{matrix.device}}" \ + --configuration TestCI \ + --scheme ${{matrix.scheme}} \ + --test-plan Sentry_Flaky \ + --result-bundle flaky-results.xcresult + + - name: Run tests + # We call a script with the platform so the destination + # passed to xcodebuild doesn't end up in the job name, + # because GitHub Actions don't provide an easy way of + # manipulating string in expressions. + run: | + ./scripts/sentry-xcodebuild.sh \ + --platform ${{matrix.platform}} \ + --os ${{matrix.test-destination-os}} \ + --ref ${{ github.ref_name }} \ + --command test-without-building \ + --device "${{matrix.device}}" \ + --configuration TestCI \ + --scheme ${{matrix.scheme}} \ + --result-bundle results.xcresult + + - name: Publish Test Report + uses: mikepenz/action-junit-report@5b7ee5a21e8674b695313d769f3cbdfd5d4d53a4 # v6.0.0 + if: always() + with: + report_paths: "build/reports/junit.xml" + fail_on_failure: true + fail_on_parse_error: true + detailed_summary: true + + - name: Archiving DerivedData Logs + uses: actions/upload-artifact@v5 + if: steps.build_tests.outcome == 'failure' + with: + name: derived-data-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}} + path: | + /Users/runner/Library/Developer/Xcode/DerivedData/**/Logs/** + + - name: Archiving Raw Logs + uses: actions/upload-artifact@v5 + if: ${{ failure() || cancelled() }} + with: + name: raw-output-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}} + path: | + raw-build-output.log + raw-build-for-testing-output.log + raw-test-output.log + + - name: Archiving Crash Logs + uses: actions/upload-artifact@v5 + if: ${{ failure() || cancelled() }} + with: + name: crash-logs-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}} + path: | + ~/Library/Logs/DiagnosticReports/** + + - name: Archiving Test Results + uses: actions/upload-artifact@v5 + if: ${{ failure() || cancelled() }} + with: + name: result-bundle-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}} + path: | + results.xcresult + flaky-results.xcresult + + - name: Gather code coverage information via slather + run: slather coverage --configuration TestCI --scheme Sentry + + # We can upload all coverage reports, because codecov merges them. + # See https://docs.codecov.io/docs/merging-reports + # Checkout .codecov.yml to see the config of Codecov + # We don't upload codecov for release branches, as we don't want a failing coverage check to block a release. + # We don't upload codecov for scheduled runs as CodeCov only accepts a limited amount of uploads per commit. + - name: Push code coverage to codecov + id: codecov_1 + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # pin@v5.5.1 + if: ${{ contains(matrix.platform, 'iOS') && !contains(github.ref, 'release') && github.event.schedule == '' }} + with: + # Although public repos should not have to specify a token there seems to be a bug with the Codecov GH action, which can + # be solved by specifying the token, see https://github.com/codecov/codecov-action/issues/557#issuecomment-1224970469 + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: false + verbose: true + + # Sometimes codecov uploads etc can fail. Retry one time to rule out e.g. intermittent network failures. + - name: Push code coverage to codecov + id: codecov_2 + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # pin@v5.5.1 + if: ${{ steps.codecov_1.outcome == 'failure' && contains(matrix.platform, 'iOS') && !contains(github.ref, 'release') && github.event.schedule == '' }} + with: + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: true + verbose: true + + - name: Codecov test analytics + if: ${{ !cancelled() && !contains(github.ref, 'release') && github.event.schedule == '' }} + uses: codecov/test-results-action@47f89e9acb64b76debcd5ea40642d25a4adced9f # pin@v1.1.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true + name: sentry-cocoa-unit-tests + flags: unittests-${{ matrix.platform }}-${{ matrix.xcode }}-${{ matrix.test-destination-os }}, unittests + + - name: Run CI Diagnostics + if: failure() + run: ./scripts/ci-diagnostics.sh + + - name: Store screenshot + uses: ./.github/actions/capture-screenshot + if: failure() + with: + suffix: unit-tests-${{ matrix.platform }}-xcode-${{ matrix.xcode }}-os-${{ matrix.test-destination-os }} + + # This check validates that either all unit tests passed or were skipped, which allows us + # to make unit tests a required check with only running the unit tests when required. + # So, we don't have to run unit tests, for example, for Changelog or ReadMe changes. + + unit-tests-required-check: + needs: + [ + files-changed, + build-test-server, + distribution-tests, + unit-tests, + unit-tests-with-test-server, + ] + name: Unit Tests + # This is necessary since a failed/skipped dependent job would cause this job to be skipped + if: always() + runs-on: ubuntu-latest + steps: + # If any jobs we depend on fails gets cancelled or times out, this job will fail. + # Skipped jobs are not considered failures. + - name: Check for failures + if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + run: | + echo "One of the unit test jobs has failed." && exit 1 From b04402446c1f60b31648306beca78deecfbf1354 Mon Sep 17 00:00:00 2001 From: Itay Brenner Date: Wed, 29 Oct 2025 18:23:20 -0300 Subject: [PATCH 02/19] Update test name --- .github/workflows/test_cirrus_labs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_cirrus_labs.yml b/.github/workflows/test_cirrus_labs.yml index 0129f8004b7..003bcf49558 100644 --- a/.github/workflows/test_cirrus_labs.yml +++ b/.github/workflows/test_cirrus_labs.yml @@ -1,4 +1,4 @@ -name: Test +name: Test on Cirrus Labs on: push: branches: From 1edc6a9a26d58ef84abbe354b78d51c6a7c4c760 Mon Sep 17 00:00:00 2001 From: Itay Brenner Date: Wed, 29 Oct 2025 18:24:27 -0300 Subject: [PATCH 03/19] Add cirrus labs runner file --- .github/file-filters.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/file-filters.yml b/.github/file-filters.yml index 7a09a0848ce..051fee09462 100644 --- a/.github/file-filters.yml +++ b/.github/file-filters.yml @@ -27,6 +27,7 @@ run_unit_tests_for_prs: &run_unit_tests_for_prs # GH Actions - ".github/workflows/test.yml" + - ".github/workflows/test_cirrus_labs.yml" - ".github/file-filters.yml" # Scripts From 5a741414b827e86f1e11a2a0487c4a231bae802c Mon Sep 17 00:00:00 2001 From: Itay Brenner Date: Wed, 29 Oct 2025 18:30:26 -0300 Subject: [PATCH 04/19] Install ruby 3.3.0 --- .github/workflows/test_cirrus_labs.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test_cirrus_labs.yml b/.github/workflows/test_cirrus_labs.yml index 003bcf49558..b284887e8fc 100644 --- a/.github/workflows/test_cirrus_labs.yml +++ b/.github/workflows/test_cirrus_labs.yml @@ -353,6 +353,9 @@ jobs: - uses: actions/checkout@v5 - run: ./scripts/ci-select-xcode.sh ${{matrix.xcode}} + - name: Install Ruby 3.3.0 + run: rbenv install 3.3.0 + - name: Install Slather run: gem install slather From 3a78a3c86df68b14ee45780da3ce0834b6581043 Mon Sep 17 00:00:00 2001 From: Itay Brenner Date: Wed, 29 Oct 2025 18:38:19 -0300 Subject: [PATCH 05/19] Fix open simulator path --- scripts/ci-boot-simulator.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/ci-boot-simulator.sh b/scripts/ci-boot-simulator.sh index 475fe10edd5..c800275331a 100755 --- a/scripts/ci-boot-simulator.sh +++ b/scripts/ci-boot-simulator.sh @@ -174,8 +174,9 @@ for attempt in $(seq 1 $MAX_BOOT_ATTEMPTS); do # Open Simulator app UI (only on first attempt) if [ "$attempt" -eq 1 ]; then log_notice "Opening Simulator app UI" - if ! open -a Simulator; then - log_error "Failed to open Simulator app" + SIMULATOR_APP_PATH="$(xcode-select -p)/Applications/Simulator.app" + if ! open "$SIMULATOR_APP_PATH"; then + log_error "Failed to open Simulator app at $SIMULATOR_APP_PATH" exit 1 fi log_notice "Simulator app opened successfully" From bf559706c5df2f55723543b6fb822498d66d8312 Mon Sep 17 00:00:00 2001 From: Itay Brenner Date: Thu, 30 Oct 2025 12:46:45 -0300 Subject: [PATCH 06/19] Fix SwiftUI test --- .github/workflows/test_cirrus_labs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_cirrus_labs.yml b/.github/workflows/test_cirrus_labs.yml index b284887e8fc..9aed76954f3 100644 --- a/.github/workflows/test_cirrus_labs.yml +++ b/.github/workflows/test_cirrus_labs.yml @@ -322,7 +322,7 @@ jobs: # iOS 17 - Use pre-installed iOS 17.5 - name: iOS 17 SentrySwiftUI - runs-on: sonoma + runs-on: sequoia xcode: "16.1" test-destination-os: "17.5" platform: "iOS" From 9550996248980869477d942502a9be399420b15c Mon Sep 17 00:00:00 2001 From: Itay Brenner Date: Thu, 30 Oct 2025 13:12:12 -0300 Subject: [PATCH 07/19] Use iOS 18.4 --- .github/workflows/test_cirrus_labs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_cirrus_labs.yml b/.github/workflows/test_cirrus_labs.yml index 9aed76954f3..11761992001 100644 --- a/.github/workflows/test_cirrus_labs.yml +++ b/.github/workflows/test_cirrus_labs.yml @@ -241,7 +241,7 @@ jobs: - name: iOS 18 Sentry runs-on: sequoia xcode: "16.4" - test-destination-os: "18.6" + test-destination-os: "18.4" platform: "iOS" device: "iPhone 16 Pro" scheme: "Sentry" From 714f8c7003943f9fb17cae2ca01dc3c2abfe0140 Mon Sep 17 00:00:00 2001 From: Itay Brenner Date: Tue, 4 Nov 2025 15:11:51 -0300 Subject: [PATCH 08/19] Update test.yml to run cirrus labs runners --- .github/workflows/test.yml | 167 ++++++-- .github/workflows/test_cirrus_labs.yml | 544 ------------------------- 2 files changed, 126 insertions(+), 585 deletions(-) delete mode 100644 .github/workflows/test_cirrus_labs.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d00da49a8c8..6142b577254 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,10 +40,24 @@ jobs: token: ${{ github.token }} filters: .github/file-filters.yml build-test-server: - name: Build test server + name: Build test server on ${{matrix.runner}} if: needs.files-changed.outputs.run_unit_tests_for_prs == 'true' needs: files-changed - runs-on: macos-15 + runs-on: ${{matrix.runs-on}} + strategy: + fail-fast: false + matrix: + runner: + - GitHub + - Cirrus Labs + include: + - runner: GitHub + runs-on: macos-15 + - runner: Cirrus Labs + runs-on: [ + "ghcr.io/cirruslabs/macos-runner:sequoia", + "runner_group_id:10", + ] steps: - uses: actions/checkout@v5 - name: Cache for Test Server @@ -94,9 +108,20 @@ jobs: run: ./scripts/ci-diagnostics.sh distribution-tests: - name: Distribution Tests - runs-on: macos-15 + name: Distribution Tests on ${{matrix.runner}} + runs-on: ${{matrix.runs-on}} needs: files-changed + strategy: + fail-fast: false + matrix: + include: + - runner: GitHub + runs-on: macos-15 + - runner: Cirrus Labs + runs-on: [ + "ghcr.io/cirruslabs/macos-runner:sequoia", + "runner_group_id:10", + ] steps: - uses: actions/checkout@v5 - name: Prepare Package.swift @@ -112,7 +137,7 @@ jobs: uses: actions/upload-artifact@v5 if: ${{ failure() || cancelled() }} with: - name: raw-test-output-distribution + name: raw-test-output-distribution-${{matrix.runner}} path: | raw-test-output-distribution.log @@ -122,30 +147,40 @@ jobs: # We don't run this matrix for all different OS versions, because the chances # of a bug solely on a specific OS version is minimal. unit-tests-with-test-server: - name: Unit with Test Server ${{matrix.name}} + name: Unit with Test Server ${{matrix.name}} on ${{matrix.runner}} runs-on: ${{matrix.runs-on}} timeout-minutes: 20 needs: build-test-server - strategy: fail-fast: false matrix: + xcode: + - "16.4" + name: + - "macOS 15" + test-destination-os: + - "15.0" + platform: + - "macOS" include: # Running the tests with simulators is incredibly flaky. Our assumption is that simulators might have difficulties # with communicating with the test server in CI. # We are going to add these back in https://github.com/getsentry/sentry-cocoa/issues/6361 - - name: macOS 15 + - runner: GitHub runs-on: macos-15 - platform: "macOS" - xcode: "16.4" - test-destination-os: "15.0" + - runner: Cirrus Labs + runs-on: [ + "ghcr.io/cirruslabs/macos-runner:sequoia", + "runner_group_id:10", + ] steps: - uses: actions/checkout@v5 - uses: actions/download-artifact@v6 with: - name: test-server - + name: test-server-${{matrix.runner}} + - name: Rename artifact + run: mv test-server-${{matrix.runner}} test-server - name: Start Test Server run: ./scripts/start-test-server.sh - run: ./scripts/ci-select-xcode.sh ${{matrix.xcode}} @@ -183,7 +218,7 @@ jobs: uses: actions/upload-artifact@v5 if: steps.build_tests.outcome == 'failure' with: - name: derived-data-test-server-${{matrix.platform}}-xcode-${{matrix.xcode}} + name: derived-data-test-server-${{matrix.platform}}-xcode-${{matrix.xcode}}-${{matrix.runner}} path: | /Users/runner/Library/Developer/Xcode/DerivedData/**/Logs/** @@ -191,7 +226,7 @@ jobs: uses: actions/upload-artifact@v5 if: ${{ failure() || cancelled() }} with: - name: raw-output-test-server-${{matrix.platform}}-xcode-${{matrix.xcode}} + name: raw-output-test-server-${{matrix.platform}}-xcode-${{matrix.xcode}}-${{matrix.runner}} path: | raw-build-output.log raw-build-for-testing-output.log @@ -201,7 +236,7 @@ jobs: uses: actions/upload-artifact@v5 if: ${{ failure() || cancelled() }} with: - name: crash-logs-test-server-${{matrix.platform}}-xcode-${{matrix.xcode}} + name: crash-logs-test-server-${{matrix.platform}}-xcode-${{matrix.xcode}}-${{matrix.runner}} path: | ~/Library/Logs/DiagnosticReports/** @@ -213,10 +248,10 @@ jobs: uses: ./.github/actions/capture-screenshot if: ${{ failure() || cancelled() }} with: - suffix: ${{ matrix.platform }}-xcode-${{ matrix.xcode }} + suffix: ${{ matrix.platform }}-xcode-${{ matrix.xcode }}-${{matrix.runner}} unit-tests: - name: Unit ${{matrix.name}} + name: Unit ${{matrix.name}} on ${{matrix.runner}} runs-on: ${{matrix.runs-on}} timeout-minutes: ${{ matrix.timeout || 20 }} needs: files-changed @@ -225,14 +260,16 @@ jobs: strategy: fail-fast: false matrix: + runner: + - GitHub + - Cirrus Labs # Can't run tests on watchOS because XCTest is not available include: # We are running tests on iOS 17 and later, as there were OS-internal changes introduced in succeeding versions. # iOS 17 - Use pre-installed iOS 17.5 runtime on macOS-14 with Xcode 15.4 - name: iOS 17 Sentry - runs-on: macos-14 - xcode: "15.4" + os: macos-14 test-destination-os: "17.5" platform: "iOS" create_device: true @@ -241,7 +278,7 @@ jobs: # iOS 18 - Use pre-installed iOS 18.4 runtime on macOS-15 - name: iOS 18 Sentry - runs-on: macos-15 + os: macos-15 xcode: "16.4" test-destination-os: "18.4" platform: "iOS" @@ -250,7 +287,7 @@ jobs: # iOS 26 - Download iOS 26.1 beta runtime on macOS-26 - name: iOS 26 Sentry - runs-on: macos-26 + os: macos-26 xcode: "26.1" test-destination-os: "26.1" install_platforms: true @@ -267,15 +304,14 @@ jobs: # macOS 14 - name: macOS 14 Sentry - runs-on: macos-14 - xcode: "15.4" + os: macos-14 test-destination-os: "latest" platform: "macOS" scheme: "Sentry" # macOS 15 - name: macOS 15 Sentry - runs-on: macos-15 + os: macos-15 xcode: "16.4" test-destination-os: "latest" platform: "macOS" @@ -283,7 +319,7 @@ jobs: # macOS 26 - name: macOS 26 Sentry - runs-on: macos-26 + os: macos-26 xcode: "26.1" test-destination-os: "26.1" platform: "macOS" @@ -293,15 +329,13 @@ jobs: # on an older iOS or macOS version is low. # In addition we are running tests on macOS 14, as there were OS-internal changes introduced in succeeding versions. - name: Catalyst 14 Sentry - runs-on: macos-14 - xcode: "15.4" + os: macos-14 test-destination-os: "latest" platform: "Catalyst" scheme: "Sentry" - name: Catalyst 15 Sentry - runs-on: macos-15 - xcode: "16.4" + os: macos-15 test-destination-os: "latest" platform: "Catalyst" scheme: "Sentry" @@ -315,15 +349,13 @@ jobs: # tvOS 17 - name: tvOS 17 Sentry runs-on: macos-14 - xcode: "15.4" test-destination-os: "17.5" platform: "tvOS" scheme: "Sentry" - # iOS 17 - Use pre-installed iOS 17.5 runtime on macOS-14 with Xcode 15.4 + # iOS 17 - name: iOS 17 SentrySwiftUI - runs-on: macos-14 - xcode: "15.4" + os: macos-14 test-destination-os: "17.5" platform: "iOS" create_device: true @@ -332,7 +364,7 @@ jobs: # tvOS 18 - name: tvOS 18 Sentry - runs-on: macos-15 + os: macos-15 xcode: "16.4" test-destination-os: "18.4" platform: "tvOS" @@ -340,7 +372,7 @@ jobs: # tvOS 26 - name: tvOS 26 Sentry - runs-on: macos-26 + os: macos-26 xcode: "26.1" test-destination-os: "26.1" install_platforms: true @@ -349,10 +381,63 @@ jobs: device: "Apple TV" scheme: "Sentry" + # These items allow us to complement the matrix and set the runs-on for the runners. + # See https://docs.github.com/en/actions/how-tos/write-workflows/choose-what-workflows-do/run-job-variations#expanding-or-adding-matrix-configurations + # for more details + - runner: GitHub + os: macos-14 + runs-on: macos-14 + xcode: "15.4" + + - runner: GitHub + os: macos-15 + runs-on: macos-15 + + - runner: GitHub + os: macos-26 + runs-on: macos-26 + + # Cirrus Labs's sequoia runner doesn't have Xcode 15.4 installed, so we use Xcode 16.1. + - runner: Cirrus Labs + os: macos-14 + test-destination-os: "17.5" + runs-on: [ + "ghcr.io/cirruslabs/macos-runner:sequoia", + "runner_group_id:10", + ] + xcode: "16.1" + + - runner: Cirrus Labs + os: macos-14 + runs-on: [ + "ghcr.io/cirruslabs/macos-runner:sonoma", + "runner_group_id:10", + ] + xcode: "16.1" + + - runner: Cirrus Labs + os: macos-15 + runs-on: [ + "ghcr.io/cirruslabs/macos-runner:sequoia", + "runner_group_id:10", + ] + + - runner: Cirrus Labs + os: macos-26 + runs-on: [ + "ghcr.io/cirruslabs/macos-runner:tahoe", + "runner_group_id:10", + ] + steps: - uses: actions/checkout@v5 - run: ./scripts/ci-select-xcode.sh ${{matrix.xcode}} + # Install Ruby for Cirrus Labs runners + - name: Install Ruby 3.3.0 + if: ${{ matrix.runner == 'Cirrus Labs' }} + run: rbenv install 3.3.0 + - name: Install Slather run: gem install slather @@ -436,7 +521,7 @@ jobs: uses: actions/upload-artifact@v5 if: steps.build_tests.outcome == 'failure' with: - name: derived-data-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}} + name: derived-data-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}}-${{matrix.runner}} path: | /Users/runner/Library/Developer/Xcode/DerivedData/**/Logs/** @@ -444,7 +529,7 @@ jobs: uses: actions/upload-artifact@v5 if: ${{ failure() || cancelled() }} with: - name: raw-output-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}} + name: raw-output-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}}-${{matrix.runner}} path: | raw-build-output.log raw-build-for-testing-output.log @@ -454,7 +539,7 @@ jobs: uses: actions/upload-artifact@v5 if: ${{ failure() || cancelled() }} with: - name: crash-logs-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}} + name: crash-logs-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}}-${{matrix.runner}} path: | ~/Library/Logs/DiagnosticReports/** @@ -462,7 +547,7 @@ jobs: uses: actions/upload-artifact@v5 if: ${{ failure() || cancelled() }} with: - name: result-bundle-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}} + name: result-bundle-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}}-${{matrix.runner}} path: | results.xcresult flaky-results.xcresult @@ -513,7 +598,7 @@ jobs: uses: ./.github/actions/capture-screenshot if: failure() with: - suffix: unit-tests-${{ matrix.platform }}-xcode-${{ matrix.xcode }}-os-${{ matrix.test-destination-os }} + suffix: unit-tests-${{ matrix.platform }}-xcode-${{ matrix.xcode }}-os-${{ matrix.test-destination-os }}-${{matrix.runner}} # This check validates that either all unit tests passed or were skipped, which allows us # to make unit tests a required check with only running the unit tests when required. diff --git a/.github/workflows/test_cirrus_labs.yml b/.github/workflows/test_cirrus_labs.yml deleted file mode 100644 index 11761992001..00000000000 --- a/.github/workflows/test_cirrus_labs.yml +++ /dev/null @@ -1,544 +0,0 @@ -name: Test on Cirrus Labs -on: - push: - branches: - - main - - release/** - - pull_request: - -# Concurrency configuration: -# - We use workflow-specific concurrency groups to allow independent test runs across different workflows -# while preventing multiple runs of the same test suite on the same branch/commit. -# - For pull requests, we cancel in-progress runs when new commits are pushed to save CI resources -# and provide faster feedback on the latest changes. -# - For main branch pushes and scheduled runs, we never cancel in-progress runs to ensure the complete -# test suite always finishes, maintaining the integrity of our main branch quality gates. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - # This job detects if the PR contains changes that require running unit tests. - # If yes, the job will output a flag that will be used by the next job to run the unit tests. - # If no, the job will output a flag that will be used by the next job to skip running the unit tests. - # At the end of this workflow, we run a check that validates that either all unit tests passed or were - # called unit-tests-required-check. - files-changed: - name: Detect File Changes - runs-on: ubuntu-latest - # Map a step output to a job output - outputs: - run_unit_tests_for_prs: ${{ steps.changes.outputs.run_unit_tests_for_prs }} - steps: - - uses: actions/checkout@v5 - - name: Get changed files - id: changes - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 - with: - token: ${{ github.token }} - filters: .github/file-filters.yml - build-test-server: - name: Build test server - if: needs.files-changed.outputs.run_unit_tests_for_prs == 'true' - needs: files-changed - runs-on: ["ghcr.io/cirruslabs/macos-runner:sequoia", "runner_group_id:10"] - steps: - - uses: actions/checkout@v5 - - name: Cache for Test Server - id: cache_test_server - uses: actions/cache@v4 - with: - path: ./test-server/.build - key: test-server-${{ hashFiles('./test-server') }}-universal - restore-keys: | - test-server-${{ hashFiles('./test-server') }}-universal - test-server- - - - name: Build Test Server for Intel CPU - if: steps.cache_test_server.outputs.cache-hit != 'true' - working-directory: test-server - run: >- - swift build -c release --triple x86_64-apple-macosx 2>&1 | tee test-server-build-intel.log - - - name: Build Test Server for M1 CPU - if: steps.cache_test_server.outputs.cache-hit != 'true' - working-directory: test-server - run: >- - swift build -c release --arch arm64 2>&1 | tee test-server-build-arm64.log - - - name: Combine Test Server - working-directory: test-server - run: >- - lipo -create -output test-server-exec $(swift build --show-bin-path -c release --triple arm64-apple-macosx)/Run $(swift build --show-bin-path -c release --triple x86_64-apple-macosx)/Run - - - name: Archiving DerivedData - uses: actions/upload-artifact@v5 - with: - name: test-server - path: | - ./test-server/test-server-exec - - - name: Archiving Raw Test Logs - uses: actions/upload-artifact@v5 - if: ${{ failure() || cancelled() }} - with: - name: test-server-build-log - path: | - test-server-build-intel.log - test-server-build-arm64.log - - - name: Run CI Diagnostics - if: failure() - run: ./scripts/ci-diagnostics.sh - - distribution-tests: - name: Distribution Tests - runs-on: ["ghcr.io/cirruslabs/macos-runner:sequoia", "runner_group_id:10"] - needs: files-changed - steps: - - uses: actions/checkout@v5 - - name: Prepare Package.swift - uses: ./.github/actions/prepare-package.swift - with: - is-pr: ${{ github.event_name == 'pull_request' }} - change-path: false - remove-duplicate: true - - run: rm -r Sentry.xcodeproj && rm -r Sentry.xcworkspace - - run: set -o pipefail && NSUnbufferedIO=YES SKIP_BINARIES=1 xcodebuild test -scheme Sentry-Package -sdk iphonesimulator -destination 'platform=iOS Simulator,OS=18.4,name=iPhone 16 Pro' | tee raw-test-output-distribution.log | xcbeautify --preserve-unbeautified - shell: sh - - name: Upload Distribution Test Logs - uses: actions/upload-artifact@v5 - if: ${{ failure() || cancelled() }} - with: - name: raw-test-output-distribution - path: | - raw-test-output-distribution.log - - # This matrix runs only the unit tests requiring the test server. - # We do this to speed up the other unit test jobs and to avoid running the - # test server with potential side effects in GH actions for all unit tests. - # We don't run this matrix for all different OS versions, because the chances - # of a bug solely on a specific OS version is minimal. - unit-tests-with-test-server: - name: Unit with Test Server ${{matrix.name}} - runs-on: ["ghcr.io/cirruslabs/macos-runner:sequoia", "runner_group_id:10"] - timeout-minutes: 20 - needs: build-test-server - - strategy: - fail-fast: false - matrix: - include: - # Running the tests with simulators is incredibly flaky. Our assumption is that simulators might have difficulties - # with communicating with the test server in CI. - # We are going to add these back in https://github.com/getsentry/sentry-cocoa/issues/6361 - - name: macOS 15 - platform: "macOS" - xcode: "16.4" - test-destination-os: "15.0" - - steps: - - uses: actions/checkout@v5 - - uses: actions/download-artifact@v6 - with: - name: test-server - - - name: Start Test Server - run: ./scripts/start-test-server.sh - - run: ./scripts/ci-select-xcode.sh ${{matrix.xcode}} - - # Note: We don't install Slather or gather code coverage for this job because it only runs - # the SentryTestServerTests target, which contains minimal tests that don't significantly - # contribute to overall code coverage metrics. The main unit-tests job handles comprehensive - # code coverage reporting. - - # We split building and running tests in two steps so we know how long running the tests takes. - - name: Build tests - id: build_tests - run: | - ./scripts/sentry-xcodebuild.sh \ - --platform ${{matrix.platform}} \ - --os ${{matrix.test-destination-os}} \ - --ref ${{ github.ref_name }} \ - --command build-for-testing \ - --configuration TestCI \ - --scheme Sentry \ - --test-plan Sentry_TestServer - - - name: Run tests - run: | - ./scripts/sentry-xcodebuild.sh \ - --platform ${{matrix.platform}} \ - --os ${{matrix.test-destination-os}} \ - --ref ${{ github.ref_name }} \ - --command test-without-building \ - --configuration TestCI \ - --scheme Sentry \ - --test-plan Sentry_TestServer - - - name: Archiving DerivedData Logs - uses: actions/upload-artifact@v5 - if: steps.build_tests.outcome == 'failure' - with: - name: derived-data-test-server-${{matrix.platform}}-xcode-${{matrix.xcode}} - path: | - /Users/runner/Library/Developer/Xcode/DerivedData/**/Logs/** - - - name: Archiving Raw Logs - uses: actions/upload-artifact@v5 - if: ${{ failure() || cancelled() }} - with: - name: raw-output-test-server-${{matrix.platform}}-xcode-${{matrix.xcode}} - path: | - raw-build-output.log - raw-build-for-testing-output.log - raw-test-output.log - - - name: Archiving Crash Logs - uses: actions/upload-artifact@v5 - if: ${{ failure() || cancelled() }} - with: - name: crash-logs-test-server-${{matrix.platform}}-xcode-${{matrix.xcode}} - path: | - ~/Library/Logs/DiagnosticReports/** - - - name: Run CI Diagnostics - if: ${{ failure() || cancelled() }} - run: ./scripts/ci-diagnostics.sh - - - name: Store screenshot - uses: ./.github/actions/capture-screenshot - if: ${{ failure() || cancelled() }} - with: - suffix: ${{ matrix.platform }}-xcode-${{ matrix.xcode }} - - unit-tests: - name: Unit ${{matrix.name}} - runs-on: ${{fromJSON(format('["ghcr.io/cirruslabs/macos-runner:{0}", "runner_group_id:10"]', matrix.runs-on)) }} - timeout-minutes: ${{ matrix.timeout || 20 }} - needs: files-changed - if: needs.files-changed.outputs.run_unit_tests_for_prs == 'true' - - strategy: - fail-fast: false - matrix: - # Can't run tests on watchOS because XCTest is not available - include: - # We are running tests on iOS 17 and later, as there were OS-internal changes introduced in succeeding versions. - - # iOS 17 - - name: iOS 17 Sentry - runs-on: sequoia - xcode: "16.0" - test-destination-os: "17.5" - platform: "iOS" - create_device: true - device: "iPhone 15 Pro" - scheme: "Sentry" - - # iOS 18 - Use pre-installed iOS 18.4 runtime on macOS-15 - - name: iOS 18 Sentry - runs-on: sequoia - xcode: "16.4" - test-destination-os: "18.4" - platform: "iOS" - device: "iPhone 16 Pro" - scheme: "Sentry" - - # iOS 26 - Download iOS 26.1 beta runtime on macOS-26 - - name: iOS 26 Sentry - runs-on: tahoe - xcode: "26.1" - test-destination-os: "26.1" - install_platforms: true - platform: "iOS" - create_device: true - device: "iPhone 17 Pro" - scheme: "Sentry" - # This runner seems to be slower than the others, so we give it more time because otherwise it frequently times out. The build step often alone takes 10 minutes and also booting the simulator takes a while. - timeout: 30 - - # We don't run the unit tests on macOS 13 cause we run them on all on GH actions available iOS versions. - # The chance of missing a bug solely on tvOS 16 that doesn't occur on iOS, macOS 12 or macOS 14 is minimal. - # We are running tests on macOS 14 and later, as there were OS-internal changes introduced in succeeding versions. - - # macOS 14 - - name: macOS 14 Sentry - runs-on: sonoma - xcode: "16.1" - test-destination-os: "latest" - platform: "macOS" - scheme: "Sentry" - - # macOS 15 - - name: macOS 15 Sentry - runs-on: sequoia - xcode: "16.4" - test-destination-os: "latest" - platform: "macOS" - scheme: "Sentry" - - # macOS 26 - - name: macOS 26 Sentry - runs-on: tahoe - xcode: "26.1" - test-destination-os: "26.1" - platform: "macOS" - scheme: "Sentry" - - # Catalyst. We test the latest version, as the risk something breaking on Catalyst and not - # on an older iOS or macOS version is low. - # In addition we are running tests on macOS 14, as there were OS-internal changes introduced in succeeding versions. - # TODO: Itay check - - name: Catalyst 14 Sentry - runs-on: sonoma - xcode: "16.1" - test-destination-os: "latest" - platform: "Catalyst" - scheme: "Sentry" - - - name: Catalyst 15 Sentry - runs-on: sequoia - xcode: "16.4" - test-destination-os: "latest" - platform: "Catalyst" - scheme: "Sentry" - - ## We don't run unit tests on macCatalyst 26 yet because of https://github.com/getsentry/sentry-cocoa/issues/6165. - - # We don't run the unit tests on tvOS 16 cause we run them on all on GH actions available iOS versions. - # The chance of missing a bug solely on tvOS 16 that doesn't occur on iOS, tvOS 15 or tvOS 16 is minimal. - # We are running tests on tvOS 17 and latest, as there were OS-internal changes introduced in succeeding versions. - - # tvOS 17 - # TODO: Itay install old platform - # - name: tvOS 17 Sentry - # runs-on: sonoma - # xcode: "15.4" - # test-destination-os: "17.5" - # platform: "tvOS" - # scheme: "Sentry" - - # iOS 17 - Use pre-installed iOS 17.5 - - name: iOS 17 SentrySwiftUI - runs-on: sequoia - xcode: "16.1" - test-destination-os: "17.5" - platform: "iOS" - create_device: true - device: "iPhone 15 Pro" - scheme: "SentrySwiftUI" - - # tvOS 18 - - name: tvOS 18 Sentry - runs-on: sequoia - xcode: "16.4" - test-destination-os: "18.5" - platform: "tvOS" - scheme: "Sentry" - - # tvOS 26 - - name: tvOS 26 Sentry - runs-on: tahoe - xcode: "26.1" - test-destination-os: "26.1" - install_platforms: true - platform: "tvOS" - create_device: true - device: "Apple TV" - scheme: "Sentry" - - steps: - - uses: actions/checkout@v5 - - run: ./scripts/ci-select-xcode.sh ${{matrix.xcode}} - - - name: Install Ruby 3.3.0 - run: rbenv install 3.3.0 - - - name: Install Slather - run: gem install slather - - # Install platform runtimes that are not included by default - # For iOS 16.4: Not included with Xcode 15.4 on macOS-14 (macOS-13 sunset) - # For iOS/tvOS 26.1: Beta platforms not included by default - - name: Install required platforms - if: ${{ matrix.install_platforms }} - run: ./scripts/ci-install-platforms.sh --platforms "${{ matrix.platform }}" --os-version "${{ matrix.test-destination-os }}" - - # Create simulator devices for non-preinstalled simulators - # Required for iOS 16.4, iOS 17.5 (on Xcode 15.4), and iOS/tvOS 26.1 - - name: Create simulator device - if: ${{ matrix.create_device }} - run: ./scripts/ci-create-simulator.sh --platform "${{ matrix.platform }}" --os-version "${{ matrix.test-destination-os }}" --device-name "${{ matrix.device }}" - - # Boot created simulators to ensure they're ready before tests run - # Based on CircleCI forum comment, booting is especially important for Xcode 26: https://discuss.circleci.com/t/xcode-26-rc/54066/18 - - name: Boot simulator - if: ${{ matrix.create_device && matrix.platform == 'iOS' }} - run: ./scripts/ci-boot-simulator.sh --xcode ${{ matrix.xcode }} --device "${{ matrix.device }}" --os-version "${{ matrix.test-destination-os }}" - - # We split building and running tests in two steps so we know how long running the tests takes. - - name: Build Tests - id: build_tests - run: | - ./scripts/sentry-xcodebuild.sh \ - --platform ${{matrix.platform}} \ - --os ${{matrix.test-destination-os}} \ - --ref ${{ github.ref_name }} \ - --command build-for-testing \ - --device "${{matrix.device}}" \ - --configuration TestCI \ - --scheme ${{matrix.scheme}} - - # Run Flaky Tests TestPlan which has a retry mechanism on failure. - # We intentionally run these before the other test plan to fail early. - # Use a separate result bundle name to avoid conflicts with the regular test run. - # xcodebuild fails if a result bundle already exists at the target path. - - name: Run Flaky Tests - # Only the Sentry Scheme has the Flaky TestPlan. - if: ${{ matrix.scheme == 'Sentry' }} - run: | - ./scripts/sentry-xcodebuild.sh \ - --platform ${{matrix.platform}} \ - --os ${{matrix.test-destination-os}} \ - --ref ${{ github.ref_name }} \ - --command test-without-building \ - --device "${{matrix.device}}" \ - --configuration TestCI \ - --scheme ${{matrix.scheme}} \ - --test-plan Sentry_Flaky \ - --result-bundle flaky-results.xcresult - - - name: Run tests - # We call a script with the platform so the destination - # passed to xcodebuild doesn't end up in the job name, - # because GitHub Actions don't provide an easy way of - # manipulating string in expressions. - run: | - ./scripts/sentry-xcodebuild.sh \ - --platform ${{matrix.platform}} \ - --os ${{matrix.test-destination-os}} \ - --ref ${{ github.ref_name }} \ - --command test-without-building \ - --device "${{matrix.device}}" \ - --configuration TestCI \ - --scheme ${{matrix.scheme}} \ - --result-bundle results.xcresult - - - name: Publish Test Report - uses: mikepenz/action-junit-report@5b7ee5a21e8674b695313d769f3cbdfd5d4d53a4 # v6.0.0 - if: always() - with: - report_paths: "build/reports/junit.xml" - fail_on_failure: true - fail_on_parse_error: true - detailed_summary: true - - - name: Archiving DerivedData Logs - uses: actions/upload-artifact@v5 - if: steps.build_tests.outcome == 'failure' - with: - name: derived-data-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}} - path: | - /Users/runner/Library/Developer/Xcode/DerivedData/**/Logs/** - - - name: Archiving Raw Logs - uses: actions/upload-artifact@v5 - if: ${{ failure() || cancelled() }} - with: - name: raw-output-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}} - path: | - raw-build-output.log - raw-build-for-testing-output.log - raw-test-output.log - - - name: Archiving Crash Logs - uses: actions/upload-artifact@v5 - if: ${{ failure() || cancelled() }} - with: - name: crash-logs-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}} - path: | - ~/Library/Logs/DiagnosticReports/** - - - name: Archiving Test Results - uses: actions/upload-artifact@v5 - if: ${{ failure() || cancelled() }} - with: - name: result-bundle-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}} - path: | - results.xcresult - flaky-results.xcresult - - - name: Gather code coverage information via slather - run: slather coverage --configuration TestCI --scheme Sentry - - # We can upload all coverage reports, because codecov merges them. - # See https://docs.codecov.io/docs/merging-reports - # Checkout .codecov.yml to see the config of Codecov - # We don't upload codecov for release branches, as we don't want a failing coverage check to block a release. - # We don't upload codecov for scheduled runs as CodeCov only accepts a limited amount of uploads per commit. - - name: Push code coverage to codecov - id: codecov_1 - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # pin@v5.5.1 - if: ${{ contains(matrix.platform, 'iOS') && !contains(github.ref, 'release') && github.event.schedule == '' }} - with: - # Although public repos should not have to specify a token there seems to be a bug with the Codecov GH action, which can - # be solved by specifying the token, see https://github.com/codecov/codecov-action/issues/557#issuecomment-1224970469 - token: ${{ secrets.CODECOV_TOKEN }} - fail_ci_if_error: false - verbose: true - - # Sometimes codecov uploads etc can fail. Retry one time to rule out e.g. intermittent network failures. - - name: Push code coverage to codecov - id: codecov_2 - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # pin@v5.5.1 - if: ${{ steps.codecov_1.outcome == 'failure' && contains(matrix.platform, 'iOS') && !contains(github.ref, 'release') && github.event.schedule == '' }} - with: - token: ${{ secrets.CODECOV_TOKEN }} - fail_ci_if_error: true - verbose: true - - - name: Codecov test analytics - if: ${{ !cancelled() && !contains(github.ref, 'release') && github.event.schedule == '' }} - uses: codecov/test-results-action@47f89e9acb64b76debcd5ea40642d25a4adced9f # pin@v1.1.1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - verbose: true - name: sentry-cocoa-unit-tests - flags: unittests-${{ matrix.platform }}-${{ matrix.xcode }}-${{ matrix.test-destination-os }}, unittests - - - name: Run CI Diagnostics - if: failure() - run: ./scripts/ci-diagnostics.sh - - - name: Store screenshot - uses: ./.github/actions/capture-screenshot - if: failure() - with: - suffix: unit-tests-${{ matrix.platform }}-xcode-${{ matrix.xcode }}-os-${{ matrix.test-destination-os }} - - # This check validates that either all unit tests passed or were skipped, which allows us - # to make unit tests a required check with only running the unit tests when required. - # So, we don't have to run unit tests, for example, for Changelog or ReadMe changes. - - unit-tests-required-check: - needs: - [ - files-changed, - build-test-server, - distribution-tests, - unit-tests, - unit-tests-with-test-server, - ] - name: Unit Tests - # This is necessary since a failed/skipped dependent job would cause this job to be skipped - if: always() - runs-on: ubuntu-latest - steps: - # If any jobs we depend on fails gets cancelled or times out, this job will fail. - # Skipped jobs are not considered failures. - - name: Check for failures - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: | - echo "One of the unit test jobs has failed." && exit 1 From 711a6918b825d2cef5f408745a494488f93bb603 Mon Sep 17 00:00:00 2001 From: Itay Brenner Date: Tue, 4 Nov 2025 15:54:47 -0300 Subject: [PATCH 09/19] Improve matrix logic --- .github/workflows/test.yml | 118 +++++++++++++++---------------------- 1 file changed, 47 insertions(+), 71 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6142b577254..abbc03a944a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,6 +39,7 @@ jobs: with: token: ${{ github.token }} filters: .github/file-filters.yml + build-test-server: name: Build test server on ${{matrix.runner}} if: needs.files-changed.outputs.run_unit_tests_for_prs == 'true' @@ -90,7 +91,7 @@ jobs: - name: Archiving DerivedData uses: actions/upload-artifact@v5 with: - name: test-server + name: test-server-${{matrix.runner}} path: | ./test-server/test-server-exec @@ -252,7 +253,7 @@ jobs: unit-tests: name: Unit ${{matrix.name}} on ${{matrix.runner}} - runs-on: ${{matrix.runs-on}} + runs-on: ${{ matrix.runner == 'Cirrus Labs' && fromJSON(format('["ghcr.io/cirruslabs/macos-runner:{0}", "runner_group_id:10"]', matrix.runs-on-cirrus)) || matrix.runs-on }} timeout-minutes: ${{ matrix.timeout || 20 }} needs: files-changed if: needs.files-changed.outputs.run_unit_tests_for_prs == 'true' @@ -269,7 +270,10 @@ jobs: # iOS 17 - Use pre-installed iOS 17.5 runtime on macOS-14 with Xcode 15.4 - name: iOS 17 Sentry - os: macos-14 + runs-on: macos-14 + runs-on-cirrus: sequoia + xcode: "15.4" + xcode-cirrus: "16.0" test-destination-os: "17.5" platform: "iOS" create_device: true @@ -278,7 +282,8 @@ jobs: # iOS 18 - Use pre-installed iOS 18.4 runtime on macOS-15 - name: iOS 18 Sentry - os: macos-15 + runs-on: macos-15 + runs-on-cirrus: sequoia xcode: "16.4" test-destination-os: "18.4" platform: "iOS" @@ -287,7 +292,8 @@ jobs: # iOS 26 - Download iOS 26.1 beta runtime on macOS-26 - name: iOS 26 Sentry - os: macos-26 + runs-on: macos-26 + runs-on-cirrus: tahoe xcode: "26.1" test-destination-os: "26.1" install_platforms: true @@ -304,14 +310,18 @@ jobs: # macOS 14 - name: macOS 14 Sentry - os: macos-14 + runs-on: macos-14 + runs-on-cirrus: sequoia + xcode: "15.4" + xcode-cirrus: "16.1" test-destination-os: "latest" platform: "macOS" scheme: "Sentry" # macOS 15 - name: macOS 15 Sentry - os: macos-15 + runs-on: macos-15 + runs-on-cirrus: sequoia xcode: "16.4" test-destination-os: "latest" platform: "macOS" @@ -319,7 +329,8 @@ jobs: # macOS 26 - name: macOS 26 Sentry - os: macos-26 + runs-on: macos-26 + runs-on-cirrus: tahoe xcode: "26.1" test-destination-os: "26.1" platform: "macOS" @@ -329,13 +340,17 @@ jobs: # on an older iOS or macOS version is low. # In addition we are running tests on macOS 14, as there were OS-internal changes introduced in succeeding versions. - name: Catalyst 14 Sentry - os: macos-14 + runs-on: macos-14 + runs-on-cirrus: sonoma + xcode: "15.4" + xcode-cirrus: "16.1" test-destination-os: "latest" platform: "Catalyst" scheme: "Sentry" - name: Catalyst 15 Sentry - os: macos-15 + runs-on: macos-15 + xcode: "16.4" test-destination-os: "latest" platform: "Catalyst" scheme: "Sentry" @@ -349,13 +364,19 @@ jobs: # tvOS 17 - name: tvOS 17 Sentry runs-on: macos-14 + runs-on-cirrus: sequoia + xcode: "15.4" + xcode-cirrus: "16.1" test-destination-os: "17.5" platform: "tvOS" scheme: "Sentry" - # iOS 17 + # iOS 17 - Use pre-installed iOS 17.5 runtime on macOS-14 with Xcode 15.4 - name: iOS 17 SentrySwiftUI - os: macos-14 + runs-on: macos-14 + runs-on-cirrus: sequoia + xcode: "15.4" + xcode-cirrus: "16.1" test-destination-os: "17.5" platform: "iOS" create_device: true @@ -364,7 +385,8 @@ jobs: # tvOS 18 - name: tvOS 18 Sentry - os: macos-15 + runs-on: macos-15 + runs-on-cirrus: sequoia xcode: "16.4" test-destination-os: "18.4" platform: "tvOS" @@ -372,7 +394,8 @@ jobs: # tvOS 26 - name: tvOS 26 Sentry - os: macos-26 + runs-on: macos-26 + runs-on-cirrus: tahoe xcode: "26.1" test-destination-os: "26.1" install_platforms: true @@ -380,58 +403,11 @@ jobs: create_device: true device: "Apple TV" scheme: "Sentry" - - # These items allow us to complement the matrix and set the runs-on for the runners. - # See https://docs.github.com/en/actions/how-tos/write-workflows/choose-what-workflows-do/run-job-variations#expanding-or-adding-matrix-configurations - # for more details - - runner: GitHub - os: macos-14 - runs-on: macos-14 - xcode: "15.4" - - - runner: GitHub - os: macos-15 - runs-on: macos-15 - - - runner: GitHub - os: macos-26 - runs-on: macos-26 - - # Cirrus Labs's sequoia runner doesn't have Xcode 15.4 installed, so we use Xcode 16.1. - - runner: Cirrus Labs - os: macos-14 - test-destination-os: "17.5" - runs-on: [ - "ghcr.io/cirruslabs/macos-runner:sequoia", - "runner_group_id:10", - ] - xcode: "16.1" - - - runner: Cirrus Labs - os: macos-14 - runs-on: [ - "ghcr.io/cirruslabs/macos-runner:sonoma", - "runner_group_id:10", - ] - xcode: "16.1" - - - runner: Cirrus Labs - os: macos-15 - runs-on: [ - "ghcr.io/cirruslabs/macos-runner:sequoia", - "runner_group_id:10", - ] - - - runner: Cirrus Labs - os: macos-26 - runs-on: [ - "ghcr.io/cirruslabs/macos-runner:tahoe", - "runner_group_id:10", - ] - + env: + XCODE_VERSION: ${{ matrix.runner == 'Cirrus Labs' && matrix.xcode-cirrus || matrix.xcode }} steps: - uses: actions/checkout@v5 - - run: ./scripts/ci-select-xcode.sh ${{matrix.xcode}} + - run: ./scripts/ci-select-xcode.sh ${{env.XCODE_VERSION}} # Install Ruby for Cirrus Labs runners - name: Install Ruby 3.3.0 @@ -458,7 +434,7 @@ jobs: # Based on CircleCI forum comment, booting is especially important for Xcode 26: https://discuss.circleci.com/t/xcode-26-rc/54066/18 - name: Boot simulator if: ${{ matrix.create_device && matrix.platform == 'iOS' }} - run: ./scripts/ci-boot-simulator.sh --xcode ${{ matrix.xcode }} --device "${{ matrix.device }}" --os-version "${{ matrix.test-destination-os }}" + run: ./scripts/ci-boot-simulator.sh --xcode ${{ env.XCODE_VERSION }} --device "${{ matrix.device }}" --os-version "${{ matrix.test-destination-os }}" # We split building and running tests in two steps so we know how long running the tests takes. - name: Build Tests @@ -521,7 +497,7 @@ jobs: uses: actions/upload-artifact@v5 if: steps.build_tests.outcome == 'failure' with: - name: derived-data-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}}-${{matrix.runner}} + name: derived-data-${{matrix.platform}}-xcode-${{env.XCODE_VERSION}}-os-${{matrix.test-destination-os}}-${{matrix.runner}} path: | /Users/runner/Library/Developer/Xcode/DerivedData/**/Logs/** @@ -529,7 +505,7 @@ jobs: uses: actions/upload-artifact@v5 if: ${{ failure() || cancelled() }} with: - name: raw-output-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}}-${{matrix.runner}} + name: raw-output-${{matrix.platform}}-xcode-${{env.XCODE_VERSION}}-os-${{matrix.test-destination-os}}-${{matrix.runner}} path: | raw-build-output.log raw-build-for-testing-output.log @@ -539,7 +515,7 @@ jobs: uses: actions/upload-artifact@v5 if: ${{ failure() || cancelled() }} with: - name: crash-logs-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}}-${{matrix.runner}} + name: crash-logs-${{matrix.platform}}-xcode-${{env.XCODE_VERSION}}-os-${{matrix.test-destination-os}}-${{matrix.runner}} path: | ~/Library/Logs/DiagnosticReports/** @@ -547,7 +523,7 @@ jobs: uses: actions/upload-artifact@v5 if: ${{ failure() || cancelled() }} with: - name: result-bundle-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}}-${{matrix.runner}} + name: result-bundle-${{matrix.platform}}-xcode-${{env.XCODE_VERSION}}-os-${{matrix.test-destination-os}}-${{matrix.runner}} path: | results.xcresult flaky-results.xcresult @@ -588,7 +564,7 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} verbose: true name: sentry-cocoa-unit-tests - flags: unittests-${{ matrix.platform }}-${{ matrix.xcode }}-${{ matrix.test-destination-os }}, unittests + flags: unittests-${{ matrix.platform }}-${{ env.XCODE_VERSION }}-${{ matrix.test-destination-os }}, unittests - name: Run CI Diagnostics if: failure() @@ -598,7 +574,7 @@ jobs: uses: ./.github/actions/capture-screenshot if: failure() with: - suffix: unit-tests-${{ matrix.platform }}-xcode-${{ matrix.xcode }}-os-${{ matrix.test-destination-os }}-${{matrix.runner}} + suffix: unit-tests-${{ matrix.platform }}-xcode-${{ env.XCODE_VERSION }}-os-${{ matrix.test-destination-os }}-${{matrix.runner}} # This check validates that either all unit tests passed or were skipped, which allows us # to make unit tests a required check with only running the unit tests when required. From 2f311bc258e9c2f21539e92e12b99ff821a7c423 Mon Sep 17 00:00:00 2001 From: Itay Brenner Date: Tue, 4 Nov 2025 16:09:24 -0300 Subject: [PATCH 10/19] Add job id --- .github/workflows/test.yml | 91 ++++++++++++++++++++++++-------------- 1 file changed, 58 insertions(+), 33 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index abbc03a944a..60d6a7a0ed6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -155,25 +155,25 @@ jobs: strategy: fail-fast: false matrix: - xcode: - - "16.4" - name: - - "macOS 15" - test-destination-os: - - "15.0" - platform: - - "macOS" include: # Running the tests with simulators is incredibly flaky. Our assumption is that simulators might have difficulties # with communicating with the test server in CI. # We are going to add these back in https://github.com/getsentry/sentry-cocoa/issues/6361 - - runner: GitHub + - name: "macOS 15" runs-on: macos-15 - - runner: Cirrus Labs + xcode: "16.4" + runner: GitHub + test-destination-os: "15.0" + platform: "macOS" + - name: "macOS 15" + runner: Cirrus Labs runs-on: [ "ghcr.io/cirruslabs/macos-runner:sequoia", "runner_group_id:10", ] + xcode: "16.4" + test-destination-os: "15.0" + platform: "macOS" steps: - uses: actions/checkout@v5 @@ -264,12 +264,26 @@ jobs: runner: - GitHub - Cirrus Labs + job_id: + - ios_17 + - ios_18 + - ios_26 + - macos_14 + - macos_15 + - macos_26 + - catalyst_14 + - catalyst_15 + - tvos_17 + - tvos_18 + - tvos_26 + - ios_17_swiftui # Can't run tests on watchOS because XCTest is not available include: # We are running tests on iOS 17 and later, as there were OS-internal changes introduced in succeeding versions. # iOS 17 - Use pre-installed iOS 17.5 runtime on macOS-14 with Xcode 15.4 - - name: iOS 17 Sentry + - job_id: ios_17 + name: iOS 17 Sentry runs-on: macos-14 runs-on-cirrus: sequoia xcode: "15.4" @@ -281,7 +295,8 @@ jobs: scheme: "Sentry" # iOS 18 - Use pre-installed iOS 18.4 runtime on macOS-15 - - name: iOS 18 Sentry + - job_id: ios_18 + name: iOS 18 Sentry runs-on: macos-15 runs-on-cirrus: sequoia xcode: "16.4" @@ -291,7 +306,8 @@ jobs: scheme: "Sentry" # iOS 26 - Download iOS 26.1 beta runtime on macOS-26 - - name: iOS 26 Sentry + - job_id: ios_26 + name: iOS 26 Sentry runs-on: macos-26 runs-on-cirrus: tahoe xcode: "26.1" @@ -309,7 +325,8 @@ jobs: # We are running tests on macOS 14 and later, as there were OS-internal changes introduced in succeeding versions. # macOS 14 - - name: macOS 14 Sentry + - job_id: macos_14 + name: macOS 14 Sentry runs-on: macos-14 runs-on-cirrus: sequoia xcode: "15.4" @@ -319,7 +336,8 @@ jobs: scheme: "Sentry" # macOS 15 - - name: macOS 15 Sentry + - job_id: macos_15 + name: macOS 15 Sentry runs-on: macos-15 runs-on-cirrus: sequoia xcode: "16.4" @@ -328,7 +346,8 @@ jobs: scheme: "Sentry" # macOS 26 - - name: macOS 26 Sentry + - job_id: macos_26 + name: macOS 26 Sentry runs-on: macos-26 runs-on-cirrus: tahoe xcode: "26.1" @@ -339,7 +358,8 @@ jobs: # Catalyst. We test the latest version, as the risk something breaking on Catalyst and not # on an older iOS or macOS version is low. # In addition we are running tests on macOS 14, as there were OS-internal changes introduced in succeeding versions. - - name: Catalyst 14 Sentry + - job_id: catalyst_14 + name: Catalyst 14 Sentry runs-on: macos-14 runs-on-cirrus: sonoma xcode: "15.4" @@ -348,7 +368,8 @@ jobs: platform: "Catalyst" scheme: "Sentry" - - name: Catalyst 15 Sentry + - job_id: catalyst_15 + name: Catalyst 15 Sentry runs-on: macos-15 xcode: "16.4" test-destination-os: "latest" @@ -362,7 +383,8 @@ jobs: # We are running tests on tvOS 17 and latest, as there were OS-internal changes introduced in succeeding versions. # tvOS 17 - - name: tvOS 17 Sentry + - job_id: tvos_17 + name: tvOS 17 Sentry runs-on: macos-14 runs-on-cirrus: sequoia xcode: "15.4" @@ -371,20 +393,9 @@ jobs: platform: "tvOS" scheme: "Sentry" - # iOS 17 - Use pre-installed iOS 17.5 runtime on macOS-14 with Xcode 15.4 - - name: iOS 17 SentrySwiftUI - runs-on: macos-14 - runs-on-cirrus: sequoia - xcode: "15.4" - xcode-cirrus: "16.1" - test-destination-os: "17.5" - platform: "iOS" - create_device: true - device: "iPhone 15 Pro" - scheme: "SentrySwiftUI" - # tvOS 18 - - name: tvOS 18 Sentry + - job_id: tvos_18 + name: tvOS 18 Sentry runs-on: macos-15 runs-on-cirrus: sequoia xcode: "16.4" @@ -393,7 +404,8 @@ jobs: scheme: "Sentry" # tvOS 26 - - name: tvOS 26 Sentry + - job_id: tvos_26 + name: tvOS 26 Sentry runs-on: macos-26 runs-on-cirrus: tahoe xcode: "26.1" @@ -403,6 +415,19 @@ jobs: create_device: true device: "Apple TV" scheme: "Sentry" + + # iOS 17 - Use pre-installed iOS 17.5 runtime on macOS-14 with Xcode 15.4 + - job_id: ios_17_swiftui + name: iOS 17 SentrySwiftUI + runs-on: macos-14 + runs-on-cirrus: sequoia + xcode: "15.4" + xcode-cirrus: "16.1" + test-destination-os: "17.5" + platform: "iOS" + create_device: true + device: "iPhone 15 Pro" + scheme: "SentrySwiftUI" env: XCODE_VERSION: ${{ matrix.runner == 'Cirrus Labs' && matrix.xcode-cirrus || matrix.xcode }} steps: From 4d475f4899283d0042c8c48321b26174f5fbe0ab Mon Sep 17 00:00:00 2001 From: Itay Brenner Date: Tue, 4 Nov 2025 16:47:27 -0300 Subject: [PATCH 11/19] Select xcode version for distribution test --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 60d6a7a0ed6..d41e462509c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -125,6 +125,7 @@ jobs: ] steps: - uses: actions/checkout@v5 + - run: ./scripts/ci-select-xcode.sh "16.4" - name: Prepare Package.swift uses: ./.github/actions/prepare-package.swift with: @@ -181,7 +182,7 @@ jobs: with: name: test-server-${{matrix.runner}} - name: Rename artifact - run: mv test-server-${{matrix.runner}} test-server + run: mv "test-server-${{matrix.runner}}" "test-server" - name: Start Test Server run: ./scripts/start-test-server.sh - run: ./scripts/ci-select-xcode.sh ${{matrix.xcode}} From 051165c45878e8e391ef4f827b7efdc4a3598389 Mon Sep 17 00:00:00 2001 From: Itay Brenner Date: Tue, 4 Nov 2025 16:56:03 -0300 Subject: [PATCH 12/19] Add CI diagnostics step --- .github/workflows/test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d41e462509c..4fc64596c9f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -142,6 +142,9 @@ jobs: name: raw-test-output-distribution-${{matrix.runner}} path: | raw-test-output-distribution.log + - name: Run CI Diagnostics + if: ${{ failure() || cancelled() }} + run: ./scripts/ci-diagnostics.sh # This matrix runs only the unit tests requiring the test server. # We do this to speed up the other unit test jobs and to avoid running the @@ -182,7 +185,7 @@ jobs: with: name: test-server-${{matrix.runner}} - name: Rename artifact - run: mv "test-server-${{matrix.runner}}" "test-server" + run: mv test-server-${{matrix.runner}} test-server-exec - name: Start Test Server run: ./scripts/start-test-server.sh - run: ./scripts/ci-select-xcode.sh ${{matrix.xcode}} From 96040b594b9a86104a3b11bee581e6e76e0bc22a Mon Sep 17 00:00:00 2001 From: Itay Brenner Date: Tue, 4 Nov 2025 18:15:54 -0300 Subject: [PATCH 13/19] Fix ruby not installed --- .github/workflows/test.yml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4fc64596c9f..dcb3de72327 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -184,8 +184,6 @@ jobs: - uses: actions/download-artifact@v6 with: name: test-server-${{matrix.runner}} - - name: Rename artifact - run: mv test-server-${{matrix.runner}} test-server-exec - name: Start Test Server run: ./scripts/start-test-server.sh - run: ./scripts/ci-select-xcode.sh ${{matrix.xcode}} @@ -387,6 +385,7 @@ jobs: # We are running tests on tvOS 17 and latest, as there were OS-internal changes introduced in succeeding versions. # tvOS 17 + # Cirrus labs doesn't have a tvOS 17 runner - job_id: tvos_17 name: tvOS 17 Sentry runs-on: macos-14 @@ -437,15 +436,12 @@ jobs: steps: - uses: actions/checkout@v5 - run: ./scripts/ci-select-xcode.sh ${{env.XCODE_VERSION}} - - # Install Ruby for Cirrus Labs runners - - name: Install Ruby 3.3.0 - if: ${{ matrix.runner == 'Cirrus Labs' }} - run: rbenv install 3.3.0 - + - name: Setup Ruby + uses: ruby/setup-ruby@13e7a03dc3ac6c3798f4570bfead2aed4d96abfb # v1.244.0 + with: + bundler-cache: true - name: Install Slather run: gem install slather - # Install platform runtimes that are not included by default # For iOS 16.4: Not included with Xcode 15.4 on macOS-14 (macOS-13 sunset) # For iOS/tvOS 26.1: Beta platforms not included by default From e5874ea3342aa3f5ceb11f7016a6e93e72020096 Mon Sep 17 00:00:00 2001 From: Itay Brenner Date: Wed, 5 Nov 2025 17:59:46 -0300 Subject: [PATCH 14/19] Use xcodes select --- scripts/ci-select-xcode.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci-select-xcode.sh b/scripts/ci-select-xcode.sh index 5096fd16417..892cfe2f1ea 100755 --- a/scripts/ci-select-xcode.sh +++ b/scripts/ci-select-xcode.sh @@ -15,7 +15,7 @@ XCODE_VERSION="${1}" # We prefer this over calling `sudo xcode-select` because it will fail if the Xcode version # is not installed. Also xcodes is preinstalled on the GH runners. -xcodes select "$XCODE_VERSION" +sudo xcodes select "$XCODE_VERSION" swiftc --version From 6aa3ba4f7a8cb4f1b3b3e60f9bbc9ae1cde2388d Mon Sep 17 00:00:00 2001 From: Itay Brenner Date: Wed, 5 Nov 2025 18:20:09 -0300 Subject: [PATCH 15/19] Run xcrun with sudo --- scripts/ci-select-xcode.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci-select-xcode.sh b/scripts/ci-select-xcode.sh index 892cfe2f1ea..651d90798d6 100755 --- a/scripts/ci-select-xcode.sh +++ b/scripts/ci-select-xcode.sh @@ -28,7 +28,7 @@ begin_group "List Available Simulators" start_time=$(date +%s) -xcrun simctl list +sudo xcrun simctl list end_time=$(date +%s) xcrun_simctl_list_duration=$((end_time - start_time)) From cf0779b16084ecbc5114875f0d6195cfdc6e0c93 Mon Sep 17 00:00:00 2001 From: Itay Brenner Date: Wed, 5 Nov 2025 18:45:46 -0300 Subject: [PATCH 16/19] Uso sudo everywhere --- .github/workflows/test.yml | 14 +++++++------- scripts/ci-select-xcode.sh | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ecfa9b8601e..295bfc379a3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -125,7 +125,7 @@ jobs: ] steps: - uses: actions/checkout@v5 - - run: ./scripts/ci-select-xcode.sh "16.4" + - run: sudo ./scripts/ci-select-xcode.sh "16.4" - name: Prepare Package.swift uses: ./.github/actions/prepare-package.swift with: @@ -133,7 +133,7 @@ jobs: change-path: false remove-duplicate: true - run: rm -r Sentry.xcodeproj && rm -r Sentry.xcworkspace - - run: set -o pipefail && NSUnbufferedIO=YES SKIP_BINARIES=1 xcodebuild test -scheme Sentry-Package -sdk iphonesimulator -destination 'platform=iOS Simulator,OS=18.4,name=iPhone 16 Pro' | tee raw-test-output-distribution.log | xcbeautify --preserve-unbeautified + - run: set -o pipefail && sudo NSUnbufferedIO=YES SKIP_BINARIES=1 xcodebuild test -scheme Sentry-Package -sdk iphonesimulator -destination 'platform=iOS Simulator,OS=18.4,name=iPhone 16 Pro' | tee raw-test-output-distribution.log | xcbeautify --preserve-unbeautified shell: sh - name: Upload Distribution Test Logs uses: actions/upload-artifact@v5 @@ -467,19 +467,19 @@ jobs: # Required for iOS 16.4, iOS 17.5 (on Xcode 15.4), and iOS/tvOS 26.1 - name: Create simulator device if: ${{ matrix.create_device }} - run: ./scripts/ci-create-simulator.sh --platform "${{ matrix.platform }}" --os-version "${{ matrix.test-destination-os }}" --device-name "${{ matrix.device }}" + run: sudo ./scripts/ci-create-simulator.sh --platform "${{ matrix.platform }}" --os-version "${{ matrix.test-destination-os }}" --device-name "${{ matrix.device }}" # Boot created simulators to ensure they're ready before tests run # Based on CircleCI forum comment, booting is especially important for Xcode 26: https://discuss.circleci.com/t/xcode-26-rc/54066/18 - name: Boot simulator if: ${{ matrix.create_device && matrix.platform == 'iOS' }} - run: ./scripts/ci-boot-simulator.sh --xcode ${{ env.XCODE_VERSION }} --device "${{ matrix.device }}" --os-version "${{ matrix.test-destination-os }}" + run: sudo ./scripts/ci-boot-simulator.sh --xcode ${{ env.XCODE_VERSION }} --device "${{ matrix.device }}" --os-version "${{ matrix.test-destination-os }}" # We split building and running tests in two steps so we know how long running the tests takes. - name: Build Tests id: build_tests run: | - ./scripts/sentry-xcodebuild.sh \ + sudo ./scripts/sentry-xcodebuild.sh \ --platform ${{matrix.platform}} \ --os ${{matrix.test-destination-os}} \ --ref ${{ github.ref_name }} \ @@ -496,7 +496,7 @@ jobs: # Only the Sentry Scheme has the Flaky TestPlan. if: ${{ matrix.scheme == 'Sentry' }} run: | - ./scripts/sentry-xcodebuild.sh \ + sudo ./scripts/sentry-xcodebuild.sh \ --platform ${{matrix.platform}} \ --os ${{matrix.test-destination-os}} \ --ref ${{ github.ref_name }} \ @@ -513,7 +513,7 @@ jobs: # because GitHub Actions don't provide an easy way of # manipulating string in expressions. run: | - ./scripts/sentry-xcodebuild.sh \ + sudo ./scripts/sentry-xcodebuild.sh \ --platform ${{matrix.platform}} \ --os ${{matrix.test-destination-os}} \ --ref ${{ github.ref_name }} \ diff --git a/scripts/ci-select-xcode.sh b/scripts/ci-select-xcode.sh index 651d90798d6..5096fd16417 100755 --- a/scripts/ci-select-xcode.sh +++ b/scripts/ci-select-xcode.sh @@ -15,7 +15,7 @@ XCODE_VERSION="${1}" # We prefer this over calling `sudo xcode-select` because it will fail if the Xcode version # is not installed. Also xcodes is preinstalled on the GH runners. -sudo xcodes select "$XCODE_VERSION" +xcodes select "$XCODE_VERSION" swiftc --version @@ -28,7 +28,7 @@ begin_group "List Available Simulators" start_time=$(date +%s) -sudo xcrun simctl list +xcrun simctl list end_time=$(date +%s) xcrun_simctl_list_duration=$((end_time - start_time)) From 86798dcdc70c90cfa2646b51ceb428409f25ea99 Mon Sep 17 00:00:00 2001 From: Itay Brenner Date: Thu, 6 Nov 2025 00:02:13 -0300 Subject: [PATCH 17/19] Tsst correcting permissions --- .github/workflows/test.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 295bfc379a3..7cc2100c3ab 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -125,7 +125,9 @@ jobs: ] steps: - uses: actions/checkout@v5 - - run: sudo ./scripts/ci-select-xcode.sh "16.4" + # Ensure the user has write permissions to the Xcode DerivedData directory + - run: sudo chown -R $(whoami) ~/Library/Developer + - run: ./scripts/ci-select-xcode.sh "16.4" - name: Prepare Package.swift uses: ./.github/actions/prepare-package.swift with: @@ -133,7 +135,7 @@ jobs: change-path: false remove-duplicate: true - run: rm -r Sentry.xcodeproj && rm -r Sentry.xcworkspace - - run: set -o pipefail && sudo NSUnbufferedIO=YES SKIP_BINARIES=1 xcodebuild test -scheme Sentry-Package -sdk iphonesimulator -destination 'platform=iOS Simulator,OS=18.4,name=iPhone 16 Pro' | tee raw-test-output-distribution.log | xcbeautify --preserve-unbeautified + - run: set -o pipefail && NSUnbufferedIO=YES SKIP_BINARIES=1 xcodebuild test -scheme Sentry-Package -sdk iphonesimulator -destination 'platform=iOS Simulator,OS=18.4,name=iPhone 16 Pro' | tee raw-test-output-distribution.log | xcbeautify --preserve-unbeautified shell: sh - name: Upload Distribution Test Logs uses: actions/upload-artifact@v5 @@ -467,19 +469,19 @@ jobs: # Required for iOS 16.4, iOS 17.5 (on Xcode 15.4), and iOS/tvOS 26.1 - name: Create simulator device if: ${{ matrix.create_device }} - run: sudo ./scripts/ci-create-simulator.sh --platform "${{ matrix.platform }}" --os-version "${{ matrix.test-destination-os }}" --device-name "${{ matrix.device }}" + run: ./scripts/ci-create-simulator.sh --platform "${{ matrix.platform }}" --os-version "${{ matrix.test-destination-os }}" --device-name "${{ matrix.device }}" # Boot created simulators to ensure they're ready before tests run # Based on CircleCI forum comment, booting is especially important for Xcode 26: https://discuss.circleci.com/t/xcode-26-rc/54066/18 - name: Boot simulator if: ${{ matrix.create_device && matrix.platform == 'iOS' }} - run: sudo ./scripts/ci-boot-simulator.sh --xcode ${{ env.XCODE_VERSION }} --device "${{ matrix.device }}" --os-version "${{ matrix.test-destination-os }}" + run: ./scripts/ci-boot-simulator.sh --xcode ${{ env.XCODE_VERSION }} --device "${{ matrix.device }}" --os-version "${{ matrix.test-destination-os }}" # We split building and running tests in two steps so we know how long running the tests takes. - name: Build Tests id: build_tests run: | - sudo ./scripts/sentry-xcodebuild.sh \ + ./scripts/sentry-xcodebuild.sh \ --platform ${{matrix.platform}} \ --os ${{matrix.test-destination-os}} \ --ref ${{ github.ref_name }} \ @@ -496,7 +498,7 @@ jobs: # Only the Sentry Scheme has the Flaky TestPlan. if: ${{ matrix.scheme == 'Sentry' }} run: | - sudo ./scripts/sentry-xcodebuild.sh \ + ./scripts/sentry-xcodebuild.sh \ --platform ${{matrix.platform}} \ --os ${{matrix.test-destination-os}} \ --ref ${{ github.ref_name }} \ @@ -513,7 +515,7 @@ jobs: # because GitHub Actions don't provide an easy way of # manipulating string in expressions. run: | - sudo ./scripts/sentry-xcodebuild.sh \ + ./scripts/sentry-xcodebuild.sh \ --platform ${{matrix.platform}} \ --os ${{matrix.test-destination-os}} \ --ref ${{ github.ref_name }} \ From e8a7288090588643de3e03df061bdfb1440ff20a Mon Sep 17 00:00:00 2001 From: Itay Brenner Date: Thu, 6 Nov 2025 00:20:32 -0300 Subject: [PATCH 18/19] Test removing sudo --- .github/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7cc2100c3ab..666ed92a233 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -126,7 +126,6 @@ jobs: steps: - uses: actions/checkout@v5 # Ensure the user has write permissions to the Xcode DerivedData directory - - run: sudo chown -R $(whoami) ~/Library/Developer - run: ./scripts/ci-select-xcode.sh "16.4" - name: Prepare Package.swift uses: ./.github/actions/prepare-package.swift From 8df036e2c81769366c44409f07ddd068db150358 Mon Sep 17 00:00:00 2001 From: Itay Brenner Date: Thu, 6 Nov 2025 00:58:26 -0300 Subject: [PATCH 19/19] Update GitHub Actions workflow to add 'runs-on-cirrus' for Catalyst 15 Sentry job --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 666ed92a233..1921b63e0f4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -375,6 +375,7 @@ jobs: - job_id: catalyst_15 name: Catalyst 15 Sentry runs-on: macos-15 + runs-on-cirrus: sequoia xcode: "16.4" test-destination-os: "latest" platform: "Catalyst"