WIP: [common] Remove dependency on pub-sub broker timestamps #10927
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # GitHub Actions workflow for running compatibility tests: Avro and Pulsar Venice integration tests | |
| name: TestsDeCompatibilite | |
| on: [push, pull_request, workflow_dispatch] | |
| jobs: | |
| AvroCompatibilityTests: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| jdk: [17] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/gradle-test-setup | |
| with: | |
| java-version: ${{ matrix.jdk }} | |
| - name: Run Avro Compatibility Tests | |
| run: ./gradlew -DmaxParallelForks=2 --parallel --build-cache :internal:venice-avro-compatibility-test:test --continue | |
| - name: Package Build Artifacts | |
| if: success() || failure() | |
| shell: bash | |
| run: | | |
| mkdir ${{ github.job }}-artifacts | |
| find . -path "**/build/reports/*" -or -path "**/build/test-results/*" > artifacts.list | |
| rsync -R --files-from=artifacts.list . ${{ github.job }}-artifacts | |
| tar -zcvf ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz ${{ github.job }}-artifacts | |
| - name: Upload Build Artifacts | |
| if: success() || failure() | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: ${{ github.job }} | |
| path: ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz | |
| retention-days: 30 | |
| DuckVinciIntegrationTests: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| jdk: [17] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/gradle-test-setup | |
| with: | |
| java-version: ${{ matrix.jdk }} | |
| - name: Run DuckDB Integration Tests | |
| run: ./gradlew -DforkEvery=1 -DmaxParallelForks=1 --parallel --build-cache :integrations:venice-duckdb:integrationTest --continue | |
| - name: Package Build Artifacts | |
| if: success() || failure() | |
| shell: bash | |
| run: | | |
| mkdir ${{ github.job }}-artifacts | |
| find . -path "**/build/reports/*" -or -path "**/build/test-results/*" > artifacts.list | |
| rsync -R --files-from=artifacts.list . ${{ github.job }}-artifacts | |
| tar -zcvf ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz ${{ github.job }}-artifacts | |
| - name: Upload Build Artifacts | |
| if: success() || failure() | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: ${{ github.job }} | |
| path: ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz | |
| retention-days: 30 | |
| PulsarVeniceIntegrationTests: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| jdk: [17] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/gradle-test-setup | |
| with: | |
| java-version: ${{ matrix.jdk }} | |
| - name: Build NAR, shadow jars, and Docker images | |
| shell: bash | |
| run: | | |
| # Single Gradle invocation: NAR for Pulsar connector + shadow jars for Venice Docker images | |
| ./gradlew --parallel --build-cache --no-daemon \ | |
| :integrations:venice-pulsar:nar \ | |
| :services:venice-controller:shadowJar \ | |
| :services:venice-server:shadowJar \ | |
| :services:venice-router:shadowJar \ | |
| :clients:venice-admin-tool:shadowJar \ | |
| :clients:venice-thin-client:shadowJar \ | |
| :clients:venice-push-job:shadowJar \ | |
| :clients:venice-client:shadowJar | |
| # Build Pulsar and Venice Docker images in parallel | |
| docker build --tag=pulsar/venice-test:latest-dev -f tests/docker-images/pulsar-sink/Dockerfile . & | |
| pulsar_docker_pid=$! | |
| cd docker | |
| ./build-venice-docker-images.sh venicedb latest-dev "venice-controller venice-server venice-router venice-client" | |
| cd .. | |
| wait $pulsar_docker_pid || { echo "Pulsar Docker build failed"; exit 1; } | |
| - name: Run the test | |
| shell: bash | |
| run: | | |
| ./gradlew :tests:venice-pulsar-test:pulsarIntegrationTest -i | |
| - name: Package Build Artifacts | |
| if: success() || failure() | |
| shell: bash | |
| run: | | |
| mkdir ${{ github.job }}-artifacts | |
| find . -path "**/build/reports/*" -or -path "**/build/test-results/*" > artifacts.list | |
| rsync -R --files-from=artifacts.list . ${{ github.job }}-artifacts | |
| tar -zcvf ${{ github.job }}-artifacts.tar.gz ${{ github.job }}-artifacts | |
| - name: Upload Build Artifacts | |
| if: success() || failure() | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: ${{ github.job }} | |
| path: ${{ github.job }}-artifacts.tar.gz | |
| retention-days: 30 | |
| # This workflow gets skipped if all upstream jobs succeeded, but it runs (and fails) if any upstream failed. | |
| # This is what we want since GitHub required actions do not block merging when skipped, but can block it when failed. | |
| CompatibilityTestsFailureAlert: | |
| strategy: | |
| fail-fast: false | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| needs: [AvroCompatibilityTests, DuckVinciIntegrationTests, PulsarVeniceIntegrationTests] | |
| timeout-minutes: 30 | |
| if: ${{ cancelled() || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'failure') }} | |
| steps: | |
| - name: NoGood | |
| shell: bash | |
| run: | | |
| echo "Some workflows have failed!" | |
| exit 1 |