refactor(bench): remove TestServer startup capability (#2611) #232
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| name: Post-merge | |
| on: | |
| push: | |
| branches: [master] | |
| permissions: | |
| contents: read | |
| packages: write | |
| id-token: write | |
| concurrency: | |
| group: post-merge-${{ github.ref }} | |
| cancel-in-progress: false | |
| env: | |
| IGGY_CI_BUILD: true | |
| jobs: | |
| plan: | |
| name: Plan dockerhub components | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.mk.outputs.matrix }} | |
| components: ${{ steps.mk.outputs.components }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Load publish config (base64) | |
| id: cfg | |
| shell: bash | |
| run: | | |
| if ! command -v yq >/dev/null 2>&1; then | |
| YQ_VERSION="v4.47.1" | |
| YQ_CHECKSUM="0fb28c6680193c41b364193d0c0fc4a03177aecde51cfc04d506b1517158c2fb" | |
| curl -sSL -o /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 | |
| echo "${YQ_CHECKSUM} /usr/local/bin/yq" | sha256sum -c - || exit 1 | |
| chmod +x /usr/local/bin/yq | |
| fi | |
| echo "components_b64=$(yq -o=json -I=0 '.components' .github/config/publish.yml | base64 -w0)" >> "$GITHUB_OUTPUT" | |
| - name: Build matrix | |
| id: mk | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const b64 = `${{ steps.cfg.outputs.components_b64 }}` || ''; | |
| if (!b64) { | |
| core.setOutput('matrix', JSON.stringify({ include: [{ component: 'noop' }] })); | |
| core.setOutput('components', JSON.stringify({ include: [{ component: 'noop' }] })); | |
| return; | |
| } | |
| const comps = JSON.parse(Buffer.from(b64, 'base64').toString('utf8')); | |
| const components = Object.entries(comps) | |
| .filter(([_, v]) => v && v.registry === 'dockerhub') | |
| .map(([k]) => k); | |
| const uniqComponents = [...new Set(components)]; | |
| // Output just the component list for manifest creation | |
| const componentMatrix = uniqComponents.length | |
| ? { include: uniqComponents.map(c => ({ component: c })) } | |
| : { include: [{ component: 'noop' }] }; | |
| core.setOutput('components', JSON.stringify(componentMatrix)); | |
| // Build cross-product matrix: components × platforms | |
| const platforms = [ | |
| { platform: 'linux/amd64', arch: 'amd64', runner: 'ubuntu-latest' }, | |
| { platform: 'linux/arm64', arch: 'arm64', runner: 'ubuntu-24.04-arm' } | |
| ]; | |
| const matrix = []; | |
| for (const comp of uniqComponents) { | |
| for (const p of platforms) { | |
| matrix.push({ component: comp, ...p }); | |
| } | |
| } | |
| core.setOutput('matrix', JSON.stringify(matrix.length ? { include: matrix } : { include: [{ component: 'noop' }] })); | |
| console.log(`Components: ${uniqComponents.join(', ')}`); | |
| console.log(`Matrix size: ${matrix.length} jobs (${uniqComponents.length} components × 2 platforms)`); | |
| docker-edge: | |
| name: ${{ matrix.component }} (${{ matrix.arch }}) | |
| needs: plan | |
| if: ${{ fromJson(needs.plan.outputs.matrix).include[0].component != 'noop' }} | |
| runs-on: ${{ matrix.runner }} | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.plan.outputs.matrix) }} | |
| env: | |
| DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Extract version | |
| id: ver | |
| run: | | |
| chmod +x scripts/extract-version.sh | |
| VERSION=$(scripts/extract-version.sh "${{ matrix.component }}") | |
| echo "version=$VERSION" >> "$GITHUB_OUTPUT" | |
| echo "✅ Resolved ${{ matrix.component }} -> version=$VERSION" | |
| - name: Determine libc for component | |
| id: libc | |
| run: | | |
| # Connectors runtime must use glibc because it dlopen()s glibc plugins | |
| if [ "${{ matrix.component }}" = "rust-connectors" ]; then | |
| echo "libc=glibc" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "libc=musl" >> "$GITHUB_OUTPUT" | |
| fi | |
| - uses: ./.github/actions/utils/docker-buildx | |
| id: docker | |
| with: | |
| task: publish | |
| libc: ${{ steps.libc.outputs.libc }} | |
| component: ${{ matrix.component }} | |
| version: ${{ steps.ver.outputs.version }} | |
| platform: ${{ matrix.platform }} | |
| dry_run: ${{ github.event.repository.fork }} | |
| gha-cache: "false" # Use registry cache only to save GHA cache space | |
| - name: Export digest | |
| if: ${{ !github.event.repository.fork }} | |
| shell: bash | |
| run: | | |
| mkdir -p ${{ runner.temp }}/digests | |
| digest="${{ steps.docker.outputs.digest }}" | |
| if [ -n "$digest" ]; then | |
| touch "${{ runner.temp }}/digests/${digest#sha256:}" | |
| echo "Exported digest: $digest" | |
| else | |
| echo "::error::No digest available" | |
| exit 1 | |
| fi | |
| - name: Upload digest | |
| if: ${{ !github.event.repository.fork }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: docker-digest-${{ matrix.component }}-${{ matrix.arch }} | |
| path: ${{ runner.temp }}/digests/* | |
| if-no-files-found: error | |
| retention-days: 1 | |
| docker-manifests: | |
| name: Create manifests | |
| needs: [plan, docker-edge, check-docker-auto-publish] | |
| if: ${{ !github.event.repository.fork && fromJson(needs.plan.outputs.components).include[0].component != 'noop' }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.plan.outputs.components) }} | |
| env: | |
| DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Resolve image from config | |
| id: config | |
| shell: bash | |
| run: | | |
| if ! command -v yq >/dev/null 2>&1; then | |
| YQ_VERSION="v4.47.1" | |
| YQ_CHECKSUM="0fb28c6680193c41b364193d0c0fc4a03177aecde51cfc04d506b1517158c2fb" | |
| curl -sSL -o /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 | |
| echo "${YQ_CHECKSUM} /usr/local/bin/yq" | sha256sum -c - || exit 1 | |
| chmod +x /usr/local/bin/yq | |
| fi | |
| image=$(yq ".components.${{ matrix.component }}.image" .github/config/publish.yml) | |
| echo "image=$image" >> "$GITHUB_OUTPUT" | |
| echo "📦 Image: $image" | |
| - name: Download amd64 digest | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: docker-digest-${{ matrix.component }}-amd64 | |
| path: ${{ runner.temp }}/digests | |
| - name: Download arm64 digest | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: docker-digest-${{ matrix.component }}-arm64 | |
| path: ${{ runner.temp }}/digests | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ env.DOCKERHUB_USER }} | |
| password: ${{ env.DOCKERHUB_TOKEN }} | |
| - name: Extract version | |
| id: ver | |
| run: | | |
| chmod +x scripts/extract-version.sh | |
| VERSION=$(scripts/extract-version.sh "${{ matrix.component }}") | |
| echo "version=$VERSION" >> "$GITHUB_OUTPUT" | |
| - name: Create and push manifest | |
| working-directory: ${{ runner.temp }}/digests | |
| env: | |
| COMPONENTS_TO_PUBLISH: ${{ needs.check-docker-auto-publish.outputs.components_to_publish }} | |
| run: | | |
| IMAGE="${{ steps.config.outputs.image }}" | |
| COMPONENT="${{ matrix.component }}" | |
| VERSION="${{ steps.ver.outputs.version }}" | |
| echo "Creating manifests for $IMAGE from digests:" | |
| ls -la | |
| # Always create the rolling 'edge' tag | |
| docker buildx imagetools create \ | |
| -t "${IMAGE}:edge" \ | |
| $(printf "${IMAGE}@sha256:%s " *) | |
| echo "✅ Pushed manifest: ${IMAGE}:edge" | |
| # Also create versioned tag if this component needs it | |
| if echo "$COMPONENTS_TO_PUBLISH" | grep -qE "(^|,)${COMPONENT}(,|$)"; then | |
| echo "Creating versioned manifest for $IMAGE:$VERSION" | |
| docker buildx imagetools create \ | |
| -t "${IMAGE}:${VERSION}" \ | |
| $(printf "${IMAGE}@sha256:%s " *) | |
| echo "✅ Pushed manifest: ${IMAGE}:${VERSION}" | |
| fi | |
| - name: Inspect manifest | |
| run: | | |
| docker buildx imagetools inspect "${{ steps.config.outputs.image }}:edge" | |
| # Create git tags for Docker images with versioned edge/rc releases | |
| create-docker-tags: | |
| name: Create Docker git tags | |
| runs-on: ubuntu-latest | |
| needs: [docker-manifests, check-docker-auto-publish] | |
| if: needs.check-docker-auto-publish.outputs.should_publish == 'true' | |
| permissions: | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Create git tags | |
| env: | |
| COMPONENTS_TO_PUBLISH: ${{ needs.check-docker-auto-publish.outputs.components_to_publish }} | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| chmod +x scripts/extract-version.sh | |
| for component in $(echo "$COMPONENTS_TO_PUBLISH" | tr ',' ' '); do | |
| TAG=$(scripts/extract-version.sh "$component" --tag) | |
| if ! git rev-parse "$TAG" >/dev/null 2>&1; then | |
| git tag -a "$TAG" -m "Release $TAG" | |
| git push origin "$TAG" | |
| echo "✅ Created tag: $TAG" | |
| else | |
| echo "⏭️ Tag $TAG already exists" | |
| fi | |
| done | |
| build-artifacts: | |
| name: Build artifacts | |
| uses: ./.github/workflows/_build_rust_artifacts.yml | |
| with: | |
| version: edge | |
| upload_artifacts: true | |
| create-prerelease: | |
| name: Create edge pre-release | |
| runs-on: ubuntu-latest | |
| needs: build-artifacts | |
| if: needs.build-artifacts.result == 'success' && !github.event.repository.fork | |
| permissions: | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Get server version | |
| id: meta | |
| run: | | |
| chmod +x scripts/extract-version.sh | |
| server_version=$(scripts/extract-version.sh rust-server) | |
| echo "server_version=${server_version}" >> "$GITHUB_OUTPUT" | |
| - name: Download all artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: rust-artifacts-all | |
| path: ./artifacts | |
| - name: Delete existing edge release and tag | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| if gh release view edge &>/dev/null; then | |
| echo "Deleting existing edge release and tag..." | |
| gh release delete edge --cleanup-tag --yes || echo "Delete failed, continuing..." | |
| fi | |
| - name: Create edge pre-release | |
| uses: softprops/action-gh-release@v2 | |
| with: | |
| tag_name: edge | |
| name: edge | |
| draft: false | |
| prerelease: true | |
| make_latest: false | |
| files: artifacts/*.tar.gz | |
| body: | | |
| Rolling edge build of Apache Iggy binaries and connector plugins. | |
| **This release is automatically updated on every push to master.** | |
| ## Binaries included | |
| - `iggy-server` - The server binary | |
| - `iggy` - The command-line interface | |
| - `iggy-bench` - The benchmarking tool | |
| - `iggy-connectors` - The connectors runtime | |
| ## Connector plugins included (.so) | |
| - `iggy_connector_elasticsearch_sink` | |
| - `iggy_connector_elasticsearch_source` | |
| - `iggy_connector_iceberg_sink` | |
| - `iggy_connector_postgres_sink` | |
| - `iggy_connector_postgres_source` | |
| - `iggy_connector_quickwit_sink` | |
| - `iggy_connector_random_source` | |
| - `iggy_connector_stdout_sink` | |
| ## Downloads | |
| ### Binaries | |
| - `iggy-x86_64-unknown-linux-gnu-edge.tar.gz` - Linux x86_64 (glibc) | |
| - `iggy-x86_64-unknown-linux-musl-edge.tar.gz` - Linux x86_64 (musl, static) | |
| - `iggy-aarch64-unknown-linux-gnu-edge.tar.gz` - Linux ARM64 (glibc) | |
| - `iggy-aarch64-unknown-linux-musl-edge.tar.gz` - Linux ARM64 (musl, static) | |
| ### Connector plugins | |
| - `iggy-connectors-x86_64-unknown-linux-gnu-edge.tar.gz` - Linux x86_64 (glibc) | |
| - `iggy-connectors-aarch64-unknown-linux-gnu-edge.tar.gz` - Linux ARM64 (glibc) | |
| ## Build info | |
| - Server version: ${{ steps.meta.outputs.server_version }} | |
| - Commit: ${{ github.sha }} | |
| **Not an official ASF release** - for development/testing only. | |
| # Check if auto-publish should run for Docker edge/rc versions | |
| check-docker-auto-publish: | |
| name: Check Docker auto-publish | |
| runs-on: ubuntu-latest | |
| if: ${{ !github.event.repository.fork }} | |
| outputs: | |
| should_publish: ${{ steps.check.outputs.should_publish }} | |
| components_to_publish: ${{ steps.check.outputs.components_to_publish }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup yq | |
| run: | | |
| if ! command -v yq >/dev/null 2>&1; then | |
| YQ_VERSION="v4.47.1" | |
| YQ_CHECKSUM="0fb28c6680193c41b364193d0c0fc4a03177aecde51cfc04d506b1517158c2fb" | |
| curl -sSL -o /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 | |
| echo "${YQ_CHECKSUM} /usr/local/bin/yq" | sha256sum -c - || exit 1 | |
| chmod +x /usr/local/bin/yq | |
| fi | |
| - name: Check versions and tags for Docker components | |
| id: check | |
| run: | | |
| chmod +x scripts/extract-version.sh | |
| # Get Docker components from config (same source as plan job) | |
| DOCKER_COMPONENTS=$(yq -r '.components | to_entries | .[] | select(.value.registry == "dockerhub") | .key' .github/config/publish.yml) | |
| COMPONENTS_TO_PUBLISH="" | |
| for component in $DOCKER_COMPONENTS; do | |
| VERSION=$(scripts/extract-version.sh "$component") | |
| TAG=$(scripts/extract-version.sh "$component" --tag) | |
| echo "Checking $component: version=$VERSION, tag=$TAG" | |
| # Skip if version doesn't contain edge or rc | |
| if [[ ! "$VERSION" =~ -(edge|rc) ]]; then | |
| echo " ⏭️ Stable version - skipping" | |
| continue | |
| fi | |
| # Skip if git tag already exists | |
| if git rev-parse "$TAG" >/dev/null 2>&1; then | |
| echo " ⏭️ Tag exists - skipping" | |
| continue | |
| fi | |
| echo " ✅ Will publish versioned tag" | |
| if [ -n "$COMPONENTS_TO_PUBLISH" ]; then | |
| COMPONENTS_TO_PUBLISH="$COMPONENTS_TO_PUBLISH,$component" | |
| else | |
| COMPONENTS_TO_PUBLISH="$component" | |
| fi | |
| done | |
| if [ -z "$COMPONENTS_TO_PUBLISH" ]; then | |
| echo "" | |
| echo "No Docker components need versioned publishing" | |
| echo "should_publish=false" >> "$GITHUB_OUTPUT" | |
| echo "components_to_publish=" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "" | |
| echo "Components to publish: $COMPONENTS_TO_PUBLISH" | |
| echo "should_publish=true" >> "$GITHUB_OUTPUT" | |
| echo "components_to_publish=$COMPONENTS_TO_PUBLISH" >> "$GITHUB_OUTPUT" | |
| fi | |
| # Check if auto-publish should run for edge/rc versions | |
| check-auto-publish: | |
| name: Check auto-publish | |
| runs-on: ubuntu-latest | |
| if: ${{ !github.event.repository.fork }} | |
| outputs: | |
| should_publish: ${{ steps.check.outputs.should_publish }} | |
| crates_to_publish: ${{ steps.check.outputs.crates_to_publish }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Check versions and tags for each crate | |
| id: check | |
| run: | | |
| chmod +x scripts/extract-version.sh | |
| CRATES_TO_PUBLISH="" | |
| # Check each crate individually | |
| for crate in rust-common rust-binary-protocol rust-sdk rust-cli; do | |
| VERSION=$(scripts/extract-version.sh "$crate") | |
| TAG=$(scripts/extract-version.sh "$crate" --tag) | |
| echo "Checking $crate: version=$VERSION, tag=$TAG" | |
| # Skip if version doesn't contain edge or rc | |
| if [[ ! "$VERSION" =~ -(edge|rc) ]]; then | |
| echo " ⏭️ Stable version - skipping" | |
| continue | |
| fi | |
| # Skip if tag already exists | |
| if git rev-parse "$TAG" >/dev/null 2>&1; then | |
| echo " ⏭️ Tag exists - skipping" | |
| continue | |
| fi | |
| echo " ✅ Will publish" | |
| if [ -n "$CRATES_TO_PUBLISH" ]; then | |
| CRATES_TO_PUBLISH="$CRATES_TO_PUBLISH,$crate" | |
| else | |
| CRATES_TO_PUBLISH="$crate" | |
| fi | |
| done | |
| if [ -z "$CRATES_TO_PUBLISH" ]; then | |
| echo "" | |
| echo "No crates need publishing" | |
| echo "should_publish=false" >> "$GITHUB_OUTPUT" | |
| echo "crates_to_publish=" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "" | |
| echo "Crates to publish: $CRATES_TO_PUBLISH" | |
| echo "should_publish=true" >> "$GITHUB_OUTPUT" | |
| echo "crates_to_publish=$CRATES_TO_PUBLISH" >> "$GITHUB_OUTPUT" | |
| fi | |
| # Auto-publish Rust crates for edge/rc versions | |
| publish-rust-crates: | |
| name: Auto-publish Rust crates | |
| needs: check-auto-publish | |
| if: needs.check-auto-publish.outputs.should_publish == 'true' | |
| permissions: | |
| contents: write # Required for git tag push | |
| uses: ./.github/workflows/_publish_rust_crates.yml | |
| with: | |
| crates: ${{ needs.check-auto-publish.outputs.crates_to_publish }} | |
| dry_run: false | |
| create_tags: true | |
| secrets: | |
| CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} |