rust-tests.yml

name: Rust Tests

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  workflow_dispatch:

env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1

jobs:
  check:
    name: Cargo Check
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@stable

      - name: Install Bun
        uses: oven-sh/setup-bun@v2
        with:
          bun-version: latest

      - name: Cache dependencies
        uses: Swatinem/rust-cache@v2

      - name: Cache wasm-pack binary
        uses: actions/cache@v4
        with:
          path: ~/.cargo/bin/wasm-pack
          key: ${{ runner.os }}-wasm-pack-0.13

      - name: Install system dependencies
        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler

      - name: Install wasm-pack and Build WASM
        run: |
          if [ ! -f ~/.cargo/bin/wasm-pack ]; then
            cargo install wasm-pack
          fi
          cd crates/temps-captcha-wasm
          bun run build

      - name: Check workspace
        run: cargo check --workspace --all-targets

  fmt:
    name: Formatting
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@stable
        with:
          components: rustfmt

      - name: Check formatting
        run: cargo fmt --all -- --check

  clippy:
    name: Clippy (advisory)
    runs-on: ubuntu-latest
    # Non-blocking: clippy warnings are reported but do not fail the build
    continue-on-error: true
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@stable
        with:
          components: clippy

      - name: Install Bun
        uses: oven-sh/setup-bun@v2
        with:
          bun-version: latest

      - name: Cache dependencies
        uses: Swatinem/rust-cache@v2

      - name: Cache wasm-pack binary
        uses: actions/cache@v4
        with:
          path: ~/.cargo/bin/wasm-pack
          key: ${{ runner.os }}-wasm-pack-0.13

      - name: Install system dependencies
        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler

      - name: Install wasm-pack and Build WASM
        run: |
          if [ ! -f ~/.cargo/bin/wasm-pack ]; then
            cargo install wasm-pack
          fi
          cd crates/temps-captcha-wasm
          bun run build

      - name: Run clippy
        run: cargo clippy --workspace --all-targets --all-features -- -D warnings

  # ---------------------------------------------------------------------------
  # Build all test binaries once, then cache for parallel test jobs
  # ---------------------------------------------------------------------------
  build-tests:
    name: Build Test Binaries
    runs-on: ubuntu-latest
    steps:
      - name: Free up disk space
        run: |
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
          sudo docker image prune --all --force
          sudo apt-get autoremove -y && sudo apt-get clean

      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@stable

      - name: Install Bun
        uses: oven-sh/setup-bun@v2
        with:
          bun-version: latest

      - name: Cache dependencies
        uses: Swatinem/rust-cache@v2
        with:
          # Use a shared cache key so test jobs can restore
          shared-key: test-build
          save-if: true

      - name: Cache wasm-pack binary
        uses: actions/cache@v4
        with:
          path: ~/.cargo/bin/wasm-pack
          key: ${{ runner.os }}-wasm-pack-0.13

      - name: Install system dependencies
        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler

      - name: Install wasm-pack and Build WASM
        run: |
          if [ ! -f ~/.cargo/bin/wasm-pack ]; then
            cargo install wasm-pack
          fi
          cd crates/temps-captcha-wasm
          bun run build

      - name: Build all test binaries
        run: cargo test --no-run --workspace --all-features
        env:
          CARGO_INCREMENTAL: 0

  # ---------------------------------------------------------------------------
  # Phase 1: Unit tests (fast, no Docker containers needed)
  # ---------------------------------------------------------------------------
  # Runs all workspace tests WITHOUT the docker-tests feature.
  # This executes all pure unit tests across every crate quickly.
  # For temps-providers, Docker-heavy tests are gated behind the
  # `docker-tests` feature flag and will NOT run here.
  unit-tests:
    name: "Unit Tests (${{ matrix.group }})"
    runs-on: ubuntu-latest
    needs: build-tests
    strategy:
      fail-fast: false
      matrix:
        include:
          - group: unit-a
            crates: >-
              -p temps-providers
              -p temps-presets
              -p temps-auth
              -p temps-core
              -p temps-monitoring
              -p temps-kv
              -p temps-analytics-funnels
              -p temps-webhooks
              -p temps-infra
              -p temps-captcha-wasm
              -p temps-geo
              -p temps-environments
              -p temps-audit
              -p temps-analytics-performance
          - group: unit-b
            crates: >-
              -p temps-proxy
              -p temps-error-tracking
              -p temps-git
              -p temps-screenshots
              -p temps-routes
              -p temps-entities
              -p temps-cli
              -p temps-status-page
              -p temps-import
              -p temps-queue
              -p temps-query
              -p temps-analytics-session-replay
              -p temps-dns
              -p temps-email
              -p temps-analytics
              -p temps-analytics-events
              -p temps-otel

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@stable

      - name: Install Bun
        uses: oven-sh/setup-bun@v2
        with:
          bun-version: latest

      - name: Restore build cache
        uses: Swatinem/rust-cache@v2
        with:
          shared-key: test-build
          save-if: false

      - name: Cache wasm-pack binary
        uses: actions/cache@v4
        with:
          path: ~/.cargo/bin/wasm-pack
          key: ${{ runner.os }}-wasm-pack-0.13

      - name: Install system dependencies
        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler

      - name: Install wasm-pack and Build WASM
        run: |
          if [ ! -f ~/.cargo/bin/wasm-pack ]; then
            cargo install wasm-pack
          fi
          cd crates/temps-captcha-wasm
          bun run build

      - name: Run unit tests
        env:
          CARGO_TERM_COLOR: never
          RUST_BACKTRACE: 1
          CARGO_INCREMENTAL: 0
        run: |
          set -o pipefail

          # Run tests, capturing the names of any failures for targeted retry
          if cargo test --no-fail-fast --lib ${{ matrix.crates }} 2>&1 | tee /tmp/test-output.txt; then
            echo "All tests passed on first attempt"
            exit 0
          fi

          # Extract failed test names from output
          failed_tests=$(grep '^test .* FAILED$' /tmp/test-output.txt | sed 's/^test \(.*\) \.\.\..*FAILED$/\1/' | tr '\n' ' ' || true)
          if [ -z "$failed_tests" ]; then
            echo "Could not parse failed test names, re-running all tests"
            cargo test --no-fail-fast --lib ${{ matrix.crates }}
            exit $?
          fi

          echo ""
          echo "=========================================="
          echo "Retrying only failed tests: $failed_tests"
          echo "=========================================="
          echo ""
          sleep 5

          # Retry only the failed tests (up to 2 more attempts)
          attempt=2
          max_attempts=3
          for test_name in $failed_tests; do
            echo "--- Retrying: $test_name (attempt $attempt/$max_attempts) ---"
            if ! cargo test --no-fail-fast --lib ${{ matrix.crates }} -- "$test_name" --exact; then
              if [ $attempt -lt $max_attempts ]; then
                attempt=$((attempt + 1))
                echo "--- Retrying: $test_name (attempt $attempt/$max_attempts) ---"
                sleep 5
                cargo test --no-fail-fast --lib ${{ matrix.crates }} -- "$test_name" --exact || exit 1
              else
                exit 1
              fi
            fi
          done

  # ---------------------------------------------------------------------------
  # Phase 2: Integration tests (Docker + Database required)
  # ---------------------------------------------------------------------------
  # Runs tests that need Docker daemon and/or a real database.
  # - temps-providers runs with --features docker-tests to include
  #   Docker-heavy tests (backup/restore, container lifecycle, etc.)
  # - Other crates that use TestDatabase (testcontainers) run here too.
  integration-tests:
    name: "Integration Tests (${{ matrix.group }})"
    runs-on: ubuntu-latest
    needs: [build-tests, unit-tests]
    # 90 min headroom for the postgres-upgrades group, which can spike past
    # 45 min when pg17/pg18 image pulls land on a cold runner and several
    # orchestrator tests recover serially. Other groups still finish well
    # under that — the timeout only kicks in for true outliers.
    timeout-minutes: 90
    strategy:
      fail-fast: false
      matrix:
        # Each entry splits the test command into three parts so the retry
        # logic can append `<test_name> --exact` *after* the libtest `--`
        # separator without producing a second stray `--` (which silently
        # disables the filter and reruns the whole binary). See the
        # retry block at the bottom of this job — it assembles:
        #
        #   <cargo_args> -- <test_args> <test_name> --exact
        #
        # so all post-`--` flags live in `test_args`, never in `cargo_args`.
        include:
          - group: docker-providers
            # temps-providers with Docker tests enabled.
            # NOTE: orchestrator_* tests run in a dedicated `postgres-upgrades`
            # group, and *_backup_and_restore_to_s3 tests in a dedicated
            # `docker-backups` group (both serial, long-running, container-heavy).
            # Skip them here so this group stays fast and parallel-safe.
            cargo_args: >-
              test --no-fail-fast --lib
              -p temps-providers --features docker-tests
            test_args: >-
              --skip orchestrator_
              --skip backup_and_restore_to_s3
          - group: docker-backups
            # Heavyweight S3 backup/restore integration tests
            # (test_{mongodb,postgres,redis,s3}_backup_and_restore_to_s3). Each
            # spins up a MinIO container plus a service container and streams a
            # real wal-g/mongodump backup, so running them in parallel saturates
            # Docker and races for host ports (`port is already allocated`),
            # causing 300s hangs. --test-threads=1 guarantees serial execution.
            cargo_args: >-
              test --no-fail-fast --lib
              -p temps-providers --features docker-tests
              backup_and_restore_to_s3
            test_args: >-
              --nocapture
              --test-threads=1
          - group: postgres-upgrades
            # PostgreSQL major-version upgrade orchestrator integration tests.
            # Each test pulls pg17 + pg18 images and runs a real dump/restore,
            # so they're 2–6 min apiece and must not contend for Docker.
            # --test-threads=1 guarantees serial execution.
            cargo_args: >-
              test --no-fail-fast --lib
              -p temps-providers --features docker-tests
              orchestrator_
            test_args: >-
              --nocapture
              --test-threads=1
          - group: docker-deployments
            # Crates that use TestDatabase and/or Docker
            cargo_args: >-
              test --no-fail-fast
              -p temps-deployments
              -p temps-backup
              -p temps-vulnerability-scanner
              -p temps-import-docker
              -p temps-migrations
              -p temps-database
              -p temps-blob
              -p temps-query-s3
              -p temps-query-redis
              -p temps-query-mongodb
              -p temps-query-postgres
              -p temps-embeddings
              -p temps-config
              -p temps-notifications
              -p temps-deployer
              -p temps-logs
              -p temps-domains
              -p temps-projects
            test_args: ""

    services:
      timescaledb:
        image: timescale/timescaledb-ha:pg18
        env:
          POSTGRES_DB: test_db
          POSTGRES_USER: test_user
          POSTGRES_PASSWORD: test_password
          POSTGRES_HOST_AUTH_METHOD: trust
        ports:
          - 5432:5432
        options: >-
          --health-cmd "pg_isready -U test_user -d test_db"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

    steps:
      - name: Free up disk space
        run: |
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
          sudo docker image prune --all --force
          sudo apt-get autoremove -y && sudo apt-get clean

      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@stable

      - name: Install Bun
        uses: oven-sh/setup-bun@v2
        with:
          bun-version: latest

      - name: Restore build cache
        uses: Swatinem/rust-cache@v2
        with:
          shared-key: test-build
          save-if: false

      - name: Cache wasm-pack binary
        uses: actions/cache@v4
        with:
          path: ~/.cargo/bin/wasm-pack
          key: ${{ runner.os }}-wasm-pack-0.13

      - name: Install system dependencies
        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler

      - name: Install wasm-pack and Build WASM
        run: |
          if [ ! -f ~/.cargo/bin/wasm-pack ]; then
            cargo install wasm-pack
          fi
          cd crates/temps-captcha-wasm
          bun run build

      - name: Verify TimescaleDB is ready
        run: |
          timeout 60 bash -c 'until nc -z localhost 5432; do sleep 1; done'
          echo "TimescaleDB is ready"

      - name: Run integration tests
        env:
          CARGO_TERM_COLOR: never
          RUST_BACKTRACE: 1
          CARGO_INCREMENTAL: 0
          TEMPS_TEST_DATABASE_URL: postgresql://test_user:test_password@localhost:5432/test_db
        run: |
          set -o pipefail

          # Run tests, capturing the names of any failures for targeted retry.
          # `cargo_args` covers everything cargo cares about (crate, features,
          # filters); `test_args` is everything after `--` that libtest cares
          # about (--skip, --nocapture, --test-threads). The retry block
          # below assembles `<cargo_args> -- <test_args> <test_name> --exact`
          # so the post-`--` flags don't collide with the per-test filter.
          if cargo ${{ matrix.cargo_args }} -- ${{ matrix.test_args }} 2>&1 | tee /tmp/test-output.txt; then
            echo "All tests passed on first attempt"
            exit 0
          fi

          # Extract failed test names from output
          failed_tests=$(grep '^test .* FAILED$' /tmp/test-output.txt | sed 's/^test \(.*\) \.\.\..*FAILED$/\1/' | tr '\n' ' ' || true)
          if [ -z "$failed_tests" ]; then
            echo "Could not parse failed test names, re-running all tests"
            cargo ${{ matrix.cargo_args }} -- ${{ matrix.test_args }}
            exit $?
          fi

          echo ""
          echo "=========================================="
          echo "Retrying only failed tests: $failed_tests"
          echo "=========================================="
          echo ""
          sleep 10

          # Retry only the failed tests (up to 2 more attempts each).
          # Each retry passes a single libtest separator: `--` then test_args
          # then the test filter and --exact. Crucially, cargo_args never
          # contains its own `--`, so there's never a second separator.
          max_attempts=3
          for test_name in $failed_tests; do
            attempt=2
            while [ $attempt -le $max_attempts ]; do
              echo "--- Retrying: $test_name (attempt $attempt/$max_attempts) ---"
              if cargo ${{ matrix.cargo_args }} -- ${{ matrix.test_args }} "$test_name" --exact; then
                break
              fi
              if [ $attempt -eq $max_attempts ]; then
                echo "❌ $test_name failed after $max_attempts attempts"
                exit 1
              fi
              attempt=$((attempt + 1))
              sleep 10
            done
          done

  # ---------------------------------------------------------------------------
  # Phase 2b: MariaDB PITR full-chain E2E
  # ---------------------------------------------------------------------------
  # Keep this heavyweight single test out of the generic Docker matrix so it can
  # start immediately, and so Actions shows each expensive phase (image pull,
  # test build, test run, diagnostics) separately.
  mariadb-pitr-e2e:
    name: "MariaDB PITR E2E"
    runs-on: ubuntu-latest
    timeout-minutes: 35

    services:
      timescaledb:
        image: timescale/timescaledb-ha:pg18
        env:
          POSTGRES_DB: test_db
          POSTGRES_USER: test_user
          POSTGRES_PASSWORD: test_password
          POSTGRES_HOST_AUTH_METHOD: trust
        ports:
          - 5432:5432
        options: >-
          --health-cmd "pg_isready -U test_user -d test_db"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

    steps:
      - name: Free up disk space
        run: |
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
          sudo docker image prune --all --force
          sudo apt-get autoremove -y && sudo apt-get clean
          df -h

      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@stable

      - name: Restore build cache
        uses: Swatinem/rust-cache@v2
        with:
          shared-key: test-build
          save-if: false

      - name: Install system dependencies
        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler

      - name: Verify TimescaleDB is ready
        run: |
          timeout 60 bash -c 'until nc -z localhost 5432; do sleep 1; done'
          echo "TimescaleDB is ready"

      - name: Show Docker environment
        run: |
          docker version
          docker info
          docker system df

      - name: Pull E2E images
        run: |
          timeout 10m docker pull minio/minio:latest
          timeout 10m docker pull mariadb:lts
          docker image ls | grep -E '^(minio/minio|mariadb)\s' || true

      - name: Build MariaDB PITR E2E binary
        env:
          CARGO_TERM_COLOR: never
          CARGO_INCREMENTAL: 0
        run: |
          cargo test -p temps-backup --features docker-tests mariadb_pitr_full_chain_e2e --no-run

      - name: Run MariaDB PITR E2E
        env:
          CARGO_TERM_COLOR: never
          RUST_BACKTRACE: 1
          CARGO_INCREMENTAL: 0
          TEMPS_TEST_DATABASE_URL: postgresql://test_user:test_password@localhost:5432/test_db
        run: |
          timeout --foreground 20m bash -lc 'set -o pipefail; cargo test -p temps-backup --features docker-tests mariadb_pitr_full_chain_e2e -- --nocapture --test-threads=1 2>&1 | tee /tmp/mariadb-pitr-e2e.log'

      - name: Summarize MariaDB PITR E2E diagnostics
        if: always()
        run: |
          if [ -f /tmp/mariadb-pitr-e2e.log ]; then
            grep -E 'Restored row counts|test result|FAILED|Timed out|timed out|Fetched MariaDB PITR|Uploading MariaDB PITR|Uploaded MariaDB PITR|Replaying MariaDB|temps-mariadb-pitr-replay|docker exec timed out|DIAG|Booted MariaDB|Base backup|archive_binlogs|Restore produced' /tmp/mariadb-pitr-e2e.log | tail -n 240 || true
          else
            echo "No MariaDB PITR E2E log was written"
          fi

          echo ""
          echo "Docker containers:"
          docker ps -a || true
          echo ""
          echo "Docker images:"
          docker image ls || true
          echo ""
          echo "Docker disk usage:"
          docker system df || true

      - name: Upload MariaDB PITR E2E log
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: mariadb-pitr-e2e-log
          path: /tmp/mariadb-pitr-e2e.log
          if-no-files-found: ignore