Added a workflow to parallelise the E2E tests #11
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Parallel Code Coverage | |
| permissions: | |
| contents: read | |
| on: [pull_request, workflow_dispatch] | |
| jobs: | |
| discover-tests: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| test-files: ${{ steps.discover.outputs.test-files }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.event.pull_request.head.ref || github.ref_name }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} | |
| - name: Discover test files | |
| id: discover | |
| run: | | |
| # Find all test files in e2e directory | |
| E2E_FILES=$(find tests/e2e -name "test_*.py" -type f | sort | jq -R -s -c 'split("\n")[:-1]') | |
| echo "e2e-test-files=$E2E_FILES" >> $GITHUB_OUTPUT | |
| echo "Discovered E2E test files: $E2E_FILES" | |
| # Find all test files in unit directory | |
| UNIT_FILES=$(find tests/unit -name "test_*.py" -type f | sort | jq -R -s -c 'split("\n")[:-1]') | |
| echo "unit-test-files=$UNIT_FILES" >> $GITHUB_OUTPUT | |
| echo "Discovered unit test files: $UNIT_FILES" | |
| # Find all test files in common directory | |
| COMMON_FILES=$(find tests/e2e/common -name "*.py" -type f | grep -v __init__ | sort | jq -R -s -c 'split("\n")[:-1]') | |
| echo "common-test-files=$COMMON_FILES" >> $GITHUB_OUTPUT | |
| echo "Discovered common test files: $COMMON_FILES" | |
| e2e-tests: | |
| runs-on: ubuntu-latest | |
| environment: azure-prod | |
| needs: discover-tests | |
| strategy: | |
| matrix: | |
| test_file: ${{ fromJson(needs.discover-tests.outputs.e2e-test-files) }} | |
| mode: ["thrift", "sea"] | |
| env: | |
| DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }} | |
| DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }} | |
| DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} | |
| DATABRICKS_CATALOG: peco | |
| DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.event.pull_request.head.ref || github.ref_name }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} | |
| - name: Set up python | |
| id: setup-python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Install Poetry | |
| uses: snok/install-poetry@v1 | |
| with: | |
| virtualenvs-create: true | |
| virtualenvs-in-project: true | |
| installer-parallel: true | |
| - name: Load cached venv | |
| id: cached-poetry-dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: .venv | |
| key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} | |
| - name: Install dependencies | |
| if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' | |
| run: poetry install --no-interaction --no-root | |
| - name: Install library | |
| run: poetry install --no-interaction --all-extras | |
| - name: Run ${{ matrix.mode }} tests for ${{ matrix.test_file }} | |
| run: | | |
| echo "Running ${{ matrix.mode }} tests for ${{ matrix.test_file }}" | |
| if [ "${{ matrix.mode }}" = "sea" ]; then | |
| TEST_FILTER="-k" | |
| TEST_EXPRESSION="extra_params1 or extra_params2" | |
| else | |
| TEST_FILTER="-k" | |
| TEST_EXPRESSION="extra_params0 or not extra_params" | |
| fi | |
| TEST_NAME=$(basename "${{ matrix.test_file }}" .py) | |
| COVERAGE_FILE="coverage-${TEST_NAME}-${{ matrix.mode }}.xml" | |
| COVERAGE_DATA="coverage-data-${TEST_NAME}-${{ matrix.mode }}" | |
| echo "TEST_NAME=$TEST_NAME" >> $GITHUB_ENV | |
| # Clean any previous coverage data to avoid conflicts | |
| rm -f .coverage* | |
| echo "Running pytest with coverage..." | |
| poetry run pytest "${{ matrix.test_file }}" "$TEST_FILTER" "$TEST_EXPRESSION" \ | |
| --cov=src --cov-report=xml:$COVERAGE_FILE --cov-report=term -v || [ $? -eq 5 ] | |
| # Save coverage data with unique name | |
| if [ -f ".coverage" ]; then | |
| mv .coverage "$COVERAGE_DATA" | |
| echo "Coverage data saved as $COVERAGE_DATA" | |
| else | |
| # Create minimal coverage file for cases where no tests run | |
| poetry run coverage erase | |
| poetry run coverage run --source=src -m pytest --version > /dev/null 2>&1 || true | |
| if [ -f ".coverage" ]; then | |
| mv .coverage "$COVERAGE_DATA" | |
| else | |
| touch "$COVERAGE_DATA" | |
| fi | |
| echo "Created minimal coverage file as $COVERAGE_DATA" | |
| fi | |
| - name: Upload coverage artifact | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: coverage-${{ env.TEST_NAME }}-${{ matrix.mode }} | |
| path: | | |
| coverage-data-* | |
| coverage-*-${{ matrix.mode }}.xml | |
| if-no-files-found: warn | |
| unit-tests: | |
| runs-on: ubuntu-latest | |
| needs: discover-tests | |
| strategy: | |
| matrix: | |
| test_file: ${{ fromJson(needs.discover-tests.outputs.unit-test-files) }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.event.pull_request.head.ref || github.ref_name }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} | |
| - name: Set up python | |
| id: setup-python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Install Poetry | |
| uses: snok/install-poetry@v1 | |
| with: | |
| virtualenvs-create: true | |
| virtualenvs-in-project: true | |
| installer-parallel: true | |
| - name: Load cached venv | |
| id: cached-poetry-dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: .venv | |
| key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} | |
| - name: Install dependencies | |
| if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' | |
| run: poetry install --no-interaction --no-root | |
| - name: Install library | |
| run: poetry install --no-interaction --all-extras | |
| - name: Run unit tests with coverage | |
| run: | | |
| TEST_NAME=$(basename "${{ matrix.test_file }}" .py) | |
| COVERAGE_FILE="coverage-${TEST_NAME}-unit.xml" | |
| COVERAGE_DATA="coverage-data-${TEST_NAME}-unit" | |
| echo "TEST_NAME=$TEST_NAME" >> $GITHUB_ENV | |
| # Clean any previous coverage data to avoid conflicts | |
| rm -f .coverage* | |
| echo "Running unit tests with coverage..." | |
| poetry run pytest "${{ matrix.test_file }}" --cov=src --cov-report=xml:$COVERAGE_FILE --cov-report=term -v | |
| # Save coverage data with unique name | |
| if [ -f ".coverage" ]; then | |
| mv .coverage "$COVERAGE_DATA" | |
| echo "Coverage data saved as $COVERAGE_DATA" | |
| else | |
| # Create minimal coverage file for cases where no tests run | |
| poetry run coverage erase | |
| poetry run coverage run --source=src -m pytest --version > /dev/null 2>&1 || true | |
| if [ -f ".coverage" ]; then | |
| mv .coverage "$COVERAGE_DATA" | |
| else | |
| touch "$COVERAGE_DATA" | |
| fi | |
| echo "Created minimal coverage file as $COVERAGE_DATA" | |
| fi | |
| - name: Upload coverage artifact | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: coverage-${{ env.TEST_NAME }}-unit | |
| path: | | |
| coverage-data-* | |
| coverage-*-unit.xml | |
| if-no-files-found: warn | |
| common-tests: | |
| runs-on: ubuntu-latest | |
| needs: discover-tests | |
| strategy: | |
| matrix: | |
| test_file: ${{ fromJson(needs.discover-tests.outputs.common-test-files) }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.event.pull_request.head.ref || github.ref_name }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} | |
| - name: Set up python | |
| id: setup-python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Install Poetry | |
| uses: snok/install-poetry@v1 | |
| with: | |
| virtualenvs-create: true | |
| virtualenvs-in-project: true | |
| installer-parallel: true | |
| - name: Load cached venv | |
| id: cached-poetry-dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: .venv | |
| key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} | |
| - name: Install dependencies | |
| if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' | |
| run: poetry install --no-interaction --no-root | |
| - name: Install library | |
| run: poetry install --no-interaction --all-extras | |
| - name: Run common tests with coverage | |
| run: | | |
| TEST_NAME=$(basename "${{ matrix.test_file }}" .py) | |
| COVERAGE_FILE="coverage-${TEST_NAME}-common.xml" | |
| COVERAGE_DATA="coverage-data-${TEST_NAME}-common" | |
| echo "TEST_NAME=$TEST_NAME" >> $GITHUB_ENV | |
| # Clean any previous coverage data to avoid conflicts | |
| rm -f .coverage* | |
| echo "Running common tests with coverage..." | |
| poetry run pytest "${{ matrix.test_file }}" --cov=src --cov-report=xml:$COVERAGE_FILE --cov-report=term -v | |
| # Save coverage data with unique name | |
| if [ -f ".coverage" ]; then | |
| mv .coverage "$COVERAGE_DATA" | |
| echo "Coverage data saved as $COVERAGE_DATA" | |
| else | |
| # Create minimal coverage file for cases where no tests run | |
| poetry run coverage erase | |
| poetry run coverage run --source=src -m pytest --version > /dev/null 2>&1 || true | |
| if [ -f ".coverage" ]; then | |
| mv .coverage "$COVERAGE_DATA" | |
| else | |
| touch "$COVERAGE_DATA" | |
| fi | |
| echo "Created minimal coverage file as $COVERAGE_DATA" | |
| fi | |
| - name: Upload coverage artifact | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: coverage-${{ env.TEST_NAME }}-common | |
| path: | | |
| coverage-data-* | |
| coverage-*-common.xml | |
| if-no-files-found: warn | |
| merge-coverage: | |
| runs-on: ubuntu-latest | |
| needs: [e2e-tests, unit-tests, common-tests] | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| - name: Set up python | |
| id: setup-python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Install Poetry | |
| uses: snok/install-poetry@v1 | |
| with: | |
| virtualenvs-create: true | |
| virtualenvs-in-project: true | |
| installer-parallel: true | |
| - name: Load cached venv | |
| id: cached-poetry-dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: .venv | |
| key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} | |
| - name: Install dependencies | |
| if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' | |
| run: poetry install --no-interaction --no-root | |
| - name: Install library | |
| run: poetry install --no-interaction --all-extras | |
| - name: Download all coverage artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: coverage_files | |
| - name: Merge coverage | |
| run: | | |
| # Install xmllint if not available | |
| if ! command -v xmllint &> /dev/null; then | |
| sudo apt-get update && sudo apt-get install -y libxml2-utils | |
| fi | |
| # Copy coverage data files from artifacts | |
| COVERAGE_FILES_FOUND=0 | |
| for artifact_dir in coverage_files/*/; do | |
| if [ -d "$artifact_dir" ]; then | |
| for cov_file in "$artifact_dir"/coverage-data-*; do | |
| if [ -f "$cov_file" ]; then | |
| cp "$cov_file" . | |
| COVERAGE_FILES_FOUND=$((COVERAGE_FILES_FOUND + 1)) | |
| fi | |
| done | |
| fi | |
| done | |
| echo "Found $COVERAGE_FILES_FOUND coverage data files" | |
| if [ $COVERAGE_FILES_FOUND -gt 0 ]; then | |
| echo "Combining coverage data..." | |
| for f in coverage-data-*; do | |
| if [ -f "$f" ]; then | |
| suffix=$(echo "$f" | sed 's/coverage-data-/.coverage./') | |
| mv "$f" "$suffix" | |
| fi | |
| done | |
| # Combine coverage files and generate reports | |
| poetry run coverage combine .coverage.* 2>/dev/null || true | |
| poetry run coverage xml 2>/dev/null || echo '<coverage lines-covered="0" lines-valid="1"></coverage>' > coverage.xml | |
| poetry run coverage report 2>/dev/null || true | |
| else | |
| echo "No coverage data files found, creating empty report" | |
| echo '<coverage lines-covered="0" lines-valid="1"></coverage>' > coverage.xml | |
| fi | |
| - name: Report coverage percentage | |
| run: | | |
| if [ ! -f "coverage.xml" ]; then | |
| echo "ERROR: Coverage file not found" | |
| exit 1 | |
| fi | |
| COVERED=$(xmllint --xpath "string(//coverage/@lines-covered)" "coverage.xml") | |
| TOTAL=$(xmllint --xpath "string(//coverage/@lines-valid)" "coverage.xml") | |
| PERCENTAGE=$(python3 -c "covered=${COVERED}; total=${TOTAL}; print(round((covered/total)*100, 2))") | |
| echo "Combined Coverage: ${PERCENTAGE}%" |