Added a workflow to parallelise the E2E tests. Updated E2E tests to create new table names for each run to avoid issue in parallelisation

msrathore-db · msrathore-db · commit 61af7729feb9 · 2025-10-03T13:22:02.000+05:30
diff --git a/.github/workflows/coverage-check-dynamic-e2e-parallel.yml b/.github/workflows/coverage-check-dynamic-e2e-parallel.yml
@@ -0,0 +1,178 @@
+name: Code Coverage (Dynamic E2E Tests - SEA & Thrift)
+
+permissions:
+  contents: read
+
+on: [pull_request, workflow_dispatch]
+
+jobs:
+  discover-tests:
+    runs-on: ubuntu-latest
+    outputs:
+      test-files: ${{ steps.discover.outputs.test-files }}
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
+          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
+
+      - name: Discover test files
+        id: discover
+        run: |
+          # Find all test files in e2e directory and create JSON array
+          TEST_FILES=$(find tests/e2e -name "test_*.py" -type f | sort | jq -R -s -c 'split("\n")[:-1]')
+          echo "test-files=$TEST_FILES" >> $GITHUB_OUTPUT
+          echo "Discovered test files: $TEST_FILES"
+
+  e2e-tests:
+    runs-on: ubuntu-latest
+    environment: azure-prod
+    needs: discover-tests
+    strategy:
+      matrix:
+        test_file: ${{ fromJson(needs.discover-tests.outputs.test-files) }}
+        mode: ["thrift", "sea"]
+    env:
+      DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }}
+      DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }}
+      DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
+      DATABRICKS_CATALOG: peco
+      DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }}
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
+          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
+
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root
+
+      - name: Install library
+        run: poetry install --no-interaction --all-extras
+
+      - name: Run ${{ matrix.mode }} tests for ${{ matrix.test_file }}
+        run: |
+          echo "Running ${{ matrix.mode }} tests for ${{ matrix.test_file }}"
+          
+          # Set test filter based on mode
+          if [ "${{ matrix.mode }}" = "sea" ]; then
+            TEST_FILTER="-k 'extra_params1 or extra_params2'"
+          else
+            TEST_FILTER="-k 'extra_params0 or not extra_params'"
+          fi
+          
+          TEST_NAME=$(basename "${{ matrix.test_file }}" .py)
+          COVERAGE_FILE="coverage-${TEST_NAME}-${{ matrix.mode }}.xml"
+          
+          poetry run pytest "${{ matrix.test_file }}" $TEST_FILTER \
+            --cov=src --cov-report=xml:$COVERAGE_FILE --cov-report=term \
+            -v
+        continue-on-error: true
+
+      - name: Upload coverage artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-$(basename "${{ matrix.test_file }}" .py)-${{ matrix.mode }}
+          path: |
+            .coverage
+            coverage-*-${{ matrix.mode }}.xml
+
+  merge-coverage:
+    runs-on: ubuntu-latest
+    needs: [e2e-tests]
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root
+
+      - name: Install library
+        run: poetry install --no-interaction --all-extras
+
+      - name: Download all coverage artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: coverage_files
+
+      - name: Merge coverage
+        run: |
+          # Install xmllint if not available
+          if ! command -v xmllint &> /dev/null; then
+            sudo apt-get update && sudo apt-get install -y libxml2-utils
+          fi
+          
+          # Copy all coverage files with unique names
+          for artifact_dir in coverage_files/*/; do
+            if [ -f "$artifact_dir/.coverage" ]; then
+              artifact_name=$(basename "$artifact_dir")
+              cp "$artifact_dir/.coverage" ".coverage.$artifact_name"
+            fi
+          done
+          
+          # Combine all coverage data
+          poetry run coverage combine .coverage.*
+          poetry run coverage xml
+          poetry run coverage report
+
+      - name: Report coverage percentage
+        run: |
+          COVERAGE_FILE="coverage.xml"
+          if [ ! -f "$COVERAGE_FILE" ]; then
+            echo "ERROR: Coverage file not found at $COVERAGE_FILE"
+            exit 1
+          fi
+
+          COVERED=$(xmllint --xpath "string(//coverage/@lines-covered)" "$COVERAGE_FILE")
+          TOTAL=$(xmllint --xpath "string(//coverage/@lines-valid)" "$COVERAGE_FILE")
+
+          # Calculate percentage using Python for precision
+          PERCENTAGE=$(python3 -c "covered=${COVERED}; total=${TOTAL}; print(round((covered/total)*100, 2))")
+
+          echo "📊 Combined Coverage: ${PERCENTAGE}%"
diff --git a/tests/e2e/test_complex_types.py b/tests/e2e/test_complex_types.py
@@ -1,6 +1,7 @@
 import pytest
 from numpy import ndarray
 from typing import Sequence
+from uuid import uuid4
 
 from tests.e2e.test_driver import PySQLPytestTestCase
 
@@ -10,12 +11,15 @@ class TestComplexTypes(PySQLPytestTestCase):
     def table_fixture(self, connection_details):
         self.arguments = connection_details.copy()
         """A pytest fixture that creates a table with a complex type, inserts a record, yields, and then drops the table"""
+         
+        table_name = f"pysql_test_complex_types_table_{str(uuid4()).replace('-', '_')}"
+        self.table_name = table_name
 
         with self.cursor() as cursor:
             # Create the table
             cursor.execute(
-                """
-                CREATE TABLE IF NOT EXISTS pysql_test_complex_types_table (
+                f"""
+                CREATE TABLE IF NOT EXISTS {table_name} (
                     array_col ARRAY<STRING>,
                     map_col MAP<STRING, INTEGER>,
                     struct_col STRUCT<field1: STRING, field2: INTEGER>,
@@ -27,8 +31,8 @@ def table_fixture(self, connection_details):
             )
             # Insert a record
             cursor.execute(
-                """
-                INSERT INTO pysql_test_complex_types_table
+                f"""
+                INSERT INTO {table_name}
                 VALUES (
                     ARRAY('a', 'b', 'c'),
                     MAP('a', 1, 'b', 2, 'c', 3),
@@ -40,10 +44,10 @@ def table_fixture(self, connection_details):
                 """
             )
             try:
-                yield
+                yield table_name
             finally:
                 # Clean up the table after the test
-                cursor.execute("DELETE FROM pysql_test_complex_types_table")
+                cursor.execute(f"DROP TABLE IF EXISTS {table_name}")
 
     @pytest.mark.parametrize(
         "field,expected_type",
@@ -61,7 +65,7 @@ def test_read_complex_types_as_arrow(self, field, expected_type, table_fixture):
 
         with self.cursor() as cursor:
             result = cursor.execute(
-                "SELECT * FROM pysql_test_complex_types_table LIMIT 1"
+                f"SELECT * FROM {table_fixture} LIMIT 1"
             ).fetchone()
 
         assert isinstance(result[field], expected_type)
@@ -83,7 +87,7 @@ def test_read_complex_types_as_string(self, field, table_fixture):
             extra_params={"_use_arrow_native_complex_types": False}
         ) as cursor:
             result = cursor.execute(
-                "SELECT * FROM pysql_test_complex_types_table LIMIT 1"
+                f"SELECT * FROM {table_fixture} LIMIT 1"
             ).fetchone()
 
         assert isinstance(result[field], str)
diff --git a/tests/e2e/test_parameterized_queries.py b/tests/e2e/test_parameterized_queries.py
@@ -4,6 +4,7 @@
 from enum import Enum
 from typing import Dict, List, Type, Union
 from unittest.mock import patch
+from uuid import uuid4
 
 import time
 import numpy as np
@@ -130,9 +131,13 @@ def inline_table(self, connection_details):
         Note that this fixture doesn't clean itself up. So the table will remain
         in the schema for use by subsequent test runs.
         """
+        
+        # Generate unique table name to avoid conflicts in parallel execution
+        table_name = f"pysql_e2e_inline_param_test_table_{str(uuid4()).replace('-', '_')}"
+        self.inline_table_name = table_name
 
-        query = """
-            CREATE TABLE IF NOT EXISTS pysql_e2e_inline_param_test_table (
+        query = f"""
+            CREATE TABLE IF NOT EXISTS {table_name} (
             null_col INT,
             int_col INT,
             bigint_col BIGINT,
@@ -179,9 +184,10 @@ def _inline_roundtrip(self, params: dict, paramstyle: ParamStyle, target_column)
         :paramstyle:
             This is a no-op but is included to make the test-code easier to read.
         """
-        INSERT_QUERY = f"INSERT INTO pysql_e2e_inline_param_test_table (`{target_column}`) VALUES (%(p)s)"
-        SELECT_QUERY = f"SELECT {target_column} `col` FROM pysql_e2e_inline_param_test_table LIMIT 1"
-        DELETE_QUERY = "DELETE FROM pysql_e2e_inline_param_test_table"
+        table_name = self.inline_table_name
+        INSERT_QUERY = f"INSERT INTO {table_name} (`{target_column}`) VALUES (%(p)s)"
+        SELECT_QUERY = f"SELECT {target_column} `col` FROM {table_name} LIMIT 1"
+        DELETE_QUERY = f"DELETE FROM {table_name}"
 
         with self.connection(extra_params={"use_inline_params": True}) as conn:
             with conn.cursor() as cursor:
diff --git a/tests/e2e/test_variant_types.py b/tests/e2e/test_variant_types.py
@@ -1,6 +1,7 @@
 import pytest
 from datetime import datetime
 import json
+from uuid import uuid4
 
 try:
     import pyarrow
@@ -19,14 +20,14 @@ class TestVariantTypes(PySQLPytestTestCase):
     def variant_table(self, connection_details):
         """A pytest fixture that creates a test table and cleans up after tests"""
         self.arguments = connection_details.copy()
-        table_name = "pysql_test_variant_types_table"
+        table_name = f"pysql_test_variant_types_table_{str(uuid4()).replace('-', '_')}"
 
         with self.cursor() as cursor:
             try:
                 # Create the table with variant columns
                 cursor.execute(
-                    """
-                    CREATE TABLE IF NOT EXISTS pysql_test_variant_types_table (
+                    f"""
+                    CREATE TABLE IF NOT EXISTS {table_name} (
                         id INTEGER,
                         variant_col VARIANT,
                         regular_string_col STRING
@@ -36,8 +37,8 @@ def variant_table(self, connection_details):
 
                 # Insert test records with different variant values
                 cursor.execute(
-                    """
-                    INSERT INTO pysql_test_variant_types_table
+                    f"""
+                    INSERT INTO {table_name}
                     VALUES 
                     (1, PARSE_JSON('{"name": "John", "age": 30}'), 'regular string'),
                     (2, PARSE_JSON('[1, 2, 3, 4]'), 'another string')
