ci(WPB-21987): shard playwright tests across multiple runners

.github/workflows/precommit-crit-flows.yml

@@ -35,9 +35,9 @@ jobs:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - uses: actions/setup-node@v4
+      - uses: actions/setup-node@v6
         with:
-          node-version: 18.16.x
+          node-version-file: .nvmrc
           cache: yarn
 
       - run: yarn --immutable
@@ -124,25 +124,75 @@ jobs:
             echo "❌ Deployment failed"; exit 1
           fi
 
+  build_testservice:
+    name: Build Kalium Test Service
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          repository: wireapp/kalium
+          ref: main
+
+      - name: Set up JDK
+        uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+
+      - name: Gradle Cache
+        uses: gradle/actions/setup-gradle@4d9f0ba0025fe599b4ebab900eb7f3a1d93ef4c2 # SHA of tag v5.0.0
+
+      - name: Validate Gradle wrapper
+        uses: gradle/actions/wrapper-validation@4d9f0ba0025fe599b4ebab900eb7f3a1d93ef4c2 # SHA of tag v5.0.0
+
+      - name: Build the testservice
+        run: ./gradlew :testservice:shadowJar
+
+      - name: Upload jar
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kalium-testservice
+          path: |
+            testservice/build/libs/
+            testservice/config.yml
+          retention-days: 1
+
   e2e_crit_flow:
     name: Playwright Critical Flow (precommit)
-    if: ${{ github.repository == 'wireapp/wire-webapp' && github.actor != 'dependabot[bot]' && github.actor != 'dependabot' }}
-    runs-on: [self-hosted, Linux, X64, office]
-    needs: [deploy_to_aws]
+    if: ${{ !cancelled() && github.repository == 'wireapp/wire-webapp' && github.actor != 'dependabot[bot]' && github.actor != 'dependabot' }}
+    runs-on: ubuntu-latest
+    needs: [deploy_to_aws, build_testservice]
     timeout-minutes: 35
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - uses: actions/setup-node@v4
+      - name: Set up JDK
+        uses: actions/setup-java@v4
         with:
-          node-version: 18.16.x
+          java-version: '17'
+          distribution: 'temurin'
+
+      - uses: actions/setup-node@v6
+        with:
+          node-version-file: .nvmrc
           cache: yarn
 
+      - name: Download Testservice Jar
+        uses: actions/download-artifact@v5
+        with:
+          name: kalium-testservice
+          path: testservice
+
       - run: yarn --immutable
-      - run: yarn playwright install --with-deps && yarn playwright install chrome
+      - run: yarn playwright install --with-deps chrome
       - uses: 1password/install-cli-action@143a85f84a90555d121cde2ff5872e393a47ab9f
 
       - name: Generate env file
@@ -156,12 +206,57 @@ jobs:
           echo "Using precommit URL: https://wire-webapp-precommit.zinfra.io/"
           curl -s -o /dev/null -w "HTTP %{http_code}\n" https://wire-webapp-precommit.zinfra.io/
 
+      - name: Start Testservice in background
+        id: start-testservice
+        run: |
+          java -jar testservice/build/libs/testservice-*-all.jar server testservice/config.yml &
+          echo TESTSERVICE_PID=$! >> "$GITHUB_OUTPUT"
+
       - name: Run critical flow tests
         env:
           # TODO: remove hardcoded precommit env in the future when ephemeral PR envs will exist
           # Overrides URL from .env file
           WEBAPP_URL: https://wire-webapp-precommit.zinfra.io/
-        run: yarn e2e-test --grep "@crit-flow-web"
+          TEST_SERVICE_URL: http://localhost:8080 # Run tests against locally running test service
+        run: yarn e2e-test --shard=${{ matrix.shard }}/${{ strategy.job-total }} --grep "@crit-flow-web" --grep "@regression"
+
+      - name: Upload blob report
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: blob-report-${{ matrix.shard }}
+          path: blob-report
+          retention-days: 1
+
+      - name: Stop Testservice
+        run: kill -SIGKILL $TESTSERVICE_PID
+        env:
+          TESTSERVICE_PID: ${{ steps.start-testservice.outputs.TESTSERVICE_PID }}
+
+  e2e-report:
+    runs-on: ubuntu-latest
+    if: ${{ !cancelled() }}
+    needs: [e2e_crit_flow]
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - uses: actions/setup-node@v6
+        with:
+          node-version-file: .nvmrc
+          cache: yarn
+      - run: yarn --immutable
+
+      - name: Download blob reports
+        uses: actions/download-artifact@v5
+        with:
+          path: all-blob-reports
+          pattern: blob-report-*
+          merge-multiple: true
+
+      - name: Merge playwright reports
+        run: yarn playwright merge-reports --config ./playwright.config.ts ./all-blob-reports
 
       - name: Upload test report
         if: always()

playwright.config.ts

@@ -17,7 +17,7 @@
  *
  */
 
-import {defineConfig, devices} from '@playwright/test';
+import {defineConfig, devices, ReporterDescription} from '@playwright/test';
 import {config} from 'dotenv';
 
 config({path: './test/e2e_tests/.env'});
@@ -44,6 +44,8 @@ module.exports = defineConfig({
   reporter: [
     ['html', {outputFolder: 'playwright-report', open: 'never'}],
     ['json', {outputFile: 'playwright-report/report.json'}],
+    // Add github and blob reporters in CI otherwise html and json are enough
+    ...(process.env.CI ? ([['line'], ['blob']] satisfies ReporterDescription[]) : []),
   ],
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {

test/e2e_tests/scripts/create-playwright-report-summary.js

@@ -29,58 +29,66 @@ const ansiRegex = new RegExp(`${ESC}\\[[0-?]*[ -/]*[@-~]`, 'g');
 
 const stripAnsi = str => str.replace(ansiRegex, '');
 
-for (const suite of report.suites) {
-  for (const spec of suite.specs) {
-    for (const test of spec.tests) {
-      const title = `${spec.title} (tags: ${spec.tags.join(', ')})`;
-      const specLocation = `${spec.file}:${spec.line}`;
-      const retries = test.results.length - 1;
-      const hasPassed = test.results.some(r => r.status === 'passed');
-      const hasRetries = retries > 0;
-
-      // Only include in failures if no retries succeeded
-      if (!hasPassed) {
-        const lastResult = test.results[test.results.length - 1];
-
-        if (lastResult.status !== 'passed' && lastResult.status !== 'skipped') {
-          // Show only the last (final) failure
-          let failureInfo = `<details> \n <summary> ❌ ${title} </summary><br> \n\n  Location: **${specLocation}**\n  Duration: **${lastResult.duration}ms**\n`;
-
-          if (lastResult.errors?.length) {
-            failureInfo += `\n**Errors:**\n`;
-            lastResult.errors.forEach(e => {
-              failureInfo += `\n\`\`\`\n${stripAnsi(e.message)}\n\`\`\``;
-            });
-          }
-
-          failureInfo += `\n</details>`;
-          failures.push(failureInfo);
+// Recursively get all specs no matter in how many suites they are nested
+const getSpecs = suite => {
+  if (suite.suites?.length) {
+    return [...suite.suites.flatMap(suite => getSpecs(suite)), ...(suite.specs ?? [])];
+  } else {
+    return suite.specs ?? [];
+  }
+};
+
+const specs = getSpecs(report);
+for (const spec of specs) {
+  for (const test of spec.tests) {
+    const title = `${spec.title} (tags: ${spec.tags.join(', ')})`;
+    const specLocation = `${spec.file}:${spec.line}`;
+    const retries = test.results.length - 1;
+    const hasPassed = test.results.some(r => r.status === 'passed');
+    const hasRetries = retries > 0;
+
+    // Only include in failures if no retries succeeded
+    if (!hasPassed) {
+      const lastResult = test.results[test.results.length - 1];
+
+      if (lastResult.status !== 'passed' && lastResult.status !== 'skipped') {
+        // Show only the last (final) failure
+        let failureInfo = `<details> \n <summary> ❌ ${title} </summary><br> \n\n  Location: **${specLocation}**\n  Duration: **${lastResult.duration}ms**\n`;
+
+        if (lastResult.errors?.length) {
+          failureInfo += `\n**Errors:**\n`;
+          lastResult.errors.forEach(e => {
+            failureInfo += `\n\`\`\`\n${stripAnsi(e.message)}\n\`\`\``;
+          });
         }
-      }
 
-      // Test is flaky if it passed after retries
-      if (hasRetries && hasPassed) {
-        const retryDetails = test.results
-          .map((result, index) => {
-            const errors = (result.errors || [])
-              .map((err, i) => {
-                const clean = stripAnsi(err.message || '');
-                return `\n\`\`\`\n${clean}\n\`\`\``;
-              })
-              .join('\n\n');
-
-            if (!errors) {
-              return `**Attempt ${index + 1}** \n Result: ✅ **Passed** \n Duration: **${result.duration}ms**`;
-            }
-            return `**Attempt ${index + 1}** \n Result: ❌ **Failed** \n Duration: **${result.duration}ms** \n\n **Errors:** \n ${errors}`.trim();
-          })
-          .join('\n\n');
-
-        flakyTests.push(
-          `<details> \n <summary> ⚠️ ${title} </summary><br> \n\n  Location: **${specLocation}**\n\n${retryDetails}\n</details>`,
-        );
+        failureInfo += `\n</details>`;
+        failures.push(failureInfo);
       }
     }
+
+    // Test is flaky if it passed after retries
+    if (hasRetries && hasPassed) {
+      const retryDetails = test.results
+        .map((result, index) => {
+          const errors = (result.errors || [])
+            .map((err, i) => {
+              const clean = stripAnsi(err.message || '');
+              return `\n\`\`\`\n${clean}\n\`\`\``;
+            })
+            .join('\n\n');
+
+          if (!errors) {
+            return `**Attempt ${index + 1}** \n Result: ✅ **Passed** \n Duration: **${result.duration}ms**`;
+          }
+          return `**Attempt ${index + 1}** \n Result: ❌ **Failed** \n Duration: **${result.duration}ms** \n\n **Errors:** \n ${errors}`.trim();
+        })
+        .join('\n\n');
+
+      flakyTests.push(
+        `<details> \n <summary> ⚠️ ${title} </summary><br> \n\n  Location: **${specLocation}**\n\n${retryDetails}\n</details>`,
+      );
+    }
   }
 }