Improve HTML test report (#2282)

Author: t3chguy

.github/workflows/build_and_test.yaml

@@ -25,6 +25,7 @@ jobs:
                 arch: [x64, ia32, arm64]
         with:
             arch: ${{ matrix.arch }}
+            blob_report: true
 
     linux:
         needs: fetch
@@ -37,14 +38,47 @@ jobs:
         with:
             sqlcipher: ${{ matrix.sqlcipher }}
             arch: ${{ matrix.arch }}
+            blob_report: true
 
     macos:
         needs: fetch
         name: macOS
         uses: ./.github/workflows/build_macos.yaml
+        with:
+            blob_report: true
 
     tests-done:
         needs: [windows, linux, macos]
         runs-on: ubuntu-24.04
+        if: always()
         steps:
-            - run: echo "Tests successful"
+            - uses: actions/checkout@v4
+
+            - uses: actions/setup-node@v4
+              with:
+                  cache: "yarn"
+                  node-version: "lts/*"
+
+            - name: Install dependencies
+              run: yarn install --frozen-lockfile
+
+            - name: Download blob reports from GitHub Actions Artifacts
+              uses: actions/download-artifact@v4
+              with:
+                  pattern: blob-report-*
+                  path: all-blob-reports
+                  merge-multiple: true
+
+            - name: Merge into HTML Report
+              run: yarn playwright merge-reports -c ./playwright.config.ts --reporter=html ./all-blob-reports
+
+            - name: Upload HTML report
+              if: always()
+              uses: actions/upload-artifact@v4
+              with:
+                  name: html-report
+                  path: playwright-report
+                  retention-days: 14
+
+            - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
+              run: exit 1

.github/workflows/build_linux.yaml

@@ -16,6 +16,10 @@ on:
                 type: string
                 required: true
                 description: "How to link sqlcipher, one of 'system' | 'static'"
+            blob_report:
+                type: boolean
+                required: false
+                description: "Whether to run the blob report"
 env:
     SQLCIPHER_BUNDLED: ${{ inputs.sqlcipher == 'static' && '1' || '' }}
     MAX_GLIBC: 2.31 # bullseye-era glibc, used by glibc-check.sh
@@ -194,3 +198,4 @@ jobs:
             prepare_cmd: |
                 sudo apt-get -qq update
                 sudo apt install ./dist/*.deb
+            blob_report: ${{ inputs.blob_report }}

.github/workflows/build_macos.yaml

@@ -27,6 +27,10 @@ on:
                 type: string
                 required: false
                 description: "The URL to which the output will be deployed."
+            blob_report:
+                type: boolean
+                required: false
+                description: "Whether to run the blob report"
 permissions: {} # No permissions required
 jobs:
     build:
@@ -159,3 +163,4 @@ jobs:
                 hdiutil attach ./dist/*.dmg -mountpoint /Volumes/Element &&
                 rsync -a /Volumes/Element/Element*.app ~/Applications/ &&
                 hdiutil detach /Volumes/Element
+            blob_report: ${{ inputs.blob_report }}

.github/workflows/build_test.yaml

@@ -18,6 +18,10 @@ on:
                 type: string
                 required: false
                 description: "Command to run to prepare the executable or environment for testing"
+            blob_report:
+                type: boolean
+                default: false
+                description: "Whether to upload a blob report instead of the HTML report"
 permissions: {}
 jobs:
     test:
@@ -65,14 +69,22 @@ jobs:
               uses: coactions/setup-xvfb@6b00cf1889f4e1d5a48635647013c0508128ee1a
               timeout-minutes: 5
               with:
-                  run: "yarn test ${{ runner.os != 'Linux' && '--ignore-snapshots' || '' }}"
+                  run: yarn test --project=${{ inputs.artifact }} ${{ runner.os != 'Linux' && '--ignore-snapshots' || '' }} ${{ inputs.blob_report == false && '--reporter=html' || '' }}
               env:
                   ELEMENT_DESKTOP_EXECUTABLE: ${{ steps.executable.outputs.path }}
 
+            - name: Upload blob report
+              if: always() && inputs.blob_report
+              uses: actions/upload-artifact@v4
+              with:
+                  name: blob-report-${{ inputs.artifact }}
+                  path: blob-report
+                  retention-days: 1
+
             - name: Upload HTML report
-              if: always()
+              if: always() && inputs.blob_report == false
               uses: actions/upload-artifact@v4
               with:
                   name: ${{ inputs.artifact }}-test
-                  path: playwright/html-report
+                  path: playwright-report
                   retention-days: 14

.github/workflows/build_windows.yaml

@@ -30,6 +30,10 @@ on:
                 type: string
                 required: false
                 description: "Whether to sign & notarise the build, requires 'packages.element.io' environment"
+            blob_report:
+                type: boolean
+                required: false
+                description: "Whether to run the blob report"
 permissions: {} # No permissions required
 jobs:
     build:
@@ -224,3 +228,4 @@ jobs:
             artifact: win-${{ inputs.arch }}
             runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-2022' }}
             executable: ./dist/win*-unpacked/Element*.exe
+            blob_report: ${{ inputs.blob_report }}

playwright.config.ts

@@ -8,7 +8,25 @@ Please see LICENSE files in the repository root for full details.
 
 import { defineConfig } from "@playwright/test";
 
+const projects = [
+    "macos",
+    "win-x64",
+    "win-ia32",
+    "win-arm64",
+    "linux-amd64-sqlcipher-system",
+    "linux-amd64-sqlcipher-static",
+    "linux-arm64-sqlcipher-system",
+    "linux-arm64-sqlcipher-static",
+];
+
 export default defineConfig({
+    // Allows the GitHub action to specify a project name (OS + arch) for the combined report to make sense
+    // workaround for https://github.com/microsoft/playwright/issues/33521
+    projects: process.env.CI
+        ? projects.map((name) => ({
+              name,
+          }))
+        : undefined,
     use: {
         viewport: { width: 1280, height: 720 },
         video: "retain-on-failure",
@@ -18,7 +36,7 @@ export default defineConfig({
     outputDir: "playwright/test-results",
     workers: 1,
     retries: process.env.CI ? 2 : 0,
-    reporter: [["html", { outputFolder: "playwright/html-report" }]],
+    reporter: process.env.CI ? [["blob"], ["github"]] : [["html", { outputFolder: "playwright/html-report" }]],
     snapshotDir: "playwright/snapshots",
     snapshotPathTemplate: "{snapshotDir}/{testFilePath}/{arg}-{platform}{ext}",
     timeout: 30 * 1000,