diff --git a/.github/workflows/precommit-crit-flows.yml b/.github/workflows/precommit-crit-flows.yml index d67d0adb608..d7a80b0a1d7 100644 --- a/.github/workflows/precommit-crit-flows.yml +++ b/.github/workflows/precommit-crit-flows.yml @@ -35,9 +35,9 @@ jobs: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v6 with: - node-version: 18.16.x + node-version-file: .nvmrc cache: yarn - run: yarn --immutable @@ -124,25 +124,75 @@ jobs: echo "❌ Deployment failed"; exit 1 fi + build_testservice: + name: Build Kalium Test Service + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + repository: wireapp/kalium + ref: main + + - name: Set up JDK + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + + - name: Gradle Cache + uses: gradle/actions/setup-gradle@4d9f0ba0025fe599b4ebab900eb7f3a1d93ef4c2 # SHA of tag v5.0.0 + + - name: Validate Gradle wrapper + uses: gradle/actions/wrapper-validation@4d9f0ba0025fe599b4ebab900eb7f3a1d93ef4c2 # SHA of tag v5.0.0 + + - name: Build the testservice + run: ./gradlew :testservice:shadowJar + + - name: Upload jar + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: kalium-testservice + path: | + testservice/build/libs/ + testservice/config.yml + retention-days: 1 + e2e_crit_flow: name: Playwright Critical Flow (precommit) - if: ${{ github.repository == 'wireapp/wire-webapp' && github.actor != 'dependabot[bot]' && github.actor != 'dependabot' }} - runs-on: [self-hosted, Linux, X64, office] - needs: [deploy_to_aws] + if: ${{ !cancelled() && github.repository == 'wireapp/wire-webapp' && github.actor != 'dependabot[bot]' && github.actor != 'dependabot' }} + runs-on: ubuntu-latest + needs: [deploy_to_aws, build_testservice] timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: ref: ${{ github.event.pull_request.head.sha }} - - uses: actions/setup-node@v4 + - name: Set up JDK + uses: actions/setup-java@v4 with: - node-version: 18.16.x + java-version: '17' + distribution: 'temurin' + + - uses: actions/setup-node@v6 + with: + node-version-file: .nvmrc cache: yarn + - name: Download Testservice Jar + uses: actions/download-artifact@v5 + with: + name: kalium-testservice + path: testservice + - run: yarn --immutable - - run: yarn playwright install --with-deps && yarn playwright install chrome + - run: yarn playwright install --with-deps chrome - uses: 1password/install-cli-action@143a85f84a90555d121cde2ff5872e393a47ab9f - name: Generate env file @@ -156,12 +206,57 @@ jobs: echo "Using precommit URL: https://wire-webapp-precommit.zinfra.io/" curl -s -o /dev/null -w "HTTP %{http_code}\n" https://wire-webapp-precommit.zinfra.io/ + - name: Start Testservice in background + id: start-testservice + run: | + java -jar testservice/build/libs/testservice-*-all.jar server testservice/config.yml & + echo TESTSERVICE_PID=$! >> "$GITHUB_OUTPUT" + - name: Run critical flow tests env: # TODO: remove hardcoded precommit env in the future when ephemeral PR envs will exist # Overrides URL from .env file WEBAPP_URL: https://wire-webapp-precommit.zinfra.io/ - run: yarn e2e-test --grep "@crit-flow-web" + TEST_SERVICE_URL: http://localhost:8080 # Run tests against locally running test service + run: yarn e2e-test --shard=${{ matrix.shard }}/${{ strategy.job-total }} --grep "@crit-flow-web" --grep "@regression" + + - name: Upload blob report + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: blob-report-${{ matrix.shard }} + path: blob-report + retention-days: 1 + + - name: Stop Testservice + run: kill -SIGKILL $TESTSERVICE_PID + env: + TESTSERVICE_PID: ${{ steps.start-testservice.outputs.TESTSERVICE_PID }} + + e2e-report: + runs-on: ubuntu-latest + if: ${{ !cancelled() }} + needs: [e2e_crit_flow] + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - uses: actions/setup-node@v6 + with: + node-version-file: .nvmrc + cache: yarn + - run: yarn --immutable + + - name: Download blob reports + uses: actions/download-artifact@v5 + with: + path: all-blob-reports + pattern: blob-report-* + merge-multiple: true + + - name: Merge playwright reports + run: yarn playwright merge-reports --config ./playwright.config.ts ./all-blob-reports - name: Upload test report if: always() diff --git a/playwright.config.ts b/playwright.config.ts index e3840efa2da..c0bc5ac57c3 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -17,7 +17,7 @@ * */ -import {defineConfig, devices} from '@playwright/test'; +import {defineConfig, devices, ReporterDescription} from '@playwright/test'; import {config} from 'dotenv'; config({path: './test/e2e_tests/.env'}); @@ -44,6 +44,8 @@ module.exports = defineConfig({ reporter: [ ['html', {outputFolder: 'playwright-report', open: 'never'}], ['json', {outputFile: 'playwright-report/report.json'}], + // Add github and blob reporters in CI otherwise html and json are enough + ...(process.env.CI ? ([['line'], ['blob']] satisfies ReporterDescription[]) : []), ], /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { diff --git a/test/e2e_tests/scripts/create-playwright-report-summary.js b/test/e2e_tests/scripts/create-playwright-report-summary.js index 2b4147532e5..b9b8f2cf0d6 100644 --- a/test/e2e_tests/scripts/create-playwright-report-summary.js +++ b/test/e2e_tests/scripts/create-playwright-report-summary.js @@ -29,58 +29,66 @@ const ansiRegex = new RegExp(`${ESC}\\[[0-?]*[ -/]*[@-~]`, 'g'); const stripAnsi = str => str.replace(ansiRegex, ''); -for (const suite of report.suites) { - for (const spec of suite.specs) { - for (const test of spec.tests) { - const title = `${spec.title} (tags: ${spec.tags.join(', ')})`; - const specLocation = `${spec.file}:${spec.line}`; - const retries = test.results.length - 1; - const hasPassed = test.results.some(r => r.status === 'passed'); - const hasRetries = retries > 0; - - // Only include in failures if no retries succeeded - if (!hasPassed) { - const lastResult = test.results[test.results.length - 1]; - - if (lastResult.status !== 'passed' && lastResult.status !== 'skipped') { - // Show only the last (final) failure - let failureInfo = `
\n ❌ ${title}
\n\n Location: **${specLocation}**\n Duration: **${lastResult.duration}ms**\n`; - - if (lastResult.errors?.length) { - failureInfo += `\n**Errors:**\n`; - lastResult.errors.forEach(e => { - failureInfo += `\n\`\`\`\n${stripAnsi(e.message)}\n\`\`\``; - }); - } - - failureInfo += `\n
`; - failures.push(failureInfo); +// Recursively get all specs no matter in how many suites they are nested +const getSpecs = suite => { + if (suite.suites?.length) { + return [...suite.suites.flatMap(suite => getSpecs(suite)), ...(suite.specs ?? [])]; + } else { + return suite.specs ?? []; + } +}; + +const specs = getSpecs(report); +for (const spec of specs) { + for (const test of spec.tests) { + const title = `${spec.title} (tags: ${spec.tags.join(', ')})`; + const specLocation = `${spec.file}:${spec.line}`; + const retries = test.results.length - 1; + const hasPassed = test.results.some(r => r.status === 'passed'); + const hasRetries = retries > 0; + + // Only include in failures if no retries succeeded + if (!hasPassed) { + const lastResult = test.results[test.results.length - 1]; + + if (lastResult.status !== 'passed' && lastResult.status !== 'skipped') { + // Show only the last (final) failure + let failureInfo = `
\n ❌ ${title}
\n\n Location: **${specLocation}**\n Duration: **${lastResult.duration}ms**\n`; + + if (lastResult.errors?.length) { + failureInfo += `\n**Errors:**\n`; + lastResult.errors.forEach(e => { + failureInfo += `\n\`\`\`\n${stripAnsi(e.message)}\n\`\`\``; + }); } - } - // Test is flaky if it passed after retries - if (hasRetries && hasPassed) { - const retryDetails = test.results - .map((result, index) => { - const errors = (result.errors || []) - .map((err, i) => { - const clean = stripAnsi(err.message || ''); - return `\n\`\`\`\n${clean}\n\`\`\``; - }) - .join('\n\n'); - - if (!errors) { - return `**Attempt ${index + 1}** \n Result: ✅ **Passed** \n Duration: **${result.duration}ms**`; - } - return `**Attempt ${index + 1}** \n Result: ❌ **Failed** \n Duration: **${result.duration}ms** \n\n **Errors:** \n ${errors}`.trim(); - }) - .join('\n\n'); - - flakyTests.push( - `
\n ⚠️ ${title}
\n\n Location: **${specLocation}**\n\n${retryDetails}\n
`, - ); + failureInfo += `\n
`; + failures.push(failureInfo); } } + + // Test is flaky if it passed after retries + if (hasRetries && hasPassed) { + const retryDetails = test.results + .map((result, index) => { + const errors = (result.errors || []) + .map((err, i) => { + const clean = stripAnsi(err.message || ''); + return `\n\`\`\`\n${clean}\n\`\`\``; + }) + .join('\n\n'); + + if (!errors) { + return `**Attempt ${index + 1}** \n Result: ✅ **Passed** \n Duration: **${result.duration}ms**`; + } + return `**Attempt ${index + 1}** \n Result: ❌ **Failed** \n Duration: **${result.duration}ms** \n\n **Errors:** \n ${errors}`.trim(); + }) + .join('\n\n'); + + flakyTests.push( + `
\n ⚠️ ${title}
\n\n Location: **${specLocation}**\n\n${retryDetails}\n
`, + ); + } } }