Skip to content

E2E Pipeline

E2E Pipeline #702

Workflow file for this run

name: E2E Pipeline
on:
workflow_dispatch:
inputs:
deployment_id:
description: 'Deployment ID'
required: false
default: 'manual'
pull_request:
permissions:
contents: read
env:
APPLITOOLS_API_KEY: ${{ secrets.APPLITOOLS_API_KEY }}
APPLITOOLS_SERVER_URL: https://eyesapi.applitools.com
S3_TRACE_BUCKET: ${{ secrets.S3_TRACE_BUCKET }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
jobs:
e2e-test:
name: "E2E Tests (Deployment: ${{ inputs.deployment_id }})"
runs-on: self-hosted
env:
APPLITOOLS_BATCH_ID: ${{ github.event.pull_request.head.sha || github.sha }}
RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
SLACK_CHANNEL: "${{ inputs.deployment_id == 'manual' && 'C08GBVA4JKT' || 'C07VDF07WLU' }}"
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
NORDLAYER_EMAIL: ${{ secrets.NORDLAYER_EMAIL }}
NORDLAYER_PASSWORD: ${{ secrets.NORDLAYER_PASSWORD }}
SEED_PHRASE: ${{ secrets.SEED_PHRASE }}
METAMASK_PASSWORD: ${{ secrets.METAMASK_PASSWORD }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
DYDX_MNEMONIC: ${{ secrets.DYDX_MNEMONIC }}
DYDX_ADDRESS: ${{ secrets.DYDX_ADDRESS }}
DYDX_NETWORK_TYPE: mainnet
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
DYDX_MNEMONIC_CANCEL_ORDER: ${{ secrets.DYDX_MNEMONIC_CANCEL_ORDER }}
DYDX_ADDRESS_CANCEL_ORDER: ${{ secrets.DYDX_ADDRESS_CANCEL_ORDER }}
SEED_PHRASE_CANCEL_ORDER: ${{ secrets.SEED_PHRASE_CANCEL_ORDER }}
SEED_PHRASE_MEGAVAULT: ${{ secrets.SEED_PHRASE_MEGAVAULT }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
S3_TRACE_BUCKET: ${{ secrets.S3_TRACE_BUCKET }}
outputs:
slack_ts: ${{ steps.slack_progress_message.outputs.message_ts }}
needs_review: ${{ steps.check-batch.outputs.needsReview }}
found_batch_id: ${{ steps.check-batch.outputs.foundBatchID }}
slack_channel: ${{ env.SLACK_CHANNEL }}
production_tag: ${{ steps.prod-tag.outputs.production-tag }}
e2e_exit_code: ${{ steps.run-e2e.outputs.exit_code }}
steps:
# 1) Checkout E2E repo
- name: Checkout E2E repository
uses: actions/checkout@v4
with:
fetch-depth: 1
# 2) Checkout upstream v4-web for its tags
- name: Checkout v4-web repository
if: ${{ inputs.deployment_id != 'manual' }}
uses: actions/checkout@v4
with:
repository: dydxprotocol/v4-web
path: v4-web
fetch-depth: 0
# 3) Identify latest release/vX.Y.Z on main
- name: Identify latest production tag on main
if: ${{ inputs.deployment_id != 'manual' }}
id: prod-tag
working-directory: v4-web
run: |
git fetch origin main --tags
TAG=$(git tag --merged origin/main \
| grep -E '^release/v[0-9]+\.[0-9]+\.[0-9]+$' \
| sort -V \
| tail -n1)
if [ -z "$TAG" ]; then
echo "❌ No production tags found!" >&2
exit 1
fi
echo "✔️ Latest tag: $TAG"
echo "production-tag=$TAG" >> $GITHUB_OUTPUT
# 4) Prepare for tests
- name: Make scripts executable
run: chmod +x scripts/*.sh
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: '20'
- name: Cache Node.js modules
uses: actions/cache@v4
with:
path: ~/.npm
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
restore-keys: |
${{ runner.os }}-node-
# 5) Find existing Slack thread
- name: Find Release Thread (if applicable)
if: ${{ inputs.deployment_id != 'manual' }}
id: find-thread
run: |
KEY=${{ steps.prod-tag.outputs.production-tag }}
HISTORY=$(curl -s -H "Authorization: Bearer $SLACK_BOT_TOKEN" \
-G --data-urlencode "channel=$SLACK_CHANNEL" \
--data-urlencode "limit=100" \
https://slack.com/api/conversations.history)
THREAD_TS=$(echo "$HISTORY" \
| jq -r --arg key "$KEY" '
.messages[]
| select( .attachments? and (
[ .attachments[] | (.fallback//""),(.title//""),(.text//"") ]
| map(test($key; "i")) | any
))
| .ts
' | head -n1)
if [ -n "$THREAD_TS" ]; then
echo "thread_ts=$THREAD_TS" >> $GITHUB_OUTPUT
fi
# 6) Post initial Slack message
- name: 🏁 Post Initial Progress to Slack
id: slack_progress_message
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
SLACK_CHANNEL: ${{ env.SLACK_CHANNEL }}
PARENT_THREAD_TS: ${{ steps.find-thread.outputs.thread_ts }}
DISPLAY: ${{ inputs.deployment_id == 'manual' && 'manual' || steps.prod-tag.outputs.production-tag }}
run: |
TEXT=$(
printf "*E2E Pipeline for:* %s\n" "$DISPLAY"
printf ">[░░░░░░] 0%% | Initializing job...\n"
printf "<%s|View full logs>\n" "$RUN_URL"
)
PAYLOAD=$(jq -n \
--arg channel "$SLACK_CHANNEL" \
--arg text "$TEXT" \
'{channel: $channel, text: $text}')
if [ -n "$PARENT_THREAD_TS" ]; then
PAYLOAD=$(echo "$PAYLOAD" \
| jq --arg ts "$PARENT_THREAD_TS" '. + {thread_ts: $ts}')
fi
RESPONSE=$(curl -s -X POST \
-H "Authorization: Bearer $SLACK_BOT_TOKEN" \
-H "Content-Type: application/json" \
--data "$PAYLOAD" \
https://slack.com/api/chat.postMessage)
echo "message_ts=$(jq -r .ts <<<"$RESPONSE")" >> $GITHUB_OUTPUT
- name: Install dependencies
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
SLACK_CHANNEL: ${{ env.SLACK_CHANNEL }}
MESSAGE_TS: ${{ steps.slack_progress_message.outputs.message_ts }}
DISPLAY: ${{ inputs.deployment_id == 'manual' && 'manual' || steps.prod-tag.outputs.production-tag }}
run: |
TEXT=$(
printf "*E2E Pipeline for:* %s\n" "$DISPLAY"
printf ">[█░░░░░] | Installing dependencies...\n"
printf "<%s|View full logs>\n" "$RUN_URL"
)
export MESSAGE_TEXT="$TEXT"
scripts/update-slack-progress.sh
npm install
- name: ► Run All E2E Tests
id: run-e2e
continue-on-error: true
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
SLACK_CHANNEL: ${{ env.SLACK_CHANNEL }}
MESSAGE_TS: ${{ steps.slack_progress_message.outputs.message_ts }}
DISPLAY: ${{ inputs.deployment_id == 'manual' && 'manual' || steps.prod-tag.outputs.production-tag }}
run: |
# update Slack to 80% complete
TEXT=$(
printf "*E2E Pipeline for:* %s\n" "$DISPLAY"
printf ">[███░░░] | Running full test suite...\n"
printf "<%s|View full logs>\n" "$RUN_URL"
)
export MESSAGE_TEXT="$TEXT"
scripts/update-slack-progress.sh
set +e
xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' \
npx playwright test --workers=2 --reporter=html
E2E_EXIT=$?
echo "exit_code=$E2E_EXIT" >> "$GITHUB_OUTPUT"
if [ "$E2E_EXIT" -ne 0 ]; then
echo "Playwright exited with $E2E_EXIT" >&2
fi
# do not rethrow; later steps rely on outputs
# 9) Sync report to S3
- name: Sync full HTML report to S3
if: always()
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
SLACK_CHANNEL: ${{ env.SLACK_CHANNEL }}
MESSAGE_TS: ${{ steps.slack_progress_message.outputs.message_ts }}
DISPLAY: ${{ inputs.deployment_id == 'manual' && 'manual' || steps.prod-tag.outputs.production-tag }}
run: |
TEXT=$(
printf "*E2E Pipeline for:* %s\n" "$DISPLAY"
printf ">[██████░░] 90%% | Uploading HTML report to S3...\n"
printf "<%s|View full logs>\n" "$RUN_URL"
)
export MESSAGE_TEXT="$TEXT"
scripts/update-slack-progress.sh
aws s3 sync playwright-report/ s3://${{ env.S3_TRACE_BUCKET }}/runs/${{ github.run_id }}/report/ --delete
echo "REPORT_URL=https://${{ env.S3_TRACE_BUCKET }}.s3.${{ env.AWS_DEFAULT_REGION }}.amazonaws.com/runs/${{ github.run_id }}/report/index.html" >> $GITHUB_ENV
# 10) Check Applitools
- name: Check Applitools Batch
id: check-batch
if: always()
env:
APPLITOOLS_SERVER_URL: ${{ env.APPLITOOLS_SERVER_URL }}
APPLITOOLS_API_KEY: ${{ secrets.APPLITOOLS_API_KEY }}
POINTER_ID: ${{ env.APPLITOOLS_BATCH_ID }}
run: |
set -euo pipefail
echo "=== Applitools Batch Check Start ===" >&2
# 1) build a 24h window (or adjust as needed)
START_DATE="$(date -u -d '1 day ago' +%Y-%m-%dT%H:%M:%SZ)"
END_DATE="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "Fetching Applitools batches from $START_DATE to $END_DATE..." >&2
# 2) fetch
response=$(curl -s -H "X-Eyes-Api-Key: $APPLITOOLS_API_KEY" \
"$APPLITOOLS_SERVER_URL/api/v1/batches?start=${START_DATE}&end=${END_DATE}&pageSize=100")
echo "Looking for batches with pointerId: $POINTER_ID" >&2
# 3) ensure valid JSON
if ! echo "$response" | jq empty 2>/dev/null; then
echo "Error: Invalid JSON response from Applitools API" >&2
echo "Response: $response" >&2
# fail the step so we can see the bad payload
exit 1
fi
# 4) filter & sort
filtered=$(echo "$response" \
| jq --arg p "$POINTER_ID" \
'[.batches[] | select(.pointerId == $p)] | sort_by(.startedAt)')
count=$(echo "$filtered" | jq 'length')
echo "Found $count batches with pointerId=$POINTER_ID" >&2
if [ "$count" -eq 0 ]; then
echo "No batch found; defaulting to needsReview=true" >&2
echo "foundBatchID=" >> "$GITHUB_OUTPUT"
echo "needsReview=true" >> "$GITHUB_OUTPUT"
exit 0
fi
# 5) pick the latest
latest=$(echo "$filtered" | jq '.[-1]')
batch_id=$(echo "$latest" | jq -r '.id')
# Use final batch fields (not runningSummary which can be 0 at completion)
failed=$(echo "$latest" | jq -r '.failed // 0')
unresolved=$(echo "$latest" | jq -r '.unresolved // 0')
new_tests=$(echo "$latest" | jq -r '.new // 0')
echo "Found batch id=$batch_id (failed=$failed, unresolved=$unresolved, new=$new_tests)" >&2
# 6) write outputs
echo "foundBatchID=$batch_id" >> "$GITHUB_OUTPUT"
if [ "$failed" -gt 0 ] || [ "$unresolved" -gt 0 ] || [ "$new_tests" -gt 0 ]; then
echo "needsReview=true" >> "$GITHUB_OUTPUT"
echo "Batch needs review." >&2
else
echo "needsReview=false" >> "$GITHUB_OUTPUT"
echo "Batch passed all checks." >&2
fi
echo "=== Applitools Batch Check Complete ===" >&2
- name: ✅ Post Final Result to Slack
if: always()
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
SLACK_CHANNEL: ${{ env.SLACK_CHANNEL }}
RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
REPORT_URL: ${{ env.REPORT_URL }}
DISPLAY: ${{ inputs.deployment_id == 'manual' && 'manual' || steps.prod-tag.outputs.production-tag }}
APPLITOOLS_LINK: "https://eyes.applitools.com/app/test-results/${{ steps.check-batch.outputs.foundBatchID }}/"
MESSAGE_TS: ${{ steps.slack_progress_message.outputs.message_ts }}
run: |
set -e
# Compose two-line header: E2E line + Visual line
if [ "${{ steps.run-e2e.outputs.exit_code }}" != "0" ]; then
E2E_LINE="❌ E2E Tests Failed"
VISUAL_LINE=""
else
E2E_LINE="✅ E2E Tests Passed"
if [ "${{ steps.check-batch.outputs.needsReview }}" = "true" ]; then
VISUAL_LINE="⚠️ Visual diffs detected — please review"
else
VISUAL_LINE="✅ Visual tests passed"
fi
fi
# Build the message (edit the original post)
MESSAGE_TEXT=$(
printf "%s\n" \
"$E2E_LINE" \
"$VISUAL_LINE" \
"• Deployment: $DISPLAY" \
"• <$RUN_URL|View GitHub Run>" \
"• <$REPORT_URL|View Full HTML Report>" \
"• <$APPLITOOLS_LINK|View Applitools Batch>"
)
export MESSAGE_TEXT
# Update the existing thread
scripts/update-slack-progress.sh || true
# 12) Upload artifact
- name: Upload Test Report artifact
if: always()
uses: actions/upload-artifact@v4
with:
name: playwright-report-${{ github.run_id }}
path: playwright-report/
retention-days: 7
applitools-auto-complete:
name: "Applitools Auto-Completion"
if: needs.e2e-test.outputs.needs_review == 'false' && needs.e2e-test.outputs.e2e_exit_code == '0'
needs: e2e-test
runs-on: ubuntu-latest
env:
SLACK_CHANNEL: ${{ needs.e2e-test.outputs.slack_channel }}
ROOT_THREAD_TS: ${{ needs.e2e-test.outputs.slack_ts }}
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
APPLITOOLS_API_KEY: ${{ secrets.APPLITOOLS_API_KEY }}
DISPLAY: ${{ github.event.inputs.deployment_id == 'manual' && 'manual' || needs.e2e-test.outputs.production_tag }}
S3_TRACE_BUCKET: ${{ secrets.S3_TRACE_BUCKET }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
steps:
- uses: actions/checkout@v4
- name: Make scripts executable
run: chmod +x scripts/*.sh
- name: Mark Applitools Batch as Complete
run: |
curl -X POST -d '' \
-H 'accept:*/*' \
"$APPLITOOLS_SERVER_URL/api/externals/github/servers/github.com/commit/${{ needs.e2e-test.outputs.found_batch_id }}/complete?apiKey=$APPLITOOLS_API_KEY"
- name: Edit original message with 'Visual tests passed'
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
SLACK_CHANNEL: ${{ needs.e2e-test.outputs.slack_channel }}
ROOT_THREAD_TS: ${{ needs.e2e-test.outputs.slack_ts }}
run: |
set -euo pipefail
RUN_URL="https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
HTML_REPORT_URL="https://${S3_TRACE_BUCKET}.s3.${AWS_DEFAULT_REGION}.amazonaws.com/runs/${GITHUB_RUN_ID}/report/index.html"
APPLITOOLS_LINK="https://eyes.applitools.com/app/test-results/${{ needs.e2e-test.outputs.found_batch_id }}/"
TEXT=$(
printf "✅ E2E Tests Passed\n"
printf "✅ Visual tests passed\n"
printf "• Deployment: %s\n" "${DISPLAY}"
printf "• <%s|View GitHub Run>\n" "${RUN_URL}"
printf "• <%s|View Full HTML Report>\n" "${HTML_REPORT_URL}"
printf "• <%s|View Applitools Batch>\n" "${APPLITOOLS_LINK}"
)
jq -n \
--arg channel "$SLACK_CHANNEL" \
--arg ts "$ROOT_THREAD_TS" \
--arg text "$TEXT" \
'{channel:$channel, ts:$ts, text:$text}' \
| curl -s -X POST \
-H "Authorization: Bearer $SLACK_BOT_TOKEN" \
-H "Content-Type: application/json" \
--data @- \
https://slack.com/api/chat.update
batch-status-polling:
name: "Applitools Batch Status Polling"
if: needs.e2e-test.outputs.needs_review == 'true' && needs.e2e-test.outputs.e2e_exit_code == '0'
needs: e2e-test
runs-on: ubuntu-latest
env:
# from the e2e-test outputs
SLACK_CHANNEL: ${{ needs.e2e-test.outputs.slack_channel }}
THREAD_TS: ${{ needs.e2e-test.outputs.slack_ts }}
FOUND_BATCH_ID: ${{ needs.e2e-test.outputs.found_batch_id }}
DISPLAY: ${{ github.event.inputs.deployment_id == 'manual' && 'manual' || needs.e2e-test.outputs.production_tag }}
# secrets / top-level
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
APPLITOOLS_API_KEY: ${{ secrets.APPLITOOLS_API_KEY }}
APPLITOOLS_SERVER_URL: https://eyesapi.applitools.com
steps:
- uses: actions/checkout@v4
- name: Poll Applitools Batch Status
id: poll-status
shell: bash
run: |
set -euo pipefail
# Build your links now that $GITHUB_RUN_ID, $S3_TRACE_BUCKET, and $AWS_DEFAULT_REGION are in the environment:
RUN_URL="https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
HTML_REPORT_URL="https://${S3_TRACE_BUCKET}.s3.${AWS_DEFAULT_REGION}.amazonaws.com/runs/${GITHUB_RUN_ID}/report/index.html"
APPLITOOLS_LINK="https://eyes.applitools.com/app/test-results/${FOUND_BATCH_ID}/"
# Helper: edit the original Slack thread in-place
update_slack() {
jq -n \
--arg channel "$SLACK_CHANNEL" \
--arg ts "$THREAD_TS" \
--arg text "$1" \
'{channel:$channel,ts:$ts,text:$text}' \
| curl -s -X POST \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $SLACK_BOT_TOKEN" \
--data @- \
https://slack.com/api/chat.update \
>/dev/null
}
# 1) Initial “waiting for review” edit (keep two-line format)
initial_text=$(
printf "✅ E2E Tests Passed\n"
printf "⚠️ Visual diffs detected — please review\n"
printf "• Deployment: %s\n" "$DISPLAY"
printf "• <%s|View GitHub Run>\n" "$RUN_URL"
printf "• <%s|View Full HTML Report>\n" "$HTML_REPORT_URL"
printf "• <%s|View Applitools Batch>\n" "$APPLITOOLS_LINK"
)
update_slack "$initial_text"
# 2) Poll every 30s, up to 1h
MAX=120; INTERVAL=30; i=0; status="timeout"
while (( i < MAX )); do
(( i++ ))
start=$(date -u -d '1 hour ago' +%Y-%m-%dT%H:%M:%SZ)
end=$(date -u +%Y-%m-%dT%H:%M:%SZ)
resp=$(curl -s -H "X-Eyes-Api-Key: $APPLITOOLS_API_KEY" \
"$APPLITOOLS_SERVER_URL/api/v1/batches?start=${start}&end=${end}&pageSize=100")
# skip on malformed JSON
if ! echo "$resp" | jq -e . >/dev/null; then
sleep $INTERVAL && continue
fi
batch=$(echo "$resp" | jq -c --arg id "$FOUND_BATCH_ID" \
'.batches[] | select(.id==$id)')
[[ -z "$batch" || "$batch" == "null" ]] && sleep $INTERVAL && continue
F=$(echo "$batch" | jq -r '.runningSummary.failedTests // 0')
U=$(echo "$batch" | jq -r '.runningSummary.unresolvedTests // 0')
N=$(echo "$batch" | jq -r '.runningSummary.newTests // 0')
if (( F > 0 )); then
status="rejected"; header="❌ *Visual Review Rejected*"
elif (( F==0 && U==0 && N==0 )); then
status="approved"; header="✅ *Visual Review Approved*"
else
sleep $INTERVAL && continue
fi
# 3) Final update: set second line to Approved/Rejected
if [ "$status" = "approved" ]; then
visual_line="✅ Visual Review Approved"
else
visual_line="❌ Visual Review Rejected"
fi
final_text=$(
printf "✅ E2E Tests Passed\n"
printf "%s\n" "$visual_line"
printf "• Deployment: %s\n" "$DISPLAY"
printf "• <%s|View GitHub Run>\n" "$RUN_URL"
printf "• <%s|View Full HTML Report>\n" "$HTML_REPORT_URL"
printf "• <%s|View Applitools Batch>\n" "$APPLITOOLS_LINK"
)
update_slack "$final_text"
break
done
# 4) Export status for next steps
echo "status=$status" >> "$GITHUB_OUTPUT"
- name: Mark Applitools Batch as Complete
if: steps.poll-status.outputs.status == 'approved'
run: |
curl -s -X POST \
-H 'accept: */*' \
"$APPLITOOLS_SERVER_URL/api/externals/github/servers/github.com/commit/${FOUND_BATCH_ID}/complete?apiKey=$APPLITOOLS_API_KEY"
- name: Fail Workflow if Review Rejected
if: steps.poll-status.outputs.status == 'rejected'
run: exit 1