RPC Healthcheck #280
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: RPC Healthcheck | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| on: | |
| schedule: | |
| - cron: '41 2 * * *' | |
| - cron: '41 10 * * *' | |
| - cron: '41 18 * * *' | |
| workflow_dispatch: | |
| jobs: | |
| check-rootchain: | |
| name: Check ${{ matrix.name }} root chain last block time | |
| if: ${{ github.repository == 'QuarkChain/goquarkchain' && vars.RPC_HEALTHCHECK_TARGETS_JSON != '' }} | |
| runs-on: ubuntu-latest | |
| # Must exceed the curl retry budget below (--retry-max-time 600 plus one | |
| # in-flight --max-time 120 attempt, ~640s worst case). Otherwise the job is | |
| # canceled mid-retry before it can write/upload its report, and the target | |
| # silently drops out of the notification email. | |
| timeout-minutes: 13 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJSON(vars.RPC_HEALTHCHECK_TARGETS_JSON) }} | |
| steps: | |
| - name: Fetch getRootBlockByHeight and check last block time | |
| shell: bash | |
| env: | |
| RPC_URL: ${{ matrix.rpc_url }} | |
| NAME: ${{ matrix.name }} | |
| run: | | |
| set -euo pipefail | |
| payload='{"jsonrpc":"2.0","id":1,"method":"getRootBlockByHeight","params":[null]}' | |
| resp_file="${RUNNER_TEMP}/rootblock_${NAME}.json" | |
| curl_err_file="${RUNNER_TEMP}/curl_err_rootchain_${NAME}.txt" | |
| report_file="${RUNNER_TEMP}/rootchain_report_${NAME}.json" | |
| result="success" | |
| error_msg="" | |
| root_height=0 | |
| root_timestamp=0 | |
| since_last_block=0 | |
| code="$( | |
| # Capture status code; use retries/timeouts to reduce flakiness. | |
| # --retry / --retry-delay: retry on failure with backoff | |
| # --retry-all-errors: retry on any error, not just transient ones | |
| # --connect-timeout: time to wait for TCP connection / DNS | |
| # --max-time: max total time per individual attempt | |
| # --retry-max-time: cap on total time spent across all retries | |
| curl --silent --show-error --location \ | |
| --retry 10 --retry-delay 10 --retry-all-errors \ | |
| --connect-timeout 30 --max-time 120 \ | |
| --retry-max-time 600 \ | |
| -X POST \ | |
| -H 'Content-Type: application/json' \ | |
| --data "$payload" \ | |
| --output "$resp_file" --write-out '%{http_code}' \ | |
| "$RPC_URL" 2>"$curl_err_file" || true | |
| )" | |
| if [[ "$code" != "200" ]]; then | |
| result="failure" | |
| curl_err="$(head -1 "$curl_err_file" 2>/dev/null | xargs || true)" | |
| if [[ -n "$curl_err" ]]; then | |
| error_msg="$curl_err" | |
| else | |
| error_msg="HTTP status not 200: $code" | |
| fi | |
| else | |
| root_height_hex="$(jq -r '.result.height' "$resp_file")" | |
| root_timestamp_hex="$(jq -r '.result.timestamp' "$resp_file")" | |
| root_height=$((root_height_hex)) | |
| root_timestamp=$((root_timestamp_hex)) | |
| now="$(date +%s)" | |
| since_last_block=$((now - root_timestamp)) | |
| echo "[$NAME] rootHeight: $root_height" | |
| echo "[$NAME] rootTimestamp: $root_timestamp" | |
| echo "[$NAME] sinceLastBlockTime: ${since_last_block}s" | |
| if [[ "$since_last_block" -eq 0 || "$since_last_block" -gt 600 ]]; then | |
| result="warning" | |
| _h=$(( since_last_block / 3600 )) | |
| _m=$(( (since_last_block % 3600) / 60 )) | |
| _s=$(( since_last_block % 60 )) | |
| if [[ $_h -gt 0 ]]; then | |
| _duration="${_h}h ${_m}m ${_s}s" | |
| elif [[ $_m -gt 0 ]]; then | |
| _duration="${_m}m ${_s}s" | |
| else | |
| _duration="${_s}s" | |
| fi | |
| error_msg="${_duration} since last block" | |
| fi | |
| fi | |
| jq -n \ | |
| --arg name "$NAME" \ | |
| --arg rpc_url "$RPC_URL" \ | |
| --arg result "$result" \ | |
| --argjson root_height "$root_height" \ | |
| --argjson root_timestamp "$root_timestamp" \ | |
| --argjson since_last_block "$since_last_block" \ | |
| --arg error "$error_msg" \ | |
| '{name: $name, rpc_url: $rpc_url, result: $result, root_height: $root_height, root_timestamp: $root_timestamp, since_last_block: $since_last_block, error: $error}' \ | |
| > "$report_file" | |
| if [[ "$result" != "success" ]]; then | |
| if [[ "$result" == "warning" ]]; then | |
| echo "::warning title=${NAME} root chain catch-up::${error_msg}" | |
| echo "[$NAME] Root chain check WARNING: $error_msg" | |
| exit 0 | |
| else | |
| echo "[$NAME] Root chain check FAILED: $error_msg" | |
| exit 1 | |
| fi | |
| fi | |
| echo "[$NAME] Root chain check OK: Last block was ${since_last_block}s ago, height: ${root_height}" | |
| - name: Upload root chain report | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: rpc-healthcheck-rootchain-${{ matrix.name }} | |
| path: ${{ runner.temp }}/rootchain_report_${{ matrix.name }}.json | |
| if-no-files-found: error | |
| notify: | |
| name: Notify (email) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| needs: | |
| - check-rootchain | |
| if: ${{ always() && github.repository == 'QuarkChain/goquarkchain' && vars.RPC_HEALTHCHECK_TARGETS_JSON != '' }} | |
| steps: | |
| - name: Download reports | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: rpc-healthcheck-rootchain-* | |
| merge-multiple: true | |
| path: ${{ runner.temp }}/rpc-healthcheck | |
| - name: Compose email | |
| id: compose | |
| shell: bash | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| EVENT_SCHEDULE: ${{ github.event.schedule }} | |
| ROOTCHAIN_CHECK_RESULT: ${{ needs.check-rootchain.result }} | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| RPC_HEALTHCHECK_TARGETS_JSON: ${{ vars.RPC_HEALTHCHECK_TARGETS_JSON }} | |
| run: | | |
| set -euo pipefail | |
| send=false | |
| overall="OK" | |
| report_dir="${RUNNER_TEMP}/rpc-healthcheck" | |
| table_rows='' | |
| html_rows='' | |
| # Iterate over the expected targets (the same list that drives the | |
| # check-rootchain matrix) rather than over whatever reports happened | |
| # to be uploaded. That way a target whose job timed out or was | |
| # canceled before it could write/upload a report still shows up as a | |
| # failed row instead of silently vanishing from the email. | |
| mapfile -t targets < <(printf '%s' "${RPC_HEALTHCHECK_TARGETS_JSON}" | jq -c '.[]') | |
| for t in "${targets[@]}"; do | |
| exp_name="$(jq -r '.name' <<<"$t")" | |
| exp_url="$(jq -r '.rpc_url' <<<"$t")" | |
| f="${report_dir}/rootchain_report_${exp_name}.json" | |
| if [[ -f "$f" ]]; then | |
| rc_name="$(jq -r '.name' "$f")" | |
| rc_rpc_url="$(jq -r '.rpc_url' "$f")" | |
| rc_result="$(jq -r '.result' "$f")" | |
| rc_height="$(jq -r '.root_height' "$f")" | |
| rc_error="$(jq -r '.error' "$f")" | |
| if [[ "$rc_result" == "failure" ]]; then | |
| overall="FAILED" | |
| send=true | |
| rc_icon="❌" | |
| if [[ -z "$rc_error" ]]; then rc_error="unknown error"; fi | |
| elif [[ "$rc_result" == "warning" ]]; then | |
| if [[ "$overall" != "FAILED" ]]; then overall="WARN"; fi | |
| send=true | |
| rc_icon="⚠️" | |
| if [[ -z "$rc_error" ]]; then rc_error="catching up"; fi | |
| else | |
| rc_icon="✅" | |
| rc_error="-" | |
| fi | |
| else | |
| # No report uploaded for this target: its job likely hit the | |
| # timeout-minutes limit (or otherwise died) before writing one. | |
| # Surface it as a failure so it stays visible in the email. | |
| overall="FAILED" | |
| send=true | |
| rc_name="$exp_name" | |
| rc_rpc_url="$exp_url" | |
| rc_height="-" | |
| rc_icon="❌" | |
| rc_error="no report (job timed out or was canceled)" | |
| fi | |
| rc_rpc_url_esc="${rc_rpc_url//|/\\|}" | |
| rc_error_esc="${rc_error//|/\\|}" | |
| table_rows+="| ${rc_name} | ${rc_rpc_url_esc} | ${rc_icon} | ${rc_height} | ${rc_error_esc} |\n" | |
| rc_name_html="${rc_name//&/&}" | |
| rc_name_html="${rc_name_html//</<}" | |
| rc_name_html="${rc_name_html//>/>}" | |
| rc_rpc_url_html="${rc_rpc_url//&/&}" | |
| rc_rpc_url_html="${rc_rpc_url_html//</<}" | |
| rc_rpc_url_html="${rc_rpc_url_html//>/>}" | |
| rc_error_html="${rc_error//&/&}" | |
| rc_error_html="${rc_error_html//</<}" | |
| rc_error_html="${rc_error_html//>/>}" | |
| html_rows+="<tr><td>${rc_name_html}</td><td>${rc_rpc_url_html}</td><td>${rc_icon}</td><td>${rc_height}</td><td>${rc_error_html}</td></tr>" | |
| done | |
| if [[ "$ROOTCHAIN_CHECK_RESULT" != "success" ]]; then | |
| overall="FAILED" | |
| send=true | |
| elif [[ "$overall" == "OK" ]]; then | |
| # Success: send only on manual trigger, or once per day for scheduled runs. | |
| if [[ "$EVENT_NAME" == "workflow_dispatch" ]]; then | |
| send=true | |
| elif [[ "$EVENT_NAME" == "schedule" ]]; then | |
| if [[ "${EVENT_SCHEDULE:-}" == "41 18 * * *" ]]; then | |
| send=true | |
| fi | |
| fi | |
| fi | |
| if [[ "$overall" == "OK" ]]; then | |
| subject="✅ QuarkChain RPC Healthcheck OK" | |
| elif [[ "$overall" == "WARN" ]]; then | |
| subject="⚠️ QuarkChain RPC Healthcheck Lagging" | |
| else | |
| subject="❌ QuarkChain RPC Healthcheck FAILED" | |
| fi | |
| echo "send=$send" >> "$GITHUB_OUTPUT" | |
| echo "subject=$subject" >> "$GITHUB_OUTPUT" | |
| echo "body<<EOF" >> "$GITHUB_OUTPUT" | |
| echo "QuarkChain RPC Healthcheck: $overall" >> "$GITHUB_OUTPUT" | |
| echo >> "$GITHUB_OUTPUT" | |
| echo "Event: $EVENT_NAME" >> "$GITHUB_OUTPUT" | |
| echo "Schedule: ${EVENT_SCHEDULE:-N/A}" >> "$GITHUB_OUTPUT" | |
| echo "Run: $RUN_URL" >> "$GITHUB_OUTPUT" | |
| echo >> "$GITHUB_OUTPUT" | |
| echo "| name | rpc_url | result | root_height | details |" >> "$GITHUB_OUTPUT" | |
| echo "|---|---|---|---|---|" >> "$GITHUB_OUTPUT" | |
| printf "%b" "$table_rows" >> "$GITHUB_OUTPUT" | |
| echo "EOF" >> "$GITHUB_OUTPUT" | |
| echo "html_body<<EOF" >> "$GITHUB_OUTPUT" | |
| echo "<p>QuarkChain RPC Healthcheck: <b>${overall}</b></p>" >> "$GITHUB_OUTPUT" | |
| echo "<p>Event: ${EVENT_NAME}<br/>Schedule: ${EVENT_SCHEDULE:-N/A}<br/>Run: <a href=\"${RUN_URL}\">${RUN_URL}</a></p>" >> "$GITHUB_OUTPUT" | |
| echo "<table border=\"1\" cellpadding=\"6\" cellspacing=\"0\" style=\"border-collapse: collapse;\">" >> "$GITHUB_OUTPUT" | |
| echo "<thead><tr><th>name</th><th>rpc_url</th><th>result</th><th>root_height</th><th>details</th></tr></thead>" >> "$GITHUB_OUTPUT" | |
| echo "<tbody>${html_rows}</tbody>" >> "$GITHUB_OUTPUT" | |
| echo "</table>" >> "$GITHUB_OUTPUT" | |
| echo "EOF" >> "$GITHUB_OUTPUT" | |
| - name: Send email | |
| if: ${{ steps.compose.outputs.send == 'true' }} | |
| uses: dawidd6/action-send-mail@v17 | |
| with: | |
| server_address: smtp.gmail.com | |
| server_port: 465 | |
| username: ${{ secrets.RPC_HEALTHCHECK_SMTP_USERNAME }} | |
| password: ${{ secrets.RPC_HEALTHCHECK_SMTP_PASSWORD }} | |
| from: QuarkChainRPC <${{ secrets.RPC_HEALTHCHECK_SMTP_USERNAME }}> | |
| to: ${{ secrets.RPC_HEALTHCHECK_EMAIL_TO }} | |
| subject: ${{ steps.compose.outputs.subject }} | |
| body: ${{ steps.compose.outputs.body }} | |
| html_body: ${{ steps.compose.outputs.html_body }} |