Skip to content

Commit 4b03ee1

Browse files
scttbnsnscttbnsn
authored
🔧 chore(ci): retry the known -fuzztime boundary flake once in deep fuzz (#84)
A bare "context deadline exceeded" at the -fuzztime boundary with no crasher is a known Go fuzzing-engine flake (golang/go#48591) that strikes a random fuzzer under shared-runner load — it hit FuzzSwarm on 2026-06-04 and FuzzHijackHeadersAndBody on 2026-05-23, each at exactly 900.10s with no input written. The Fuzz step now classifies the failure and retries only that case once: - real crasher ("Failing input written to testdata/...") → fail immediately - genuine hang / build error → fail immediately - boundary flake (context deadline exceeded, no crasher) → retry once, then red the job if it recurs so the flake stays visible Exit-code fidelity is kept via `run_fuzz || rc=$?` (suppresses set -e in the condition) and `return "${PIPESTATUS[0]}"` through the tee. Summarize now reads the step's `kind` output so it no longer mislabels a double-flake as a crasher.
1 parent 7fb8270 commit 4b03ee1

1 file changed

Lines changed: 67 additions & 9 deletions

File tree

‎.github/workflows/quality-fuzz-nightly.yml‎

Lines changed: 67 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,12 @@ run-name: >-
2222
# input is recoverable. The triage workflow is: download the artifact,
2323
# commit the minimized input under `app/internal/<pkg>/testdata/fuzz/<Fuzz*>/`
2424
# so it enters the seed corpus for all future runs, fix the bug, push.
25+
#
26+
# One narrow exception to "every failure is a crasher": a bare "context
27+
# deadline exceeded" at the -fuzztime boundary with no crasher is a known Go
28+
# fuzzing-engine flake (golang/go#48591) that strikes a random fuzzer under
29+
# shared-runner load. The Fuzz step retries that single case once; only a real
30+
# crasher or a genuine hang reds the run.
2531

2632
on:
2733
workflow_dispatch:
@@ -47,9 +53,9 @@ jobs:
4753
name: "🔀 Fuzz ${{ matrix.fuzzer.name }}"
4854
runs-on: ubuntu-latest
4955
# Job-level cap is the hard ceiling regardless of what -fuzztime asks
50-
# for. Keep it comfortably above the default 15m + setup overhead and
51-
# above the longest documented manual budget plus the fuzz watchdog
52-
# cushion.
56+
# for. Keep it comfortably above the default budget run twice (the Fuzz
57+
# step retries the known boundary flake once → ~2×15m + setup) and above
58+
# the longest documented manual budget plus the fuzz watchdog cushion.
5359
timeout-minutes: 75
5460

5561
strategy:
@@ -109,6 +115,7 @@ jobs:
109115
echo "Running ${{ matrix.fuzzer.name }} for ${FUZZTIME} with ${TEST_TIMEOUT} watchdog"
110116
111117
- name: Fuzz ${{ matrix.fuzzer.name }}
118+
id: fuzz
112119
working-directory: app
113120
env:
114121
FUZZER: ${{ matrix.fuzzer.name }}
@@ -119,11 +126,54 @@ jobs:
119126
# -run=^$ skips normal unit tests in the package so only the fuzz
120127
# driver runs. TEST_TIMEOUT is FUZZTIME plus a cushion, so hangs
121128
# produce Go's goroutine dump before the job-level ceiling fires.
122-
go test -run='^$' \
123-
-fuzz="^${FUZZER}\$" \
124-
-fuzztime="${FUZZTIME}" \
125-
-timeout="${TEST_TIMEOUT}" \
126-
"${PKG}"
129+
#
130+
# A bare "context deadline exceeded" at the -fuzztime boundary with no
131+
# crasher is a known Go fuzzing flake (golang/go#48591): under
132+
# shared-runner load a worker can miss the coordinator's stop deadline
133+
# even though no input failed. We retry that one case ONCE. A real
134+
# crasher (Go prints "Failing input written to testdata/...") or a
135+
# genuine hang (the -timeout watchdog prints "test timed out") fails
136+
# immediately and is never retried.
137+
LOG="${RUNNER_TEMP}/fuzz-${FUZZER}.log"
138+
139+
run_fuzz() {
140+
go test -run='^$' \
141+
-fuzz="^${FUZZER}\$" \
142+
-fuzztime="${FUZZTIME}" \
143+
-timeout="${TEST_TIMEOUT}" \
144+
"${PKG}" 2>&1 | tee "${LOG}"
145+
return "${PIPESTATUS[0]}"
146+
}
147+
148+
emit() { echo "kind=$1" >> "$GITHUB_OUTPUT"; }
149+
150+
for attempt in 1 2; do
151+
rc=0
152+
run_fuzz || rc=$?
153+
154+
if [ "${rc}" -eq 0 ]; then
155+
emit pass
156+
exit 0
157+
fi
158+
159+
if grep -q "Failing input written to testdata" "${LOG}"; then
160+
emit crash
161+
echo "::error::${FUZZER} found a crashing input — commit it to the seed corpus and fix the regression."
162+
exit "${rc}"
163+
fi
164+
165+
if ! grep -q "context deadline exceeded" "${LOG}"; then
166+
emit error
167+
echo "::error::${FUZZER} failed for a non-flake reason (exit ${rc})."
168+
exit "${rc}"
169+
fi
170+
171+
echo "::warning::${FUZZER}: known -fuzztime boundary flake (context deadline exceeded, no crasher) on attempt ${attempt}/2."
172+
done
173+
174+
emit flake
175+
echo "::error::${FUZZER} hit the -fuzztime boundary flake on both attempts; failing so it stays visible."
176+
exit 1
127177
128178
- name: Upload fuzz corpus on failure or cancel
129179
# `cancelled()` covers manual cancels and job-level timeout-minutes
@@ -145,6 +195,7 @@ jobs:
145195
FUZZTIME: ${{ steps.budget.outputs.fuzztime }}
146196
TEST_TIMEOUT: ${{ steps.budget.outputs.test_timeout }}
147197
STATUS: ${{ job.status }}
198+
KIND: ${{ steps.fuzz.outputs.kind }}
148199
RUN_ID: ${{ github.run_id }}
149200
run: |
150201
{
@@ -155,11 +206,18 @@ jobs:
155206
echo "- Timeout: ${TEST_TIMEOUT}"
156207
echo "- Result: ${STATUS}"
157208
echo ""
158-
if [ "${STATUS}" = "failure" ]; then
209+
if [ "${KIND}" = "crash" ]; then
159210
echo "A crashing input was saved to \`testdata/fuzz/${FUZZER}/\`"
160211
echo "and uploaded as artifact"
161212
echo "\`fuzz-corpus-${FUZZER}-${RUN_ID}\`."
162213
echo "Download it, commit the minimized input to the seed corpus,"
163214
echo "fix the regression, and push."
215+
elif [ "${KIND}" = "flake" ]; then
216+
echo "Failed twice on the known \`-fuzztime\` boundary flake"
217+
echo "(\`context deadline exceeded\`, no crasher — golang/go#48591)."
218+
echo "Re-run the job to clear it."
219+
elif [ "${STATUS}" = "failure" ]; then
220+
echo "Failed before the fuzz driver produced a verdict (setup/build error)."
221+
echo "Check the job logs."
164222
fi
165223
} >> "$GITHUB_STEP_SUMMARY"

0 commit comments

Comments
 (0)