|
| 1 | +#!/usr/bin/env bash |
| 2 | +# SPDX-License-Identifier: MPL-2.0 |
| 3 | +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell |
| 4 | +# |
| 5 | +# test_secret_scanner_canary.sh — prove the secret scanner can still FAIL. |
| 6 | +# |
| 7 | +# WHY THIS EXISTS |
| 8 | +# |
| 9 | +# Every defect this suite guards against was invisible in CI. Ranked by how |
| 10 | +# long each went unnoticed: |
| 11 | +# |
| 12 | +# * `continue-on-error: true` on the gitleaks step. The scan ran, found a |
| 13 | +# live Cloudflare Global API Key in a PUBLIC repo, and reported success |
| 14 | +# anyway — for months, across ~200 repos. |
| 15 | +# * `rust-secrets` grepped `./src` only. A Cargo workspace keeps code in |
| 16 | +# `crates/*/src`, so `grep` exited 2 on a missing directory and the |
| 17 | +# enclosing `if` read that as "clean". 8 of 64 Rust repos (12.5%) were |
| 18 | +# never scanned at all. |
| 19 | +# * `const.*KEY.*=` never matched `static AUTH_KEY: &str = "…"`. Found only |
| 20 | +# because a canary planted 5 secrets and the job reported 4. Reading the |
| 21 | +# regex would never have shown it. |
| 22 | +# |
| 23 | +# The common shape: **a gate that cannot fail is indistinguishable from a gate |
| 24 | +# that passes.** Nothing in a dashboard distinguishes them. The only defence is |
| 25 | +# to plant a known secret and assert the scanner still trips. |
| 26 | +# |
| 27 | +# A security gate without a canary is an unverified claim. |
| 28 | +# |
| 29 | +# WHAT IT TESTS AGAINST |
| 30 | +# |
| 31 | +# The step body is extracted from `secret-scanner-reusable.yml` at run time and |
| 32 | +# executed. It is NOT a copy: a copy silently drifts from what actually ships, |
| 33 | +# which is exactly the failure mode that let the defects above survive review. |
| 34 | +# |
| 35 | +# Usage: tests/test_secret_scanner_canary.sh [path-to-reusable.yml] |
| 36 | +# Exit: 0 = every canary behaved correctly, 1 = the gate is not trustworthy. |
| 37 | + |
| 38 | +set -uo pipefail |
| 39 | + |
| 40 | +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
| 41 | +REUSABLE="${1:-${SCRIPT_DIR}/../.github/workflows/secret-scanner-reusable.yml}" |
| 42 | + |
| 43 | +WORK=$(mktemp -d) |
| 44 | +trap 'rm -rf "$WORK"' EXIT |
| 45 | + |
| 46 | +pass=0 |
| 47 | +fail=0 |
| 48 | + |
| 49 | +if [ ! -f "$REUSABLE" ]; then |
| 50 | + echo "::error::canary: reusable workflow not found at $REUSABLE" |
| 51 | + exit 1 |
| 52 | +fi |
| 53 | + |
| 54 | +# --------------------------------------------------------------------------- |
| 55 | +# Extract the shipping step body. Any failure here is a HARD failure: silently |
| 56 | +# skipping would restore the very "gate that cannot fail" this test exists to |
| 57 | +# prevent. |
| 58 | +# --------------------------------------------------------------------------- |
| 59 | +python3 - "$REUSABLE" "$WORK/rust-secrets.sh" <<'PY' || { echo "::error::canary: could not extract the rust-secrets step from the reusable"; exit 1; } |
| 60 | +import sys |
| 61 | +try: |
| 62 | + import yaml |
| 63 | +except ImportError: |
| 64 | + sys.stderr.write("PyYAML is required to run the canary (pip install pyyaml)\n") |
| 65 | + sys.exit(1) |
| 66 | +
|
| 67 | +src, dst = sys.argv[1], sys.argv[2] |
| 68 | +doc = yaml.safe_load(open(src)) |
| 69 | +steps = doc["jobs"]["rust-secrets"]["steps"] |
| 70 | +step = next(s for s in steps if s.get("name", "").startswith("Check for hardcoded")) |
| 71 | +env = step.get("env", {}) or {} |
| 72 | +cutoff = env.get("ENFORCE_RUST_WIDE_SCAN_FROM", "2026-08-21") |
| 73 | +with open(dst, "w") as fh: |
| 74 | + fh.write("#!/usr/bin/env bash\nset -uo pipefail\n") |
| 75 | + fh.write('ENFORCE_RUST_WIDE_SCAN_FROM="${ENFORCE_RUST_WIDE_SCAN_FROM:-%s}"\n' % cutoff) |
| 76 | + fh.write(step["run"]) |
| 77 | +print("extracted rust-secrets step (cutoff %s)" % cutoff) |
| 78 | +PY |
| 79 | + |
| 80 | +SCAN="$WORK/rust-secrets.sh" |
| 81 | +chmod +x "$SCAN" |
| 82 | + |
| 83 | +# Build a fixture tree. `mk <dir> <relative-file>` then feed stdin. |
| 84 | +mk() { mkdir -p "$WORK/$1/$(dirname "$2")"; cat > "$WORK/$1/$2"; } |
| 85 | + |
| 86 | +# `expect <dir> <wanted-exit> <label> [env...]` |
| 87 | +expect() { |
| 88 | + local dir="$1" want="$2" label="$3"; shift 3 |
| 89 | + ( cd "$WORK/$dir" && env "$@" bash "$SCAN" >"$WORK/out.txt" 2>&1 ) |
| 90 | + local rc=$? |
| 91 | + if [ "$rc" = "$want" ]; then |
| 92 | + echo " PASS $label" |
| 93 | + pass=$((pass+1)) |
| 94 | + else |
| 95 | + echo " FAIL $label (exit $rc, expected $want)" |
| 96 | + sed 's/^/ /' "$WORK/out.txt" | head -6 |
| 97 | + fail=$((fail+1)) |
| 98 | + fi |
| 99 | +} |
| 100 | + |
| 101 | +echo "Running secret-scanner canary..." |
| 102 | + |
| 103 | +# --------------------------------------------------------------------------- |
| 104 | +# 1. REAL SECRETS MUST BE DETECTED — the load-bearing assertion. |
| 105 | +# `static` forms are included deliberately: `const.*KEY.*=` missed |
| 106 | +# `static AUTH_KEY` in production, and only a planted-secret count exposed it. |
| 107 | +# --------------------------------------------------------------------------- |
| 108 | +echo '[package]' > "$WORK/.keep"; mkdir -p "$WORK/real"; echo '[package]' > "$WORK/real/Cargo.toml" |
| 109 | +mk real src/main.rs <<'EOF' |
| 110 | +const API_SECRET: &str = "PLANTED_CANARY_VALUE_NOT_REAL_1"; |
| 111 | +const SERVICE_TOKEN: &str = "PLANTED_CANARY_VALUE_NOT_REAL_2"; |
| 112 | +static AUTH_KEY: &str = "abcdef0123456789abcdef0123456789"; |
| 113 | +let api_key = "PLANTED_CANARY_VALUE_NOT_REAL_3"; |
| 114 | +let db_password = "PLANTED_CANARY_VALUE_NOT_REAL_4"; |
| 115 | +EOF |
| 116 | +expect real 1 "5 planted secrets in ./src -> BLOCKS" RUST_TODAY=2026-07-21 |
| 117 | + |
| 118 | +# Count them: exit 1 only proves >=1 was found, not that all five were. |
| 119 | +# The `static AUTH_KEY` defect passed an exit-code assertion but failed a count. |
| 120 | +found=$( cd "$WORK/real" && RUST_TODAY=2026-07-21 bash "$SCAN" 2>&1 | grep -cE '^\./src/main\.rs' ) |
| 121 | +if [ "${found:-0}" -eq 5 ]; then |
| 122 | + echo " PASS all 5 planted secrets reported (not just the first)" |
| 123 | + pass=$((pass+1)) |
| 124 | +else |
| 125 | + echo " FAIL only ${found:-0}/5 planted secrets reported — the scanner has a blind spot" |
| 126 | + fail=$((fail+1)) |
| 127 | +fi |
| 128 | + |
| 129 | +# --------------------------------------------------------------------------- |
| 130 | +# 2. WORKSPACE LAYOUT MUST BE SCANNED (regression guard for the ./src-only bug) |
| 131 | +# --------------------------------------------------------------------------- |
| 132 | +mkdir -p "$WORK/ws"; echo '[workspace]' > "$WORK/ws/Cargo.toml" |
| 133 | +mk ws crates/core/src/lib.rs <<'EOF' |
| 134 | +pub const SERVICE_TOKEN: &str = "PLANTED_CANARY_VALUE_NOT_REAL_5"; |
| 135 | +EOF |
| 136 | +expect ws 1 "secret in crates/*/src -> BLOCKS after cutoff (no root ./src)" RUST_TODAY=2026-09-01 |
| 137 | +expect ws 0 "secret in crates/*/src -> advisory before cutoff" RUST_TODAY=2026-07-21 |
| 138 | +if ( cd "$WORK/ws" && RUST_TODAY=2026-07-21 bash "$SCAN" 2>&1 | grep -q 'NOT YET ENFORCED' ); then |
| 139 | + echo " PASS advisory run reports the finding and never claims a clean pass" |
| 140 | + pass=$((pass+1)) |
| 141 | +else |
| 142 | + echo " FAIL advisory run did not report the outstanding finding" |
| 143 | + fail=$((fail+1)) |
| 144 | +fi |
| 145 | + |
| 146 | +# --------------------------------------------------------------------------- |
| 147 | +# 3. CORRECT CODE MUST NOT BE FLAGGED. Flagging `env::var` fails the very |
| 148 | +# practice this job exists to enforce — a gate whose only route to green is |
| 149 | +# to stop reading from the environment is inverted. |
| 150 | +# --------------------------------------------------------------------------- |
| 151 | +mkdir -p "$WORK/clean"; echo '[package]' > "$WORK/clean/Cargo.toml" |
| 152 | +mk clean src/main.rs <<'EOF' |
| 153 | +use std::env; |
| 154 | +fn main() { |
| 155 | + let api_key = env::var("SOME_API_KEY").ok(); |
| 156 | + let password = std::env::var("DB_PASSWORD").unwrap_or_default(); |
| 157 | + const TOKEN_ENV: &str = option_env!("BUILD_TOKEN").unwrap_or(""); |
| 158 | + const MS_TOKEN_URL: &str = "https://login.microsoftonline.com/common/oauth2/v2.0/token"; |
| 159 | + let api_key_file = temp_dir.path().join("config.js"); |
| 160 | + let lookup = request["api_key"].as_str().unwrap_or(""); |
| 161 | + // const API_SECRET: &str = "documented-example-only"; |
| 162 | + let api_key2 = "fixture-value"; // scanner-allow: rust-secrets |
| 163 | +} |
| 164 | +EOF |
| 165 | +expect clean 0 "env::var / URL / lookup / path / comment / pragma -> ALLOWED" RUST_TODAY=2026-09-01 |
| 166 | + |
| 167 | +# --------------------------------------------------------------------------- |
| 168 | +# 4. EXEMPTIONS MUST NOT BE OVER-BROAD. A literal secret with an unrelated |
| 169 | +# `env::var` mention later on the same line must still trip. |
| 170 | +# --------------------------------------------------------------------------- |
| 171 | +mkdir -p "$WORK/trap"; echo '[package]' > "$WORK/trap/Cargo.toml" |
| 172 | +mk trap src/main.rs <<'EOF' |
| 173 | +const API_SECRET: &str = "PLANTED_CANARY_VALUE_NOT_REAL_6"; // fallback when env::var fails |
| 174 | +EOF |
| 175 | +expect trap 1 "literal secret + later env::var mention -> STILL BLOCKS" RUST_TODAY=2026-07-21 |
| 176 | + |
| 177 | +# --------------------------------------------------------------------------- |
| 178 | +# 5. THE CUTOFF MUST NOT BE SILENTLY DISARMABLE. An unparseable date would |
| 179 | +# select the warn branch forever, restoring a gate that cannot fail. |
| 180 | +# --------------------------------------------------------------------------- |
| 181 | +expect clean 1 "malformed cutoff -> REFUSES TO RUN" ENFORCE_RUST_WIDE_SCAN_FROM=soon |
| 182 | + |
| 183 | +# --------------------------------------------------------------------------- |
| 184 | +# 6. A repo with no Rust must skip cleanly rather than error. |
| 185 | +# --------------------------------------------------------------------------- |
| 186 | +mkdir -p "$WORK/norust"; echo 'hello' > "$WORK/norust/README.md" |
| 187 | +expect norust 0 "no Cargo.toml -> skips cleanly" RUST_TODAY=2026-07-21 |
| 188 | + |
| 189 | +echo |
| 190 | +echo " === $pass passed, $fail failed ===" |
| 191 | +if [ "$fail" -gt 0 ]; then |
| 192 | + echo "::error::Secret-scanner canary FAILED — the gate is not trustworthy. Do not merge." |
| 193 | + exit 1 |
| 194 | +fi |
| 195 | +echo "Secret-scanner canary passed: the gate can still fail." |
0 commit comments