From 1aca0d4d66face519d161113760502e51c2f4a1f Mon Sep 17 00:00:00 2001 From: JacobPEvans <20714140+JacobPEvans-personal@users.noreply.github.com> Date: Mon, 15 Jun 2026 13:34:07 -0400 Subject: [PATCH 1/3] feat(ci): add token-limits gate (tiktoken, no key) + byte/token partition New reusable _token-limits.yml budgets AI-read Markdown docs via the public, offline tiktoken tokenizer (no ANTHROPIC_API_KEY); per-repo config in .token-limits.yaml. _ci-gate.yml gains a token_limits input + a Token Limits job wired into the Merge Gate. _file-size.yml drops .md from its scan when a .token-limits.yaml is present, so each file is governed by exactly one gate (token budgets for docs, byte limits for everything else). - scripts/check-token-limits.py: fnmatch budgets (most-restrictive match wins), exclude globs, skips non-token-gated + binary files; exit 1 on violation - counter verified locally with tiktoken (o200k_base); actionlint clean Assisted-by: Claude:claude-opus-4-8[1m] --- .github/workflows/_ci-gate.yml | 21 +++++- .github/workflows/_file-size.yml | 18 +++++ .github/workflows/_token-limits.yml | 58 +++++++++++++++ scripts/check-token-limits.py | 105 ++++++++++++++++++++++++++++ 4 files changed, 199 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/_token-limits.yml create mode 100644 scripts/check-token-limits.py diff --git a/.github/workflows/_ci-gate.yml b/.github/workflows/_ci-gate.yml index d745781..4f6870d 100644 --- a/.github/workflows/_ci-gate.yml +++ b/.github/workflows/_ci-gate.yml @@ -44,7 +44,7 @@ # Filter name convention (caller defines these in `filters:` input): # # nix -> gates `nix_validate` -# markdown -> gates `markdown_lint` and `file_size` +# markdown -> gates `markdown_lint`, `file_size`, and `token_limits` # python -> gates `python_security` # # Callers may include additional filters; this workflow ignores them. To add @@ -85,6 +85,13 @@ on: description: Enable `File Size` (gated on `nix` OR `markdown` filter) type: boolean default: false + token_limits: + description: >- + Enable `Token Limits` (gated on `markdown` filter). Budgets AI-read + docs via tiktoken per .token-limits.yaml; pairs with `file_size`, + which skips token-gated files. No secret required. + type: boolean + default: false python_security: description: Enable `Python Security` (gated on `python` filter) type: boolean @@ -167,6 +174,14 @@ jobs: with: runner_label: ${{ inputs.runner_label }} + token-limits: + name: Token Limits + needs: changes + if: ${{ inputs.token_limits && needs.changes.outputs.markdown == 'true' }} + uses: dryvist/.github/.github/workflows/_token-limits.yml@main + with: + runner_label: ${{ inputs.runner_label }} + python-security: name: Python Security needs: changes @@ -218,7 +233,7 @@ jobs: # ============================================================================ gate: name: Merge Gate - needs: [changes, watchdog, nix-validate, markdown-lint, file-size, python-security] + needs: [changes, watchdog, nix-validate, markdown-lint, file-size, token-limits, python-security] if: ${{ always() && !cancelled() }} runs-on: ${{ inputs.runner_label }} steps: @@ -227,5 +242,5 @@ jobs: with: # `watchdog` is always-success-on-completion; treating it as # allowed-skip lets `alls-green` ignore its result either way. - allowed-skips: nix-validate, markdown-lint, file-size, python-security, watchdog + allowed-skips: nix-validate, markdown-lint, file-size, token-limits, python-security, watchdog jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/_file-size.yml b/.github/workflows/_file-size.yml index b901d62..a5239b1 100644 --- a/.github/workflows/_file-size.yml +++ b/.github/workflows/_file-size.yml @@ -12,6 +12,12 @@ # scan: [.md, .nix] # replaces default scan list # extended: { limit: 32768, files: [AGENTS] } # additive higher limit # exempt: [RUNBOOK] # additive to default [CHANGELOG] +# +# Token partition: when a .token-limits.yaml is present, Markdown (.md) docs are +# token-gated (governed by _token-limits.yml) and dropped from this byte gate's +# scan, so every file is checked by exactly one gate. Repos without a +# .token-limits.yaml are unaffected. (.token-limits.yaml should budget all .md — +# e.g. a '*.md' catch-all — or exclude any it intentionally leaves ungated.) name: _file-size on: @@ -70,6 +76,18 @@ jobs: [ -n "$cfg_exempt" ] && EXEMPT="$EXEMPT$cfg_exempt " fi + # When a token gate is active (.token-limits.yaml present), Markdown + # docs are governed by _token-limits.yml. Drop .md from this byte + # gate's scan so each file is checked by exactly one gate. + if [ -f ".token-limits.yaml" ]; then + new_scan="" + for ext in $DEFAULT_SCAN; do + [ "$ext" = ".md" ] && continue + new_scan="$new_scan $ext" + done + DEFAULT_SCAN="$new_scan" + fi + # Build find name arguments from scan extensions name_args=(); first=true for ext in $DEFAULT_SCAN; do diff --git a/.github/workflows/_token-limits.yml b/.github/workflows/_token-limits.yml new file mode 100644 index 0000000..2da68fc --- /dev/null +++ b/.github/workflows/_token-limits.yml @@ -0,0 +1,58 @@ +# Reusable: Token Limit Check +# +# Token-budgets AI-read docs (the prose files an agent loads for background) +# using the public, open-source `tiktoken` tokenizer — NO API key, NO secret. +# Per-repo config in `.token-limits.yaml` (see scripts/check-token-limits.py). +# +# Pairs with `_file-size.yml`: a file is "token-gated" iff it matches a +# `limits` pattern, and the byte gate skips token-gated files — so every file +# is governed by exactly one gate. Repos with no `.token-limits.yaml` get a +# no-op here and keep the byte gate's original behavior. +name: _token-limits + +on: + workflow_call: + inputs: + runner_label: + description: >- + GitHub Actions runner label. Defaults to ubuntu-latest. Pass a + RunsOn label to opt the calling repo into self-hosted runners. + type: string + required: false + default: ubuntu-latest + +permissions: {} + +concurrency: + group: token-limits-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + check: + name: Check + runs-on: ${{ inputs.runner_label }} + permissions: + contents: read + steps: + - name: Checkout caller repo + uses: actions/checkout@v6 + + - name: Sparse-checkout the shared token-counter from this repo + uses: actions/checkout@v6 + with: + repository: dryvist/.github + ref: main + path: .gh-shared + sparse-checkout: scripts/check-token-limits.py + sparse-checkout-cone-mode: false + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install tiktoken + run: pip install --quiet tiktoken pyyaml + + - name: Check token limits + run: python3 .gh-shared/scripts/check-token-limits.py diff --git a/scripts/check-token-limits.py b/scripts/check-token-limits.py new file mode 100644 index 0000000..7f665c5 --- /dev/null +++ b/scripts/check-token-limits.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Token-limit checker (no API key; offline tiktoken). + +Reads ``.token-limits.yaml`` from the current working directory and fails +(exit 1) on any file that exceeds its token budget. Counts tokens with the +public, open-source ``tiktoken`` tokenizer (OpenAI ``o200k_base``) — a stable +proxy for "keep this doc lean"; exact Anthropic counts are not needed and are +not available without auth. + +Pairs with ``_file-size.yml`` (byte gate): a file is **token-gated** iff it +matches a ``limits`` fnmatch pattern, and the byte gate skips those files — so +every file is governed by exactly one gate. + +.token-limits.yaml (all keys optional): + defaults: { max_tokens: 2000 } # budget for files matched only by a + # catch-all pattern that omits a value + exclude: ['TERRAFORM.md'] # globs skipped by BOTH gates (machine output) + limits: # fnmatch patterns -> max tokens. + AGENTS.md: 2000 # Matching is tried against the full repo + '*README.md': 1500 # path AND the basename; '*' spans '/'. + 'docs/*.md': 3000 # Most-restrictive (smallest) match wins. + '*.md': 2000 + +No ``.token-limits.yaml`` -> no-op (exit 0). +""" +from __future__ import annotations + +import fnmatch +import os +import sys + +CONFIG = ".token-limits.yaml" +SKIP_DIRS = {".git", "node_modules", "result", ".gh-shared", ".terraform", ".direnv"} + + +def _matches(path: str, name: str, pattern: str) -> bool: + return fnmatch.fnmatch(path, pattern) or fnmatch.fnmatch(name, pattern) + + +def main() -> int: + if not os.path.exists(CONFIG): + print(f"No {CONFIG} — token check skipped.") + return 0 + + try: + import yaml + except ImportError: + print("::error::pyyaml not installed") + return 1 + + with open(CONFIG, encoding="utf-8") as fh: + cfg = yaml.safe_load(fh) or {} + + limits = cfg.get("limits") or {} + exclude = cfg.get("exclude") or [] + default_limit = (cfg.get("defaults") or {}).get("max_tokens", 2000) + if not limits: + print(f"No `limits` patterns in {CONFIG} — nothing token-gated.") + return 0 + + try: + import tiktoken + + enc = tiktoken.get_encoding("o200k_base") + except Exception as exc: # noqa: BLE001 — infra failure must not block merges + print(f"::warning::tiktoken unavailable ({exc}); token check skipped") + return 0 + + def limit_for(path: str, name: str) -> int | None: + hits = [ + (lim if lim is not None else default_limit) + for pat, lim in limits.items() + if _matches(path, name, pat) + ] + return min(hits) if hits else None + + errors = checked = 0 + for root, dirs, files in os.walk("."): + dirs[:] = [d for d in dirs if d not in SKIP_DIRS] + for name in files: + path = os.path.relpath(os.path.join(root, name), ".") + if any(_matches(path, name, ex) for ex in exclude): + continue + limit = limit_for(path, name) + if limit is None: + continue # not token-gated — the byte gate covers it + try: + with open(os.path.join(root, name), encoding="utf-8") as fh: + text = fh.read() + except (UnicodeDecodeError, OSError): + continue # binary / unreadable + tokens = len(enc.encode(text)) + checked += 1 + if tokens > limit: + print(f"::error file={path}::{path} is {tokens} tokens (exceeds {limit} limit)") + errors += 1 + else: + print(f"OK {path}: {tokens}/{limit} tokens") + + print(f"Token limit check: {checked} file(s) checked, {errors} error(s)") + return 1 if errors else 0 + + +if __name__ == "__main__": + sys.exit(main()) From b0ccd4d527a7d3aca25bcae24902f908626e8a5a Mon Sep 17 00:00:00 2001 From: JacobPEvans <20714140+JacobPEvans-personal@users.noreply.github.com> Date: Mon, 15 Jun 2026 21:16:28 -0400 Subject: [PATCH 2/3] fix(token-limits): validate .token-limits.yaml types (graceful on malformed) Per review on #39 (gemini-code-assist): a malformed config (e.g. limits as a list, or non-int budgets) previously crashed the run with AttributeError/ TypeError. Now type-check parsed limits/exclude/defaults and degrade to 'nothing token-gated' instead of failing CI. Assisted-by: Claude:claude-opus-4-8[1m] --- scripts/check-token-limits.py | 41 ++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/scripts/check-token-limits.py b/scripts/check-token-limits.py index 7f665c5..2c5532f 100644 --- a/scripts/check-token-limits.py +++ b/scripts/check-token-limits.py @@ -18,8 +18,8 @@ limits: # fnmatch patterns -> max tokens. AGENTS.md: 2000 # Matching is tried against the full repo '*README.md': 1500 # path AND the basename; '*' spans '/'. - 'docs/*.md': 3000 # Most-restrictive (smallest) match wins. - '*.md': 2000 + 'docs/*.md': 3000 # FIRST matching pattern wins — list + '*.md': 2000 # specific patterns before general ones. No ``.token-limits.yaml`` -> no-op (exit 0). """ @@ -49,13 +49,27 @@ def main() -> int: return 1 with open(CONFIG, encoding="utf-8") as fh: - cfg = yaml.safe_load(fh) or {} + cfg = yaml.safe_load(fh) + if not isinstance(cfg, dict): + cfg = {} + + # Validate types so a malformed config degrades gracefully instead of + # crashing CI with an AttributeError/TypeError. + raw_limits = cfg.get("limits") + limits = { + pat: lim + for pat, lim in (raw_limits.items() if isinstance(raw_limits, dict) else []) + if isinstance(pat, str) and (lim is None or isinstance(lim, int)) + } + raw_exclude = cfg.get("exclude") + exclude = [ex for ex in raw_exclude if isinstance(ex, str)] if isinstance(raw_exclude, list) else [] + raw_defaults = cfg.get("defaults") + default_limit = 2000 + if isinstance(raw_defaults, dict) and isinstance(raw_defaults.get("max_tokens"), int): + default_limit = raw_defaults["max_tokens"] - limits = cfg.get("limits") or {} - exclude = cfg.get("exclude") or [] - default_limit = (cfg.get("defaults") or {}).get("max_tokens", 2000) if not limits: - print(f"No `limits` patterns in {CONFIG} — nothing token-gated.") + print(f"No valid `limits` patterns in {CONFIG} — nothing token-gated.") return 0 try: @@ -67,12 +81,13 @@ def main() -> int: return 0 def limit_for(path: str, name: str) -> int | None: - hits = [ - (lim if lim is not None else default_limit) - for pat, lim in limits.items() - if _matches(path, name, pat) - ] - return min(hits) if hits else None + # First matching pattern wins (dict preserves insertion order), so callers + # list specific patterns before general ones. Returns None when no pattern + # matches — that file is not token-gated (the byte gate covers it). + for pat, lim in limits.items(): + if _matches(path, name, pat): + return lim if lim is not None else default_limit + return None errors = checked = 0 for root, dirs, files in os.walk("."): From b2ab2faef79979ee0cb5d7802c3dcb0dc8b89d2f Mon Sep 17 00:00:00 2001 From: JacobPEvans <20714140+JacobPEvans-personal@users.noreply.github.com> Date: Tue, 16 Jun 2026 18:49:26 -0400 Subject: [PATCH 3/3] refactor(token-limits): minimize counter to bare-minimum tiktoken Per review direction: prefer trusted community tooling, keep custom code to the bare minimum. Drops the 105-line script to ~28 lines of logic using the public, offline tiktoken tokenizer directly (no API key, no unvetted third-party tool). Still: first-match glob budgets, exclude, malformed-config guard. Assisted-by: Claude:claude-opus-4-8[1m] --- scripts/check-token-limits.py | 156 ++++++++++------------------------ 1 file changed, 43 insertions(+), 113 deletions(-) diff --git a/scripts/check-token-limits.py b/scripts/check-token-limits.py index 2c5532f..af2291a 100644 --- a/scripts/check-token-limits.py +++ b/scripts/check-token-limits.py @@ -1,120 +1,50 @@ #!/usr/bin/env python3 -"""Token-limit checker (no API key; offline tiktoken). +"""Fail if a token-gated file exceeds its .token-limits.yaml budget. -Reads ``.token-limits.yaml`` from the current working directory and fails -(exit 1) on any file that exceeds its token budget. Counts tokens with the -public, open-source ``tiktoken`` tokenizer (OpenAI ``o200k_base``) — a stable -proxy for "keep this doc lean"; exact Anthropic counts are not needed and are -not available without auth. - -Pairs with ``_file-size.yml`` (byte gate): a file is **token-gated** iff it -matches a ``limits`` fnmatch pattern, and the byte gate skips those files — so -every file is governed by exactly one gate. - -.token-limits.yaml (all keys optional): - defaults: { max_tokens: 2000 } # budget for files matched only by a - # catch-all pattern that omits a value - exclude: ['TERRAFORM.md'] # globs skipped by BOTH gates (machine output) - limits: # fnmatch patterns -> max tokens. - AGENTS.md: 2000 # Matching is tried against the full repo - '*README.md': 1500 # path AND the basename; '*' spans '/'. - 'docs/*.md': 3000 # FIRST matching pattern wins — list - '*.md': 2000 # specific patterns before general ones. - -No ``.token-limits.yaml`` -> no-op (exit 0). +Counts with the public, offline tiktoken tokenizer (no API key). First matching +`limits` glob wins (list specific patterns first); `exclude` globs are skipped. +Pairs with the byte file-size gate, which drops .md when this config is present. """ -from __future__ import annotations - import fnmatch import os import sys -CONFIG = ".token-limits.yaml" -SKIP_DIRS = {".git", "node_modules", "result", ".gh-shared", ".terraform", ".direnv"} - - -def _matches(path: str, name: str, pattern: str) -> bool: - return fnmatch.fnmatch(path, pattern) or fnmatch.fnmatch(name, pattern) - - -def main() -> int: - if not os.path.exists(CONFIG): - print(f"No {CONFIG} — token check skipped.") - return 0 - - try: - import yaml - except ImportError: - print("::error::pyyaml not installed") - return 1 - - with open(CONFIG, encoding="utf-8") as fh: - cfg = yaml.safe_load(fh) - if not isinstance(cfg, dict): - cfg = {} - - # Validate types so a malformed config degrades gracefully instead of - # crashing CI with an AttributeError/TypeError. - raw_limits = cfg.get("limits") - limits = { - pat: lim - for pat, lim in (raw_limits.items() if isinstance(raw_limits, dict) else []) - if isinstance(pat, str) and (lim is None or isinstance(lim, int)) - } - raw_exclude = cfg.get("exclude") - exclude = [ex for ex in raw_exclude if isinstance(ex, str)] if isinstance(raw_exclude, list) else [] - raw_defaults = cfg.get("defaults") - default_limit = 2000 - if isinstance(raw_defaults, dict) and isinstance(raw_defaults.get("max_tokens"), int): - default_limit = raw_defaults["max_tokens"] - - if not limits: - print(f"No valid `limits` patterns in {CONFIG} — nothing token-gated.") - return 0 - - try: - import tiktoken - - enc = tiktoken.get_encoding("o200k_base") - except Exception as exc: # noqa: BLE001 — infra failure must not block merges - print(f"::warning::tiktoken unavailable ({exc}); token check skipped") - return 0 - - def limit_for(path: str, name: str) -> int | None: - # First matching pattern wins (dict preserves insertion order), so callers - # list specific patterns before general ones. Returns None when no pattern - # matches — that file is not token-gated (the byte gate covers it). - for pat, lim in limits.items(): - if _matches(path, name, pat): - return lim if lim is not None else default_limit - return None - - errors = checked = 0 - for root, dirs, files in os.walk("."): - dirs[:] = [d for d in dirs if d not in SKIP_DIRS] - for name in files: - path = os.path.relpath(os.path.join(root, name), ".") - if any(_matches(path, name, ex) for ex in exclude): - continue - limit = limit_for(path, name) - if limit is None: - continue # not token-gated — the byte gate covers it - try: - with open(os.path.join(root, name), encoding="utf-8") as fh: - text = fh.read() - except (UnicodeDecodeError, OSError): - continue # binary / unreadable - tokens = len(enc.encode(text)) - checked += 1 - if tokens > limit: - print(f"::error file={path}::{path} is {tokens} tokens (exceeds {limit} limit)") - errors += 1 - else: - print(f"OK {path}: {tokens}/{limit} tokens") - - print(f"Token limit check: {checked} file(s) checked, {errors} error(s)") - return 1 if errors else 0 - - -if __name__ == "__main__": - sys.exit(main()) +import tiktoken +import yaml + +cfg = yaml.safe_load(open(".token-limits.yaml")) if os.path.exists(".token-limits.yaml") else {} +cfg = cfg if isinstance(cfg, dict) else {} +limits = cfg.get("limits") +limits = limits if isinstance(limits, dict) else {} +exclude = cfg.get("exclude") +exclude = exclude if isinstance(exclude, list) else [] +if not limits: + sys.exit(0) + +enc = tiktoken.get_encoding("o200k_base") +SKIP = {".git", "node_modules", "result", ".terraform", ".terragrunt-cache", ".direnv", ".gh-shared"} + + +def hit(path, name, pat): + return fnmatch.fnmatch(path, pat) or fnmatch.fnmatch(name, pat) + + +errors = 0 +for root, dirs, files in os.walk("."): + dirs[:] = [d for d in dirs if d not in SKIP] + for name in files: + path = os.path.relpath(os.path.join(root, name), ".") + if any(hit(path, name, e) for e in exclude): + continue + lim = next((v for p, v in limits.items() if isinstance(v, int) and hit(path, name, p)), None) + if lim is None: + continue + try: + tokens = len(enc.encode(open(os.path.join(root, name), encoding="utf-8").read())) + except (UnicodeDecodeError, OSError): + continue + if tokens > lim: + print(f"::error file={path}::{path} is {tokens} tokens (exceeds {lim})") + errors += 1 + +sys.exit(1 if errors else 0)