examples/pdf_overlay: SHA-preserving PDF annotation overlay from profiling

Closes the extract->profile->annotate loop: bb_pdf extracts the box table,
DuckDB SQL imputes a domain per box, emitted as a standalone XFDF sidecar that
references the PDF via <f href> so the source SHA256 is invariant (no baked
copy). Coordinate flip bb_pdf top-left -> PDF bottom-left; validated visually
on IRS f1040sc (1682 boxes, pixel-accurate, source byte-identical).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: phrrngtn

examples/pdf_overlay/.gitignore

@@ -0,0 +1,5 @@
+# sample data + generated artifacts
+*.pdf
+*.xfdf
+*.boxes.json
+*.png

examples/pdf_overlay/README.md

@@ -0,0 +1,22 @@
+# pdf_overlay — context-sensitive annotations from profiling, SHA-preserving
+
+Closes the extract → profile → annotate loop on a real PDF: `bb_pdf` extracts the
+box table, DuckDB SQL imputes a domain per box, and the result is emitted as a
+standalone **XFDF sidecar** — the source PDF's bytes are never touched, so its
+**SHA256 is invariant** (baking annotations into a copy would mint a new SHA; we
+don't). Acrobat/Reader render the `.xfdf` over the PDF via its `<f href>`; viewers
+without XFDF support get the same findings as a draw-on-render overlay layer.
+
+```bash
+./fetch_sample.sh                                   # public-domain IRS Schedule C
+BBOXES_DUCKDB_EXT=/path/to/bboxes.duckdb_extension \
+  python overlay.py f1040sc.pdf                     # -> f1040sc.xfdf (source untouched)
+```
+
+Notes:
+- **Coordinates:** `bb_pdf` is top-left-origin (y down); XFDF is PDF bottom-left,
+  so rects flip against page height (`lly=H-(y+h)`, `ury=H-y`). Validated visually.
+- **Imputation:** the SQL `CASE` is a stand-in — replace it with a join against
+  blobfilters domain fingerprints (catalog/authority) for real domain matching,
+  and colour out-of-domain values (residue) as violators.
+- Deps: `pypdfium2`, `pillow`, and the `duckdb` CLI with the `bboxes` extension.

examples/pdf_overlay/fetch_sample.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Public-domain sample: IRS Schedule C (Form 1040). Born-digital, complex layout.
+set -euo pipefail
+curl -sL --max-time 40 "https://www.irs.gov/pub/irs-pdf/f1040sc.pdf" -o f1040sc.pdf
+echo "fetched f1040sc.pdf ($(wc -c < f1040sc.pdf) bytes)"

examples/pdf_overlay/overlay.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+"""Context-sensitive annotation overlay for a PDF, driven by blobboxes bb_pdf
+extraction + SQL domain imputation, emitted as a standalone XFDF *sidecar*.
+
+The source PDF is NEVER modified: annotations live in a separate .xfdf that
+references the PDF via <f href>, so the document's SHA256 is invariant. (Baking
+annotations into a copy would mint a new SHA — deliberately avoided.)
+
+Coordinate note: bb_pdf returns TOP-LEFT-origin boxes (x,y,w,h, y down).
+XFDF rects are PDF user space (bottom-left), so we flip against page height:
+  llx=x, urx=x+w, lly=H-(y+h), ury=H-y.
+"""
+import argparse, json, hashlib, html, os, subprocess, sys
+from collections import Counter
+import pypdfium2 as pdfium
+from PIL import ImageDraw
+
+COLOR = {"year":"#2563eb","ein":"#dc2626","currency":"#16a34a","number":"#0d9488",
+         "lineref":"#ea580c","integer":"#7c3aed","text":"#94a3b8"}
+
+# bb_pdf extract + SQL domain imputation (a stand-in for catalog/authority
+# matching via blobfilters; swap the CASE for a join against domain fingerprints).
+SQL = r"""
+LOAD '__EXT__';
+COPY (
+  SELECT page_id, x, y, w, h, text,
+    CASE
+      WHEN regexp_matches(text, '^\d{4}$')              THEN 'year'
+      WHEN regexp_matches(text, '^\d{2}-?\d{7}$')       THEN 'ein'
+      WHEN regexp_matches(text, '^\$[\d,]+(\.\d{2})?$') THEN 'currency'
+      WHEN regexp_matches(text, '^[\d,]+\.\d+$')        THEN 'number'
+      WHEN regexp_matches(text, '^\d{1,2}[a-z]?$')      THEN 'lineref'
+      WHEN regexp_matches(text, '^\d+$')                THEN 'integer'
+      ELSE 'text'
+    END AS domain
+  FROM bb_pdf('__PDF__')
+  WHERE text IS NOT NULL AND trim(text) <> ''
+) TO '__OUT__' (FORMAT json, ARRAY true);
+"""
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("pdf")
+    ap.add_argument("--ext", default=os.environ.get("BBOXES_DUCKDB_EXT"),
+                    help="path to bboxes.duckdb_extension (or set BBOXES_DUCKDB_EXT)")
+    ap.add_argument("--out", default=None, help="output .xfdf path")
+    a = ap.parse_args()
+    if not a.ext:
+        sys.exit("need --ext or BBOXES_DUCKDB_EXT (path to bboxes.duckdb_extension)")
+    out = a.out or os.path.splitext(a.pdf)[0] + ".xfdf"
+    boxes_json = os.path.splitext(a.pdf)[0] + ".boxes.json"
+
+    sha_before = hashlib.sha256(open(a.pdf, "rb").read()).hexdigest()
+
+    sql = SQL.replace("__EXT__", a.ext).replace("__PDF__", a.pdf).replace("__OUT__", boxes_json)
+    subprocess.run(["duckdb", "-unsigned", "-c", sql], check=True)
+    boxes = json.load(open(boxes_json))
+
+    pdf = pdfium.PdfDocument(a.pdf)
+    H = [pdf[i].get_size()[1] for i in range(len(pdf))]
+    ann = []
+    for b in boxes:
+        p, x, y, w, h, dom = b["page_id"], b["x"], b["y"], b["w"], b["h"], b["domain"]
+        c = COLOR.get(dom, "#94a3b8")
+        llx, urx, lly, ury = x, x + w, H[p] - (y + h), H[p] - y
+        ann.append(f'<square page="{p}" rect="{llx:.2f},{lly:.2f},{urx:.2f},{ury:.2f}" '
+                   f'color="{c}" interior-color="{c}" opacity="0.30" width="0.75" title="{dom}">'
+                   f'<contents>{html.escape(dom)}: {html.escape(b["text"][:40])}</contents></square>')
+    open(out, "w").write('<?xml version="1.0" encoding="UTF-8"?>\n'
+        '<xfdf xmlns="http://ns.adobe.com/xfdf/" xml:space="preserve">\n'
+        f'<f href="{os.path.basename(a.pdf)}"/>\n<annots>\n' + "\n".join(ann) + "\n</annots>\n</xfdf>\n")
+
+    sha_after = hashlib.sha256(open(a.pdf, "rb").read()).hexdigest()
+    assert sha_before == sha_after, "source PDF changed!"
+    print(f"{len(boxes)} annotations -> {out}")
+    print("domains:", dict(Counter(b["domain"] for b in boxes)))
+    print(f"source SHA256 {sha_before[:16]}… PRESERVED (byte-identical)")
+
+if __name__ == "__main__":
+    main()