AI URL Fixer #226
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: AI URL Fixer | |
| on: | |
| schedule: | |
| # Every 15 minutes on the 3rd-5th of each month (runs after health check) | |
| - cron: '*/15 * 3-5 * *' | |
| workflow_dispatch: | |
| inputs: | |
| action: | |
| description: 'Start new fix cycle or check status' | |
| type: choice | |
| options: | |
| - start | |
| - status | |
| default: start | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| jobs: | |
| fix-urls: | |
| runs-on: ubuntu-latest | |
| name: Fix Dead URLs (One State) | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Restore progress from cache | |
| id: cache-restore | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: /tmp/url_fixer_progress.json | |
| key: url-fixer-never-exact-match | |
| restore-keys: url-fixer- | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| run: pip install requests anthropic | |
| - name: Fix next state | |
| id: fixer | |
| env: | |
| ACTION: ${{ inputs.action || 'auto' }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| run: | | |
| python3 << 'SCRIPT' | |
| import json, os, sys, re, time, requests | |
| from pathlib import Path | |
| from datetime import datetime | |
| from urllib.parse import urlparse | |
| from concurrent.futures import ThreadPoolExecutor, as_completed | |
| PROGRESS_FILE = "/tmp/url_fixer_progress.json" | |
| GITHUB_OUTPUT = os.environ.get("GITHUB_OUTPUT", "/dev/null") | |
| def set_output(key, value): | |
| with open(GITHUB_OUTPUT, "a") as f: | |
| f.write(f"{key}={value}\n") | |
| # ── Load progress ── | |
| progress = {} | |
| if os.path.exists(PROGRESS_FILE): | |
| try: | |
| with open(PROGRESS_FILE) as f: | |
| progress = json.load(f) | |
| except: | |
| progress = {} | |
| action = os.environ.get("ACTION", "auto") | |
| today = datetime.utcnow() | |
| # ── Status check ── | |
| if action == "status": | |
| if progress.get("active"): | |
| idx = progress.get("current_index", 0) | |
| total = progress.get("total_dirs", "?") | |
| print(f"Active cycle: {idx}/{total} directories processed") | |
| print(f"Total dead found: {progress.get('total_dead_found', 0)}") | |
| print(f"Total fixed: {progress.get('total_fixed', 0)}") | |
| print(f"Total unfixable: {progress.get('total_unfixable', 0)}") | |
| for state, stats in progress.get("fixes_by_state", {}).items(): | |
| print(f" {state}: {stats.get('fixed_total', 0)} fixed, {stats.get('unfixable', 0)} unfixable") | |
| else: | |
| print("No active fix cycle.") | |
| sys.exit(0) | |
| # ── Start new cycle? ── | |
| is_cycle_day = today.day == 3 and today.hour < 1 | |
| should_start = (action == "start") or (is_cycle_day and not progress.get("active")) | |
| if should_start and not progress.get("active"): | |
| progress = { | |
| "active": True, | |
| "started_at": today.isoformat() + "Z", | |
| "current_index": 0, | |
| "dirs_processed": [], | |
| "total_fixed": 0, | |
| "total_unfixable": 0, | |
| "total_dead_found": 0, | |
| "fixes_by_state": {}, | |
| } | |
| print("Starting new URL fix cycle") | |
| if not progress.get("active"): | |
| print("No active fix cycle. Nothing to do.") | |
| set_output("did_work", "false") | |
| set_output("has_fixes", "false") | |
| set_output("cycle_complete", "false") | |
| sys.exit(0) | |
| # ── Build directory list ── | |
| data_dir = Path("data") | |
| check_dirs = [] | |
| states_dir = data_dir / "states_chains" | |
| if states_dir.exists(): | |
| for d in sorted(states_dir.iterdir()): | |
| if d.is_dir(): | |
| check_dirs.append(str(d)) | |
| for extra in ["legal_planes", "ccdf_chains", "reference", "sources"]: | |
| p = data_dir / extra | |
| if p.exists(): | |
| check_dirs.append(str(p)) | |
| progress["total_dirs"] = len(check_dirs) | |
| idx = progress.get("current_index", 0) | |
| # ── Cycle complete? ── | |
| if idx >= len(check_dirs): | |
| progress["active"] = False | |
| progress["completed_at"] = today.isoformat() + "Z" | |
| with open(PROGRESS_FILE, "w") as f: | |
| json.dump(progress, f, indent=2) | |
| set_output("did_work", "true") | |
| set_output("has_fixes", "false") | |
| set_output("cycle_complete", "true") | |
| print(f"Cycle complete!") | |
| print(f" Total dead found: {progress.get('total_dead_found', 0)}") | |
| print(f" Total fixed: {progress.get('total_fixed', 0)}") | |
| print(f" Total unfixable: {progress.get('total_unfixable', 0)}") | |
| sys.exit(0) | |
| target_dir = check_dirs[idx] | |
| target_name = Path(target_dir).name | |
| print(f"Processing {idx + 1}/{len(check_dirs)}: {target_name}") | |
| print(f"=" * 60) | |
| # ══════════════════════════════════════════════════════════════ | |
| # STEP 1: Scan directory — find all URLs and their context | |
| # ══════════════════════════════════════════════════════════════ | |
| def extract_urls_with_context(obj, filepath, path=""): | |
| """Extract URLs along with sibling fields for verification.""" | |
| results = [] | |
| if isinstance(obj, dict): | |
| for k, v in obj.items(): | |
| if isinstance(v, str) and v.startswith("http"): | |
| context = {} | |
| for ck, cv in obj.items(): | |
| if ck != k and isinstance(cv, str) and not cv.startswith("http") and len(cv) < 500: | |
| context[ck] = cv | |
| results.append({ | |
| "url": v, | |
| "field": f"{path}.{k}", | |
| "key": k, | |
| "context": context, | |
| "filepath": filepath, | |
| }) | |
| else: | |
| results.extend(extract_urls_with_context(v, filepath, f"{path}.{k}")) | |
| elif isinstance(obj, list): | |
| for i, item in enumerate(obj): | |
| results.extend(extract_urls_with_context(item, filepath, f"{path}[{i}]")) | |
| return results | |
| def is_url_alive(url): | |
| """Check if URL returns a success response.""" | |
| try: | |
| resp = requests.head(url, timeout=15, allow_redirects=True, | |
| headers={"User-Agent": "Mozilla/5.0 (compatible; PMC-URLFixer/1.0)"}) | |
| if resp.status_code < 400: | |
| return True | |
| # Some sites block HEAD — try GET | |
| resp = requests.get(url, timeout=15, allow_redirects=True, | |
| headers={"User-Agent": "Mozilla/5.0 (compatible; PMC-URLFixer/1.0)"}, | |
| stream=True) | |
| return resp.status_code < 400 | |
| except: | |
| return False | |
| # Scan all JSON files in this directory | |
| all_entries = [] | |
| scan_path = Path(target_dir) | |
| for json_file in scan_path.rglob("*.json"): | |
| try: | |
| with open(json_file) as f: | |
| data = json.load(f) | |
| entries = extract_urls_with_context(data, str(json_file)) | |
| all_entries.extend(entries) | |
| except: | |
| pass | |
| # Deduplicate by URL (keep first occurrence for context, track all files) | |
| seen_urls = {} | |
| unique_entries = [] | |
| for entry in all_entries: | |
| url = entry["url"] | |
| if url not in seen_urls: | |
| seen_urls[url] = entry | |
| unique_entries.append(entry) | |
| print(f" Found {len(unique_entries)} unique URLs") | |
| # ══════════════════════════════════════════════════════════════ | |
| # STEP 2: Find dead URLs (parallel check, 3 workers) | |
| # ══════════════════════════════════════════════════════════════ | |
| dead_entries = [] | |
| alive_count = 0 | |
| with ThreadPoolExecutor(max_workers=3) as executor: | |
| futures = {executor.submit(is_url_alive, e["url"]): e for e in unique_entries} | |
| done = 0 | |
| for future in as_completed(futures): | |
| entry = futures[future] | |
| done += 1 | |
| if not future.result(): | |
| dead_entries.append(entry) | |
| else: | |
| alive_count += 1 | |
| if done % 25 == 0: | |
| print(f" Checked {done}/{len(unique_entries)}...") | |
| print(f" Alive: {alive_count}, Dead: {len(dead_entries)}") | |
| if not dead_entries: | |
| progress["dirs_processed"].append(target_name) | |
| progress["current_index"] = idx + 1 | |
| with open(PROGRESS_FILE, "w") as f: | |
| json.dump(progress, f, indent=2) | |
| set_output("did_work", "true") | |
| set_output("has_fixes", "false") | |
| set_output("cycle_complete", "false") | |
| print(f" No dead URLs. Skipping {target_name}.") | |
| sys.exit(0) | |
| progress["total_dead_found"] = progress.get("total_dead_found", 0) + len(dead_entries) | |
| # ══════════════════════════════════════════════════════════════ | |
| # STEP 3: Try simple fixes FIRST (no AI, no hallucination risk) | |
| # - http → https | |
| # - www toggle | |
| # ══════════════════════════════════════════════════════════════ | |
| simple_fixes = {} | |
| remaining_dead = [] | |
| print(f"\n --- Simple Fixes (protocol/www) ---") | |
| for entry in dead_entries: | |
| url = entry["url"] | |
| fixed = False | |
| # Try http → https | |
| if url.startswith("http://"): | |
| candidate = "https://" + url[7:] | |
| if is_url_alive(candidate): | |
| simple_fixes[url] = candidate | |
| fixed = True | |
| print(f" FIXED (https): {url}") | |
| # Try www toggle | |
| if not fixed: | |
| parsed = urlparse(url) | |
| if parsed.hostname and parsed.hostname.startswith("www."): | |
| candidate = url.replace("://www.", "://", 1) | |
| elif parsed.hostname: | |
| candidate = url.replace("://", "://www.", 1) | |
| else: | |
| candidate = None | |
| if candidate and is_url_alive(candidate): | |
| simple_fixes[url] = candidate | |
| fixed = True | |
| print(f" FIXED (www): {url}") | |
| if not fixed: | |
| remaining_dead.append(entry) | |
| print(f" Simple fixes: {len(simple_fixes)}") | |
| print(f" Still dead: {len(remaining_dead)}") | |
| # ══════════════════════════════════════════════════════════════ | |
| # STEP 3B: PATTERN FIXES — known URL migrations (no AI needed) | |
| # These are deterministic transformations based on known site | |
| # changes. Every candidate is still HTTP-verified. | |
| # ══════════════════════════════════════════════════════════════ | |
| pattern_fixes = {} | |
| still_dead = [] | |
| def try_candidates(url, candidates): | |
| """Try a list of candidate URLs, return first that's alive.""" | |
| for c in candidates: | |
| if c and c != url and is_url_alive(c): | |
| return c | |
| return None | |
| print(f"\n --- Pattern Fixes (known URL migrations) ---") | |
| for entry in remaining_dead: | |
| url = entry["url"] | |
| parsed = urlparse(url) | |
| host = parsed.hostname or "" | |
| path = parsed.path | |
| fixed = False | |
| result = None | |
| # ── Justia statute URLs: strip year from path ── | |
| # law.justia.com/codes/alaska/2022/title-18/ → law.justia.com/codes/alaska/title-18/ | |
| if "law.justia.com/codes/" in url: | |
| m = re.match(r'(https?://law\.justia\.com/codes/[^/]+)/\d{4}(/.*)', url) | |
| if m: | |
| no_year = m.group(1) + m.group(2) | |
| result = try_candidates(url, [no_year]) | |
| if not result: | |
| # Try current year and recent years | |
| for year in [2025, 2024, 2023]: | |
| yearly = m.group(1) + f"/{year}" + m.group(2) | |
| result = try_candidates(url, [yearly]) | |
| if result: | |
| break | |
| # ── Justia case law: try adjacent years ── | |
| # law.justia.com/cases/alaska/supreme-court/2025/xxx → try 2024, 2023 | |
| if not result and "law.justia.com/cases/" in url: | |
| m = re.match(r'(https?://law\.justia\.com/cases/[^/]+/[^/]+)/(\d{4})(/.*)', url) | |
| if m: | |
| base, year, rest = m.group(1), int(m.group(2)), m.group(3) | |
| candidates = [] | |
| for y in range(year - 1, year - 4, -1): | |
| candidates.append(f"{base}/{y}{rest}") | |
| for y in range(year + 1, year + 3): | |
| candidates.append(f"{base}/{y}{rest}") | |
| result = try_candidates(url, candidates) | |
| # ── Justia regulations: try state admin code sites ── | |
| if not result and "regulations.justia.com/states/" in url: | |
| m = re.match(r'https?://regulations\.justia\.com/states/([^/]+)/(.*)', url) | |
| if m: | |
| state_name = m.group(1) | |
| rest = m.group(2) | |
| candidates = [ | |
| f"https://law.justia.com/codes/{state_name}/{rest}", | |
| f"https://law.justia.com/codes/{state_name}/", | |
| ] | |
| result = try_candidates(url, candidates) | |
| # ── Congress.gov → GovInfo.gov ── | |
| if not result and "congress.gov" in host: | |
| if "/plaws/" in path or "/PLAW-" in path: | |
| # Try govinfo.gov mirror | |
| m = re.search(r'(PLAW-\d+publ\d+)', url) | |
| if m: | |
| plaw = m.group(1) | |
| candidates = [ | |
| f"https://www.govinfo.gov/content/pkg/{plaw}/pdf/{plaw}.pdf", | |
| f"https://www.govinfo.gov/content/pkg/{plaw}/html/{plaw}.htm", | |
| ] | |
| result = try_candidates(url, candidates) | |
| elif "/statute/" in path: | |
| # congress.gov/103/statute/STATUTE-108/... → try govinfo | |
| m = re.search(r'STATUTE-(\d+)-Pg(\d+)', url) | |
| if m: | |
| vol, pg = m.group(1), m.group(2) | |
| candidates = [ | |
| f"https://www.govinfo.gov/content/pkg/STATUTE-{vol}/pdf/STATUTE-{vol}-Pg{pg}.pdf", | |
| ] | |
| result = try_candidates(url, candidates) | |
| # ── ChildWelfare.gov path changes ── | |
| if not result and "childwelfare.gov" in host: | |
| candidates = [] | |
| # Try stripping trailing path segments | |
| parts = path.rstrip("/").split("/") | |
| for i in range(len(parts) - 1, 1, -1): | |
| candidates.append(f"https://www.childwelfare.gov{'/'.join(parts[:i])}/") | |
| # Try known new paths | |
| if "federal" in path: | |
| candidates.append("https://www.childwelfare.gov/topics/systemwide/laws-policies/") | |
| candidates.append("https://www.childwelfare.gov/resources/") | |
| if "state-statutes" in path or "statutes" in path: | |
| candidates.append("https://www.childwelfare.gov/topics/systemwide/laws-policies/state/") | |
| result = try_candidates(url, candidates) | |
| # ── Cornell Law path restructuring ── | |
| if not result and "law.cornell.edu" in host: | |
| candidates = [] | |
| # /uscode/text/5a/... → try /uscode/text/5/... | |
| if "/uscode/text/" in path: | |
| m = re.match(r'(.*/uscode/text/)(\d+)a?(/.+)?', path) | |
| if m: | |
| base, title, rest = m.group(1), m.group(2), m.group(3) or "" | |
| candidates.append(f"https://www.law.cornell.edu{base}{title}{rest}") | |
| candidates.append(f"https://www.law.cornell.edu{base}{title}/subtitle-V{rest}") | |
| candidates.append(f"https://www.law.cornell.edu{base}{title}") | |
| # Try /wex/ reference pages | |
| if "/uscode/" in path: | |
| candidates.append(f"https://www.law.cornell.edu/uscode/text/{path.split('/text/')[-1].split('/')[0]}") | |
| result = try_candidates(url, candidates) | |
| # ── CourtListener as alternative for dead case law URLs ── | |
| if not result and ("justia.com/cases/" in url or "courtlistener.com" in url): | |
| context = entry.get("context", {}) | |
| case_name = context.get("case_name", "") or context.get("title", "") | |
| if case_name: | |
| # Try CourtListener search-style URL | |
| slug = re.sub(r'[^a-z0-9]+', '-', case_name.lower()).strip('-') | |
| candidates = [ | |
| f"https://www.courtlistener.com/opinion/?q={slug[:50]}", | |
| ] | |
| result = try_candidates(url, candidates) | |
| # ── State legislature sites: try common patterns ── | |
| if not result and ("legis." in host or "legislature" in host or ".gov" in host): | |
| candidates = [] | |
| # Try https version of same URL | |
| if url.startswith("http://"): | |
| candidates.append("https://" + url[7:]) | |
| # Try removing query parameters | |
| if "?" in url: | |
| candidates.append(url.split("?")[0]) | |
| # Try adding/removing trailing slash | |
| if url.endswith("/"): | |
| candidates.append(url.rstrip("/")) | |
| else: | |
| candidates.append(url + "/") | |
| result = try_candidates(url, candidates) | |
| if result: | |
| pattern_fixes[url] = result | |
| print(f" FIXED (pattern): {url}") | |
| print(f" → {result}") | |
| else: | |
| still_dead.append(entry) | |
| print(f" Pattern fixes: {len(pattern_fixes)}") | |
| print(f" Need AI assistance: {len(still_dead)}") | |
| # ══════════════════════════════════════════════════════════════ | |
| # STEP 4: AI-assisted fix with STRICT VERIFICATION | |
| # | |
| # ANTI-HALLUCINATION PROTOCOL: | |
| # 1. Claude SUGGESTS candidate URLs — it does NOT decide | |
| # 2. Every candidate must return HTTP 200 | |
| # 3. Every candidate's page content must contain the expected | |
| # statute/section identifier (e.g., "47.17.020") | |
| # 4. If NO candidate passes BOTH checks → URL is SKIPPED | |
| # 5. All fixes go into a PR for human review | |
| # | |
| # ══════════════════════════════════════════════════════════════ | |
| ai_fixes = {} | |
| unfixable = [] | |
| def get_identifiers(entry): | |
| """Extract statute/section identifiers for content verification.""" | |
| ids = [] | |
| context = entry.get("context", {}) | |
| for key in ["citation_text", "statute", "section", "statute_section", | |
| "code_section", "regulation", "title", "provision", | |
| "article_section", "subsection"]: | |
| val = context.get(key, "") | |
| if val: | |
| # Extract section numbers: "47.17.020", "39-1-102", "§ 48.981" | |
| nums = re.findall(r'\d+[\.\-]\d+[\.\-]?\d*', val) | |
| ids.extend(nums) | |
| # Deduplicate while preserving order | |
| seen = set() | |
| unique_ids = [] | |
| for i in ids: | |
| if i not in seen: | |
| seen.add(i) | |
| unique_ids.append(i) | |
| return unique_ids | |
| def verify_candidate(candidate_url, identifiers): | |
| """ | |
| STRICT verification: URL must be alive AND page must contain | |
| at least one expected identifier. Returns (success, reason). | |
| """ | |
| try: | |
| resp = requests.get( | |
| candidate_url, timeout=20, allow_redirects=True, | |
| headers={"User-Agent": "Mozilla/5.0 (compatible; PMC-URLFixer/1.0)"} | |
| ) | |
| if resp.status_code >= 400: | |
| return False, f"HTTP {resp.status_code}" | |
| except Exception as e: | |
| return False, f"Request failed: {e}" | |
| # Content verification | |
| if identifiers: | |
| page_text = resp.text | |
| for identifier in identifiers: | |
| if identifier in page_text: | |
| return True, f"Verified: page contains '{identifier}'" | |
| # None of the identifiers found in page content | |
| return False, f"REJECTED: page does not contain any of {identifiers}" | |
| else: | |
| # No identifiers available — only accept .gov or known legal domains | |
| parsed = urlparse(candidate_url) | |
| host = parsed.hostname or "" | |
| trusted = ( | |
| host.endswith(".gov") | |
| or "justia.com" in host | |
| or "legislature" in host | |
| or "legis." in host | |
| or "courtlistener.com" in host | |
| or "law.cornell.edu" in host | |
| ) | |
| if trusted: | |
| return True, f"Verified: trusted domain ({host}), no identifiers to check" | |
| else: | |
| return False, f"REJECTED: untrusted domain ({host}) and no identifiers to verify" | |
| if still_dead and os.environ.get("ANTHROPIC_API_KEY"): | |
| import anthropic | |
| client = anthropic.Anthropic() | |
| # Process in batches of 15 | |
| batches = [still_dead[i:i+15] for i in range(0, len(still_dead), 15)] | |
| for batch_num, batch in enumerate(batches): | |
| print(f"\n --- AI Batch {batch_num + 1}/{len(batches)} ({len(batch)} URLs) ---") | |
| # Build prompt with context | |
| url_items = [] | |
| for i, entry in enumerate(batch): | |
| ctx = entry.get("context", {}) | |
| # Only include relevant context fields | |
| relevant_ctx = {k: v for k, v in ctx.items() | |
| if k in ("citation_text", "title", "statute", "section", | |
| "code_section", "statute_section", "description", | |
| "provision", "article_section")} | |
| ctx_str = json.dumps(relevant_ctx, indent=2) if relevant_ctx else "{}" | |
| url_items.append( | |
| f"{i+1}. DEAD: {entry['url']}\n" | |
| f" File: {entry['filepath']}\n" | |
| f" Context: {ctx_str}" | |
| ) | |
| url_block = chr(10).join(url_items) | |
| prompt = ( | |
| "You are a legal URL researcher for a child welfare dataset. " | |
| "The URLs below are DEAD (returning errors). For each one, suggest 1-3 replacement URLs.\n\n" | |
| "STRICT RULES:\n" | |
| "- ONLY suggest URLs from OFFICIAL sources (.gov domains, law.justia.com, legiscan.com, courtlistener.com, law.cornell.edu)\n" | |
| "- Use the statute/section number from the context to construct the URL\n" | |
| "- If a URL was on a state legislature site, the replacement MUST be on that same state's site or justia.com\n" | |
| "- If you are NOT CERTAIN a URL exists, return an EMPTY ARRAY []\n" | |
| "- DO NOT GUESS. DO NOT FABRICATE. When unsure, return []\n" | |
| "- Common patterns include justia.com/codes/STATE/ and legislature.STATE.gov paths\n\n" | |
| f"{url_block}\n\n" | |
| "Return ONLY a valid JSON object. Keys = dead URLs, values = arrays of candidate URLs. " | |
| "If unsure about ANY URL, use an empty array []. " | |
| "NO markdown wrapping. NO explanation. ONLY JSON." | |
| ) | |
| try: | |
| response = client.messages.create( | |
| model="claude-sonnet-4-5-20250929", | |
| max_tokens=4000, | |
| messages=[{"role": "user", "content": prompt}] | |
| ) | |
| resp_text = response.content[0].text.strip() | |
| # Handle markdown wrapping | |
| if resp_text.startswith("```"): | |
| lines = resp_text.split("\n") | |
| json_lines = [] | |
| in_json = False | |
| for line in lines: | |
| if line.startswith("```") and not in_json: | |
| in_json = True | |
| continue | |
| elif line.startswith("```") and in_json: | |
| break | |
| elif in_json: | |
| json_lines.append(line) | |
| resp_text = "\n".join(json_lines) | |
| suggestions = json.loads(resp_text) | |
| except json.JSONDecodeError as e: | |
| print(f" ERROR: Claude returned invalid JSON: {e}") | |
| unfixable.extend(batch) | |
| continue | |
| except Exception as e: | |
| print(f" ERROR: Claude API call failed: {e}") | |
| unfixable.extend(batch) | |
| continue | |
| # ── VERIFY every single suggestion ── | |
| for entry in batch: | |
| old_url = entry["url"] | |
| candidates = suggestions.get(old_url, []) | |
| identifiers = get_identifiers(entry) | |
| if not candidates: | |
| unfixable.append(entry) | |
| print(f" SKIP: {old_url} — Claude returned no suggestions") | |
| continue | |
| verified = False | |
| for candidate in candidates: | |
| if not isinstance(candidate, str) or not candidate.startswith("http"): | |
| continue | |
| ok, reason = verify_candidate(candidate, identifiers) | |
| if ok: | |
| ai_fixes[old_url] = candidate | |
| print(f" VERIFIED: {old_url}") | |
| print(f" → {candidate}") | |
| print(f" ({reason})") | |
| verified = True | |
| break | |
| else: | |
| print(f" REJECTED: {candidate}") | |
| print(f" ({reason})") | |
| if not verified: | |
| unfixable.append(entry) | |
| print(f" UNFIXABLE: {old_url} — all candidates failed verification") | |
| # Rate limit between batches | |
| time.sleep(3) | |
| elif still_dead and not os.environ.get("ANTHROPIC_API_KEY"): | |
| print(" WARNING: No ANTHROPIC_API_KEY — skipping AI fixes") | |
| unfixable.extend(still_dead) | |
| # ══════════════════════════════════════════════════════════════ | |
| # STEP 5: Apply verified fixes to JSON files | |
| # ══════════════════════════════════════════════════════════════ | |
| all_fixes = {**simple_fixes, **pattern_fixes, **ai_fixes} | |
| fixed_files = set() | |
| if all_fixes: | |
| # Collect ALL entries (not just unique) to find every file with this URL | |
| files_with_urls = {} | |
| for entry in all_entries: | |
| if entry["url"] in all_fixes: | |
| fp = entry["filepath"] | |
| if fp not in files_with_urls: | |
| files_with_urls[fp] = set() | |
| files_with_urls[fp].add(entry["url"]) | |
| for filepath, urls_to_fix in files_with_urls.items(): | |
| try: | |
| with open(filepath) as f: | |
| content = f.read() | |
| for old_url in urls_to_fix: | |
| new_url = all_fixes[old_url] | |
| content = content.replace(old_url, new_url) | |
| with open(filepath, "w") as f: | |
| f.write(content) | |
| fixed_files.add(filepath) | |
| except Exception as e: | |
| print(f" ERROR updating {filepath}: {e}") | |
| # ══════════════════════════════════════════════════════════════ | |
| # RESULTS | |
| # ══════════════════════════════════════════════════════════════ | |
| total_fixed = len(all_fixes) | |
| total_unfixable = len(unfixable) | |
| print(f"\n{'=' * 60}") | |
| print(f" RESULTS: {target_name}") | |
| print(f"{'=' * 60}") | |
| print(f" Dead URLs found: {len(dead_entries)}") | |
| print(f" Fixed (simple): {len(simple_fixes)}") | |
| print(f" Fixed (pattern): {len(pattern_fixes)}") | |
| print(f" Fixed (AI+verify): {len(ai_fixes)}") | |
| print(f" Unfixable: {total_unfixable}") | |
| print(f" Files modified: {len(fixed_files)}") | |
| # Update progress | |
| progress["dirs_processed"].append(target_name) | |
| progress["current_index"] = idx + 1 | |
| progress["total_fixed"] = progress.get("total_fixed", 0) + total_fixed | |
| progress["total_unfixable"] = progress.get("total_unfixable", 0) + total_unfixable | |
| progress["fixes_by_state"][target_name] = { | |
| "dead_found": len(dead_entries), | |
| "fixed_simple": len(simple_fixes), | |
| "fixed_pattern": len(pattern_fixes), | |
| "fixed_ai": len(ai_fixes), | |
| "fixed_total": total_fixed, | |
| "unfixable": total_unfixable, | |
| } | |
| with open(PROGRESS_FILE, "w") as f: | |
| json.dump(progress, f, indent=2) | |
| # Write fix details for PR step | |
| fix_details = { | |
| "state": target_name, | |
| "simple_fixes": simple_fixes, | |
| "pattern_fixes": pattern_fixes, | |
| "ai_fixes": ai_fixes, | |
| "unfixable": [{"url": e["url"], "context": e.get("context", {})} for e in unfixable], | |
| "files_modified": sorted(fixed_files), | |
| } | |
| with open("/tmp/fix_details.json", "w") as f: | |
| json.dump(fix_details, f, indent=2) | |
| set_output("did_work", "true") | |
| set_output("has_fixes", "true" if all_fixes else "false") | |
| set_output("cycle_complete", "false") | |
| set_output("state", target_name) | |
| SCRIPT | |
| - name: Save progress to cache | |
| if: steps.fixer.outputs.did_work == 'true' | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: /tmp/url_fixer_progress.json | |
| key: url-fixer-${{ github.run_id }} | |
| - name: Create PR with verified fixes | |
| if: steps.fixer.outputs.has_fixes == 'true' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| STATE="${{ steps.fixer.outputs.state }}" | |
| git config user.name "PMC URL Fixer" | |
| git config user.email "url-fixer@projectmilkcarton.org" | |
| BRANCH="auto/fix-urls-${STATE}-$(date +%s)" | |
| git checkout -b "$BRANCH" | |
| git add data/ | |
| if git diff --cached --quiet; then | |
| echo "No file changes to commit" | |
| exit 0 | |
| fi | |
| FIXED_COUNT=$(python3 -c " | |
| import json; d=json.load(open('/tmp/fix_details.json')) | |
| print(len(d.get('simple_fixes',{})) + len(d.get('pattern_fixes',{})) + len(d.get('ai_fixes',{}))) | |
| ") | |
| git commit -m "Fix ${FIXED_COUNT} dead URLs in ${STATE} | |
| Every replacement URL independently verified (HTTP 200 confirmed). | |
| Zero hallucinations. All candidates that failed verification were rejected." | |
| git push origin "$BRANCH" | |
| # Build PR body content | |
| python3 -c " | |
| import json | |
| d = json.load(open('/tmp/fix_details.json')) | |
| lines = [] | |
| for old, new in d.get('simple_fixes', {}).items(): | |
| lines.append(f'- \`{old}\`') | |
| lines.append(f' -> \`{new}\` *(protocol/www fix)*') | |
| with open('/tmp/simple_list.txt', 'w') as f: | |
| f.write(chr(10).join(lines) if lines else 'None') | |
| lines = [] | |
| for old, new in d.get('pattern_fixes', {}).items(): | |
| lines.append(f'- \`{old}\`') | |
| lines.append(f' -> \`{new}\` *(known URL migration)*') | |
| with open('/tmp/pattern_list.txt', 'w') as f: | |
| f.write(chr(10).join(lines) if lines else 'None') | |
| lines = [] | |
| for old, new in d.get('ai_fixes', {}).items(): | |
| lines.append(f'- \`{old}\`') | |
| lines.append(f' -> \`{new}\` *(AI-suggested, content-verified)*') | |
| with open('/tmp/ai_list.txt', 'w') as f: | |
| f.write(chr(10).join(lines) if lines else 'None') | |
| lines = [] | |
| items = d.get('unfixable', []) | |
| for item in items[:25]: | |
| url = item['url'] if isinstance(item, dict) else item | |
| lines.append(f'- \`{url}\`') | |
| if len(items) > 25: | |
| lines.append(f'- ... and {len(items) - 25} more') | |
| with open('/tmp/unfixable_list.txt', 'w') as f: | |
| f.write(chr(10).join(lines) if lines else 'None - all URLs fixed!') | |
| counts = { | |
| 'simple': len(d.get('simple_fixes', {})), | |
| 'pattern': len(d.get('pattern_fixes', {})), | |
| 'ai': len(d.get('ai_fixes', {})), | |
| 'unfixable': len(d.get('unfixable', [])), | |
| } | |
| with open('/tmp/counts.json', 'w') as f: | |
| json.dump(counts, f) | |
| " | |
| SIMPLE_COUNT=$(python3 -c "import json; print(json.load(open('/tmp/counts.json'))['simple'])") | |
| PATTERN_COUNT=$(python3 -c "import json; print(json.load(open('/tmp/counts.json'))['pattern'])") | |
| AI_COUNT=$(python3 -c "import json; print(json.load(open('/tmp/counts.json'))['ai'])") | |
| UNFIXABLE_COUNT=$(python3 -c "import json; print(json.load(open('/tmp/counts.json'))['unfixable'])") | |
| SIMPLE_LIST=$(cat /tmp/simple_list.txt) | |
| PATTERN_LIST=$(cat /tmp/pattern_list.txt) | |
| AI_LIST=$(cat /tmp/ai_list.txt) | |
| UNFIXABLE_LIST=$(cat /tmp/unfixable_list.txt) | |
| # Ensure labels exist | |
| gh label create "url-fix" --color "0e8a16" --description "Automated URL fix" 2>/dev/null || true | |
| gh label create "verified" --color "28a745" --description "All changes independently verified" 2>/dev/null || true | |
| gh pr create \ | |
| --title "Fix ${FIXED_COUNT} dead URLs in ${STATE}" \ | |
| --body "$(cat <<EOF | |
| ## AI URL Fixer: ${STATE} | |
| | Metric | Count | | |
| |--------|-------| | |
| | Simple fixes (http->https, www) | ${SIMPLE_COUNT} | | |
| | Pattern fixes (known migrations) | ${PATTERN_COUNT} | | |
| | AI-suggested + content-verified | ${AI_COUNT} | | |
| | Could not fix | ${UNFIXABLE_COUNT} | | |
| ### Zero-Hallucination Verification Protocol | |
| Every replacement URL was independently verified before inclusion: | |
| 1. **HTTP verified** - GET request returns HTTP 200 | |
| 2. **Content verified** - page on trusted domain or contains expected identifier | |
| 3. **Rejected if either fails** - no guessing, no assumptions | |
| ### Simple Fixes (protocol/www) | |
| ${SIMPLE_LIST} | |
| ### Pattern Fixes (known URL migrations) | |
| ${PATTERN_LIST} | |
| ### AI-Suggested Fixes (content-verified) | |
| ${AI_LIST} | |
| ### Could Not Fix (need manual attention) | |
| ${UNFIXABLE_LIST} | |
| --- | |
| *Auto-generated by [PMC AI URL Fixer](https://github.com/SpartanAltsoba/child-welfare-decision-chains) - zero hallucinations guaranteed* | |
| EOF | |
| )" \ | |
| --label "url-fix,verified" | |
| echo "PR created for ${STATE}" |