cleanup_dashes.py

#!/usr/bin/env python3
"""
Cleanup script: remove em/en dashes and AI patterns from all draft articles.
Handles paired dashes (parenthetical), definition dashes, and single dashes.
Preserves code blocks and table formatting.
"""
import re, os, glob

DRAFTS_DIR = os.path.join(os.path.dirname(__file__), "content", "drafts")

AI_PATTERNS = {
    r"\bfundamentally\b": {
        "fundamentally different approach": "structurally different approach",
        "fundamentally different": "structurally different",
        "fundamentally": "structurally",
        "changes fundamentally": "changes completely",
        "is fundamentally different": "differs structurally",
    },
    r"\bessentially\b": {
        "is essentially": "is effectively",
        "essentially a": "effectively a",
        "essentially your": "effectively your",
    },
    r"\blandscape\b": {
        "regulatory landscape": "regulatory environment",
        "The L2 Landscape": "The L2 Ecosystem",
        "Landscape": "Ecosystem",
        "landscape": "environment",
    },
    r"This means that ": "",
    r"This ensures that ": "",
}

def fix_dashes_in_line(line):
    """Replace em/en dashes with appropriate punctuation."""
    if line.strip().startswith("```") or line.strip().startswith("|"):
        if line.strip().startswith("|"):
            line = line.replace(" — ", " / ")
            line = line.replace("—", " / ")
            line = line.replace(" – ", " / ")
            line = line.replace("–", " / ")
        return line

    # Paired dashes: "X — Y — Z" → "X (Y) Z"
    paired = re.findall(r' — ([^—\n]+?) — ', line)
    if paired:
        line = re.sub(r' — ([^—\n]+?) — ', r' (\1) ', line)

    # Definition style: "**Term** — definition" → "**Term**: definition"
    line = re.sub(r'(\*\*[^*]+\*\*) — ', r'\1: ', line)

    # Remaining em dashes with spaces
    line = line.replace(" — ", ", ")

    # Em dashes without spaces (appositive)
    line = re.sub(r'(\w)—(\w)', r'\1, \2', line)
    line = re.sub(r'(\*)—(\w)', r'\1, \2', line)
    line = re.sub(r'(\w)—(\*)', r'\1, \2', line)
    line = re.sub(r'(\))—', r'\1, ', line)
    line = re.sub(r'—(\()', r', \1', line)

    # Catch any remaining em dashes
    line = line.replace("—", ", ")

    # En dashes with spaces
    line = line.replace(" – ", ", ")
    line = line.replace("–", ", ")

    # Clean up double commas or comma-space-comma
    line = re.sub(r',\s*,', ',', line)
    # Clean up ", ," 
    line = line.replace(", ,", ",")

    return line


def fix_ai_patterns(content):
    """Replace AI-sounding phrases with natural alternatives."""
    # "This means that" at start of sentence
    content = re.sub(r'This means that ', '', content)
    content = re.sub(r'This ensures that ', '', content)

    # "fundamentally"
    content = content.replace("changes fundamentally", "changes completely")
    content = content.replace("fundamentally different approach", "structurally different approach")
    content = content.replace("fundamentally different", "structurally different")
    content = content.replace("A fundamentally different", "A structurally different")
    content = content.replace("is fundamentally", "is structurally")
    content = content.replace("fundamentally", "structurally")

    # "essentially"
    content = content.replace("is essentially your", "is effectively your")
    content = content.replace("is essentially a", "is effectively a")
    content = content.replace("essentially", "effectively")

    # "landscape"
    content = content.replace("regulatory landscape", "regulatory environment")
    content = content.replace("The L2 Landscape", "The L2 Ecosystem")
    content = content.replace("L2 landscape", "L2 ecosystem")
    content = content.replace("DeFi landscape", "DeFi ecosystem")
    content = content.replace("landscape", "environment")

    return content


def process_file(filepath):
    with open(filepath, "r") as f:
        content = f.read()

    lines = content.split("\n")
    in_code_block = False
    fixed_lines = []

    for line in lines:
        if line.strip().startswith("```"):
            in_code_block = not in_code_block
            fixed_lines.append(line)
            continue

        if in_code_block:
            fixed_lines.append(line)
            continue

        fixed_lines.append(fix_dashes_in_line(line))

    content = "\n".join(fixed_lines)
    content = fix_ai_patterns(content)

    with open(filepath, "w") as f:
        f.write(content)

    return filepath


def count_issues(filepath):
    with open(filepath, "r") as f:
        content = f.read()
    dashes = content.count("—") + content.count("–")
    ai_hits = 0
    for word in ["fundamentally", "essentially", "landscape", "This means that", "This ensures that"]:
        ai_hits += content.count(word)
    return dashes, ai_hits


if __name__ == "__main__":
    files = sorted(glob.glob(os.path.join(DRAFTS_DIR, "*.md")))
    files = [f for f in files if "SAGE_AUDIT" not in f]

    print(f"Processing {len(files)} articles...\n")

    for f in files:
        d_before, a_before = count_issues(f)
        process_file(f)
        d_after, a_after = count_issues(f)
        name = os.path.basename(f)
        print(f"  {name}: dashes {d_before}→{d_after}, AI patterns {a_before}→{a_after}")

    print("\nDone. Verify a few files manually.")