scenarios.py

"""Predefined evaluation scenarios for Forge eval harness.

v4.4.3 Phase 1 expanded the gold set from 5 to 30 scenarios covering:
- 10 Python epics (Forge's primary language)
- 8 TypeScript/React epics
- 6 Go epics
- 4 cross-cutting / mixed-language refactors
- 2 failure-mode scenarios (ambiguous spec; gold outcome is
  "escalated_with_question", not "landed")

Each scenario carries:
- name, epic (id + description + spec), language
- expected_traits (informal)
- expected_outcome (landed | escalated_with_question | shelved)
- gold_subtask_count_range (a decomposition landing outside this
  range is usually a signal of planning failure)
- gold_file_set (files a reasonable decomposition would touch;
  used by PipelineEval's file_set_iou scorer)
- difficulty (easy | medium | hard) for stratified metrics
"""

from __future__ import annotations

from forge.eval.decomposer_eval import EvalScenario

SCENARIOS: list[EvalScenario] = [
    # ---------------- Python (10) ----------------
    EvalScenario(
        name="python_cli_tool",
        epic={
            "id": "CLI-001",
            "description": "Add CLI command for database migration",
            "spec": "Create a 'migrate' CLI command using Click that runs pending DB "
            "migrations. Include --dry-run flag. Test: assert migrations are "
            "applied in order and dry-run shows but doesn't execute.",
        },
        language="python",
        expected_traits=["2-4 subtasks", "each spec > 80 chars", "has test description"],
        expected_outcome="landed",
        gold_subtask_count_range=(2, 4),
        gold_file_set=["cli/migrate.py", "tests/test_migrate.py"],
        difficulty="medium",
    ),
    EvalScenario(
        name="minimal_single_function",
        epic={
            "id": "MIN-001",
            "description": "Add string truncation utility",
            "spec": "Create a truncate(s, max_len) function that truncates a string to "
            "max_len chars, appending '...' if truncated. Test: normal, at "
            "boundary, empty string, max_len=0.",
        },
        language="python",
        expected_traits=["1-2 subtasks", "single package"],
        expected_outcome="landed",
        gold_subtask_count_range=(1, 2),
        gold_file_set=["utils/string_utils.py", "tests/test_string_utils.py"],
        difficulty="easy",
    ),
    EvalScenario(
        name="python_flask_endpoint",
        epic={
            "id": "PY-002",
            "description": "Add GET /api/users/<id> Flask endpoint",
            "spec": "Create a Flask blueprint exposing GET /api/users/<id> that returns "
            "the user as JSON. Return 404 when user is missing. Test: 200 with "
            "valid JSON, 404 on missing user, response schema.",
        },
        language="python",
        expected_traits=["2-3 subtasks", "has test description"],
        expected_outcome="landed",
        gold_subtask_count_range=(2, 3),
        gold_file_set=["api/users.py", "tests/test_users_api.py"],
        difficulty="easy",
    ),
    EvalScenario(
        name="python_dataclass_to_pydantic",
        epic={
            "id": "PY-003",
            "description": "Migrate User dataclass to pydantic BaseModel",
            "spec": "Convert the existing User @dataclass into a pydantic v2 BaseModel. "
            "Preserve all fields and defaults. Add field validators for email and "
            "username. Update all call sites. Test: validation errors raised on "
            "bad input, existing valid input still parses.",
        },
        language="python",
        expected_traits=["3-5 subtasks", "cross-file updates"],
        expected_outcome="landed",
        gold_subtask_count_range=(3, 5),
        gold_file_set=["models/user.py", "api/users.py", "tests/test_user_model.py"],
        difficulty="medium",
    ),
    EvalScenario(
        name="python_async_scrape",
        epic={
            "id": "PY-004",
            "description": "Add async page scraper with retries",
            "spec": "Implement scrape_page(url) using httpx.AsyncClient with exponential "
            "backoff retries on 5xx (max 3 retries). Return bytes of page body. "
            "Test: success path, retry-then-success, retry-exhausted raises.",
        },
        language="python",
        expected_traits=["2-4 subtasks", "async code", "tests include retry logic"],
        expected_outcome="landed",
        gold_subtask_count_range=(2, 4),
        gold_file_set=["scraper/client.py", "tests/test_scraper.py"],
        difficulty="medium",
    ),
    EvalScenario(
        name="python_sqlite_cache",
        epic={
            "id": "PY-005",
            "description": "Add SQLite-backed TTL cache",
            "spec": "Implement a Cache class with get(key), set(key, value, ttl_seconds), "
            "and delete(key), storing entries in a local SQLite file. Expired "
            "entries return None on get. Test: roundtrip, expiry, overwrite, delete.",
        },
        language="python",
        expected_traits=["3-4 subtasks", "persistence layer"],
        expected_outcome="landed",
        gold_subtask_count_range=(2, 4),
        gold_file_set=["cache/sqlite_cache.py", "tests/test_sqlite_cache.py"],
        difficulty="medium",
    ),
    EvalScenario(
        name="python_argparse_subcommand",
        epic={
            "id": "PY-006",
            "description": "Add 'status' subcommand to existing argparse CLI",
            "spec": "Extend the existing argparse-based CLI with a 'status' subcommand "
            "that prints current config plus uptime. Preserve existing "
            "subcommands. Test: status prints expected fields, help still lists "
            "all subcommands, unknown subcommand exits nonzero.",
        },
        language="python",
        expected_traits=["1-2 subtasks", "extends existing code"],
        expected_outcome="landed",
        gold_subtask_count_range=(1, 3),
        gold_file_set=["cli/main.py", "tests/test_cli_status.py"],
        difficulty="easy",
    ),
    EvalScenario(
        name="python_csv_parser",
        epic={
            "id": "PY-007",
            "description": "Add robust CSV parser with error reporting",
            "spec": "Implement parse_csv(path) that returns a list of typed records and "
            "a list of per-row error messages. Skip blank lines; coerce ints and "
            "floats where possible. Test: clean file, row with type error, "
            "missing columns, BOM-prefixed file.",
        },
        language="python",
        expected_traits=["2-4 subtasks", "error-path coverage"],
        expected_outcome="landed",
        gold_subtask_count_range=(2, 4),
        gold_file_set=["parsers/csv_parser.py", "tests/test_csv_parser.py"],
        difficulty="medium",
    ),
    EvalScenario(
        name="python_type_hints_coverage",
        epic={
            "id": "PY-008",
            "description": "Add type hints to forge/utils module",
            "spec": "Annotate all public functions in forge/utils.py with type hints. "
            "Run mypy and resolve errors. Do not change behavior. Test: mypy "
            "passes, all existing tests still pass.",
        },
        language="python",
        expected_traits=["1-2 subtasks", "no behavior change"],
        expected_outcome="landed",
        gold_subtask_count_range=(1, 3),
        gold_file_set=["forge/utils.py"],
        difficulty="easy",
    ),
    EvalScenario(
        name="python_class_refactor_split",
        epic={
            "id": "PY-009",
            "description": "Split oversized OrderService class into OrderReader + OrderWriter",
            "spec": "The existing OrderService class has both read-side and write-side "
            "methods, violating SRP. Split it into OrderReader (get_*, list_*) and "
            "OrderWriter (create_*, update_*, delete_*). Update call sites. "
            "Preserve behavior. Test: all existing order tests still pass, "
            "mypy clean.",
        },
        language="python",
        expected_traits=["4-6 subtasks", "cross-file refactor"],
        expected_outcome="landed",
        gold_subtask_count_range=(4, 6),
        gold_file_set=[
            "services/order_service.py",
            "services/order_reader.py",
            "services/order_writer.py",
            "api/orders.py",
            "tests/test_orders.py",
        ],
        difficulty="hard",
    ),
    # ---------------- TypeScript (8) ----------------
    EvalScenario(
        name="typescript_react_component",
        epic={
            "id": "UI-001",
            "description": "Add user settings page with dark mode toggle",
            "spec": "Create React component UserSettings with a dark mode toggle. "
            "Persist preference to localStorage. Test: render, toggle, verify "
            "localStorage update and body class change.",
        },
        language="typescript",
        expected_traits=["2-4 subtasks", "each spec > 80 chars"],
        expected_outcome="landed",
        gold_subtask_count_range=(2, 4),
        gold_file_set=[
            "src/components/UserSettings.tsx",
            "src/components/UserSettings.test.tsx",
        ],
        difficulty="medium",
    ),
    EvalScenario(
        name="typescript_react_form_validation",
        epic={
            "id": "TS-002",
            "description": "Add sign-up form with email + password validation",
            "spec": "Create SignUpForm component with email and password fields. Email "
            "must match RFC5322-ish regex; password must be >= 8 chars with at "
            "least one digit. Show per-field error message on blur. Test: valid "
            "submit, each invalid case, submit blocked while errors present.",
        },
        language="typescript",
        expected_traits=["2-4 subtasks", "has test description"],
        expected_outcome="landed",
        gold_subtask_count_range=(2, 4),
        gold_file_set=[
            "src/components/SignUpForm.tsx",
            "src/components/SignUpForm.test.tsx",
        ],
        difficulty="medium",
    ),
    EvalScenario(
        name="typescript_redux_slice",
        epic={
            "id": "TS-003",
            "description": "Add userProfile Redux Toolkit slice",
            "spec": "Create a Redux Toolkit slice 'userProfile' with state "
            "{profile, loading, error}, async thunk fetchProfile, and reducers "
            "for setProfile and clearProfile. Register in the root store. "
            "Test: reducers handle each action, thunk dispatches pending/fulfilled/rejected.",
        },
        language="typescript",
        expected_traits=["3-4 subtasks", "store wiring"],
        expected_outcome="landed",
        gold_subtask_count_range=(2, 4),
        gold_file_set=[
            "src/store/userProfileSlice.ts",
            "src/store/index.ts",
            "src/store/userProfileSlice.test.ts",
        ],
        difficulty="medium",
    ),
    EvalScenario(
        name="typescript_api_client",
        epic={
            "id": "TS-004",
            "description": "Add typed ApiClient wrapper around fetch",
            "spec": "Implement ApiClient with get<T>(path) and post<T>(path, body) "
            "methods that throw typed ApiError on non-2xx responses and return "
            "parsed JSON on success. Support JSON request bodies. Test: success, "
            "4xx throws typed error, 5xx throws typed error, network failure.",
        },
        language="typescript",
        expected_traits=["2-3 subtasks", "generic types"],
        expected_outcome="landed",
        gold_subtask_count_range=(2, 4),
        gold_file_set=["src/api/client.ts", "src/api/client.test.ts"],
        difficulty="medium",
    ),
    EvalScenario(
        name="typescript_custom_hook",
        epic={
            "id": "TS-005",
            "description": "Add useDebouncedValue custom React hook",
            "spec": "Implement useDebouncedValue<T>(value: T, ms: number): T that "
            "returns the input value debounced by ms. Clean up timer on unmount "
            "and on re-value. Test with jest fake timers: returns initial "
            "immediately, updates after delay, cancels pending update on new value.",
        },
        language="typescript",
        expected_traits=["1-2 subtasks", "uses fake timers"],
        expected_outcome="landed",
        gold_subtask_count_range=(1, 3),
        gold_file_set=[
            "src/hooks/useDebouncedValue.ts",
            "src/hooks/useDebouncedValue.test.ts",
        ],
        difficulty="easy",
    ),
    EvalScenario(
        name="typescript_error_boundary",
        epic={
            "id": "TS-006",
            "description": "Add ErrorBoundary class component with fallback UI",
            "spec": "Create ErrorBoundary React class component that catches render "
            "errors in its children and renders a configurable fallback. Log the "
            "error via an injectable logger. Test: renders children on success, "
            "renders fallback on thrown error, calls logger with error + info.",
        },
        language="typescript",
        expected_traits=["1-2 subtasks", "class component"],
        expected_outcome="landed",
        gold_subtask_count_range=(1, 3),
        gold_file_set=[
            "src/components/ErrorBoundary.tsx",
            "src/components/ErrorBoundary.test.tsx",
        ],
        difficulty="easy",
    ),
    EvalScenario(
        name="typescript_i18n_wrapper",
        epic={
            "id": "TS-007",
            "description": "Add I18nProvider + useTranslation hook",
            "spec": "Create I18nProvider that loads a locale bundle and exposes a "
            "useTranslation() hook returning t(key, vars?) that substitutes "
            "{{var}} placeholders. Support fallback to key when missing. Test: "
            "provider loads bundle, hook resolves keys, var substitution, missing "
            "key returns key.",
        },
        language="typescript",
        expected_traits=["2-4 subtasks", "context + hook"],
        expected_outcome="landed",
        gold_subtask_count_range=(2, 4),
        gold_file_set=[
            "src/i18n/I18nProvider.tsx",
            "src/i18n/useTranslation.ts",
            "src/i18n/i18n.test.tsx",
        ],
        difficulty="medium",
    ),
    EvalScenario(
        name="typescript_table_virtualization",
        epic={
            "id": "TS-008",
            "description": "Virtualize the existing 10k-row DataTable",
            "spec": "The DataTable component currently renders all rows; with 10k rows "
            "it freezes. Add virtualization so only visible rows render. Preserve "
            "existing column APIs, sort handlers, and row-click handlers. Test: "
            "scroll shows new rows, existing interactions still fire on visible "
            "rows, virtualization disabled when < 100 rows.",
        },
        language="typescript",
        expected_traits=["4-6 subtasks", "performance refactor"],
        expected_outcome="landed",
        gold_subtask_count_range=(3, 6),
        gold_file_set=[
            "src/components/DataTable.tsx",
            "src/components/VirtualizedRows.tsx",
            "src/components/DataTable.test.tsx",
        ],
        difficulty="hard",
    ),
    # ---------------- Go (6) ----------------
    EvalScenario(
        name="go_http_handler",
        epic={
            "id": "HH-001",
            "description": "Add health check endpoint",
            "spec": 'Create GET /healthz returning {"status":"ok","uptime":N}. '
            "Must include unit test asserting 200 status and valid JSON body.",
        },
        language="go",
        expected_traits=["1-4 subtasks", "each spec > 80 chars", "has test description"],
        expected_outcome="landed",
        # R11 FORGE-EI-01: widened lower bound from 2 to 1 after the
        # mechanical archive showed HH-001 LANDED in 1 subtask under
        # Phase 4c prompts. See docs/r11/forge-ei-01-mechanical-archive.md.
        gold_subtask_count_range=(1, 4),
        gold_file_set=["handlers/healthz.go", "handlers/healthz_test.go"],
        difficulty="easy",
    ),
    EvalScenario(
        name="go_middleware_auth",
        epic={
            "id": "GO-002",
            "description": "Add bearer-token auth middleware",
            "spec": "Create AuthMiddleware(next http.Handler) that requires "
            "Authorization: Bearer <token> and rejects missing or invalid tokens "
            "with 401. Allow an injectable token validator for testing. Test: "
            "valid token passes, missing header 401, bad token 401.",
        },
        language="go",
        expected_traits=["1-3 subtasks", "has test description"],
        expected_outcome="landed",
        # R11 FORGE-EI-01: widened lower bound from 2 to 1 after the
        # mechanical archive showed GO-002 ran in 1 subtask under
        # Phase 4c prompts.
        gold_subtask_count_range=(1, 3),
        gold_file_set=["middleware/auth.go", "middleware/auth_test.go"],
        difficulty="easy",
    ),
    EvalScenario(
        name="go_grpc_service_method",
        epic={
            "id": "GO-003",
            "description": "Add GetUser RPC to existing UserService",
            "spec": "Extend the existing UserService gRPC service with a GetUser(id) "
            "RPC returning the User message. Update the .proto, regenerate "
            "bindings, implement the handler. Return NOT_FOUND on missing user. "
            "Test: valid id returns user, missing id returns NOT_FOUND.",
        },
        language="go",
        expected_traits=["2-5 subtasks", "proto + impl + test"],
        expected_outcome="landed",
        # R11 FORGE-EI-01: widened lower bound from 3 to 2 after the
        # mechanical archive showed GO-003 LANDED in 2 subtasks under
        # Phase 4c prompts (the new ## Required imports from spec
        # block let the decomposer compress proto + impl into one
        # cohesive subtask plus a separate test subtask).
        gold_subtask_count_range=(2, 5),
        gold_file_set=[
            "proto/user.proto",
            "service/user_service.go",
            "service/user_service_test.go",
        ],
        difficulty="medium",
    ),
    EvalScenario(
        name="go_goroutine_pool",
        epic={
            "id": "GO-004",
            "description": "Add bounded goroutine pool",
            "spec": "Implement Pool with Submit(func()) and Close() methods that caps "
            "concurrent goroutines at a given limit. Close waits for in-flight "
            "work to complete. Test: submit N > limit jobs, verify max active "
            "never exceeds limit, close waits, submit after close errors.",
        },
        language="go",
        expected_traits=["2-4 subtasks", "concurrency primitives"],
        expected_outcome="landed",
        gold_subtask_count_range=(2, 4),
        gold_file_set=["concurrency/pool.go", "concurrency/pool_test.go"],
        difficulty="medium",
    ),
    EvalScenario(
        name="go_context_propagation",
        epic={
            "id": "GO-005",
            "description": "Thread context.Context through HTTP -> service -> DB",
            "spec": "Update handler -> service -> repository call chain to accept and "
            "propagate ctx context.Context. Cancel DB queries on request "
            "cancellation. Preserve existing behavior. Test: cancellation "
            "propagates, timeout is honored, in-flight query is cancelled.",
        },
        language="go",
        expected_traits=["2-5 subtasks", "multi-layer touch"],
        expected_outcome="landed",
        # R11 FORGE-EI-01: widened lower bound from 3 to 2 after the
        # mechanical archive showed GO-005 ran in 2 subtasks in BOTH
        # Phase 4a and Phase 4c (the original (3,5) was stale even
        # before the Phase 4c prompts changed decomposer behavior).
        gold_subtask_count_range=(2, 5),
        gold_file_set=[
            "handlers/users.go",
            "service/user_service.go",
            "repository/user_repo.go",
            "repository/user_repo_test.go",
        ],
        difficulty="hard",
    ),
    EvalScenario(
        name="go_interface_extraction",
        epic={
            "id": "GO-006",
            "description": "Extract UserRepository interface from concrete implementation",
            "spec": "The UserRepo struct is referenced concretely across the codebase. "
            "Extract a UserRepository interface with the currently-used methods, "
            "and update callers to depend on the interface. Keep the existing "
            "concrete implementation. Test: existing tests pass, fake "
            "implementation of the interface compiles and is usable in new test.",
        },
        language="go",
        expected_traits=["2-4 subtasks", "interface extraction"],
        expected_outcome="landed",
        gold_subtask_count_range=(2, 4),
        gold_file_set=[
            "repository/user_repo.go",
            "repository/user_repository.go",
            "service/user_service.go",
        ],
        difficulty="medium",
    ),
    # ---------------- Mixed / cross-cutting (4) ----------------
    EvalScenario(
        name="cross_cutting_refactor",
        epic={
            "id": "REF-001",
            "description": "Extract logging into a shared middleware",
            "spec": "Move inline logging from all HTTP handlers into a single logging "
            "middleware. Each handler should no longer import the logger directly. "
            "Test: middleware logs request path, status code, and duration.",
        },
        language="mixed",
        expected_traits=["3-5 subtasks", "has dependency chain"],
        expected_outcome="landed",
        gold_subtask_count_range=(3, 5),
        gold_file_set=[
            "handlers/users.go",
            "handlers/orders.go",
            "handlers/health.go",
            "middleware/logging.go",
            "middleware/logging_test.go",
        ],
        difficulty="hard",
    ),
    EvalScenario(
        name="multi_file_rename_import",
        epic={
            "id": "MIX-002",
            "description": "Rename forge.legacy module to forge.archive and update all imports",
            "spec": "The forge.legacy Python module is being renamed to forge.archive. "
            "Move the files, update every 'from forge.legacy ...' import to "
            "'from forge.archive ...', and leave a deprecation shim in "
            "forge.legacy that re-exports from the new location. Test: both "
            "import paths work, existing tests pass, deprecation shim emits "
            "DeprecationWarning.",
        },
        language="mixed",
        expected_traits=["3-5 subtasks", "cross-file imports"],
        expected_outcome="landed",
        gold_subtask_count_range=(3, 5),
        gold_file_set=[
            "forge/archive/__init__.py",
            "forge/legacy/__init__.py",
            "tests/test_archive_rename.py",
        ],
        difficulty="hard",
    ),
    EvalScenario(
        name="upgrade_dependency",
        epic={
            "id": "MIX-003",
            "description": "Upgrade pydantic v1 -> v2 across the project",
            "spec": "Upgrade the pydantic dependency from v1 to v2 and update all "
            "BaseModel subclasses to v2 idioms (validators, Config -> "
            "model_config, etc.). Preserve behavior. Test: all existing model "
            "tests pass, any BaseSettings usage continues to work.",
        },
        language="mixed",
        expected_traits=["4-6 subtasks", "dependency migration"],
        expected_outcome="landed",
        gold_subtask_count_range=(3, 6),
        gold_file_set=[
            "pyproject.toml",
            "models/user.py",
            "models/order.py",
            "config/settings.py",
            "tests/test_models.py",
        ],
        difficulty="hard",
    ),
    EvalScenario(
        name="monorepo_shared_types",
        epic={
            "id": "MIX-004",
            "description": "Share API types between Python backend and TS frontend",
            "spec": "The Python backend defines User and Order pydantic models. The TS "
            "frontend currently has hand-written duplicates that drift. Add a "
            "codegen step that emits TS interfaces from the pydantic models and "
            "wire it into the build. Test: codegen output matches expected "
            "snapshot, build fails if codegen is stale.",
        },
        language="mixed",
        expected_traits=["3-5 subtasks", "cross-language contract"],
        expected_outcome="landed",
        gold_subtask_count_range=(3, 5),
        gold_file_set=[
            "models/user.py",
            "models/order.py",
            "scripts/gen_ts_types.py",
            "frontend/src/types/api.ts",
            "frontend/package.json",
        ],
        difficulty="hard",
    ),
    # ---------------- Failure-mode (2) ----------------
    # Gold outcome is NOT "landed" -- these scenarios test whether Forge
    # correctly recognizes ambiguity and escalates with a specific question
    # rather than blindly decomposing.
    EvalScenario(
        name="ambiguous_auth_spec",
        epic={
            "id": "FAIL-001",
            "description": "Add authentication to the API",
            "spec": "Add authentication to the API. Users should be able to log in "
            "and then make authenticated requests.",
        },
        language="python",
        expected_traits=["should escalate", "spec is underspecified"],
        expected_outcome="escalated_with_question",
        gold_subtask_count_range=(1, 3),
        gold_file_set=[],
        difficulty="hard",
    ),
    # ---------------- v4.5.1 Stage 4 canary ----------------
    # Single-scenario smoke check derived from the live HTTP-101
    # epic. The full ``_default_runner`` is still deferred to
    # v4.4.3.1, so this scenario is primarily consumed by the
    # offline ``forge eval canary --events-file`` path.
    EvalScenario(
        name="http_101_canary",
        epic={
            "id": "HTTP-101",
            "description": "Add normalize_encoding(name) helper to httpie/encoding.py",
            "spec": "Create a pure function normalize_encoding(name: str) -> str "
            "that canonicalizes encoding names: 'UTF8'/'utf_8'/'UTF-8' -> 'utf-8'; "
            "'latin1' -> 'iso-8859-1'; 'ASCII' -> 'ascii'. Unknown encodings "
            "pass through lowercased with hyphens normalized. No I/O. "
            "Test: happy cases + unknown pass-through in tests/test_encoding.py.",
        },
        language="python",
        expected_traits=["1-3 subtasks", "pure function", "unit tests in same PR"],
        expected_outcome="landed",
        gold_subtask_count_range=(1, 3),
        gold_file_set=["httpie/encoding.py", "tests/test_encoding.py"],
        difficulty="easy",
    ),
    # ---------------- R10 Phase B Capability Sweep ----------------
    # Four httpie-sandbox scenarios designed to exercise distinct
    # orchestrator code paths beyond the canary's "easy happy
    # path". Each is a baseline-success hypothesis (expected
    # outcome: landed); post-capture data may surface escalations,
    # retries, or other surprises -- which is exactly the sweep's
    # purpose. Spec text lifted verbatim from
    # ``docs/TASKBOARD.md`` Active list (lines 22, 21, 25, 26).
    # See ``docs/r10/`` postmortems (created during B.5-B.10) for
    # observed outcomes.
    EvalScenario(
        name="http_103_content_range",
        epic={
            "id": "HTTP-103",
            "description": "Add parse_content_range(header) helper to httpie/downloads.py",
            "spec": "Add parse_content_range(header) helper to "
            "httpie/downloads.py -- parse an HTTP Content-Range header "
            "value (e.g. 'bytes 0-1023/2048' or 'bytes */2048') into a "
            "dict {'unit': 'bytes', 'start': 0, 'end': 1023, 'total': 2048}; "
            "return None on malformed input. Add unit tests covering "
            "valid, wildcard, and malformed inputs.",
        },
        language="python",
        expected_traits=["1-3 subtasks", "pure function", "unit tests in same PR"],
        expected_outcome="landed",
        gold_subtask_count_range=(1, 3),
        gold_file_set=["httpie/downloads.py", "tests/test_downloads.py"],
        difficulty="easy",
    ),
    EvalScenario(
        name="http_102_safe_filename",
        epic={
            "id": "HTTP-102",
            "description": "Add safe_filename(url, max_length=200) helper to httpie/utils.py",
            "spec": "Add safe_filename(url, max_length=200) helper to "
            "httpie/utils.py -- sanitize a URL into a filesystem-safe "
            'filename by replacing /, \\, ?, &, :, #, *, <, >, |, " with '
            "_, stripping leading/trailing whitespace and dots, and "
            "truncating to max_length characters. Add unit tests in "
            "tests/test_utils.py (create if missing).",
        },
        language="python",
        expected_traits=["character-class edge cases", "pure function"],
        expected_outcome="landed",
        gold_subtask_count_range=(1, 3),
        gold_file_set=["httpie/utils.py", "tests/test_utils.py"],
        difficulty="medium",
    ),
    EvalScenario(
        name="http_203_truncate_bytes",
        epic={
            "id": "HTTP-203",
            "description": "Add truncate_to_bytes(text, max_bytes, ...) helper to httpie/compat.py",
            "spec": "Add truncate_to_bytes(text, max_bytes, *, "
            'encoding="utf-8", suffix="...") helper to '
            "httpie/compat.py -- truncate a str so that its UTF-8 "
            "encoded form is at most max_bytes bytes, never splitting a "
            "multi-byte character; if truncation happens, append suffix "
            "(also counted against max_bytes). Return the input "
            "unchanged when it already fits. Raise ValueError if "
            "max_bytes < len(suffix.encode(encoding)). Pure function. "
            "Add unit tests in tests/test_compat_helpers.py covering "
            "ASCII-only, multi-byte boundary (e.g. a 3-byte emoji at "
            "the cut point), exact-fit, and suffix-too-large cases.",
        },
        language="python",
        expected_traits=["multi-byte boundary", "pure function"],
        expected_outcome="landed",
        gold_subtask_count_range=(1, 3),
        gold_file_set=["httpie/compat.py", "tests/test_compat_helpers.py"],
        difficulty="medium",
    ),
    EvalScenario(
        name="http_204_mask_auth_headers",
        epic={
            "id": "HTTP-204",
            "description": "Add mask_auth_headers(headers, *, mask='***') helper to httpie/output/utils.py",
            "spec": 'Add mask_auth_headers(headers, *, mask="***") '
            "helper to httpie/output/utils.py -- return a new dict with "
            "the same keys as headers but values for sensitive auth "
            "headers replaced by mask. Match these keys "
            "case-insensitively: Authorization, Proxy-Authorization, "
            "Cookie, Set-Cookie, X-Api-Key. Non-sensitive headers pass "
            "through unchanged. Input must not be mutated. Pure "
            "function. Add unit tests in tests/test_output_utils.py "
            "covering sensitive-only, non-sensitive-only, mixed, case "
            "variants, and the no-mutation guarantee.",
        },
        language="python",
        expected_traits=[
            "case-insensitive matching",
            "no input mutation",
            "pure function",
        ],
        expected_outcome="landed",
        gold_subtask_count_range=(1, 3),
        gold_file_set=[
            "httpie/output/utils.py",
            "tests/test_output_utils.py",
        ],
        difficulty="medium",
    ),
    EvalScenario(
        name="scope_creep_trap",
        epic={
            "id": "FAIL-002",
            "description": "Add a search bar",
            "spec": "Add a search bar at the top of the page. It should search across "
            "users, orders, and products, with autocomplete, keyboard shortcuts, "
            "recent-search history, fuzzy matching, saved searches, and analytics "
            "of search patterns. Should feel fast. One subtask should be enough.",
        },
        language="typescript",
        expected_traits=[
            "should escalate",
            "spec claims one subtask but contains many concerns",
        ],
        expected_outcome="escalated_with_question",
        gold_subtask_count_range=(1, 3),
        gold_file_set=[],
        difficulty="hard",
    ),
]