ordeal.toml.example

# ordeal.toml — Configuration for ordeal's coverage-guided explorer.
#
# This file is the single source of truth for an exploration run.
# Shareable, versionable, usable by both humans and AI agents.
#
# Run with:  ordeal explore
# Override:  ordeal explore --max-time 300 --seed 99 -v

[explorer]
# Python modules to track for edge coverage.  The explorer checkpoints
# when new control-flow edges are discovered in these modules.
target_modules = ["myapp"]

# Wall-clock time limit for the exploration run.
max_time = 60

# RNG seed — same seed + same code = same exploration.
seed = 42

# Maximum saved checkpoints.  Older/lower-energy ones are evicted.
max_checkpoints = 256

# Probability that a new run starts from a checkpoint (vs fresh).
checkpoint_prob = 0.4

# Checkpoint selection strategy:
#   "energy"  — favor checkpoints that led to new discoveries (default)
#   "uniform" — pick uniformly at random
#   "recent"  — favor recently created checkpoints
checkpoint_strategy = "energy"

# Max rule steps per exploration run.
steps_per_run = 50

# Probability of a nemesis action (fault toggle) per step.
fault_toggle_prob = 0.3

# When true, seed mutation stays within common bounds implied by your
# declared Hypothesis strategies. Useful for config/control-plane tests
# where "nearby but still valid" is better than unconstrained fuzzing.
# seed_mutation_respect_strategies = true

# N-gram depth for edge coverage hashing (AFL++ NGRAM model).
#   1 = classic AFL single-edge (prev XOR cur).
#   2 = path-context: hashes the last 2 locations with the current one.
#       Captures which branch led to each edge.  Sweet spot for Python.
#   3+ = deeper context, diminishing returns (Python lines are coarser
#       than C basic blocks — ngram=2 already captures the important signal).
ngram = 2

# Rule swarm: each run uses a random subset of rules.
# Disabling some rules per run forces others to accumulate state
# (e.g. only inserts, no deletes → cache grows → GC triggers).
# rule_swarm = true

# Per-rule timeout in seconds.  If a ChaosTest rule takes longer than
# this (e.g. buggify-induced hang, slow ONNX inference, blocking I/O),
# it is interrupted with RuleTimeoutError — a real resilience finding.
# Default 30s.  Set to 0 to disable.  Also settable via --rule-timeout.
# rule_timeout = 30.0


# Each [[tests]] entry is a ChaosTest class to explore.
# Format: "module.path:ClassName"

[[tests]]
class = "tests.test_chaos:MyServiceChaos"
# steps_per_run = 30   # override per test (optional)
# swarm = true          # enable swarm fault selection (optional)
# rule_timeout = 60.0   # override per-rule timeout for this test (optional)

# [[tests]]
# class = "tests.test_chaos:AnotherChaos"


[report]
# Output format: "text" (stdout), "json" (file), or "both".
format = "both"

# Path for the JSON report.
output = "ordeal-report.json"

# Save full traces for replay with `ordeal replay`.
traces = true
traces_dir = ".ordeal/traces"

# Seed corpus — failing traces are saved here and replayed automatically
# on subsequent runs for instant regression detection.
corpus_dir = ".ordeal/seeds"

# Print live progress to stderr during exploration.
verbose = true


# Long-lived services — use instead of [[tests]] with:
# ordeal explore --runner compose
# Full guide: docs/guides/compose-runner.md
#
# [compose]
# # Compose file and optional isolated project name. Relative file paths are
# # resolved from the directory containing this config.
# file = "compose.yaml"
# project_name = "ordeal-test"
#
# # Host-reachable service URL. Readiness polls base_url + health_path and
# # accepts any status below 500.
# base_url = "http://127.0.0.1:8080"
# health_path = "/health"
#
# # Exact Compose service names eligible for SIGKILL and restart.
# services = ["api", "worker"]
#
# # Values available to {name} templates before any response capture.
# initial_state = { tenant = "acme" }
#
# # Stop after either request-selection budget is reached.
# steps = 50
# max_time = 60
# seed = 42
#
# # Each faultable request has this chance of entering a fault cycle.
# fault_probability = 0.3
# faults = ["kill", "restart", "delay_response", "corrupt_response"]
# delay_seconds = 0.5
#
# # HTTP and post-start/recovery readiness timeouts.
# request_timeout = 5
# startup_timeout = 30
#
# # Repeat a recorded failure this many times, reporting attempted N /
# # reproduced M instead of claiming deterministic replay.
# replay_attempts = 5
# # Optional clean-control response-oracle mutations for workload protection.
# workload_mutations = 20
# trace_dir = ".ordeal/traces"
#
# # False: take down only a topology ordeal itself started. Existing running
# # topologies are always left running. Named volumes are never deleted.
# keep_running = false
#
# [[compose.requests]]
# name = "create"
# method = "POST"
# path = "/{tenant}/items"
# headers = { Accept = "application/json" }
# json = { name = "demo", owner = "{tenant}" }
# expect_status = [200, 201]
# capture = { item_id = "json.id" }
# # POST is not faultable by default. Opt in only if repetition is safe.
# faultable = false
#
# [[compose.requests]]
# name = "read"
# path = "/items/{item_id}"
# headers = { Accept = "application/json" }
# # Do not select this request until create captured item_id.
# requires = ["item_id"]
# # Quoted TOML keys preserve the dots used for JSON traversal.
# expect_json = { "json.name" = "demo" }
# # GET is faultable by default; this explicit value is documentary.
# faultable = true


# Mutation testing — validate that your tests catch real bugs.
# Run with:  ordeal mutate
#
# [mutations]
# targets = ["myapp.scoring.compute", "myapp.scoring.normalize"]
#
# Preset: "essential" (4 ops — fast), "standard" (8 ops — default), "thorough" (all 14).
# preset = "standard"
#
# Or specify operators explicitly (cannot use both preset and operators):
# operators = ["arithmetic", "comparison", "negate"]
#
# Minimum mutation score — exit 1 if below (useful for CI gating).
# threshold = 0.8
#
# Parallel workers for testing mutants.
# workers = 4
#
# Skip mutants that produce identical output to the original.
# filter_equivalent = true
# equivalence_samples = 10


# Auto-scan — exploratory smoke-test for every public function in a module.
# Run with:  ordeal audit <module>
#
# [[scan]]
# module = "myapp.scoring"
# max_examples = 50
# mode = "evidence"   # or "candidate" for stricter ranking
# security_focus = true   # opt in to trust-boundary-biased sink detection, path probes,
#                        # and small artifact/config mutations for deserialize/IPC inputs
# shell_injection_check = true  # static input->shell sink oracle, runs before execution
# include_private = false
#
# # Optional: focus scan on a subset of callables instead of the whole module
# # surface. Exact names and glob selectors both work.
# targets = ["myapp.scoring:Scorer.score", "normalize", "Scorer.*"]
#
# # Load project-specific fixture registrations from importable modules that
# # call register_fixture(). This supplements local conftest.py discovery.
# fixture_registries = ["tests.support.fixtures"]
#
# # Suppress noisy mined properties or relations when they are domain-shaped
# # constraints rather than defects.
# ignore_contracts = ["quoted_paths"]
# ignore_properties = ["commutative"]
# ignore_relations = ["commutative_composition"]
#
# # Contract packs expand to the concrete built-ins ordeal already knows about.
# auto_contracts = [
#   "shell_path_safety",
#   "shell_injection_check",
#   "protected_env_vars",
#   "cleanup_teardown",
#   "cancellation_safety",
#   "json_tool_call_normalization",
# ]
#
# # Keep expected validation failures visible without promoting them as bugs.
# expected_failures = ["validate_input"]
# expected_preconditions = { "*" = ["ValueError"], build_env_vars = ["protected key"] }
#
# # Mark domain-shaped findings as expected semantics instead of surprising
# # exploratory output.
# expected_properties = { "*" = ["ordered_arguments"] }
# expected_relations = { "*" = ["equivalent"] }
#
# # Per-function suppression / override knobs.
# contract_overrides = { build_env_vars = ["protected_env_keys"] }
# property_overrides = { normalize = ["idempotent"] }
# relation_overrides = { normalize = ["equivalent"] }
#
# # Custom fixtures: keys = parameter names, values = comma-separated choices.
# # These override Hypothesis strategy inference for parameters that need
# # domain-specific values (e.g. enums, categories).
# [scan.fixtures]
# category = "violence,cyber,sexual"
#
# Shared fixture registries for every scan in this project.
# These modules are imported for their register_fixture() side effects.
#
# [fixtures]
# registries = ["tests.support.shared_fixtures"]
#
# # Shared object factories for bound instance methods. Scan and audit reuse
# # these automatically. Hooks run in order: factory -> setup -> scenarios.
# [[objects]]
# target = "myapp.envs:ComposableEnv"
# factory = "tests.support.factories:make_composable_env"
# state_factory = "tests.support.factories:build_composable_state"
# setup = "tests.support.factories:prime_composable_env"
# teardown = "tests.support.factories:cleanup_composable_env"
# harness = "stateful"
# scenarios = [
#   "subprocess",
#   "sandbox",
#   "upload_download",
#   "http",
#   "state_store",
#   { kind = "stub_return", path = "sandbox_client.execute_command", value = { returncode = 0, stdout = "ok" } },
#   { kind = "stub_raise", path = "upload_content", error = "RuntimeError: denied" },
# ]
# methods = ["build_env_vars"]
#
# # Optional semantic probes for shell/path/env helpers during scan.
# [[contracts]]
# target = "myapp.envs:ComposableEnv.build_env_vars"
# checks = ["shell_path_safety", "shell_injection_check", "protected_env_vars", "json_tool_call_normalization"]
# kwargs = { path = "tmp/my binary", env_vars = { PATH = "/bin", HOME = "/tmp/home" } }
# tracked_params = ["path"]
# protected_keys = ["PATH", "HOME"]
# env_param = "env_vars"
#
# # Lifecycle contracts for stateful async/framework methods. These reuse the
# # matching [[objects]] factory/setup/state hooks above.
# [[contracts]]
# target = "myapp.envs:ComposableEnv.rollout"
# checks = ["cancellation_safety"]
# kwargs = { marker = "demo" }
# phase = "rollout"
# followup_phases = ["cleanup", "teardown"]
# fault = "cancel_rollout"
# handler_name = "cleanup_alpha"
#
# [[scan]]
# module = "myapp.validation"
# max_examples = 100
#
# # Expected failures stay visible but are not treated as bugs.
# expected_failures = ["validate_input", "check_permissions"]

# Audit defaults — stronger test-quality signal than raw scan output.
# Run with:  ordeal audit  (uses [audit].modules when omitted)
#
# [audit]
# modules = ["myapp.scoring", "myapp.pipeline"]
# test_dir = "tests"
# max_examples = 20
# workers = 1
# validation_mode = "fast"
#
# # Optional object-level targets with per-target factory/setup hooks.
# [[audit.targets]]
# target = "myapp.envs:ComposableEnv"
# factory = "tests.support.factories:make_composable_env"
# state_factory = "tests.support.factories:build_composable_state"
# setup = "tests.support.factories:prime_composable_env"
# teardown = "tests.support.factories:cleanup_composable_env"
# harness = "stateful"
# scenarios = ["sandbox"]
# methods = ["build_env_vars", "post_sandbox_setup"]
#
# # Optional outputs / gates.
# show_generated = false
# # save_generated = "tests/test_migrated.py"  # must stay under the current workspace root
# # write_gaps_dir = "tests/gaps"              # must stay under the current workspace root
# # min_fixture_completeness = 0.5
# include_exploratory_function_gaps = false
# require_direct_tests = false
#
# Init defaults — bootstrap starter tests and optional follow-up files.
# Run with:  ordeal init  (uses [init].target when omitted)
#
# [init]
# target = "myapp"
# output_dir = "tests"   # must stay under the current workspace root
# ci = false
# ci_name = "ordeal"
# install_skill = false
# close_gaps = false
#
# # Optional deeper defaults for the bootstrap loop.
# # gap_output_dir = "tests/gaps"   # must stay under the current workspace root
# mutation_preset = "essential"
# scan_max_examples = 10

# Revision diff defaults — both refs run in detached worktrees/subprocesses.
# Run with:  ordeal diff  (uses [diff].target when omitted)
#
# [diff]
# target = "myapp.scoring"
# base_ref = "origin/main"
# candidate_ref = "HEAD"
# max_examples = 100
# seed = 42
# # rtol = 1e-6
# # atol = 1e-9
# include_private = false
# fixture_registries = []
# replay_attempts = 2
# save_artifacts = false
# artifact_dir = ".ordeal/diff"  # must stay under the current workspace root


# API chaos testing — built-in OpenAPI engine, no extra deps.
# Run with:  ordeal api (or use programmatically via chaos_api_test)
#
# [api]
# # ASGI/WSGI app to test — "module.path:attribute" format.
# app = "myapp.main:app"
# wsgi = false                     # set true for Flask/Django WSGI apps
#
# # Or fetch the schema from a running server:
# # schema_url = "http://localhost:8000"
# # schema_path = "/openapi.json"  # path to OpenAPI schema endpoint
# # base_url = "http://localhost:8000"
#
# # Fault injection during API testing.
# # faults = ["ordeal.faults.timing.slow", "ordeal.faults.io.error_on_call"]
# fault_probability = 0.3
# seed = 42
# swarm = false                    # random fault subsets per run
# max_examples = 100
#
# # Extra headers sent with every request (e.g. auth tokens).
# [api.headers]
# Authorization = "Bearer test-token"