|
| 1 | +"""DNA Enhancer Design grader. |
| 2 | +
|
| 3 | +Evaluates programs that generate 200bp DNA sequences optimized for |
| 4 | +HepG2 cell-type-specific enhancer activity. |
| 5 | +
|
| 6 | +Scoring tiers: |
| 7 | + - Always available (no ML): GC content stability + population diversity |
| 8 | + - With Enformer model: adds HepG2/K562/SKNSH expression prediction |
| 9 | +
|
| 10 | +All scorer code is bundled in eval/scorers/. No external SAGA dependency. |
| 11 | +""" |
| 12 | + |
| 13 | +from __future__ import annotations |
| 14 | + |
| 15 | +import json |
| 16 | +import os |
| 17 | +import textwrap |
| 18 | + |
| 19 | +from coral.grader import TaskGrader |
| 20 | +from coral.types import ScoreBundle |
| 21 | + |
| 22 | + |
| 23 | +class Grader(TaskGrader): |
| 24 | + """Grader for the DNA Enhancer Design task.""" |
| 25 | + |
| 26 | + def evaluate(self) -> ScoreBundle: |
| 27 | + program_file = self.args.get("program_file", "solution.py") |
| 28 | + top_k = self.args.get("top_k", 10) |
| 29 | + timeout = self.timeout |
| 30 | + |
| 31 | + program_path = os.path.join(self.codebase_path, program_file) |
| 32 | + if not os.path.exists(program_path): |
| 33 | + return self.fail(f"Program file ({program_file}) not found") |
| 34 | + |
| 35 | + # Path to bundled scorer modules inside .coral/private/eval/ |
| 36 | + scorer_dir = str(self.read_eval_path("scorers")) |
| 37 | + |
| 38 | + try: |
| 39 | + result = _run_evaluation( |
| 40 | + program_path, scorer_dir, top_k, timeout, |
| 41 | + self.get_python_command(), |
| 42 | + ) |
| 43 | + except TimeoutError: |
| 44 | + return self.fail(f"Evaluation timed out after {timeout}s") |
| 45 | + except Exception as e: |
| 46 | + return self.fail(f"Evaluation failed: {e}") |
| 47 | + |
| 48 | + if "error" in result: |
| 49 | + return self.fail(f"Error: {result['error']}") |
| 50 | + |
| 51 | + score = result["composite_score"] |
| 52 | + n_valid = result["n_valid"] |
| 53 | + n_total = result["n_total"] |
| 54 | + gc_mean = result.get("gc_mean", 0.0) |
| 55 | + diversity = result.get("diversity", 0.0) |
| 56 | + has_enhancer = result.get("has_enhancer", False) |
| 57 | + |
| 58 | + parts = [f"Composite: {score:.4f}"] |
| 59 | + if has_enhancer: |
| 60 | + parts.append(f"HepG2 (top {top_k}): {result.get('hepg2_mean', 0):.4f}") |
| 61 | + parts.append(f"K562: {result.get('k562_mean', 0):.4f}") |
| 62 | + parts.append(f"SKNSH: {result.get('sknsh_mean', 0):.4f}") |
| 63 | + parts.append(f"Diversity: {diversity:.4f}") |
| 64 | + parts.append(f"GC: {gc_mean:.3f}") |
| 65 | + parts.append(f"Valid: {n_valid}/{n_total}") |
| 66 | + explanation = " | ".join(parts) |
| 67 | + |
| 68 | + feedback_lines = [] |
| 69 | + if has_enhancer: |
| 70 | + feedback_lines.append(f"Top-{top_k} HepG2 expression: {result.get('hepg2_mean', 0):.4f}") |
| 71 | + feedback_lines.append(f"Top-{top_k} K562 expression (off-target): {result.get('k562_mean', 0):.4f}") |
| 72 | + feedback_lines.append(f"Top-{top_k} SKNSH expression (off-target): {result.get('sknsh_mean', 0):.4f}") |
| 73 | + feedback_lines.append("Composite = HepG2 - 0.3*(K562+SKNSH) + 0.1*diversity + 0.1*gc_bonus") |
| 74 | + else: |
| 75 | + feedback_lines.append("[Enhancer model not available — using GC/diversity proxy scoring]") |
| 76 | + feedback_lines.append("Composite = gc_bonus + diversity_bonus") |
| 77 | + feedback_lines.append("Install grelu + model checkpoint for full expression scoring.") |
| 78 | + feedback_lines.append(f"Population diversity (Hamming): {diversity:.4f}") |
| 79 | + feedback_lines.append(f"Mean GC content: {gc_mean:.3f} (ideal: 0.45-0.55)") |
| 80 | + feedback_lines.append(f"Valid sequences: {n_valid}/{n_total}") |
| 81 | + |
| 82 | + return self.score(score, explanation, feedback="\n".join(feedback_lines)) |
| 83 | + |
| 84 | + |
| 85 | +def _run_evaluation( |
| 86 | + program_path: str, scorer_dir: str, top_k: int, timeout: int, |
| 87 | + python_cmd: list[str], |
| 88 | +) -> dict: |
| 89 | + import subprocess |
| 90 | + |
| 91 | + script = textwrap.dedent(f"""\ |
| 92 | + import sys, json, os, time, warnings |
| 93 | + warnings.filterwarnings("ignore") |
| 94 | +
|
| 95 | + # --- Run agent solution --- |
| 96 | + sys.path.insert(0, os.path.dirname({os.path.abspath(program_path)!r})) |
| 97 | + module_name = {os.path.splitext(os.path.basename(program_path))[0]!r} |
| 98 | + program = __import__(module_name) |
| 99 | +
|
| 100 | + start = time.time() |
| 101 | + sequences = program.run() |
| 102 | + gen_time = time.time() - start |
| 103 | +
|
| 104 | + if not isinstance(sequences, list): |
| 105 | + print(json.dumps({{"error": "run() must return a list of DNA sequences"}})) |
| 106 | + sys.exit(0) |
| 107 | +
|
| 108 | + # --- Validate --- |
| 109 | + valid_bases = set("ATGC") |
| 110 | + valid_seqs = [] |
| 111 | + for seq in sequences: |
| 112 | + if isinstance(seq, str) and len(seq) == 200 and all(b in valid_bases for b in seq.upper()): |
| 113 | + valid_seqs.append(seq.upper()) |
| 114 | +
|
| 115 | + n_total = len(sequences) |
| 116 | + n_valid = len(valid_seqs) |
| 117 | + if n_valid == 0: |
| 118 | + print(json.dumps({{"error": f"No valid sequences ({{n_total}} returned). Must be 200 chars of ATGC."}})) |
| 119 | + sys.exit(0) |
| 120 | +
|
| 121 | + top_k = min({top_k}, n_valid) |
| 122 | +
|
| 123 | + # --- GC content (always available) --- |
| 124 | + gc_scores = [sum(1 for b in s if b in "GC") / len(s) for s in valid_seqs] |
| 125 | + gc_mean = sum(gc_scores) / n_valid |
| 126 | +
|
| 127 | + # --- Diversity (always available) --- |
| 128 | + import numpy as np |
| 129 | + diversity = 0.0 |
| 130 | + if n_valid >= 2: |
| 131 | + total_dist = 0 |
| 132 | + count = 0 |
| 133 | + for i in range(min(n_valid, 200)): |
| 134 | + for j in range(i + 1, min(n_valid, 200)): |
| 135 | + total_dist += sum(a != b for a, b in zip(valid_seqs[i], valid_seqs[j])) / 200.0 |
| 136 | + count += 1 |
| 137 | + diversity = total_dist / count if count > 0 else 0.0 |
| 138 | +
|
| 139 | + # --- Enhancer expression (optional) --- |
| 140 | + has_enhancer = False |
| 141 | + hepg2_scores = None |
| 142 | + k562_scores = None |
| 143 | + sknsh_scores = None |
| 144 | +
|
| 145 | + scorer_dir = {scorer_dir!r} |
| 146 | + sys.path.insert(0, os.path.dirname(scorer_dir)) |
| 147 | + try: |
| 148 | + from scorers.enhancer import EnhancerScorer, is_available |
| 149 | + if is_available(): |
| 150 | + enhancer = EnhancerScorer() |
| 151 | + hepg2_scores = enhancer.score_hepg2(valid_seqs) |
| 152 | + k562_scores = enhancer.score_k562(valid_seqs) |
| 153 | + sknsh_scores = enhancer.score_sknsh(valid_seqs) |
| 154 | + has_enhancer = True |
| 155 | + except Exception: |
| 156 | + pass |
| 157 | +
|
| 158 | + # --- Compute composite --- |
| 159 | + gc_bonus_vals = [1.0 - min(abs(gc - 0.5) * 10, 1.0) for gc in gc_scores] |
| 160 | + diversity_bonus = min(diversity / 0.5, 1.0) |
| 161 | +
|
| 162 | + if has_enhancer and hepg2_scores is not None: |
| 163 | + # Build per-sequence score for ranking |
| 164 | + indexed = [] |
| 165 | + for i in range(n_valid): |
| 166 | + h = hepg2_scores[i] if hepg2_scores[i] is not None else float("-inf") |
| 167 | + k = k562_scores[i] if k562_scores[i] is not None else 0.0 |
| 168 | + s = sknsh_scores[i] if sknsh_scores[i] is not None else 0.0 |
| 169 | + indexed.append((h, k, s, gc_scores[i], i)) |
| 170 | +
|
| 171 | + indexed.sort(key=lambda x: x[0], reverse=True) |
| 172 | + top = indexed[:top_k] |
| 173 | +
|
| 174 | + hepg2_mean = sum(t[0] for t in top) / top_k |
| 175 | + k562_mean = sum(t[1] for t in top) / top_k |
| 176 | + sknsh_mean = sum(t[2] for t in top) / top_k |
| 177 | + top_gc_bonus = sum(gc_bonus_vals[t[4]] for t in top) / top_k |
| 178 | +
|
| 179 | + composite = hepg2_mean - 0.3 * (k562_mean + sknsh_mean) + 0.1 * diversity_bonus + 0.1 * top_gc_bonus |
| 180 | + else: |
| 181 | + # Proxy scoring without ML model |
| 182 | + hepg2_mean = 0.0 |
| 183 | + k562_mean = 0.0 |
| 184 | + sknsh_mean = 0.0 |
| 185 | + avg_gc_bonus = sum(gc_bonus_vals) / n_valid |
| 186 | + composite = avg_gc_bonus + diversity_bonus |
| 187 | +
|
| 188 | + eval_time = time.time() - start |
| 189 | + print(json.dumps({{ |
| 190 | + "composite_score": round(float(composite), 4), |
| 191 | + "has_enhancer": has_enhancer, |
| 192 | + "hepg2_mean": round(float(hepg2_mean), 4), |
| 193 | + "k562_mean": round(float(k562_mean), 4), |
| 194 | + "sknsh_mean": round(float(sknsh_mean), 4), |
| 195 | + "diversity": round(float(diversity), 4), |
| 196 | + "gc_mean": round(float(gc_mean), 3), |
| 197 | + "n_valid": n_valid, |
| 198 | + "n_total": n_total, |
| 199 | + "gen_time": round(gen_time, 1), |
| 200 | + "eval_time": round(eval_time, 1), |
| 201 | + }})) |
| 202 | + """) |
| 203 | + result = subprocess.run( |
| 204 | + [*python_cmd, "-c", script], |
| 205 | + capture_output=True, text=True, timeout=timeout, |
| 206 | + ) |
| 207 | + if result.returncode != 0: |
| 208 | + raise RuntimeError(result.stderr.strip()[-2000:]) |
| 209 | + stdout = result.stdout.strip() |
| 210 | + if not stdout: |
| 211 | + raise RuntimeError(f"No output.\nstderr: {result.stderr.strip()[-1000:]}") |
| 212 | + try: |
| 213 | + return json.loads(stdout) |
| 214 | + except json.JSONDecodeError: |
| 215 | + for line in reversed(stdout.splitlines()): |
| 216 | + line = line.strip() |
| 217 | + if line.startswith("{"): |
| 218 | + try: |
| 219 | + return json.loads(line) |
| 220 | + except json.JSONDecodeError: |
| 221 | + continue |
| 222 | + raise RuntimeError(f"No valid JSON.\nstdout: {stdout[-500:]}\nstderr: {result.stderr.strip()[-500:]}") |
0 commit comments