From f91c326135ea7eb915360c5284f809e828ab6a1f Mon Sep 17 00:00:00 2001 From: "OpenClaw Agent (basd)" Date: Thu, 16 Apr 2026 21:28:41 +0000 Subject: [PATCH 1/8] chore: ignore .worktrees/ directory --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 84361ce..1a93e1c 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,4 @@ skill-optimizer.json cli-commands.json tools.json tasks.json +.worktrees/ From cc72ee73d83ead185253165e102132ebf6c4584e Mon Sep 17 00:00:00 2001 From: dmn Date: Thu, 16 Apr 2026 17:00:50 -0700 Subject: [PATCH 2/8] feat: stable task IDs + optimizer loop diagram (#24) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: ignore .worktrees/ directory * feat: stable task IDs + optimizer loop diagram in README - fix(tasks): derive task IDs from sha1(action names) instead of LLM-supplied id field, which changed on every regeneration and broke --task filters. Action names come from the discovered surface and are stable across runs when the surface hasn't changed. Duplicate action-name sets get a -1/-2 numeric suffix. - docs: add horizontal optimizer-loop SVG diagram to README top, showing the full init → baseline → iterate (analyze/mutate/ re-benchmark/accept-reject) → output flow at a glance. Closes #17 Co-Authored-By: Claude Sonnet 4.6 * docs: add PNG version of optimizer loop diagram Co-Authored-By: Claude Sonnet 4.6 * docs: use SVG in README (PNG kept as companion file) SVG renders natively on GitHub and scales without pixelation. PNG is included alongside as a companion for external use cases (email, Office docs, tools that don't render SVG). Co-Authored-By: Claude Sonnet 4.6 * fix: address Copilot review on PR #24 - Delete SAFE_TASK_ID / isSafeTaskId from src/tasks/generate.ts — dead after the stable-ID refactor (still used in src/benchmark/config.ts for external task-file validation, so that copy stays). - Extend stable task IDs to the prompt surface: fall back to a SHA-1 hash of the prompt text when expected_actions is empty, so prompt-surface --task filters survive regeneration. - Rewrite four README links (optimizer-loop.svg, docs/reference/*.md, CONTRIBUTING.md) to absolute github.com URLs — docs/ and CONTRIBUTING.md are not in the npm tarball's files field, so relative paths 404 when users view the README on npmjs.com. Co-Authored-By: Claude Sonnet 4.6 * docs: final 1.1.0 polish — CHANGELOG, error message, README consistency - CHANGELOG.md: fill in all additions and fixes that landed on development after the initial 1.1.0 bump (stable task IDs, Codex auth, SKILL folder, diagram, model-ID slug overhaul, error message). - src/errors.ts + docs/reference/errors.md: fix E_MODEL_ID_FORMAT — was "missing the openrouter/ prefix"; now lists all three valid provider prefixes (openrouter/, anthropic/, openai/). - README.md: use catalog-correct openrouter/google/gemini-2.5-flash (dot, not hyphen) in answers.json example; change "skill-optimizer benchmark" to "skill-optimizer run" for consistency with other examples. Co-Authored-By: Claude Sonnet 4.6 * fix(tasks): stable dedup suffix order + accurate CHANGELOG wording Sort validated tasks by (id, prompt) before the dedup counter loop so that numeric suffixes assigned to same-action-hash tasks are determined by content order, not LLM output order. Previously, if the model regenerated two create_wallet tasks in swapped order, the -1/-2 suffixes would swap between runs, making --task filters unstable for multi-variant cases. Also soften the CHANGELOG entry for "stable task IDs": clarifies that SDK/CLI/MCP IDs are stable across regenerations (action names come from discovered code), while prompt-surface IDs are only stable when the LLM produces identical wording. Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: OpenClaw Agent (basd) Co-authored-by: Claude Sonnet 4.6 --- CHANGELOG.md | 6 ++ README.md | 18 ++++-- docs/images/optimizer-loop.png | Bin 0 -> 176284 bytes docs/images/optimizer-loop.svg | 99 +++++++++++++++++++++++++++++++++ docs/reference/errors.md | 11 ++-- src/errors.ts | 9 ++- src/tasks/generate.ts | 74 ++++++++++++++---------- tests/smoke-generation.ts | 9 ++- 8 files changed, 181 insertions(+), 45 deletions(-) create mode 100644 docs/images/optimizer-loop.png create mode 100644 docs/images/optimizer-loop.svg diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b673ce..89a5b19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,10 +24,16 @@ The config file `skill-benchmark.json` is no longer auto-detected. Rename it to ### Added - **prompt surface type** — benchmark and optimize prompt templates, Claude Code skills, and agent instructions. Discovers phases and capabilities from markdown, evaluates output quality with content-based criteria. +- **Codex auth** — direct OpenAI model runs can use browser-login tokens stored by Codex (`~/.codex/auth.json`) instead of requiring `OPENAI_API_KEY`. Set `benchmark.authMode: "codex"` and use `openai/` IDs. +- **SKILL folder** — bundled AI-agent guidance (`SKILL/SKILL.md`) so agents can use skill-optimizer reliably without extra setup. +- **Optimizer loop diagram** — README now includes a visual workflow diagram of the optimizer loop. +- **Stable task IDs** — task IDs are now derived from a SHA-1 hash of the action names (SDK/CLI/MCP surfaces) or prompt text (prompt surface). For SDK/CLI/MCP surfaces, where action names come from discovered code rather than LLM output, IDs are stable across regenerations and the `--task ` filter works reliably. For the prompt surface, IDs are stable when the LLM produces identical wording; if it rephrases a task the ID changes (fixes [#17](https://github.com/fastxyz/skill-optimizer/issues/17)). ### Fixed - **benchmark:** Strip provider prefix from model ID when using direct `anthropic` or `openai` formats. Previously, `anthropic/claude-sonnet-4-6` was sent as-is to the Anthropic API, which expects `claude-sonnet-4-6`. The `pi` format is unaffected. +- **model IDs:** OpenRouter model slugs now preserve dots in version numbers (e.g. `openrouter/anthropic/claude-sonnet-4.6`). Presets updated to match OpenRouter's catalog exactly. The dot→hyphen rewrite in `validate`/`fix` now applies only to the `anthropic/` direct-API prefix; `openrouter/` and `openai/` slugs are exempt. +- **error message:** `E_MODEL_ID_FORMAT` now lists all three valid provider prefixes (`openrouter/`, `anthropic/`, `openai/`) instead of directing all users to use `openrouter/`. ## 1.0.0 — 2026-04-14 diff --git a/README.md b/README.md index 535227e..8e22aff 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,14 @@ skill-optimizer runs your SDK / CLI / MCP docs against multiple LLMs, measures w **Requirements:** Node.js 20+, plus either an [OpenRouter](https://openrouter.ai) API key or a local Codex login when using direct OpenAI models. +## How it works — at a glance + +![Optimizer Loop](https://raw.githubusercontent.com/fastxyz/skill-optimizer/main/docs/images/optimizer-loop.svg) + +`skill-optimizer run` benchmarks your callable surface against multiple LLMs — it discovers actions, generates tasks, calls each model, and statically evaluates action recall and argument accuracy to produce a PASS/FAIL verdict (exit 0/1) usable in CI. + +`skill-optimizer optimize` runs the benchmark as a feedback loop: it copies your SKILL.md, mutates it with an LLM agent, re-benchmarks, accepts only when scores improve, and repeats until stable. Your original SKILL.md is never modified. + ## Installation ```bash @@ -89,7 +97,7 @@ npx skill-optimizer init --answers answers.json "models": [ "openrouter/anthropic/claude-sonnet-4.6", "openrouter/deepseek/deepseek-v3.2", - "openrouter/google/gemini-2-5-flash", + "openrouter/google/gemini-2.5-flash", "openrouter/qwen/qwen3.5-397b-a17b", "openrouter/moonshotai/kimi-k2.5", "openrouter/z-ai/glm-5.1", @@ -119,7 +127,7 @@ Benchmark how well models follow your prompt templates: ```bash skill-optimizer init prompt -skill-optimizer benchmark +skill-optimizer run ``` The prompt surface discovers phases and capabilities from your SKILL.md, @@ -142,9 +150,9 @@ SDK/CLI/MCP guidance. ## Configuration reference -See [docs/reference/config-schema.md](docs/reference/config-schema.md) for the full generated config reference — auto-updated at every build. +See [docs/reference/config-schema.md](https://github.com/fastxyz/skill-optimizer/blob/main/docs/reference/config-schema.md) for the full generated config reference — auto-updated at every build. -See [docs/reference/errors.md](docs/reference/errors.md) for all error codes, descriptions, and fix instructions. +See [docs/reference/errors.md](https://github.com/fastxyz/skill-optimizer/blob/main/docs/reference/errors.md) for all error codes, descriptions, and fix instructions. ## Interpreting the verdict @@ -210,4 +218,4 @@ export OPENROUTER_API_KEY=sk-or-... ## Contributing -See [CONTRIBUTING.md](./CONTRIBUTING.md). +See [CONTRIBUTING.md](https://github.com/fastxyz/skill-optimizer/blob/main/CONTRIBUTING.md). diff --git a/docs/images/optimizer-loop.png b/docs/images/optimizer-loop.png new file mode 100644 index 0000000000000000000000000000000000000000..2c2b59cd5cf136909432d91aec91d8efa4e10fd5 GIT binary patch literal 176284 zcmeFZbyQT{8#a7~A*DqDX=$XTdlVFqM!H2Q=^7da5ecPBX$k30$w3JL>F(|rdL#zE z@p*pV6W?0T`|tP9yVqi!!TlDRKqTC}b&*|Mc3&J}( z^(hd^e|?Hl{1`1O_a7}Dto-9XAZo-O|TBJF9bV?d8_z|6U+ zxNWf6cdm{-)Rm8q#}?kE{U~RSZSF4nme2g2A7Y;$+f#b!$%i6tlB)-9dXp@W=H}$w zZGVP^bzdHGOURgO0u$#n82kGv_=N6(#I|V&D&D`n0*g(kpAfStu8GW-dv)f$2e-*6 z^xoiT!_Z}eMo(ny-|KvI>mOz~B(otGG-Ac;3*sqm6}<6&*G}zx|Im8KAC4D!+KZAy zRpKowjhC_4OUt6{FKaIbM`_S+bAS7)6UE_Lg)`t`Fi%@Qit0H@p*q@=y7VxdY~dY5 zXd@(yIUo2V93^fRYFAp%>Z@Dk-Pkj6q&6Q&C}=w64%L{{HNo_QHnh>V|KjU>nXa2t zeee*tyn&)Hdl1W4AYx%NIc6b>(+Uuzg)RIeUS}ZEE}k{@Kz*g#kv8Ji=zENy(_Y1 zJ9`P(nP`bIB*^&Ga^eHdGa0u{?JQ~%Z@%Kkp7e_vvvGSoQ26(6QS*{1_is!fHL#q_ zF-Ve^)B-UcsI;VIa`Oi(+^pUoHZN;YchOamarj1?vN^IV|EMx+tD4D+ZQ0sD+&CHqLzE#1Wp>MfG zS>14RKxYN}78CrKtL*ySj_n%^b`S%lzb9&EKy7DF-Gu7NCiu&rm-yq7lXlWJlAuz$MGm0*Uyma6BO zoq$ZLp*g z5B&_%g|fOLr7~MtL-zDKLRVq>1tAW<;HV8K1iz%#&!-9JR?>rY#N_S#i}LZW>)_R{ z%fX4?4rPvdQS_agF%@Qf1kx_dBuEPPf{RG(2)Gm~0Jwt9(m2mjy}6$9_U)+z0Q z`R3vw;T@!@hVAK)%55{(H2bu(cJNtRuVhgRm=C|^YWaZNY2L)@wG!COJf-0KhBnAN z!d$-HLht^|fKNefT?n9r*DZAax#m3mtBIGh+ln5RrdW^5f@^M>rk{G<2=h#BbPY(; zWjtQHj9?eHTpEooMh@)f6}S3F6)&h1N|&7aN_pMznrmEz#c`|qKUxzwl?{~E#%_#k|huJ?-wtvhkNFEsx3atwrzIs0@ZK|;yM~`T^pEKpnl-~W^nV$Z- zX%mq58KSn_&$QzRTo|I3L4*Ca=@u6^|0kmdVXnchAW4wIMa*YK=-hu{&?r z@jXXrR@jbp6N>z}TsT8ErqfkpGM~jhizO11na>c9B>T(3)`X1yvE3e}{s@er&-R63 zZzO+(4uhKFq*;G?0|3F`Z9ttAaANQa5NXmsD&<>*E)&j60cURWt;^QZW;ZLw6cdeL zA=)C5VokvfXXaT21*KGCxA!~Ew)upM3o{FX2MtY4WAo$|R3V^a@d1}lf=?SNK35Ky z?Bh$2nDA#2^DTBnx2;eCbsqD^zxd<~CI!AnKCKTsg1a`sintGazrClNkbqYZv2hLT zgTELm^GJHL4NkNsc_gJz$y^`L^Dca4!8nHU-ZI5v(3d!OaWMn}>j?llZpo!RJWn{; zJP_$F>bz7;T(0My{=K>n(v@@8XIP4*+`*a_!31*-l2@UbB^N~=WQ}W^E=Kek$|Adt zRSc?IS33Z(GsuJN-C?s?|BQy}H}I8X4Ys|iA(?gi7O z1TdrJ0hMr4kgkyNdV2Kv6~B|~ynW5(K4eC-WU%@gx${!SLI#y|xfLbuvh7(SAOgdC z|Aipqa;@lXX`Cpqhyep1U3MUQ@|rV1Mr}f~!{EpdQgr^_(&g?w@5p;20UXd88#Y}@_JTBr)Da3HXsS3J2Is$*2Zbf})?s7Yb= zY>^-%xH&M7!icI_G8(m#@n<*&Ia*w`=tcw{&Gj}UCgW%!^zC(WZD0mCLP}$P;aw?O zz6KrV{znOH%3L4VDA#a0`zcxO34+UcbtS9wvzHGa1h`d}@jY)b4U9RT(4YN@I^M23 zIwrYEhRr{1e%zn~8B0qrm3RiE&4{N}cX96DOm1uyZxq14L~fB?)hyUVjZ9-{iRp?} zX8DaT`1ouY&xX*#gM*U>7Bv)zzi;nvEL#prC;WHLt zo zIhGO-dCnCuw7WJo!827Bg%YH_+-ejFw9I7)|?Z&$}*Vaqcl-^4D;|=X!Q92(wxa5?vc>XN7pg=d$T%bPFP2k!#ymdpI+wiX#Brt-o| zXOnNps5aI)6A{U+r@Py95b%I^*=y+-2P0x;RfP?0jnh3V=kn)gD^rCHpJyuPAysc#8@C;?fD;VZM8*f8=WBs&EV@Thvh8nR0EvFc*Ju^BNZ^8u7h=_P8xERzL%L}s1T$m_~%C<7SErUTTgktymDP4se6q<7cYNYIp5n(v8vo`>xqNd zR@;q>Qtv61#7y(p3zt{7e^V3q8CZ$mYc1gkyq482$``djHK!TYH5t`VDpaWz#V3Bw znC!S4#WAgP@jW~>194zOS`ag>&qk-0P*bd*c|mOS!|?~}{N<9sm4F@b8L|GIi+iwp z;~P2BXCrRAc|V?`1j|0H3-8RE%QLJghZ}~!es|1ld~VZvQJO}51U85hjI#MrDoeOH zUyV~jj}NOAk)YkueSGa(PvAH&`9gY8RJNyv@&IEVfc;89WtW)^379!i;P@ET*)Y!#f;6@HMO?3_Hm>T zJ}Crf?PmHN=`bbPuQvOt?hQW{I`?j_uJh!grAsknf;zU zNYJ8!9qjB?Ca6dbc-mi!5)4XxF;vM@9N*KMwt2C1j~tRcPv|galK9zd=f*K$uReKb z?95Vw@@Qp4yBYfK?h}ItiSo6P4b*;gKy;eqJFf2Af&fr^v`1BWSEcgwzhFd z4n(t7OLCpEFY0?QJ4>g=QVdXuyRS{M!p5O}%T%9I<)?0`aF+_&&O0$XVdOc@tD9V_ z?+jTa)k2I%pPtG^uKdO4iGBiUC%2ln4YC!mXjN7#=84e9+&#))>ZXmG89G#213N|Psz@_jP`g5 zZ>wV7y&jO8uYBKQE#~{n|EeGeOKkDc&oZOB$o&)rJ_v8i6vq*lpHEO9p8*Cu~Lns=k-LBOnQUz%) z<9+Y}&o`22?^%g?CzL58-x_K{K*G<>$f9XzvY(5v-+hkUh+5H@s``W^2jC#CL`Pn} zhc^wrA|T-Q=r4b%-8?fH6walM>bcS3+)f=R)G<1m_x^WH-pY2^I8hm`O6N!w=oxs_|`xi!x{R<;hpYJ?H>^ucokU7+eIug(jN^7j`GAz0v zIY%5`13Y9V?e)_=3%_`e538cO^3=_!q011aMcCA5dA5?kk@2qe-DE)2Gcnb&Zu7Eh z!aqqGB=N=bXY3^1(cXIzF5dcm_Q40a9W9*~u!6~)_Zn|22gb{o_+#!w$46gBzhEq5 z&^u5(paP;x+~2%)d`lc0k}LB{fnC8U#Z9K*C$g0nm;4u=37GHv<}%Cy&w^|=h4DM4 zH}~gt{yt1~9zzei?3>69cxj&x8(se^_i!x4y5uLLMG4LyucBXA6u-E~b|F&ZaMEIO zC$mnYHM(>RxpSUq(i5;Y=zKM&8)KY5lBj3(Bpn>N#&I&?!QV zvN_;~xXD#N9%a2e?YS+GMV9sQ<(Y+x(blb3faR+^G?X`mp#&YzUmNwc2cU4_H%dRw zU$+v(zFK)5bn=$wUtUK3FE8tn;a%W3CNYznCON_4Gq<|P+a=udP@N<8b0exB-h5qV zI1dlQ&KofKh4DXb(~i|C(cyz3QM#)rc?X-TE6YaUX5jEABDpA;2&Z3C%Ch|O{-Bp_ z6i3i0jMz_V-RnsYnVaQl+U1KaMzgslZN1FhxRZc*gh_wNiKLm)ZhmN+JsP%8j~EB= zOE4JH;*IUxDyD@gK)wQDmtKe@#O!SlkZ^74H?K%fCjCiC_w0u#o(NjFAe_8+huYGR z9KsG$)t@+i3~ds)kwLpv^2sT5rlj_H_$CM#zj-%uJt0Bn-qN4Y1K))nJ8}hJnjK86 z1${YnoO=lxtkSOgO#8s9)xSkQWXkkm*I9S zEfdp@7tM9pH0rmZH+gjW<`2msOe@ev><%G&pYDKlQPsbo^TZY}ffo2H1dRF6J4}IH z${lzsyU@{W>y2|x3f-8QUBq-}$Tza@d${f3%~aV2%1=k}HojsSdZOPG z!A-XBbemG*W&Eqb4~T%;>6t9$Y`$}3n&5*o4y0VaumTT@6 zkv?XrsC)4vG5%3^ujc&CQSWh=mv>w4P%juBU4RyL95B$7=STUlEP97~Hb#g4Q%`1X z>7$fbS?e1v&?>dODs7Sb#nbVH#);!cVOCuQNN02#XYQc zg2>>62tx;?_F9JN#bpvP?$!Cl&aYAL9ZAysJ1csvvl<;o`U2}U#;FmFXjkkv`LT+mW$O!X z7Kog=+5V>)G&%+UoqOmJ*rg($DBr`v98d|6ZQ1QR%vcQJ1*klQQ^pJM)n{@#QH5-v z)9LNUr`@om^tv!L6}W|gP+QyuTbtZkPh`P1hq#gibj{~wV)%u9m3!ufR^n`86UW7~ce8+blS8z7Kz6N_$cB^Ifk>nhJCX9@%DKJ7p9UR^hOmx^F|90-qPRVb4xY96VJ1dro zH*J}d*n0V&3fUNo)*ey(FRO4#?y8TcB-Bop@JPQzbSbR zxLmtQ)Moo*n-qwv{8EnP~_u$qUwY*o(TE zxaM8;JSC8}47wnuMe3>dYbYg80JtdxDLyI+%-Xe zvEWd*^5nMK$WUmH;qd#SHFtXrrjukKXm1Bfg>$5e#3%7Kef=$P3?6A8zlh)g1LF@a zY1Ub@#ow_JJOy_-Q|)B-7iUZL4LK!)Oj!A@)~5M=4B_wZx-jBj!j7*f;$+1 zt9AB==t^m7eUf+I9yH=Kvfb%XRa*Cld>jlik!_~`RSiDdkXd!{Wg^+lXn7oW?o-A1 zB?Yi2-qSWv0|6t0s#Cfv2)3P@IvtaHE?KPQyB!bWBKqM)&&r@JQ-q$=o(U+?+kAIZ zpu$8r&kB=7b+4HC79k5PbIm!Nj|JnkzwC~$wqP(FmsQo{8O`~A9$q4XhY9By@%VyN zi_&(y3V08iki8xbiV!sG)LH6XBilN5>_`5bC2*qCbvMinf|TjZg+PDs8V4@lg_&{P ze5#*RyW!r=`SgbkHcKw|@idTt_|J|5l*d<8*k{%c@>fGJ8-0iLL z|Cf*Gt{aFiRTs22xEYrsXGbR#1E?=KNG*Z~aQ?dIVgmP{0n}fS)So9q8E^lYJ^U-| z`t#)fobV^%lmPzfhLI82Od00 zu&jWH|F=3q1IqCK>Gkhl12F!TK>J5$zfUjX{%=*m|1ZT8km6n7?<{91JA#CgElI%% z7?dn;-FOLK zW1jEE0zN-`{o1YC+-sVkySV);gsPmh(O=p(fh&?JE;d#oFYG_Z^S`l{8f7M0CMVNF zsxl``vU4*khAZ`940@-ZJ5>O`XYQqS#L=k&UOJ+LkZ?GpoIA-J?GhJ{9{;oWLr$)T zX%glaiwE0`$gx>gh3uBrs=UfalgYmt@R2#VqZM$v*qR<;0UmwJjkHY!-aI4a1zU1T zh@zi3ffxHnW4>*H^Z0Lnl52&L;1A-G{B+5<&U^sdqRXSR0Y9EOVxbbjc{|^U#ie|w zmG#rjT5gvcq+H!KQ}q#$S8YrE5uDGGo;R&L74eq`X0?VtYM+FwGwj(Mw5TNn3UZdE z@v>cIpfc`B_uqh`pBnnRPQZfXK3=pk2J=ABoNpdb&_D3Jj4qbqedCzu{2^xautrg}KEPn} z;l&$d4^eVx=(W(lDxxk@aECM)0n9Gr(o{n1!3Snh8}rIgAEQl!?rB*0vx*MY-~?L1 zzu<7_W+@HIcuL1`PU7+%AylS@4;y6vI&4>!r8!G6bZg8mj+7olRcG zk6WKnKL9uv@;k95d*b`?q+2BgAl!F`ZngWxyVieWx=S#o4Wy?TvM}HQG79G)zov;+ z=1ey7Dy)=cV$5ny(EC>{*w6r(R=DRqK5BS2*KKFv#M@(BJSy!4h5ve@n!yT1`h+i# zN_2Ad3j`yZe?(29#m}LmKp#CTP+l9V4vm+SJEil#>gf{`Z2mVXD08^}z?_ z=R_#Xl<4sYRzz^g$WLkfy^<*L$3$NqGKvioR%nwjmccb9Zu_86@tKS=a~#}fDVusw9jMdazR?<$Szf_+lcao$ zAg+u#9|NQ4?eGweixD?^$-iJK{?TrSRf7K-2_s`xD%p0Xj{z+tKDvSFET=O2@6R+_ zs=E*f83&@AgiFf3X-3&cWt#NcgaGpl{s;EYpLf~ALO0dA{*lc>+ppQ1`?2O6CGoUa ze_5r*{QY8E;m$&RR;)izm0dy{?cIj|wm@Rm@+3he6;0@!zu(5|2glwt_WCEgvfnVA zSQ+%&!h-*8VH*8-f50tX&Kcxlr%rP*#2%(MoB1e1x+gO7FRW1r+kN71?tUKi=8%dp ztr?cn(q8nF=DL>WHoheT8^Le zuX_kne)X>d2nRWdEA{{<0LwNSu9D z8}Z5mFvFhxa3!XYh5V^sSZ@$rfJy47JB{QB0f36MIKcvgwbAN5lZ$rm2*AHIqGZ1m30&Ra@zvM3J>17IU zmJA5sUmW)Q5%$z4f!2P%+oAD9H;H!54vSLNYF4->VBA1Grwk>D z9@|DZu5Sz!0YnRmtU_RTW|H;E0dcH7k=uyFPG{Ab>v;(LZ7(sg^S9k7nX+|79mPVR zVxxJcX^Y5up@v7@9FBN@>0#V8`8wUdOavr8VrQE)d~@h+o^xtl`arN%kCB0$*MN=g z`m&#bqq`0zXT4%jS+VR8u;aSrvv+$>8#&fQbVq!|Z?9e*pa^~vjFUhsgnG@dojV{a zpyWDp3pS|g{=pkP_=}d!i~T}g_p0m0PO}#s>or8+Kn>m0$aYQ%gFnJP-OvPDcm`{! zrSAx)Bg1R$4{Ea#4C1U`T#QH=I`}M%3NHpG(EE;8oaejzkJh}I&t)Zao?{IPjf$1j z;0$^e(iM0EBwS=C6!BWn_yFl7hK%4?>&3HjBpXW}OhR>kfF&ugT!6odc_x9D7QJB= zoKRcqkh%edrVq{_gP8d>Kl&dHl9k7;HnRKz^2Nd^K7XvtzB+bBivn4ufoPVeu^*~< z?BpatY>i(s_v{R3k-+wq!fdF?*?zW?41{eE8ulFoI6vkcid~?$GMJF-{WT49DoZYV z%G2~ox^eG;R$-p?l??zpM>x9g$EXxvIleIua1C>NV=m&W33UsITYKUVXO1f+r-#>j_ z8%v=iU7fo{IMc7UbL>75pnkipm1lmzkJe&EugxayHL)Q!;CUi@s->c4GH-xQl5wrrf~PmGci4nTQ$5m^zWU8o&hRwT5z#+L!#MF4%^XJ9 z?$fV0?y23N{M5fg6$?&U@mZJCDzkf@&YsR{n{ETsr|9_+4*a$tWDs$SzG{Y0PWQ@b zFF`E&HqFc1_wYhhs+G33I3&E$MUFe^FV#QkA_86er29MvLG^q+WozHH80tO0Ejig* z$ji&M1`Xk!ws`|o^qlJDNAj6o+DgdRey~c?cU|Gqz-V05eBIjdH z+A9GPZ#G}Wcrp?is$`naWxpdW_Oge@wvyeMcg^bMW%Tl-7(#E)-d5IvYI*5CZe5M= z#G)DUu6(YEH+g*o?Hy&ekbi;>bP8LJZSPR5;TX*5PTsG_wcad9{bqWC-9FptoEjQ> z6dT}2LxJ0$Vft0mx$$qHJ&Mf(UnXl&fS&56U2Ug;f~77BR8SaAGy-3oitHg;y>(EY zs-m-tL=e(v@*P}GIU)Gv z(kc^jS*wK8uls?CVNEkHNH#DUqmC#84@cQ_RKU)V;Fs1pq+$#TxhA&1MO2SPXy3%cLw2(WzV_Rq_ArAVk(O3rJ^GY#gPl0J#6 zm+N=6$r1u1&>Cd*euBGCP>&Iz%mp1^>+~;u3aiEUI|aNFPr=Uee!UGi7~+u~=u8Ff zodJ=1NQH$W(|T*@v(kcvRNJA|P8jc*O-L8s;?_7we)|LF zGu2dg7b`<}Yk%<3qmL0cT!})c3SV0(>Dk=i5|r0RBT2elSFi_0`{#y zn}iQy3@X^8Z)B-^uj^LVqHNt}R`T&C^NG?V2k>!epzaG5MG(6Wb>lMzkl=p9-i4l< zW7HF~J+P>Ry*7UiY3cU$po0a zGTX7YfDvw|fqJ%=(c5}cHBhveR#%u_izBII|5N5AmU_W#;P(OQS2vQYCz2Z{;;g2j zZbWIvD+9-)*XX1;FMt%OYu2yCGfR1xwUX5a@f>68p6L29P}ShrA{LAQLx*>A$Y0l% znQ3?YO84Iusauz2xM@p#e^CwM>J$pwKf4z#sYDxFN%c#M!%y;78tTX1HW@=pZ&Elt zK1KuE-h5&Pk=_pRw0IF==8#l?R!pu~E8>-lt?&7~=W8AZr_7b<;uYKv4cl(abO*Xp zBJK!zxY4XrK%P6?YJiN&5EUpFvDjm3fbr#znZz!qnwWDfW?gi*9?8<*gB>gl=*}O; zloh{NFb_ZO7Whv%nuXUXdZM^%iQ>$nzQ3BT5pjPJJ$GdZ^jHr8fh`JK@EX7El zplPYLvB{hqI=rJYnhRU5#F7kKiLZ8DkK{9S#ZDVLk+a(r4_}qd3Qkx?=Cz(d;tmf;nbw<;cm+C>NYi~UrQy81yPKvVJlx1);U@N z%+dDd6Y8>SA=jKvnBOFZHmhrtwBFJUBdtfYhHDH3>Xy6xJ)PoG{XQL_U0Yx&I zrzyJ8Oj%9xE;99r2a8WlzJR8A8I{>JtvESZv)&72pQoN(c#u+9dp+1Q}h9~mJ%OXcMhBNcoB23m*cF)htmyZ}hR9;-~6)H9`O&A$_sB2YQ zPI*oR2J6=ZO<0mPK!q+6hQYXWMQhfN-)T<5lhc2XjVM>-U#JJW#O7SGCvEHd$^Mvye==xf48x zN<8m*@|gv$N%V|;o!|RK)ZuB)>sjM9LK~XEn)ORWX2e041%70Zbe!!C?p)Gl6H3Nu zDiw0|MbtuOH5MN3KROD_W*O+aSKq5BeK_IQSIP5oo0E>MaWDMq;@rBu2c$?N<|VXI zpc;7?w31W;nizZj)PtK~vyR)fz(6QdE9Lhcf=nFAEiQ{i97XXhxJdW5tO) z$6jVxg$Nu$?+kbX4R?|yWAx&vBrzFT2{wK&%n-1sg&rVKawd1Y9$2DJ%3eE`6fnyjZfxIrYghl$`HUvo zjX%p^)5vRCBd}&~n}`ncS5bOOWfu-^y|Xhi@JaaTs*dCmSw?!x6rE?Q-{A?VE)W;g zKa(N`5Ivztl`1DG_Z{3q9n`^Yit*bnk>E#u4nL(n-RGARCY}7Ef6;E%hs5%ks>Vo7 z8^}N&#;6e_u@ru7Krgq~ZL`eh^NsK57oA?2p!_@lro5gKYR_D?$+Q6K(2~TKhgwjU zKbRJN&Pi0sT|dyEf!W-EzX|JQ^V!163VzN-dq!y5Eo)x(;X1i@cVk6`3onftqm zHye{2G~F@OUA`4RZ=44GhP;FnB2U`sCQluS^LC3*8Ga_H2?S!jc;RN#tAF1;{(Ne~ z>nHO7x8s+^R!okCRTPu%7|sV(?dCCSVzWFMuj@`l*kWvX{-)W5-@I5{Kt>!3IkN&= zsO^GM#z$^`)zTzN%zB5k`q~k12D{~v5fO1fC!z5!?Q@#)9l?sO1`3GFc|Ow%!oUfn zRsN~b{V29P7oXpFdm&Xp(KOw(#}jbf@eIKx5!8JwqKv`yT4Z&wk+knCF9wauRQ1#hjlTAC_fHNJ_AA zCKV&PlV8W?47r}WS?nT^M(3rMXN@toE1nKgYyqeB2aR37anv*ir2@U#p=I~#*Pvy0 ze!0GEFAr))u?&d42FmgVTsC)@=?7jRU5zT#B&1XU)A76sCZ$vkFNZVuTmmqsetCDz z2V^pCcza_tZh_+A*D>DMwC~1m%D}vxA9!Y6sG}P=4po{6(Kfpqr|kz69d3oe(uuah z@-%wq{e0)u4iU+YVRboWl%E)y2f{)%X^mM;@R9+$Aw*OR4j3%p=f|yF_~r5knbn29-X~g)qq=%clBd zOfmAgefH6#+*@Yq8aFxpq!6jceflN$7{s}@83?qXoM;s`>0Z!1J-qgoP#0~pDJNML%Q}fM3K`Itk&QurLaSk+ITt3 z*m3W`kNzQpeZ%tgN;PxkYwUvgSC4XbF!Y5I5kvb}sa_Vin z1ZCtotak8=ad!d$y9Gp#GB7h{N=8cxHK@tqW@?xJ7`>g4V=9fQV8E8gECrTSE0YmN z@vCRr*^XQb!`6QG$C4U?-hRFhY{H4wUcy2dd8KEiu;kjm?tXQ>&1bJmT_Vs*Yy}<~3$jE4;Kxt{ekESwVOneTwJ7}{ zoh7QJ9RO%upsl|B0n6*PU65E;&#gsU2o|DmU*rSuIXk(mQwfONx>VH3nPl?-&F`7y zh;WfB6eUUDHxRCnuGIOpHqA% z`(j4{|Ab)*aQGd?>RkXNXW#m)SF~Gq^M1$XhV~`Eq&2T8gGWMld7*N`A17JYlzt0N ztHMvpu2qn-_t6~I9b)jrkQ~fh68E0D51gKwK2jU7L66|Ay!&X`^m5_k5K9?b0!Od} zdvf&NYheUAOk5#uK17MLABS;Ts@1KIibUgN?cC=*2w(7t$EyEL#KKwsc zHxe`pNLjt!8i;!YWV)79_5cS(&eiQtRe%Z14XkDmFei|{@ev(no2X$`Q|Pr;)PV*m z_8unUq&G^gl=Ejw^^KRt)1m(^1(kU}7sL#B5?YtAyL4V-Mt_NoPtC**)!lXwZ~_BA zfAqUYmTaG&pRz*MlGAIKEFm>U7vapeE>(`EFDxHqYD!3HJU#sw6Z{D}jtsJUf|bpx z?I1(|8f5ls^(9Mra$-mPcF(Kwu#?H=+eaaW$&_^Kv+My*0RE7nHaTQrM+J5^npLBO z$_VRTxR^S=YTjW_F+)`i9Dfv4!0mu`S9S?!M zK$*nik)?vKtSLhaA`df)4E9ct^#0Ri_?t)304x!@q}P}NyNSSWB$C5$o{-tQa-v~h z@EGadu?O8PIk^NQfoklx^V=eVy`ZuNWy<*$AlmHCU^kL3lgL%ia>X~#U8VX%S=!}_ zlFkYTOmpzk->70^WZ=D}(<8$a1-N(jVxKEF@bMc>1|X2;T{mNc;V&A<-94P}Fcp;j zTzgssUGmF(pK355I$bZmiLDa7tpin{lkVn?cQG&5#t9`sL!u>8;t?=-evxg|F4l6I zuZJ%Q_<ix5%adxhTt?ac8?bVEmvXrYJZ6>3$v^ZM!rVRb`I0j1SDIYxFP zJ4t~j&ZNJF8U3CVo1&T`5)z9xw|s{Tfqd+HJ1HUH4Ew0LStBNj&O;+_)$48NC65Ti zLzx*~&B%_`_k&o{r87fbOC*^;B0)p~aLkrR7 z&T8hIi==vCVYh6GEKJcb;uJfTLghqF!H-ctMwqC6>3Zwi2oXGez%e}C1CUN7D7&yJ zU%pOTn41+mrs6wxm`d?;gSCu>kK2u-pbK>9u#7sv8zH)4izrGwAqQ{iijvO~{@ z6yRBDHutiHVcV`_4G*LNSi{X;I|w~AL>)mD-|fLStbzF8=VEx{X{3GoctE^$uB<@U* zY^UqOCk=}Dh9G|Jk%hOs(?$D5FL15()_pYdsbL)X7(iRv{)zbIEd@!sT9E^AG&7MD zNjdoeSL-85RK@b)ZJ{@umvuOTwRa9G6f>Oc-+uaqP?@aYpHFtg0X?ZjB4k!@8T6P- zb7Jqmc)S+*NGxs|CiNv@1_Pog)xo7tsIa+gI+O>HEUYO7M_~o>(&asKY$ddwC^t6h z0Re-AQ4gyxCH7*Qy`*@4)UD$1LK@L*;p`H2Fqs$Rc!q9*ENq~#+>vXW*5rW%k?PWmIK+3JG+(=`bsmCwk{pd$TGg%f0uP)vS@6PqE#G3i6v zYR&2C9U!P23k$T@I8LTisDLk3X{u(nRkS2O7kHInIKI1ri;FBw8~m|~cr;e`@U?jp6@5mPK6+FsXOPO= z1%`j`?9zR0c&TjqECm>E`~6p;?$~A$wt9WZaTu2GZg9i$rGP!fN0XXZWgY z$t~{Qo?h3Y)>q1M-ym?!T-U9oYXF9&(368YBcJ2q4s6G6~n$oj9~-_6ojmdz_6Q=?t~OO zSq-%BhZ&@H2l+mQAx=@^oR`ly`v%oCpFTz6wSvvq3cpFCylqU9YQJtE2Rwd#MVpn* zHP(s}`HBs+qtny2r36c_j=sq)k_oVy{m$m)b;Wu^LXXE_64x^PR>M3u>6GaXTIBm> zk?e67GBMiUGy^3Yt7UW){u95_4JSl0dsu>Gxq(B&5{ z@BGQKT%PpZadPk+pKE*wd;cx#F(*UjK30g|21q;bGklbsk8YRC=7CCN&*V*9VJ@3a zVMy{Gr`}zXi$ZE^OHa_&z}frJm5TRS4*(E8kE)a?X8A~pf!|AApuhgzUigN#)TmvC zg*I9twy$XjSzZvji78!;GrDgs`@;yLZASUZn;cMQ*zmw19m=rXF-E3XK`{iZq=;vs zniaEfLu2DQ%R+Ln^)$qdU@gcR8aQdsUG>}*uZ#;qK!q}I`Yzr%*C9y zv-#OaTug&pwfO7A(y0DN+}iN^s3j&pCO~8mn--lfLMtaih3;DY1e~wAc@*-k7^o2_w9=*p^;uG>jM3d3yf|*k%WjTiF|AC88y2Cf z#GH*YcVb?LbWj6B8cW{A&!MB}%yAiq=zM;@+3oTNnU}sb9&oyv#v;u^Kt#TINck)8 zf?DlE z2Ac`25#{}+E)3B$WS{SfQ}96x#S5G-QubL)YtRTC&iRMT&0zoJS>G#ONRjChRfi*B zVo}T7*NtP%UB{#D0>HAqRb{dKh_$o8Uun~}6rtjn(^IHu_I{eHquY}G3l+ckQ8y+A z^6e)Cpy+4PZca~ z5+S=5^g+l%Sft_5dkYgVFs!cSeAAA7pY(bSG=}?~;khGZv&i4XRd;_Z?gfVX&ctyb zuKK|c;@3N_x|so(VKPgRn35yv25Y7A46P}gH0A=C3-ZKcP0?e zaiX+gXHq+2EI1fI=Z2+ZX9J*tGfp>FV_x&8t^Z)+N z2ZjSU_N>ib>pa)_i*va^i^Hw59I#~?UYM$G0@X?PuD_NJs0y=k9J9&;IjA4tEaruy z&TB2uvk)^LhQqLTTKv%XY~Ff?D^T0im4%@h@pS{lpq1$bbSrRRxy}Welb{IWbVP%N z&6TmP@KXyn?TaZ98`hAMq_|~bSo&? z+S7gIgI;i)SFq|~EVof~!OGb|zp7Zei&IiR{)demLx$PQ(3EX-2KIL1khx(Arh^%UV}J1_G* z|Io-d$r5HGTv_kTo)L3OO21=xZ8DVz?fHpBCJ!jC+bF&uZr=f-5btK!totazjzQlM zVVCB7!{zazd^0sq%kxvLn{*~}ztGtds3 z(Z92;TKVijgW4UcO-<_uL6IiA^_8h8x~`U|*?Hfpavy^48^dTK%JV+HdAfQFJDcu! z5xLn!?rrN7jP_Zal52t+ZxxOC)rugX8;h@fKN|*zdNf#tg^)nWXT#*4^s_ZtJAV|O zhR7Fzbn3;=mMyGX5f9R=-l1LL zu2}G(d@kguo)AFDo#eJ&5=J6RYUoyQh2UTqXOlT139Gjl#l6*QTPbwBjs202MDE-J zi&_Egn&_hC#8>5xNIq_jVH)QnvFpDN*r)E0J}oju6Z13wF_%X^*8;>m@#KG)33jnn z%8FX_e@gho%_IVdNExbpemv~yQu1Fop~dl;z)?T+G9l;1k6h3|znOr-ux6=Z6~m_L zDW?Csi2f&t=RK6nu00tVnS~{%G6m)LPBMbav%g-Nm}h8n?Y{DRwkkjSscGmtctX%S zXPyH!Qi@X2+UVPPM67KmNCDJBU&)txbY0f{O2|ri*X6-p%K8tPcvh0OENuFlB z9g8e?RKpBQL?Nm}Ex#~9x3A?qZ%L+v8l+^-;>*sxFLseC4j9vB=UYjRDlZeJrN30kxw)D74h4*>~xvk&1FeF;LzXYl44?1tO0(`Gn>%>Ny8K_hKl@7 zYt4tv8ak4}B3Oq8Dnh)@RJcM#(dcH<)$iYnVBcz=LDMdQLFhh1O64>T-rj67WBP7cllTGBU$_|kphfo_(^qZ-BxMZ zLUmz=Yt*=oNTp`YKiUy~xm+y@_2}IVPRn2E3~Vb_43JLVnGT#(=SPtJS@X+W5<dD>~uR8rS~qwG=c(d6ar;_PyL$R{krD)BtLa~dnOH}V|B1PJrR1%Wx*EW1qLl5 zL6OK@EbteRfh`1^4Oyunr2l7f5MdK^R93#!haFw`yY9;p8QFH^M&Y(Mf*H03pS|P> zzP>j$y6A-7$Y5KSP^Ij%sOgSY)gKybU!rZ`uv0MAVey@dW8{5~DFdRr{S-T2ZI?tx z!TDk3qMvI}pt-~|;mgdQAk7k2m`y%jSVEBL!@NfaiMLpNALCa+=Ants0|}4=)u(B* zJ*4q2dCRk_qwRmj%kd@f5d`_V0ze;Mmo+}JCFoB#<2=%XiTA0eI_uZ<2Z*`t4Q_h! zkcoeG$oK2px8}5<^|+}fp~Q;;if%umADAmu+eJNzNa3Z5c@M#>0dzc#+ooA5`zv6+ zMhhFvuQw$FFzby-$6}&Gho1@y4-GOB7JSH5vy4{nxZWXM z@rgnmCFiyOy8fGBOj>n=xq)>Y#H?BYo0lXbXy{b8#QV;%;2p&iAywL7R--VfRESEkB1E#6iDC%~2D+938mF7} zSH1Y_b01^R3&8*48r@4TkHWYkZm{{GvF-lzD&v|S%|2jnw@$Jwlh?I`YY32WHWJG* z5usMbGuq51DJ2WIyl=b$eJ4}*QjERi-1DNmeM?XJUJu~nxrp5LhBtaofAd9wJM{(v zFTtl83^_Gw5pG1z(Cej;rVMNFL7>o%+s1=}svv&*3^n)wRsLpvM2Ev5&2h0$!m#b7 ztiF_lA*twJj^nM}{gL3a=ZOzjXqP?mF>^=S9QY!>Hd|KKGR`?>${P5Uu0n+e&b>Iq z^*?^)nqjSQf&R--k0(6oKBZT$!C`Zt!kzKMN~K_ob0?PdlJycJljKh252rA z5K?u3=2gS>A_AvSmBP3U6|7rqcD01HF4dL)`YRlI9f9#6Oj8SaT z9@~y7V0INpGt@iO_j2}-X*WwznJNaq)7rsQtrETur&u&|w>o{;OWI+NFT}!muj&F$ z0CXaJwM^=bQFqyeR!6`5RG=NB7m|MB#5`vzNi+bUG zVZ`Ley#d(=^-CAHGqd-hpqUDbN>RU}qg-Oi30w1ATey8;GCk7HBnkQXh!3OyG-2hT zowRWE*{StNNwNSwdcEh)3Og^pk(6EAMfEm3@Sn|NRTr_+KblCH@v_mwP|U~!r6uAd zf@;j;YSFFcA~elao7;^}HPQ-$eXUHs!_e_oC48=QWL^TSE+^rXT>KQV~F*n($IeqC0ink zR7IuwGw3{%b?})TG%^vu&9`tiR~j=t_oU#5BByD)SeLZBBqK1`2c6z~gX1JAO*xbM47;$6F+8Z+6ptZBiKv z5gx+uPa%=#qI>~5sEFZ@(9rs(CRfbvrMgnc%Kqes?*N57!{ zO}Tq&IW{@iF^lt1<}L#iz&ft0B0@-H2dV=UwOSl_c(`gYqv(t?j2YLV!Umgy5wY6L ztCE%8r)9oHnWO_OFrmF%s%H>SUO*vj&Ya5bjDl`C_?`KgE;oCk8^pia#*yTUsYoqT zP(1Cy$oiw17xE3_0;nYWl}pdnR;nTf=U+O_#Hnp;{;R@8##9J3lUhRN(WnwDi@b|o z=3@?-S`g+3<(RiO-knn&@9bw^IQ`y~4osMjhiqXzvELplF4zx!Nhp~b1Ieb)82oS^ zF}0%3nndV3p*wSk&lvl8m)IN$LnT;y`-bgFb`(TD6A9XWpK<^Y-A85QJGjxPVdD`| z4sPSb&)?2-&Gmzfqp=}^C*zeKa9lxpP$#Y16#%SaXi7>0x*q>P$q{D}fa)U{Uo(%! zUV;MU=~D7N?4d?D3iJBmfW@rnSxLn8$}^S8FKUdWH;&HZ^Qdyt{_TI|f0s(H-@?6* zO_?ebRhr$uFX6(w%wj2>2ARDRG@v+rklW)F_YUv9Jn!^raQ=d$nu`bKDEq$U`bOOC z#UG;2UE`!&cXnnN%k36Yq0(!AqR@Pgkjf9grTS!Gd=hqAo^-)U9yCG z+H#-XEOU1_(qwKO`X5!ad6exkOpxk7a#xTC1NR9LT+S}#IyUnt&==H|Cg=teKqIn2 z=qkS{5L1u@WAHXKAK#I5K>9?}CE=UwN~nts4YwT(4h(VlmdiEwrI$}dVca9yXa%K- z4>X$FLGWPTjggcN%p`g`E^NQ6`L}Ofi|<5p;cb@OY5EV_A%7Tn8UEHj(9v5wkrz8L zQ%Mv#vyhO!Y5~AG)2!eTX2l3U>A;M3Us}&Se>xpbNh>Y;$AM^tJoINE|5v}E_a?_B zoE$nUAPN4{;?P>L7ESw?AKq&Gaj&+csRHqcnGw8k#D*PkdkkwKAD^z6I@J*qG!9H% zT46t#y#&ON!J)$Hp7WPiXy<4NuSyI_cnr;a@~Ss|HLBm4(pywfXjH2t9_KBIoCUnO zvU=zBA-a@E%9_RB#Xp_bYCmUG-|xv)26e^{p{>kDqE}1t*UZakm97~Xt2SOg#tQwnxrmV5riNHMq~A|vpS zQu7nH>PY`DGPy7_W9dDoHLNeAnMv)bE(Ch#yIdSFPxr|cK0ItAbX>l@cKdq;>`?Yi zCoTgCt}Hk*q>uKsYsC&ZD&$*`lj$(|$65g0NzFEKpm+Q@Y(c8 z2G+M3KfiJx@^AGYGU-gBA~NYdNbLSBjA@6A+DlDJs;d2Hp^+1{_MY@>wWPJo&Rx+o zRQb7h@zsJUA{)?s`~Opvx}8^i2KxONeQhN)uUm_BMkS-1^Z6YJd=w|LM7VQ+ppyVW z<6gqhZyYz}DNpXDhJL@SS8eyg`I1O?^cl^~ETry%yi?_AtcaoOiUJUx;?~THuSwfO z{e9r?dtu^!n>WO?7xjteIyRiguE*~>I@Q3!=??VGaAe-`BXZyLjVde2Cp9|MWf`C# zwqQdW#N79KcAq%A0Ey^Gnh_B~Q46`VWyx3&yuGG$u9xt*sJebElD4Bc?rZ>$4u??> z8?aEI#S0k^-PyW~5%oZR2=22lJDTI^KEH%#n)!vKtT7gUgKr<(B|ZesL2i zq&u7l?P<(~@cjZiUhdM#M9fWCxCEo*`KVQXEYVU5yA6se%)e$R5 z7@8XMW-J!7Z0X-%@!B?B(A7?;W*#q`5*-T#EpLQkgJM&Fu;`o6CfbInsSs>at>XhU zZi){;ouGuz{4r41di5#!kqOe3L4+9@hQ>hctG95|6(TS0oFNfqsTr0}b=G}WgujAViUS2JJO)?xP{{N9w#;Iax@ixTV{?7}N) zju|{QQpnuEN44U+oKy_0>QvC3aAQ(q7u0uvMp=9fkI@mu;C`G>R$@sdxX=sh)m!x= zMK{_lNbOo^kv~wV@#i6qc~;Nd>u<-eItD6#mKa~5-RX^J%-s@JI!F}`IV*PThaise z3aTOLHtaPdrVHhVSubQp%g-YwDzG#vT^^7=#jAhf^2+aR@_2fRxw}$s%G6B%hTRq5 z{Q*H>y3)gU;Qohn{a>dTqkf}=PyZQ!T>6vh2nM6#lr^VRn8qb!N{*~F$249@B$a{U zA1ntxv@U>p-hW3<6#y^JaP09l(XD#8lnrc9K@a=Zj{4Qmm76(Rz4qB2X@_6$DTgx! zPPBq+imdNXpbgQN|!r__B*W<)~!H|!}BbyzH2s1nKqy*-WWbkG2 z#H}mgvw?&+Z7f&4NrSzQ`?o{egjr(1NtIwwom^L+y^xs zkYu4GG*bDTOA@>ZT%U_*gQjVQad@xNSX@F}Ee)b4Ojv|mn&H}bw;+=Tb+__~e>v%J;L zN)-S@#G_BIWcJ=ecAXwMl=yw`hs`86;aB5sWoKwS+%9X5v#{PhgF4b$E>7hIyzf1p zO2!{QZ@&oM9nvjB2Z(EQj3dBK4*0p~&-}8s7i$7KZo=4mlqi?GrhnBeD*b(7xsiQ? z|G_MQb~ECHbJZeGr;WbeZOLsQ7$d(Hz_Q2{U@ zp;rea^vUO?9u@F33&^4YQ?H=22|nv_Rq*L*J~X>qnm(gi6}sWkF#{V$(3F`(Y-Vp>)Q61)JN%+1>y9T`uH*v)F{JUdOoRyR^61kg#2XQw zgE7g3=%1$tOX#zOLN^$Lnfh&0Q1bqm(;2v({-208(p=IcpmINe8pjoXt;&7$)uji7 zrB)mBcOQ)L@+PcDvWhol`Zjmo_Q!bw&sWa&eb`9+qmgKAS%0b-;KU^OvFC_O619bntQCq)JEpxE#`zp-#?Wi`mexnTX`&RTua)*%_l{$ zNVqx}gJ5c|tZ)5#W#t}bLbq;e_TwPt9?|>Qs_;gqE6-o7AsrnkJu0rd0bAz{-Oq*> z@1=q_G9dvBF@tcSkW4xf?Zv~&al@tjR^Uc*AWB8W%=j*N&t2R9b=Lsn+e7eUiaxK_ zUMa9W&6gdc{u^M$`RKNhgNR!}tQ=l2t3dN9@;|HS8TfcfbFXPH1}ibQYwmcBi+p_c zkBbntZ9iq;7Nr-t@O!vxFwXh$?|>Nt*s6x^FCT3&Ft@nM=aNqq17Ca%xjX}11e0l$ z-i553s0}%x*QerbJd9V_()R+l7{3YwxF_iS?Ra!a%)@_gv(SJiB?E%1Dd4Ev{o|*! z1wF91*u3<9-TYCl+dDEyEB>9}@d+k89<5bYP*%|(`Cr4(zjtOt_?=*%fJ5uiI(c4C zY)V8!RI2Yo(w^B+HA>^tAM*|v-}WbT3|g(1b6Tv~OaJ70?&FGox{g5@5ZjxP2INRK zBNZ;**?S{1Njd;RY0FjyN)SieRCRo~Ur#FJWB2H_|Bt}|_`zeBX2kO@uBHPVF3?Ny zDc;?P{+N$|hn~<0%OdizYh3%+hMZB2bNq_IixHL$8Coh$9UlTN#`3Qz5R6iq5GGqpG8UiA?L+z^# z55k^g+0ypHyPe*ctz`Tys9QY3_cB?^u}BzNkq#M*=_-3S)q%LkkeBNXrCwS@k0OGK zhfALmw&`BZJ!R;>{l2i^s=|&tX{_mGUW}$`^VR!}ql!#(v$0zr2C!hZSN(uzna5 z*t5uo%z$soEYx{kLroI>+*W0l(v?ns-oNsoU)6-^=$)W|9Q{6Gg79)o%<=s8BG`DHLGJ~8wrAWCmViFEFcs26TW@bclCK(ZM%#vA^((sZO2 zKcio1c-&&IYiP&|LVIXux=IF^Q=i{%09WnC>z1A(Qi1lU&BzsAp_&_p1TYn!oL$`S zh|mGQHm8Z@>pcM`2*DR6Yvg`nCG zYYFQ{qY5ku^cPAEhJ0VLtL4zCt-0H#My#}xMnE09HC8l(zh7J_yZwwLxY@A zMES>HqC;ksXw)??Y=_;^dt4jiV>cGW&^Woq?(`qrmP3?)H@FEi&m1kGAe%N*fYHKX$ zxQC1Mn!ZO`y&2c^Il^5*R}g;XNU2AY&5m}yRzr<7v9U0sxWq}Xhh{kc)@}~^8*24= zS?~mH6}E1(EU$lee+}`K0J1<-yl7F`7yQ|ypf;mqA@^tJeZ?@{Co8yfe zHhEVU+f0QInq-pZqD`o_kSFsC=P^P$mC-#_g<^3CkcDaJdgWT-(&EbQxirR)zvV1Sd+@gCyIZh0+i<&+qpW@eN=lH=hWXK?8Tf2vMfrDvOACZ&4 z`TMu|=Hs7dBEv1Qg6DcZk7VLWlUM9Vr%Z5YSoAyRq+~?oN3D|aD9zqMi}5VWbLcrm zKr&MLd3P~q%5vug zVf}c}$Ao|ZY7x3#QmsOn=8z(wouu(I+}BG>Hk0wnoh5Yu%=+o4a*hDIM~YOEwm(3> z@$>=5BZA^PO~K>#4rX9<88q{yL&S0$XtF9hj3l+!rmNZl!f?f?%PzdzZWSDE!=^=6 zJ!lbH9ni8-#LXhR=`hQHzHRQRelhk8Y86brf9FMm0bDWe*GW{HVgNmA8HUL>33QPs zYN`uILD@#hb%0Y|ySz%A7Nt9b&6T^L{8GvbZ*o{M6ST;);jZXsm ze7}t-zlPQ<#H|@Jub3a{{YVk>-Ml!k=fAai%Dq|3UC}Ko7e9mAh7ql@gJv4p0O8^L zEj_9svfNzmkK(RwbHJ5`b((1doUKvKe@StCnj08(E!ZxlK0t7$SN!~RyLtcYc>IT> zJQipI51-XW3xz8hmoS~1 z&iA`HQvC1{q%V6asWnjfa?NH3D-ytoDZUw&HRyD7H_fOoYJiq|nxdqq`vV{g{nOT* zBOo>9NeeShLfDBe%gKA*)zKHVFAWuCN>VL7`qgdfv1X?ZJMBEAyAJ~30v1@V;AeC~ zB*g&1Kx9P{`AD$kv?sBeQ$*%;d@8`z(E| z(}x=o_4pKDV=%m>11Qbw&$5>wK?7-Z8OdXHK1UuN2bdFg=g&j<#i; zI(sxBnaq7QTE6XXXfh_I%~~pmzdVDcIY5wLl4sWp_R)ggSGii?cRa}xZ^-;wjR5?z z1`#Lk;2?9lIOz?v015V)Ujm-8wn$X|SpM19i4Ohs0)}UqmX|IFVzfr@Rs68(;r6_)|OMV z!q|O;yDaV2=tjYFtR2YmjY=kPG8_zL|3NqWxg3qdrLo$%!=Zymp`ykQ+1=|G-w^|4 z8+|GDVcaRwtjBhq8H}%^`>HI^l!)M-ao4&Zt0ODN_2~&~*7qM~T55X0fJ%NUq=DAo z-t2Xy&PltiL~T3H=Dr&XhQeLy1&%YnAOcLJikGIdM0h0XJY~P(2&+1#Px4I0wZC}q z8Rf5qc|$GII&S)iLEl*4zXj~;X780lniinyYYNP0Tu@~}4}S1UgNB@cN3XDj5Mz1V zJpodonpQ30Fvzi4)N#idjKYtiw~8T9sArQ_E?&uWIeYO1S0>r#O?vzb=ytP)JP|Q; z{6pl+b@HC)%EZyc&PZOgwJ{hg!Dh@Tnx6U}ZpG}N(bh5hSSQHu+`^wy%FLJ5ti{RUXq6Ypx>s0wp{95reBtldR;&T`E_^`>Vq2BaXKU7b%iBy`kF5_5U(0liU|~zt zh;(Yx(L*OlC+Nwosmf(yxbgiIgAtf3z~k`pjIj7;ha8P*Typl^wrDT*I?hj+P@f8H)O$AJDeJ9g2HmvkGS?lNt zl_%`79ww#y?ub|CV}#kR6v46k zEv6P^$f^PMnL_*clg*}Yy{7bJW@HmM(}2!k>8QkEzgi}^fbJ&j|C}rMReK-E9zPOT z+;8r8!>8+^h=3?DG7q3V#vDtUh(yTksdr18ffVF6F`^V93pLsk z*1vm8c3la&stS@_gC?9KY|$ZiU&?y@Chb145ke>jrk88$WrCE+D~_oECc$FKQtg8Z`vI1 zZ!~ud&lMboe@A5|F>O3J!-GcxwQs+zcUMl0F`?z@SV$5F8CXZ%-IEA|!QM&x&roSKRxKCsrw!d~7t%5OFpeZuCMw^PDyua$9+s;ue@2j%ht z9k`^kw9XuD&hh&8`VKrCcDDr8q(+d{#muCs?(r~;3G0Mx@Unc>x2yksEuryok>T(MBTd=t;TBYRGri%|(=G!BuqoSb6 zx(=S9qbKWzprFA=qR)dt67xq*96~q*KQMc(h@T`sEse}S4a+CYBx`~wBV}B-9+J=O z&D3i!cWv&W(5RrFQB2JMcpG2)&6U)Ra2HL48NC}i&5mEdL#+pNAR_7aN+bKBfu&ws zV?Gfm^O!QUKSpf(P~&)#I9+$qJfKl8tB~BXOQ2kgBlJ6*D7lVG(8_GxU z!CMdS14bBf)ToqAHlB?m~pA?qkCQuA(i3I z?1O$>9`hhHateTS#rnK%Kk+qF*r*uPy87Z>r7n8s=Vj8wNOuJaxmWHts_F+ycG{)x z$!G`Kiixpi?Nei_0~lAhR9JT#>vm7|(c3IW;)=)*-R7`NGY<11S6l@APqUq!tvjA^ z*+;|@S7y(9M4O}nM9FIHJV+O(PZb&J7e)YE3=nMyC!&sYRoFZ4l|fu$dI zv8yZqZZy*xa|wKem)XoTjT|=h;RwYV>f%~GO>Ug9jJM~}fzup{d=~u$a#R*>uz7te z^En`>uZFYHm(IrR%gRv|Yn{6ReEU%U40!bTBpeQOWj6|K?d7@Cgk$1a2k!%7&)o*< zSFrtfnE)qH*YtsvvmQJ|F^y}%BoD9B>0PMl%{;`*jj&#!?E%{7Ql@4L%Fo3Kk0|J? z-v(0UDBFf(q||Xhvt7lt&?zGEg2qWVOUnI~-AVPYkx&=rY9ef0umDSu8&PTji#nqV zxl`^x0s6+;s% z?c^2w{0qT%9&X-m3BRaxi)fAEf||D})lS{{(14l%S;4gL^onz?s&6wHA`KRo-{_<` ztg-(Coxb@OfiUv8jdneJFAwQdhS0OfbfA=9o;QR#6 zHN2svW05d9c}vUFxHGsH1sqa}?ly6@Z+(N|`9C6CT z=wL=gdv$fzpfSp^AT3;*`bc1R9>}4dI;j%@GOjQ5G=k7TKZ1I|dYbbV*fMS5%$d4s zq1+Yk?3k8Ne`P3ys2YJ~kRtfj>mGFTG6T3*Z!XFZkx0PF9#0Y%k(Pzwg_)X^B8~S7 zK!fVl-HmY6+##^n?aeLb6|qyEH@IfN;ck5Y-e$nJ1f2|Ju~dpA$^cbvM(*P~>W+Og zS&1`JG>!nO+v4Q@E8nxz2I?aas1g-g*!%N9I|{jsr|PF26?7hkJxBd#g5@3L!X&)p zT;xJi)Lb!QCj1jC`(uoFN?*Bi|COrK3~C)=aPTV$yylQY{EoY$rUP6|_8fXILJQIQ zW17soR7n(>OsE>sw`wT^KgB;|bS5XXaWSQ(I}Cqs}`ZQ+!{ zgX6xt&!aHFbJfGt>M52fp9Z=(*}>$iI;fs;D}_M_3&)EFHOr)A8sGWI3mSj}EjTc^ zf-pzhOZXKhDLd@sviRL6_FzjeF3{<<-?(<}G-~iu&3(^oj9PRM)Cr^s(c7eK8t%}^ zoyoKYhe<~(KOkh;%mqD?@{m?fE_Fu2P(Zby2IN57j3TOeoZ0vM!LmzWj3*$FY8ajR zK>JZUkk&*oFczp9xodGP{S+#qV8+Ai_!BXu-}o>@^x;^)+PU7!;({XFWwFLBm;^u+ znvDl)>sm7Y8sgqVNbSd6wtqDct+A(xNaUxqJe(NUf;Z#OVZ@e8dB=Z|e4rESr`sy# zpe~^tzcD`>Mflkv&Hbq2wYEoKTO=CQJ2DVd4mc52PEMvR6%FlOA_yQ|%4TO?~&Eu?Nmgb7tk;2ct@A@e;-nJ-8 zTN_%BX~7)G0UD&<=`@_{Bt|f({>g^C%ME~Q{Rjsdh^Ux8pQd;BhHwU~5Q{hmwoK0< zWG2FKrKiTgVF_WVsAH!`v=GQmXE*n?8|U*@yQVEM2brk1YUtBxxS)oj{W^r!Zq{=# zcOJ1=?+vhWoc<1uC%pyRXpi*s)nU>NFhR;Zr<@>LI zrm#$nvHTra>5|^>(7@zGM{S?>(L2OjTdsfpN}3`E!xfURi*=EgJ3M=i?jGKtSD2ep zEMqnZJY?u^(7`XY#+J%cTtLbf98HGnAJ(pjwwe6%hLAwB47h*jZH;AWHEysZ=I4Oc z4P*EELsR1hZ$-`#t$+Z4U)*!xex;a>e+x`BZ69f z%!;%W7`<*h(_Av{uq?nMRVz{H{r3KGb>s-iPbu&qgT9yeDI?s^>LbfF2gw@+5 zYkeG99l_jK?b5Wus;TZKM243^SEC&w9lvJYv?vp44v&`?=I>TW{I zX1pm7s!`qUj4_W+!pG(}@mdN{`ACeVVUE?4T@i53PK4fj`*=u5bH0<*7(;jq-6mx^Gy&sD*8CGs?xI`=y1bD^nqTrfWF=RUjy{kpc@U zqLWFX?Uohb2J@rgb2yJ-jNaxUonF48CM z*7y`x@PiAH3vR?fQq(Adua?my79SjqoGIM7SGy;ZwL7BBj}Cj&!NX4_wF=}M_6n$? z@9$6KxYe7pd2mNxX-^OxYFO~Hlp&4Ev@0u0DrPyqJ~^bK-@F|l$|u@m(ukvpcQdHS z2~ep#>Rso7(+B!e66j5%x!+0u9;F2sXf|eMAK&cRrF19Xvc0Azr@f}~C8zq?4@;qD zh8U5BUi>E^pk*s^GjCk4D|j&FQRDXa+r_|MQRKz#?-&bf#ji)TW@PL#eR{3-k3Zyn za|~>@x399(EDAp)@Dw_1*%C4B_ilc@k9(}z|L|+@>;S={s(e191|o|KSGswt=B-v( z&h}*BZ-j;e(u5(-c8^7dzb8X{q4^ulwrxxIS>PzLxVF?64V_fTzV6Scjiy6>tzU6# z=J)gSG!&ebDL?lFfuJBIxfi<1$w~Ql1hqSTJl?lF3Qq%RqJx52|#;NwSXg%q3obT2& z+sh$$*GNip#8hp;?PA|-6Ky5GbnT4z(`Ia9;s+5TK|SYD(!!7&XSzNTh;iOHUekfL z#we=F>&%kj6r<_dEq{*!Ee8BCPk7@gX`uubalQF>?VAo*KUfl544gCDf)IP0hf-2E z2M2Y{NcfB3CB~(obbLdN>Na~gx&Vd#NWm839MR()(@ZN(ytQqieTYvEOKd#KAvD)) zH}D+bl9tgJxL=75z*6Ba3#!ZpGuEk^X2uH#OY@GrzU|d}PbJvWx==UOZoU(Iu{bog zvv(xvUioW`_9kE`T0H*n>?6xJ?GbL^!1lBiK^j6l@E&rt)E+3;HbyvY7p?MqhOhcP}=*Fj?xo9XMKK!X2J%TY2;6CT`*M<+xyNSpclLVtdD8Sg^SUmsvWc z4YI6f@3>GGSTQ}CtJNNy%l7MrAr1Y;@ZDwUWznG<-8fwL$_(RaSFfnZs;W%gB03h1Elj4YrYgERayWc13R`O7$J|`n!bivsFoXzLNA~ zBh-WopS+hlyxE9;fiZq+CXnyQzptxx)E)BBiJu=jGb`126dUfjR6+jfYpurz0*lZ| zluswv5b2viQife7mbH!j1MO}RP}|CEwPxb?EGoN7vl9+ctrvl&55*aU4&P6ql*jNM zr6FDysin{7y|Lxg;ws`eTqw)?bPJSt&eCT5e5NbI-a3iD6hto;G968oB}Xph6(5cp z7|uiPb>uqcHb^aSc3_Q1(>Fu3^Jn-;vAX}_ z7?PtBzr2>hA1Q-8{Ag2ms>zv+ioQA9TRGS!)u<1;zs6xNY#}AHG#T$7vi#B}aT?_1 zsV0S}GwP3&DZ0?1<#_88^hW-ZPy~CS8P2`B1`Ctv``~W%6#4Ry2a&LAld&!WY&%6$ zV?S&0MKf>6br{yV9ACN{Lh!`a|MwA(xVUp z5>Jwd$i(B>oz3eG85pmh{5Ox8zZ;3St=!R27HW&w0DZcc^Cro^&O)%Y=9-1V0Rkz> zOZzvB$sGLq_KS3Ax{ClA-)V z4cU_SPfiP*-*5E+rquDuuOl}Uoai1h@?$?FeAV~JXeLAq4Na;qg2O`j*RL>G7v{0- z-@X((&Ty!tMv3$uNZmH2wfO4pt`|}kkWL_k#G2BQevmzo%rOKGoX;;4uG@{Rw(|SgR8CUVGI$WnT_Zdp9zw2!dTIu(etsPsHhTXS_IZol^#M=URZZNcdkVul{ zymAGYsB>*vJp6ca{Gqct>ncQtP+X!%(Eye4DMvt>4h{HSV6*SH4PyGg=1qEW14oYiQXW{xnNNkc^OD?h?R6;BTYTskwh( z##sY*n3J^@1+TQ0F^f&V&!6o0I3lc$CIAWHJbCIjag_RCVJEYP=k*F+k6Q*NSts2j zSsn!rTX=qpl7)<$e7n>0<0Y{ty|7xCdj+fAP%wrgH6P;`9+f_o`*tV6pYeh0kAErr z{GXRrh>;OFF-MP=BTpch9c>tO`%F)ZH3PJ;{_tEA{8ek582~V8k=K0r`g=&)x<_h~Ux=AJ2Mv$mc=B++Zt3>oq>vWC;h6_x zqQ;pLKiGCtHmi``Iy*!6y-*tVA2zOskdDn@^a|23;Vb*|Q`h&m_4TpbUp7)lL8%#c zoD05ZGMm<;N>kv;Wh(JnV~YyUjL3F@sdpDT6WwHi?e}LyYu10TMAxLBZ#UA+nL$Dw zMcIq`fB3t5_m>$>Y+Z%UPW=>Jf7h8GbAiIneZ0K)qeP1@Sb8tgnW}OE>0P2!AT15L z?ZFUTs#s?WyRf*V$INks~7pr6|;oZqz z-0y1J(1_+ft@XuKa=)RoX-binOIWT>s9UT~t%~4W7GLSB9lM!UwS>eaT zvHGj${v3+eSy$4_&Gszz8tdQM?xr@6+jb2R{g{y8U^D60(M`rTKF@A_gV#1N(sk$r z)XgIPepuNv^bT~Ni-SY|lECz{m4eZ+uX!yjAa=wE51yYaH9d2yyB;?o6i!v_p1fq$#myC)T`b#opU758 z|C+b}E9?tec6Xc*RK8^n*G8CirWocKrN*;2R*5#Z^>SKMk^Hv+KQCgT_Ztm$Rk@HD z#@WJW+I>fKrXz9=>kzETDS_-Jwyd-)@gHZQe7@LjKVcZ2be}~s$_Pk;5K0)ilaq1oEOcO$Clf3Z4y7L%|^hxif>b$evIs58{`FDS48ug^K&biMN{J8fkUC%_O z%%+upK?2*w<^7q)@TtfQ>vfyAY*Jgys@TNx;2foDAQe0|u(9MxtkTD!WviLa@7 zrH_!Loz=*6jg?vx7d?pkPzyF+ky_du~0ZE>f#dvOWw?heHv1a}DB z^xJ!%v&Z>!^C!v3O4eF$ratp|=WNm=Vm;ih$HKsNu}-*UB-v4yA-4;Cm0rA8RR(H> zI-R!o`YwvsBi|1s`(s!zobQw#k(?f3^t89%cr=E21dp&fq?|!Al3!6v{Y?4$D4sEQ zdy~BaS%J0c@{a2e-tRsxt8+jsWKTNf6qG$L5V3vq+l?%kyjM9r4R1h-PA#nF9aE(j zNp@eFkFxPVDf1$?1`Koyn#f_1(mlp{$D~Di%$dNf|N5rn+oe z+u?T_6igU%5=g2BtT$+TZKt#tracj^fL=BZbttH6e7+{~q^8!Mc9DL*24ADU*9&|4 zBLHF{>EP72Ke+$1;P`a#MbCmKRBeBvvdQY=4+NFYpRYOH+U2eC+=qt?Y1|}gg9Req zn^i&6g>X-~_EWHMi6*vBlV|0$g~9`PX}|F#Z$&AT_MR-EB3=VPbxJ12l7qyZwxPS$ zgEw0@TQoO*20i|XkHRlr@O4*2ZtVW**5K(=)Fxd(aJWeJZ(N&<0MF;AW>?FJtSO0S zSByZT>X2`qB z1~~3I{>>Lz`^&l`AbpAwM)eflu~}~aX1}UelHaOETKfm$>B6xiBS#MCUj#^FVJBF5 zv;8yb(_hPEu(8ZvsR^6*;AZ{ck~uzNyV)5PbCT`qyi}c|l6SRB$6Fs^m_dwH%hRPN2?m0|NUwUWZVx(DGTcZ-MR^hic$4r;g)FWTP{(qx%Y`Y z@tiru#j;y+dAx~=xwyO!qb9}Mf>r81W<%4TB6cPJHqFJK)rGncH zm&w3^S&j}b-Hh)6!@~@G&hYz2MbRjA>atm8L|)&YO;gm3$PCc6gKG+iR&b5Vo~ zVO4{$!>X`+pW;7eG&(Oj0w*i!y-I9iOUcUE7G||#=P8gu8VlKLN2q-18_y@}ME<$5 zN$ZNHl%K$`=7U7EAL-;jTMHWkvl$Z@S4w(){ZuB_IJ813Oas*d4IZH5T{^rLqF6h5 z%#>HVwD7b&QEXq=KCej3d%g)tZp%W;+3slx$%bp#MBO;?C;ngK@iKZZnmn*df-q{# z-+H~9_?RpNK1B8S)4&2UdprK(9r|=;MPk+dtY1FbY5U>7qXYncVyOqD(%^dK%NG=s zq$EIb1OWAGPTQ_SsiBMJ*U~6kD)bM` z@L2-Ljc6tCo!0Bss>5~66M7-)s8mh_bWY5vLu&!Yt=o}G3d&Q@&H9kcjDv3VJ>Vod zwaPK=J$w*V_JyE@;HJ}HCB0kmijQA>nzhFr+&0Z&T1&AHJ-4faZ}aWX9l{`!vaa2+ z031FC@tS+-Gq{Y75&bdeHWwReG#jMsu3a}s_(Mc^rJUK6Q&8|f2ma?PA7y@gzX-WQ z>pLOJIynCfYOmVq;u+vRGn&nlRH{52ar~D(!|}Pp5#tcEe|(ECLIrLdOG@A4BSi`e z#qLp`x!w&oyz|K@MdP?P!Rck4zXge1IDxl;W>*?sz_3sa=g!Ej0e+4DS&n}n1|XlIl6j`3tv2&b zo{K*P11^<30upS^WIoaLE=X$*zTSHGJ8Lf2B3OEJ<=}3ei*S>)i^Xs@Fl*wWH98An zUaP!L`x*Xq&s*7#?o*8;)_>*~c;Lux&l^g)xC^SW>HFz@nA&pgl%#EmmT@hc*$*(B z%CCn`EM2>5TIoD9A@J&!toEHJxsccC-{IW;e3Su1-&(#ZYb(@86IZ&CXaoA~hWLtm z{6gD#5E!^V4$XO(W`qvM2KLy-iV9v6JoMqT-#{e>)B2WXz91^OB0paVQ@CNeU%KP} znk~Wej#!phswr3%(Rd}d3c2H>5Th~g%ei@M(W2%?Qx8_(Rri*#FN2M_m|C#)EHqE< zpXn+MH5ZNKX8PcN^awM=|7W%S^R{Hus41Ad?Hqa@s|PO-GZ^2oD zQ_vUndu4puSZhHmOgIIx{IxrjJXsumH11A1|LO-I;|P5nL+&qwR84M_jCa*N` zGR0uLYV&J2<5O2ZSIauUSWIUV%_f2(B&W36P&e+$*Om*2h-sEt{6asWdQxuKlU zVKX1A;_hGHzF+`rLU{ zdt!}r7iQ;!x$Nyz5hh*aU98Iw)i$DbqGH=B<2t02nXe@M+Hpd&Mu_GJ756|-?8_ek z>zk5^pXTd2XLx{ATzN{s)qZmnhX5zkx1+hY*3H)^%UEB4<6{m`ItJ4%Uw4 zgcb&2L=g>m@eqyQzj4Avx#TP?NJtbl5Z+6zi!@ zBf>i(L262G@Y;TSNfSg^ifizN+tM?Be6n3~<^hN3hkN99Ko0|{KQAEb>FL%vPO;SA zZA$s^(NO3_Y$jPrhv)O33$6sWb8Wi@;k-Nom>Bf}Q~yfNbA5t&!V-bX4@b|e?Pg9v zlK+*G^ErXF&8J-3^TUxfpB9^pQ*$@B)4~~VZ)^&*urM+Ob#<9f?>anM&|7BiI=|`H zB_XW^rjJm$#iXMg=aB?j=blsUN~1ZU8KxG0EfBNcPUI2>U6m@uiD5+_3?|D53bb9X z1`U?LJuzS(cnyBoKJaEAEHkjOc&XZl=Yd^Dw$@_b;6R!VyCs-nbg5}QZTkUQ z&_LW`Vu%qacjw>SWN3hMS{4&Z98d++Xbc~fj>p$iTeE(&`rJloHDKd;)Ni?Zy#j)| zjSV{StM42SeFoA4TeZJR3nv5^ew#=oiB}mQlzj3OZhvj}Q~iy(sglg6OI-B=rrS3m z%5k*aEjqq!QtFGLiJjJshR=MYP?#{QwK+ z42V5@|j*lB~n9pK}? zBS7L*YX2keaUa02?=U~Wi*6oIT`s3T^kS*4NNB**P-a1*f49ANm7+{9WjxCDum!U{ zcgI8|j=I;i@2z5O%Z@C*8uMsagoXz6U$H6ZeDVIo@e#B&HdcTuIKUG$>I&i0E-djZ zgCg3m7AHg>TtP(nu0JZ6W=O1j^hH}q=#Cf~uta&^uKakg=l^gnk{81ia(C+O^KOxI zN(v;h-!+I|iA?iY$Q&B&($SuIk2VuJoEgYgAZqF7ePQNkQj1sULE5%z9@(L+O`7->>Y`rcH-S>~Utfc89_0c5{?M_pNOPXnZ7$7<7Xa}b3S;ew0 zvxZ^d%G=9+(ysgsB1bF-tUoik{Ux3~>#}r?pBh4LvaV|`hYsrgWzDu`euPl`ubjhP zy?n8-9E+1D3Ze^bzhFf;+Dx3{KtFH(m=)_#IfkkeO{qnih}1}|<4TqpraxieL6#18 zA4sEoVm!6K7izR!iWq{Rt`MLsZ(n}XNs4t_Hv2K4gCF;=Op*u4-pC-_h~hFmgF+w-`qRx=VmJi9N>m z()`h7Ib|Q`obAS{14Gq6gG7 zg^IKl9r|hp__7+>dd^jz#nKlM(khp4st+S|zr1^+mS0g+N< z3zl;d_{BX|TEBgIFzWqohdi_3sAIBt8aujKps~37UW{aCrbA(`XVc!WInCXXq{(J@ zut;Dt>9i-wdE^>0e%01O=Yz#9OEG!grMT$tLUm$)^$5@nHPC$Qy1-w#1ZBY|BRXtE zDuh&H{*0Zg>uktROt*RE)v`BXYJp_H9UZ>RHe2O&rWDJx9rbEU3I-h7{nQS8p}W{W z{NuCXg+&s};iC1lHf#p4N>rFLM)d7dsn#{>_~K1B7PGiR48|sMj|z-H$(mvD>5liF zIC&4p#uj?5F;IAVYI~o-JxPY8#*9>^3KU=yM^{f(-uhjD%S=*hy;~;kUflP1 z480`#s7-WX0$9Ry&63u(^fHG8ZKAe$tzxi7t`D$CHBCSovz_CtuK#M|DraA9_jCe2 z*bscfZ;bz>=?Ohf?%50g2X5YtZPxN!2IMhJij)jtqAJl(jZ4|`%5Et7>LOkwP#LTxr`rXka%rW8HFZW8 zJZEmFeO-dbujj-g^U)YH*~YEFJ`!9ul%>8M2vG7`fj<0^c9h!Rfq5}!q+=2fHw<3Ry8AVjoID*sw+JRrEsbDltkC z+^<^UVK(9BZnJ>c$rzPTcg-{KgT1$DzxaN23Cf?f>jlSyV`RR!dfgFek7d20;qU7U zaq>3S0jFhk(09X~Ph&#Vee~=8Pnpt+O3`B2h7l?e3!OC=TK;%?hqtz;hd$0IZEZ5q zK2Wk7A~66C5W=EpSY>oJFHqvGO z4s4nG3{A#vH$*9NJk>k!bDL#Ju>-w7xS!r3^xqo!+M;`bX!e~h&d+VisS&bGbMEo* zG39d2(W_Um4VY1HKit^BSg{eXLz0j(N&NL#5+Q#2{Y}2;H+Pydi`x{%hYKFkBa!XaN|@cYxx`7(a4FtgB_2Azj$xM?&Se8 z3oZGc|8mt-VwUk*(yzG46=3(-4r{)Eg_m?SsNxX^7b6=5>Z(w3cZCu){!j*~tKGzR zJ^tM5_lwb>IeAK@==SI$8{bJEQ$U&=Mzj8FU$D}N1QhE~H5ok^4fB8oc1=8&Wh3l& zL%+jMPfxN?RV?(L%|=!}zk$YW<*tWX3FiY{%g2oH{b)l<+1jZE zIcW1)QF{G)Xn7x-zpPaaeata;ylI)o1ZFM z0A3W}?6JmjDPBu;S{-=npDN^Turm_&;Dtx~;lY+J|NSd~-1jQDt0Nv~rA`w5$75Zu zE1hqDsRUf|VjXOi&E{7l?y%ZieIWnQlYRcypiI17JdT(cT6q%n- zJT?GgA|8K8N$wxN+k@5~T+Rb*Ahf6Tj{#J68{02i+L2K)GJZ^Eq5ld_&gohNTAWV0 z2A?Cb-%a#v=XAOAY)I7gC4Ney2)Gj>yih)PsXOEVxHa)b(bMg(VeYz#en&y8A z<_q?}vPvz&C?ut9dnNf4&OqRMGn$2MbDce+QMz2;>M?e3(A#LTf0idEJsZ3<^GtHP z_4d^;`iaC|?u|uQneo{;jy(k~A{FyfhI9USG5Hatvufs^M4oJF1fBL3@~*0Fo)B~Ii(i~wFpB20S0NX zfd;J4&T6EWTp4<<{xb6P${{t@D@CGjKE9bGE#+e{7_mW$;y!NaE(z%{tn$OB7Uc*+ z^~~Cn{}ztP08NTu`nJ^ClYh1Km2V6X zDle;lb>aP2Z_*i6l*C_AuEw^mK=!tZ;gWCR7X*<*KMR5XgxJPTU56@4k&;L%y`&!| zoa~)Y(=tn}?OiG86ZK!zt2a_ox}9>Cqlr1vudg-dv&~L4Da$?egt?r;VHuI>oZhdx zhsU?E3xR-rp+wV_?i@)=gIq>&g<%uY3xSOZG7DGdjJy-B+oB_V{ZP*EwS*VTiA^P#KiQ?~^*q zwL`IR)kox!_Fp%1%ySyZ7r<2h_Hsu3+6zRGq32hV)pxh&pK4oa>10U9f=z|k45KPB@g3N-&+Kqk zZ-CmoC)^*?dkJS*QP>z)?|y6#^S=wZQ~p9XC)#4=mE%oW=71~6wPu}(%WNUHmz*4Q zY^GSUoRH|0Wcv}Z(T@6e3>)&h1IfL!32Mt~2w3w=fl<~WN@TLNmRng|8=iHs`9~i2 z(;Y(NT*|JuQ^#Qvo(HLuxxqcsoA7j-;KvNCY$6`Tc5+R{l_9Vg9lv*cn~8$d-Vvs6 zj6)LBjF(fJ&4QA1Fk`z=7Amys%wB<<$Lh(l^;;tQV6JL~jVa3kc?=1{ zor#;0z$I8&4kvf{*Gh9sx2{fY&GB7vyVj@x7wFe+Dzb41wio;Xn(Ny^mJcc4(8jx>3alxgcK zaj7usqX?~SxqqM_Ddp0JuRA>}yD)*KXmhidC&K}$ddjUfoZ`2a_pl1bKOh*;rTVI% zH>7M+jG6wk0n#28^d#fj*s0>!Yo(K=K=mTnfCr9IgT>753du)3RpTAS;<59SjlTK8 zL%zfFt#{|n#@=xG<5--7gwf>4AF>bS4q`_I5VUGB3mzqFwD$Ab!{LAUmdY^% zvmZ2DTU)Cle2QF*B6a< zA@$dQMV@|;V=Y|+jgOK+7E3FoZos7_)CjL#AJ}l+-##i|xZuXyS_EiSkXar-OO692 zY`QrJxdicOL=Af=b?P$LkvvwS)m1dBxxnl1M>Hqj6jvG-6#vBvqJEkNkJxMnPgVow zKBkK}@jtId7c-o7pSpMD_p!g(NC;QLH_oz&$^)CTtCmR;P_rZMJPHzc%zt3y`FSk7 z<2m8o|C@pzr&cf5r=UuU{>`n(%*Up2|E<6t=y-k9a#Gw6uKl$EYcpNK&i99_tg9~n-^17T4$aZ zDFi@lstMm4PG@Z8OyyF^43#HY`!8=!pq1fW#)v|GIm`Fc=R{=QubzKUSEHSN-dK&Gz%tQI^B=vnyxxK+aB(~&2SY@ zQpAA|i+On9qk71WVnPyk2pqiR7~{GB#evvPWff%O_V^;dJ;WsTbjWcJW*8{>vcx~4 z^m6lB@+mSz_oL>*RoJeWgBWxe_2@1!w%!3Z)a}Gm0gp?tMPowKK<-CR`60|Y~OY-uf@i$b6_-pReZS_Kai0PF zgjxTsxbg&BZIW?8)uiwj3cl2G*N`E@Cov#6(%$1Az$^Ds90kFm4L(D{RAYE zk@`KZgdF7-g&dkakL;Jr7nqf6ZFpJ-H$lZU?dUP<$rFXF%n=(pG%K)B3a)t31c4jX z8U9j(t#d4^{1kyJSnx3FG;b-w%JXW~;qwks3TqBW^kR?pgysU&oeUV`bddNU zb5#@=P9Yet;uMtL`=c$cvH|IbIO^Tg2L5u!*+orc*gW0Q!=em$L=FE8&a05^1-cq` z$rjrsH&ET1FYtiQ_|V_jk~KsP#~1S1xVkAq0?fV96Ca0lqf2Y%roIJ|jEXs0t0itwHDAI7FRw4PgP6IS(rkVEq<@%$?(M><{({p0;{UN&wGNv<1)RaZ z3XRV#`qet;hHAbCJ+6S$@6+DX*Gj3)MA#}6^<3)Zh;SRgx#|{Fc)CvnYuS)%=PUtv zd0%G{`2YOmk5kLP$i}IS5?z?Ekj5RDCPt8L{S+ql`Ps>AH@Nu2kCDD5{d)8hQq*TccdP zYmX&nQ2VX98dgGE~veHPh&5#qR(-Fgp;)?dV=bZ9t-Mq+CEU^o(N6tw(5-Rir_o zG%EJTw3O3on%zI?>Mf|w8$2vIFQTcHJ9b@jRCd>&N|y=$b^1N5Z`2jh_a52Ti;(KD z6g#XBz17|%ynDzrP&V6bKTDC0JC}Y?TBclcB!W`N|o5=W3nu9?FclGY! zIj^(ApX;o_`sJk`?IwGbj8%;ww)t5R0_p`N>f5Ncc26D3Y2b!802SE8&EG}bKrJF5 z7$6|bbpDW22%i$mp81O&hcKI%(QFl`C4Cl?FTL`mgRXajV>r)E_2oz0xA;A%NpqJ= z^pY!_pt)O=q*;QbQXI7tEB7oV{;%DV<;3G<@s*%F1S;>IV05J~XF;}1CsffZxc|`S zM2#gCb7djLtypwEfB-5&xTjH{uU+D06#Z|tOBPGKG7CFg+Hy1Ax=SxLJ2^nEQ#o>8 zkrROw$+0Z*P94w;jP^PzA>J%^T!j}=jH%TNAoe90mQ6dMGzD{SaK9Ryhu z-iK*!n6MP%hMO4!W|2C*$ z>KJw`f-&@>aYt+Q4Mg(!jurS1iKi&N657Nzk|A8!H;HijL6((=8*$Du zXy-W@-H?Tn@t$10ZD$46;qxMlM-_bZF}nF9YcLjg>sAyOf-^;1gUl1D)}h50^c3%E zg!gp6HA-L&w$>(SW7RmeC4@10)MwfX8Si$RTDckkvL><;DbD6|o@^#Ga^i)5!oF`F zrisO-P8B_7v`Jh{;8Twne^Y23wfj4I^2;|t zq?Hv4QoPF0X@c&IiI@AA8nGAT$MzTn8{F9*6rdA@*}{aA{_r^Wr{5ID4h5_5^$ZKc zjP|JE@PIdX^l5u{yV0=>3vm11z2ta7SigsjBX)Rf*otUJ+HfP=_kyts;j2@2A6)uW zlH!P~b9e%;sYWarfG>@YYN#C%1v@k!L>SvApdwx7a(hr2wLog?USIj9^J&Iom$>Wz zp*_1*?=k1I61P!K6HypBvT>{$7fl@PJlolNFLJLvf3BB8346{$zDN+g4#6E-0ZzVcy>+acV#F8}(mK|9AmeTd+- z_)@lu)cVJjFO^;Fes)7Pl+*J1dj{+aHxYeCjRQS5bXr}jueH?4Z+pxbC->X)e1IBZ zjui7{0$3!6hpVMb{N>-3cepnevQJ)X=3d6cg>%edaVwW1?AaLN9EEMI{@YC$^R!a%mtrs7 zmw&eQx4GMmN5=|4(t?uh80NWAr&6~o*NW+5!Q`&g8uAaX_jZ=284CGH%sAQAdZp@< zrGujorl~DWgY$&8a({i(VnywNBgvla8ZUi|6z%J-rv^_XeVRt*3ai)D3sn$MB1M&^Eaz(F2{qm8muX*x*n1yhGSl>4k>2saB&u(*`;w>F{e&Y?lUP|kUnU!iB z@kftnNbcoVYAVlB^J@&r@o@7gyWr;3M!tv=943()P>cvpuvL;#Rv|R=3BGkM{158d zaA$GBq={u>SRr}N$ZAVK*t1jhYQfgIr_t5V0=aj#-EEujspZfay7weF5d{LfeKNap_Fxpryjh<$Q-j^%W(b3}SPyR0V z#;qnXJ}S8iS)$S7{3Q1EnQ)I6OAJB!cXJE*L(i?wienu!+(4_L(9Z)nF!e)a)EM@z zjqfTS#+Q34Q(Ups7h^(m?DKp4OD7^97VGNCrw=Kt+(&s{DCw9KATm#afPk6?Cv{rw zG5Tj#vAx?A$_f+7oJHi@j4+{Jhi9@WrN&~!UNa%i;gW1GDX);ov19EcxxQw%xLeyK zlGZi@wcn2SW!}oZR|wVF=Ua$6!C=8EN!XZL#2NfruXT2iqOxkA;G|m7ovPfFC4jRS_0PvlO~ik1JHdkpGx`q`IxAJ( zj^NE;nE+!?OBA?%N*^Iw^OeHV!`9nm8EZTRJdH0g%Og&Vcf&9cO*y_+JIt>Ge0N#@ zK@253aDTy2@iRscQCPj^a$;cC`dF-h9bBK6G^}x-=X!P;aye4xo@q?7V7%*1$2f76 zYgat_#OFm!J;Lz9u-lM<7Rw%dq9f`nrfG)Q;6s#6HeAdMLm-cwysLJG`AELj0^Av@ zm0?*@Pz9lvjz*K&?(GeKMuX5SI~J-9qNdTl>Dv4z`<1L!)=C$>KXmpZ6U5m*z#?#X zs#9@_&)jl*!V?(sYNNv{e(~$_2L|#);{mD-%ReWbR4s8H!peUJ&7pgT*8w`Y>A_B&`)E^+x!#5CB8e355%PJe!Ko}0L$UJ`9dEfgT3LAR^|aK#E>hV#qq3{Y$KlhX8}V%m_Y3Rxowr(aIOR`o%ZXK+|0^018C)W@&9?l0! z#eA&g+Y^zcxXIUnHOJ2nVlP}Uak8{Rd9A1t&%LmD9p!{yk2$$Hacwf*pSeI4E+^oq z@lE0KvEZU25=GCBb`3*dqjcg~q$_zR zu&?r3`IRWs2kimfSrwa&qE9rvAOo|hUQLcUBd^5*ASB9c{>@8&Wa@C-N}ZfXves@Oq_pOMa%22|mkEdF2_3kk`&-rD~}6u9CG$oAjC zZC4^no#no7uC}q}vR9u-7{KD}c{8y^WuGJkJbfon2>3dpBQ?A>gq)R5p|sC*bvR+# zbyb$6htYMwTHxu913v%0@mUhDb?g z5eQU;BrZ87E$?D!xUnLOZ2#(gLwEW_mjeiX=U+sKy2dlxf<^+Uwre=~L{k&$(c;53 z`fxv;$umGx?)ua~FSUuGe0aIRcYgW?T>u95eMD*MX_v$$J8XPs7+Qt%LZJ4|-`Jc& zmY3~<3LRr><(1WSZ4xa$Tq_)Basy?79fE$&(!f>g()Qh4)39>2Zj60VD}`C;~jbnmt?#U05;w+lO7;sf}W zX-|SP*6QT7kj3X+a2R({+a@lX&4qFpG1|Yj6{Tf|iN#RZP$y%D0rY0sOn9|fc?=5d zu(R$j64=Y(%(M2WE!;4NW0_ONir*KvVMlQ8K%k!hk&!c$AvMsX#E2v0oS47qY^MgK zsn>{`{*pe;<*^QA(BV_aaO=CNq;RP{3riQn*v>;eaCj_IH9|ArDZP*YiA$?)^er=p z8QJKc7OkizW32*pm1OzT2trssC<;e`;qylkYP*C4Xv&w%Oq!z*f0;scR}}IBABG-9 zoTQM2?H-=;EtuP^$ioWVWdPXcB})J$VJApnz~gZlW-}IwrJFjnV!!jx`7a=_J^oEM z?5XuR&Z#DHiQz>SDHhvd4!izGNx8eDxx}ye?3b*DsWYo3e$%tjCrD0Yn*C4B8{~<#zL;yQJtrn8e9@#^_D*sr)+;Lws=cB2eB){b)FDn z=Ik_?(#20~bU5HD`-RI>5-?9lk@qyy5uD!H3zN6So_vc8sb!4ym$9+RF0xRXfp+Q_ zB6?l9+nU!PRBEp}lHD+6v^wi0tH0LIq~P$XJT1y72`~7`GaJh|)6Gv%+$$GoD6!tK zmuGyM;ri$N_RT@7R&sr_DxM2fK4@H_K_kNv#;<$q+xC42PT;2!pYxRtj<)S7G#H5K zL+<(EF}AP$8$PnTkc}{!W(8${TE4z|X6LHPndq#%7c98UIfO2-8ZOVkL^M|hXh}NP zp>Nq(Tbq2HlI^D-?<3<$fM}fJ@Mrm(^Q#r%E)`;I3)=o*0;0q1(Vw*@N{yI?Gh-3( zx<2s7``vdT+_5qDbBc?Z5*aXgWx?e-Cil0TZ;8k5$b2rC~#`E{ytVcQ*xUX zCyJoX!t)aIwO8Hs_VM9{!}w=mT=*^;JsvUmGxU92{1?sUM?DW*8+d}QNbb`Kx*|kH zB%;uwdc?Vj3mEP5RH@(jTq%W;{fnwh_byQ*WB>cu6fk^5xQ`nz7{fdRQ!hO^NaXtf zFqS>|%vxo8w)GmahG7P=7lsQ={~a2PIF9sDKyn&JCd=e%LbW6})knJ68Cvi4+I<{w zr@tzfGt+{4B*CX5U{U6{rAr309IvmRtS*DO@}$%>i_wF6{!*z{ad;Sf`!6y-L7kSj zTl_`?_oD&WO0~#w@w(NEmZF~pun92w)Z4m1?&40}v`xOJ=Sg=l;&UKj@n_sQ8}*xC ziP+3pjpkQ!M$kq7W~l*0{P8&f$-v#AIwdhHajIrUL1<+sID|2`;i{Bxr! zAG&NK19$H#jt?>q?uZ+0LGA5I>}e{Cw}pcrsT@^Pfj>WU0L|*ygO^G-hX2+ijJ_jB zdlG7RPm6=2U66h5v$cH6Y1xjm%kUsg^@CW_uhqtog{$iVX=e?>+j-}hWua~K+j*`m z?uRy)b%KY%#s053wBo6JHLm&5+D#u^smT?=-!3`|Qh#~n1j6)_UxMpQhl z4|~%U!C%+bFjE8ryNrXA^F{CcAP}ey%q8)B4lv>-=-gm1Qtjq}4hxmw@={}nfbr&# zyIEyKHDBVhK~Z>`N3wA2f{=Gr5*{=`-{s)r_DMM@ik1t#P%P>ch#2b*271~X63M6>OjqyHhUpayz@HjACz7kY9TKIJ zifl>o3@vwv5yILam^`|T)N_?jSK^b({uU?8C*g@f-PC?JRdJ`~Wb}Ag|8owWJPE$Z zVOVvMp6&&pvTOVD#Fp~T6FHuIllu0zLmSb}$4q)E6|A;4Cf3?YocY#xtu&ngKi{f_ zu4?zIKRCuSoz{9Txhe%ceR?`qAO7X~p(1~z>N5}K23eVQe6#Wbzr5zx-jf$M`|uCr zz?C9NX-)J-1D-@XmokRb|NLXvOlE)%j}cP_saxJ|Jo7LA@?!HVu7juYCr*)0o2%)M zt-t(L4m5?I@LZo82?N+!b4Rl(C6q_|rbJII74H2b6B3OW=RKVnThJAiym(L3Fa(w1 zs<`JAsc?_H=1A-c$(dMIf{UU}L4GA5YQLUoz7NCZRGA=Lb??-Q$Rf(o7NGTVOJ4Ow zqo9h|4vM*4Xu;NVkx=wJ?uGO$6;9%sL6}!1Jns~;UUB=B{pZx->XxAjOW!u5j89AJ zYd(Pm!NyUk)Mg8WrPG_pb~w!oXUaHe+~2YYO%R>?5M${G!UrsV8f0yRW20Yd!~>C% z{}9zvQp2;ADCn_6a)%x4aDvkwMALQfNk;gtwBG%k`WJ_G5N?1tlfVezUq>#s7I{EG080;xU`bu>E?Qad6DX@Cf@SL;qVMaeSJoG0I^TA=Aa6kH2 zqIR;f61ZlkemMHprojFKWM+&Nb3#`NDoX6EoK$21lguy@tgrE68))*vxO0YtWmavZ zpSQ4Hj0z)Av_`P`7*O49lLT#45gcZHpY&AG^j%u{xJno^ssb$aP5AidFOGg9eP~i2 zQXdn?1i2j>E-qvIgB7(rq_=9%+?ZqN2r_Y+ZW6w|v>rSo*e6&u9EVO1rI0x0>`;zU<^GD zsN1XzY|*@A7cd;VxS1QEqoD=VNN?k|MrNdwWuu0Nh;PzoTOu4 zBEEVoyhKC8S;{>R&mMg#CN{?|eL(3WLFl7!QQ}-pdrI=jzFh-o_!$=eP!TG&%6v-8 zpPx167G}GD2r8sm4WLj~@kc(v#jKse3ZfO8-Op-MDH_%* zgw7*%J>}G}Edh(h4f+7l+wK-3ntPfS6+4bG6D!a1XQ@ZhAD`kWw{O4E{fpR@!R8LN z8-4S}Ty(pCzm%tL40VibzTEVZk89C^Q$$&f5dCs#@$!RH33!aKM(>S_jl8QpaW$iP z$DtJMq8G9Ghd#WSxmQI!ZZ zp}bJy@I{gajVyG}DgjPS2g+_W*ex#*TTyMYD@jRl=wZDk&JMN$HhM|#UJ^usRc1oC z(f9P)d#(0|Wy+HpbMABZ;=lj_O1PJ7f%W6S@4DWkFbYmHlkX9=d;s}B)!JEfT*mG> z3Qv#zn_!yELJ*EH9G4(=y=MtBm4Pb@-x>mQ9;O0KH?vQ+47$P%)65&)OhMpp`3{%|ov978*$>K^83n zA?bHy1!T;Lt7L%%v@0DQ z6GMS}6ICBXksKckE(K&gw9WN})Is$Npz=<7wt_<=awMkfoNhQPU;2v6e2 zR;QioW zJN>7p;+VX$jYm>8=6s+B-397`!R<`|l}2pwcWhDJ6P&7LHU@WMQL}Kz1jt>s9Ic&&{ z^la-RXeSa;zzo|3kC+;|PI%c#pwfZi8|LKWi{F=G=gG&*@9`5mFu$tE8 z_nVnc7p05)d{q%O3Z6kM93FYTrRAW8;>wO2q}5gm53XNyO-y#Rtr@7vD@4vQnKQQl zpN5$UDxOZWtfP+e51Ytys+tb?TbciK?}BCC&wT{2K3;dPSbyT6Vi_Ja0)AtYS3-B8 zD`KSACM6bpMK>zS;ugXzeUPvM@vIfAq@de0r*cmXyfLsA|g|W2znjDxf+t< zHN?$x`&YOj0n9zYrHsrWCW!IWc98kOvg9RtZ8L$ULIv6suhtRdGW;_a$)@@nlJX_b zMQ)Fil}Fe1xciP$Zokv%fKqdKSccgeVvJnOlhZ37Z&Ph`#A+aq`0a@7QU7*jVBn=f zbx(!$*J>*m>i!|D@w8 z{)uv@EOmM13l)D)Hj>R-Mt#3$(65F1xd%UB+`6Tizp;93?m^6xe&av56zGojGi4dwx% zOe1e*1R^x4ft1)6^u*6=cUK+T(}2uY=M~WsJU3y!0>>uQ-@XD%OvkTBzc023w-EQX z((cXqcgCI16A>XNfMAUoAQDO#AM68LVhZF^%aOKe!Ym2XjUk;qb=&hDFV4@54tgdQ zQO2@m*NM5n5L%^y4eWNDVArPr#$0*|Mt=r|KV;f`e0n86!pnOfeMY=)hW865%Cflx z`Dr?sVYsLOEM}RJSgZ92E3~x9suMxck)kw+YAF<+6#9RRy=7FJUAr!dQ;N04y;N{3 zUMv&}w75G2N^y6B1eZdAwzykyD@B3^hvM!M+}#Nz?DT!Vz1G^}>~qH9N63$4jEp&- zd(K-fq5pmD4mB~Pm}lKnsy)U7c+%)TFAatw3S9=*)sKtNvz1crvuVaVav;dE)(gOn=mWK+~PL9(ze3j>$CRzS(&h z*wJSwJ8_=U4;|joLHNnKMP$vTCfO9gDU(g^gZ9gH(6?%c0$E}B4s?D_OJ_1K3wvwU5gt_{-={>;8ZYVW21SSCH#gD$$UEBfAp6GbMt@VrkQ#@A@f```&W+#oB zRk7|IpSf%tE$XQi2WS{{DwL3h0(hZ0PwB)nB0B<>?1O2S_(b3rza9J4yI7Pk5Zx6J z?Jm2I#&dFM77kaAo37&)$(!%B#1UA;8ajmP78wNHA{%L+;KF{0%FAZEFg<1dB3uQ1 zMXRJ192Dp1eA33cf5fs|Qs*^DX7c{%?MQN>f?ic&3K)CBKTQQyRB6~Ip7{`oGVIrt zbY_cZHvgD#be|a_qDmaoQF(MK@aMCs$fAYl;2zL?V+GS>rDVsN=W&lVz}=x;v&hGN zAPfoR@i-_(KL<)%wcOl(#~#0SMckK`BSATdmx{lG@EbQ0 z{7ST3#SL8g?-&3IKS8W_e4_qEcL z`EI%Q>6mv*Of6-O2&L|Lvakcsw2R-3x64cT=zq15*sFF|>&Sdu19>6iQme#D0)%N( zpxE$Mw=IU!s`I#jQIuuBOH+C5vNNL z%otg&MxpJg0}MVnIT83SxbkG>iS0HVIkWBBSF{ec?%m5Aba7xslySO_V_Bim7?96h#84GoEr}!w zBBb#hw~S;6pG5-(v<9nD7&A8JR4DcLbP8RN(777S0L<54=w|W2attwFVOU%Aa4&SN zGrtr_KlEFWsCsb{UuYPxXkesWioqBaY?SRX7Y^(_rYI|-nj%YQo_FLI#zS>f?T3z_ zE#OnR24X6~Al@kabiq_y^lsygu|=#abh?QdZeF2n{wYkJbAgYlKlR$P_c1DaEtXad zLl>^Tc$%DEiQ|9r4{NVXQ{s6z%4&Z3UVoZ=>z2$ljv-##izVmOIQlHN$!TPg@sjpJZ#D>~4HU<3Q^^NPMLoH`=23 z3nzJ~l2Ssq#xt#Bw9-JK`%kyGrK9xoT&Ue5@`yrSh?@ZAX=9O%9?CtB-!e$(HIab{ z4D6q{ljvU+Q=9GK3^{1?@`5n_?G~OC?VgXNRylodk|u%W!5)~I#J$gvpD=rMGlUTzUxJJ?tluKS-$jC{XDA>&|UVh5hFmCjT_f=Yd#me4&fW)>GXJr)ni>I+4(i8qVL97=6 zTgR#QZB$#quge}9T^E}cEKz4zfj*&J42Rwit}H>NcIf)KXQSMMwc}daGp7MjvW(AQ z!Wde;GYAQT2*oh}@-GZU;G2+#?S*=D)u#blT|eU!F?L*knine;Uyi@(@lsp;G4=dJ zL_P3f(3j@R61>}WQIoaez)eUO<46c??N<2>LcN8cI!_2xq+@>Zlk-iWox zFCRWj79U(Q>`jnOHt%g9*b&^f~#8$yd%SILUI zw9w?O@(A?S)c@gNsG#Avb`H(Uz{iwZZU$(T%bFAG&q<>GRoPq=8j(#n*zE|GOUATT z*u%q+yobO3&e>e4^80W`V>0fCFcmooY|Cpn(K)%MI|LgZBI1oG#-~NQUbxKo`2ZNa z{{@!z5BD~fx0_N@k9{9R<2(JiL&IjAJCZgNqMR52WgVQCX|zwv#%SqP?f_bDIac zMd<0)X9vrd3MTqWm({15JT2BM=Dv7h_OzYn86T||w-M_*v-OhK-rJ_vOW{c>GHvQv z`+$QF##I7A)xdiXXdFU#VqH*rb&H{rSRx(0ZRCCktLgjc7f~~M%s(~6YX}r%<)%)_ zH1`u$JgdRkhp}pt-A{VGVow6TeI+_nc*OQlb{(a`1M}`16;L~Iy~e}qw0l>j`$lF= z9J6x7e*^mY)H!usm!V$f=L>g>D96ipil^P1FGns77cLiW>0GvlGsRId953{uGPI~u z^gG7KeQ>*sF)LvT-N`iphFrs2I5ZP8{{JEcgD1EfUKxh|I6oH@lJ^^}1#B$-cs;hl zEbO^nC;jQ8m1e#elcVra0}bSmg#BF|VM+;=)w|lkY^(YA>6$v%VqOWK@$ra0E)+fP z!6u+tFNOJw99d|7dVki>9~ru$zTB<>!aaG-Xb&(?BTYt4+iGRQx4TlMLhNhQA!vKt z+!~V`5jX*30pT0zo5WAv& zN#in1=r!n`Zosm@m9#vk$q`YuRl(D;5k{molMVOmHl|vTd|GKpRC1zn!E{TEst-7_ zIqJKG^(YBKGyt%JR8``BRg>YgXVb$Ne~Iblt) zhWIdl9`}_`{lZBxM1-7_vp7V&W>DKarC6)~I*%VFYCIciT^kkne1&9Caz}UGqUQG( z@WQ_O?n|a7le-9wymgw+NSeFt9{NDFnw~XGc+0tPb`borW*wVLzZJtqy>DYc6#6e24nKy4n1ckCQB%NVs5o;lGU- z5uW60Tfcuct2W`^cM3FhsJ1Z=H^$t*?U9eV8hy@|q}P4*#E{3Mmz_lTJK$FpFebhy z=>qYY)dbnb2yn^>G7OV7K1yMC42i}*k`VSnBY$OFJNpTrWIw)ehWRV(AFNs;&(~)! z7liy3^7%K(xGn$I>4WZ#XG>w_*?YFXhE{MiQxb-Ht0`D~*h*ND?O4@46ogEaWMRi! zwG2h+g61QF`W%0QG zO;3r>q$bDSpU*y)4E-8o=&SHg!Ob9TTFCQ)lzQp1LW7-nmadMlsyJ~>%}< z1NU>J9wRzQKzT|;9ga$gbB;rj6sV4EeWl9W~#9m{9cJ*5nMfJ@m z4tdCXvuL)YEW2^4uPDMO7GW|nO`%I`7P2Q57t-!j4(rU>GjtLWeCuo_m7`Ue7qcG= ztCzpcQWAYQU)S-`7~prX+U!JnNdq<89;bil0bm%i*PQ zr=+9%!S?=k+gubQUYbeXYSr&kSzl*P8_i-nI4u#R_1-qGi^+F?q?YIfiN%eDpwAC}i#0JW6E!!*A6l=S+zyv&WtBjb zmR58(rp~_jnINQL3C>9?$nV)#)D);S1dh#`p@0n~1xzHGHfld}Ph+Y4gqDHR40M9j z>GH}aQyGEfQOa0d?lYA)uD1#{^e>i+Ih$x;#qYzt?a0 z9_wyjLd02+*~_8dmkF5Td0pyVAZ+uE%A1h$+0umwZ49B8*DBHCVjRkbX>xNLoand;YtzHZl(2k|W*tT9h?EnCHA)awDkm~T; zG2hosS0ZKQ>TO8{xB7gz8?I0LW1T8A>gnpc_1*uBKMtc|P=UKMrW-J%BMR8lGvRg* zwE{ehYl9QrT5bI!UM=0ugWG%eq)xIa%#4RHs;3I_ zmFA`Uth@0k=pPHXX}NC#vSHFMeS0%f9~b`4Tyo!~x}aY=^j^q%u3;O_=&^6|f2;R^ zD`KVe2R`>K%^wKHpu8A&ogR#WQ`sE>zjY!<0@qBtr%8W*gn`YG0!ENU^zFxbw7gj% zh3rA}fMWs`Z(Kv7)XG+4nQ5|A<7~3YWpy*{ZYFqwqr+h%UoVVCTd&fuMPMj*qj)A~ zBEw@Kbu+w~&jmL0=H_t5@}P>rKTD6S*>Z+c-yV&YSgdxh8UMXSp@B&zqv`661L>abTlA?4JcEodXNaw`DWrkYDqlqgKj{=h$@lurb^c;8yQ-0BIyBrrPfji5J+EhK37daXgacpKtd7EdCV6E=iHJ` z`B-Ku1{>B&bJiAJt`)#ZP?(v!ip5(9F@~m!frQ3>bDgzZFcc^F-Kp4wKWPNkua~7-Tck&o+ z#-u)|(8(##Idk`TG<}_x`lM#R*AL>`+j$t~Sq(qNQX+F_i00m4Ir3mxFt@5NH4ddV zHgO)wjk%>s)K+R~EjJLLG4!XSopbTMnk#tn=~n$xME^mebh^C%#3d@nE%TXA)FQ(7 z!4U=DBMJ_e7iu=O7+P*B^XX^|hqA8~w0#VxA^MVFqS0s8yG9-4PT~sOWcdT&^U>DA z-{Ji^>{hLZ>Pb2)uI3zb3!Iv(msebu+nIod5L%xqI-o1~2I~mM0US|LOdU#-iJI!$ zoA{6)C62L`6=u8YJ*92W^>+)3hygwodNj4f@t}BoGV`3}pS-n-Tki@nYhzQRvFMSA zTXJZYM!pDfU<)u&#{#2QUGOr2;bdh8I0vb-$>R?JR{Bv`8WAQQ!yw#Ov^TUOEi6bf z6o`3AijTJJcWNm6xm4!+Bt(gD@|kUH*mUBGI;Vy}{?+=iSt~{Kh@M-B@oKeEr$T3{ zU%1kzZ7F6Ovnyh_=&omreTT8|uBV?)2W*m0tO&0BhQ{NSQkTO;;W4wo_ub;50j^Z+ z(KYi;p^Ey822Dd_E65mHM6LDw+$VIqyW7v7??JOw$jQ~4Fx8^Z!&N0MJiir6*U5=^ zI5*o(QP<==>I|Q24(M|+=RVol@Gur2f-5^vnt;(MRbC}rT&nlB?~b)N7&r!m&iGOH z)C8U~u^YVC#w~qgfg8V`V-~8PG;|~#yu2x;bOwf=v;B+E>9Mn)={;jCLu~%CJ^;;n z!L1Y*8CJx`kh$%6HxjmYd5PFvHb`uN>^P{8o~V8`;ZT+G(#j2lB4ll7kiQWVFMT5_+$Hi{=6Gj?DfuEu8)RGA)*k zD3NwvL%@SkF&|eO6Sf4kG(83ms}aY|+9=CJ5A0Z>RJf#j z*_MqCj^D)NUHq%g2X>#npf_pComK5ui5|oRGYAs ztYsU=PQ-5n;bieE)6vdG?NJGpIUnmAEN?1Oy_pO6-8HaWzpnaqffPFPUBg@DHb>ja zNf;$wt9g0P34t(>o#Zc5LG?zHDR1%niIa}kmu%IP+?yQ2S_ZZiskr&Mg6MS=j`US;b{`B`RS2<}c z(ANRt2y1=V_y|8s$FHs&ZX`_Z(1qTuILJiK(m2!+F`*WbH zn-h|pVk5gtyb9Hk0WO5x29C6~faSI%-&T$@M;?YXs@Rs7t^G(p38E7dyQp`ORMPbY zfrc)-P1hLKGYM=kOKeNJ!$Jd}ew|PPc}m5sYISy(yciuUP)lnxH4*9knS6>iD`w+^ z^aDt{%_VnMSu)P#(+U`g=XexT*ChsPDH{$mGvVKm= zP}|ZuWY)uZd;Q^qzn?VtamtAeEyr%l*;xZula4pEMqBgeH6M9c{58uuI8oYwuttWMGT$e)Sn1sJ+q|=Q;$btr zcH44px=G)g!-h>Y2s@gjgX5_XG`(8C1gTQTEKzE2saCBs?kjKH@N{*#kc51-H}DwR zVt80#6ROXLVsNyo`Nr1cROLVSI<004G&kvED4b4P3QZze_ta$w2;rTl3v-4cg){p{ zRr9_?_V2J{ONClbC@M`PaStS(wS3C`SFAeF!?1s(K?Qd2dej%!8P?s6LLfe*|COpf zwf}bdq30#)9D-Fi+SBTkULOM+Q6ksM+nWMTCbDwZBMtR-v!g|7@H3J$eHrdyd>lQp zr39+)crQY3M#ivq$!!azy0`W@8(U3H1KQdWIU$ZHGzB2bzdeh(d**q)*~zbOS|}*q zsn;4~QEfhaP+qC6r>C*pTzI1^+79jIpWFr9o;N+2vV9uWLf#tw$a4mA|8bf+#f35O zgMw96@s|u8zCTsNXh;EPTPxGD?WE;bu7?l?@3ZqT>KxgS7bBXBs~fc)9%n za9qo0OS^!oj$~v>iwNZsn182Fyo=&RiQS75qSW0adfHb$F!ehIYOIft*NdSW3p~!C zffBlK4(Yw{NxJA&b5I*r7sSRNd}CX}JtH_pH0}_%M}iI%V9GYsK%+aJ(u)Sm1m zlp8_z^y)jNCLjUdK-~U#hxm1~f8pv8{?&3Q2%1e#hqVLp8zp+BsO-_G{<}?b88+&N zM`at-Kfa-)a!|ANE+Nh<(8XC%;>kwQL-)dw>cs2w_a&Xf%Om~qet38~&t$3eKmVf$ z|If$&_e+Fh3U40flLsQAa$1~)a?FHEo6L7)x>CG0h%%lp; zqwfvW4SM;<%%9f=@G5)T_mua;?19)g`Gf}q!|#s%7)CE~@WDA&Z_8KyJR$z$&kJL} zZv8m-r}gD=wISi0&a+&)%j?NoXe*KEy`q(v>Qsm#IcIfuPSfzxzP^Bw^i>guUC5k{N0{w zyWgKI^NCq@y|Ch~S)CG@7@zU+Si3EyAbalYu!iUYJGDJHUGy#P0<%PLYW?X^gp}3Q zr?xNbC$24G<%}tPT9Mw@0ja4qTVs03bCLv z!t@Nj;{XwN8xbv3W9hQ_Z!9O6@Gk!hI1K^3Uc3J?#WP?_KGD|ji9CFe`2uA!+o#ug zn7rS9vAJbASelZ6-dPt`5oW7?l!Yj4B0`-u#VJ^nOIn7wd8je^-2KSioP{?pFIeQj z)^!zp{|d!|^NGA={bgVcQhJ&)X36^R*4>ciTF5emd8z>EUFD5tYZOt((k{!DF-Pr7 zcB^wk2iNRsRAK!;@~OXHJH72g%?aZ^C_ULxFO%LR7g!yQcuCG&*fyUfh^%*g$%4NM|5J~EQeTT_zQ^Gw`W+;%Fp$JDh0V2jvG2RiNLwc=vhGId zwJvix^5lViM#mxtedYSXd(nR8;!fe>WKhkD@x9j9VNzFOj^Siter|kH?24b2j$^a; z@wb8~IS9_tgzz2SH{oU@L1c-#{7MERYw@>r@zF%PcU1>A&SRfge}xoZ1bhfD${4F` zOW`o01+3n(ld~7FX)vhPXMkIIJ0{y6+wof4q18WiW}74YI88%!?*xZ(dLl~929(f8aAlGx z2Jyx~dDq>GnE00Gdltv1CK2W@qckiQo}U?OF6}f2Yzlp ztoOEeziIwRYb(f}G0j<9$VBHJx+kZY<9qqLpgT5T$#mgKnoSnh(NDiU6D6aFVG=)) zEy6F)sri(O3ix#`>UKP(@*7-YXykp2#}O6hPElbi_K~c(2yS9Xb{E}#IZA~Vipzl{KLzC{`gnD z?VHYKf%ma7RHRJ;`PKT40AG*?SGz$WvQR>E4EZiG0fX;Bc}#qVr#y zv1Uth`kdpwHuB*C$FN%I`cF2)PU3zT>k&%+_;KUt{7|4} ztzyH3!kN)VjC{65^}g}`J4F6{hg3pg!hN5i#_2Njr2bwzede=ihmyT{flunicAT9o z>($R}M$#x&`@ffRJ8Fj;Cb>*HUHW>q_H*{@^%to2LbeRh_DcXJ3oVap0FPr{wEiem>ow zHc4eX5gw`ymMy?^uuqVRY?DwIs@S4(I$92G1cP#xJ!@UH5M(vTd8-W3uH}eQI-u$U zHQ@O&w`mO2?>77QWt3>={+?;fYPjhVT#3(bzAbM+#Ri8w+-ydf>n#v#SX09PqTiXt zvqceAuz&cT29tp@zY)< z4PTX~UecGo@GG?GzrzDa+$2pz-79ZWGjziow|t_XaT7H)@HIyMzryT4pFMTV?MLBF zT7%DvInMc#g17K#04ICsa_pIZ-0IlU`#~Nfo4qE+QwO3Gd8#`Fa_^ThRJU!0x)UB_ zs_yy0x%tP;q?Vn(l}9PWaIl+PJKkl)Tm{qPii$!?PO(R-rO0u6Nk-kl%p-~ayFuG= ztDVGTxbWy|!d|*8Tw23EkI9`>wM_rJ05(x;ORKtqLL$pNcKne%emY%jCoN12)H4Eu1WqMS; z42cKdD(rxYq-5k<<(hgtX(o#ka6Kkr?q{R)v$MXX1DQkF6(!KZQzc_OW(~QW#p0#1 zf`NAU4=ut0{@w5875w<6KvEA**!3R>i5={SZ%xunRcEYh>PAYL22wZIk+WJMNM_>| zxR#k4^Fl=yX7pb{8Br;h^QF6`e?I8SXUSW+v%>{{pnD*M>AxQYR0@-o_m728G1wsI z19)n;xfzFO@E^}R-~GR_tlsTII-uvoNP4Vh2Y;Yovc}d_)^T^9kKTGq<5_ZtHhS;( z;FomiH3`+|mu1eDIRp5w%7wM&AU7NU1PqNV=F*6xv+IOXnHatbnrz1?`Wh9^11vFS zE_2(VyNkRtfn-ZnvOS>^gPM6ks{o2A^a3c5wwV3V?B*hbS;BS8tm7lG zsMrbGUifG*SB)}PV=VKDZL0j+OFk+@GX@w5n-zi+9 z7z}&8WoDuTzHnnt>HiTQs!0V@A&QS6 z)AX20AHp&FV|CxzUbY0cJJ*_C7rSPjHsoACI}Y8d%swlAfl`+ONEjcA8*vXyA_DX5 zf!l$*m_B4R%V^j(=8mO%n9H|490Z)WfLCtwpWo6(0{Z>_{2EJY%9M(5oC(-HH+)ec zB>}Nx)|AQdtj2je=uS1G5-!+ULnsuYv--3m)g<|@23Vtn|Hpj^RMEcb3CXJeTr0ST zoj&JntC;&3C*2y}@rh)Th`nmCUPl&q=m;yM2eig(v@Uw3dZB}3m*;qyNT=`l;WhU5 z2&7g6kw+vuktaQ_rsiGyXQi2n#8k&?-_R!vT+0Hi+@aa1v7DZCYj79!M`&Z_iCp+t z=s{VVL;j|K%*Mh(<-hXD=~-ZuAmGd7Tc6`NZ50#_510Tyw-4sp^iCF*TvEu7p*)6P z^xi@qd@stss_-2O#m4qM#vx8{@aja{XUZ6&K#7X|6D(P=XX{OL(VpBM@j-^7A?-d=v8EUt_bFly4b&i2slGw*3JIc$6Bx;EWW zAdMoPn)mG#wHhNsgW#p5?_wYGMTRepF*o}boqQa#baOZ9Jciz`)4v9oe6B+4=itf~ z^gi8P(>mCsIB4H59IsUz4k$0&e6!j86&m@s5Xo3}aaCONQRGh5)bNYXbTY5=ul~6} zMR(acQ%DVH6$Cvq64P{6_11+tge@Qu+w379j-|#khQ7Rf{;|Ab$O>idV3^H ze{F$6N`&0-QDyy;07I4kTAxt0v5Yz;mio#wD}Uv9 zCO|}ug4$PVJ<-h^`~KgV!967OF5DR8j= zpRD;cXt{V7@1a@xpoEc@kXA#jbR+j302Igt)8Z);UgUv^?u1E9t^iuO)sHbtl!3-S zbZ;;G8|)&8VL^TdE6?o@RM1UJ>fk9Pij8kCsCfo)>l}3@v4hsFrZ7i%0)X5o*1CSsAxG znQb9KNi#)imB7w@_wTl~ACF;Rt%~4`wa8AZ;QR-(V=l}z`9_?WhnGrAPM(oXq@JFh zNk-&)zRpJEx}Q(G{hU2Hj~uWcld{Vw7)|`S%NZgumLE%ezJ2}rqJPnKJ502pFFmE3 zk?q`NV)7EvIKoMij_e$0O)PwAEW{Tdwu(r!d)$YuWY>Fv?-a1CvGo(b@i9dmjFnm( zB`TVr3vLoY7^~vVYHS$h7dd7J$Du;Q^G^SIzwvGO&^0$U1s|;ZS(uRV zF~ppss>)Kg2dUVI6ike})Fh!Qx``1OIuRU5O|xlhd_+N+-hM*M)T|o^s3ELj6+!B@ zVOzy}YQB7UxdrD< z#lG>%*UVopjc(=sDBYM{2N}PcXuWT|=RTUHLN#+2mc<4c=9^PhMOV=IscA4CY? ze(!WoLSbrcMBNWhWX2}K;-st{>;PU$Zs?9`~^5M#I9F|%(M0f;py|s=B z%!Q0&Lq-6|z%hzD8I-v#nZ*ruZEzT-Z00%43d%jyTIcHIwM!iP!O=#?+aaYGpBi^4 z3-OjO`?Vr#9n@&YTo<(6b{?_E;E>dMfoxn|8O!@#JkWxSUxee&rEmQRg#Iqa6!NvTxAkEG zHsil{YrJ%LSm+ukG4`-U7|AnZ*`!TkfUDom7TTK}F_vWvkc#NCgx`apXNPw4T&gan z4lP^Qq!SxGH_3(<}C;GG)T%n0V9MI*JFhO()j#-=+)lc0e@ENvt!;D-(aW;g{kt%DmT?yB4guF#DCG7dQabQ9 z&^kBy{GR*#uG@0AbPXQBu5BHoX&r>IdwJ)l#=A=(2uGB9R8(>bY3`<$umqvJ_ixa zZ%sZY)yen9f3wvO+t(g4iAS|382uH(|C$i3`M7as9UnK!pD3gh%V0S8*Gs27fd%ha z+2bEND9D{&QbyO>T<&7pxL@vNh^7I%V_&bRTY=2pxORl>UiOsX5;fNSf*yUhf6D08 z0B`XbcG=cun&a1{ z^coD(@@w^DkcbXf9lQ}puXRz=S_5j9{Cgz)8V3@nvgkn_-6kWVFVWrd;}5 z`b}xNQW-Mr?<%;8pbvR!7CG)eTFA<9iEbvul{MO-aA$FNyL_8@HLep@6{Wo=Z7kU* z3qp?X48ltHymt(K5lka|gmt=tZ`MB+-IeyaALIJV%mle6I9f&@5#+Jgj_gfR;k>d< zoM(KwIB;E)RKd$}Ts6Wj76)n#u>p|4n-vecFMgz{hY5!Mz<1r}=QjB~?-57q8fStV3!fJ(L0%78oLquX&It0r`d6zuA_6r|l z8gh)cf3(iN3N~UMCXE?%02}kC@VgwAk77Dipw67JjGQz_BMMP&N-yZ##(0<`LcmqkNAnF@O4dybiPwwAWSMxr zDz`?H+g~5VEb|xF=<6_I+?%yUZE)O%jm{&Du+MMj2WQ+g#X{A>6Mb;GgHHYwS@(<* z&%k0`T9K1iHwszOB&ExWn7U;U5q(}M%$J`44Ee{&_v!tYy$!}vk4#yOQv*z6)!#*J ziqa6#;Dx#OJTdc7kh?q_SbE+!IM|!*)=U_KBbe#6uM1&KBc5WH zmHDp8^7@6vh^f9n2y-;GIYB5RoqncUGP*~&JsUf(6d1-)6u1z_P#6uy@Nvv&CDEd4 zyq$KhwYA*c`Ny%YYec?Dru3`L<8*X-0}*P*XFXroXAI;H14(o$j(zv9+cFem5lQ4{ zWdoXX{b+p(A`bm8BT>q(yFkvYkuGYy5u5-}xuU*0&{Pb8dlRiaXm?LZ#{C1bw9|i@QBP7pt~QZ~=aqS>ZA{BMs{E5r-#~Yl z+H&%;y3(uZb4F$Tl1VtVjdKE6)LmsGRNr#y8hLl7;0TUzY%l#X0FuBt}gv8yoEd!H9w>klCE|ny3lvY#6M1qBgJ@=2* zvKZ*#t4!^Fy-u|RCib=YfJFjEGEMbzLETneE-?ZBFp!QPqa#f=pC>FRBN~h?YGWoZyhu#!jl$gYC zRW)R$D94YAv86wshTxDCg@n!N+a4S z@74XmcHLi<$9?**C@u_sSt~lh4_Vg&+d-GMDd7#moeOHUVlTlM#LqB(byQH=W zjp#Dj2nEEa>RKm`Mdgs3t!>4f6IVnrgfziWn~mn(mAthYx=!a&QuOd+Pkxo(`=6dE zI|kfJ+;tyXVr0!U&W3ErgF}Do1H!J{OYCb2(LY&^2^dx0MQ3G<5{FZiV~#@ZoqQR5 zpv2zd426|=-&<|UzjEfZ_|(4j$ozI+q44XBcH`v$y?}|Bm6*KEuNG5r1?Qd4UX;d* zICF)eBWk(Ev6&r9y|H(L;*2z*7+k$&DsSdrzt^HyPH25vPKT>iLD3^}I1@X;zamxq z1te>e{snY&78dFGSOQW(MB_m(PJK!Xhgger=8y?*K>YHp9_`NLQ`*}G910d%$O^Yj zwSLK-0aoU>veLM%$teTM zj#p**8R+@A+cMIF1ny%E?Ur)`pS&$~y~-bCl74^dd08BO_VX=xe=wCTvYh9|%#Tm) z8fFxALY4j?E~jQwK3b>NqR7U_d>WU{TV(-dGZTYq?ikPG6TX#7_mC3}WUudijKe%{ zuOPJKS9@%wpD!jjvR2uO>oHQeSNIx<(1%F-hZa8(mbi$|ksTbY?e`osv`xk9i4Qod zlBg9&j&F~KR6y9P@J#t=?1rBT62My>M5Y;^otY09>L7aBM1cFKTm@>RCUcorGt&kS zviZu?nxFUdO958tE|wbeg-mWnn|^t}slbxUYJ5X--N*Lil;yM&JU%G?DO1XSy!!6r z6*a~lXzE%($$%P=K#oTn=K17yFc|oRy!mXx5)5i#@~UN|p07xE5`Ru8whTSq8S9I;50F|C+MD@>U>8?#HXg67)eSW52^_7MHe`q_=n zjCTe6isQ3*LZU|Cq^yxb^FAuoN=%%+);rY95LFu7UuaOiz*EHL zqkV$Xde~U?aha>+Q`Rn7EN{E&Kv8?O`Z!1s{VE4c> zqu`smLqBh%CaWOF_`aiLfz{&k`cf0hz&n-yXg@Nbeb*=}*e1q>lg{LGuz8Y{gy~PX z%BB{qFougZ(r2UIvX>YERLL?Jxz%vGHhTfK(M-h#kEF z$XWd8YFQ;SY&W&XBU#xN3r53RBg*&45+fT$%*q}t5vmxyy}kBdF6B+4i$b0O zyngh&wJHmb(^E-7d(m_@)DN)jGDn#l=$xSbW@M9CPK83jY)IC6M#W4_DjvJ{Q7hed zJbFSY))Z#7$Z9d17wh}5d`qRslXx`=$8e&0va>mS!2c9gWx3qGxG zE2$`Xj#yR8p_p^G1Q_}Q_rZQ3cni}QBpbNZz#NPpbj;eS5GG`SCE~l!C%y*)Yq${n6EV)p?H1WYS zX+;7!zvb>3j*-N&Tb<~=;Gs!)t7i)2oQ>hz?U?G$u20d-#;&TsLXWlhLAF}b;W{!I z!&V=*Tzfnf9t$Zw_nX4QwMz3H=|hp6axJ8XVHT1t-wk4~86Hi<8m2b)+^kZWdvGBwE+SeQAvP@R#Iit;ALxm~07`iMeI;x;1sQ&2=FmkTBaLoYNk@V=_A&$GSC*`7r<8HQ3c3>5i;CDEIb( zr|kE>ygx;pLvl=K4%cy%16^CPh)=~HHS+g7hEwpgp4I{dv%)QEJ+!HQKVuu2bAsHx z-f=~KK>AO&#kGwNbH{dxD?d=kZ4dtWETQbHEaaQG;cd>6Z6Xc!u;gy~T#I>B`yT$SQGc9C`pmXEHvB&P-wQguV3 zjf1}6)$YA}Mti7n3*AL4zhFPbv;iF_RZiejqj@$uja0ehd$o$B8~#Q$5$2^_ zZf?1V<1f}5OI+tR;x)(Aac$WP*x=W}NC7XVMdu4W0^+%+agw%0=)=8qb?iFbtu3sk zFQ8ptaIZ#?V@ko^ItmC=1GYNp(U0en6pr|%l-BmXSgDxWWTfZs+<_@6jm|)g`X(YR z!JuiMlG)iIj0Nmz)Z%b@`u%+}_#0_!0C%3TZDONaQ#bnP#3r(%TlqOl&v}Cd!{20W zuHESl6A?8e@U3&H6zz|*>I`1<)=JCsSAz(B3^H3uZYl0P+~F;4&;A!?~V`q+Z@LX2Q&A5 zUvaK;t>3lt$zJ)MHR$FnUn{Bg*E9mX8g;Rj=%yz~w+1(K3~(!%vhs2oRR^J^F05 zV&F^>v-hGD|LHjLlYw{w!rDq=+&yj5*M&vTy-^9}fw>%lLj)H0>>Ll%U_Tcxjlmq} zN5)OZpp1NMp>NcuWmc8@ZM%zQjoVrBk7Zzpn}8*b5#A9a-9uF%xfE+T$i{M)W0Q z0JJY?NHqhmc+3=%UprTy?&H{ByJlLJJAKDQk*8TORzMtxxUp$;U^#YK^AR=i!t9RI zrfgtk7vGj98h>?v-vd$q9LL?TWQGO7R$=5NUHXyW97#oGjmh0rxC*1hbDPg zo4$l>_u!n$(bE=Fyku)-HECbcN#etJ6B#vPB2S zjfy{~T6;*&X_vtdA;u)N*a2TQ$KA{9p2dVE2W1?CF&R>ALW=1bFAp#94mFD-G_U>q z17qfsmr%Qvc9kUBKOkE)@xgy*D*=EB6Sb^CSy^_DI%46dFG%vMqgtbE^Fm}+rc z;T1GWB1}$g4Ft+7nkYV$ke=Tzh`-0F&qw|ju=``A-Eys;UO{un=&|AR&3iW`UzF$G zB?~H1J2d7KbG|q4?>U`b$Tg5XT^jH;b>P9a#0Kntbx(@X8`+5eRK8bJAtye-ws-%^ zZ_xeIEN7|Lieq2P^O3Q>EbXTn*(!dfZ+&wKIruwYrW&yw%)P#z+_s}S>KUJE%6Y3n z*v9vEpotzpD8Um%BV*Y97WIMhD&mqjwp@3aPUW-yZcEIj{vG8)YFcZ^;Q2CX|AT%E zcKAeC%Rf=}TW~G_mYSI1jd#~dzs9+B(RCPTTbd``x`CBi^u&x}j z0hpdjw1J=y(OzLdhxn7VJ}!^KZ=xot?{d8AR1+4#VnP`usT9M5A63R-Swh#)9i}Z4 zj*|8=Y1#MjfF=m<2Qz#jqL**dW-nU*W#v3hPJ)P)zrFG{?g1Ev*DSG7)QG z`qmq4vIi&*IumXokLUdKxZiZ(MK0p_?+z|L{_7-ArCuf`sqG>b+6rq)ZbkR#H=#;C z#`6FN*KX@or1! zhZkQix2UBVfzgLfnx8RFhzi#nensvN?9cm+3M84YeHAMF$e;4k9C;u#lYN%WwazEa zoaT)(((nMgb7aGErEZ_wg~OPrv&8G(Bi@XMI!9L(@* zG*n`6B@Qv-hFqS~8}Y93{^rVV%8i2~X)CeCQPecYAnZVBWm>dzSdiBe`+~{|^bTH4 zXv&)v#1xmNY==l;j)5A<+nz`v>y8C57VCtE%H*nqN=y|wB6J5wY_#+zp1ke&@B zU1VXy&XH&M6>ak%}jUnQLNWv>=Vanqd(5cE$~2>h=bYqt@L% zCcTi8)qm<^--O8c!+`*5XhI0kkpg$}w)4_qwM8IQ9A1V%zXl_DY7BLpf^!op#dHy% zbncSxVlV9Pt%p_cTB*G%3|t-tIH(iX(Z7YY2~BVWVciC^{LD`UsPwZuO>?UN)B^ZK!=RsonlMr=8s7kcIRF{`gCJfoU} zqrExCYz60pVKHW9Bic#UOEj4){Vd%}2CRNckkWVp(V ztrO5xYD8)0nJ}(K@RKg-wbTuGlC>Nl=3QeuyA?1#XCYkYiB6x`$fS7J6@cG@5vmt( z7oA_a>=ZCN^ovQ6 zPERF6H1x$M?bx!~RZ_6a`e~s;&baIRJ65?i^`mVr!uhZGGCvzXgH#{ZQFX`S@#Sg7 zd;7#&6PjfF*Y8yKSU!#Udr)eWCbnR8B+x4Y&J44$CJ8ZOTg6@0cRy#%B%Sy4NOy5% z@go6md@sCNL8Ng~N=B|b-qa^Om1*Y`w#KQ&j)hI>EeA-6w~?WRbKR$&1GkBmapMJw zBI@_UmYdM@$(VOWpmaL$>CFHK%lp7XA#I)l5W=38fV|Z03uG+1f@<-vsAcgbH2eLX zpNSm_IZfh|$^t~ENF^nRPWRr_Z1-i^^#aZI@_(M5+%e3dlKBTL{c?4%d^U4n65(Hw5%?S96^WwI9isQChVOrwO+uSXDlmabR8w_uswyL?! zF97t_29dAh4)nhO_K>K1Qom_?lQ@a{gEnIXM|(Fm6MU!?BtWZltpsvIpF3z|V+v zW3fOoDVzRGn2zRq%@1xce5*}=xnC(zpF>9<+RpcL`A1&+QKdz86pLD7D&LP=(o1qi zRF9D3yMXH)p?xs7|pM=hI`Xs5=u8y=t1+urFx6H;rT$|89dG=DLuwJ$q{zq$zG(3JMs+fUu82y3L$Pkja@U#sf~DkG z5P{G&D0=YKeegZYAUB`2=A6o|UH09Jmf5qIT>-Pdt>5+OqZYzC+H6HDG!hJ8AU2xMfWJz1-tGcP@<38>q~|%OEC}mi{#p6Fq<9MD?X(7=XPzIRfF>s9to-Ot zX<_%lV`vpFL zi!Du$v*UIqexIbD6-tdCN|X-=j2T5Gbq6fSJ*RxBB?dC1SQN@CNketv=Jrm&i3O$N?!?nU|l7aZrFf#Wu<69D;y0xlz2mUxOBt|Ps zuRIn%jUtscmjqmfN(Sp1FJwI0J?1kflvSN`s~69C5^QeRg6j*#z`gHJEv}O#=dCpJ zz1-8x6pIuZqEwrNwU2?aRL)l;g4f1&qNhUJu0yB&ksm`@wKa;FpCMa}TH>#e1q_GG z@ifT)=D}tw(>^%HxgPt$TksRD^7!g%rm1)QyB%D9Tf6g$RfJsK+>XfIdF~K-&N_b! z-le$cIc{lyX+Eud;p7afg-WV)dhH&NJ4jBMWW5RmQ=OlIvK>dc#p=A{8ChK0hojBX zYX9*NqqC&VVy|T3yNR0xv%SE^tyJW6j!)cpb4^RJsFpj3ZrTk5@JGi^I3Q`93iqB7X2o%M&AnBG|wG zRw3ZC`mu-G+;wp&CXX+Rd3&Ciw-&7<8eJUW7bL8ZV02gQnKN14XmqFPj!h3x(W6~M z#Y7SXYfi<;Q%Q|d8MIkkskZ<%gZfYo*zU=c&WOWcsmpwy0%`!=5*w%!;ce}em05NW znxGq(1oc?0LU%+$p9}sX9jpOmosJ3}6h00U-U^m@JE~ET&rM_GF?+l#gPjrwT8*v( zhs<^t?DG}R2WO`!g&p`~U%cE!(9^~;b=P}O}t zGpYEhQ1EK3)tb$En(_+?++KC>qN4kfO9oaZ%wPFY))LBLUvaa#hG&Qt-=!Mb%=ct# zeypbE?&zL*QJJ5AbaLz$jM5 z5z-XDchn7P^*y;Uj>4SYDi4h)2oEFkOpx%V@cEL_D3&YQ-Q7Nh*p6Irp33IvL9LDg zhqCve;M;K^GJEpM{ylG&jgD+UzJc!wB*hRYHzP|&OS^J9)o#>}0K1~Rjgp_NmvD*&#AWGG1w;vEmT4KQA?v%2Ze5*~k>fFIiJ+ zY{63DW{2q<eWipt+qEVIp&Wdyf9-`{o~%_qVHe}H!&68R+kBf;>;tvDP1~YI*Xnk z9aWE2x8spIEYm7E7x*04(9CMde$yv#Gik%c9X5&^8c_Y)zWyxgN;auqT@;3Ne1es} z>50y8ES1%|>xqiRvOtp|mjgY~5TP|S%^-~E`Fw->?BvitCTou{dk+H7!^sIlWL7Y{B@CNjoeSeKbXOp%{N-%`B978_ z!?k=W$uz68Rq}ZhxW&^N8G40|*hi#EpV<9#OIU@KE7CI~DZS0!%=2^A@^9I^Wde@P z=o1oTf%jwvj-&1fTQ;#yraMMaXo*Nf({aC6Q+H9oBCE0$wiq^Y^W~ z_^r!$8E%l~!oeo#ug>}j-A3>h`NaY7EKxgIs}epy;kLQl3}aoNtNwE1<(k++j{5)V3qe8OccgX%^>P~Nhg32h>fKaK#q_OlU zmP=p_E9r|mw7(i#vA?@c1B^J46gs+CT{ANubl~?o3*MEdj$%POnt-lSvEjlSc1c;(ttUt*g%FSb%2QrH0#eL|OU*ndk;NhOSF%ilQLf~PR+sC z%6znFe4s*W^!c18GvvvJ7`kfqtBJ0u*u zpVN12OI~9w2wJ{1cYPMJ{qV5){?fTRCd1z~g{43|(fqTXGFq2A{Jo%U1jc#~ip{YT zN6Ox$dqn{wk2Yo47abk9KS>u2D>LjXiacmzqNNz&w^hfWo~C!?V(3}gaPJJd8ty;a z6?P#Hh_pZ@O#m;ApIG6a>(A}4snEu4jei~Y@l#kui9mPdjzg+yvEtfQu{+J8GmLzQ z;0(_O?9nO%_qK%(#2b}$>FWl`67#BGN=oZ;}qdn>ckiJyq)mviJaw1ID_R?vY99eH% zWT`(VbFAZIX^<)3Mdykmgg@O#re#qnFMvi6>Xss@#(z#$1H%ZeZSns82cP0}ls+Mc zwHcm=fQV1~z-33t*F;xRtxiVqJj6&GmwBZX4?WHk&umK3qAFJX}dq1UaHP z>QenD{VJFE+x><9-YfY9iO((PSiB?vJ8sRQz^P6|S#thV{WX^nHu9kE6$g;cSsxUz^tu7f!*RB0pss^lAe&yr!BlwGBz#5m?Sc7+Xh zOU7$2xBK@{?s0kQbx78W?SKL=sJ#wQ3t;ax5O4MLJ0uQyXqSt!7-EW)`h43gBZnVs zi#HUTwNtlI)_y9gxyUz<1q*J-?wbi!c1%s|1#L6A6ost-sUPEjCpS}OSsl>^)dV&2taR6xrYkoe$9Et&;+&XA znSNTh3zkIOG!OS;@3H=m_9xOH&4>w^4npi_Wi$g7UAX2#!^u)~umb}$Nu3k}bhHlI z!(V0TW}2g>jnDSt>^kLq&R!*JRKEVAq{C}<xf$lFZh1L+LeeV?Ky(UX7i{cY zj^nK$E(F%2Pn=_5xx)Le{UchFC|@2Q>PJAqQ8~U4x0P?iyp4R?eA(DqK^J@W2n_xG zW7k?6(+#PZnXCM zS0qCdyl|rz@?CX^?;<*L8$;6)4p&U}{kX%tU&80PPX@%*CD)-70-#kU3kcs$n%=DZ z$J3D0wEla$yJR!*a$?mI96{U@bjM+vHd0lhkkEQ`Xu8+jzs3?mctpmK|8CAAxowKG@09++;Bu+BQCKBNeKlG z;VE&pw}QK#rfu+@F{o$=C8KoC>-+UfB_?}~?8`tDS{DMGj%N@u8MXSoXT@>U8rjAB zjY3u7vI<)S?LdJHJ>mv;`XS{qV4pWq7I+nAWFtkGUKV~%(7N@=fst(bX|JwrhoI%0 z?R;@Msw-ndv+~~3NrMtr#-u#E#ztKSWClC4!S15(49fN6$IeZ=61vQjSADkWzUvh% zQ@6Azo&b_vF0Gen3*}1Nk=$KO_RZ_j;mRfGIWc11H9&8B5f5W~e14Jvu>!7w=IFH@ z&>aGL`UM9nBxe1`G6#GkNRgU=rWWPLIQX6}D!eAaL`&Sv*sIt*I84U`+C<#(6m!f& zdvGVZANQCk?_y7S4bUNEI~1zxc<@XOwFk$DCy&R8R4p&WTX)gW^BtWbs~z^5SBDpq zGSXp^Zc=72@i(U;Y)2j&U%Udp;&Pr{u~o|Erizei4tQy?s<2Af|H*l9?-Jox6R(h} zhp$KLpH+<4SXk+opCw~5g+j4won*6&t`0k(`*Q~)BezRlMHwHe`LmCUlW~B#u=hig zy^LV%f~y?2>2!FHXg&MBFKp0qeT{a(vXoUy-wRlIdmvvYf<7|y}krUrt=8lBIeDQ!GMrdg3Gsw`{M5M>#?6`zBH)Ry!2*E-O;F|R=`<{oEf$2WED zefDSy>U+L#Wg1znY`yj?rO0OQM%tRA5UmI`D^*+l!Unzqq89w#N`IXvRS;X%!(;PGib)T^%sOYOwJD?^jiQ3^HA2*2dzp;zU= zXw`D1TO3`eW!2jL?pBy!)#G9D*-xrY4O`#Od>F%=!!6QzfjEyAjlgZ>yNeeW7tq$< zLN9*|amI@NN{Yrz;En~D!G#82gvM6e4*21XG_@EAg)>(p33=2!P#&4a$o*w z>QaKf0lrHTS5Yro+J4E6uoDSNWGZ9&bl@}>B~A(Y!PKR?y#Yvpumed9JM_Gb-cl>y z3s##YLFJe9Rfn06b4CoN54mu1th-E-jlT5he&@<)@1y;V<$hpyh1OpgP<73XEiX;2w)))tjEGx!(^b27 zpmd#3wQF9_(3AKv+PH$AV;<=hrztv?yo=Y!Vd+PErCfkr=%=hTX@f zzq{f;_||_Lf6U~^dQm3S=JOHv}H|!V#(yAJX5zc zl}>>~XYVH|97dh3Q*&N#;eJ`;1TGsg97B_7O>6~3xV#fC@G-00t`JdJVX;C zQc8{KNCD|2Gd57?Le__CL+jc3S+<(7*xHUtgqGHZd#kl6#n1&Dkr&mN&^cypSQf$J zXdV6c84{l*P_=dhL-t<&kSQ1D73ku*&`+{)nEl!E+z^}91x3CGCtH%^{Z$6_wPRO} z2~22Z@vLVcNnxU)R>J6dPPe34Qsb^fa+AVU>FXp;ElZ_G1zUs1qz5 z?Uf+RCBu$g&M&viDmjP59O782ljCg`^gwd9^2Y*m1A?55SoR2O7twxndVx|4T5a#T zL8a<8>z}S=VF>%luu(g{KAi-C-Ps&YL|c@7xZr28;-Cw$gqcd8DT7i&rg= zZ^k@VaN7Z&ST|At@+!=GM-(f=_(LA6F@akA9&nb&mC0Wrh!EbTig+W|ReRN=(JcHy zA@#zhbml0W6rjz4WTLi#7+j#2OKyMi)x z?gUIw3JY9D2=cN$W(?JlS!+Z(cXPPpEtez4Q1h3!+Il4*z_WT3A`tJ5PUg(Mgik z^R%ma^ZMn5gcqo{6!7YpMLZ#D)OVqI_i3fDB6D*1ZY-|uVgxEX6=w@dZc!Hjp|J!T9VBLf zk4``Yi5H3`uc#;S;@-A!LB$iT;_q1*$>OczNx%R#RRm(C?J)`Z@PU;9*OSbT)7RW;)f;1KBRhvFCbHeqb9Ywwa*h2TYOUwi&!NP!$3OzZ5AC;nv ziTbQE=wYO`sB9e3bB$~vX-zLpm0oXchlsOW3HoEqv%&Hbe%>E9X7&pcP?OHTm78V` zIh>{X7O}Z5HUJ>Er{^({j)@KOR!3+IY!yZtVfdKo@S69kbpIFWV&@}IlI30LM9m8iMG8+t z-CG#Pt2(36&`l?z9Yv*T!q1bhCH+`}@GsiR(eP&O`c`O-{v38wy7V&-T(&N+l@m8m zPG(vxBGo;BXuy2qXnogS@r&1%Go2DCJ<2)zBruoT8WZx0wMRb5u(RqTWig>pADgxm zVL;zw%{na2X6`YhmRS*h_W}av8s$*I7h7Dm#BrIzWLJg3(1vsem-)!*F|XDqiQiVn}^_6^kTUf5+GHsKiQv zwG%8Au<@l+2O=M8W9HZ{PWr}}0L6rDh>5p1zEWW=cnqCi?g!-(O^dOV9!d1pWwA#! zWjr-A3h2f452L5v7J|s@_LGO<`Y&e6MIub}9-G%k+i%xDe>;oim>lE=Hidscr_tlx zI%_d$Irv5Sc!HP6&hOkS)wp~;H{DX7(>c3ER?{&h+yUjN(Vl$O*tRILK;`?r={YJ*@;La+Nv`^UCF%(1@$gC#5LF_eRcV@Z#-W0h6WvkxZq{y`$K_ek$TYeaS;AO{}YY4YP(!kBBv^yfy*SzHdDB zx4vx6(8$6Sm5;8(xv<8?zJDsTo&0zr~5lR^*H;$M;wYl^7wjD8rt2(%5tq4 z`bj4r@srdJKW{!bC|S~AuLeeLv;Q2oL|Aag@$@2wq>bUOy29DYJ;U$IL-Ctu8F%&0 zmh64$VNXc52P|1K2U$H%ANqUI2BTLJ`YAS`6g%ehGzZVGPU5F!Pt6pC!}vdZozmrN z18)D6z~i26RT~xFm1R_RU3qsk6{N(KWo5R9qI^qSX_mqAg;qbgix`pc8x)7tadk-@ z6u3dTwZKhMjrkg}srh?Dzx<;xzG=m3I+fF#n_hX1_WvXn7dF3W>F^g#Ros{!`2I=U z{Fn88?Fx*e_d;|7doIi9-5zAP!Yu}Csu6F!LtlYaD*lGpVvX!RT&yxoX>E9!R9a2x z=sTP_ABnLBZW^}C4reAdOT(l5k6*MHGz_+_=}eLark06l2(=c3FWKS3I-|K0Ut=>6 zr~pPX@>avaYDX^y(tOEH$I6e{$;cbBV4;N<(a{k{N{k+2V_SU@z%oA~OCIoZ5>)b1 z)+Gsv?mo~lX{0U|zE&AM1v_^`M%q7l$?Frw=jQ1lj8dDPIo&nn+}iFxk)HD*wgCdE zw3!}Syt7y5OyxqP@;-dl-#!nDFpe0PdQz}lTZ8L$N%GM8e8`8;eb)}1*7kUx+@cm> zlqey%Enf&JaP>77?lLlBlGq4IgwNey*)yIU=FM7Z%j!+-_|9gzdT72>K42rdI_-?p zO4she{*xMfO8|?=!6()7J!*7$|IlK8-&4iIA14_7*c!VCF}>4X6xC@9%0P{F{hNwg zDTe*7V0k#uVLh|v``KsohsBrI!wJ-G4P1|3-ki``eF~C;xRvjTK#>lA2H?UsgA{D> zQ@TZB^k?;Sr)RIjA?@wG|<)!jCZhAAZKff$gQC0l7>KPg$8W&&#VCcxxlMl&{##WB@GlB{IXyzoS z$hrU?W+1oLq6z!3BlkbcxYGEB*y@TpcGmVxp)?e7riHMKAJRP~zvHk>I#S=Z^KFsi zO7vQJQ~ztn0U)RMX^e~TU;#m#On=3l5PXu!@)4G9|)sU~0mUq#?ItH?nd$gF0 zo#;o(rKPLue?(Zx*H};kStu;|N64n5?cGweu?ckZ1r&{Cy7kNo8B^=8LbIWZR^;1c z`i6^qE!%4P6VOnW!p!cJ&b&QgY9n{h|3a1=nd>>|D z)kFz2lj(@Q77*XSrdI$DC!IoAUnY*mQuRvmLO>eV5!N~C1MgDKS%XV38FiRvs9j75 z-@44EGa1t$oRk%&FC z@VM|8(Y3}%(M6ih6 z==FHPT+IfMzd&mECS?EKQ2OxO`bo;(7$^ID@4U{vfLzd0QUf^-3G(C4qbK{#hu;cW zxxa=H;AvdY{wpLRrF^;yj7Tl{b$I_PYR+J0)2NmTM>H|N*s&BWraGN{hXJ;3CWp&0 zdnfVO@Z8iN@@7(+G2KnP<=5IFe2L!fpP>|{Th?Q;F(Wz%wtFIWSvSoP| zr!EbNn}lbA~K4xh*ueXeG224Ad~0YvghQA~)pb@Hb~Lw&ebj?NJuMXZ0f3x>!F@4sOC zV5t^3BQ{8y$q-`^-y4KaGH~)v0wW8H?}GvKzW0YeGS$1k^I4%su?))M zR*+w2&Ky?ZhAph#H{?dl?V~Y=5uNe(Sop0c=pVHKt$twrO?RyHO{0JMD<&MmdnNT3 zNQCg-wd%3QgmF|vnF|(M!1g%aM{ns9jEEbPnfenWy;MuVQ3|u;ZbxKcm!4DZ$h2%l z*oM!W;T6%O65{4bmK=bS{!*2!{lnaAB`ko}-W@UV_uBljQ_|tk{0_aF*RqRt4Y$){ zRr0Paeg4Xm4YD>C+rh&wL^ZEQ&^H#GAXKbf%gun6BM{<&_=zC@A_4|V!EX8{Y9zDb z-Bo2Ce3mg6HYELsasPWa!I4$^g>Cf)-r{20lc6Uo%e;@m=qUPiKas#$nf{OYl}C|rCZs0%Uz@KN#OE}fT87!M^%@=@1EYOI=rv;)XqMZtM&Y`fg_bEkVRE;$3x13T03Y*Qp!1H z@7ucle59*Ja#ew?W{GPxj?*?QZpic8Ep5KO_(j_EJt?_yem$Jfcp}r6``+J`e)|r~ zHjVd8@9`{(%8JP+;#4!5rI;BHbG+4nG7)YZulW5Cwx+Zs10S15#a3P5{|G9Ww5ZoC zpHHTn(ejqDS#9>bp2}++sU4kYWIuYnZW}^HWf0*ODJNh69yL?cmxC5JX?=WjsK-@v zSD@MK*@+n!)WE&Q6{dqVVtn(^p)8ptNlD--SbaE<*Sk6CTP`XuHZV@3H1pL%baA)7 zURv>!E3qcICf;c7SKiAH1~of3E#B!#@iF)54Xtt1??FAub7*UNZbevhTlk4Wtf6_| zVo-M=aJ;&Pg9ss$))lBHifoho+KBoaAKVl;RSknGzu1?r)TmOj<-OY43U}Ol%j-Va zpeQ1hz>qHo4^chn=MmO3`Y-r-@{!91$C=BYwBEexw*B-iVh{}lbH{ban}|5QLb=!i zeccvnj2C2s&PcX~*RZwN?g}Go^gT6b8c^vPny79UXQxZ5aBBG@Y*)50bO<&TFFv3y zAsE6uwvo7gb$a(_245sXG}4HB1qS)51q!iO0}m$%dG<#p;+$Wol=0f<$`iWY{=(|W ztZ~N~t_~+KuK7#y?FD~`7~Z@Fw5jDu&olykPmUVJsy(kS_w1AYdT$#(2DqoN$I$|p`*m2d z7~#|m;CT*s<)jU{uI>qo?F}E1%YU)hl_%w73wu{bTCKdt4tdq8<-*=fWUFAAB=}>1 ziv5R0q&-Jsx0^Iux^%bXVKXj*;79pWa5hRD(1Ocvl=&lH;+*uNWUax;PRxtEtPK!&73@XQK>%%)yr<^4b?zP-gU#P~1c&4Tz2Al`7E%?p9R zh_J8o72ie-4Ppy*D*Ts{#Q$lESix5W$tf-TyA%75kmJd+@0)rSSurVmv`dapSU{34 zk9cRBgAeNfkguw~bEC>UGVdzoikmX$kH9?bN*X1d4RlzjcBG>6VrSiEbJBjbUB$(DFQacWwe4BRWY1EFs_CdmPn;RhKy&Q7I^2vlszhyMu*hkmxGOU~nCHS1WGPexhxAn)2UC`Kzp>Wv48uBFQFWayzm4GV z0h(@2%h96v7X|C)h@zNNoIH$`VJyY>2PJJ<%B zk!&7;^q6xqeeyv_>8b75*V{B(N*0HhcwPd4++@_uu6X~NKN zG<(ivjNB3Fa|Mmr1hGzxX?NK)jrpdTS`<40yuJ@v+0g+%gw9-dvIb_Y-VeXX>aVWt zLAme4+kf56GkVYi3#*EplsF6jN^fJ*nPX6q26Lh&@%Q@=^%X?yeE z&-{J$NAf*f$#0bmCm#xVpMw0*^?Va#XWo-Q%aiKk)=}u31@UR>d2p33eKl;ntRUiZ z`(V=Cs2BXcPuthI2_db(inYr2EnC=&2kX7%TWG#e$`0sT{)Waq7+J3z0XB?CkA&N| zIv$@)aK@QyV@&)%)!pAq_6e&m6SEJHj}i#XSG^ZV)%(jY`QKObhXUMf=7tMQXy~(E z79(I#B3{h!(YvQo(Dyvjp~X24NfWT0XjCI?T@azWZRGpUn5g0!h;F+-7ONhTI0sWa z$8jaKSZwjWJ5%XX`4SJ#o+|Z#l+XsEjZtKtY2boswVS=I2a~cuap~~DONYlyCqvzdGEG^G>$?oa2epO#CHcBB{X$i z50&e&!u9CmwL3J6M(_LaU-wP=tI()gS5>r@R+oEtm$B{UfBiVON=;TtkvGpn(khql z<3Ds@oG~}?OfIPDAy~{2vu{Bx!$$Ek0=*KW8GQZ)CB@+VczC1$UR%Qf`h|D2d;MPx z%nhEBj0UU1J|aD2Mm+Lqz|ypPit_14OIIK{w>zc(tyafBcF47WWB0=*ERw?Kbvb zHYTn(wj@z}0>>Pex1fIUXmN5%MJHfLkonq;EPdo8qGe32h`)GoJ1|klV)VmlEPFBb z9p8n5-)iyI5~({*(=?ba2S=*J?HvaUU4HBrHqXXFGiC-{Wju)EZNlB->zrmm6_q-+=i13^?F$oXf4^|n zlg0Dq*=?OW79VHq2}m&5vDf&)^VD<)#_`S&1_@KGhFK5*(p$Y9aUIi8@rgz#3W9x4 zMWY55I#i6YQ75@gye-;Sp9t@4{Rq2SX-KPLCInOhmg$2I7t za?%63dOeH=+AGuTwI6h?YCb64AHt%wvc;DNKVqNUf7|*plC<#I zyz-@R6RPIwGaAFX8Yea}^}*}A+rKd9Kg#q^d%pkvd$H#~@!~ZSI*Pe9_%SGZ@7+Y) zi3==I|02>VkmQJ?eD)|%{s-dm*R=P-wc~cd1qX-KdW}_BvR`K`uN))F>+|Ya2hMUa zziX-&Z$+k->3%?eSHk}FvAJINGHIw%=t<~*u#sIP-9VCoj~T+^I*+UC=a-&xxwd_J zx=9hrwWv1#W4XLWg}b{klQ;Q^y~^tv(CYs28aamrN`As#!0Yi762W%`nU`aKSoJ_X zlL01GUSKHdi|9$d+xTFAnbB?B6M(U@6pNecA;eqUTSm_rz5{epbM?I6)6Q17>~If! zm%ZWl6a2s<_Wyte9+#w|QBiq_)^-^UpuR%P%C8JRK<=~~Sd5xcES-;~gcCT&9%EGZ zpmH}C&3W#BiUC{{VA0JdFvS@hU^|=+3&`v&i`QE1NgUA86UttEnPwBquQ455LS41H zu$t0M_NX~HwP)fuh5JOF}acE!dM6cFvJu7(&)%i2s0sX6^F-4_M3vzSx78Y}U2h1c|NIKVs(2 zeZ!vPBT$oxEQuzY){4}w%+y#{{U_vOXxKZpZ}o2WVHo31ZN=g8=*R^VEiYzQTJ@f8 zlwM2M?Xd1rZc8sS&*4ef*M1pzls@$#$2!}+tJ(!|ZuDqljQ~s{0l9y;0NI`HP7Tho z{h3mViA7~A?SZ4bVIODmjOq}Ha^{9Q8qmtvvtsmyvVE~L*{oo{4k_ug&$_2F9*b;lhr5GN1z6aY;lB=voDI}%#i^N%RWwve~S4{Q3KPq~HiEI4G99SXf& zSlW_&^!m%{H8B4l0^)zxyFa1XviTNy+NOJSL3k$||4@EKtu>P;^9F4D9xdvE|KB7zMHxMB_}Hn69p9LUtTm8r zCxCVhQ0my#JeH`jxoC|(mg}Y}LAm)1`F%pJ1C+OgJinj#Hm(Fhllnj3c6*fOQB!fU z<;rN~8%axnKHTyp94rWk0EYKeOEN$H+Lm94@3PZ9sQ)Se4;uXI3v`pk&{{k1Wgpgh z-Evz0u_=VAH6>n6jsYqnG6>Vt-Fj&E1+@>Hef?N0|6~lXueNh+XzmJD&WpF`jUFkS z_E*988>cPcvL*k}_3*A1%R2JU1=a#XA9!XDw9g6CjH?cMQ@VdHZ8$(YuXFVR>_a}p z3b6hE=3+o9((wKaB>1W((yLx~DnranLh2RwtsQ#3**Ost#ml9q{9H17J4h*Zj}AW9;~Qh^CA5;_n$|*K*T44X%r+|ZA`<+*xnUk7`J4)BrYTOukOP; zJJ0IUzcvA^WiAk~|e{xxT0S&1&Z)Zv`fj2#rX)zIdbk&PYmq1!ii@}2zE7kG)A z7YApOFfZ!ffyuwp#^OVX;^CT*ck`3E<@s4&Y!K=Htr!(~X5wIiIrl{3c>dFk*qWYW z!3AlS7_2*X$GO##CQ&YoM#T?M;VXD{FUoE7D%jxu*9_)6y_D;bT1H0&2$tPbhPen8 z%m5lM@iuzDRg^ZRRrLHRe^HY0Y&&rWXtH|0xv8|ELhxIJZZ-bm6}70lUdorR1Fb8+ zjq?OIIr(|xp!4y&uJ5m0mFZs^Px{StjtX-q z5nhUDTAbGhvaGxscyANb`VX`wx>dQH-_JSe<5HjQf;)CSM58=>RNaI~c0X5SkVR-k zqS>MNsZVb$VKFEs<$(LCv-haWLv*lCGE=S521KH54E^yw_-QKur}=o*!Oh}NKPAn} zGN*x(m#1E@f)wTU#N?}1MnuPAz;Iba*|!fz{?4zHeZ2?PtEP8;?%`|ZF`IywW&M;~ z+)25{KaeHOttnPZbv4W1bn=S#zz6vd4m5rM) z>(-7Xye-$MQjk2Yhu5VfZ9PS9RGsZoX=HV*rB5y zQs~QTU>DP}Un2FLkwm{P+&+5y!|V!$Gqt)U%du zn}mbxIoF-qX%|l&pSMGfa1YR$jIR7?X?w0xBFO?N;lVeY_^gE4{RawJQ-x1}Cl%|~ zbMR-^%il-FbF6QKKAx@5FKo2HzT5$%gyNp_1=FE0erJDc! z4QH5{+;sUiwQD8e{mE~K`I{I3(*B2;=STQ;8+Dvg;W>b_=IxjQFa@0ITu|NYSW9lT zh_pq&t|RjU8Z;ir>Cwy%Qn9Fje*mJ9RGX;32O#m%&rv@N$`)n9yo^go&r{g&4sBX( zoptUHjKy_5?dk(1NJ>jq9|^gj^srEWm~b||U8~;c-^%8|a=4l~TTB8fTZ&4Z?(QZ- zz6MO9%#rslY^I+1{C3lH-8Cw}ZEtk3lslp?a)ttTYs>uYRlcDSx5U1{l>&J{>@{7e zY^6W^$f$N>Xv7@ zL{=ek|0{6WK%%HgT7X%0(#bw3-oNIg&XND5A%S#MQvD5&QaP*1KONzTeNq`pNrNeN0cHHqM%XRneTCXD} z>~p5#o!U*nWwkIwuQ=+dK#zI1N)*z=89mlAX zlu@PUH;$d}*PBvoQ))|Nxz$`Y`<&1CeLFIIAs3O^I<;A7Mqmm`RF$6^flj$c8zA}29WL42&L5{>(22NsxFRspx%Ul}) z=}qKa@)C$3{lReXM}i;h70u*vX+aM#`@p>+fJ_|h;Ra}Zr6_*1m#^%54_#)@$ne$( zvWHtTXB_B-QoejjYJ?SBYLD6}tZ+$fZ=L1rN3?;f@Q!};Z5Cru+O``lS?>xdr@0R^ zuJQ$8$+$zLTq$p8w0NZwE_&oIa2W@AEwE(KPSEQ5E|G@Nl^6?@nE*I~jLT(p+ zVjsrZZroJt@)~fnyrauA<*73d?%P4 z%H95dp?v7?e}miWJXZc~Ru^Z*4Ll#~iSMIAW4WUT zyReItX=w5=dq7EkrGYY?a#ts!P4MZ&*tPHr90SRxlaV3w2L;v^d(az-q4;)W`VyWQ zu7HYSy$iw}{aDS8UOq41(P@q~NmsI;iz>#}Hz0NWHO~U$_5|N(K7VCzXJil zuDAfZ#x0nWjYmIoJzV0}Ur031yhg~7j$ZY*7Q4NHi$*9k`t)pIGCd$W8L6;^Y^X8D z8j}Hv0_uOMX9@No-4A0%ZwNMN7Ed=$+a#7yE>HRjBhMjxXW@q0eohQ zR|jodddbKY#>%;zhurn#eUx=)*2)Gr1QTy2mxAvQ&we$^Mj~=5 z4}FWCg$v5_IP1OQ7t`dNkmH3amI&?L34q330092u{pLF>zL1Aaj|?Lcxj%hoXpp44 zhRk-sb3EQ3EY(iwwoo5ZwA%@@w4nmR zq>$}b*h;0h;Tg>^dCIjRFXSEdSDE!FpkjE9Jmgzr;V3z6{}q{CuT@*{NHq_NrX~Z&R#dZJ7>y!BDP^o0#v)l~9SO zU;Hw1@|g9}ct`qn^Pm;+uY~ifi~}e2B+{%+6AwR)q6x9Q@Lzi~x2VI)L`)6=rgX ziISq&d5yf{da-SAylzqIJ1&LW~2J>bH)C*!eN(W$k#pqc%V=9W}l{P|w8 zRuj0dbo*U-YUw_cL_95KS-T?@#Y*;_Gu|QOjl?ZN456+uJr`fzRBO@OT~m8bn+$Y7 zy>e$jU|swHx5Y5)8jZWeg;#+OIr-`eqPzLhp-#<>aAUMEaBI?3v>DgK8!hc!VZpC4 zPfKcWy2dcvl44O;oh?DwO4VuR_*VbST5VElNbPe|Eo)=`j1-Px5*x?u`@F_z-aw0? z`PjIA&0u6(R{TH@CYiOi(_s(yd!732o3Zh4Uc~lqSmbr{wytCAMPFz$(A*x6_6*Ys zezt>U=Q#2SJ+Hsw*f*0f-=oBFe+kq~@gcm@(KI(+VEUH1UAzrL-s^H6R7(gs8jx;m zV5n``=(=g}c;Gh9IVVBRJLd;e?lNw(75UjIc4K|w!8qHcg2eKNn|$*8`L|EgbUc^x z-i~L-00BV(*XM51f^2Q6)xG2CA2~ikR5)wtV={``C_1>@aS&z*Yh;CIUjlpCfT)5o z9GhhoQ}?loI`fnH)@Ea3`3j)c-Svrli8+U9_KyhxPF;36)#~?>UE=u%k#_NS^OIJs zuKOD{QtAh&{nvy*2ejpg{-n!Z%EweC{N&=(3oxp;T;+o=@sN4fjg1tPZY@8M{*oKC z1eVl-g?iCjPY50}(EQf%lCIgZ`FgYD9vr=n!PWN-P`gnG2OW4bXTOL($!KGujwPTs| z_y#=#%qI07mB%gsQp5u|q~2%l$+00i&8?kE3-h%_Eq&++_chXA!rRrdmVUDYD;aZt*W`i1NPVmcOhS}yUq>g)%-{m#) z&_gWU{f%c~TIaWV5+d@?77624s08a*dp1}L4|FWMeclkM>q{DE};*!spgQE{x& z)>cb;+Uns-wUn^o;iN2&jk(uYzr3AK3y*2jD!JRrB@NePu$~{uYe^#FW;Bj}`!Z|& z67VbF*m*0GQa|B(I6Z2}GhtsZYl)MC5z==8PHPk2+6v7U0SmtA6ZWHF8a8ockpeNt zs<>9~>z-2F(i4F-KV4cu(wsStT@jt++1{)f3IO?{NVf4;)%~!$u|G)oAdLi^2v@nfK8d8JzFRZAar_}3HBqVz1Wz3P9!AI z#+1}Gmq(i>jgFbM2dp0Jg(@W)E zzs^I)t=?s4NU;)X!2EOUS#8}1%-2%>>B%_nHyp(rnAh|)+m&2;lhB`78i2w-#V`k@ ztmN%2EqY$yicV%!%Pl%Mj7k0)WBY5^?jyvW=p!_fr&FCeM8CesUGGciL!?30#pBkh zR0crn2Ar_N$f?i#@9Itr40G<`WH0p4Xh-ic-|P;}1u)pR z#Y7F?2IuMuykv9(gx&@-*2`_xByyJNn(U7i{9PDdN1-iK*Uop*AJ20cuIE)yYFtO4 z(t(yK6z%oo4Pu`M!)KEG0%tDt(eMVFl$u^gV6{-8P1 zj|oPUY<~Ve#pp3 zS#~k$va5Bw!J-eToBeJ50dnU`8p}iFFdm!P!`ZRBn}2-`*Xp?_3*1Cv-rOBMo`eXc z_a$z5N_o}BCPj4}%a3vyhm&LEerh{>C0uc7DT3rGpmtD_o3B|Wn6S&iEO_jk5bgtp*^2C zSJCeXbQu@r&R(#* zxal_xxbl|nUKrmk6n_+)DNBGO{t0Ti-r!>cIT)wkh46{m!ReR%wE-GfC)6@#h8#%O z%eu5hn{oOoHlTpITlAKqL6B) zbd`dNy7a~KUy*MrtPz6h?x%cWlr$}%sYd+FqeBz{8aWg{YhrwvyyuF(co8C$__8b# z*~$z)AU8nkr4ub^H>S+_$F#Wc=Eoniic#{jXYwMu@u`Kd*GoP;u3bz|fa}~#j?EOj z(^;P;F9(_l&UTFj1J|p>ymIZmqWc8==#cYXF)lj5E$|)D;t;z_-qHyR0W-hbkPfSjx-`I=|&{v`jcT_%r48VzVXrc9BFP2~<2 zCUAWQY%Dr^c4thO%jNcLa{>o+Do+b41PGWmXi|_G`b85DB`N+m1VJ7a?f4Br$h z-`H4DjdQyF0-$CrDeVsD(3a`!nzM(S?49i)-qr!ZB{|x=*DWckwSZTYqG@`wq%@#~ zJqdN%aB702W5p&>IT&kIS~bfel&+N>0f0)X5-9Ai#rdWhCoQG1c2j#s9zs5K&>!Zz2n8? zQw~4+b>S7{-m~gz0SOK;!P{Lr21?c)Iy~I9Up@uHgtjl>osw`Zy(Z~*0C&KSX4X(>#H0^g13{`FQP|05OBamC{ zIMw!;E09rK%NSC>2ZUkL{5&o4T2Ap3LretS@qa(wU z{IvOA!;304&OC3WQs?73r@5s;*>5LyAPk3wTo^Y{9qs3vaW9uh80?#jQrj^g*(2_^ zociJ3R7ed+(TCt}f!9#^p=SX3$85`Y&u186)zg^-9S(ml;*elc#0!_!J zo=v`qdd&rircN|i8ONU98*L4#GB6cc^=6V0mpRN=hse`oJQYl`ydF%H_38vb|2@H;PrB3G2kVR3y+(3`Cl+}u)Ci-KwKKZVb3a+lPuB^zZs!;ULpH>ex`8;yJvZS_wfxxQp7JiRdIFu zt?RK|jk$_$<{T9(aK1~&Go$E)xyIYPk&64W5vb5O-&Lhq^B;YKZf=L1i z`U|W@L4o!b#$PdfqQ($9 zgITJvJt%XIB#jZ@clk3S>!Co4jk&qPCz)=wZKBfA$%J(@=XS_Rv~E}ee#NVMOPd;7 z=#Wl4ETvbx=AyfO>U>s!O))a^#MNZiWOA)44xVGK%vPdZ6{<%15D)$%#}&QF8crS-YxkK$1dF}a4E}GGH9H~;w4oyZt?S@t zRYo#4YW05lv+rwZ?uQB(cKiDhD_ebhixaUNFsdy&vS0w*=lW5++A1(ZXt^gkYn>M8 z9{zB1_-QD3*kx-*nbR0xMgRAE#mNN$PetH-N9`|Ka;pS|>k_pc+U>KY)+d7*p*2n4 zF}m}0PfQu(efweeD}qAvU8H{pY;MD1^zvsXJ$rWU%?Y~|Ms~%y{0uAXr~8O}2I@vz zIY6eVYjMPw9Zu%vQ-i@;Ou3I<)@9brAI5xJDH%}|*`XO?Edl%MF6>e9f&csaAFb^c zcYVNaZGw{) zTc2nKf3ON;Ji9m~?EiGKu)w(x(Q>0=`jEBMj5<@~X0iva>KR(7j)J%0(oGa(Nuvct ziPDVRKUh$$Bc%UyQu}9}?WnZe6s~F+6C<*I2OUh=VbBGhIjz@kxgpM7ZW`2SOunxf zY*x(%lxwjX8yvkaqbCa#;KH!X36D%fwSYddLaikqZ;U7xRJkt1F%-YfEyhsaj$C{X zJ1Mslb&D1pvv#veexqLFnxg|mJD*WKR@$QAZWQPZt3 z@wZ#JnLKW)INwpr95hRmyu{lR)zEvh>XJ$rUEcIS?h@wzz^eTh^r6 z=2cH1wN&aS~Lhf+MrDqC9Dx+C}_+TriAm}Ji;-^dn|L^@?0uY&jHa)Abx z-j~eL<`A0q(>2~!RTn$DK~EDDx3Be~cU&s@Ze2Fq7}p(srm?BVo$q4y|5#zN66G;kS}OL$jD_a)cs8d)_sMrl>4fWE_XF9T4x4oC_bJR zw*}bCqtb5&pCx98>&@j8iDon|Vk=k4F1B$Q(dtS%aNk-&&)qaZCPGiQB>YrHf6I9F z&cOlFX0xN?92@tl1``f90#b56F+78i;(9yYnId03sILDEYeLqfpZ3JgRqb7|pJ+%+ zTcM{$-1EU09~2$XCLx{&5zYSj!mZs@n?}$&gdCOeX_iju)or@!w)&A-moz4}EdKry0Kd5VSqf?7=YKU#&n#?-xYaXUU;<@hpd_;JB(G_~ah-L*p z?*%gbZfiivlzK7$OnRv$-Tj?pidQlls#&Y5WBuD4t}-GT_F8mJ!ubPvKEA0+MXfhX zp9B;JGQKEzcwd!~EEQFuEw{bM*SHzG(bSz&!YI7E7EN*1avR~T#zO4db8>Z+dupWT zKD&`KSUA{j_yI57ANru+{ui#%{j44*s=Vt_1K+(v1O6$0J>y`|>89$BXk4^Jx}Jny zEr|28-4Mw^s}nnygn>>p>5{7xjZ=k&QmZ`@YNGl2@N6FwF{l(~nHprCB|3 ze)V#?wXF$;C+>+(%e1g#%Ub{7#0XkZkw>FH)?&@_8bPhVCgvVLcZ;8k1ZYKvps#JRJat9cd?x z5G>E>Q`${Ete5m)o-Li-`+dasbKy;KX}VEgGD`;?`P5x#HR-Y^F6`CCile!b7^B84 z#>tDiPqg}{hSo-63+NW7h%J_P=O%u|ct)u329=xe#dV_8di%18S&B@mhUpDsbA5_) z;0U&eHf+U*uh&XzI{5Eu1dpJkTF0}0jgJfxf5Mz0av*6#$oDff@#^lnq9~?KPU>aNqO?Yn0a7sDMUq!>!dsly36m2myaG;YnTXTgP?O~v z;&qqEoqR3RY#wiEzxZC{lZ^(%VVS`X362;g8F!cuS}-%`f8_UvQ58|!lzuXLJZJpV z-P?&?LF5{+Ll_i&iw@>ET*q!l8K3ny&y(K=L%*cQoEd7CZwk9A@D{%*%~;0FVWe=@ zL!IFGNS6WKdl9RH zPaNO#6?+t2aO}7w+YFo}1`e_te?s!yD(tlshl2V7B__oGv8~e@eewkuy=yFNusS(e zNRZR3%`lI2-lO^3P}3x|0*npK2C=do3v-fg4QM;(Z46v@!nGhwxETz5LSYhFiU38|G9u#GCf!KI0pR4x_j=$!jkB0;) zZG@ph?|d87A$O#;)Rauf-bnZi8-yJUd7na$At*DUhZu5U$`lR*9rJ!yn@}IAf}@=) zy4ve~^1|qLQwA@&Sd`o#Br@@4Z=aR-TS48N^)Rk$r!)Q%*AJ`N5+fFR=Zf z(fS$~IVT>#?++U%euO>r+PqHv$^IEG64$ZDRiJB_j%|F(C#ckh^u(Ij(~E3oL9@CD z?EN5Qj#jN-hp+v*nIv+Z_f&ctzSAkCkk<6fkUI@qkLQQ=;;6PX(t%CnJGfV}E2@^* zMw*lGLJiB1Mvy9a#5FDVK3T~8%GlWG)lJkdFbVo98KvHd`^NKvoej_V2{;bflHoin zC6#ry?wBHad2wkEzM=)Zey`2zi$p&1J^k%Vl+%J%<{uif?q$rjd#yK?ITM99^~iDA zb@OWt?I7)l8)pvs(vxh4mlTmYh$}!3AP++4cSJIS7FU?JoX-xl(LWZY+`Y>73|r+W z`n;YrW8@|A7CSHG*AYfcb5UspaxxkRdlt5X_Re4obp$d&yN}aPjFBaHh5jz3J8EKL z;^;IK4&cXsjpNQh3QRWW(ovNe<5jm~XI$dZ-BhAOEJ7WI>}*_Pa8&)GRf`n@ON(=4 zix)FOblo&98R2ZwOI6?T=2)Y^Ub{Ru8nVTnS~v+@xK7SaEkEc`D`~p1Z>?ucnuf54 zSSh4IT~1jVPoEKG3GLztEGd&UIhDiElgG%F!(JeUyw63!4dHk{a(H#pn_e_(w*6)N zu-a!Bf?wsWVfOsVlGQ?ROc0O14_sRtuK`{L^)kb^9&NTB&e)*!yjr{YgWHO!GXT+*b9{gTHDA!9WE$p34pM>gn9Hp&v@C%r1dV>wW zQb|Vu;u0Mo4cfiah1(Dl;hYLMCaWa#{rCx9cm z>MX7KQJe?j-yT{{FvL}WHuD~3>^&1!aUl#bm$Z0vVu^g`c$@yxaBm-mHlFV{IT z1qwDV>Sl`)0;_E{)E6o2{6coYp%K3ncKEGpE?0rfj3zz~Y-Y#7NB+78Of}_n`mf1pw(pZA)c13)L zyobtg(=o{@^xaG*@fV*W*yn1h-OQeQTD`cRafu1dA#mV!8zH*xqTkDVDplhOZ6wY7N`^B7r>rO^xR(8xtH#blsHC^@s96B@j?4 z^K2sUSD0&8@F%>z0Vz?BiCnz1hm%B|yxzYntWrfnGKL>(f_|^f!f}7GuXiAq=kZ=h zs9mlgq7f1O-w{66Z~SfPxjmZv5WCM1d9hKq!Q~nR*}+g@W}A;ow?0SUgz?;fYSpb* zkFITrf35^|>KUBkLL$G^q~^M@!tuLLnK_=9@l`efgt(O>FAiMsVQx^YkQJp!;4F%) zbD5JD>yF&>$@`w2dQ5q%WwxA6fkSICN5i2?RE+H~Vt@%j=4+?iH*V@f$A{{>zG@!4 zx6F-gJvKiNv>Q_T7F}ls622a4+r_fY5HU2ie5^eSk zu^08|%q6u;srQ&!QPc|C=R3~+)W^mRm9VYYVdZjtE?cF`iGH}P6*DOlRpWb;4h30Hd6P9@6PvxMz4PIO?2WI1VJSn6sSh99y$A9s<)t+5(HA$^weRW$y`CO zoFMs6YBaESG#uacRZOA=g*H$6;S05D)K%kZRpQrhBhj;S9h&Q03s}0s5wmgmR(yB5 zcZ`Tn@gF$2fz02Fr6qJ265;Hks-d7ExF47J*HxB*jQzRU|2Kn1aO^WU!7ayI+WUrVFDVl0wuya&?tvntXO4 zcN*LnK~o6i?1nzGD8}DJ=cbAR;b(9VM^JcJ=lD7OF*(GFB)jY5 zGVi3T(VfSR%jn%*#IUk~m5?e)IF3qxS4y3C+nzP zLbvTifRz>*f*020e4QEOB_3M8u+ILyIN?mtzQy1GE)Ew)mR66aCdN8@Wpdygo`XoA z|D(ZRe995Zz&@Xomt z#gi>}d)ptzxA4s~OZ=wFQugnz%cL`NE0O@)1A{9;*5g5aR|PA3VGHa=uKx==UE?+1$l3pWOrBeO&2ve}|81MfDw zyFXuk?SH%D5o~L^{o1l%d5>Y5f3f=T=dMq8%Uhcom?uuH(k0{E)&=9rg7WzC`?iet znBYa)JOfGL{g&|qXqmV3uDmfj+{)cmXo_^k@JcO*-styQ(We}V(;o(BwFk|gx4?wh7%*Se0zrR}3;(KqkBJ^JcqS(`-W=`iZQ@wNO+>PNSJ!`ry61*qen zFYUVj(8_EAju(E1-R}T5_yMC>qjUsiU+tTsPGn~KTWZx#6DFJ2mMV_s=Pi{>N}P+x zaD($y?=@@$Bh`_6A%mSBqy(K0lv1*TQ{oe|Ulz*CIfn!hXUV z?e*v-WXw9n3Y*gz#ME%-FJ+0Si^PBmi<3$Dgrq}r``kn!>5+(bEz>i`#^RkP%v-P6 zHM6O3-r&5_eYQ2hA)Q+4RAXaRH~kXY#3Fw|R4-X}2xJXt^T9cwm3d)z+9+v5VdnfiOO|XWSIS0 zXf9w&6l!%RxiNkTj=hS={ORr@S@_kxNyg@C$b-8W`6WfL$$-gmLeeIss{hC@97wWm_muxHAf)|Kt!2_0**DVyng6kLR9t`!6TdVAkS5a`G#h1^y zw>hq4;ZQ>AwgfN{xA*;8D{VG(kGcIa3Hzrq03Y z&ct`GwmI|A9|(@12PNBWu)-TV*gu0=uv`=I%hJOw2OCfOWJ#j!kP*yv6=S&B8LEw8 zxaLibpTCMN&2@4fCsM1Q$%F54Fo0T*%J!8&T3>Yp5W>KgZ4J1DxkTL1!M0I)Ku&CZ zK@LUE@@TElX{t8Q#32y|Pfea^UF#$P!uSbxYjnx8`mgFCqpVj05<L0uhx$FRFqnVex!lTV%cU z+(n5o9MAkIZ(=($MI&t_wL~&EOvpcHKv4f-WE|mMj|BStXJim$#aT)Ah{pXE2Bz**)95&Lb=sBq?u(bK%HWl>rb_WeZ2~%x_)(`x3vLL(0S!F zKekw5e7##$^Jyz*UL94p&EzLWks(6gfXs*6N-z?ZDrcTj%r?om2&Gul&ML$|o6(mHG8)`jyLQ`8A? zFJDa>Z21Yzt6yu7Lk5GFQ0bXq7^AJv(|}8;@C>=%p@6JELjlLa@HI!9@06)u6Izas z40q@En3a05tTorhD|jSAT@H`DZ9pm5#(BOBh55>9L!0k%0sbn|0L_Zjklj5j)iEB$ z6=uHyBq2LGGTg`^cVqYS<85V9p~Lc%s~vrcc1MYgucF7IW@lQ)^i2_gcN9C7j-DH? z*^Twc_#`91vwt05R%aVfuP(XG)gQmD6RE$f zli)+ku{JG1-Y3B!bc3x<33H(@f7-q*CTx3Nqz#8__~f zc3tpZPS`jf0#GdKBS6oy9}KGyZb4EE;L0_lazkWQ>c@uM>RI-N`hzj__jDDjoPBG( z%yrEyzJVg~oQuZ^HjT{ZP{zWve{cmcu~_+Uf_;aZX1y$BlOZa+df($ z;=EqiVr4zsw%gL(+U(OU!*N)GQ^a}go0*P|9zH29thEmX&hZ0sIi37B3a>?lR-<0v zf$DS9yn?$O-b8Z|Z2xP{v%>eqD@eS?K;8syDuI8hC@Zyv z`eSj&*Uv9P>U%x{Dv_A@&C*1;$YuXyF_HqIzhAvvtBLS8rn&$8q1)uOtKjOvt?m{@ za?;t@_Jo;9EMIV(anaT*ovj=pp;f!oKi`x%LSI33Er~#l#_GDT5S0K20_4UM6YE;< zGX!r0($9CI%;%p523a=oOjR{ z+kF$v45>+0c}k>5fc#v-WNTBCkHzrj_QgsVyW2Qegj@t%@i?uLSH^GURV)nRhUi84s9nZQJeV9B$Gq-|(ZNE`%Wm;=pMo&>S z^6#tHN8`KOz*jWIvfmAl4=Lzy(1PZ6x@scbl=KKc?a6X7Uh_qVhG@*QGUDi}b`C7Y zW~O7JJK3}|2PXBr#n~&b*VDILUMmDi@CC4bIRUzj1iR6K%tmr~U!Xg&VC&8!dIBG;jP$uLak`@$>UJX)k$CJ#v=v6Jqb zP^PHfj$#AqS7rSpM&ZXFfs>d)GcGbH{J=T&0UtvFEl!$5#P~2qK9$?P{&$OrDRr)V zDpo2(zctVB$9bJ3j4pUI+3ols1AG(6blUL3@|3CMoQ2gd_WNtM^oyde?tsewnLZB= zRa5fPoIe3i?N!+_6BAnyjCS8N<~2pg!vPfwmNVfY05zD;oo@M$26w_D zU!RpdoVAGQ+&|e?l*-Tq4+8@qNJ2@TYimBt_2FR1UrSD4ozv!@Y++`-)_RT}t`DA0 zx7BAJ(vf>swYjwzKb{)%GbuCO+3sNjoH#xb-gy8V5g~K5?CH7Sk-~nrIb8mk@1}2L z*gr$MSc0eMLDWPIL!#Pw_Ujv%l*Iwblx7f=lF@a$q?J>Uwg09|T2)tlM{Pfc*9B zfE(dtRqXQrHRs;H5+ou|skp_<PCvVCS#8)fmU#?DCf!MgZ&go6|Q5{Gz8p?p|0$`w7g4U<6vPAAdcx@w3q zr@!<8N%#&1QcQ>XWfB%5U0G_Y{x4?~!{a45pVivF^#~0pt+clS>07A#9gGO`Pk66FM48u|@sZ&oI| z5GkYsBr#P5^}{Pu@8zEq0d&Z}TMp>G;Kh2je#&gQ56N;b=aC8^M#E$hU2p!SKJY0w zFzT<(|MPJGg{7GPK0x}+te=L82`8sJ+n>UzalZQV-^1AgfefMljBvI9kMKGr=<~Mu z064frWdua>TeHc{r%UhKWJJ9GA|CJ^G33AZlH%uiCvoTe;>}XW2p9=1nNHWeo!D`b zD?{s(4p0jB_d9t0&4?XmdNCKHu956_n)f$lFo=(4@qgC!zs!ZQ2V$M37f*Bt+99vp z%>h!WjOj7HNsyKfE==H^yQTbhLMtSmT@$V~Mi1-WC8G_`e^e4HkW}~Y#;qEZGQJwX z&H8fkomm4!$lvW7y6tWDZD#M_WfQpP&s`U_{+X;gS8IlRa*;!^!$ODt&-3E{lpph= zQ%R+TgXN<@`rDsF)Hvf$!{%7{m}+&ZWrg;{;R1GRzfP+1;WYg*B3?}@b3*+=35jk- zqEXR7Wyh2WW>cJ$Nd~dHe;F!V$$effH|_0VJT~{uswp zf4vC$Yl(D8JMIfI>5CM*s-*|~Q(zzd7@A~%8B)+0duyuTOsnP7-UIa+_npSfbcw!_ zb?qqwmDdv+mUU-IeJOv;25j0fQuyN#PS1q-V@JY`j2H?udE8x>ZGTI5Kz055#p2`~ z3KJ7->_^+a7!mJpuCOS%Nay5HM}1q#)-(z9N>kKKsoS>kYjkZha?=2?GXCp1$IfNH z;>N4?olDGP3uW8(wLwV+a)mn0gq5QjCXd1;{Qq!uoncLF+ZIGXq$(;%6&n_cNbe#l zDpimknv{U_UXy?*s5AlTC87e-I|(hJi1a4Ch9WgUfRF?TNnSkXo_mjP{|S5_>?CWi zwdR;(jyaa`AMxpJXODrZ*VFk&Z@a-8<50ZX$j2(EHt~FDIL&c1ySWJe*MH`4XGV+E z9UUpz{*pM<!!mt!(7uasL+l%N#H$oyK{q{;Mc(-I1CPSc)3`x#|t=_GQMT!1T0e zI!TpH`bY*rXTO!}9;r*2ROnxzyXipz1?S&ev(lV-?-%jSlj8IJzoQm!tTh6!veF`Q z-lv!?KCiy^J$6U6(Z7}|LZ`y-8pnu#0ls*O$J#-KCs2K1f6p5z-bDY4F7`gSs8QGp z!S3X7Yr_Ne^BLmk#fpV^L!t5mmWfGZM=Z7YMl#N0SU%B`7eIzig|He~!Bu*}Y$gt} zQ#ED3@!x;0U(&-@y*k^hF8S@N#}(OMI@+0J!%pAV!cJ%4mmfC<9pq>9*YL35v9};{ zfubg`zFOo0pCtk~awRwkHf4%!hNc*MA zUCLX_9icv_QP&;7E}b7~{7&vd;K}>OZ$JWw-;XTlb0w6AGEB`t^Ug?b-7?(gf5vgc zF4aYu8ZqAg^UXUUx9kokgHBBt7W&vcc!rH0kBj_P^VLm-oH1qcuE`4a=U6y z=iP8wU){ngtmEfU)BU-)jXe*^2QiAo_G(p5@mf_t(@LCt*j=MI7c-N;?(+A%47_g7 z(jueXLVk>cfkl!7JgHOk_nHY%EgWvcC9@e(F~;&AAc^{Z%9{23rO>N7$W(lHE3do= zSIW@DCp9G4`fuiLMOX7(Mct(L@Xq)3G0nT9xBTh<9?WV$$Ulqw^4=m#7J#oR>m(X^ zSNaB5YCvH0R-d>`Uxm*{>feZ`{5#&iJr;}zj2Mb=(5q5Q^0RllR|LzjEcF)PA?vR` zTfR=sjYrQ^Z5+KfL|)f{-AscRWd#0vBTNX_-aS^WIaKfr>g`5*8-u9U64d8xDfVxy zcqWj+bQl=ByUE73jSt_X5^d?my8Fw#DrhwIrbjb4FrMzun~rjovR6ShGs$0(?9^x1 z*#E~|c%ZMN%In_YlFatsb@w2ly+fv*pSpLQqUM>#;eUp-Coux7tgf7X=lbpItl>Ot z>&9^7rwtbxjqSIVafaQ3C~K3ZpyPnIv1I@H811GYi+U3U|FfMAkAb6NWE8dk?$Lke zZo<{5mhQsOx#0nR5d!#{EUz>qZzNB;NzKW-L)2LPJ4Wa${`uu;HTOy#YY+1d8J19o;_3zy)e*Dd!5c5gQ9^?a| ziVxI>Y6G7VKl_YMVE;6LqDRU3E&loMwP*H6mK)EdSN;rfEDyV;qdKz<@2R7*vwwy& z{e(ZJMI+7XL#Hu$`>^Pay8#U;GWn=zgzG+q`+lslnV=DMz5V-hM>k-`;P{lo;lT;y zXzucVK-eGQG2vt`N^GcU{^KJI!S&YxmMwVeK) z$o#n&ukPH)a)X(Bd-!@+=okoW7EWx{#=Hi!RVLa~Q_Md>%}47$QvrQxu1wEYdD=YH z-?mTQV@{_X7UfUvgtq%SAO!TFpJfZ%N!M>Nt?!sPl$kAC z`!sO_6V~QoEI!Wk4i@}jHHhG>Y`v^_xiM zSdhSW!&$Ggg6EY_pJ_sUH?YJHv20sh>TkzM3WU7-F{a8@I8*vJY$KFho6}@*GcQ5H zLkuj%6|9Bg6EqA9P82_1w{~*byAe1o!Tu54Flyy0d<{f#Pag*J-YRQi%HX1WbOQiA zZe{hW&RVYl$Z;TS5f#@2jsmtT%G$FBtPOL@t{@Z`jBXdlnIE=kpvj2(mABM5hBMhX zRG0xh$$leh?H`X~)?$~*Cv4QbXs`E8?mvH~Ct!5W`!np(>j%YQ>$DgFx6xCd-QL78 zmC^4zTtnw9wWd*xU{Cam4ikDlkh+M0jhPY!GZoJD+fAR~-ifMnP9Fmrs0U;X3tR|w z!cL1iue<6%jzN31lr+oyO`ERPfl@Gl)a7ZiTu%ORr0c;ue?Hww_$xYND}q3O1G=Q# z5F6Osgz*2e*=K1*~BO?(x8L+hX2+Qe05ijMs>yn~jy+ql5CtaI&} z3dv@!bqD)IG)jzRF3_)|q1?82@fr2V!s160Ms3r22LU;r6_ESVQ?>MRBwYP9W@Vei z=>sRa;ZTDwKCE})>M4BQ!5CG$^(rdQ@kHQ{$S@fcu?*1c3@U}@wad91#j`?YkD4k4jPEWtSvzOZDVJiFQd{Z$& zE7^rMfg|Nj=4qw8;>_nijt?D2NNv=N%CqD|r;tQKr|C$!tVN42(smHz3BX5{Vej(+gCN2~O+%~gN6-m`! ze6k6PzX?&vLkY15%^e^8I0rx1L?F!a+NhO)4TJD^2L`EWZo**g6Xbw6dT@Yvb&?69NP93~QhzMABb?ke7JI_(U2+n7<$Kxxo;LhgeTC1WMU{ zU>=}H{IR8}*gP0n1CWM1DrT9)57L#s5lAmzV$@t+ac=IAo0MaCvHAJu!&#li3F=zm z3RcFa%99?=SLA5WU4l=|GW&yiuWTa>9QMm>EAKC>TlhoP==v}rD6EEJxy54nXbaB^ za7>VR;T^zQLzTHc54oveN+y*Xi&kHoa&Sw0IDQE-Zr0-ap4uk8y7P*V(I~JNJT7P0 z(;Y&oh3;ss6x2~q0cqTJ!^nAW>~ok%I0hhcU^aQqkr%dJS*P^uPa>rwh@>*DdL?Sj z?s`_Xc)W4*?W1h((ypNVN2=Il~mOt zAMk<{H)lGvR`yB^1~E~q^=jdsRtFHKuW3nc5y2O+nCPtBJfqCTBEpk1XWrQsvbU}< zYO@#G>6EQK6Vr7uydifLa{CKC@GdonsCa!Y_3N{L!W9*`;Rv6mN8?vB7jz0F71`v% za5NciwW{iq?O6?MhfKwpCE^pkv@r9uE+(m$hhqSCRg)n{e#Nt!z`pJndjFVL1eCjW z+bYwiFp6;s<2{#fh_I^4+3~vhJ`$z>bh8nSc;pFJ%?ncE$i9ngbO?=FOLM<7UgvIS zlsy_5X2(oBIfvOxc+7aWXQ|H2lV4+Ji~GYY>>Gml!nDv$LAjkar#p*GJaE2VY8jd>zZK7TjQ@sP_J5i|nBE!TAeOyvIYcPdy9NuE8p~H6e26(b2po8BZoVY+?Ppdw6}~8&F%+WsT7adO z`_5?>NyTEH8+7ZRtJu1Xm;hygqPjZJ-Li6v}f^UF{QxTVkov@k@y;XUgtbAo&Q z)m477OR29s06l*IPXjTON#y&Jy!yA^fSQGxai7f9Nt&HXL@3_)K#>)>lANtTptbuDwlT}|FNyHX7qwQb*-eOzG2IyA@K=v`#RQ$ zP`SN*S|}1D-muCl?SShY1D-#(VF`_3B6EpW{`!K8n`(_k<Ya;-^K67r8_T%a$y~7~G z%xMmbWUr1bRK|7KMPH%?a4V<=f!JNFj;?<4tZ4VzOyjD(#CL4EMt%V_Lz*|>L1~6HgisF z;43?r$T5QsmhGMsS;i!GWHr_14(6FmC{gn5ml|6N+!ZK3;W?6!JE}!>IU`@O+A2Y4 zNvz*!gg9Rwm+>l37rxM@aHn=NKw5`#Vnar%p$i1L+njud{9vU z;sHsYw2{#3I^?^%I8*%c;9fXeoA1Dvi?JbWfn(2andaUYv66N?_#n|`9GjMT%4F8D zU5yGCwFeA`IL@kiKNzcSe{@clQT}O?yj7_vhmLf@;HUbva#({bn5Nc%UsKWOHP&eQ zORZnTmGfjv_|+4!JF=Si`$wq&TQB3V6TAHUpqIvILt7EKT7TZ`vF*t?SlJL%6Q~_Q zhr7_5N|d$ROmGhd%XaRXIzcOC7Y^PgA;tXT8c?>eYBH2Xe@!r*v^$}6gu;;NJJASg zBX}&f6h6AmmJX#v=ctq(n%%2W7A)B?m<6yaqp|jPcd&k@6AT}yT4Ag`Z5iQcwR8E6(wNYg81zH(R&p*z^*)AM(}nmKszS@ zmi4`qxt@Rqp19?u1(h1AG#(6c%%8TRIj1}9Go~0be@fHzF2k`qz!9P%d~&o7N9>!O zzd%vhD=zjjf!LCSGfj~TNA|**+zY$C#g^IF>uy zCa3Bcj395%|s5(1&y{3H)<)a7b z!btteAlgtP5CSmrrf_Dysr1_Gng41g_a9)G6ASeA2SIodEtBI z`MM9Pcd3gsO^I1c~=tSf(uoom(x>e|@Ng zDQl`Yc>5@0FnA85DzCS{9hW2liq!Wxj{4lNYU%##dygOFw|U`;`tA z)zeg`>+q8-Y3b+ro^8x061YZhZ^mgLo9DcYr)KK;U=;qQ`J5j9xQy^--{HM)x$Vuw zN0YkF(9oZWs1oH%{3hZx8JjzTbSSN+W4}_7D?$zycG+*4^u~i6>n<7#brSMWxoIBV zK@V%w1l6%Fi7RfIr(>0ev-<`Y+A+TipZtHY3csj=Tm|FB9l!Ytir<)Zl4f~<#iw9w z0vWYMmouYH{UBy+Z_Uoi$o&$qD>dAFu{`kUp!}?H`b6{4LOud0UNEKAD!5BxY3FI49mRjUPv(|0FF>yQv^g`sE||3OKjrJxxhSq(RWxK<-QA4ElQ8!rq~C=rq6vV9Ay_X@2+T_omm~ zVnRWOD*#H{MYm)*PxJ(j;|3P@3$slJr9)j;87Mgq4kYHg?l{^ue-Ev;HoWWhBfb0) zJ4qm@wFy<~aWP;J_oHE4T+Q{!l~~RF)itwDePvItSpQ2|hGnE}k`c=xF+C8O$8gz_ z`7}U`dt9%G;~u$GT5#+p?P0-_EXR5NT1g2{Kn}lv51Sl~zc}Dv>p1Wd0u)}bL>ptU zU#oky2hlqF-jWV|vli#7o6c)_lq5%-O{@G6Uip!6TT9h=*@a_}jGE5)IUL)il?=xi z4lvMOgwuY=b&1z%&kOejPEY#1>7BlISg$IRaLC#2KSG702VZmIJ%WkU#B7Row@UT( zIP*pb#jw&!=r451dtCvJQl)z!ZGaUu0W0gGR z4eJx_o*d5$1;%&wsR`_~lddWBm?@boAFvH{6~Z^9ioYlf#agom$Gx%c<&wQjN-Ac2 zim{>}N2=)WdN4YgymXv+tJ=~K@T5b(sE^fv3xV5hoa+~ww~q3uxComuUv*G=fb*CP2TxxC!&+`_!xCw5nZ+|jw0xw5QD?wdvEdv~s~(^ft|Ahk~)! zHVeVe3K8VQFcZ}{gZU{2pTg5@K;CV6%UBb?p1x4iQpEFRfMxfn?v_KnkccHdY)dJjl2FTLdwb~cFA`S);Hlc8_>SV0uz6QRUA+w`y>p8OZIbVwCq#EsGN<%RE`Ij_^)IoF(5 zXTjURedmZf3h1gHH^tsh`O!;IGhgs33X0e3H``_raX zWHrij;6^U$i=|%^{_2yGGHqm}QPRQG&M40!8C=EB=M4uh9RZJ{?J=#c6iJ{Ch#BTO z;87?xg1oafIB)wwSof#Gp6mLZ^`qAj9EQTjZU=GJ-+-eX-E^(v7WLqYn=Z-+%&L>^ z{xgTXPs@n@br}F81ZwD|{C80O4F_awZEx&q3>)6^6zPTRpTsetLZV&%11ns zbXT%53edvp6?7o9bwx(?BGQEZxLp_?rcHt)b$PJd{L#qf(fY`KSF4lbwF28nTiy|( z=e#?ojQSk^sEXtB;YQ@z(HQ|7S8MQsWy)MBU04__9bn3G94Z>sqB2sKF-;3cl73DY z*w3q^cx9g`jhV87EM0aKxLbhB!&@)AG&MW)CoZYLLP*_#kqa-|Ko?~~=$j1zVcgV= znDs@{zQw;N)yBZqI3*Xs!Qo?NA7JyIVuQ`-2k|jjJ*#oqmU9DwdFGb)@z3orj}Ma( z7*>x-|CKe}{AOaDnXdDJz%SA4B4BO1xIjlDpbYHOR(g5oo7=r;$g4Diz20vjJ0(3I zI_kQ|S!+Xc*q!T6KSL~RsWY}N+Y;MkR&^VZ<(XliZT>L2#Lc76R6IbeYQkc|cpMf5$f=ZTlahToTX7?d&W4pT{7@_BWp8zXnf05xYB zi?5~dOdiB0y6;S$Bv7(0?y^2vw1-;0^j-(AU~YnD1^pd}5vt=gk^>)J@Tk=W7GbI= z@bfAzwcj=>9XqRP^}B98T;#nx3&xW&CLH{ZM?4b)k+be%$7Br82T)L#Hq!tl=m-4F zp|<5dc-oHwb3uySjni}GiHF(#O~a)4K$6YiXW!R4`BXpQdV6>^$bVmC!pLTCTGdq= zvpN_TCDV&^$zE?C`P3!l-5Z&~#VBshN{=j+T`Mt&lm4dlO0r$1#vIymJS&BLM97Fj z5E~*6hoy#FR1b}pJ3p(}u3!RHkNn%(Zlm~>kgA%COBO|5cAl+UAp*|j2bJ52zY-2- zTYKLBij0kNh|U0AG&?7V-8#5y{nmY+d-_J6W3liA`Mm5E-3d8a#UY!AgY8s=rO zEHC_obW$>(-Y-I{))M6veQwLUZzYVapzZ42@)!;~hRYf&ifax7OuH|10bCU1ww{<@ z-x@nf=p!(KIOk+#d$KZvW~M{3wD})vOV(dCmcI~mXAs+RLoU@eht$}(vXt>Cfl|J+ z7)Q3aGo+rMeiIFCz33#786`;1NnA9f3LZc7jukB+tpK0j6YtK5vg1B<+MKSoq!FnK zbd33G(Fb(NO1I7{I<|0DE;eXA4#tK{F?*4HIrenEDb;Mac#|r>j8s202?wub{2Uoo6Cvims%7yJ6#`A$ zRy0r!L^RhPnOlA03&|ucNcm>1suOpScG>AYArxf+N=wD>$^2ql&mrOWc7MGN+L1yc zB(3XQg|f1?j6&pGxejS+H|b(=B}k7tbRs1 z*;I>3+)pNym@{jZ3b|;mJ#(=72<~nKq|G?E=(+8Vlv;nTDOcy8Mj1KOjic1gzDo|y zP_L`E=x5)e!|8wy$1%JuV!G*$N4=?Q1<#stHaBBw zt%@VWE+!|{d5s+lR$PbWLl0baW#K`t?$f>&`eF$0mJA{(YxH;)4GTeXL6|ek>{7Gq z&VM~@UmA|SEi|0mn$#^KQMMXvhpt)X;>2M3Q3zC)BlFNMrNz~?T-!E<>wS`&e$&#x z3J$;zNXJT!Y?)xaHbuYjk6^YtAYmPan?~}PEtI1-c!d0=sgdSOh?dZTg|v50ua_b; zBk{as;Ss+zic$H7j73y?l3iuMosg*hL?O9XM9p_*ml+dsK z9J{lz!1O4;QF?!WDlsNI6VjfQ(HoQ>U)K^=kYkbCo29Pu?ZlX^>#Zptnx*;)bVWz+ z6}+&lfqL|ENhEnUQ1sLu{&yuTfWJ%j4n@WB*>R+MM+!X^A6u4k>Tmi56>eU8s*tR6 z=f&pM7|<(wPC)jRvz-lmoU->-vu#l^bzgSiwcSgOv(8*uVjHz}Hu3KW`}!H(`?}?* zbBOJ@hT0v+{5B?@gI^oQLZE9`?Dj{Xsynjait9jCQ*4LKL z|5Jo&3Mw(FaO3~D>y%j0#0luFX_zSuDtDiST2KNQPwWBEVCj}q+s|OFKVDzsmXOuW z?uHc}rIk&8iS(2_H+wt4X|O$Sg~2dR6%}Y(BvpW)>p|)!XfDv1}Yey{Fm1bP*~ou z4un#=^>vnE-yX@F?va;Oee1@R0y~rA5-lW;eUFKjlajZV(*a=5c1#~Z%`3@PmcWnn zEsmf-I2$hV(rq+c2O~_otT!di+?j|I&qfS0x zbl>as=LR0o>^Z@aVngrX z$NoaoWpjy~GmtM0A_td5wi~vdU~y8j;$0*n!AXB3FvJw%Uhz>;a{vNq6nzj4d4Boe zW8nO!TNqy$#f!y+y~u#c!>vVYivj#A5GvnYoZ4lX4$A=EEb!Ltt?`f&oExu+W&wsbAz@bTbpXRK)QaB4POTQ!~ zP(F)?NCUfHFRb8LO<76B0yKB{=U>*)X~N|0+xdKhbiTx%N@(dJam`LJex-jOZa}F< z2=q1A8v{BrJM7I?M>F>31FEjq5NHEgL`xvyoODz`?8%YJ%)-!tcWWD-3LEW^sUa89 z>W)=E)VSc!Sbz5{;ov)JzPKYV3Ee{@P2=y)m<^CwRx~Om?T74%q~v>LXEy_z`@Z+g zmfD-89%_ENdfpMJBO9~7@m#@z;v+AV+AxzvAJ@w&6`R0vr8#{DuR2WpsH}9b=Wxht za0wI#5cY~}RfK|^rTtEr#MGqO3=18`59IkN^!2X!H zc?je}(MI}{v)oK|nQY>0T}oex58cD9Mf3(`enWEQ^ZcI7x-+w(4< z4u^{;4-d~b)|?LgtQA(xD;aFZ#E?!722FExXzE;R$$b(Ln~iuPXhU7$lbQLjwuoh0 zZ;-JnBIh}~hu`kR2L!L3Qii(**!c0dB>IDkVC(P1d{2-lk)&!$ikcJOW*e8E+P$-q zcIM~GK&7~*)16Z3N4yQGy!~~!_o`I;W3F?>?y7r^2pGnkNCo0z)05ziRi?7_)q ziRP_O(^Os5Ef2iv7`oNFPdH&_xzIamA0%nVh*qi@ z)t9)$9yfxgSksg%l};{}USn^=xZs)TF6p-7v?Q@SHk#`SP{cqSf^A#X1M2JBqt|9}$R~B(QzK>gBp)Ny(0Z#=fiGMRB@4B`IB$$^wJ*tvt2|rl9$-S0XdFcAPlGx&d6nWA(<5z7! zo3GYv_kMm~&{Gf_moa$-JjM|jCt?KS3(Y*NuM0=UO}Ji-=*F zvRDDmcSsyXF5jg)H3M=U>8tQ4AZDxjXT8pNqdTKOmYAtl+#U9`9w6M|^l;udzfnV1 zxV2tUsq4EVX8E&X7OSl|S)zJC-zvCYepI-Hda z$fn^{B<8SzIht-QS_jadQdlxxs8_bT#Qr8MvIis5SivQy$^SmIr~4D1Eb+e2w6A2o zE@QZm8|3dsW3{Oe(?&1(`!!?t=n6T=zorYpURJ;s3#GxxQVG z?&mZnr@qpvUn|l#B8Hk8(UmMd(jgmTnIf%b6>W zT#Y-J6C9JxVS4TISux<$Qr&`Xr+@y1KAupl__`|2j44FzW@ve~S|Z(%`d{)L;S;9c z;yqN!#O^quX!^F8aNwxf*Ktk(ZY z^=5j{6Nc5M4k8pwFG`N}K0hC7<@s6)0w3@YXHJUnaT_Xmx)>r6!}k&(D7#o>NQZWJ zQ3-Rrys8nB89<6$@UQ(Y>vG6tK=*F&H+!>G>Y+db(`~vhxkWQY(JZH=0mZ!hu;m1kQ$k<12au@KI%Z|24Yx|eM!nt& zo(a18VDt1YYyXLHRrz}2V@tV<4J%&^Jz33DXzl!O%_6mgXpE=_mj0e>X^T9ISJ&(5 zc32Kqt7To&e9UPlm^_5R-={ks%Eo;Y5TLWkdIe@PV^TUUTM17Fra$Szi$~J3$*i0S z$NHDx%5|7J(`y|cdlF9aewV!(BX+j#(t`Jks_)<;_4DNv32lqjK#sAs9F+Q{PNH9t z`ISZEr-LSa6E|udqlI7FpW3(pKO2gx2JK=eK7cmYF(pEd(N?#SxAYbrB=c9MbABf1 zEY-)*AY`b*nb0@0#DVQmX1H3gmr5W+1z1#4;>lO){jn_-_{c*w2&joaDS7!|Kbp5L zQlQIepAqWZp~00lJm+cT=N;VzL$B{^u`UrbEuBv#*0QK{4l&MrsH|siT)>bvEcRrFeFdp z`^Wc|7nGTw0-a6swRJkVP~{R?hGUD9R1-IAOa0jEWeUpUSJB?A_jXet{8o6P;mf54 zPxSXbC{5s+&Z~>eY%2gqRaNmj@!>QhQIwG0L#x^j7hL$BEo$lE6xvAsoNQGqj$Q+- zI76!go8hS5CVW&xWe};m&>Vd$o{^O~S}H&IS9gU1a3*mq+AdZ8dM^ANR%umUf~+Y@ z<>qFj4ip|F2UKrAGsi|dUC&{pQx=SVwbpcI`EsrBgzj;)3{98(eCurp!go~{~0u&+C25gu$SAMy}<>u}XcBrAj3Mt;-i3F^)NSme$O zVa@dyDYrljY{3B1a9{YT{0|js_Vdr**?Z@cPE^=_&VPi9C~_hI%!#5i(Mt7>M5~o8 zNrrQGKKfCmVNI~zvDTaOpwXO{6M5e|%y$V>CjNu^y#sPwV8M;4ZBxY&R-yq zOF{#i=1YW>*V&xi-Hzu9<%*ezev{v7Jjy6ZPf{YofMZJDA*G=>lZ^K={Ff9EwC(Xo za$$wz*FaVFjsy8a!?5KNWG0T<3F7Uv9~O`9BX%i#4X9gW44d<0jW~V}C2eb&?jSvL zn%rMS5ZL#=OMl8VD{%FRD!r5Yh&kmt>pWEzEXbZ2k51* z--){GXfj-jTEV?(GI~tHr{s@kyPPCkw-WOj>I$*ie5qoKt`--c8GP zncZZ#;Na~I-gK1;UN7$Ac(ylmr150!tu26S{1RrD!JZPWw9PwyX4SGqGh9rDp5nkY z(+`ClfxQX86;eY0vDq}w(v8VW@Q@&ATlV4#Zcppmo_P($@k#O&CNbsOZ?mqh^xCz~ zMW&Txg(gs2rtenoc>DK}UF>G4DAct$(S!Kn3T5=(z*WR?U`RxCld*#a2|RSHY`4vAI*JtcevSDI^~> znij3F4OVospe6%zLVgiJ3fhNpv2uHl2EF-_sFY)H!i>K2@c_}1)To8wq}^S0O^^jY z%Khw*>YoZwThtx)O8j)N+J@>PEY!6R2u7M8_e!-p$S^tL69ZSWxX$AOC(_C>mwu}w zH3T?}gm&&(`@;7KVPxV;YlMO=F@*Rf5H#E?S0`>bW9($nz3lI`+eMtCvuFLU z=Lzt%n2lDQl=Uizt4pb%_VOsTf_YneSK1xJJ?}SIV)N?aGfU5#zKAYTHl;90GC3&j z=|}OKR19~$0EcTu5>+>s>8i>=1AuiI*UWzX=A3z6IrMiOf-)y->^zAC%9Az|P4 zfu{M50-RG8vCjU1s5e@uefiQ-@+)Xc9dHg+dOGng+e^kXP^BQWjaVo7oBDx&*1Q`} zMz-a6FS=x^9zp8Y`!+mW;FNCjLH_#@Mx#yW1t4ImywXD&+mbmAJ~4miyDxm*gXKNY zab-NWH*sKfZ8kcka`p|smi;-8rr%PWAhiSdd}A2qUb>YMxgUP^X_E8kQCm+rwC=d^ zj2}CZR3(9F%`x?3E~f7D9go9v&fSSI)e7H_^RnzZl>KFDPykA5u?GF@pWD{BVVc+e zW~^^M6ce&S6TC@2a9WA<`1;dI`U3FH&a9A4iz2N?zaH05jn&@lqO-sM*DmQ^xr!5^ z$>PSZB8w*fKEZ{*i(ujHs*s?@GS#oMsdaYAGdrJFW6Ip67?;N23>V}XlJPgaIlUZi zX5@!3z7^s1yS}fwcjcw?gU8>^4s3N`X8f7WN8;3twUz_>WItwYYz6nU%z3uX!hkBz zN5KVOg%hXAAi@c#dp9pIl}><)t=?2Mqvs2sB_bNDS>ujVe+Ze|~xrKWNr5p_(F77H8InR|j zeO=sq^VIORH;dTergO!rcMGnLGj3@!JU?Wcm8|E)NPqFSkuS7)>tvJ}m2%D`52V4< zlGv)Qc3i%+KTnmvd^sy3+xM-S?(v|9*kreaRoe@7!_I>yU1G@UyiSNZCj2%RLb6Pl zPyWz&`XT?NJWD{hYDTc%@D}(aSvLzSvMt-l6MXHYPQMu%sZe@?zu$XABTeH%v6Ume z9lu#I;9d9tBC9cdAJBnOP0Mz+HMsJK89*)GfN_lp=%%BlcDP5|QE$hj=C&o!^tCE;h_?9wN00EIv30-?~X6Ov~c=*}Z zmt^)0+>H}5R#S@8g0THblkJd1qB6zsHn8!xCU)=y=2@KPdQcygJoU|^rx)WYWaw}! zPQ?+VCuBCqO%p>&7gqxNRCo!&L)%whKZjF=Ge%fH`ufozlYBXD{MC}Ust0DM{hUZOK1XQ}`=+ zbW{N-R~zn342-naec8I#nUBky`?K*wJ{A;9gi7)9t>?@J>A~H|81bH^pRwd2I^eAE$L=Dr#Qx>CdH?Rf~7g zUIpLj`&mbI!6|t2Z_kA@PhK6%Z)`bYPT&f`xFJDiA!l5|mm(m21FO?ZxQSXlXwFZ? zQtzmZAQR#z_%af*yX2k(MWFT@zUUsZzy1k52YuEozq`kpcpnJNb|!-><_-rXAH97i zDDm^^Ky)`Fs{VL?X*t{SaL}B|;rlz#?H|r=t(+Ncv(?`y ztdvGo7|m4)xB`9txtzpm|(Ia<6l*GDYfwY|FB~* z2!1~|yS#%l*|ROBiNm@&gcjdWW5+yxz2KLS6kT?8g(8>srbHWtnEDviIw=oUJ6`Hu zu^a9=kl%Lw@zTcJ%oUE8u4qN#Qmx*aP19x89~rQQb5;H;+c7Kg^MWaaFNI^Hl4Ok| zTgmHJyD87TpeAPeJ)^7kzmxoM>FUGf{QF3``xOZ>x22Ssx49}w8_E|y&oPw=2so>N z3#`lmjy2q7T9ll<*NpFg%>kJOHco%n%?C5xg2di!{;HH!T(rip z;GxpGXJ?Vx01nwyOi28=SNMbS6IHaNc$^uBxo4ZulXpM=eN9XGB1Kk2R&+qV?s)Mu zDr0!Se9NTPw>FR>p2XI~o0o+p!siq}@s2#OZ4 z1MAkT3&_$H%sO_Qh@}%UFIq&V6w~%ahQL-jU_zL*LTBf%uVUqHf)F=xe<4h9qVk64 zO^dR$t2KzynTn!df~B53yeC*tBywNbE31?Lbj7Y@Y6R>ZI>7P$v=6E9`KAoKWAy8) z!Fo^0<*CfbUd(wjqpPaa;7J_bel1VtsS0vHS8a6d3Xfw@z@&d@z;oH7^I+nS!Tkx@ zh)LEZ2Y3O;Ng2?V-_&&2?1K(Mmg)(N(={XG>{ES3lyc{1k~=q_DF-jjydwas#>j}J z5Jzmw_c)`h$j{?Xl|;<>em1SD+#-3rJ(9X*^-FMmP7(Mx?26tC3$zmGhkBJb*sNij z!GyJZ-GNw@>rHh?&}aUS%j3}5aF_q(#MF}w>H!NIdFEo}UpnT3HCEcDm(UDGZZsEl z;eIuzsdf<@JPPjFF#koLjtr{D0DGscqZ2>Xwuv(;^WxO%|N3)=GgJ zR5iDcuwJi<|M;2s3-!G8eIs}uKsB;zJbG@`qv_iMzwv{ttgbjX^e7=iamH>tO z7Bgo$)d_jbVyyjd{HF2rYK37E1K4Z-c64cMzxk-geCt zo>JW>)Rzl8(Ha9aRNJ&7U0hA$bFS#97l2D5JufzrG#e+n3S6h1$Eofgrat@izQg&# zIxQfuP_zgrz2zs&!)zs}`?4=v?ExP#qy!?wd?nY~qR*J1_m@e3y?SWo@C2 zf|2(4FRpQF#N`+^!!^?4?$@z@-C&PT-!w8a)y>>?#Z>|#%O)*V@sjC>mh`w!O!?n- zW&dvmJyG!!s)e~ssRzD~=e?mo72wy-c`VM?$;&3UbmgrI&KwLkf(y*vAbR7MyNJ_> zW$PjDF^(CjubKgZov9KeyYYeMt$6I+d_v36uuakyiUS+AuoaSZNu6;i$6Hb~i}`;~ z#ag+h-gla31@r9Lm@{DH4As6Y;s4ls>!>Kd?r(U8p<5|w6bV7;u0chR?rtgRX6R4} zgAhR^hY;!RE)fLjuAv!v=o$vz(Qn*(KhJNicdhr2cRg!8*FV=e3$7FU?ETsM>^W!e z*MDB9|CSOadNze3TxFLnhGxVWXGl(!v>j&z%3&P5pxT&scaJa?MQc&rz$V^H7)Qhf zuNdnW>N8iwR{9|z#f%*Qqr)f_m8zuVAu+UjI8&{zj%c>usG#}}!=O*+Q2%ka^Z>i; zb!DE}4w2}0gWE4{?{;7F-}(HQ7D-r5aeFhc()7nr4q`r^1Y$7{nvNYGhI9O9KIogQ z8h`u*A(NJ?JyOGw;2C+Xar}KNG^{@vz7zRdOp$KRj#{Ksq^~PNO~n7Y`OoyQ9e)96<{%CJ6{aRH?a!zOzt0jqPv!rqIdsD-qJIQK z`b~t<#cub~vOi;GV*I~}O7urS2l>!dWv!hzF)oG*iq}_F!Y^l`Jq0e!?%ud+`qhnL!#O@ z$LPh!3-NB$F?@xIUvAGJ)2{5h3GFArc1nq=70Dyqlm`2KP}xZfu$$9BqpE6<{)=QI zgxCIkhvw1wdTm<~1}W5AM08?;v*+c}Of`%7p&_ce%E*eVByH{a%=d6P>);L{@D5$l z#wv7TM23ja)w`B)+-SqULd8==!(rIa)T+`$OCAonT$tXuDz&wm=T1(j@({?AJDUuS zgd|IAEynCVRC%2M-l+R1{2|YTXN*i0IqO}1>fG19;PL?qK0Ba$^`H(i&h6OKS;M7V z6TPV$>er_-F7DDXTxa#=v{JWsq(}1HxdeTsq%rDRaAzB`vG8>&5rJB;-gq9%6wYhE z?d9%+Tv$IJVdG}GHa}`?ywzli4*b@=rSeC7CmE9)5G3PPxij(CFj>{t(|BRMI>Lno zZK>-W+Fo-%>ynG>cvf+x>QA zjGVY8+Tf>R(DM}|uN(coC;$5ER%jQ5w6WHz6INHUN9cI!lzOqNMkUe|(BJ-g9_%AI zgGlKx?tRq9zYH^8z__A{TANKKX|JEqAw1nC}iA1hmr@y9U5j@dH_;S*!eVO1^g~8fhy~i z6D>bJbN_{pK7nC^IK5oG7-dizcUfJY9K-XEDTa_Q*;yZ`JpGPB-p$}YoHp`tqDmst z2`wZrfMc|%lAJr30$`&OgJ=2KS>*3+7VC;-ir+W_YV9}l(w=-uxc+QN#x!RJefgv~ z#JpA;yfyiCOLU-Di*H7<_Htr+pJ3!sb;TI5;B)M#krBk9f!#($@Tc-cxsJ`Y&|Isa zAN2q)wv-@G0<6rC8C6*S)q|P`yPU03<2y4REVoB{TMmm_+TI^|>tB3ciGM$xX75vH zRBr8<><;|QnO0+Fj_9IX!#J%xfA)p}!y@`q5^u+XWfs^g8jQOL#x6W{CL|`B+gJML zji2TXQGm+ro1()ILcZP|=XZPZmA&+!#@-Sq>l2)v=#a$9Asek>nzHr zO&D%EW$hFmQ(Z%1tVqI@RpGtHj(cuU>9I*kwCjE#{2*|u(fCK()6=~b-7TJB-yj&3Q9&2{fswED>fEm>wfO7Ejb~(D-_>i#FKjX<{ zNz4wdUK@x(j8OVCIA9dta?;Zd6(5qz-#&QxxQ2CwVS21_eyqhK-x$fszC03y6b^cX zJ^xI**QWYNTeTk9FmNPwfRkca$?lL(gh|z~Kfe0T%urC7GapMepd(K7ibFMD$gsyW@02k>lMh9o&#rF3r#p)M zJ0M53MB0J$w&KTBF8UuG?RGC;M^}gkN>1R3HXoe>@Fx(Z+1L0yQLpzS^RVE$*K>=f zG%m=+s}F{hk6&poRN?f;?CQV}_?}^p7@LEd5jhJ2qwew_gVCvNMh-o{%wA?`ZnmZF zhG%;DV-B6RzX^-Nt|1V0KRc6dBSTzG!*A4D_D!?GU*sT!?@YX$^b0qi=whx96rhqg zByTZVOgX>wSG^Xr3=_oUYO3A!&|w$%ZVh47NPj%nCI%oG zUTN+=l=t1&K7hjbXZ}(bkJI)(Cg?1axgT5)k+r>3+j2vybs18@2#~wF1E;wqV?6L&bb|75WZU!|k_Da82J)&w_ zfW>PH6N4Z-Dl4-=WE`v1UOqi_Y~YD6TMRs#{u^Ylm}321Uo7X=c!W?G_XC`tYZ)4a zecC34^TsGN6XDS(y!UVkj+k9~Vy@|UyLP?rB0(~vQnkf#epbo9`HRR4e{NmL1nM3E z)aPBu1X)SKsIR_8nBusDH%Iae!SF(X&|YBGXc#(z zzM17S?l!Q3K9oH8MDxqetC#`j3sX@d6&A@V4w%>=w@H#pnn+F%}8a~F`XJLlp9S9iPp8D*>9^itK}O@*)S5MvcI&E%ijEEr56 zpQZU+yE3vJ2QUSGd7$a@EGgr@@qQx~0o>OIf%jb*q3hccyJGQo?6aT7^*9$ga@DUe z))zsdzgvX_4fc!^gRE<2ul&xOx^ICW6~gOkv0r~1DSK&A*c#M6RN@5*ItavQFg_AP zG0^SN>pHqnc9ZH?8c!Xn`TDnT%LM>zXPYo8j>7`7nz{Lz#9wVU3a)M5Jv<42tKzVy zE$ba~ewnVxzeMwbW0@wvzicsB?1YOfBfRSK?Dae@g9b+%KHEQ2GpQzEl;L20~^ z{Z%E@6HMDBRW4qxMsnmI-UYl3(%#o)Pw-0bos3#L@=a@bV2Tp7%+(agpB+sI3dl-* zOdHmS6>a29RX5JJfqYAIYiZ8~Ot&c>_%&vpsn5@Ztehf*k5;OhVUEJhp`>CV)wz#o zeb-WQ)T`^!;R!^Tp=N5HC@Ng_dI@O}V{SCGu{xbtqPx54oM11EQLHsHxK{GeCE)O6 z#{MF?26FhlPICT%ON8_K%KWunLV!D27qu;g$AeaCjAdE@t7&y>r7FzRneSB}picy{h?*=cn=!qV*j1MFfiFN^i82`@ z!w#~V*(Gn&TJGly%h@f(zP;2~1Znc^EBe4Lv&yBQrc&u2SU&fXf@7+IV$b z>W^oUA8xL6poyaBSclUUl+x^~M0R-0ZCru~ErGuXC^YmsB^7CM?9Y5X&xZ1qP}bN= z&7$~npK*-tG9b!v-cNkDc@srVPJ7U{#J&4?J=V_#HJn6ynM5-y^?kpBA9meFR34_y zpoPHIdUtEuE0)_ElhRx78EcB2z}D72&h5h9Cbz)8A2ZrC!Vc3b)vj9 zt=QS)^1CGwXBR_igLZMA_eFHS^D)W2pmxxx4^%2fff0Vo$p>jCwVD%GlPJx{Px*i9 zT%?-I!-gZtjt>aYfT7bAn+K;a9E*1v7qF1gQKI|{H$U${3ly70)vPyjYFdyWzN!b1 zTv=ZcSFbfBxVEiG!_P>F+|Ml#vkS6PSL?Y76E%I*{P6*O!;=s{|E|{w$AOmYh`i6s$TG+kdP~D;>m3Y`)(!i z2{D6bDZ(=%g)W87MQLf7yqZQ*&qB3t&wXlaeqz@)x1GbmV1#uAikh9-0hGPXlzMwfu{$pAR1pN!q8Hd4 z*m-nhc7eW?GZ}%5A>7P$EUTI;wn7}&eYhBZyLrL)wj41d6h3ddS0cR4r5v_i zcLVWi>R~hR!8l|JmpFOW*@yD=K+d$Zq6LlxQ7H_um;GTQ&=!lggZUzS>S1->*fScs z7yjplVqOFo27}4^F~p2-wm~hs;YquVgtvQ_X}wpMEP6mhiJg?)leu4y-*CB1AiPmc z0ZgawYgDTmo<|+3Tz1cTu^F*8>|jQVV#$)XV3(T zSm;a=O@4sA2?J%|^`W?2TV5P|BJr&hs+J)O|0CB1kx5H@l5RFdidW)@eC+k2167=qY(mW1;|Unraq&)zCNfFCPZGzJD;2^LT0cljm273#~u5S>%A4{6NpH)cqriT4h6K=09 zf9VLn)3#SSbxcPUz*ISH>;!h+8(4g@MgRDrz2O$Ltpu5oP9Knd`Er0N8j81TdpJdr zN*GKPtXO=X%KgX7zHq6!xf78yNeKZ*IO$%dz_q8J56YmYy2r9xs)jk}N@3 zFml>o@Qv4t7i_T9VYvEbyHQM3BRK;QofJMF$@uE+1Kre5RG`E>>D24)=)sPwXeavc zd{hg{V;mauP13?_97gK3hjwOpbwsXl!%=%rwz%OviHugNrHmz}ic{K&gs=&_UAt#V zP1Qhh59jOj36-X;q)}@KME3${`P9C@Vx%Ejz+GE6o_r?^E0zRY_-OkTj z%#8og2b^hH?B4#4)gpx@DmT$P*+}{sp(A3E2)T{MudbOu;#TRKKggFTz?zX%R=Jdz zWf?;svb-LxcZ>Hq*FRpEr<#EFNGXY(ZMG?=8&^N+3nzA?F)FM0WurMbYyf&8n(_b%wI`sGaJoXak z#aj_a8Mnoy6kIZiLW<112UrKIg)!&rr?wRmYVW6aSJ*Twju$$*JqV^Hfdn>;p9XEu zSxIXtGGgzA6Wfx;XvtuLZm4fNb?!QC*Lp#=s?debT^&L#Ld+X78kidfApzj+-3Fug zoZU;Ar{@77lDy(MYd=KAuc@U2Qqx+7OoOLjn-(;Um>w7kCZ(YCm$9Wt{B$xJbD>}X zqfU2Qv69)7`+6wyv%-Ca^Y2pWM?*T_=i&j`L;0qotE3Uk8RL&m!$R27g~zSaQGrZ$ z^#c|q(ZRQSnq1N_?iyX`iKUCjjNcdHxga&$Tpw<9a^1M#UUS~4?Iu+l)nX{npye6l zELhf4kl^3xnn##TqB=|OweE0h9$$!JFp|t~2MJM>J_1GbR?I7WH&iCBF~yL8w$v@( zd7SCcDY=%X^yz{w;iuKhK-|)~k&H)<_d}sddV_~bwT`?92 zkOW9{*QJydX{r$gEi9+fa!LpT9%UXuB0^if<5E-3&LJgTWZnb-c}Ry2rZXVT$zNqm85=8x4H80|^_DbP0;mjsr)rlC;mkfB8@wLVJGWk6>-^u68 z)Jsyet?Y(x=@{wh7pOE-K{H%3wsfUluXR6=T2hgTfdw_dOy*>DRKn9{vjK4H1@%1A zw~EqNm{0=f4^U%#%1GV7@5=%R;+*W+JF^@c$bHYt`1JU4c;zthMNUH9UV)-8Q>Qq`JtLCto}S4g!5eC?6qv^(J~P`&YZt*^MP$>5K;WtmX6FFjI_@18KD5A4 zZUut)d=P6-Jb2;&tA~3hUM!y@Jt^DXDhAV1N6{~Bn$TO{*Jd2wnQ0g)qCnEpuOpVGSkmrt`&}&G09O4~TO&CP zW`T_#o%x$mHtss%sS_Om6vTvVVHu)(w>R9bmzP6gg(Asj{1l_#76$yNx7|r3{LJ|+ z;4+gXwRozmw#l|opoO)kA7Yn?VXlj?H@3wV@z@G82V{;*BnQmX^L%p9Eq+0iexih^ zTJ*LR+jz6j*zz1`*LUo6%NIjI%8qR83DFh99TG`hdk=EK7|zwf_VixK*6ZzkYVkNg zg;Q}6O*LR4?A$xnT~b~;b1&uj-JEMx?!>Wsy`-dE1AD#eIF^y%~bOMVOX9(LI)o#(1@i!U?TgR0LautV@yb4A(Aj8w78vr5^fb9$y{j z>l}}!W*{&=TXl{_mr<#{8>mY;Fjry`F_h2lpu~a`VqCWQ#t)?s5Keq{ijXp zI&bVGcMS8#rv;DufdIa8oR#iPv3PAsHsM&`7n?k(T1~I(xO3L>_%-LTNQ)`@ZxCZ) zGL!j!D+Y|RZg#LEuS$oDSzwIUwWgQ(7v>KUrlL{KbI**c*g&KJ?}Ss7!Dw-pUPbWt z5(q$u%3u`XMuI!%bGn{A-P*vNL%Wj-ihHd3F6GF%%hUte-F)lIG}R&e+g*?-UcNZMJe{9zHzMWp|iz+t%}eqZ8!? zLI}IjnG64YVn0N%L`OQu+(xNJ~hs9R!GKro%K4R%cr}F^SEZ4Wd(#m{1JDZJXe1NX%72TBp+4Tnj(iKn z?nzHAEhz=y#giLY&S5)oCpz(^ODr5X6&|Cv1qPHW4k{M*!;5Z&w(nPPpQh=tb0Y(^{X4yb6n2C#$~g1%oO-S@4c z-cbT`-M_k6Z;dqAW-^jBxaMHGfu97lj?2UaKUMU-;^J~~P(P;M^GO+?DlrH?ps!lp z6a;3I+3})Z9iqe*DGVUJ2sJ$*%`#CEIha{$9u&gIGUK!zKu{+{dg&{E?y{sIy&rb) zcr_7VZ9#$A&CfzLge!EijsXFjh>M%jx?x4w(;Xe}P%d=5??p!EfVfp00vfa!2d6Bv zHLRJ_XLZRs?VgV08Tp;P((eGyL0A;0r-1Jjp`YixFfMVRm9urkxe-8fGJ@MoaQjY~ zQ*UMI1={QAU^)s=WJl+hH{kmiN^zX?c~<%BRm4cQl;Q*L7ajZM?zY%w$3@NN30>ljDWc(EiLm_dSgZX$u# zm<+*hxVG{ssm*>Jw1d$+dvxk+mFg%{d1J~EXqIVou3z+}{5tJYqiQ%W55l0vp0l1EhlPejNN=E&zoHB02AX^T z;eFByODOaO3^H1Nq(B>ogPDA@_}c><9>&(p*%nnd`t|#z9%WTHW+V#+{j1~#&uZF< z09Sw?mMwGOQ4MopCjelLTy|jlCiPHW)V)~gGEY&s{X1cY`@KlM)yV45C#M?Wzprjzd5Tc9*>q)f4YZApZ3gd%=-*kqnR(Z0L-srb;XYTmg4dvWEC#$p*IuAj7v%9%EQ%+` zYN15~{Bf~Z>CfTaI$({-+(9+8!K-~H2k0%yXQVv3%HD7&v|9V*7@~upi6kBb zQTl+n-wYm40-Q46r{p`}Y}-IQZzT~wzb~l7pGR`GD|&V+>zTd+W0j}(Z>~VaE+gKn zv3iN7iz(6is?Vo7)7H?b`4!Nomw*=VgC7PPd`zo)7`wd%s$&T9nIH>{1OEjzXe8lS zB414f{tGF0?qpum$S?HeytDD#eS%@_XDlb|kEa#LyVRbdT6g^*!?@_t%YJFCDXJRA zFhyD&`E^UZv9d_a%$XL46OsF;h*PHqr|$b;CjYgUWV$U+X)Ut}FO42#(#L2lk0h2x ziB2FL*e+n2AyejSl!>#<8H51cY6iCs+?WAf6YqM3$l;u!Qb zx{X1HF8&kC)>Yw;(yyC}EohoCJ;wD0xR#02Dee-b0s!}Hc{Rb3*RFx@xV6z9F`_g7 zF}Y5Ap1pe`4gF23!M3@*ZS}@6S>_pNJ`+yv7{<1_Ar;LHyHe?BI#PEPx6r|u0CnAi zxB@dV(h_{tgszdjib1b;Yk|_1sa<5wk*(K$vE!N)sWtJ{B!MEUuX5W8mmHX^0=2q$ zI#M>Cdyz(BX0b~ldOmK|x=6E<<_Fq*@9dF|wa|OJ9pX{0aZxo>X@W0(UfBF``DS*I zqO7p{c3e?N>i9GF<=9m~U-uHT!aLxjZRZ_HDB!LUF&F(HJ&&h{tx1h{{}D@9#-|Y5 z!p^5myaGpRW)qiiOmm5Rsd&Gtjp>8!)DqQ2Le;CCv1$piR|}s`c}t zvB)K`U(?FO01Q$&_h3Yk1glv#M~syl`nruO8geW2YL$#gp%NwRU@ZGbn==Agr5fn9Lk(BEHlpdezORlA85wQ}qi z=!)92L70|nj26ogd1gIfRT&=8c|#;sJ%k}J8cCU~g?*_JYMOvlNqrAUoz%=Z4w75}U_NjlZ}z_;gj0YKIJC|B7+*JP{Q5 zO|IWbAM@@`8xH;h0;9&P5@83u3sd|Z$LO$6+M zGGcRluB%ZX-%H#VvXY}NzsMa7Hx=68MrX8reWiTyxZP<7f9AkLqu$(em0yOFcVS+ol?!uYIsQ_-I3=puFR?Q(Dp zYxn?qY+iJR$`0!u4T|Lu$e zm5mF7rPjBqHjOnfPr%X-4AvGH@aF~%ZH}A#m>+h1e>+C&UgeV*-WWcfdIL*8rmgh} z@({QF%Uej@e8V{cd-+Eyiwe$64#+0C z1XbWrJ~xR32S{RTci$n<)plV1TVVX=_}OcQ#CQ^W8r$sGG&h}^aAdF=*^ko;D?_G} z4UtjBhO7+xImGFHW3p3$gV7^`jCuuj6<)#P&kmPZdgTWcefZVAnl*bdag*F#kKdOB zU+K(5xcLFAewm8!=CsV(hq-&6YezpYF5ehELgQmijpHWioz2&%ro6^e1sL9ZM78O# z^oe1{GD4uv1siu%LU`LO)f5U&&MJy+-~?v8Kb19N05#IO9$XarNY8#hO_qKhNumTUei*iZczm zK;BY8?m17wq(_tB6^ylSG5Dc7z%&yROBlLI@qR+82Ddz#aQsCVCayYhT5GmK zJ`vGKr!bY~I0Hy;dA9BB5iTrG(N^?pVL8y&Y9n~LuaV2T6C^<=S1g=QR(*d&w#%@H zXWL8b_o+O_nZUSnlcA~JhcLqV=BIrx!~RtetIx+d2tBT6Z4<&DgA{o=m#iBS3|YT; zT+yH)hkt$|!Igg;VszR|QcN{o66n-IAv9ismiFAzKfM6!RBw$ACRcQPq>DmfL?G3l zLh6+r2sRlg)v5E0M*{LoDuuQ_18yv(>0g7d_3I|IVW56W=7v5|LYYQyw?!}VJvHaD z-P)^GxMRs3Mg+(xDar?Y&7@XXvq95~KYVPw{cX;OoRuad<4A46=UCZ>(i79<{^YRj z``c!fr73SKDAbCbO1QFEa>q`>N^`0X)#ilr9L=DI&^VtvW@C8WdWlKviRwF7wR=at2=i_OqadaQ(eGRw0KVWS@U5<8vV`5`4Z*BTab zN+Mq~!o9lwrRWs!L8$+wJrd(Yyn&A1?MZ#}V~qiKW^Tad*3kuRBu--*_i5@aT#4lB z#X}=gX-MR;gE%DJ?Q$=nBI*szt)@vl|8+l~%Y7Qr!z_&&J>8w}!EGK5T@){X*;94} zqgcUgVaiyAoB|BPDrDX{E(+!~7z+WAeM0RSv+*mQjShY%o zUOkSQ@8{jmu6XARxvu z0iJ&7&;VoyF{QOQzQ|mzqPZNP`m#T^FFl}ny=lw`W*P_&e;s?6YVbw+i%4+Kk;>TW zu`s#B+$Bd%rS9~)UGNKUFzVi&PG*l#ky#joYkwlB@X z8Q$Hua#FeeF13e02I_K{pVNcECwQcK$L$HsRl#>t_ujOn9{#eRQy2&x`B5Tfjrqe~ z6AZ1KHFE?yl2~bIV^gGrKizeX=Z80)nxvQAWj=j3@6yM@M3O@7B3p5$i>qXHSVM@- z(!sheFVwW+u?47AL7m)%1FyFJ*u||uiY+9!D)R=r!h|!bgb4Oa`wmmzq1jNl<2^NY z^r3ioAT%H-PWu9FIRBT-iYXVtW*9K@MU_4DoIqcbu#^Iu=)JY> zj_K2R{@V;Zeg@2C-`_&#xm)rB^#+j%h22Qs%Nv@m!A&p40Yq&UaF(H!>lO_goQ?U`3JNtrM@NU^Rl@gkHl(%Q+Qy!r6Z=vvRI{dPec8gQ_ZPwM& zH*BKcq`h^EGP*GByck1-92a2VlQQ&;Vr%q8XC&J9PbDSIH3VIdEsnkM&1hxcd{38dPYJyyTaZO2X= z<6i?oOD@>KG=!Fd-MMMJjB0N*9E+rkVd!g!7l&nm$A0F&3@OV{;2-1By35{>8!(iM z#lEX0neX)kR(Jctd8oxkIDg40T|OZaI*<#jE}2n695bE?>Q6mgeo~ev-PH6}4sigq zyOQ)vmLu_j=kMLy)j)qu;nFC???GZ{YGg&n`7w3~O{}2`V)GdGZP+eD5b$Ni&i6h%^62 z!?++0j>N>*uMhgMVq)F)(&M%JH)zaLUA_|p7d^7cz7wPR?>3Ah7_c~`v_Q>wKQJk*Z;kb>E9t`f}@7VFBIlz zpMrx>`5q6&(MyDuA#iABBmZwXs(+g=x_BgRE=~OVX|&(V>Qd~z;-Ei2_@`FR+IU^p z)*yLiZ2 zM8tYc4m*PIw_Nl?jQ5p3#e15<691DrIx(V?Y4M?t%?q^tes1>1s43D~_|tMs5+JPq zcJ-8%(w{#5K>b$>95lCL9{)3P^c(AoQBQlz#%jt)1trTmW~^iT>vxKJ1m>hK7ogO(-w{)2Z3*@|1k@*f8}`w{x7z8e2*k;ZJL!4|pDSd4RqG z_xDr&VG&i0$k-Ll;jAk6uwUguvnHAW4gT;GUJ&6;#0j_Fz;}x7QAzz0+#`atdf7ZeX6?J0Om*c8Tqsi$Uyf_!SNvzZGCl_)1$y6jndZ*d zj$2;1_3KdEv^-bi2KZ@JZwbUftkL%u0w4TAkoP}wF%uw+J4x>Kr49zx4*b$Ch_s#i zjDV8REIz7k@tS|zJ~A!|?cmD+z3@BTv2FssvKqlGo5!nF$KwKfc^LZGe2j>AD)q-o z9nosk?tPxNb!Y1BZ|GXR8g6lesC(v`SV#x>FefvQxb^pYM1MM27m7JKc35_*0Q^Oh zgMNX3=>!aRv3X|q6pLwRYl?MlO@1C8pcwEZt6Yu}qc&GeZ9s6|6wMk(a%#UcjWl~4 zMD|}^h=^cEkdyaAO`F8v9#*T?eb~ftkloLGBw#ycJjOwln;=Qn~GKKy6?N8z@dpOQ+kyA9&f$&4twy|28EGN`4k!y^t& zeCtZSc-5b9M2iJ$KzikhPI~>A4-i|rEG&+WK5#V4Uy=V8%b>V`qBr#qGaDIqv4&ZV z4!8Kw^8T;$IrB2+T||+{%w*Mtnh$&p{)-(f<<()-{}CuOHb9r|TfjG)fPKXDn*LE3 z9Lw8OM4K!KqkpM4bej9F!c?f6esFt`7iQCiV2nUhOY2tLpFXJzlDeO?q)>af8d6~d zjF+Ooj&dZl2v;o(IICg?|T)pI=s^Bj7@AF1WZ~1jHk02zYi`JTF3@H4Ie%0f4BUFvYLPO z7u;*I{}1XEA0IwYVv_1!?`AYodPN4jb&pW(@(#l2=P8xH9@L}#cFRRKz~M+eTPKJb z-*tJ(3n*fiYk?~}pjgt$+(fv*V=$QR09o=mBz z$Pajav?<19MC-bM0{+6szgvFl^=gZ`xVZ(#QZ(r7qMR+J46z%tY%ZJN&gHtVjaS=5zOwMtGd=#Gp{c#X{F$g zJ_#I)7DJCe=l&AcW~Q&em88}7LrFH{N(yv~Wb{8W@kIO6phN*%jOV;_tZqytD3eYl z>4a;JfV98D(fEN1ERhwBJoJBv+tpqDPj3hSVxp!FV}R>xpuwa4Qcpb}T^}2C?|(n* zrZNN&OKL!S7qli+3&5wug_xWo?an{R7%JGB*Ebn{sGiVcY(Ni+34M(J$jAAoU~G{8 zvhzp}KB5`mpXZ28yAU;1vN@E&e?|wk0owl?q6CHmFpwm@fafC#kYCyts{`?h2 zx>}j{D{X4&p=@u}UL3%)W1Ua8di*T@8BxS~3lsD5aQ4p3Vd=szvQEhoCwSacTpMbQXStlHYTnLt{_m>zK@I)l@eY>Q?3KLt z49lPNpSK_1h@MA+5+4UN&o+K=l+7QOw~_=z<|}WAQ2xad#S7UU^bI=d?m-%np z_#W+oA$IQ_X1Q!Y{iQ2eF!n+JfBuTvueSL(K!=0NOku(ig#D_9!ijwh`&9zPE_)5u zE7KoJ9QXi_eH3z04}juG6b`_}9k?}LNexJN7MmjQnj*ctzj^nZH$>m5h(Cz=1~v=%0Q5 zf6$2kcbfmdtmez|Mt-$XE23VpJNL2gN-C+S)`9LSsrtqq0G{553h-rD{cc13$wXQ% z(B^?ijiRlTy`g|?aY|_h*=8BzeOYTp#Q1*`|S8JmMsR-qB9oJ zD`wJViUzTaFKCC!pKbnIJKCbw8DlJRr{dAz8yu?wk}p$`+#8R96t6@BMW5Y+Xd=D{ zW4s`Gtb7O3EiU};Xa7}r675;oX*2RL*jhO<$xe4AmG=C60B7Z{kmB_FaspgqdUd$_ z%h`mH%HhI{V&HnyFm+ZFwwiItvqA|Pwb?I|rbBAKnd<-5cwCVF%L*^603c>oi>yr{ z&eHZxox8*!jdqR3Va&P`8{X=#sY}9u9Yq6>0P?#ZIyUyTgS1KyY81Jl9BhUfTzWs_ z*01ZO{zk_D7*}g{kuZcS-Kl;4vAn3!UHQNI`lqUO)-3fSH+i2p;yFc~m@jk8zQ_oV z`=kEVLW}!SiNpW_d;3wH23Ca8wb+}3{??L@*~w!MwjS<=h$IaN+#F>D>J;7g`n1VI z`ulOW--Yjo-4mX%%sR!pg4|J{TCDG16V6EHi56B zNq&eL*X3)x8^&+-sN28W!4Eij23qU+ZFl$2!2f=`Rnl(p;P4nbNlXorsr{P!d-t;E zo!sB|8CO%MOywT_^6h1)!Km*!=yy~^;l2e1X2Eo?&RF68z3hiD)_cRaTz|mTZRNYi24kr@R zcW(dM#s3MHf7eh_MZyGj4&6fOeT88nRwC-q@ zl}~*i)Du|)b(mqirB^nvUfc;d46zRTLURC1GKI^*cS3u0?dBzj3XQ6Dg0^l*Uq=PT z72~)F1Vm5B()+;EEO)BEOY(DbFC66B*C4(EUxpu&Z1r-@(`BI#Y1Z|t_oxK*1eB3*xEsOhzFa3c_Xc&;W78i$4w;(~ z3NlF~%nj0l=APVz`)(CDFjV_VxhPyatgqHIa>w}Bf*Ca3rZpB&qpfe+q@;reU>wFy z07=36l;^ZuNCfA$?^Up4M=F~&`j~d;r|{GFDv(suv6h1;X6BLlfn?7Dr*=`d$4Oql z;qqxpIGdXf@4kN>IY=WJ++aW=1!iC>;trjB#q~}E|8CS%4S5x8rF_a7!4%*ml+$nK z1b-0q1U}|?6IkFxX>RTJ`MV@FB6X9DP|1h4p?BDJc*F4KM6sy@-lTVYd9^_-KP|(J znP;bS*iolabMSWS%WKK*IJY2Wy6PLxFYDXNr#pfl^L9B(N`^QZLqzGs?iycjYO4;Q zkl9chRYPxoZMg+lR(L~NF4d)^LAt%#Ue~==(w!mZoUo@k+<5}0-78OmdlhVmSQ|l%pg4^r%xF&t94Ya!x)2(56OSQ$#-J8zs?h0fGj0{;C~;+ zZugVjzN(}bsaR^9iRWt-h^$*qh{kwD5X$gK_2*V?-86!>`)MF<6KXTf zJczX^R+?6z*Y@f&CLhx`%64~S1n$TSic))DtopPhM74fe{vh;Yhp_P|sAy$E6;_vX zUH@suG^)!0fYc;`h*+^AW=T=Pm_|e?H;r!FFhza+un)84ei3r^8jtIB(@Lr#8g$=F zitxbC=*(0O?ktaqKOw4`mxvj!z%<4R$FvF|-FNSZSD$d6e+`;tjN}sZyte=br`xDS zLa(o~7*)TdIHx`!m2>zSc6$VQKLLA*^v2LYoyOJ0ia0~?#E&kofh3TrssGr(C%`0E zwU6x%F1w2EtuK=Xu)4cHkuwua{SR>?MZ)61j`QT7mIVX4LA)MAk*8IA3f0NOom*o;gIV{Z|qiZyjB z4a#b~Vn#8C;2maDi!Z1|sHtu}2mk&>>C!MZBh&7)%k5W(t(~=;GtO67M+Sp^wWxt0 z6k}GI?aPTO1fVKP+jMsco1IQgBxjITS`LtdcFkPq)2cZ5l)Sxc7sQ9b zn5_=#=-yJ$65xQAfOp!|Nd6ze8jghnhT|CH42D;BDl{?WcgMK;7(RVy?6w?JX@z7w zzguS~N^jiV-i1R*a#Sv?3A{r3O9p&g-rty&Ch%2R7xGkXM zJ>XUYIa-ME{PW1{>A-~DdhsEIkOk{g@gl})496jStXqR0BCbHyktWbOOOY5)rAeK2 zyyuho^BS}yOcw)}u*^GF&iQEGD}{W^IA&`C-X~XvGz{_;xN4_cObH1lCBK!W6J}t> zZ^SDE-sd$u|F+CU8r97OWd$UAIV0*#sm^HMzK6nJ&}KUc2Q6SaT$L^lUr{Z+mJF7f z@U2Co;WWXVwQw>MqDiKOW*$zuYHU)H@Z2?+thDXcBsL$?_zh7SIlEJ+7hof2jM)xTfFt-_IB& zjfyBCC?W__(lMApOLv2UboZ!XfFhkr3?!sOx5?2Fj8Q{sz+jx|_xt;u^FI&H z^Rp*=!0WY*?Y{5py5Dte&w>re=G*6;Hs9=!$Gm$AUg$iYZrvxY^`2jKTc^|Xh^oo% zBDI|Czo+p%l2qnf-S?d)>JJ$4O~(gx*H6ahb1+ZYYeI@#XOQh>;W)CJIA+4;Ga_t0 zs5P$OcV;nY4X1h_dV^e7r~JX7J@Igp6}Fr844CLBn%6z^KqAN@V#@qONPHM(efR8g zy@(?l85|@HOu&SC;aqD+Qxb!8YM2F8q}ewi*FY6t7I-pFLPx!5x>)KC&)#jrJFbA*RTS3>u zcbVw7L0xEy#|(48KIl&sg_7a^tO-yQY+OLE|FWFd$+&lhG+!vkPPaii2_5g-0G4t} z-3^nGA6Be_bH2Oh3!mU4oE^NR>L+bwcdWf-hud^iwDz>-sMZrl5#@O)&I+V(mt{au z!_u>bm6bM~Oja(^oOK1ThV-0%$fE`X2N4MCCmCk6SZbLg@kPkio2g>^(fubDdgVnf zGK*VUy*1x%XIXfZA7!Y4a;P4Qp4*IiT`s!w*`H0G`7`8EqgH2P9fN0HauYzNN$VX=0|h$pH0G@30+!Z`&B-Z@SQ;MGsW?8km=yk`V2 zbR!r;4pm@nX*ICEIU$8nv`|hPOH2*12&03`=rN_z!w9vaX7MA&(`FIaAtPB9E)l#Q*(zWNL-zk5!95(E`{nYo&BypQNt6q8ogTDE+nwr7@WE|Db zj(ubOX|OUJt+OH@x->!w-&5=_t=l9g46DLvPkkLdJ*%_!&OATDb5CDbEqe`COO*Q^ zzFTj2{uP&w#m)UG|h2ki$Q|{mU-FZ4wUOUyqFLNHvJr*Fk#yP;2C~@h) zX`@5J)OW(4uBy{R}Z|I@;87odWV}Qe#0&+9?-i0fJrAf|8;u`Dck^Ome&zz}{8RqfEgFRH(doPh zmUfWWzdchc3A3TF)EeCiMJuECQSs)y&SJ*z7UT*}LHYeR8|!T-vo4ns<})utM{}RmLRqvdT{~ZY|^6YuF69-!t2+ zISE|W1As8mbi5FyyjVYNUqyKcL}hHOjH7yt|Mas+Qh4 zYW^rbtoU=jsv~&X8HI7N{Sw3#qX6I>Sr*(Kh%7_Oww!1C;hPXfX)f?$HFkO+#rB=z z+bK}Gv}913Cg^&Iy>97OuF(FRGGyx&+H@oL!R-h>x*8Ff_fyGs*#Y(ZYI;Eho8t!1 z^E_;fYi}cBpTj>WlcN}`E2nL^3Vpa4P#HYPQ#3@sCwoD;>6(0|r{Orw#f0Ih!_r+E!Z&P@uu-Hu+CZGmSk&rO@0Q z?1^|(svQ_8$ri}oGL^I{4*!gqY0uf$D9yAW5o;{nGjda%iPc=TU$yNv z=O=x^G>B~3V*q}y^F~ZC@G&HShQb*KEttvd#3y0dtF(&}Gaa`uxFyBe!_kYHcndkp zZzX85Ym>we9BtJ5&u-(FLWE;$G}q&7Nd43TxAsaTcfIDSQUy9fNmgt>N7Is;>PtCF z7=tyw2jrg;gN4iaH4Xi#pd5*h6roc(wsAf|(2Ct+{M4oP&B4}IdPkQAmvn64A$zyH+a?62ERPFE@d=hX01)CvRsqRzWZ*ALK zQ~+jJtC<}puAXNZ*gqNcdZkWQN8Uhof3}&&evAZa|1m!4+3ncEkLVApG;_h;0(&FZ z7uCL-%=`8Ov|l&6-+S#}bX1iLlJ%w4Q@U4O83YCJu%rjvhDx0m$7M}T<+pF5TnCVs z;6p+3p%P)<)KVMaL<1&bBgG+x&AU(0v}hfZf!5Y7QM$w%a}VxZ$zGsAfoE82TpTh; zv!g>?*v78a8y-5e^8sw}kLNmkh&CtmJBJ1)hiyN*JzPmD$_^{XED%M+Ffd$Y1Bdlf zBM2|Qq{hJ<5<#RcE>(A;Lc2IYiA=$d!K3!^J3?|MlHxgEn{-`c;45wb?b0bmR6E zNp{ln^k*3$pt!5bbeOXKi{ zF#R0YD>70)lce|=Y?(7%xd6bmfLOYeg1D=fynfEER5#_H9nHolDO5JrT8s-etSr<9 z7}s9}!s1u7m+Tr2ZKOfvyQINIf9_c}2yrq8vm%-lozYdc_n4e`QV&iG;|8NnqRGsX zxgXC%5vq+o4!)O_SYsQ-oG)q0eqXAYcL(4?MM`76bSb<9Xz`<*xWRHNV|wEQTdUV> zBNT`l*7nmdE)N*)RGr7s?TXAuFXahW`0vipzE8~K*i^nc8FZw*T(eV$1)0mnY$HU; z7QyWxDCEaC+)LMs(<>%{M>1ru_X!ti_ws=cl+B8kU6uf)ssPqVj%ZkGw6J&^=p2l! zc~5&qAcgAAt$XbLt@WSoz~hBWz7zge!cSqQ*AR0MFgVa;9ds^Lv2g_J-ev$Lc0GkM z6|kFfJ7pemQxUnh)^`m;SksqCI?dmS zMzaAU`$i`04X_9smD?5Ee)5x~Y*10-ny?5v?v6Xhzgp!Y9D>kBMwv$CsvwIO-G~_J zI~fbRk6ZpMfywX5U)P30$#xwu)QbcAb>+xBF#YE^vup}M4?Qag?cH#$7J{YZ*aBmOq-CFlu%#)F@gMKpm>DR%bRB)m3{&) z8sQnxlTu=?Y&AOZK{sVUqe>jyNEc`O%GIGdz$Qp0d2OZCud&Nf_`@Tq+kyx55p{4K zWCSP6qzc&GLudGOw|un3tmAbfDzvE{`RfbMt`(t`Bl!I0?mWZhhrvJAhnC!-+x<<9 zxh(q#=mtrEqBvPC)04N|&n760Wf>;}_>53Gmn(#}zs^zjOTPmT3U}>a4ecycW{3}h z7nY(-Elz$J*X_(7gOG(N>VB(=C>>)#@lgQ*Qb=G2urq)+ACcg$jzphnk0l3eXQVVf4vc3@a zV@qEaL8L`?OfQhN<~8+PZw4&7#1mv{ceH|`q!XVrXKS1G@Fin^t{0}JaA)|J_bZ#4 za1e(y)yJQ<9B{```JRfC89pm#4Pu}ooo zN0OVq{?W&Js1U1pHf;V)wA&2TPk(oTXb}K#0yXG*+0AO); zzCWd$<9w(%nM!3TGx_}9Vw1MW@|RrM*?PrS+BNq<+x#>1#$@vOccm^Q%#2zt?r*G8 z{ex6R|5s(`&!{csW!0%=uaO(y#_Ki3GBZkM_}f|y!#DkHhZE9}>{%&bp}=Pgqj$Y6 znv)Hz6m>aAy&xJZ;Syoix@6~HY}6HI68b7BI$AP^Y6JgDct3L%Xn49}% zp*l9S=5v)KhB<9NU_NB_k7;^ot@95C!_0$u&t;V#$F;eZ`_8qRNwq9|oiQnHBO7#< zO0IuSQ!yEEQY>i?{ro~d??h&_)1LL?wQlw? zeN1c{PsC7y7zDSSG3?C zgv?jNt54Aw_>Mvb(M&xECB-IP-%qpKV;U6jl?bY+A@U&6+Bz@i7FWVh&0q-L##>f% zL5XPxWAC-t>mGte627e{NDlS!K~T-alxE}E;;r)(-yxz%vxr%y!9;azxxhx8;QCXy z{@8$Jc>UWLVj0eBW#7Ra9PR{>DRL~|brsuL z@48*x%PRDXd1@#V zC;V&p>50$g-HcYNUz73yT+00`+5F~nbcMM0gT=FVKq%@M?A)P~spgJPCPpCFZIJ=?5Jr3|{IV*B9<#G&HjfM*)0rGa=x70v; z_3pp=<-hy`(UDgxuawz#XZ1iK+_~n+SczFgh2^^MLEjBuG&xm#9OWI3SY-s#m`5~i zl-+1X=#fhVEC}uJE6M{~Ns_IEAlSf+UK5ZS2Xl9|Vh`O^fp?CtT*}(&4p41%gxej- zKiF*TVCh`i4~B`Dx;r4%m>=Tlrc=firV=GktG^{?vdC_FFEoa@2vxXrmve=*1?JvvF^^KP;IO%2bP-Sqr zp5aOJx`z$2^eMV0CO%MU&--Y8q*40p35RTbS==O-HQ&Aa{$46zw2x9>)Cq-fnKj!- zjlFcNuFT~(e{NMa@*YU0gNAaoy0KCE08S2;`x5X{`8XfVxv8Edl3QY0MQz_$Zi9(eSa*B zu`=fR^XT&)Vmh+L*n{w#-zf@eEX`VTa7Rk`hmg-)C!cJBb8ZZVcoXSn>nXgzDErJ; zsOk7n#|FE6$$jT?-T^g2KA3l|Jg3vc7j(dyqM^@|NXHy%hzD0LyHF7!l7fUby_K^# zX0l1(yEo~ZIDCOjr|f3g%=j|7#iy=GgW{c#X$N4{GUh>d=rww^XPZdUXbgnFMJtxc*cKm;cavJ-K=oEk zK(28fzlLWo#H^P#_dGtJ4ik7yQ$_1Qi;Ao-l9$zj0-6?HN-8FKX)Dd0oC_r54fFY7 zdkNl&KWswXLRQMMwQeh?wrVuxOe&X-LI2d@iS+v!h#JBE$Lp7l@!NF=^Zbiulei|f zYecF+7v?)cM8oKURf{c3&27Z{j0Di4VC$CkY;gO;Z$ttX3)aTvpME~4J|3WTBjDrd zN3y738!@+1?0_ux5g^mOxpksMtyG+Xrx_L=Gb{FF)pRWk)+wzKp>x8c^5`8M0a0Dk ze93&GK?dFi{b91B4^HRQC^41K|*mX_pG1^j38)*^7MrxWx9| zCt&!x%lf4(rY!&fEw?=rfGn6E=#Vhw5U95|^%t{}bsnw{Xe$Don@Yw7fo1mm?{U=q zQ-xp|^x==SJcfUeT3#~^^Qre!Ke&8`(!iiGB3|PDY}=l&ZyW83d;11;$s97>3a`7p5>qpRCy^7h}TB(X!mdMD*uQZZL5v)NY zd)~^Hqeowab_(GqP~pcqQO)^WH*Q!zKIBUdK!Akm{=k|5Q5k6~bs6xX^F5Q{@r}2z zSdvAMMixFx%DH&x!u>QDOSmMPQvP6WvOZ$+W>-OZPJcLo=E*V)uuOu^FE%Dx_ruRH z;n~3pc$bJ%{F&Y=G(ns?W#R@+x@-u3aDumeRzH~Bjv}4MUH@Xw1X$ueTG0*O>YNp( z0#PTwnzrA?SRpFHF@uzv)rndF!Mz!0++k|${<=rUyKa_CMZ=IpYqw*L`pzt}(Py7J>NeGD@YdoBq@q? zI4hAmYgnQ97kNK3T}%q;&$fzvZV0s>5k@!D+8xqZBocH%f`O6hlf_f(DCFBMT~G&y z)3W8x@m|tg$U4(2ybgCSC?vK#-u~Jh5KdI*#z?EG%KVnZHmp9}oVYK`t4;x7Y>p-Y!@sbZ7AIM7GDfQ-ZDFcDKT%X8W3(!4qM}H znirJo7A;dpgDYLAF8eHsbWISvwJ2D0-jDs-m+!xUXC&@1mk$8ooX*G(mDEq66|YhVK_=$k%zlawSq;243Z$u!DONX;^wQPROA6 z+j3hY&w>NI2WF%rLqh~UsK2dP{d!&Yq(l;EzlMOeXk?!zMC5ps>`}DmW160~zPAZs z2}zpkLp8(12u#K%dUS{+sQhs3k^ed3;UhX3rAsAFq6Y;%Z!`QcEvuqyyvY>#1v0?IkY>`|1k& zoJeJV{`ck0C$zxu<=A;o*q6v>!-?}R&f7v^EgChx4Q1EF*Xu~k-(ytq)C-#g;^|Xb zNBnJVP^Ln;G42QmEU~aZL4eD0EvA|rshoC=^5iUZTVJd0j7%%+VVT|09&BD_l8N$Ds**t4ZUijR=>yeG6e(Pssn zO`Lq3=0f^NF-#ybop>%PxB{chu-Ju2S!%TjW%O&9WCR?**EwW^U%`bG!!x$osOn3u ziav4JoP*ho`IBfUnlD^${zFMs)K96TMd?GI#IA>P<*0@kP=yiEp!ycRIFlLj82#XO zzj_R}e7*LG3%k60b1O;vYS7hrkHNZFkuCmEZs4u)k^dp1a+dLE%vj3diKb?^Pll9- zEa)$jY7kALfw3Jh5y@Gpzp@NS5(Xb&`8pcISPvhC#&yhTU?*@d)}8_bLN|c5I|Akn zjY4j9+d(o%#OnT;RZSkW1<4zgqxKoq-8~cDVz8~ZMEs{_@7g4Zp^RGENA^COD&oM8 zz9c0fA<~6K=M!J#B|z(3pd8>1@V z3Y|*`BFTEAGG>3M6S~AnipG|KbA45PmE`WIdx=@rwmq>(2!(Dth@Lj&;{DnZ(I$;P zg=pXN>Ji~<;QVONrky9`+ih|A(hl!)CC%n0n3-+wjJg&fe`9rDVfJ|9GCpGr*Sny z&j2mA?k*<~htbIezfs@#kF&zKbq|hoo`W8yTX;c*8DXze;H$BF%qYjM5C*NRuu$0C zA@upodhs3t8-V8Crk|?JZAd};Kn=A%+%lz1)Vgoh^y`3)46j5)BWAFmwFH-nEle;H z+iwxswS4j;RwalR=o{%1P5QRq#ul7H$bJ9av^DfdfUJ|cKyltz?x!!G;PQ+Gs&Xcfr8=GMx*NkY_8^+a|t@ z%vMRrJ+o0Aw_9_z17yDhECt)DIXUJ^&ep^~I!Y9~0nmWRaNevYQ-e`Y�~U_mq;s zOrP`z59A0{jX|ys1b9g~WR6OK?O4YgOtz#C?fz=29)U!CYn~V3M-d-5Cgm57o0+)B z3cNP8Geh0RdQviL0f5C_1iPnKRi}^@cF(fJv^jHxo&5)w%-sj5XK#X8LMG4enloug zt1?Hcic}O9AGE}cGQCqIZu#u)dDfx@eXI8K<=Y(i#Tk&1l?R`HI(wdX^Vd0!a?@XP zUjIDV_y#bwc-TUJBT@UR1@}9tR=d5Hz}~awNA`)vU|lnG$&MYh(|4M!81TYEh_9d< z>mn}2VjlZ&8PSslW_Z=lmE=s48>OM4#zuW?*y@1uiI9NX(#QTb_l z-_(B0uy2pSVhNRKH4km96@S@F%75^kWAowBA@0*|S~FJNjWD$@un; zIlHa#t7cp}TJ*?EiOCb5Fsr!8#OZ6LOd_D+qoCq)wI|p-J3z@z{jHKN5FISja2lBH zbqCp=OUo>442l^=l)zeVYQEypJ@@!5a6<@YSpKjjz?o<+B{CRN8eMO7bjvatmEG-l z6L1Vn-W=Ltr%ulaIrXuIUV5`hf$BW`1REwB??M&B3TEUK%KB^5`6EJ5l${)-1O|(P zw-Tnosm#^(rwBB*8VsXKklT{k!$ToLMo6{}pVuMpe$>jTb++l;v*DT7KONk+AXs}N zl9{dRkk;}qFml34viItUzNvBa2VHJ4;L-jMq;~UP{e3V8XWz%W%*^1R*Ymc%A5Jq! zW~^$DT^CFZB)D8Y?9T`HJiQ!=WJ)4^woCE_osGa`Gl78}4g!O@#Cf~%i73(TO>rxV z>jr!-juMZhEdGcSc6?zkgAC;Uh=RDWT{owLC4E z|ADi#OGlIQ8-RfsQI&##n{E_7PkB;z zD||bSnN&cU_Ua#0xZOae@{#1C!!PQ&^At>iNV@%77>6Sfxeh_)Gk3QIza#K_POP3PGzhLygQwp-$j}F)K!9hQ%%-Y?`zS6WvwZ|^vl_9NF!kiGe zZA5^G{*i8K9t$Tb126E)yJftmXtKwMq#)EGD%}ARurY@X6U{YNBLdJp(C6 z-AYnHnsyE1z^jS5-Dk4jcB-U%1m}h7xb!~c(0KVcShP+USq>+)CN$B25*2v)wEGm< z2kV$fCz=dbBzJYOlIBOC=1$16LiB}Khs^bMn-P!rI8KkMwPMBMjAGDG9Ezb_mQ=SZ zH2jJ{G_}_SxrkV)TuHJuaV!x_3Uami-0X}Ww{MUHwsA~Gy9hJcm5m)+wk))F6&YZW z4&-y@0j_wC=K$rBR_ys0e_4&Za6tPuCfS#{2~6NgUoqGcDSZ|E^}MssRUUG3#t)N? zO+ql-9#4+Ge~12-^Dqf?h8_g4LIX2d32C=OJcJ!n>_!JAk>D_SyiarFTqG_I-r$+K zTwgZWfT&FIP~#K@)Jdi{Nw_32mgI{Jv2HS7%(R)=8=Us?68WG)zx}!B|G{2j%aV6^ zlw#GzeeMx<)s<{OkOQjP_H*&seCRRHrMPzy7X={~k3YXiex58%Cnpr__DBFYs*fy{ zOyMP*fK>!B6L*NgD7y?fH|kC+!r;1q&|Y6dAj7M^CLThh**IR3=%sZoOVh{No+0wv z@`wJKYbd?cb`Ubkdfb9l|+-m7C^fzJyqS)5F=_f zfV5roQAtRzf7^BZ48X?#O|^L&)4VtU?pcYXl*6o_1Xao0F&j$(baj)8c~Dl?>8{E| z%_Tv69RpaRuapE{R3h&tK1fefo2Oi2r-M=eHJ_>&;65_SxO~e{XhrWNyUms%Q^;5c+To z339)(GR1j7?8e+37fz!Q%wQy!&o~a*Cd}QBVpMp`Jkyit$(29Ez@sFW zX&ziQz+i%=T*^!r%;nh1^uej8dqIj_lK&{cGdlvpFwt}T=OqvH8$NqT5Y*GcnmMlz&YA^R zRptiD8c-;Q6Xt{+m{NHgeHC6xaEx-;@sqeg9mDX~R<)?Lwf5zEyW-U zxWH6EZgb@m-TMmJ_Khgjp&@OMQ?vEeVRqS_L$V2b6Nt%Bru5uVFk}>Z+TimnYFW=0 z=vxVP`IVH-6a4I@>N9~X*m+TLIu6*8CS}>7r{(r10J!lFDYLe4YLYCw&Z`FES@+!9 z-ld$54UVpEu$5Hho#N6rEuxx@@aC-TRd2ZI6rXLS$LWL1mBF&Dd@0&TpJ#Hfh4L#? zZZ;>)WeS+KYU9@I7ngtWy4T_JniGGd+-Li6VD|P_nxUlh4?9T$@ML8 z^Ujv^=tcbjO!AL1f8qhOgnh?7P?pHuT7D8!L!o+dJ)G8O8j)fSYbMCL?+)buI{t^c z8FD2`kDxPx%(xk~@>fGp||q`o6%#~0plelmqY9wwH)Ibe;LhIcY~KbZC?;&4&9_G^dysH8gj)!c@Mkubd2-Qm@cl}<)7CKG)_Tr+ zN#;0Cce8pGt+$WZsk?(QyE8Fw5Dq0r(@*~VYhdGHnK+208=>PZ8J-5N$Im}sWCz;i{IFF^(C;&mxEstMfd41?^-0n70yYY-kP&+~ zW-X0P3$3lGOxi^8fBZC{nVTC-A_bjtFGMf8Kj?3R>IvDqhc{=oABOV)rvN$8zk-Bp z1mf1VeZ^RW`IpC7Fi(s;5M4m~oO=L@$3ea?9r&Co{kB(1w7rsWbjf}G~k?Ocw?@l0)7)k73~8h=?pZR0)D zw~5sI+qZ&r`-n4*41oR1F9ttZ!_G>6AM9IoqPo(-Vdtyl#7uR)zM3|Ee=Mo*}n2o0HAEJl`_Djsp(lp{QI z_0efOVz$_+3Uep31>cUnt`zDtzTE=ga5t3-nm*M#6hni7 z)>)RPo6cX{Q(Z%KT4=2tZxNHD^sF-KuxWnwGTT&J_FmA7V|wU4`NH^RpDf8#;t+vo zhlvd8q*`aP2bmE}r_@q&u6lLU*|1C-&?w=Y%NAES`Al$9axGe}8`52cYoEEO*>3Nfs)q@DApezq-BSJ)jtxtUF>5n2IQm?b|R zDd=w9du!lSpJ@E$c(rb(>ezOD{ygH^FH1(<$i$x} z)?KVaY~^m2v9e8h9|bg46^zXGVR7|olm1yO5{piL_rqZnEv=7nhFSAgfCJofJKOGl zdgZjwIfbkGz-V6n8Kj5Mt1bBu9@w+da2e`6@cny7+N8UezkY*f*S+>Z$AAEq8(?zgOD2>7z;nS%o}hLX zTX*pA{W2r<6#?jg*?`srs}dtw^*PU9+INQ5wJK|xc(HmwvYa;`E@1YYtP|uG&Yjt} zlIVIbh2-`A>-j1fP@bcTZ4%AZmUb4xvCCX4^tzR5@44*ok6~T#`m39sQiGtQMIahn z|Bu#jk<kOc^nOe^u5k z!i&#;_svex`Q6-DIU^V=8V5;ZShe7#j)j(+_ef~3Vl&x>g)2yhubByP)Asw`n_N;T zCbuL%z1T!Uus@c&i_?JY-~GW`Tc4jF`Zg!DlKs^Y)q!KH?;FY4@DA^5O_^;Mfo$Sc zQI1a}jS{>eiu%@v2#V4X#>0^FV7`*mg&|p6hMH4bty`P8=M4f!f6ln0nr&DzA!Jk! ztKr8+y`{n6-Q#w={%*(iJT%fd8znb!w`f+6_WhIqFQz3fp=2TC+Jr~gUyj&1hU1?A zEUmITa_KIq&9!IW7)Ng&mqhW2r-_q|5{+);w}Dk^zR0>7Q4;j%_u2=j4X2*TZ#P7F zVosX+*JxHuz}FA|`dPAmsq%`yH-!EJQDyaRlJ+~$f={`y4#);NrgoLRrlkcW4pj;& ziU=4VaO+P`b=NrqpBdQ?^%P6R2hvsxnyyJ;H{JY}e zyA2JX1C~vibfqYATKW$7uKm%%8!wnvzq{O2luWQ zg(g%#wB7Dx>N4{InAgtdgIbHdtZ({f(B5H&4S1tAs;?f(VCca23q>$JiSqD-%#|~b z>C_(vgte{HRhS(o0s9VCNTDjfE}=Xm*XwQ?V&G-UP|R!0b5~_5<)G75sxSB9?=S5< zZJZIpIF*38#Pqg!HkF!%YmP;d;WO-tqh&-Fz#2?_C9ya9F*-hIqR2=jKb`%QaK`e@c_2-M1 zsqA*f-G+}T3!W`j7iZH}%sfZGG2(%~%4%eElo|SHPVk(p@I(RECO>S83n`=`tUJdp zuRhg@j0#oe>b9qK1{u24cs=Mpc9(tM=ibcZpK~@ZRVhbfKs!g($e4;LmGO01oQN0c2g9dl_s z$9=iA*4C1k=hLY+jkA%YcVk$59qmdEWbgRjOdx)7-!t>Vazhu|^s1owVuE6`CAq}@ zXp($fkYuvBG%^!wbUJshfh`pyFfwHgR9Jq$Kl^R~$cUcds@c2F@|(AJL(**Wl)PzZ zN6}%HCcdcqS`F92i2+&!%j~) z(E+p%4ZhxbX!NgNGSoUca#xi>ACjMCeU4BuxBh!))z>NvxyJCuJH*t>NBxzuC_LKA z`ML2N(S4OvIc|6D(ZmN*VU3#eBnOAE!Qwj=r_3F)N;QWP0yQYu}nx6AGMv zFhtPgQI1O+1~?vK+kY%4s3}4cv6Fwp^hU6L5>CYpLqrlT!85y$AX(prJqG$I;QzQU z2#HvPbjcvKod?1%l;s!EDX4REDWX?2*!}HE>c5tkZCWS6GmGC2V`=Y$i~a?}mx1K& z(Xivo?u;w8TmKzAFBkQ>CSq=kugAt%%Ldp?7+?f1@2&7|QS}2zZ{d zOi9&Gbcmd?H+e2gN4Qh1C^{~pBW(YCRF$!$uzS(qxlxl^r-RGDA|J;2KhXm`PkkpB zGO<4mrLq&S6^u;fpTT|i2<(P9Sj}8~-N5!byb-t_yFVL|dEJ8Z3uDEocPW0-3uqSu z>ItUaIs*|)!70n|VAjQ*_`kHZQJ)doXa@cd6aeU(@bRRz5D%*@nEW>|?Ev{7kN>H6 z+0uO)pnv6>Q<@LC#4zUV44St6oRd}>Qc9NIxc)KN{&Lp*!=}8+8(AC6JX1WJMDqjE zZ)zLJb*s5EjeJs?#R*w-e;)kUIB{ ztuB+Gg8OwsWSa{iFVTY3^U!yF0z{QX3juZ$eF>9d{`26Lmj9p0p#S;9-{*DTZ~s4q zGyFdTgB}|FCwnj(IrcS5F~u($Fb@p;xa^pLJ*+;jNoA}8f6n}%>-nVVCF-$+p@dOA z!v_Zm6KDVXg7Lr-etM?;Fay@sZN;>)`doA$PB^6bv$r;cBp+LyehcSoH=#R81= z9mgp(FX>H}KD&i^dhv6qzk4dvkyNG$e|y6_hY%U_ZqlXLhB)!#3tZ*+qbxm@{a4f? zzyJkP%DBb(aX=YWtL({+*kbCES`E$uzQI*I>E%FHz3=I`FwT!TIlS@PVPlrM-7JcX z3y}-fGR{nT9uH6pA69KF!6*DEn=bQ$44R4<)6amk{5&QTNfg37TWcAeYq!{!$XvC< zM?xgCU~nSHqSTmSDp$jAsXwTjf6Scq%W88Y*$jSHm3gpS^oC~Rr|O4=!C$c#v$gm& z%n%?eY%y?@M(gE{7R+6JZp3OfeNvv}x|h84K}e#5?Ctyavzb|B*MmeQ1tr$){@typ{m>oC{v9+ z+e=`GO}bM};(~3q-S!6j%oYq#^URB5X9W#H!XRqls49mgXkolQ>i$dk=WgZ;vpNpT z#cWJj+IBG38PaqP7mG=ACNteO<=5RyKprtv<3ozO<}aHw=HYCPBfbW=;Ny`;XC)F? zzqHo_?U&D!TEx1*V^mxSCj5=zj|JN~HD}$MD4&G|Zch{3Ltt@sa%k)P1{&Y%e2Upn zM7?V++sxKS(DlB|#UJi2sLR8urxzZ#Qb&{$lp^5rg&_4(<8imUq!cfk{mx-B~gbi=j7F6Q^I0(2yRVmRV{ zkQJrn;l#Jzr#I{(6bc4NH$x%=)@X-hsmaRbn)&Nz0Xtll_mhMT_+-bkjEDh}LBI4S zUz_W0cQ!i9_P;|lyyR!S{83UEkEYg2Qc5n$I*?Ui&Bn&>y zBAY=G)aS((0()#^+-BOw`Quc66AQj4O$pFhfl_9%L-19qtGd5Av4@%WkAiU?JZ^KR zUSHln%35lIZ9DuVDt#e}b(J9NafG|<&H_l_jcQ+^)8{{cD)Ja znjK1b!AT~n>jyE6#7otI5yU;i<8Y7oS4fDhY0ujqPN4LnCI533n=H&6f?4_p>Eb<% z@Dk3j_1xI1=397BBbnd2OIn2-y`hYs%juIKs8#Vd?KfZHdSP8w!Zz#;dMeZ7`|X>M z)_x=(X&=`8gW(HodA}`u5U0a-IhT5C?kIxja)?3vo`&{K{M)Zu=b9~p0PGtD&T~+} zu0intl6=mmlDb12;m!Y|(EVz=-N~=V$MW28P(@Ae;4E27=-VC*6S)*%W4ru(#QWax zFH9JvV+_gi=kK#zOn~qFPK>_{0m$FEnGVuR0 zM=~nH z8zTF^!;Z6wv?roi8+>vDNvR zUn|Hl5^i;%(9|38fA##@lO=O3@P>@Sfr6%e!Gp|OJ)I;zDqbT$kTe292pOnqZ1%i( ze&G>h_Ej(|u}btlV0~)s(k}%vl?d=3P!ngLxVvrCcq7Z|^G@M4gs@pJxtNxY>`(;a zlE}KIuT#`Yd(@S>Hkozuz(`tuhP&vG#}<3OA(<(gV++s;yq=2G4soiJ^KteO`|lqQ z0Yn78=HWd`4t^CTQgbt(u?zfF zG&v_w)Ekg_^>tz4JJ|1@z@tQb#;Ud64t-EeM@HpHZA}-O10#1Kk$>Zf(W0s^`@vJc zdWaF0POn$~;T%aL#4z8CSR|u?OT)7) zA4m=8ecjru+IDDRSk**ww@Q9yYf#zOBpAUn-T9%w3JNR;;DC0dxMjfN4a;Bl*a}(a zJ;YG?H49uYHOb(Ubl;{IG|kSNwnuo8EM2u;O*H+Y=2&pSHY97J=SA(|6sS+O&m@U8 zZ?jGAvhK=rgT-gBcsX2<#Il#NE3@@L9gd`0FzV2m=MFN(m81oiP5eXTTjus}-H7|8 zzH$Eef3^4BZ%s98yD9>&qVie*rK*5P7wKI^N%-2D3r__Io3KWHLwOl~XewAH`)8a2#X;n7BQnDZW$YJ5)=j3%dTc)qAhHOh z34P7T=K{&T9TNVijN{ByYLR}5Q`fW8 z#{^Hl@jUAq6BXGQP8DR>h!pu8cNTU3o`Ef(lHr8@LyAy4=+8&x;;eweK}GH65QhjP#c+OJh$A&p~HN z+GH!4oj7I2I7(4qzsCoqNw*y^oC%4XH+{2mHtJ-DXAJfV^I~3}H*ou$hWN)J=%KWm zIAdLEjKA={iAt?Cl~joRC}e26eqt>Pi!hZv9eKiHVnvU$C`ZY^VQgt1b5X}n$c^G6 z&de$4sQ}*>`1zu6;nqqe!=@p#%;x5~53z9O_dy-63*TFOX%6N9d-a`sggi&i?Ed43McV3kIsY6z+=Bq%I^EC%bnaK`h~82 zKP3sVb!;7FO7#-)dR)nBEM4w{0LuJCw3X_?PJXFANMV-v#lV-Q(el>$L`Voi)qg&6 zAbS$!C3U*$-Q8KMn^TNiwUoL3iAMKRxJXq5^D~2yWP=P`;9=WFePIg2Q!g+1FP$8p z6)KPXT1>vI8mx#_ax>nXd3Y<#HjlI;j%r{efc&9Xww@?L-Gi8}A6&o9f0Z#cy4{em z=(vO~5{BR=gF8ZH4r5K_3t!T@vCcDAOa}sOJz-^x0_pmQq;*zvym_vS;fYCAm+2~i z|Cg$gueaOu$mo@&NOu4|T$|72;nL7`JaPqb8qnLw%11Z&rk^gqE0*{{n1Lj!*1C9n zRwp}~s&$9!wgu)er-S*UUdjHkWh*m%1(~8gGNfS7d6iBlGFrbiZYZg`b|?ICSkfG7 zByePOG@%T3(!bw51I~Wx`#S9zH`YkSh>3r(6^uvm51t%fs`C0--S}nQ)~HR3a8hR9 z&De+_USWqeuzBI)FZHp!+D0D@bR zJJuU%q_eg(5+EdPu2^s~g7oa=_<%S2w)RViV8z3(C7tNO^xPBT5f#fF$l=FfWa4Ok zqui-SW|?9#oXmUs>qrC`PY+A^hCtsEGxU3u^;bDx)r1p4;y1sj!gxmRGvIjTr}HSm zR_1rPrJ0-$PB^dFg!wwi{RWXLKhGsus5=BLpZ{o_1~C(Fd)eTuc|H_1Jqw{6EQ=^t z^6oJyIW=sISYK2yTL&g`3E!LiQ0MIGequLNQ00+ZjMh7MoR!SCEboSkV>_0KheQ50 z=EQUyGBCPlrczt_ByF3}!&Gs!P0ia2!WG?+i2N#he7o&!jVP-#E~{~y!1U!hK70ki zbh1%{uo?XPi~u*FJXzu5)rn!%azoyMXg-*OyUT9fbC#dOO`liu&6dPC1OzCeBR9gH zez$u11v?Y1EDBhmt@)UnE*?`0K2pSk#c@7)b zPaA{GyPJ;ca|pfyX!V*9Dnxcx?hjuX-JsK^9mc-}*Mi0 zZXUJ)IFq8w&k+-^(iTusX{cWvY8jO@8gC`O4motdzU5aNNM8k4S@$kETjsWeE~z~B zFRcZ4XhtsmW@ogWcH+BSjaP?z47%##qvJi-KHH%Nw$|h$?T4A$*6Xj+rkV+GjbTGL z)2qvB4XoIcN}g_f<34{4K6h;ouD^8>-b-0ShVoPGZ4D8&udNn8e6IbnZ{O0&>UBzt zl9?ehede6?VDW<$*L$$@Y#dXGClW;Slmedo^@o4z7v{_oq~}LR8o44Jh7A<0%}=Ge zd#WSiga(TPSPL%%p75l{YD%}E)e#Mnq>ZHYh{QKjBAeM4K1|E+-YF68H*EiKxGGD0 ziFX9Ks{VC?S5w^+Z=~X^WX1;i$$KLIqGa(Q=(OdYiQiYK_Ik*;{rtU+5l$ZKk&_jO4v@=Y;hgjQ{`XP<11vv2kE?UlY@j*EhsDtd z#u5oCGH~|Pyk0cNvCxDkt5Y>?tgUg>)2ge^{b#c>WNep_N4+Lq?1zaAJ)9Y;T8qNK150|m4V9oF z&9J;{VM40g#2c&k$y;d-_yIYFSFKUmuT4+Bo?o4Hdp7vE-t^ql`oWr46MIJl)W}Rp zB4+E}hJ7HkK9Wkb!5aD|%)A?ei!0C(MRnX7!R;q88ezvVZR3Ii1r4{(n z->D(ylWtnpilUS5BPrJ8?^gtpAL{H-78Z}@<$#K>ALoeTYjpbBa6Lug2`BG*2Lp#&BR-8*7}nVZ8v3GQi`8Tk0XMR-Izqd zh!7%np>t(2^+_{|&zUoS%Evp`Z$0QcsaQ`JoIpH6I#iAL+jRAcbLl*hEoAC=4JcEt zq^*o;E@$Mc2#^z%Q{v9_HlBk0{CVj(h8Imftk=dW0E1IU8bH1A@jf>$c0qMI!LvU> z_MR6-a}!&}sSv&8Bfa`hOg0yKTM2emh;dmFS$D-ECOFM55A}8Og$&+jS&f#>`F7L1 zx@CZx4#f8GFxTPu{qbX~`4uYwDX9#CH4!pKbzk_gu~0LM?qLvRdiy9vcT8;CvU+2g zuNa3!XK6jk-X@o)u*@WfrH6p$CPZrOLAFwY^L*;O+wH3mVD^M~smz+Nzb9rj-n0zH zRUE%UsO#xunw(H2Wzr)*!CrvwGzS=y5WWL@JDsfZG{r?4J!yMb^n)b2oR*W4Lpo=4nIzA5u*a1A6u20Y$f%yCP|z)A&~)ic;9$l4|a+}<i97MeL0d_29N4{Tu+`oRr4Y<2=epZIo`lG^!cNx$%hfBxKs;g{z zOADSpN45VsVGF)noJ~JKDzrzRrQ~R9A<2qh*X-1!<(A_oT&NZf2D5~sf8^+^2<4%E zY~uvBUjT;}|8{k%P>f=K17IBBBDEW<8);v~zxCp7J~mo?M|v_vnQhNYQ(!tlPmzjO zZ!rJPY?(hgzxs$Dr}8Ls5EG^60SWnBHkppj0j=Gy!Gy9g^d6a6PP%uD`>O*+!^?oR z4>VRwX|j}YN!ZL%Xj1?#zZwW%Z*;I2#Y|UO)f^1m8F>8gE>5;j-W-tLA;xoF$QK#$ zr2Z4L&4rYkYdK;sI*m9Fzy2ICQ-ho}*yCQ1GRB-Irb1Ki3cGKDQccCIq+Na8g;nK_ z9c@zONaO5PEhWYR&0CWX8i3hL;&?f?WQ*{~BREAD(R^UKbg+*pTDA(@v=%gJGLj%t zPpZk8PC+78Kqb$jcJg3YJ$4ykuS=h4}Z)h@X z*@Ix=pp^G}{OFSruRgQ)#JoH;0zqNJQX#!91AGd{^7s%e_PYUcVnpFtSK3?PF&2MR zc94<~z<*($O--9N_8HXg*d|To-WA z89t?+R&=}r#y8stQrH4aJgop7D2e^qo3tcd&+|yj@`NjQ>qaPoYUtL@ezt4eqms)_ zi&ePr8XJ~w@7i|eeG-ql(Sxoz^@JIvA*&GIyH$62%XmltjEj8IA`B!$6nDE0x zRkw~G#07!T5qH+fyk7H_g#mu(T5Rc<*sOs_`lSQ=$a?6Rn87KCKQ>Hxm#xPm$ggW) zJhIa%RbLoCq?qW1?jhzGjd_`K)aHr#(+{n8+6@4z!NVF%rGO5NOk?$-I`*}J`V&~a z(qCpEv(;E&^H<09C?UNatRE3q}eM3)TOFolgJqXihswFz$LmN8uBx3>m`Ou z&z3^!^%Tf=A(Sw-2>{&d4bSS_mKj+wbIqCkJ~Mn1ma9ziP_42qEuI`2_3>8U8~`-G zXpG*&gFbZN_ zC5m&IpZ7iLZb75{W_MHx_yuC2Uu7>DoUyF;^(=5m4)Z4NHR8*HrW}P?&)LtEL!d*h zb!c~asW0!RK7XVM0q75YV<#Y37UfX`LgQ9wp^k)oRH)8c2%^xj>?XvWBZJpqSCB8b z>?d&blk;;AKiO4WGmB~Ro7!3QV2b~O`<31U=W=q_>J#*?x|d|V!`fh-9V}mhc0_@A z3AV%2FNTj0nRl!kj{{xeZGglzlNk{d zzi+@7%!6vA39LH#vXUV&C14%bHg}KM$9v$5SKS7vHa_nf=n5#&Plc)rw(({9dmMGo zV7Bo>NOhZgAz&L?uPUdGsc%KASPMX&dFE_fb~I51p{bq1R=_cUez@mB@brbwe~cLa#JsMEvqnipZmQUoRCgRV)*?TB=#~s#$(@3^t{VRpl@S+ zqMdE6Q+(T6MQW$v(anvi0?enu+1?7yTlKLXqaUl`&pANyT@8D~Q>fUyui)@>?-@%TK>?9gUcm74Z7H#vGmlvpBt&x zz?-EmR<79hs)cmNP}qiUOJ0Co?0j#wL2476oJkr>wItM41dT1tt1N_eu%-ZS74PW;Yak|8$8wq&AuTv2 z#O+;X3b>IK#}b&DqpWn^-&!DTSJJ_5r*%K(~gy%7!5ZZ=Zl4LF?ca`>i3dEI+)p}`R(R>oX~pLKO_IG5=O4IB8YQIK^@n0>O=w=0?^w_HWkRMS<}1sbLn5ONW2@@=Hwirk zl|zZu5PcuS^2pc_|4Nr4HTt0E%&`DPW8Z(hH}jU$K;k*%2wl!UCpa{YG<;n2?QR*3 z;#}hht@wrfW+XVBzLl-Q1*B@V-G#ct;s7ocBt{DytY-f-aaEt@x zs#)_7cR7X|C=y$32xM%FowsL_qR;#Hs}a#r``PR2DRi)BxZ@%2T6Jp{g`ASX4i7Lu z_s0Ebdq>R?+A(*LSAo=Wy`v+n*=Tu zdPV$1nsNHWZ?jogHGvAWBAS0`vu1sm@@6%}7Ixw zDk(f8L_kryRviE}rEcT|$4I}42?ZYukS~iL9`9r6lk4Ph;=p;61$Wa%Nsj}rUG%Uy z+Yaga`r5I!oD#0Yj$z^7m%*aofS=lyxu~7Rw=7QC}40x6@Nn>#Pbp z6veNXjXmn&$n!JG7b4;yMnxBeIpXc^ zx{V3}e=Q@3?jvbOw#52<>OOKILgv<{?kX-Zx>@$->@ep9;Wr?L&;jw)aM?#PetndM z^eH<0T{55gD}NZJ_x0<2_!ZT3f34?8``7ZeAt@L#`k+iNJr|Z=EO9_3 z-3L}c>3ekvcV5(ujZ0puDCe=+I5}&TwZ9vj=zF#YRWo;+?6esyY~LCVqO__MaAX?9;@z><^|7a6G%bR*Y5d~s?msWyicbEkpE3- zx2ZZmS-We8c7qG11hE6a!np#z z5pE$96`OD^ou`{%i3|QbauZ(Ze#U9WO1>!W#p@K*Bq1Q7kS9E)4hXEx@j|<$j9%$I zgp58bYfj(rLFN^F_&QW2v5^05nNV!#>6Q4(3f9nTHHJiKKfav(Fgti`ANZ1mSG|c@ zC!I^Y*nA9)Y;C`snDG9p$V5i9l_!UUXDv%ved8}MOOCNGqho@C-$86)DF0hseWcq# z{-ok(_|z$^PR0Px-{c8-07!(H7~2AOk`~nhZvj{w*oEM0q7`C|MYQEHpoPYdS3hOC z1^b(8hZ#5zvqK(Bu>a=AfDRBBtwC(WxL^f#IX~yDqPt+fnw;0bydnl+D6h}LB~fpN zZ_r1g$=(Ku`i6i&dAX-!F93Y2?Ua>SPChdU`w!0}rjZG4!b2-3vDNKoDM|t2f}_K? zJiEg~yT08l%BvIux}(>fhW}yr#6!zmA{I+y8jBAsSIrQ zi;@a&z+^`>TT5v;C3flO&3NC5h=?Oa)rq(=4DYL1UPGU1_iI;(H4`j*W9nrY5dQiU zCM_?67=iI71XsHbkHG`q=o$!dXDgltq%j%)bxw-^s|;Ml?MTXols;OH6AuKu)qHzZ zBRcMZ^7CnfH*HHrY3YmC?0$-}cE!Cvqgdu(Fm6yI?xifSkiU^=GQz-iF#N-}I@jw8 zqWMme zDHkt0VgEokk4ewf`L{pHhT@@XvBhmCiT zi=(9<8=-vvq_|km4OK=FZ772N@IhqNqXe>pUxqhmf4s@OZKLPe{ zkM^2oPX@kn-7?)z>Ix#hd>~pdrIT)ODhOA*V*UmQ^0r#LxvZ%J*pk_j6W_ zyQ-Ttl2Qk0`){Xab?LE2=tay1`}?U&8G7tp;^pEq$=qubUNy5~*9@+9TAT*FefIX~ ztr4)*>|4*&5M=L5f@a!?sB0j`12DQ+Ic>4hlHjknEc0N8VChnf4lA(_IcDV@sdic0 z92u`G1!7=}W8Fwd`-SL@eU#&XJh5-mTHn|XSUcV+Oyr0DIF&J=X_wxBYU6hyoL@jB z2jZf)VV5fbzn3bdOwP&=$w@}5T5l8`tsTSLcj`;aw45ro)gPVz&QNrST2=n_GS;{c zpeR3p%elyLTQj|tj$dTEX|@GOHB$_Inkd;(ZsK^#nr`xSRs^_CdbDk zbQq~~Qs2NaerlV~D7T$m4_{u3kEj%guL^Sh`mwI>HA0M%?9C&ejSYG*o!iN4%^w56 zo~l&*ZkuF1pwwGZC9|+QA;ga459a<6Fy?ecfST3k3+`=S#`)JrF~INTDN4In zvAwfPAraQi{@i{3lo!v2p?`O~mMz$74-=<^;E8X^kKPBwV>E`{m;4?3lBVd=%9(TS zhr8=nDc%a0E`vze#hud)4iv9!%vd2zxgZb$^XMao6fwO86%YI3q?8R-)eHn=qFsRjwN z+u7dE;^&zO^6P*97L#=n-`)ro@&_u*hMqd#blHk?*5T5*PNuqB4XU1jQ++Acol0mw z&KrWQfuSz*pW-qjhLXquz27|nq5^~f+krf7!Q)Z`){rRO&*7oq`3v3mJ6Gz&mPVS| z=Jqdi5B(hcqP78uh%rQOhKG`PkM1iGcQU6}&;T#FYcfqFx9&TbJ3U*Gr&ThlCAxC8 z7sFMK@-He9R1c{tAjLqfS;Y|NRA@e5O|B4kG@%l~kuqtm`hf`3O)gPk{146rc?mZqZ)fH8BO`SD&-DfjGO08-G?n8#(qqMpI zFwR!?()Cgu5y#;S&QX_fG1HL}()j$(}yZ8`5Z!5^F;cBf}yp)b#kwvKDw z73YuDapl3hEs}ufKDr(L>4$p%>`di;=r^jlScPT1oRi*e4o%f&u`gzKrEk_2q^z?2 zE8-?qnErrcFG`%hg5)+n!FclnkD(h@7PS4N)+r&+*&b`^k8+3n+$PEDZ^I+^j%^%R zi4=~bi@^%a{LT;S<>anP3x7Zl|4jCorH$n=B`O1(`hQbV2Zol^CleVNc~U;cetk=e zWiy%h?B2$P#^@C}VQS-R%f=~ZIcury9T69+>sq>fI#qWB`;I0`yatSw)NfNZtoP#9 z*}$4LKNKnpTGl#E;cgz|j;cz&&0Nl)fu=MTe%)i6N|N)E!UufMZ>t8aagl3Ty8dZ@ zm!u@z+U~M4*~w^kA%E4&wtJ(A)XFvXv*Oe-R0&XcL`CJ^xww}RXD_*kD*65UBRU*(&*wQ!|eXlsUbR+t`^$edavD?b2ZDKvdz1# zcc~IU3ntZS7^#OI_>R;L_}NX!Z+ywzdI=n9_~mZ4_8fI2u7P`XAma+3qVmqk!!Aj` z8erSVrVx9u{Q0QSoW7t|e`E92wnAxjq2Z%QsW%V8X5?p_;i>5cG@ph}awTx+Sd){y za+{G%PPHOgG{#*#_lWJ3Yp8l5KrH%30Ea)sf z=_Fl6&j5FREfxl90ag%sv)9OaV?OgJ!_Ph~`F^edW8+6teWcIp-0oa+l#lpPzkKz8 zqwlQFx=yLSd#m6)!6=OX&+;{NV~Vz_S-F ztZ^K0@Uve;$G=;N*dXwtS#AF^RN=j@KTC61AgA3eo|0^Ydt!m5cagw}ns%rNI~zx; zeoCcD6II=NlW?i)&pel8*ZOhtcTR=fD6x)%RV(jZxbg}Ug~~zJ}j0| z@|YK_zt&JXoNB5s&~-VXpZgUw@|YTbECSHuH|v}HDMU17^>^9Y+PHf6l4YeKVv}9@ zq3#F&7Pn`i-PPa$Ew$S~sizgQY9%^zkxSvuFvb1d;S6&<@glj3D@r=Lf9OC)x}j~G z+whqZ>yEpZbvH7^7{sT~7sC+3^|qCk+J~jv^88MfDd|M(X7~0Cpd#(=z93r9nUwDp zRt>3hAKm=G5G&2)^7*9-0j}MaR$o{qpR%lU4Dssr?A}1&c{l<~UOPXA-^_r$$mW0h z?uxLRyWecD|J{g>v|7syx4P0r{tcHV${3Z1X}!iWAig0winD9;hW)+zYP6&B2?@$W z3=rbJEm42YOYV-$LyHh-eeQFsuL1AsL>xVv#m9ii269+M51^qz4mBy@yjMp}dEVSD zf+ZX(#k!R&ull9EPuEx?s7F%O8fR(-? zt-v|vcw{tww6UDNC7BRM=YNc*Vyz8>65YZ1!#BE6zn)+n%XhttFG6T-4|c)jP`a@7 zH2khZG|?Uvoak>TbDPq0GZr_l51{W7S+UtB3`7j=bT(LtLZ3y&Y>KLw)Bx(p6O*G>SN9i*9K`!>c5N+$|xD*T!zhSG#gbyH#^78LNQ z#w5#)1`|}eN)`e&hp^P;goOPZ5O6iD07qT82Ib22I-K@N3K{#gZ>emxo0up(5Tq}r zQ1s>TD()i-C?fzMX!6Nz$hMPdKFg>^aofI|Zl+m6e=*8sU_PR0ge= z)}mkn5de<3^HpNcq#GBuwW--4HfUYk!%r1nAil=X8e?j-j|5CkC;4l}VF!0gdPM!< zUMCx*LD<{@CYoSYQ7NQDx3w(#&Nc?K#{My0V14ik=8xpBhVq7M3+^@hl{k%j?6n7e zRm|g1k#n3u%UIKOluG&G1-*jKC1cF8R2y<(+=EU>xyV`SQo>ovTb-qqYnLB`8$#7n z-YdUdee2B**EMCSn5hd(H5|buclZE6U2WQI*TPEd_Po16uQtw1-I$Pc=IrjqQGV7I zIR(| z>bCLXo63QD3I04Wlb*@12&v|L9}Iz?mY0atVOK`W%q;BeqwBmTU*4Q!fO*A)p`L9j zmV{F$kIOoB!@);q=|)kHgC>VJ+sHJ8XnnYwYDsWyr>(!4uG01gwr;H#m_JZqgT=Xo4r z_IWXNTjh!Cy*FQyTovtTbo0a>HG=1!vr8 zA3W0)WDa(;+w9b|y0!W1w&IBYxQp|hTX8G-H$4+30-Vhhg@r{*OllI1rCTju#;{(! zwCjS0!geCiwo?sK=#>6^kjoeUp52 zkc->XHg7_@y>gk3JyN-BlO8||HdVQ~<>^Wywt2FK=w!HQo=4D$K5+ZpP1&7;+8)N= zHi}^J#u=2)nmu}@Yf|eV&-P~hS~a(V4(#h11PBP>QHgxNTVV8J&&`F1SN+{aa!I0 zF8uYW-Xh&%zye@vNDVFaQ|$&A_#DON8PUouA1|dX-GnNuZl1q>E1)a(lYLmm|er? z!50;EVQBv5VxHPEHF#g{< z*}}cSJLBQ05ghg!zk6r6Tz7{8IWRuVAf*1vOjTTRL6+w#-) zFJ>Xcq_KvHpS2sxt~LwtR^P^Sx77uV4SoqVhyTK~2iF;wn!G^M9cA9aM`z7;-MiP4 zM*YfaGMDI%Os=w}7A1qMf_N&~X_;bR8EEf-35ZzT7kXcqLd^{ZBX_!c^;%kmCIl=K ztMVXN?d_p!G1s`o4FFLyp|Y*czOEV=r!Jlb;KBDo2ZM8IaupNltz{*PNiMY7V}S6- zc$ybd54wC(f2JhR)Q1}%gtPyiQf4<)`BIr-Sr3o(Eaez87x9}2Y#@R4h4e}Cw90Vh zHXA724rFVlt7p5)*cAlrRd{7`VR39X!e9e1g(WB5-?6lO5vT25f{8q9_xeLMwJ>i* z-rqe4XP4^EnOQ*A(?7z8>PkOz5|Z44&>2$8<^z7fE}!V^pvUwDKg3XJYn?}DX)PB; zhi;bPK^C?7!Nb>ZBk{nG?sL3vK)Rk8ZN@E6Si0w5Yf8s{T<^tGLsl+&7U|$8>Fz!K zb_mWF749CT#xG)5t|o?H+;1G42IfmD+V~z)@4rtlq^JZN2d?1S=!%^Gd5SxYF2$NL;$G@2aQ*7#=w5PHP+j3P3SYPbkU8yx-&0)VYMUn?B z?7d`K@fZI?E>0wwK~e~Qtq!8`@VnkW8!1q11(V-uuvv{A$gT_d#_#srR@;I>XP!kA z+Iip4!^dg5IfR2tPrX$gza{AaWmOL&{IDvFQ$4n-4BHEtBkzEZ%J*5Up*0t7mR+HNXJ^*` zZj$yRDZ&}d+dgYGKL;)9zCpijYDZ4v?q%{8334S3cbLFRu!}JJzKhBral})fJrF7h z%;r_&Vb{BF+JU&zoPn)B^?OBB@oy)-M?wb9sm$Vml=51=bTM0%>(}9; z)4$rP6u_p;75WQyV?1`rrzfq>eGFrRN}Ul-2Bux4?_rwIkFb9BTVvG=i<)kqRa7z* zq4O-yg5zMDP`YW&IUoFP+HI;`VE$-&o64xt0t_tfEHX*ES($URT3)7zaQnq4=&SSn}7XhCw^K0xBbZ1ZoC8zdUd!nBE>CC>v5oQaU!_I?D0y&Z=WTa zxjNu5Y-^E(q*&IlR+o8@_=JO;1`ocd0e55doySNisUNVRuZp`>2P@#c*Glq@y!?}) z-dQDc78$*PcV_*{e1OG18;A1hx0mqA=vg-dTP!cV-`VyE`y9=P?tolR)$C442Wq7M zSrhaLL~DiK&&6B4rRkfk5y#+*=zOnY(L9^LaMYerc7F76!=tmAPu8s8BY_3G;f>(4 zj$$K%aQ8F6pX|%9V>0AV%FDyD1!EF;l!JGU$mPQf#f*LX$NCrr@W-#gHhRD7{v$=M ze0c%>1cyp9ph<`kHT*h6FqVV$uOE0_HO05BmG|CZ(_!XvE5aXL2b zhq7eY0|QI?(SWWjI|%mwu8B^pOOOmu>tsnuTJZb6ElP2;C@wyS3|*SZ2)0rFulkWI z+CABU@M($&es6t8o(i)4@S1x?Dj{vd{dsPu@Rt96d;dpMDlMCvt!AVd zfbceOzH%|Oe3u8Ws4yx218=DoEcxg`f_-U0Y=@_CgluDDPJ7KjB8YdFTrUhMEqi0z zji3Bq8q_E)Yd+K+5_dp0?fZ{Skjt9dR$KM&hI#hQ9<==K-6ItYj>r0_mVQWz{3t(> zN0@FE_r_Z3tyReH+1q0OdqA(V5;5vBx-r)RmWJ`3(9lreZZR^&Uy4;PkR#(uYvuaz z?l-~3tC!B9uDStEVvi+4>9(bu?`Hu6-6k-W-kX$8To1SIpZDf}JUJZ&YkCJ_+d@bN z0FHERf3+OxZ1}|=)oTmaq3aXerpnRy$wi(^K2(8z!3wL%3#*oH>lIecP<$vT z?3$S}+@0~iTG-lKf-mo<`+itVUf%pUFL~|Hh%Wh8?C_pZWKG@=nUq|;0Q|N*b>=-6 zv}x!J^JS7Q-7Lf;)X;;U{I{6-AC0Q074s+9O-QEXbSaoNz;7s;23A$W`L7jmBFu3E z_(|1JoquVo{>MG^Y~Rx#__O~k`g4W8e97m3DgKXx)jz-b@bWeHKTCdJeK{wv_}`@` zoUZ@Nb@!iN-C_8bC&NEWBd`7Moxb-kKK0MaPnrIIm;djG{J%5G?X;8LEq5+8Yrip} zpBEz)Vr=f`8{n+u;N|J*2=Y5|;&$?zC%tZ`q^?X;*qloqbAUaajGTTwJ}n#7shN?v zUHbKd^7nHMm54Rw)bIj3jDl(Lxd%6rUZA6V4U;}UYxA?7P*h5qM7UE1rM*}?wqtxcvWphelnrM1+X2g++J&L)yS-r5(3^Qz6k>kHm=nWE3M zMwZ6A&t`GNY!yUD)&(x~@OAm?+Z1P%r2q)GFQ!Q{gt;2_J(o+-w?id9)DhOY%95F< at$t_WN4Xd_2=)8b0B9OKtbFkF)&BvLKWAA0 literal 0 HcmV?d00001 diff --git a/docs/images/optimizer-loop.svg b/docs/images/optimizer-loop.svg new file mode 100644 index 0000000..7fc32ca --- /dev/null +++ b/docs/images/optimizer-loop.svg @@ -0,0 +1,99 @@ + + + + + + + + + + + + + + + skill-optimizer: Optimizer Loop + + + + Your Repo + skill-optimizer.json + SKILL.md + + + + + + + 1. Initialize + Copy SKILL.md + → skill-v0.md + + + + + + + 2. Baseline Run + Benchmark all + models × tasks + + + + + + + Iteration Loop (repeats until stable or maxIterations reached) + + + + 3. Analyze + Bucket failures + by type + (missing · wrong args) + + + + + + + 4. Mutate + LLM agent edits + skill-vN.md + (surgical patches) + + + + + + + 5. Re-Benchmark + All models × tasks + with mutated skill + (static eval, no exec) + + + + + + + 6. Accept? weighted avg improved ≥ minImprovement AND no model drops below floor + ✓ Accept → save skill-v{N+1}.md, continue ✗ Reject → restore checkpoint, continue + + + + + + + + + + Output + skill-vN.md + ready to review + + + + Progress Table — Baseline → v1 → v2 → … → vN · per-model Δ shown · your original SKILL.md is never modified + Exit codes: 0 = PASS (gates met), 1 = FAIL · Use in CI with: skill-optimizer run --config ./skill-optimizer.json + diff --git a/docs/reference/errors.md b/docs/reference/errors.md index 555b16b..4388373 100644 --- a/docs/reference/errors.md +++ b/docs/reference/errors.md @@ -12,7 +12,7 @@ The catch-all `E_UNEXPECTED` appears if an error slips past the known list. |---|---|---| | `E_INVALID_SURFACE` | Invalid surface value | Set target.surface to one of: sdk, cli, mcp, prompt | | `E_MODELS_EMPTY` | benchmark.models is empty or missing | Add at least one model to benchmark.models, e.g.: | -| `E_MODEL_ID_FORMAT` | Model ID is missing the openrouter/ prefix | Prefix all model IDs with openrouter/, e.g. openrouter/anthropic/claude-sonnet-4.6 | +| `E_MODEL_ID_FORMAT` | Model ID is missing a provider prefix | Prefix all model IDs with a supported provider prefix: | | `E_VERDICT_OUT_OF_RANGE` | Verdict threshold is out of range | Set benchmark.verdict.perModelFloor and targetWeightedAverage to values between 0.0 and 1.0 | | `E_MAX_ITERATIONS_ZERO` | optimize.maxIterations must be a positive integer | Set optimize.maxIterations to a positive integer, e.g. 5 | | `E_INVALID_FORMAT` | Invalid benchmark.format value | Set benchmark.format to one of: pi, openai, anthropic | @@ -53,11 +53,14 @@ The catch-all `E_UNEXPECTED` appears if an error slips past the known list. ### `E_MODEL_ID_FORMAT` -**Model ID is missing the openrouter/ prefix** +**Model ID is missing a provider prefix** **How to fix:** -- Prefix all model IDs with openrouter/, e.g. openrouter/anthropic/claude-sonnet-4.6 -- Browse available models at https://openrouter.ai/models +- Prefix all model IDs with a supported provider prefix: +- openrouter// — routed via OpenRouter (e.g. openrouter/anthropic/claude-sonnet-4.6) +- anthropic/ — direct Anthropic API (e.g. anthropic/claude-sonnet-4-6) +- openai/ — direct OpenAI API (e.g. openai/gpt-4.1) +- Browse OpenRouter models at https://openrouter.ai/models ### `E_VERDICT_OUT_OF_RANGE` diff --git a/src/errors.ts b/src/errors.ts index 7a0c0c3..0fa8e8e 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -41,10 +41,13 @@ export const ERRORS = { }, E_MODEL_ID_FORMAT: { code: 'E_MODEL_ID_FORMAT', - message: 'Model ID is missing the openrouter/ prefix', + message: 'Model ID is missing a provider prefix', fix: [ - 'Prefix all model IDs with openrouter/, e.g. openrouter/anthropic/claude-sonnet-4.6', - 'Browse available models at https://openrouter.ai/models', + 'Prefix all model IDs with a supported provider prefix:', + ' openrouter// — routed via OpenRouter (e.g. openrouter/anthropic/claude-sonnet-4.6)', + ' anthropic/ — direct Anthropic API (e.g. anthropic/claude-sonnet-4-6)', + ' openai/ — direct OpenAI API (e.g. openai/gpt-4.1)', + 'Browse OpenRouter models at https://openrouter.ai/models', ], }, E_VERDICT_OUT_OF_RANGE: { diff --git a/src/tasks/generate.ts b/src/tasks/generate.ts index 52cda9d..6c7b420 100644 --- a/src/tasks/generate.ts +++ b/src/tasks/generate.ts @@ -1,13 +1,17 @@ +import { createHash } from 'node:crypto'; + import type { ExpectedAction, CoverageReport } from '../benchmark/types.js'; import type { ActionDefinition } from '../actions/types.js'; import { computeUncovered, buildRetryPrompt, computeCoverage } from './coverage.js'; import type { DiscoveredTaskSurface, GeneratedTask, TaskGeneratorConfig, TaskGeneratorDeps } from './types.js'; -const SAFE_TASK_ID = /^[A-Za-z0-9._-]+$/; - -function isSafeTaskId(taskId: string): boolean { - return SAFE_TASK_ID.test(taskId) && taskId !== '.' && taskId !== '..'; +// Derive a stable task ID from the expected action names. +// Action names are surface-stable (they come from discovered code, not LLM free-form output), +// so the same surface produces the same IDs across regenerations. +function stableTaskId(actionNames: string[]): string { + const key = [...actionNames].sort().join('\x00'); + return createHash('sha1').update(key).digest('hex').slice(0, 12); } export async function generateCandidateTasks( @@ -115,7 +119,21 @@ function parseGeneratedTasks(raw: string): GeneratedTask[] { throw new Error('Task generator response must contain a top-level "tasks" array'); } - return tasks.map((task, index) => validateTask(task, index)); + const validated = tasks.map((task, index) => validateTask(task, index)); + + // Sort by (id, prompt) before deduplication so the numeric suffix assigned to + // colliding IDs is determined by content order, not by the LLM's output order. + // Without this sort, swapping two same-action tasks between runs would swap their + // suffixes (e.g. id-1 and id-2), making --task filters unstable for multi-variant cases. + validated.sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : a.prompt < b.prompt ? -1 : 1); + + // Deduplicate IDs: two tasks with the same action-name set get a numeric suffix. + const seen = new Map(); + return validated.map(task => { + const n = seen.get(task.id) ?? 0; + seen.set(task.id, n + 1); + return n > 0 ? { ...task, id: `${task.id}-${n}` } : task; + }); } function resolveStringField(obj: Record, ...keys: string[]): string | null { @@ -159,31 +177,11 @@ function validateTask(task: unknown, index: number): GeneratedTask { resolveStringField(candidate, 'prompt', 'user_prompt', 'description', 'instruction', 'task', 'action', 'method', 'name', 'command') ?? pickLongestStringValue(candidate); - // Resolve ID — fall back to deriving a slug from the prompt or action if omitted - let taskId = resolveStringField(candidate, 'id', 'task_id', 'taskId', 'name'); - if (!taskId) { - const basis = taskPrompt ?? `task-${index}`; - taskId = basis - .toLowerCase() - .replace(/[^a-z0-9]+/g, '-') - .replace(/^-|-$/g, '') - .slice(0, 48) - + `-${index}`; - } - - if (!taskPrompt) { - // Only reachable if the object has no string values at all. - const received = JSON.stringify(Object.keys(candidate)); - throw new Error(`Task ${taskId} must include a non-empty string prompt (received keys: ${received})`); - } - if (!isSafeTaskId(taskId)) { - // Sanitize rather than throw — strip unsafe chars, then handle dot-only segments that - // survive character replacement unchanged (e.g. ".." → all chars allowed → still ".."). - taskId = taskId.replace(/[^A-Za-z0-9._-]/g, '-').replace(/^-|-$/g, ''); - if (taskId === '.' || taskId === '..') taskId = ''; - taskId = taskId || `task-${index}`; - } - + // Resolve expected_actions before computing the ID so action names can anchor the ID. + // LLM-supplied IDs are intentionally ignored — they vary across runs for the same task, + // breaking --task filters after regeneration. For SDK/CLI/MCP surfaces, action names come + // from the surface definition and are stable across runs. Prompt-surface tasks have no + // actions (expected_actions is always []), so they fall back to hashing the prompt text. let rawExpectedActions = ( ['expected_actions', 'actions', 'steps', 'calls', 'expected_calls', 'tool_calls', 'cli_command'] as const ) @@ -200,6 +198,22 @@ function validateTask(task: unknown, index: number): GeneratedTask { } } + const actionNamesForId = (rawExpectedActions ?? []) + .filter((a): a is Record => !!a && typeof a === 'object') + .map(a => (typeof a['name'] === 'string' ? a['name'].trim() : '')) + .filter(Boolean); + + const taskId = + actionNamesForId.length > 0 ? stableTaskId(actionNamesForId) + : taskPrompt ? stableTaskId([taskPrompt]) + : `task-${index}`; + + if (!taskPrompt) { + // Only reachable if the object has no string values at all. + const received = JSON.stringify(Object.keys(candidate)); + throw new Error(`Task ${taskId} must include a non-empty string prompt (received keys: ${received})`); + } + if (!rawExpectedActions) { const received = JSON.stringify(Object.keys(candidate)); throw new Error(`Task ${taskId} must include an expected_actions array (received keys: ${received})`); diff --git a/tests/smoke-generation.ts b/tests/smoke-generation.ts index 164a4d3..28502e2 100644 --- a/tests/smoke-generation.ts +++ b/tests/smoke-generation.ts @@ -142,7 +142,9 @@ await test('generateCandidateTasks: parses strict JSON response', async () => { const generated = await generateCandidateTasks(surface, { maxTasks: 5, seed: 7 }, deps); assertEqual(generated.length, 1, 'should parse one task'); - assertEqual(generated[0].id, 'task-create', 'task id should match'); + // ID is derived from action names (content-stable hash), not the LLM-supplied id field + assert(/^[0-9a-f]{12}$/.test(generated[0].id), 'task id should be a 12-char hex hash of action names'); + assertEqual(generated[0].prompt, 'Create a wallet named alpha.', 'task prompt should match'); } finally { rmSync(fixture.root, { recursive: true, force: true }); } @@ -374,8 +376,9 @@ await test('generateTasksForProject: runs discover -> generate -> ground -> free }); assertEqual(result.kept.length, 2, 'two tasks should remain after grounding'); - assert(result.kept.some((t) => t.id === 'kept-task'), 'kept-task should be in kept'); - assert(result.kept.some((t) => t.id === 'balance-task'), 'balance-task should be in kept'); + // IDs are now content-based hashes; verify by the action the task covers + assert(result.kept.some((t) => t.expected_actions.some(a => a.name === 'create_wallet')), 'task covering create_wallet should be kept'); + assert(result.kept.some((t) => t.expected_actions.some(a => a.name === 'get_balance')), 'task covering get_balance should be kept'); assert(result.rejected.length >= 1, 'at least one rejected task expected'); assert(existsSync(result.artifacts.benchmarkPath), 'generated benchmark config should exist'); } finally { From e090ae070e6840264918743b45f4278dc83d1f0d Mon Sep 17 00:00:00 2001 From: Xiaohong Chen Date: Fri, 17 Apr 2026 09:56:09 +0800 Subject: [PATCH 3/8] Update README with Fast team and payment info (#27) * Update README with Fast team and payment info Added information about the Fast team and payment infrastructure for AI agents. Requested by Jessy. * Update README.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: dmn Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 8e22aff..4b288ce 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ Benchmark and self-optimize SDK, CLI, and MCP guidance so every agent model can skill-optimizer runs your SDK / CLI / MCP docs against multiple LLMs, measures whether they call the right actions with the right arguments, and iteratively rewrites your `SKILL.md` / docs until a floor score is met across every model. +Built by the team at [Fast](https://fast.xyz/) — payment infrastructure for AI agents. [Give your agent a wallet](https://github.com/fastxyz/fast-sdk) in 3 lines of code. + **Requirements:** Node.js 20+, plus either an [OpenRouter](https://openrouter.ai) API key or a local Codex login when using direct OpenAI models. ## How it works — at a glance From 39869c480465563d1d4d7a3c9bf36c30b194df82 Mon Sep 17 00:00:00 2001 From: dmn Date: Thu, 16 Apr 2026 22:23:22 -0700 Subject: [PATCH 4/8] fix: prompt-surface correctness (P1/P2/P3/B/C5) (#28) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: exempt openai/ model IDs from dot→hyphen rewrite OpenAI's direct-API model IDs use dots in version numbers (gpt-5.4, gpt-4.1). fix.ts already exempted openrouter/ but not openai/, so a manufactured model-id-bad-format issue would corrupt an openai/ ID. Defense-in-depth: validate.ts already skips emitting the issue for openai/, but fix.ts must independently respect the documented invariant in CLAUDE.md. Also wires smoke-model-ids.ts into npm test — it existed on disk but was not in the test script. * chore: remove unused src/discovery/prompt.ts and its tests The active prompt discovery lives in src/project/discover-prompt.ts, imported by snapshot.ts and benchmark/runner.ts. src/discovery/prompt.ts was a dead parallel implementation — only referenced by its own test file. Removing both the dead module and its test file to keep the codebase lean. The live discover-prompt.ts API (discoverPromptCapabilities) is covered by other smoke tests in this PR. * fix: prompt surface not blocked by coverage violation Prompt-surface tasks don't guarantee 1:1 capability coverage the way SDK/CLI/MCP tasks do, so coverageViolation=true was hard-FAILing every prompt benchmark regardless of actual scores. computeVerdict now only appends the coverage-violation reason when config.surface !== 'prompt'. Coverage is still computed and appears in the report. Regression guard in new smoke-verdict-prompt.ts locks in both halves of the behavior: prompt PASSes with coverageViolation=true + scores above floor; mcp still FAILs under identical conditions. * refactor: extract resolveCriteriaForTask from runner Pure refactor. Moves the caps→criteria lookup out of runner.ts into src/benchmark/prompt-criteria.ts so it can be unit-tested without running the full LLM pipeline. Behavior is unchanged in this commit — tasks missing capabilityId are logged as eval errors (FAIL with message) rather than silently vacuously passing; capabilityId tagging by the generator lands in a later commit. Adds optional capabilityId to GeneratedTask (SDK/CLI/MCP generators don't set it). Runtime enforcement (throw on missing/unknown) lives in resolveCriteriaForTask — no silent fallback, per the no-legacy-compat policy. New smoke-prompt-criteria.ts locks in: match, distinct-per-capability, throws-on-unknown, throws-on-missing, and noActiveCriteria flagging. * fix: evaluator flags noActiveCriteria instead of vacuous 1.0 pass Previously, when every criteria category was empty the evaluator returned score: 1.0 — any response (including an empty string) scored a perfect pass. Now the evaluator returns score: 0 and noActiveCriteria: true. The runner treats that flag as an evaluation error with an actionable message pointing at the SKILL.md section for the offending capability. Evaluator stays dumb (no pass/fail policy). Runner is the policy layer. * feat: per-capability prompt scoring via capabilityId The runner's caps[0] global was collapsing every prompt-surface task into evaluation against the first discovered capability regardless of what the task actually exercised. With this commit, each generated prompt-surface task is tagged at generation time with the action key of the capability it exercises, and the runner looks up criteria per-task via resolveCriteriaForTask (wired in previous commits). No legacy compat. Prompt-surface tasks lacking capabilityId fail to load; users regenerate with `skill-optimizer generate-tasks`. Regression guards: - smoke-generation.ts: valid tagging plus rejection of unknown ids - smoke-verdict-prompt.ts: three caps produce distinct criteria (caps[0]-collapse detector), P3 regression guard via evaluator, and mock-LLM verdict matrix (threshold + weight math) Co-Authored-By: Claude Sonnet 4.6 * docs: v1.1.0 correctness fixes CHANGELOG gets a Fixed block covering P1/P2/P3/Bug B/C5 for v1.1.0. README prompt templates section reflects per-capability scoring. SKILL.md audit for any guidance contradicting the fixed behavior. * test: release-readiness coverage check smoke-changelog-coverage.ts parses the top block of CHANGELOG.md and asserts every item in Added/Fixed has at least one test file referencing relevant keywords. Guards against 'shipped feature, forgot the test' — the class that let P1/P2/P3 slip past v1.1.0 before this PR. smoke-release.ts also gains an assertion that the CHANGELOG contains a section header matching the current package.json version. * fix: plumb capabilityId through TaskDefinition and tighten changelog coverage check - Add capabilityId?: string to TaskDefinition and update normalizeTaskDefinition to read and pass it through — without this, resolveCriteriaForTask threw on every prompt-surface task because loadTasks silently dropped the field - smoke-changelog-coverage: require ≥2 tokens to co-occur in a single test file (whole-word match) instead of any one token anywhere in the corpus — prevents false-passes on generic words like "prompt" or "coverage" - generate.ts comment: clarify that parseGeneratedTasks attaches capabilityId; membership validation is in ground.ts, not here Co-Authored-By: Claude Sonnet 4.6 * fix: output capability criteria and prompt coverage computation - discover-prompt.ts: _output capabilities now store section: section.body (full markdown with fences) instead of section: snippet (extracted content without fences). generateCriteriaFromCapability requires fences to extract format patterns, so passing bare snippet produced empty criteria and forced noActiveCriteria/FAIL for every output-format task. - coverage.ts: actionNamesOf falls back to capabilityId when expected_actions is empty — prompt tasks always have expected_actions:[] so coverage showed 0/N covered. capabilityId matches action.name for prompt capabilities (key===name in capabilityToAction), so this correctly attributes coverage. - Tests: regression guards added in smoke-prompt-criteria and smoke-coverage. Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: OpenClaw Agent (basd) Co-authored-by: Claude Sonnet 4.6 --- CHANGELOG.md | 5 + README.md | 12 +- package.json | 2 +- src/benchmark/config.ts | 7 +- src/benchmark/prompt-criteria.ts | 41 ++++ src/benchmark/prompt-evaluator.ts | 13 +- src/benchmark/runner.ts | 47 +++-- src/benchmark/scoring.ts | 5 +- src/benchmark/types.ts | 1 + src/discovery/prompt.ts | 180 ----------------- src/project/discover-prompt.ts | 2 +- src/project/fix.ts | 5 +- src/tasks/coverage.ts | 6 +- src/tasks/generate.ts | 29 ++- src/tasks/ground.ts | 8 + src/tasks/types.ts | 1 + tests/smoke-changelog-coverage.ts | 87 ++++++++ tests/smoke-coverage.ts | 17 ++ tests/smoke-discovery-prompt.ts | 321 ------------------------------ tests/smoke-generation.ts | 162 +++++++++++++++ tests/smoke-model-ids.ts | 23 +++ tests/smoke-prompt-criteria.ts | 104 ++++++++++ tests/smoke-prompt-evaluator.ts | 14 +- tests/smoke-release.ts | 9 + tests/smoke-verdict-prompt.ts | 213 ++++++++++++++++++++ 25 files changed, 771 insertions(+), 543 deletions(-) create mode 100644 src/benchmark/prompt-criteria.ts delete mode 100644 src/discovery/prompt.ts create mode 100644 tests/smoke-changelog-coverage.ts delete mode 100644 tests/smoke-discovery-prompt.ts create mode 100644 tests/smoke-prompt-criteria.ts create mode 100644 tests/smoke-verdict-prompt.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 89a5b19..23e237d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,11 @@ The config file `skill-benchmark.json` is no longer auto-detected. Rename it to - **benchmark:** Strip provider prefix from model ID when using direct `anthropic` or `openai` formats. Previously, `anthropic/claude-sonnet-4-6` was sent as-is to the Anthropic API, which expects `claude-sonnet-4-6`. The `pi` format is unaffected. - **model IDs:** OpenRouter model slugs now preserve dots in version numbers (e.g. `openrouter/anthropic/claude-sonnet-4.6`). Presets updated to match OpenRouter's catalog exactly. The dot→hyphen rewrite in `validate`/`fix` now applies only to the `anthropic/` direct-API prefix; `openrouter/` and `openai/` slugs are exempt. - **error message:** `E_MODEL_ID_FORMAT` now lists all three valid provider prefixes (`openrouter/`, `anthropic/`, `openai/`) instead of directing all users to use `openrouter/`. +- Prompt-surface benchmarks no longer hard-FAIL on `scopeCoverage.coverageViolation`; coverage is informational for prompt runs (`src/benchmark/scoring.ts`). +- Prompt-surface tasks are now scored against the specific capability they exercise via a required `capabilityId` on `GeneratedTask`. Previously every task was scored against the first discovered capability (`src/benchmark/runner.ts`, `src/benchmark/prompt-criteria.ts`, `src/tasks/generate.ts`). +- Prompt evaluator surfaces `noActiveCriteria: true` (score 0, runner-level FAIL with an actionable message) when a capability's section produces empty criteria, replacing the previous vacuous 1.0 pass (`src/benchmark/prompt-evaluator.ts`). +- `openai/` direct-API model IDs are exempt from dot→hyphen rewriting in `applyFixes`. OpenAI's API slugs use dots (`gpt-5.4`, `gpt-4.1`). (`src/project/fix.ts`) +- Removed dead `src/discovery/prompt.ts`. Active discovery path is `src/project/discover-prompt.ts`. ## 1.0.0 — 2026-04-14 diff --git a/README.md b/README.md index 4b288ce..28b28bb 100644 --- a/README.md +++ b/README.md @@ -134,10 +134,14 @@ skill-optimizer run The prompt surface discovers phases and capabilities from your SKILL.md, generates scenario-based tasks, and evaluates output quality — not just -tool calls. It scores responses on required sections, format patterns, -forbidden keywords, and structural elements (code blocks, numbered lists, -tables). This lets you optimize prompt templates the same way you optimize -SDK/CLI/MCP guidance. +tool calls. Each task is tagged with the specific capability it exercises +(`capabilityId`), and scoring is performed against that capability's +criteria — not the first discovered capability. It scores responses on +required sections, format patterns, forbidden keywords, and structural +elements (code blocks, numbered lists, tables). Coverage violations do +not hard-fail prompt runs; coverage is informational for the prompt +surface. This lets you optimize prompt templates the same way you +optimize SDK/CLI/MCP guidance. ## How it works diff --git a/package.json b/package.json index 0a96499..8422dea 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "build": "tsc && npm run gen-docs && chmod +x dist/cli.js", "typecheck": "tsc --noEmit", "lint": "tsc --noUnusedLocals --noEmit", - "test": "tsx tests/smoke-code.ts && tsx tests/smoke-sdk-python.ts && tsx tests/smoke-sdk-rust.ts && tsx tests/smoke-cli.ts && tsx tests/smoke-cli-entry.ts && tsx tests/smoke-mcp.ts && tsx tests/smoke-llm.ts && tsx tests/smoke-discovery-sdk.ts && tsx tests/smoke-discovery-cli.ts && tsx tests/smoke-discovery-mcp.ts && tsx tests/smoke-discovery-prompt.ts && tsx tests/smoke-prompt-evaluator.ts && tsx tests/smoke-snapshot-prompt.ts && tsx tests/smoke-generation.ts && tsx tests/smoke-optimize.ts && tsx tests/smoke-mock-repos.ts && tsx tests/smoke-release.ts && tsx tests/smoke-scoring.ts && tsx tests/smoke-scope.ts && tsx tests/smoke-coverage.ts && tsx tests/smoke-feedback.ts && tsx tests/smoke-verdict.ts && tsx tests/smoke-dry-run.ts && tsx tests/smoke-errors.ts && tsx tests/smoke-e2e.ts && tsx tests/smoke-import.ts && tsx tests/smoke-init.ts && tsx tests/smoke-gen-docs.ts && tsx tests/smoke-actions.ts" + "test": "tsx tests/smoke-code.ts && tsx tests/smoke-sdk-python.ts && tsx tests/smoke-sdk-rust.ts && tsx tests/smoke-cli.ts && tsx tests/smoke-cli-entry.ts && tsx tests/smoke-mcp.ts && tsx tests/smoke-llm.ts && tsx tests/smoke-discovery-sdk.ts && tsx tests/smoke-discovery-cli.ts && tsx tests/smoke-discovery-mcp.ts && tsx tests/smoke-prompt-evaluator.ts && tsx tests/smoke-prompt-criteria.ts && tsx tests/smoke-snapshot-prompt.ts && tsx tests/smoke-generation.ts && tsx tests/smoke-optimize.ts && tsx tests/smoke-mock-repos.ts && tsx tests/smoke-release.ts && tsx tests/smoke-changelog-coverage.ts && tsx tests/smoke-scoring.ts && tsx tests/smoke-scope.ts && tsx tests/smoke-coverage.ts && tsx tests/smoke-feedback.ts && tsx tests/smoke-verdict.ts && tsx tests/smoke-verdict-prompt.ts && tsx tests/smoke-dry-run.ts && tsx tests/smoke-errors.ts && tsx tests/smoke-model-ids.ts && tsx tests/smoke-e2e.ts && tsx tests/smoke-import.ts && tsx tests/smoke-init.ts && tsx tests/smoke-gen-docs.ts && tsx tests/smoke-actions.ts" }, "dependencies": { "@clack/prompts": "^1.2.0", diff --git a/src/benchmark/config.ts b/src/benchmark/config.ts index b8ac3fa..b5f9923 100644 --- a/src/benchmark/config.ts +++ b/src/benchmark/config.ts @@ -50,7 +50,7 @@ export function loadTasks(tasksPath: string, baseDir?: string): TaskDefinition[] throw new Error(`Failed to read tasks: ${resolved}: ${err instanceof Error ? err.message : err}`); } - let parsed: { tasks: Array<{ id?: unknown; prompt?: unknown; expected_actions?: unknown; verify?: unknown; expected_fetches?: unknown }> }; + let parsed: { tasks: Array<{ id?: unknown; prompt?: unknown; expected_actions?: unknown; verify?: unknown; expected_fetches?: unknown; capabilityId?: unknown }> }; try { parsed = JSON.parse(raw) as typeof parsed; } catch (err) { @@ -65,7 +65,7 @@ export function loadTasks(tasksPath: string, baseDir?: string): TaskDefinition[] } function normalizeTaskDefinition( - task: { id?: unknown; prompt?: unknown; expected_actions?: unknown; verify?: unknown; expected_fetches?: unknown }, + task: { id?: unknown; prompt?: unknown; expected_actions?: unknown; verify?: unknown; expected_fetches?: unknown; capabilityId?: unknown }, resolvedPath: string, index: number, ): TaskDefinition { @@ -105,12 +105,15 @@ function normalizeTaskDefinition( } } + const capabilityId = typeof task.capabilityId === 'string' ? task.capabilityId : undefined; + return { id: task.id, prompt: task.prompt, expected_actions, verify: rawVerify as TaskDefinition['verify'] | undefined, expected_fetches: rawFetches as string[] | undefined, + ...(capabilityId !== undefined ? { capabilityId } : {}), }; } diff --git a/src/benchmark/prompt-criteria.ts b/src/benchmark/prompt-criteria.ts new file mode 100644 index 0000000..7c57360 --- /dev/null +++ b/src/benchmark/prompt-criteria.ts @@ -0,0 +1,41 @@ +import type { GeneratedTask } from '../tasks/types.js'; +import type { PromptCapabilityWithSection } from '../project/discover-prompt.js'; +import type { PromptEvaluationCriteria } from './prompt-evaluator.js'; +import { generateCriteriaFromCapability } from './prompt-evaluator.js'; + +export interface ResolvedPromptCriteria { + criteria: PromptEvaluationCriteria; + noActiveCriteria: boolean; +} + +function isEmptyCriteria(c: PromptEvaluationCriteria): boolean { + const s = (c.requiredSections?.length ?? 0) === 0; + const k = (c.requiredKeywords?.length ?? 0) === 0 && (c.forbiddenKeywords?.length ?? 0) === 0; + const f = (c.formatPatterns?.length ?? 0) === 0 && (c.minLength ?? 0) === 0; + const structure = + c.hasCodeBlocks === undefined && + c.hasNumberedList === undefined && + c.hasTable === undefined; + return s && k && f && structure; +} + +export function resolveCriteriaForTask( + task: GeneratedTask, + caps: readonly PromptCapabilityWithSection[], +): ResolvedPromptCriteria { + if (!task.capabilityId) { + throw new Error( + `Task ${task.id}: prompt-surface task is missing capabilityId. ` + + `Regenerate tasks with \`skill-optimizer generate-tasks\`.`, + ); + } + const cap = caps.find((c) => c.action.key === task.capabilityId); + if (!cap) { + const known = caps.map((c) => c.action.key).join(', ') || '(none discovered)'; + throw new Error( + `Task ${task.id}: capabilityId "${task.capabilityId}" is not in the discovered capability set. Known: ${known}`, + ); + } + const criteria = generateCriteriaFromCapability(cap.action, cap.section); + return { criteria, noActiveCriteria: isEmptyCriteria(criteria) }; +} diff --git a/src/benchmark/prompt-evaluator.ts b/src/benchmark/prompt-evaluator.ts index 0067f44..8946629 100644 --- a/src/benchmark/prompt-evaluator.ts +++ b/src/benchmark/prompt-evaluator.ts @@ -44,6 +44,12 @@ export interface PromptEvaluationResult { keywords: number; structure: number; }; + /** + * True when every criteria category is empty. Evaluation cannot produce a + * meaningful score in this case; runner treats this as an evaluation error + * rather than a pass. + */ + noActiveCriteria: boolean; } // ── Weights ─────────────────────────────────────────────────────────────────── @@ -237,9 +243,11 @@ export function evaluatePromptResponse( if (structureTotal > 0) activeParts.push({ weight: WEIGHT_STRUCTURE, score: structureScore }); let score: number; + let noActiveCriteria = false; if (activeParts.length === 0) { - // No criteria specified at all — vacuously pass - score = 1.0; + // No criteria are active — treat as evaluation error, not vacuous pass. + score = 0; + noActiveCriteria = true; } else { const totalActiveWeight = activeParts.reduce((s, p) => s + p.weight, 0); score = activeParts.reduce((s, p) => s + (p.weight / totalActiveWeight) * p.score, 0); @@ -255,6 +263,7 @@ export function evaluatePromptResponse( keywords: keywordScore, structure: structureScore, }, + noActiveCriteria, }; } diff --git a/src/benchmark/runner.ts b/src/benchmark/runner.ts index 9aa8edd..3976ed5 100644 --- a/src/benchmark/runner.ts +++ b/src/benchmark/runner.ts @@ -25,8 +25,8 @@ import { evaluateTask } from './evaluator.js'; import { computeCoverage } from './coverage.js'; import { computeVerdict } from './scoring.js'; import { buildSystemPrompt, buildTaskPrompt } from './prompts.js'; -import { evaluatePromptResponse, generateCriteriaFromCapability } from './prompt-evaluator.js'; -import type { PromptEvaluationCriteria } from './prompt-evaluator.js'; +import { evaluatePromptResponse } from './prompt-evaluator.js'; +import { resolveCriteriaForTask } from './prompt-criteria.js'; import { discoverPromptCapabilitiesWithSections } from '../project/discover-prompt.js'; function buildWebFetchTool(): McpToolDefinition { @@ -178,15 +178,12 @@ export async function runBenchmark(options: RunnerOptions = {}): Promise 0) { - // Use the primary capability's section to derive criteria. - promptEvalCriteria = generateCriteriaFromCapability(caps[0].action, caps[0].section); - console.log(`[prompt] Evaluation criteria derived from ${caps.length} discovered capabilities`); - } + // Discover prompt capabilities once; criteria are resolved per-task inside the loop. + const promptCaps = (config.surface === 'prompt' && skill) + ? discoverPromptCapabilitiesWithSections(skill.content) + : []; + if (config.surface === 'prompt') { + console.log(`[prompt] ${promptCaps.length} capabilities discovered`); } @@ -365,16 +362,28 @@ export async function runBenchmark(options: RunnerOptions = {}): Promise= 0.5; - console.log(` [${slug}] Prompt score: ${promptResult.score.toFixed(3)} → ${taskResult.metrics.taskPassed ? 'PASS' : 'FAIL'}`); + const { criteria } = resolveCriteriaForTask(task, promptCaps); + const promptResult = evaluatePromptResponse(rawResponse, criteria); + if (promptResult.noActiveCriteria) { + const msg = `Task "${task.id}" has no extractable criteria — fix SKILL.md section for that action`; + taskResult.metrics.toolRecall = 0; + taskResult.metrics.taskPassed = false; + taskResult.error = taskResult.error ?? msg; + console.error(` [${slug}] Prompt eval error: ${msg}`); + } else { + taskResult.metrics.toolRecall = promptResult.score; + taskResult.metrics.taskPassed = promptResult.score >= 0.5; + console.log(` [${slug}] Prompt score: ${promptResult.score.toFixed(3)} → ${taskResult.metrics.taskPassed ? 'PASS' : 'FAIL'}`); + } } catch (err) { - console.error(` [${slug}] Prompt eval error: ${err instanceof Error ? err.message : err}`); - // Leave vacuous scores in place — task shows as PASS rather than crashing the run + const msg = err instanceof Error ? err.message : String(err); + console.error(` [${slug}] Prompt eval error: ${msg}`); + taskResult.metrics.toolRecall = 0; + taskResult.metrics.taskPassed = false; + taskResult.error = taskResult.error ?? msg; } } diff --git a/src/benchmark/scoring.ts b/src/benchmark/scoring.ts index dbf5fec..4e041fb 100644 --- a/src/benchmark/scoring.ts +++ b/src/benchmark/scoring.ts @@ -50,7 +50,10 @@ export function computeVerdict( ); } - if (report.scopeCoverage?.coverageViolation) { + // Prompt surface: coverage is reported but does not veto the verdict. + // Prompt tasks are not required to map 1:1 to capabilities, so a + // coverage gap here is informational rather than a failure condition. + if (report.scopeCoverage?.coverageViolation && report.config.surface !== 'prompt') { reasons.push('coverage violation: some in-scope actions have zero tasks'); } diff --git a/src/benchmark/types.ts b/src/benchmark/types.ts index ba63cd9..c1c77ab 100644 --- a/src/benchmark/types.ts +++ b/src/benchmark/types.ts @@ -145,6 +145,7 @@ export interface TaskDefinition { expected_actions: ExpectedAction[]; verify?: TaskVerification[]; expected_fetches?: string[]; + capabilityId?: string; } // === Extracted from generated code or tool_calls === diff --git a/src/discovery/prompt.ts b/src/discovery/prompt.ts deleted file mode 100644 index c17106f..0000000 --- a/src/discovery/prompt.ts +++ /dev/null @@ -1,180 +0,0 @@ -/** - * Prompt surface discoverer. - * - * Parses SKILL.md files to extract phases, capabilities, and structural - * information. This enables benchmarking how well models follow prompt - * templates — not just tool calls. - */ - -import { existsSync, readFileSync } from 'node:fs'; -import { resolve } from 'node:path'; - -import type { DiscoveryOptions } from './types.js'; - -// ── Types ───────────────────────────────────────────────────────────────── - -export interface PromptPhase { - name: string; - description: string; - hasCodeBlocks: boolean; - hasNumberedSteps: boolean; - hasDecisionPoints: boolean; -} - -export interface PromptCapability { - name: string; - description: string; - source: 'phase' | 'instruction'; -} - -export interface PromptDiscoverySnapshot { - surface: 'prompt'; - phases: PromptPhase[]; - capabilities: PromptCapability[]; - sources: string[]; -} - -// ── Phase parsing ───────────────────────────────────────────────────────── - -const PHASE_HEADER_RE = /^##\s+(?:Phase\s+\d+\s*[—–-]\s*)?(.+)$/gm; -const CODE_BLOCK_RE = /```[\s\S]*?```/g; -const NUMBERED_STEP_RE = /^\d+\.\s+/m; -const DECISION_POINT_RE = /\b(?:if|when|unless|otherwise|decide|choose|either)\b/i; - -function extractPhases(content: string): PromptPhase[] { - const phases: PromptPhase[] = []; - const headers: { name: string; start: number }[] = []; - - // Find all phase headers (## headings) - let match: RegExpExecArray | null; - const headerRe = new RegExp(PHASE_HEADER_RE.source, PHASE_HEADER_RE.flags); - while ((match = headerRe.exec(content)) !== null) { - headers.push({ name: match[1].trim(), start: match.index }); - } - - for (let i = 0; i < headers.length; i++) { - const start = headers[i].start; - const end = i + 1 < headers.length ? headers[i + 1].start : content.length; - const sectionContent = content.slice(start, end); - - // Extract description: first non-empty, non-heading line - const lines = sectionContent.split('\n').slice(1); // skip the heading itself - const descLine = lines.find((l) => l.trim().length > 0 && !l.startsWith('#')); - const description = descLine?.trim() ?? ''; - - phases.push({ - name: headers[i].name, - description, - hasCodeBlocks: CODE_BLOCK_RE.test(sectionContent), - hasNumberedSteps: NUMBERED_STEP_RE.test(sectionContent), - hasDecisionPoints: DECISION_POINT_RE.test(sectionContent), - }); - - // Reset lastIndex for the stateful regex - CODE_BLOCK_RE.lastIndex = 0; - } - - return phases; -} - -// ── Capability extraction ───────────────────────────────────────────────── - -function extractCapabilities(content: string, phases: PromptPhase[]): PromptCapability[] { - const capabilities: PromptCapability[] = []; - - if (phases.length > 0) { - // Derive capabilities from phases - for (const phase of phases) { - capabilities.push({ - name: phase.name, - description: phase.description, - source: 'phase', - }); - } - } else if (content.trim().length > 0) { - // No phases found — extract capabilities from top-level instructions - // Look for bullet points or imperative sentences - const lines = content.split('\n'); - for (const line of lines) { - const trimmed = line.trim(); - // Match bullet points: "- Do something" or "* Do something" - const bulletMatch = trimmed.match(/^[-*]\s+(.+)$/); - if (bulletMatch) { - capabilities.push({ - name: bulletMatch[1].slice(0, 60), - description: bulletMatch[1], - source: 'instruction', - }); - } - } - - // If still no capabilities, treat the whole content as one instruction block - if (capabilities.length === 0) { - const firstLine = lines.find((l) => l.trim().length > 0 && !l.startsWith('#') && !l.startsWith('---')); - if (firstLine) { - capabilities.push({ - name: firstLine.trim().slice(0, 60), - description: firstLine.trim(), - source: 'instruction', - }); - } - } - } - - return capabilities; -} - -// ── Frontmatter stripping ───────────────────────────────────────────────── - -function stripFrontmatter(content: string): string { - if (!content.startsWith('---')) return content; - // Match the closing --- that appears on its own line (after the opening ---) - const closingMatch = content.slice(3).match(/\n---\s*(\n|$)/); - if (!closingMatch || closingMatch.index === undefined) return content; - const endIdx = 3 + closingMatch.index + closingMatch[0].length; - return content.slice(endIdx); -} - -// ── Public API ──────────────────────────────────────────────────────────── - -export function discoverPromptSurfaceFromContent(content: string): PromptDiscoverySnapshot { - const body = stripFrontmatter(content); - const phases = extractPhases(body); - const capabilities = extractCapabilities(body, phases); - - return { - surface: 'prompt', - phases, - capabilities, - sources: [], - }; -} - -export function discoverPromptSurfaceFromSources( - sources: string[], - options: DiscoveryOptions = {}, -): PromptDiscoverySnapshot { - const baseDir = options.baseDir ?? process.cwd(); - const resolvedSources = sources.map((source) => resolve(baseDir, source)); - - const allPhases: PromptPhase[] = []; - const allCapabilities: PromptCapability[] = []; - - for (const sourcePath of resolvedSources) { - if (!existsSync(sourcePath)) { - throw new Error(`Prompt discovery source file does not exist: ${sourcePath}`); - } - - const content = readFileSync(sourcePath, 'utf-8'); - const snapshot = discoverPromptSurfaceFromContent(content); - allPhases.push(...snapshot.phases); - allCapabilities.push(...snapshot.capabilities); - } - - return { - surface: 'prompt', - phases: allPhases, - capabilities: allCapabilities, - sources: resolvedSources, - }; -} diff --git a/src/project/discover-prompt.ts b/src/project/discover-prompt.ts index ba4a384..7f242af 100644 --- a/src/project/discover-prompt.ts +++ b/src/project/discover-prompt.ts @@ -312,7 +312,7 @@ export function discoverPromptCapabilitiesWithSections( addWithSection({ name: outputName, description: `Expected output format: ${preview}`, - section: snippet, + section: section.body, type: 'output', }); } diff --git a/src/project/fix.ts b/src/project/fix.ts index f143323..28ad3a3 100644 --- a/src/project/fix.ts +++ b/src/project/fix.ts @@ -31,8 +31,9 @@ export function applyFixes( if (issue.code === 'model-id-bad-format' && !prefixFixedIndices.has(idx)) { const currentId = models[idx]!.id as string; - // Never rewrite openrouter/ slugs — they're passed verbatim to OpenRouter's API. - if (!currentId.startsWith('openrouter/')) { + // Only anthropic/ direct-API IDs get dots rewritten to hyphens. + // openrouter/ slugs are passed verbatim; openai/ direct-API IDs use dots (e.g. gpt-5.4). + if (!currentId.startsWith('openrouter/') && !currentId.startsWith('openai/')) { models[idx]!.id = currentId.replace(/(\d+)\.(\d+)/g, '$1-$2'); } } diff --git a/src/tasks/coverage.ts b/src/tasks/coverage.ts index 503b4c9..ac791cd 100644 --- a/src/tasks/coverage.ts +++ b/src/tasks/coverage.ts @@ -3,7 +3,11 @@ import type { GeneratedTask } from './types.js'; import type { CoverageReport } from '../benchmark/types.js'; function actionNamesOf(task: GeneratedTask): string[] { - return task.expected_actions.map((a) => a.name).filter(Boolean); + const fromActions = task.expected_actions.map((a) => a.name).filter(Boolean); + if (fromActions.length > 0) return fromActions; + // Prompt surface: tasks use capabilityId instead of expected_actions. + // action.key === action.name for prompt capabilities, so capabilityId matches. + return task.capabilityId ? [task.capabilityId] : []; } export function computeCoverage( diff --git a/src/tasks/generate.ts b/src/tasks/generate.ts index 6c7b420..25f2586 100644 --- a/src/tasks/generate.ts +++ b/src/tasks/generate.ts @@ -27,7 +27,14 @@ export async function generateCandidateTasks( const prompt = buildPrompt(surface, config); const completion = await deps.complete({ system, prompt }); - const tasks = parseGeneratedTasks(completion); + + // For prompt surface, pass the known capability keys so parseGeneratedTasks + // can attach capabilityId to each task; membership validation is in ground.ts. + const knownCapabilityKeys = surface.snapshot.surface === 'prompt' + ? surface.snapshot.actions.map((a) => a.name) + : undefined; + + const tasks = parseGeneratedTasks(completion, knownCapabilityKeys); return tasks.slice(0, Math.max(1, Math.floor(config.maxTasks))); } @@ -35,18 +42,22 @@ function buildPrompt(surface: DiscoveredTaskSurface, config: TaskGeneratorConfig const clampedMax = Math.max(1, Math.floor(config.maxTasks)); if (surface.snapshot.surface === 'prompt') { + const capKeys = surface.snapshot.actions.map((a) => a.name); return [ 'Generate benchmark evaluation tasks for a prompt/skill document.', 'These tasks will be evaluated by content quality, not action matching.', '', 'Return a JSON object with EXACTLY this shape:', - '{"tasks":[{"id":"string","prompt":"string","expected_actions":[]}]}', + '{"tasks":[{"id":"string","prompt":"string","expected_actions":[],"capabilityId":"string"}]}', '', 'RULES:', - '- Each task has EXACTLY three keys: id, prompt, expected_actions.', + '- Each task has EXACTLY four keys: id, prompt, expected_actions, capabilityId.', '- expected_actions MUST always be an empty array [].', '- id: short snake_case identifier (e.g. "deploy_service_to_staging").', '- prompt: ask the model to perform a realistic task from the skill.', + '- capabilityId: set to the action key of the discovered capability this task exercises.', + `- Valid capabilityId values: ${capKeys.join(', ')}.`, + '- Every task MUST have a capabilityId from the valid list above — no other values are accepted.', `- Produce at most ${clampedMax} tasks. Seed: ${config.seed}.`, '', 'Full SKILL.md:', @@ -102,7 +113,7 @@ function stripCodeFence(raw: string): string { return match ? match[1].trim() : trimmed; } -function parseGeneratedTasks(raw: string): GeneratedTask[] { +function parseGeneratedTasks(raw: string, knownCapabilityKeys?: string[]): GeneratedTask[] { let parsed: unknown; try { parsed = JSON.parse(stripCodeFence(raw)); @@ -119,7 +130,7 @@ function parseGeneratedTasks(raw: string): GeneratedTask[] { throw new Error('Task generator response must contain a top-level "tasks" array'); } - const validated = tasks.map((task, index) => validateTask(task, index)); + const validated = tasks.map((task, index) => validateTask(task, index, knownCapabilityKeys)); // Sort by (id, prompt) before deduplication so the numeric suffix assigned to // colliding IDs is determined by content order, not by the LLM's output order. @@ -164,7 +175,7 @@ function pickLongestStringValue(obj: Record): string | null { return best; } -function validateTask(task: unknown, index: number): GeneratedTask { +function validateTask(task: unknown, index: number, knownCapabilityKeys?: string[]): GeneratedTask { if (!task || typeof task !== 'object') { throw new Error(`Task at index ${index} must be an object`); } @@ -221,10 +232,16 @@ function validateTask(task: unknown, index: number): GeneratedTask { const expected_actions = rawExpectedActions.map((action, actionIndex) => validateExpectedAction(taskId, action, actionIndex)); + // Extract capabilityId for prompt-surface tasks. The field is stored as-is here; + // grounding validates it against the known capability keys and rejects bad values. + const rawCapabilityId = typeof candidate['capabilityId'] === 'string' ? candidate['capabilityId'].trim() : undefined; + const capabilityId = knownCapabilityKeys !== undefined && rawCapabilityId ? rawCapabilityId : undefined; + return { id: taskId, prompt: taskPrompt, expected_actions, + ...(capabilityId !== undefined ? { capabilityId } : {}), }; } diff --git a/src/tasks/ground.ts b/src/tasks/ground.ts index d1e3055..5b97f6d 100644 --- a/src/tasks/ground.ts +++ b/src/tasks/ground.ts @@ -36,10 +36,18 @@ function getRejectionReason( } // Prompt surface tasks must have expected_actions: [] — evaluated on content, not tool calls. + // They must also carry a valid capabilityId referencing a known discovered capability. if (surface === 'prompt') { if (expectedActions.length > 0) { return `prompt task "${task.id}" must have empty expected_actions, got ${expectedActions.length}`; } + const knownKeys = [...actions.keys()]; + if (!task.capabilityId) { + return `prompt task "${task.id}" is missing capabilityId (known: ${knownKeys.join(', ')})`; + } + if (!actions.has(task.capabilityId)) { + return `prompt task "${task.id}" has unknown capabilityId "${task.capabilityId}" (known: ${knownKeys.join(', ')})`; + } return null; } diff --git a/src/tasks/types.ts b/src/tasks/types.ts index 43351b6..cbd51bb 100644 --- a/src/tasks/types.ts +++ b/src/tasks/types.ts @@ -5,6 +5,7 @@ export interface GeneratedTask { id: string; prompt: string; expected_actions: ExpectedAction[]; + capabilityId?: string; // prompt surface only; SDK/CLI/MCP don't set this } export interface TaskGeneratorConfig { diff --git a/tests/smoke-changelog-coverage.ts b/tests/smoke-changelog-coverage.ts new file mode 100644 index 0000000..4041e73 --- /dev/null +++ b/tests/smoke-changelog-coverage.ts @@ -0,0 +1,87 @@ +import { strict as assert } from 'node:assert'; +import { readFileSync, readdirSync } from 'node:fs'; +import { resolve } from 'node:path'; + +/** + * Parses the top version block of CHANGELOG.md and checks that every + * item mentioned in Added/Fixed has at least one test file containing + * a relevant token. + * + * Guards against "shipped feature, forgot the test" — the class that + * let P1/P2/P3 escape in the first place. + */ + +const repoRoot = resolve(process.cwd()); +const changelog = readFileSync(resolve(repoRoot, 'CHANGELOG.md'), 'utf-8'); + +// Grab the first ## block +const blocks = changelog.split(/^##\s+/m).slice(1); +assert.ok(blocks.length > 0, 'CHANGELOG.md must have at least one ## version heading'); +const topBlock = blocks[0]!; + +function extractSection(block: string, name: string): string[] { + // Split block on ### headings and find the named section + const parts = block.split(/^###\s+/m); + for (const part of parts) { + if (part.trimStart().toLowerCase().startsWith(name.toLowerCase())) { + return part + .split('\n') + .slice(1) // skip the section heading line + .map((l) => l.trim()) + .filter((l) => l.startsWith('-')) + .map((l) => l.replace(/^-\s*/, '')); + } + } + return []; +} + +const added = extractSection(topBlock, 'Added'); +const fixed = extractSection(topBlock, 'Fixed'); +const items = [...added, ...fixed]; + +// If no items exist in the top block, skip the check (pre-release state). +if (items.length === 0) { + console.log('SKIP: no Added/Fixed items in top CHANGELOG block'); + process.exit(0); +} + +const STOP = new Set([ + 'this', 'that', 'from', 'with', 'into', 'when', 'then', + 'some', 'have', 'been', 'does', 'must', 'will', 'true', 'false', + 'none', 'more', 'less', 'only', 'each', 'other', 'also', + 'added', 'fixed', 'remove', 'removed', 'change', 'changed', + 'every', 'their', 'where', 'which', 'about', 'bench', 'mark', +]); + +const testFiles = readdirSync(resolve(repoRoot, 'tests')) + .filter((f) => f.startsWith('smoke-') && f.endsWith('.ts')) + .map((f) => readFileSync(resolve(repoRoot, 'tests', f), 'utf-8')); + +let failures = 0; +for (const item of items) { + const tokens = item + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter((t) => t.length >= 4 && !STOP.has(t)) + .slice(0, 8); + if (tokens.length === 0) continue; + // Require at least 2 tokens to co-occur in a single test file (whole-word match). + // Prevents false-passes where a lone generic word like "prompt" or "coverage" + // appears somewhere in the corpus but no test actually covers the claimed behavior. + const minMatch = Math.min(2, tokens.length); + const hit = testFiles.some((content) => { + const matched = tokens.filter((t) => new RegExp(`\\b${t}\\b`, 'i').test(content)); + return matched.length >= minMatch; + }); + if (!hit) { + console.error(`[FAIL] CHANGELOG entry has no test reference: "${item}"`); + console.error(` searched for tokens: ${tokens.join(', ')}`); + failures += 1; + } +} + +assert.strictEqual(failures, 0, + `${failures} CHANGELOG item(s) have no matching test file — ` + + `either add a test or remove the CHANGELOG claim`); + +console.log(`PASS: smoke-changelog-coverage (${items.length} items, all with tests)`); diff --git a/tests/smoke-coverage.ts b/tests/smoke-coverage.ts index 890421c..9df5146 100644 --- a/tests/smoke-coverage.ts +++ b/tests/smoke-coverage.ts @@ -20,6 +20,10 @@ function mkTask(id: string, actions: string[]): GeneratedTask { }; } +function mkPromptTask(id: string, capabilityId: string): GeneratedTask { + return { id, prompt: `do ${id}`, expected_actions: [], capabilityId }; +} + function testFullCoverage() { const actions = [mkAction('Wallet.send'), mkAction('Wallet.receive')]; const tasks = [mkTask('t1', ['Wallet.send']), mkTask('t2', ['Wallet.receive'])]; @@ -54,11 +58,24 @@ function testRetryPromptMentionsActions() { console.log('PASS: retry prompt names uncovered actions'); } +function testPromptCoverageFromCapabilityId() { + // Regression guard for Issue 2: prompt tasks have expected_actions:[] but + // must count as covering their declared capabilityId capability. + const actions = [mkAction('summarize'), mkAction('translate')]; + const tasks = [mkPromptTask('t1', 'summarize'), mkPromptTask('t2', 'translate')]; + const coverage = computeCoverage(actions, tasks); + assert.strictEqual(coverage.coverageViolation, false); + assert.strictEqual(coverage.uncoveredActions.length, 0); + assert.deepStrictEqual(coverage.coveredActions.sort(), ['summarize', 'translate']); + console.log('PASS: prompt tasks covered via capabilityId (Issue 2 guard)'); +} + async function main() { testFullCoverage(); testPartialCoverage(); testUncoveredDriver(); testRetryPromptMentionsActions(); + testPromptCoverageFromCapabilityId(); console.log('\nALL PASS: smoke-coverage'); } diff --git a/tests/smoke-discovery-prompt.ts b/tests/smoke-discovery-prompt.ts deleted file mode 100644 index 2145104..0000000 --- a/tests/smoke-discovery-prompt.ts +++ /dev/null @@ -1,321 +0,0 @@ -/** - * Smoke tests for prompt surface discovery (discovery/prompt.ts). - * Mirrors the structure of smoke-discovery-mcp.ts. - */ - -import { readFileSync } from 'node:fs'; -import { resolve } from 'node:path'; - -import { - discoverPromptSurfaceFromContent, - discoverPromptSurfaceFromSources, -} from '../src/discovery/prompt.js'; -import { discoverPromptCapabilities } from '../src/project/discover-prompt.js'; - -let passed = 0; -let failed = 0; - -async function test(name: string, fn: () => Promise | void) { - try { - await fn(); - passed++; - console.log(` + ${name}`); - } catch (error: any) { - failed++; - console.log(` - ${name}`); - console.log(` ${error.message}`); - } -} - -function assert(condition: boolean, message: string): void { - if (!condition) { - throw new Error(`Assertion failed: ${message}`); - } -} - -function assertEqual(actual: T, expected: T, message: string): void { - if (actual !== expected) { - throw new Error(`${message}: expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`); - } -} - -// --------------------------------------------------------------------------- -// Inline sample SKILL.md for unit-level tests -// --------------------------------------------------------------------------- - -const sampleSkill = `--- -name: test-skill -description: A test skill for benchmarking ---- - -# Test Skill - -## Phase 1 — Requirements Discovery - -Ask clarifying questions until the stopping condition is met. - -## Phase 2 — Implementation - -Write the code changes: - -\`\`\`go -func handleRequest(ctx context.Context, req *Request) (*Response, error) { - // implementation -} -\`\`\` - -## Phase 3 — Testing - -Create table-driven tests with named fixtures. -`; - -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - -console.log('\n=== Prompt Discovery Smoke Tests ===\n'); - -await test('discovers 3 phases from sample SKILL.md', () => { - const snapshot = discoverPromptSurfaceFromContent(sampleSkill); - - assertEqual(snapshot.surface, 'prompt', 'surface should be prompt'); - assertEqual(snapshot.phases.length, 3, 'should discover 3 phases'); - - const phaseNames = snapshot.phases.map((p) => p.name); - assert(phaseNames.includes('Requirements Discovery'), 'should include Requirements Discovery phase'); - assert(phaseNames.includes('Implementation'), 'should include Implementation phase'); - assert(phaseNames.includes('Testing'), 'should include Testing phase'); -}); - -await test('each phase has a name and description', () => { - const snapshot = discoverPromptSurfaceFromContent(sampleSkill); - - for (const phase of snapshot.phases) { - assert(phase.name.length > 0, `phase should have a non-empty name, got: "${phase.name}"`); - assert(phase.description.length > 0, `phase "${phase.name}" should have a non-empty description`); - } -}); - -await test('code block detected in Phase 2 (Implementation)', () => { - const snapshot = discoverPromptSurfaceFromContent(sampleSkill); - const implPhase = snapshot.phases.find((p) => p.name === 'Implementation'); - - assert(implPhase !== undefined, 'Implementation phase should exist'); - assert(implPhase!.hasCodeBlocks, 'Implementation phase should have code blocks'); - - // Phase 1 and Phase 3 should NOT have code blocks - const discoveryPhase = snapshot.phases.find((p) => p.name === 'Requirements Discovery'); - assert(discoveryPhase !== undefined, 'Requirements Discovery phase should exist'); - assert(!discoveryPhase!.hasCodeBlocks, 'Requirements Discovery phase should NOT have code blocks'); -}); - -await test('empty skill returns empty capabilities', () => { - const snapshot = discoverPromptSurfaceFromContent(''); - - assertEqual(snapshot.phases.length, 0, 'empty skill should have 0 phases'); - assertEqual(snapshot.capabilities.length, 0, 'empty skill should have 0 capabilities'); -}); - -await test('skill with no phases returns capabilities from instructions', () => { - const noPhaseSkill = `--- -name: simple-skill ---- - -# Simple Skill - -- Validate the input format -- Transform data to JSON -- Send the result to the API -`; - - const snapshot = discoverPromptSurfaceFromContent(noPhaseSkill); - - assertEqual(snapshot.phases.length, 0, 'should have 0 phases'); - assert(snapshot.capabilities.length > 0, 'should have capabilities extracted from bullet points'); - assert( - snapshot.capabilities.every((c) => c.source === 'instruction'), - 'all capabilities should have source "instruction"', - ); -}); - -await test('phases with capabilities have source "phase"', () => { - const snapshot = discoverPromptSurfaceFromContent(sampleSkill); - - assertEqual(snapshot.capabilities.length, 3, 'should have 3 capabilities (one per phase)'); - assert( - snapshot.capabilities.every((c) => c.source === 'phase'), - 'all capabilities should have source "phase" when phases exist', - ); -}); - -await test('discoverPromptSurfaceFromSources reads fixture file', () => { - const fixturePath = resolve(process.cwd(), 'tests/fixtures/sample-skill.md'); - const snapshot = discoverPromptSurfaceFromSources([fixturePath]); - - assertEqual(snapshot.surface, 'prompt', 'surface should be prompt'); - assertEqual(snapshot.phases.length, 3, 'fixture should have 3 phases'); - assertEqual(snapshot.sources.length, 1, 'should track one source file'); - - // Verify structural detection on the fixture - const phase2 = snapshot.phases.find((p) => p.name === 'Manifest Generation'); - assert(phase2 !== undefined, 'fixture should have Manifest Generation phase'); - assert(phase2!.hasCodeBlocks, 'Manifest Generation should have code blocks'); - assert(phase2!.hasNumberedSteps, 'Manifest Generation should have numbered steps'); - - const phase1 = snapshot.phases.find((p) => p.name === 'Requirements Discovery'); - assert(phase1 !== undefined, 'fixture should have Requirements Discovery phase'); - assert(phase1!.hasDecisionPoints, 'Requirements Discovery should have decision points'); - assert(phase1!.hasNumberedSteps, 'Requirements Discovery should have numbered steps'); -}); - -await test('discoverPromptSurfaceFromSources throws on missing file', () => { - let threw = false; - try { - discoverPromptSurfaceFromSources(['/tmp/nonexistent-skill-12345.md']); - } catch (error: any) { - threw = true; - assert( - error.message.includes('does not exist'), - `error should mention missing path, got: ${error.message}`, - ); - } - assert(threw, 'should throw on missing source file'); -}); - -await test('frontmatter is stripped before parsing phases', () => { - const skillWithFrontmatter = `--- -name: fm-test -description: frontmatter test -custom_field: true ---- - -## Phase 1 — Setup - -Initialize the environment. -`; - - const snapshot = discoverPromptSurfaceFromContent(skillWithFrontmatter); - assertEqual(snapshot.phases.length, 1, 'should discover 1 phase after stripping frontmatter'); - assertEqual(snapshot.phases[0].name, 'Setup', 'phase name should be "Setup"'); -}); - -await test('decision points detected from conditional keywords', () => { - const skillWithDecisions = `--- -name: decision-test ---- - -## Phase 1 — Routing - -If the request is a GET, return cached data. -When the cache is stale, fetch fresh data. -Otherwise fall back to the default response. -`; - - const snapshot = discoverPromptSurfaceFromContent(skillWithDecisions); - assertEqual(snapshot.phases.length, 1, 'should discover 1 phase'); - assert(snapshot.phases[0].hasDecisionPoints, 'phase should detect decision points'); -}); - -await test('stripFrontmatter: does not terminate early when YAML value contains ---', () => { - // YAML value "A --- B" contains ---, which indexOf finds first - const content = `---\ntitle: "A --- B"\n---\n\n## Section\nBody text.`; - const result = discoverPromptSurfaceFromContent(content); - assert(result.phases.length > 0, 'should have at least one phase'); - const phaseName = result.phases[0].name.toLowerCase(); - assert(phaseName.includes('section'), - `phase name should be "Section" but was: "${result.phases[0].name}"`); -}); - -// Bug 6: preamble contamination -await test('discoverPromptCapabilities: preamble lines do not contaminate first section body', () => { - const content = [ - '# My Skill', - 'Run this preamble step before anything else.', - '', - '## Phase 1: Do the thing', - 'Write the implementation here.', - ].join('\n'); - const actions = discoverPromptCapabilities(content); - const hasSpuriousInstruction = actions.some(a => - a.description.toLowerCase().includes('preamble'), - ); - // Should be false — preamble should not bleed into instructions - if (hasSpuriousInstruction) throw new Error('preamble content bled into discovered instructions'); -}); - -// Bug 7: frontmatter -await test('discoverPromptCapabilities: strips YAML frontmatter before scanning', () => { - const content = [ - '---', - 'run: my-skill-command', - '---', - '', - '## Phase 1: Setup', - 'Configure the environment.', - ].join('\n'); - const actions = discoverPromptCapabilities(content); - const hasFrontmatterCap = actions.some(a => - a.description.toLowerCase().includes('run: my-skill-command'), - ); - if (hasFrontmatterCap) throw new Error('frontmatter must be stripped before discovery'); - if (actions.length === 0) throw new Error('should still find Phase 1 capability'); -}); - -// Bug 4: heading-less files -await test('discoverPromptCapabilities: handles heading-less instruction-only files', () => { - const content = [ - '# Deployment Skill', - '', - 'This skill deploys services to production.', - '', - '- Ask for the service name and environment', - '- Generate the deployment manifest', - '- Validate the configuration before applying', - '- Run the deployment command', - ].join('\n'); - const actions = discoverPromptCapabilities(content); - assert(actions.length > 0, `should extract at least one capability from bullet list, got ${actions.length}`); -}); - -await test('discoverPromptCapabilities: returns valid ActionDefinition[] shape', () => { - const content = [ - '## Phase 1: Requirements', - 'Ask the user clarifying questions.', - 'Generate a requirements document.', - '', - '## Phase 2: Implementation', - 'Write the implementation code.', - ].join('\n'); - - const actions = discoverPromptCapabilities(content); - assert(Array.isArray(actions), 'should return an array'); - assert(actions.length >= 2, `expected ≥2 capabilities, got ${actions.length}`); - - for (const action of actions) { - assert(typeof action.name === 'string' && action.name.length > 0, 'name must be non-empty'); - assert(typeof action.description === 'string', 'description must be string'); - assert(Array.isArray(action.args), 'args must be array'); - } -}); - -// Regression guard: both discovery modules exist intentionally for different layers. -// This test documents the interface difference so future refactors stay aware. -await test('production path (discover-prompt) vs standalone path (discovery/prompt) have different shapes', () => { - const content = '## Phase 1: Do the thing\nRun this step.'; - - // Production path: ActionDefinition[] — has args field, no source field - const productionResult = discoverPromptCapabilities(content); - assert(productionResult.length > 0, 'production path must return at least one capability'); - assert('args' in productionResult[0], 'production path must have args field'); - - // Standalone discovery path: PromptDiscoverySnapshot — capabilities have source field, not args - const discoveryResult = discoverPromptSurfaceFromContent(content); - if (discoveryResult.capabilities.length > 0) { - const cap = discoveryResult.capabilities[0]; - assert('source' in cap, 'discovery path capability must have source field'); - assert(!('args' in cap), 'discovery path must NOT have args field'); - } -}); - -console.log(`\n${passed} passed, ${failed} failed\n`); -process.exit(failed > 0 ? 1 : 0); diff --git a/tests/smoke-generation.ts b/tests/smoke-generation.ts index 28502e2..f786705 100644 --- a/tests/smoke-generation.ts +++ b/tests/smoke-generation.ts @@ -529,5 +529,167 @@ await test('cli discovery/task generation canonicalizes option keys for extracti } }); +// ── Prompt surface fixture ──────────────────────────────────────────────────── + +function makePromptFixture(): { + root: string; + benchmarkConfigPath: string; + skillPath: string; + // Capability keys produced by the phase headings in the skill file. + capabilityKeys: { summarize: string; translate: string }; +} { + const root = mkdtempSync(join(tmpdir(), 'skill-optimizer-prompt-')); + const skillPath = join(root, 'SKILL.md'); + const benchmarkConfigPath = join(root, 'skill-optimizer.json'); + + // Two ## Phase headings produce exactly two capabilities: + // phase_1_summarize and phase_2_translate + // Body text avoids imperative verbs and decision-point words so no extra + // instruction / decision capabilities are generated alongside the phases. + writeFileSync(skillPath, [ + '# Translation Service Skill', + '', + '## Phase 1: Summarize', + 'Condenses long documents into brief summaries for quick reading.', + '', + '## Phase 2: Translate', + 'Converts text from one language to another while preserving meaning.', + ].join('\n'), 'utf-8'); + + writeFileSync(benchmarkConfigPath, JSON.stringify({ + name: 'prompt-smoke', + target: { + surface: 'prompt', + repoPath: '.', + skill: './SKILL.md', + }, + benchmark: { + tasks: './tasks.json', + format: 'pi', + models: [{ id: 'openai/test', name: 'Test', tier: 'flagship' }], + }, + }, null, 2), 'utf-8'); + + return { + root, + benchmarkConfigPath, + skillPath, + capabilityKeys: { summarize: 'phase_1_summarize', translate: 'phase_2_translate' }, + }; +} + +// ── Prompt surface: capabilityId tagging ───────────────────────────────────── + +await test('prompt surface: generator tags tasks with capabilityId', async () => { + const fixture = makePromptFixture(); + try { + const surface = await discoverTaskSurface(fixture.benchmarkConfigPath); + assertEqual(surface.snapshot.surface, 'prompt', 'surface should be prompt'); + + const { summarize, translate } = fixture.capabilityKeys; + const deps: TaskGeneratorDeps = { + async complete() { + return JSON.stringify({ + tasks: [ + { + id: 'summarize_long_doc', + prompt: 'Summarize this long research paper into three bullet points.', + expected_actions: [], + capabilityId: summarize, + }, + { + id: 'translate_spanish', + prompt: 'Translate the following paragraph from English to Spanish.', + expected_actions: [], + capabilityId: translate, + }, + ], + }); + }, + }; + + const generated = await generateCandidateTasks(surface, { maxTasks: 5, seed: 7 }, deps); + assertEqual(generated.length, 2, 'should produce 2 tasks'); + + const summarizeTask = generated.find((t) => t.capabilityId === summarize); + const translateTask = generated.find((t) => t.capabilityId === translate); + + assert(summarizeTask !== undefined, `task with capabilityId "${summarize}" should exist`); + assert(translateTask !== undefined, `task with capabilityId "${translate}" should exist`); + } finally { + rmSync(fixture.root, { recursive: true, force: true }); + } +}); + +// ── Prompt surface: grounding rejects unknown capabilityId ─────────────────── + +await test('prompt surface: grounding rejects unknown capabilityId', async () => { + const fixture = makePromptFixture(); + try { + const surface = await discoverTaskSurface(fixture.benchmarkConfigPath); + const { summarize } = fixture.capabilityKeys; + + const deps: TaskGeneratorDeps = { + async complete() { + return JSON.stringify({ + tasks: [ + { + id: 'bad_task', + prompt: 'Use an unknown capability.', + expected_actions: [], + capabilityId: 'not-real', + }, + { + id: 'good_task', + prompt: 'Summarize this document into bullet points.', + expected_actions: [], + capabilityId: summarize, + }, + ], + }); + }, + }; + + const generated = await generateCandidateTasks(surface, { maxTasks: 5, seed: 7 }, deps); + + // Only the task with a valid capabilityId passes grounding. + const grounded = groundTasks(generated, surface.snapshot); + assertEqual(grounded.kept.length, 1, 'only the valid capabilityId task should be kept'); + assertEqual(grounded.rejected.length, 1, 'task with unknown capabilityId should be rejected'); + assert( + grounded.rejected[0].reason.includes('unknown capabilityId'), + `rejection reason should mention unknown capabilityId, got: ${grounded.rejected[0].reason}`, + ); + assertEqual(grounded.kept[0].capabilityId, summarize, 'kept task should have the valid capabilityId'); + } finally { + rmSync(fixture.root, { recursive: true, force: true }); + } +}); + +await test('prompt surface: grounding rejects task with missing capabilityId', async () => { + const fixture = makePromptFixture(); + try { + const surface = await discoverTaskSurface(fixture.benchmarkConfigPath); + const deps: TaskGeneratorDeps = { + async complete() { + return JSON.stringify({ + tasks: [ + // capabilityId field is entirely absent from this task + { id: 't1', prompt: 'Do something.', expected_actions: [] }, + ], + }); + }, + }; + const generated = await generateCandidateTasks(surface, { maxTasks: 5, seed: 7 }, deps); + const result = groundTasks(generated, surface.snapshot); + assertEqual(result.kept.length, 0, 'task without capabilityId must be rejected'); + assertEqual(result.rejected.length, 1, 'rejected list must have one entry'); + assert(result.rejected[0]!.reason.includes('capabilityId'), + `rejection reason must mention capabilityId, got: "${result.rejected[0]!.reason}"`); + } finally { + rmSync(fixture.root, { recursive: true, force: true }); + } +}); + console.log(`\n${passed} passed, ${failed} failed\n`); process.exit(failed > 0 ? 1 : 0); diff --git a/tests/smoke-model-ids.ts b/tests/smoke-model-ids.ts index bd0feb7..6ca1ba8 100644 --- a/tests/smoke-model-ids.ts +++ b/tests/smoke-model-ids.ts @@ -60,3 +60,26 @@ await test('openai/ direct-API IDs with dots are NOT rewritten', async () => { assert.equal(result, 'openai/gpt-5.4', 'openai/ direct API dots must be preserved (OpenAI uses gpt-5.4 not gpt-5-4)'); }); + +await test('applyFixes directly: openai/ IDs are NOT rewritten even if model-id-bad-format issue is present', async () => { + const { applyFixes } = await import('../src/project/fix.js'); + const raw = { + name: 'test', + target: { surface: 'mcp' as const, repoPath: '.' }, + benchmark: { + format: 'pi', + models: [{ id: 'openai/gpt-5.4', name: 'GPT-5.4', tier: 'flagship' as const }], + taskGeneration: { enabled: true, maxTasks: 5 }, + }, + }; + // Manufacture the issue that validate.ts would normally never emit for openai/, + // so we exercise fix.ts's defense-in-depth exemption directly. + // fixable: true is required so the filter in applyFixes actually processes it. + const issues = [ + { code: 'model-id-bad-format' as const, field: 'benchmark.models[0].id', message: 'synthetic', severity: 'warning' as const, fixable: true }, + ]; + const fixed = applyFixes(raw as never, issues as never, '/tmp'); + const id = (fixed as { benchmark: { models: Array<{ id: string }> } }).benchmark.models[0]!.id; + assert.equal(id, 'openai/gpt-5.4', + 'fix.ts must exempt openai/ from dot→hyphen rewrite (defense-in-depth; OpenAI API uses dots)'); +}); diff --git a/tests/smoke-prompt-criteria.ts b/tests/smoke-prompt-criteria.ts new file mode 100644 index 0000000..becd39f --- /dev/null +++ b/tests/smoke-prompt-criteria.ts @@ -0,0 +1,104 @@ +import { strict as assert } from 'node:assert'; +import { resolveCriteriaForTask } from '../src/benchmark/prompt-criteria.js'; +import type { GeneratedTask } from '../src/tasks/types.js'; +import type { PromptCapabilityWithSection } from '../src/project/discover-prompt.js'; +import type { ActionDefinition } from '../src/actions/types.js'; + +function cap(actionKey: string, section: string): PromptCapabilityWithSection { + const action: ActionDefinition = { key: actionKey, name: actionKey, args: [], source: 'prompt' }; + return { action, section }; +} + +function task(id: string, capabilityId: string): GeneratedTask { + return { id, prompt: `do ${capabilityId}`, expected_actions: [], capabilityId }; +} + +function testResolvesCriteriaForMatchingCapability() { + const caps = [ + cap('summarize', '## summarize\n\nInclude: date, author. Use a numbered list.'), + cap('translate', '## translate\n\nInclude: source language, target.'), + ]; + const result = resolveCriteriaForTask(task('t1', 'summarize'), caps); + assert.ok(result.criteria, 'criteria must be returned for matched capability'); + assert.strictEqual(result.noActiveCriteria, false); + console.log('PASS: resolves criteria for matching capability'); +} + +function testDistinctCriteriaPerCapability() { + const caps = [ + cap('alpha', '## alpha\n\nInclude: x, y. Numbered list required.'), + cap('beta', '## beta\n\nInclude: totally-different-thing.'), + ]; + const a = resolveCriteriaForTask(task('t1', 'alpha'), caps); + const b = resolveCriteriaForTask(task('t2', 'beta'), caps); + assert.notDeepStrictEqual(a.criteria, b.criteria, + 'different capabilities must produce different criteria (caps[0]-collapse guard)'); + console.log('PASS: distinct criteria per capability'); +} + +function testThrowsOnUnknownCapabilityId() { + const caps = [cap('known', '## known\n\nInclude: foo.')]; + assert.throws( + () => resolveCriteriaForTask(task('t1', 'unknown'), caps), + /capabilityId "unknown"/, + 'unknown capabilityId must throw loudly — no silent fallback', + ); + console.log('PASS: throws on unknown capabilityId'); +} + +function testThrowsOnMissingCapabilityId() { + const caps = [cap('known', '## known\n\nInclude: foo.')]; + const taskWithoutId: GeneratedTask = { id: 't1', prompt: 'test', expected_actions: [] }; + assert.throws( + () => resolveCriteriaForTask(taskWithoutId, caps), + /missing capabilityId/, + 'task without capabilityId must throw loudly', + ); + console.log('PASS: throws on missing capabilityId'); +} + +function testNoActiveCriteriaFlag() { + const caps = [cap('empty', '')]; + const result = resolveCriteriaForTask(task('t1', 'empty'), caps); + assert.strictEqual(result.noActiveCriteria, true, + 'capability with no extractable criteria must set noActiveCriteria: true'); + console.log('PASS: flags noActiveCriteria when criteria are empty'); +} + +function testOutputCapabilityWithCodeBlockSectionProducesCriteria() { + // Regression guard for Issue 1: _output capabilities store section: section.body + // (full markdown with fences), not section: snippet (stripped content without fences). + // generateCriteriaFromCapability requires fences to extract format patterns. + const outputSection = [ + '## Output Format', + '', + 'Respond with this structure:', + '', + '```json', + '{', + ' "name": "",', + ' "count": ', + '}', + '```', + ].join('\n'); + const caps = [cap('my_output', outputSection)]; + const result = resolveCriteriaForTask(task('t1', 'my_output'), caps); + assert.strictEqual(result.noActiveCriteria, false, + 'output capability whose section contains a fenced code block must produce non-empty criteria'); + console.log('PASS: output capability with code block section produces criteria (Issue 1 guard)'); +} + +async function main() { + testResolvesCriteriaForMatchingCapability(); + testDistinctCriteriaPerCapability(); + testThrowsOnUnknownCapabilityId(); + testThrowsOnMissingCapabilityId(); + testNoActiveCriteriaFlag(); + testOutputCapabilityWithCodeBlockSectionProducesCriteria(); + console.log('\nALL PASS: smoke-prompt-criteria'); +} + +main().catch((err) => { + console.error('FAIL: smoke-prompt-criteria', err); + process.exit(1); +}); diff --git a/tests/smoke-prompt-evaluator.ts b/tests/smoke-prompt-evaluator.ts index c8170c2..8024404 100644 --- a/tests/smoke-prompt-evaluator.ts +++ b/tests/smoke-prompt-evaluator.ts @@ -238,6 +238,7 @@ kubectl apply -f manifests/ assertEqual(result.categoryScores.keywords, 1.0, 'all keywords present, no forbidden'); assertEqual(result.categoryScores.structure, 1.0, 'code blocks and numbered list found'); assertEqual(result.score, 1.0, 'overall score should be 1.0 when all criteria met'); + assertEqual(result.noActiveCriteria, false, 'populated criteria must have noActiveCriteria: false'); }); // --------------------------------------------------------------------------- @@ -336,13 +337,20 @@ await test('minLength enforced as format check', () => { }); // --------------------------------------------------------------------------- -// No criteria -> vacuous pass +// No criteria -> noActiveCriteria + score 0 (regression guard for P3) // --------------------------------------------------------------------------- -await test('no criteria specified -> score 1.0 (vacuous pass)', () => { +await test('no criteria specified -> noActiveCriteria: true, score: 0', () => { const criteria: PromptEvaluationCriteria = {}; const result = evaluatePromptResponse('Any response at all.', criteria); - assertEqual(result.score, 1.0, 'no criteria should vacuously pass'); + assertEqual(result.noActiveCriteria, true, 'empty criteria must set noActiveCriteria: true'); + assertEqual(result.score, 0, 'empty criteria must score 0, not vacuously pass'); +}); + +await test('non-empty criteria -> noActiveCriteria: false', () => { + const criteria: PromptEvaluationCriteria = { requiredKeywords: ['alpha'] }; + const result = evaluatePromptResponse('alpha present here.', criteria); + assertEqual(result.noActiveCriteria, false, 'non-empty criteria must not set noActiveCriteria'); }); console.log(`\n${passed} passed, ${failed} failed\n`); diff --git a/tests/smoke-release.ts b/tests/smoke-release.ts index 1f19ae3..df53ad3 100644 --- a/tests/smoke-release.ts +++ b/tests/smoke-release.ts @@ -59,5 +59,14 @@ await test('mock-repos README matches the tracked templates', () => { assert(!readme.includes('mcp-demo'), 'mock-repos README should not mention removed mcp-demo template'); }); +// CHANGELOG must have a heading for current package version +await test('CHANGELOG.md has a section for the current package version', () => { + const pkgVersion = (JSON.parse(readFileSync(join(process.cwd(), 'package.json'), 'utf-8')) as { version: string }).version; + const changelogContent = readFileSync(join(process.cwd(), 'CHANGELOG.md'), 'utf-8'); + const versionHeaderRe = new RegExp(`^##\\s*\\[?${pkgVersion.replace(/\./g, '\\.')}\\]?`, 'm'); + assert(versionHeaderRe.test(changelogContent), `CHANGELOG.md must have a section for version ${pkgVersion}`); + console.log(`PASS: CHANGELOG has section for v${pkgVersion}`); +}); + console.log(`\n${passed} passed, ${failed} failed\n`); process.exit(failed > 0 ? 1 : 0); diff --git a/tests/smoke-verdict-prompt.ts b/tests/smoke-verdict-prompt.ts new file mode 100644 index 0000000..f6b7112 --- /dev/null +++ b/tests/smoke-verdict-prompt.ts @@ -0,0 +1,213 @@ +import { strict as assert } from 'node:assert'; +import { computeVerdict } from '../src/benchmark/scoring.js'; +import type { BenchmarkReport, ModelConfig, BenchmarkSurface } from '../src/benchmark/types.js'; +import { resolveCriteriaForTask } from '../src/benchmark/prompt-criteria.js'; +import { evaluatePromptResponse } from '../src/benchmark/prompt-evaluator.js'; +import type { PromptCapabilityWithSection } from '../src/project/discover-prompt.js'; + +function syntheticReport( + perModel: Record, + surface: BenchmarkSurface, + opts?: { coverageViolation?: boolean; weightedAverage?: number }, +): BenchmarkReport { + const entries = Object.entries(perModel); + const summaryPerModel: Record = {}; + for (const [id, rate] of entries) { + summaryPerModel[id] = { + passRate: rate, avgRecall: 0, avgPrecision: 0, + avgToolSelectionAccuracy: 0, avgArgAccuracy: 0, + avgHallucinationRate: 0, tasksRun: 10, + }; + } + const overall = entries.reduce((a, [, r]) => a + r, 0) / Math.max(1, entries.length); + const wavg = opts?.weightedAverage ?? overall; + return { + timestamp: new Date().toISOString(), + config: { name: 'syn', surface }, + skillVersion: { source: 'local', commitSha: 'local', ref: 'file', fetchedAt: new Date().toISOString() }, + results: [], + coverage: [], + scopeCoverage: opts?.coverageViolation + ? { + coverageViolation: true, + inScopeActions: ['a', 'b'], + outOfScopeActions: [], + coveredActions: ['a'], + uncoveredActions: ['b'], + tasksPerAction: { a: 3, b: 0 }, + } + : undefined, + summary: { + totalTasks: 10, totalModels: entries.length, totalEvaluations: 10 * entries.length, + overallPassRate: overall, weightedAverage: wavg, + avgToolRecall: 0, avgToolPrecision: 0, avgToolSelectionAccuracy: 0, + avgArgAccuracy: 0, avgHallucinationRate: 0, methodCoveragePercent: 1, + perModel: summaryPerModel, perTask: {}, + perTier: { + flagship: { passRate: 0, avgRecall: 0, avgToolSelectionAccuracy: 0, avgArgAccuracy: 0 }, + mid: { passRate: 0, avgRecall: 0, avgToolSelectionAccuracy: 0, avgArgAccuracy: 0 }, + low: { passRate: 0, avgRecall: 0, avgToolSelectionAccuracy: 0, avgArgAccuracy: 0 }, + }, + }, + }; +} + +function testPromptSurfaceIgnoresCoverageViolation() { + const models: ModelConfig[] = [ + { id: 'a', name: 'A', tier: 'flagship' }, + { id: 'b', name: 'B', tier: 'mid' }, + ]; + const report = syntheticReport({ a: 0.9, b: 0.85 }, 'prompt', { + coverageViolation: true, + weightedAverage: 0.875, + }); + const verdict = computeVerdict(report, models, { perModelFloor: 0.6, targetWeightedAverage: 0.7 }); + assert.strictEqual(verdict.result, 'PASS', + 'prompt surface with scores above floor must PASS despite coverageViolation=true'); + assert.ok(!verdict.reasons.some(r => r.includes('coverage')), + 'verdict reasons must not mention coverage for prompt surface'); + console.log('PASS: prompt surface ignores coverage violation'); +} + +function testMcpSurfaceStillBlocksOnCoverageViolation() { + const models: ModelConfig[] = [ + { id: 'a', name: 'A', tier: 'flagship' }, + { id: 'b', name: 'B', tier: 'mid' }, + ]; + const report = syntheticReport({ a: 0.9, b: 0.85 }, 'mcp', { + coverageViolation: true, + weightedAverage: 0.875, + }); + const verdict = computeVerdict(report, models, { perModelFloor: 0.6, targetWeightedAverage: 0.7 }); + assert.strictEqual(verdict.result, 'FAIL', + 'mcp surface must still FAIL on coverage violation (regression guard)'); + assert.ok(verdict.reasons.some(r => r.includes('coverage')), + 'verdict reasons must mention coverage for non-prompt surfaces'); + console.log('PASS: mcp surface still blocks on coverage violation'); +} + +// ── Helpers for scenarios 6-7 ──────────────────────────────────────────────── + +function makeCap(key: string, section: string): PromptCapabilityWithSection { + return { + action: { + key, + name: key, + description: `Capability: ${key}`, + args: [], + }, + section, + }; +} + +// ── Scenario 6: distinct criteria per capability (caps[0]-collapse guard) ──── + +function testDistinctCriteriaPerCapability() { + const caps: PromptCapabilityWithSection[] = [ + makeCap('summarize', '## Summary\nProvide a summary section.\nInclude key points.'), + makeCap('translate', '## Translation\nList the translated output.\nSpecify the target language.'), + makeCap('classify', '## Classification\nShow a numbered list of categories.\nInclude confidence score.'), + ]; + + const tasks = caps.map((cap) => ({ + id: `task_${cap.action.key}`, + prompt: `Perform ${cap.action.key} on the given text.`, + expected_actions: [] as Array<{ name: string; args?: Record }>, + capabilityId: cap.action.key, + })); + + const criteriaList = tasks.map((task) => resolveCriteriaForTask(task, caps).criteria); + + // All 3 criteria must be mutually distinct — none should be equal to another. + for (let i = 0; i < criteriaList.length; i++) { + for (let j = i + 1; j < criteriaList.length; j++) { + const ci = JSON.stringify(criteriaList[i]); + const cj = JSON.stringify(criteriaList[j]); + assert.notStrictEqual( + ci, + cj, + `caps[${i}] and caps[${j}] criteria must be distinct (caps[0]-collapse guard): ` + + `got identical criteria ${ci}`, + ); + } + } + console.log('PASS: distinct criteria per capability (caps[0] collapse guard)'); +} + +// ── Scenario 7: empty criteria → noActiveCriteria via evaluator (P3 guard) ─── + +function testNoActiveCriteriaViaEvaluator() { + // A cap with empty section produces no extractable criteria. + const cap = makeCap('empty_cap', ''); + const task = { + id: 'task_empty', + prompt: 'Do something with empty_cap.', + expected_actions: [] as Array<{ name: string; args?: Record }>, + capabilityId: 'empty_cap', + }; + + const { criteria } = resolveCriteriaForTask(task, [cap]); + const result = evaluatePromptResponse('any response text', criteria); + + assert.strictEqual(result.score, 0, 'empty criteria → score must be 0'); + assert.strictEqual(result.noActiveCriteria, true, 'empty criteria → noActiveCriteria must be true'); + console.log('PASS: empty criteria → noActiveCriteria via evaluator (P3 regression guard)'); +} + +// ── Scenario 8: mock-LLM verdict matrix (threshold off-by-one + weight math) ─ + +function testVerdictMatrix() { + const models2: ModelConfig[] = [ + { id: 'a', name: 'A', tier: 'flagship' }, + { id: 'b', name: 'B', tier: 'mid' }, + ]; + const policy = { perModelFloor: 0.6, targetWeightedAverage: 0.7 }; + + // 8a: both 1.0 → PASS + const r8a = syntheticReport({ a: 1.0, b: 1.0 }, 'prompt', { weightedAverage: 1.0 }); + assert.strictEqual(computeVerdict(r8a, models2, policy).result, 'PASS', 'both 1.0 → PASS'); + + // 8b: floor inclusive (0.60 == floor) + const r8b = syntheticReport({ a: 1.0, b: 0.60 }, 'prompt', { weightedAverage: 0.80 }); + assert.strictEqual(computeVerdict(r8b, models2, policy).result, 'PASS', 'floor inclusive at 0.60'); + + // 8c: below floor (0.59) + const r8c = syntheticReport({ a: 1.0, b: 0.59 }, 'prompt', { weightedAverage: 0.795 }); + assert.strictEqual(computeVerdict(r8c, models2, policy).result, 'FAIL', '0.59 < floor → FAIL'); + + // 8d: weights 2:1 → wavg 0.733 (a=0.80, b=0.60, weights 2:1) + const models2d: ModelConfig[] = [ + { id: 'a', name: 'A', tier: 'flagship', weight: 2 }, + { id: 'b', name: 'B', tier: 'mid', weight: 1 }, + ]; + const r8d = syntheticReport({ a: 0.80, b: 0.60 }, 'prompt', { weightedAverage: (0.80 * 2 + 0.60 * 1) / 3 }); + assert.strictEqual( + computeVerdict(r8d, models2d, { perModelFloor: 0.6, targetWeightedAverage: 0.7 }).result, + 'PASS', + 'weight 2:1 wavg 0.733 > 0.7 → PASS', + ); + + // 8e: wavg below target (a=0.70, b=0.60, weights 1:1 → wavg 0.65) + const r8e = syntheticReport({ a: 0.70, b: 0.60 }, 'prompt', { weightedAverage: 0.65 }); + assert.strictEqual(computeVerdict(r8e, models2, policy).result, 'FAIL', 'wavg 0.65 < target 0.70 → FAIL'); + + console.log('PASS: verdict matrix (threshold off-by-one + weight math guards)'); +} + +async function main() { + testPromptSurfaceIgnoresCoverageViolation(); + testMcpSurfaceStillBlocksOnCoverageViolation(); + testDistinctCriteriaPerCapability(); + testNoActiveCriteriaViaEvaluator(); + testVerdictMatrix(); + console.log('\nALL PASS: smoke-verdict-prompt'); +} + +main().catch((err) => { + console.error('FAIL: smoke-verdict-prompt', err); + process.exit(1); +}); From c447db71ec58db1cf2e8ddd62998c6ffcdb63f8d Mon Sep 17 00:00:00 2001 From: dmn Date: Fri, 17 Apr 2026 09:45:49 -0700 Subject: [PATCH 5/8] fix: address PR #26 review findings (prompt surface, docs, precision, error quality) (#30) * docs: add implementation plan for PR #26 review fixes (13 issues) * fix(preflight): exempt prompt surface from maxTasks check; surface-aware discovery hints Co-Authored-By: Claude Sonnet 4.6 * fix(init): add prompt surface next-steps guidance * fix(wizard): accept anthropic/ and openai/ model IDs in custom model validator * fix(generate): allow missing expected_actions on prompt surface in validateTask When `knownCapabilityKeys` is defined (prompt surface), LLMs may omit `expected_actions` entirely even though the prompt requests an empty array. Fall back to `[]` instead of throwing, so task generation is not blocked. * fix(docs): correct config path to .skill-optimizer/ and update stale model ID Replace all occurrences of `skill-optimizer/skill-optimizer.json` (without dot) with `.skill-optimizer/skill-optimizer.json` (with dot) to match the actual path written by `src/init/scaffold.ts`. Also update stale `openrouter/openai/gpt-4o` model ID in `SKILL/references/setup.md` to `openrouter/openai/gpt-4o-mini`. * fix(snapshot): include snapshotPath in unsupported-format error message * fix(runner): set toolPrecision=1.0 for prompt surface tasks * fix(docs): correct apiKeyEnv description and loop.ts agent cwd comment Co-Authored-By: Claude Sonnet 4.6 * fix(generate): guard generateCandidateTasksWithCoverage against prompt surface * fix(tasks): replace brittle string match with NoTextBlocksError class --------- Co-authored-by: OpenClaw Agent (basd) Co-authored-by: Claude Sonnet 4.6 --- README.md | 8 +- SKILL/SKILL.md | 2 +- SKILL/references/setup.md | 4 +- .../2026-04-17-fix-pr26-review-issues.md | 916 ++++++++++++++++++ src/benchmark/init.ts | 3 + src/benchmark/runner.ts | 3 + src/cli.ts | 2 +- src/doctor/checks.ts | 10 +- src/init/wizard.ts | 4 +- src/optimizer/loop.ts | 3 +- src/project/schema.ts | 2 +- src/project/snapshot.ts | 2 +- src/tasks/default-pi-critic.ts | 4 +- src/tasks/generate.ts | 8 + src/tasks/pi-simple-complete.ts | 11 +- 15 files changed, 965 insertions(+), 17 deletions(-) create mode 100644 docs/superpowers/plans/2026-04-17-fix-pr26-review-issues.md diff --git a/README.md b/README.md index 28b28bb..9bb2340 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ For direct OpenAI runs you can also use your local Codex browser login instead o npx skill-optimizer init cli # or: init sdk, init mcp, init prompt ``` -The wizard asks for your repo path, models to benchmark, and where your `SKILL.md` lives. It creates a `skill-optimizer/` directory: +The wizard asks for your repo path, models to benchmark, and where your `SKILL.md` lives. It creates a `.skill-optimizer/` directory: - `skill-optimizer.json` — the main config (commit this) - `.skill-optimizer/cli-commands.json` — CLI surface manifest (template to edit, or auto-extracted) - `.skill-optimizer/tools.json` — MCP surface manifest (template to edit) @@ -68,13 +68,13 @@ npx skill-optimizer import-commands --from my-cli --scrape **Step 3 — Run a benchmark:** ```bash -npx skill-optimizer run --config ./skill-optimizer/skill-optimizer.json +npx skill-optimizer run --config ./.skill-optimizer/skill-optimizer.json ``` **Step 4 — Run the optimizer** (iteratively improves your `SKILL.md`): ```bash -npx skill-optimizer optimize --config ./skill-optimizer/skill-optimizer.json +npx skill-optimizer optimize --config ./.skill-optimizer/skill-optimizer.json ``` The optimizer never modifies your original `SKILL.md` — it works from versioned local copies in `.skill-optimizer/` and prints a progress table at the end showing per-model improvement. @@ -113,7 +113,7 @@ npx skill-optimizer init --answers answers.json } ``` -**Key config fields** in `skill-optimizer/skill-optimizer.json`: +**Key config fields** in `.skill-optimizer/skill-optimizer.json`: | Field | What it does | Set it to | |-------|-------------|-----------| diff --git a/SKILL/SKILL.md b/SKILL/SKILL.md index e725323..93bc72b 100644 --- a/SKILL/SKILL.md +++ b/SKILL/SKILL.md @@ -39,7 +39,7 @@ Before doing anything, figure out where you are: | Run optimizer | `npx skill-optimizer optimize --config ` | | Compare two runs | `npx skill-optimizer compare --baseline a.json --current b.json` | -`` is the path to your `skill-optimizer.json` — typically `./skill-optimizer/skill-optimizer.json` after running `init`, or wherever you placed it. +`` is the path to your `skill-optimizer.json` — typically `./.skill-optimizer/skill-optimizer.json` after running `init`, or wherever you placed it. ## What Do You Need? diff --git a/SKILL/references/setup.md b/SKILL/references/setup.md index 936c51b..72d6189 100644 --- a/SKILL/references/setup.md +++ b/SKILL/references/setup.md @@ -78,7 +78,7 @@ npx skill-optimizer init --answers answers.json { "surface": "cli", "repoPath": "/absolute/path/to/your-repo", - "models": ["openrouter/anthropic/claude-sonnet-4.6", "openrouter/openai/gpt-4o"], + "models": ["openrouter/anthropic/claude-sonnet-4.6", "openrouter/openai/gpt-4o-mini"], "maxTasks": 20, "maxIterations": 5, "entryFile": "src/cli.ts" @@ -130,7 +130,7 @@ npx skill-optimizer doctor --fix --config After successful setup: -- **`skill-optimizer.json`** — main config file (commit this); when created by `init`, the default location is `./skill-optimizer/skill-optimizer.json` +- **`skill-optimizer.json`** — main config file (commit this); when created by `init`, the default location is `./.skill-optimizer/skill-optimizer.json` - **`.skill-optimizer/`** — working directory for task artifacts, surface manifests, and versioned skill copies (gitignored) Your project is ready for benchmarking. Read `references/benchmark.md` for next steps. diff --git a/docs/superpowers/plans/2026-04-17-fix-pr26-review-issues.md b/docs/superpowers/plans/2026-04-17-fix-pr26-review-issues.md new file mode 100644 index 0000000..6cf502d --- /dev/null +++ b/docs/superpowers/plans/2026-04-17-fix-pr26-review-issues.md @@ -0,0 +1,916 @@ +# PR #26 Review Fixes Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Fix 13 bugs found during code review of PR #26 (v1.1.0) as a single PR to `development`. + +**Architecture:** All fixes are isolated, targeted one-to-few-line changes to existing files. No new modules. The highest-impact fixes (prompt surface preflight, init guidance, wizard validator) are done first so they're independently reviewable. Doc fixes are batched. Minor code-quality improvements (comment accuracy, structured errors) come last. + +**Tech Stack:** TypeScript, Node.js, `npm test` (smoke tests), `npm run typecheck`. + +--- + +## File Structure + +Files to modify (no new files created): + +- `src/benchmark/init.ts` — add `else if (surface === 'prompt')` branch to next-steps output (issue 1) +- `src/init/wizard.ts` — fix custom model ID validator to accept all three prefixes (issue 2) +- `src/tasks/generate.ts` — add prompt-surface fallback in `validateTask`; add guard comment in retry path (issues 3, 12) +- `src/cli.ts` — add `project.target.surface !== 'prompt'` guard to maxTasks check (issue 4) +- `src/doctor/checks.ts` — add surface guard to maxTasks check; fix discovery-failure hint (issues 4, 5) +- `README.md` — fix `./skill-optimizer/` → `./.skill-optimizer/` in all examples (issue 6) +- `SKILL/SKILL.md` — fix config path reference (issue 6) +- `SKILL/references/setup.md` — fix config path + update stale `gpt-4o` model ID (issues 6, 8) +- `src/project/snapshot.ts` — include `snapshotPath` in unsupported-format error (issue 7) +- `src/benchmark/runner.ts` — override `toolPrecision` for prompt surface tasks (issue 9) +- `src/project/schema.ts` — fix `apiKeyEnv` description: "by provider prefix, not format" (issue 10) +- `src/optimizer/loop.ts` — fix stale comment about agent cwd in local-skill mode (issue 11) +- `src/tasks/pi-simple-complete.ts` — export `NoTextBlocksError` class (issue 13) +- `src/tasks/default-pi-critic.ts` — catch `NoTextBlocksError` by type instead of string match (issue 13) + +--- + +## Task 0: Set up worktree + +**Files:** (no files changed — environment setup only) + +- [ ] **Step 1: Create the worktree** + +```bash +cd /root/openclaw-workspace/skill-benchmark +git worktree add .worktrees/fix-pr26-issues -b fix/pr26-review-issues +``` + +- [ ] **Step 2: Install dependencies** + +```bash +cd .worktrees/fix-pr26-issues +npm install +``` + +- [ ] **Step 3: Verify baseline tests pass** + +```bash +npm test +``` + +Expected: all smoke tests pass with zero failures. If anything fails, stop and investigate before making changes. + +--- + +## Task 1: Fix prompt surface preflight in `--dry-run` and `doctor` + +**Issues fixed:** 4 (maxTasks guard) and 5 (discovery-failure hint) + +**Files:** +- Modify: `src/cli.ts:218` +- Modify: `src/doctor/checks.ts:14-51` + +### Background + +`tasks/index.ts:51` exempts prompt surfaces from the `maxTasks < inScope.length` gate with: +```typescript +if (surface.snapshot.surface !== 'prompt' && maxTasks < inScope.length) { +``` +But `cli.ts` and `doctor/checks.ts` have no equivalent guard — they hard-fail for prompt configs. + +Also, `doctor/checks.ts` catch block (lines 14-19) shows "Check target.discovery.sources and your manifest file" when discovery throws. Prompt surfaces use neither — they use `target.skill.source`. + +- [ ] **Step 1: Fix `src/cli.ts` maxTasks check** + +In `src/cli.ts`, find this block (around line 218): + +```typescript + const maxTasks = project.benchmark.taskGeneration.maxTasks; + if (project.benchmark.taskGeneration.enabled && inScope.length > 0 && maxTasks < inScope.length) { + console.error(`\nERROR: maxTasks (${maxTasks}) < in-scope action count (${inScope.length}).`); + console.error(`Raise benchmark.taskGeneration.maxTasks in ${project.configPath}, or tighten target.scope.exclude.`); + process.exit(1); + } +``` + +Replace with: + +```typescript + const maxTasks = project.benchmark.taskGeneration.maxTasks; + if (project.target.surface !== 'prompt' && project.benchmark.taskGeneration.enabled && inScope.length > 0 && maxTasks < inScope.length) { + console.error(`\nERROR: maxTasks (${maxTasks}) < in-scope action count (${inScope.length}).`); + console.error(`Raise benchmark.taskGeneration.maxTasks in ${project.configPath}, or tighten target.scope.exclude.`); + process.exit(1); + } +``` + +- [ ] **Step 2: Fix `src/doctor/checks.ts` — both the maxTasks guard and the discovery hint** + +In `src/doctor/checks.ts`, replace the entire `checkDiscovery` function: + +```typescript +export function checkDiscovery(project: ResolvedProjectConfig): Issue[] { + const issues: Issue[] = []; + let discovered: ReturnType; + + try { + discovered = discoverActionsOnly(project); + } catch (err) { + const isPrompt = project.target.surface === 'prompt'; + const hint = isPrompt + ? `Check the skill file at target.skill — ensure it has parseable capability headings` + : `Check target.discovery.sources and your manifest file`; + issues.push({ + code: 'discovery-failed', severity: 'error', field: 'target.discovery', + message: `Discovery threw an error: ${err instanceof Error ? err.message : String(err)}`, + hint, + fixable: false, + }); + return issues; + } + + const { inScope } = resolveScope(discovered, project.target.scope); + + if (inScope.length === 0) { + let surfaceHint: string; + if (project.target.surface === 'cli') { + surfaceHint = `Add target.cli.commands pointing at a cli-commands.json manifest, or fix target.discovery.sources`; + } else if (project.target.surface === 'mcp') { + surfaceHint = `Add target.mcp.tools pointing at a tools.json manifest, or fix target.discovery.sources`; + } else if (project.target.surface === 'prompt') { + surfaceHint = `Ensure the skill file (target.skill) contains parseable capability headings`; + } else { + surfaceHint = `Fix target.discovery.sources to point at your SDK entry file`; + } + issues.push({ + code: 'zero-actions-discovered', severity: 'error', field: 'target.discovery', + message: `Discovery found 0 in-scope actions`, + hint: surfaceHint, + fixable: false, + }); + } else { + const maxTasks = project.benchmark.taskGeneration?.maxTasks ?? 0; + if (project.target.surface !== 'prompt' && project.benchmark.taskGeneration?.enabled && maxTasks < inScope.length) { + issues.push({ + code: 'max-tasks-too-low', severity: 'error', field: 'benchmark.taskGeneration.maxTasks', + message: `maxTasks (${maxTasks}) is less than the number of in-scope actions (${inScope.length})`, + hint: `Raise benchmark.taskGeneration.maxTasks to at least ${inScope.length}`, + fixable: false, + }); + } + issues.push({ + code: 'discovery-ok', severity: 'info', field: 'target.discovery', + message: `${inScope.length} action(s) discovered (${project.target.surface} surface)`, + fixable: false, + }); + } + + return issues; +} +``` + +- [ ] **Step 3: Run typecheck** + +```bash +npm run typecheck +``` + +Expected: zero errors. + +- [ ] **Step 4: Run tests** + +```bash +npm test +``` + +Expected: all pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/cli.ts src/doctor/checks.ts +git commit -m "fix(preflight): exempt prompt surface from maxTasks gate in --dry-run and doctor + +Both cli.ts and doctor/checks.ts hard-failed when maxTasks < discovered capability +count for prompt surfaces, even though tasks/index.ts already exempts prompt from +this constraint. Also fixes the discovery-failure hint for prompt surfaces: the old +message pointed users at target.discovery.sources and manifests, which prompt +surfaces don't use." +``` + +--- + +## Task 2: Fix `init` next-steps for prompt surface + +**Issue fixed:** 1 + +**Files:** +- Modify: `src/benchmark/init.ts:92-107` + +### Background + +The `else` branch in the next-steps output catches all surfaces that aren't `sdk` or `cli`, including the new `prompt` surface. Users running `init prompt` see instructions to configure `target.discovery.sources` and `tools.json`, which prompt surfaces don't use. + +- [ ] **Step 1: Add prompt branch to next-steps output** + +In `src/benchmark/init.ts`, find this block (around line 92): + +```typescript + if (surface === 'sdk') { + console.log(' target.discovery.sources → entry file(s) for SDK discovery'); + } else if (surface === 'cli') { + console.log(' target.discovery.sources → CLI entry file (for code-first discovery)'); + console.log(' .skill-optimizer/cli-commands.json → replace template with your real commands'); + console.log(' (cli-commands.json is used as a fallback if code-first discovery finds nothing)'); + } else { + console.log(' target.discovery.sources → MCP server file (for code-first discovery)'); + console.log(' .skill-optimizer/tools.json → replace template with your real tools'); + console.log(' (tools.json is used as a fallback if code-first discovery finds nothing)'); + } +``` + +Replace with: + +```typescript + if (surface === 'sdk') { + console.log(' target.discovery.sources → entry file(s) for SDK discovery'); + } else if (surface === 'cli') { + console.log(' target.discovery.sources → CLI entry file (for code-first discovery)'); + console.log(' .skill-optimizer/cli-commands.json → replace template with your real commands'); + console.log(' (cli-commands.json is used as a fallback if code-first discovery finds nothing)'); + } else if (surface === 'prompt') { + console.log(' target.skill → path to your SKILL.md or prompt document'); + console.log(' (no discovery sources needed — capabilities are read directly from the skill file)'); + } else { + console.log(' target.discovery.sources → MCP server file (for code-first discovery)'); + console.log(' .skill-optimizer/tools.json → replace template with your real tools'); + console.log(' (tools.json is used as a fallback if code-first discovery finds nothing)'); + } +``` + +- [ ] **Step 2: Run tests** + +```bash +npm test +``` + +Expected: all pass (smoke-init tests do not check `console.log` output so no test update needed here). + +- [ ] **Step 3: Commit** + +```bash +git add src/benchmark/init.ts +git commit -m "fix(init): add prompt-surface branch to next-steps guidance + +The else branch caught 'prompt' and showed MCP instructions (target.discovery.sources, +tools.json) which don't apply. Now shows the correct guidance: target.skill path, +no discovery sources needed." +``` + +--- + +## Task 3: Fix wizard custom model ID validator + +**Issue fixed:** 2 + +**Files:** +- Modify: `src/init/wizard.ts:129` + +### Background + +The validator at line 129 rejects any model ID that doesn't start with `openrouter/`. This was written before `anthropic/` and `openai/` direct-API prefixes were added. The `validate.ts` validator accepts all three prefixes; the wizard must too — especially since v1.1.0 introduces Codex auth for `openai/` models. + +- [ ] **Step 1: Fix the validator** + +In `src/init/wizard.ts`, find: + +```typescript + if (!v.startsWith('openrouter/')) return 'Must start with openrouter/'; +``` + +Replace with: + +```typescript + if (!v.startsWith('openrouter/') && !v.startsWith('anthropic/') && !v.startsWith('openai/')) { + return 'Must start with openrouter/, anthropic/, or openai/'; + } +``` + +- [ ] **Step 2: Run tests** + +```bash +npm test +``` + +Expected: all pass. The smoke-init.ts tests don't exercise this specific validation path. + +- [ ] **Step 3: Commit** + +```bash +git add src/init/wizard.ts +git commit -m "fix(wizard): accept anthropic/ and openai/ model IDs in custom-model validator + +The validator rejected any non-openrouter/ prefix, blocking direct-API model IDs +from being entered via the interactive wizard. validate.ts accepts all three prefixes; +the wizard now matches." +``` + +--- + +## Task 4: Add prompt-surface fallback in `validateTask` + +**Issue fixed:** 3 + +**Files:** +- Modify: `src/tasks/generate.ts:226-231` + +### Background + +`validateTask()` throws if `rawExpectedActions` is undefined. For prompt-surface tasks, `expected_actions` is always `[]`. If the LLM elides the key instead of emitting `[]` (a common LLM behaviour with empty arrays), the generation crashes. The function already receives `knownCapabilityKeys` to detect prompt-surface context — it just needs to use it as a fallback. + +- [ ] **Step 1: Add the prompt-surface fallback** + +In `src/tasks/generate.ts`, find (around line 226): + +```typescript + if (!rawExpectedActions) { + const received = JSON.stringify(Object.keys(candidate)); + throw new Error(`Task ${taskId} must include an expected_actions array (received keys: ${received})`); + } +``` + +Replace with: + +```typescript + // Prompt surface: expected_actions is always []. If the LLM elided the key, + // default to [] rather than failing — the grounding step enforces emptiness anyway. + if (!rawExpectedActions && knownCapabilityKeys !== undefined) { + rawExpectedActions = []; + } + + if (!rawExpectedActions) { + const received = JSON.stringify(Object.keys(candidate)); + throw new Error(`Task ${taskId} must include an expected_actions array (received keys: ${received})`); + } +``` + +- [ ] **Step 2: Run tests** + +```bash +npm test +``` + +Expected: all pass. The smoke-generation tests exercise the generation path and should still pass. + +- [ ] **Step 3: Commit** + +```bash +git add src/tasks/generate.ts +git commit -m "fix(generate): default expected_actions to [] for prompt surface when LLM elides the key + +LLMs routinely omit empty-array fields despite instructions. For prompt surface +(detected via knownCapabilityKeys !== undefined), expected_actions is always [] +so we default it rather than throwing. The grounding step in ground.ts enforces +emptiness for prompt tasks regardless." +``` + +--- + +## Task 5: Fix doc config path and stale model ID + +**Issues fixed:** 6 and 8 + +**Files:** +- Modify: `README.md` (multiple occurrences) +- Modify: `SKILL/SKILL.md:42` +- Modify: `SKILL/references/setup.md` (two places) + +### Background + +`scaffold.ts` writes configs to `./.skill-optimizer/skill-optimizer.json` (dot-prefixed, hidden directory). All user-facing docs reference `./skill-optimizer/skill-optimizer.json` (no dot) — one character off, causes immediate config-not-found on first use. + +Also, `SKILL/references/setup.md` line ~81 uses `openrouter/openai/gpt-4o` which is not in the current MODEL_PRESETS. Current presets: `gpt-5.4`, `gpt-4o-mini`, `gpt-oss-120b`. + +- [ ] **Step 1: Fix README.md** + +In `README.md`, find and replace all three occurrences of `./skill-optimizer/skill-optimizer.json` with `./.skill-optimizer/skill-optimizer.json`. Also fix the table at line ~116 that references `skill-optimizer/skill-optimizer.json`: + +Find this block (around line 71): +```markdown +npx skill-optimizer run --config ./skill-optimizer/skill-optimizer.json +``` +Replace with: +```markdown +npx skill-optimizer run --config ./.skill-optimizer/skill-optimizer.json +``` + +Find this block (around line 77): +```markdown +npx skill-optimizer optimize --config ./skill-optimizer/skill-optimizer.json +``` +Replace with: +```markdown +npx skill-optimizer optimize --config ./.skill-optimizer/skill-optimizer.json +``` + +Find the table reference at ~line 116 (`skill-optimizer/skill-optimizer.json` appears in the "Key config fields" table header): +```markdown +**Key config fields** in `skill-optimizer/skill-optimizer.json`: +``` +Replace with: +```markdown +**Key config fields** in `.skill-optimizer/skill-optimizer.json`: +``` + +- [ ] **Step 2: Fix SKILL/SKILL.md** + +Find (around line 42): +```markdown +`` is the path to your `skill-optimizer.json` — typically `./skill-optimizer/skill-optimizer.json` after running `init`, or wherever you placed it. +``` +Replace with: +```markdown +`` is the path to your `skill-optimizer.json` — typically `./.skill-optimizer/skill-optimizer.json` after running `init`, or wherever you placed it. +``` + +- [ ] **Step 3: Fix SKILL/references/setup.md** + +Fix the config path around line 133: +```markdown +- **`skill-optimizer.json`** — main config file (commit this); when created by `init`, the default location is `./skill-optimizer/skill-optimizer.json` +``` +Replace with: +```markdown +- **`skill-optimizer.json`** — main config file (commit this); when created by `init`, the default location is `./.skill-optimizer/skill-optimizer.json` +``` + +Fix the stale model ID around line 81 (inside the `answers.json` example): +```json + "models": ["openrouter/anthropic/claude-sonnet-4.6", "openrouter/openai/gpt-4o"], +``` +Replace with: +```json + "models": ["openrouter/anthropic/claude-sonnet-4.6", "openrouter/openai/gpt-4o-mini"], +``` + +- [ ] **Step 4: Run tests** + +```bash +npm test +``` + +Expected: all pass (smoke tests don't assert these doc strings). + +- [ ] **Step 5: Commit** + +```bash +git add README.md SKILL/SKILL.md SKILL/references/setup.md +git commit -m "fix(docs): correct config path to .skill-optimizer/ and update stale model ID + +scaffold.ts writes to ./.skill-optimizer/ (dot-prefixed hidden dir) but all user docs +referenced ./skill-optimizer/ (no dot). Also updated the gpt-4o example in +setup.md to gpt-4o-mini which is in the current MODEL_PRESETS." +``` + +--- + +## Task 6: Include `snapshotPath` in unsupported-format error + +**Issue fixed:** 7 + +**Files:** +- Modify: `src/project/snapshot.ts:71-73` + +### Background + +The invalid-JSON branch at line 62 already includes `snapshotPath`. The unsupported-format branch at line 71-73 doesn't, making it hard to debug which file is the problem when multiple snapshots exist. + +- [ ] **Step 1: Add snapshotPath to the error** + +In `src/project/snapshot.ts`, find (around line 71): + +```typescript + throw new Error( + `Snapshot file format is not supported — delete .skill-optimizer/ and re-run the benchmark to regenerate.`, + ); +``` + +Replace with: + +```typescript + throw new Error( + `Snapshot file format is not supported: ${snapshotPath} — delete .skill-optimizer/ and re-run the benchmark to regenerate.`, + ); +``` + +- [ ] **Step 2: Run tests** + +```bash +npm test +``` + +Expected: all pass. + +- [ ] **Step 3: Commit** + +```bash +git add src/project/snapshot.ts +git commit -m "fix(snapshot): include snapshotPath in unsupported-format error message" +``` + +--- + +## Task 7: Fix `toolPrecision` for prompt surface + +**Issue fixed:** 9 + +**Files:** +- Modify: `src/benchmark/runner.ts` (the prompt surface metrics block, around line 370-390) + +### Background + +For prompt surface tasks, `knownMethods` is an empty `Set` and `extractedCalls` is `[]`, so `toolPrecision` in `evaluator.ts` always computes to `0.0`. The runner already overrides `toolRecall` for prompt tasks (line ~381: `taskResult.metrics.toolRecall = promptResult.score`), but never touches `toolPrecision`. The `0.0` then rolls up into `avgToolPrecision` in the summary, making reports misleading. + +The fix: set `toolPrecision = 1.0` for prompt tasks in the same block where `toolRecall` is overridden (the `promptResult.noActiveCriteria` false branch and the no-active-criteria true branch both need it). + +- [ ] **Step 1: Locate the prompt surface metrics block** + +Find the section in `src/benchmark/runner.ts` that looks like: + +```typescript + } else { + taskResult.metrics.toolRecall = promptResult.score; + taskResult.metrics.taskPassed = promptResult.score >= 0.5; + console.log(` [${slug}] Prompt score: ${promptResult.score.toFixed(3)} → ${taskResult.metrics.taskPassed ? 'PASS' : 'FAIL'}`); + } +``` + +and the no-active-criteria branch: + +```typescript + if (promptResult.noActiveCriteria) { + const msg = `Task "${task.id}" has no extractable criteria — fix SKILL.md section for that action`; + taskResult.metrics.toolRecall = 0; + taskResult.metrics.taskPassed = false; +``` + +- [ ] **Step 2: Add `toolPrecision = 1.0` to both branches** + +Replace the entire prompt evaluation block (covering the `try { ... } catch` around the evaluatePromptResponse call): + +```typescript + try { + const promptResult = evaluatePromptResponse(rawResponse, criteria); + if (promptResult.noActiveCriteria) { + const msg = `Task "${task.id}" has no extractable criteria — fix SKILL.md section for that action`; + taskResult.metrics.toolRecall = 0; + taskResult.metrics.toolPrecision = 1.0; + taskResult.metrics.taskPassed = false; + taskResult.error = taskResult.error ?? msg; + console.error(` [${slug}] Prompt eval error: ${msg}`); + } else { + taskResult.metrics.toolRecall = promptResult.score; + taskResult.metrics.toolPrecision = 1.0; + taskResult.metrics.taskPassed = promptResult.score >= 0.5; + console.log(` [${slug}] Prompt score: ${promptResult.score.toFixed(3)} → ${taskResult.metrics.taskPassed ? 'PASS' : 'FAIL'}`); + } + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + console.error(` [${slug}] Prompt eval error: ${msg}`); + taskResult.metrics.toolRecall = 0; + taskResult.metrics.toolPrecision = 1.0; + taskResult.metrics.taskPassed = false; + taskResult.error = taskResult.error ?? msg; + } +``` + +The rationale for `1.0`: prompt tasks make no tool calls by design, so the model cannot hallucinate a wrong tool — precision is vacuously perfect. Using `1.0` (not `0`) prevents the metric from dragging down `avgToolPrecision` in mixed reports. + +- [ ] **Step 3: Run typecheck and tests** + +```bash +npm run typecheck && npm test +``` + +Expected: zero errors, all tests pass. + +- [ ] **Step 4: Commit** + +```bash +git add src/benchmark/runner.ts +git commit -m "fix(runner): set toolPrecision=1.0 for prompt surface tasks + +Prompt tasks make no tool calls by design, so toolPrecision was always 0.0 due to +the empty knownMethods Set. 0.0 rolled up into avgToolPrecision and made benchmark +reports misleading. Use 1.0 (vacuously perfect: no tool calls = no wrong tool calls), +matching the convention used for toolRecall override." +``` + +--- + +## Task 8: Fix schema description and stale comment + +**Issues fixed:** 10 and 11 + +**Files:** +- Modify: `src/project/schema.ts:68` +- Modify: `src/optimizer/loop.ts:144-145` + +### Background + +**Issue 10:** `benchmark.apiKeyEnv` description says "default: OPENROUTER_API_KEY for format:pi, OPENAI_API_KEY for format:openai…" but the actual default is determined by the **model's provider prefix** (openrouter/, openai/, anthropic/), not by `benchmark.format`. A format:pi run with an `openai/` model defaults to `OPENAI_API_KEY`. + +**Issue 11:** `loop.ts` comment says the agent in local-skill mode "runs with cwd=targetRepo". The actual code in `pi-coding.ts` (lines 19-24) sets cwd to `dirname(localSkillPath)` — the skill output directory — precisely to isolate the agent from the target repo. + +- [ ] **Step 1: Fix the schema description in `src/project/schema.ts`** + +Find (around line 68): + +```typescript + apiKeyEnv: z.string().optional().describe('Env var name for the API key (default: OPENROUTER_API_KEY for format:pi, OPENAI_API_KEY for format:openai, ANTHROPIC_API_KEY for format:anthropic)'), +``` + +Replace with: + +```typescript + apiKeyEnv: z.string().optional().describe('Env var name for the API key (default is determined by the model provider prefix: openrouter/ → OPENROUTER_API_KEY, openai/ → OPENAI_API_KEY, anthropic/ → ANTHROPIC_API_KEY; leave unset to use the per-provider default)'), +``` + +- [ ] **Step 2: Fix the stale comment in `src/optimizer/loop.ts`** + +Find (around line 144): + +```typescript + // In local-skill mode: restore the repo to undo any rogue writes the agent may have made + // (it runs with cwd=targetRepo), then skip scope validation (the local file is always in scope). +``` + +Replace with: + +```typescript + // In local-skill mode: restore the repo as a belt-and-suspenders safety net. + // The agent runs with cwd=dirname(localSkillPath) (the output dir, not the target repo) + // per pi-coding.ts, so rogue writes into the repo are unlikely but not impossible. + // Scope validation is skipped because the local skill file is always in scope by construction. +``` + +- [ ] **Step 3: Run typecheck and tests** + +```bash +npm run typecheck && npm test +``` + +Expected: zero errors, all pass. + +- [ ] **Step 4: Commit** + +```bash +git add src/project/schema.ts src/optimizer/loop.ts +git commit -m "fix(docs): correct apiKeyEnv description and loop.ts isolation comment + +schema.ts: apiKeyEnv default is determined by model provider prefix (openrouter/openai/ +anthropic), not by benchmark.format — a pi-format run with an openai/ model uses +OPENAI_API_KEY. loop.ts: the comment claimed the agent runs with cwd=targetRepo in +local-skill mode; actually cwd is dirname(localSkillPath) per pi-coding.ts." +``` + +--- + +## Task 9: Protect `generateCandidateTasksWithCoverage` retry path + +**Issue fixed:** 12 + +**Files:** +- Modify: `src/tasks/generate.ts:269-275` (function signature area) + +### Background + +`generateCandidateTasksWithCoverage` calls `parseGeneratedTasks(retryRaw)` at line 298 without passing `knownCapabilityKeys`. If a prompt-surface snapshot were ever routed here, all `capabilityId` values from retry tasks would be silently dropped. Currently safe because `tasks/index.ts:73-80` explicitly branches prompt surface to `generateCandidateTasks`. The fix adds a defensive guard at the top of `generateCandidateTasksWithCoverage` and a comment on the retry call. + +- [ ] **Step 1: Add guard and clarifying comment** + +In `src/tasks/generate.ts`, find the `generateCandidateTasksWithCoverage` function (around line 269): + +```typescript +export async function generateCandidateTasksWithCoverage( + surface: DiscoveredTaskSurface, + config: TaskGeneratorConfig, + deps: TaskGeneratorDeps, + inScopeActions: ActionDefinition[], + outOfScopeActions: ActionDefinition[] = [], +): Promise<{ tasks: GeneratedTask[]; coverage: CoverageReport }> { + // Iteration 1 — existing one-shot prompt + const firstPass = await generateCandidateTasks(surface, config, deps); +``` + +Replace with: + +```typescript +export async function generateCandidateTasksWithCoverage( + surface: DiscoveredTaskSurface, + config: TaskGeneratorConfig, + deps: TaskGeneratorDeps, + inScopeActions: ActionDefinition[], + outOfScopeActions: ActionDefinition[] = [], +): Promise<{ tasks: GeneratedTask[]; coverage: CoverageReport }> { + // Prompt surface must not enter this path — it uses generateCandidateTasks directly + // (tasks/index.ts:73-80) because coverage enforcement does not apply and the retry + // path would silently drop capabilityId from tasks (parseGeneratedTasks is called + // without knownCapabilityKeys on line 298). + if (surface.snapshot.surface === 'prompt') { + throw new Error('generateCandidateTasksWithCoverage must not be called for prompt surface — use generateCandidateTasks directly'); + } + + // Iteration 1 — existing one-shot prompt + const firstPass = await generateCandidateTasks(surface, config, deps); +``` + +- [ ] **Step 2: Run typecheck and tests** + +```bash +npm run typecheck && npm test +``` + +Expected: zero errors, all pass. The guard will never fire under normal operation since the caller already branches away. + +- [ ] **Step 3: Commit** + +```bash +git add src/tasks/generate.ts +git commit -m "fix(generate): guard generateCandidateTasksWithCoverage against prompt surface + +The retry path calls parseGeneratedTasks without knownCapabilityKeys, which would +silently drop capabilityId from all retry tasks if prompt surface were ever routed +here. tasks/index.ts already branches away, but the guard makes the invariant +explicit and catches accidental future misuse." +``` + +--- + +## Task 10: Replace brittle string match with structured error in critic + +**Issue fixed:** 13 + +**Files:** +- Modify: `src/tasks/pi-simple-complete.ts` (export `NoTextBlocksError`) +- Modify: `src/tasks/default-pi-critic.ts` (catch by type) + +### Background + +`pi-simple-complete.ts` throws `new Error('Model returned no text blocks...')` when the model returns no text content. `default-pi-critic.ts` catches this by checking `err.message.startsWith('Model returned no text blocks')`. If the error message changes, the critic silently fails to catch it. A named error class makes the contract explicit and type-safe. + +- [ ] **Step 1: Add `NoTextBlocksError` to `src/tasks/pi-simple-complete.ts`** + +At the top of `src/tasks/pi-simple-complete.ts`, after the imports, add: + +```typescript +export class NoTextBlocksError extends Error { + readonly contentTypes: string; + constructor(contentTypes: string) { + super(`Model returned no text blocks${contentTypes ? ` (content types: ${contentTypes})` : ''}`); + this.name = 'NoTextBlocksError'; + this.contentTypes = contentTypes; + } +} +``` + +Then in the same file, find the throw at the end of `piSimpleComplete`: + +```typescript + if (!text) { + const contentTypes = response.content.map((b) => b.type).join(', '); + throw new Error(`Model returned no text blocks${contentTypes ? ` (content types: ${contentTypes})` : ''}`); + } +``` + +Replace with: + +```typescript + if (!text) { + const contentTypes = response.content.map((b) => b.type).join(', '); + throw new NoTextBlocksError(contentTypes); + } +``` + +- [ ] **Step 2: Update `src/tasks/default-pi-critic.ts` to catch by type** + +Add the import at the top of `src/tasks/default-pi-critic.ts`: + +```typescript +import { piSimpleComplete, NoTextBlocksError } from './pi-simple-complete.js'; +``` + +(Replace the existing `import { piSimpleComplete } from './pi-simple-complete.js';`) + +Then find: + +```typescript + if (err instanceof Error && err.message.startsWith('Model returned no text blocks')) { + return '[]'; + } +``` + +Replace with: + +```typescript + if (err instanceof NoTextBlocksError) { + return '[]'; + } +``` + +- [ ] **Step 3: Run typecheck and tests** + +```bash +npm run typecheck && npm test +``` + +Expected: zero errors, all pass. The behaviour is identical — only the detection mechanism changes. + +- [ ] **Step 4: Commit** + +```bash +git add src/tasks/pi-simple-complete.ts src/tasks/default-pi-critic.ts +git commit -m "fix(critic): replace brittle error-string match with NoTextBlocksError class + +pi-simple-complete.ts now throws NoTextBlocksError (a named subclass of Error) +instead of a generic Error with a sentinel message prefix. default-pi-critic.ts +catches by instanceof. The behaviour is unchanged but the contract is explicit +and won't silently break if the message text is ever edited." +``` + +--- + +## Final Verification + +- [ ] **Run the full test suite one last time** + +```bash +npm run typecheck && npm test +``` + +Expected: zero type errors, all smoke tests pass. + +- [ ] **Check help still works** + +```bash +npx tsx src/cli.ts --help +``` + +Expected: prints help with no errors. + +- [ ] **Push and create PR** + +```bash +git push -u origin fix/pr26-review-issues +gh pr create \ + --base development \ + --title "fix: address PR #26 code review findings (13 issues)" \ + --body "$(cat <<'EOF' +## Summary + +Fixes 13 issues found during code review of PR #26 (v1.1.0): + +- **Prompt surface preflight (critical):** `--dry-run` and `doctor` now exempt prompt surfaces from the `maxTasks < inScope` gate, matching the existing exemption in `tasks/index.ts` +- **Doctor error hints:** Discovery-failure and zero-actions hints are now surface-aware for prompt configs (points to `target.skill`, not `discovery.sources`) +- **Init guidance:** `init prompt` next-steps now shows prompt-specific guidance instead of falling through to MCP instructions +- **Wizard validator:** Custom model IDs now accept `anthropic/` and `openai/` prefixes, not just `openrouter/` +- **Task generation:** `validateTask` no longer crashes when the LLM elides `expected_actions: []` for prompt-surface tasks +- **Doc config path:** All user-facing docs now reference `.skill-optimizer/` (dot-prefixed) matching what `scaffold.ts` actually creates +- **Model ID in docs:** Updated stale `gpt-4o` example to `gpt-4o-mini` (in current presets) +- **Snapshot error:** Unsupported-format error now includes `snapshotPath` for easier debugging +- **Prompt metrics:** `toolPrecision` is now set to `1.0` for prompt tasks (was always `0.0`, making reports misleading) +- **Schema description:** `apiKeyEnv` description now correctly says defaults depend on model provider prefix, not `benchmark.format` +- **Loop comment:** Corrected stale comment in `loop.ts` — agent cwd in local-skill mode is the output dir, not the target repo +- **Generate guard:** `generateCandidateTasksWithCoverage` now throws if called with a prompt surface (invariant was implicit, now explicit) +- **Structured error:** `NoTextBlocksError` class replaces brittle `message.startsWith()` check in `default-pi-critic.ts` + +## Test Plan +- [ ] `npm run typecheck` — zero errors +- [ ] `npm test` — all smoke tests pass +- [ ] Manual: `npx tsx src/cli.ts init prompt` — next-steps shows correct guidance +- [ ] Manual: `npx tsx src/cli.ts run --dry-run --config ` with `maxTasks:1`, 12 capabilities — no longer errors + +Closes findings from: https://github.com/fastxyz/skill-optimizer/pull/26#issuecomment-4265764667 +EOF +)" +``` + +--- + +## Self-Review + +**1. Spec coverage:** +- Issue 1 (init.ts next-steps) → Task 2 ✓ +- Issue 2 (wizard validator) → Task 3 ✓ +- Issue 3 (generate task fallback) → Task 4 ✓ +- Issue 4 (maxTasks preflight cli + doctor) → Task 1 ✓ +- Issue 5 (doctor hint) → Task 1 ✓ +- Issue 6 (doc path) → Task 5 ✓ +- Issue 7 (snapshot error) → Task 6 ✓ +- Issue 8 (stale model ID) → Task 5 ✓ +- Issue 9 (toolPrecision) → Task 7 ✓ +- Issue 10 (schema description) → Task 8 ✓ +- Issue 11 (loop comment) → Task 8 ✓ +- Issue 12 (retry guard) → Task 9 ✓ +- Issue 13 (structured error) → Task 10 ✓ + +**2. Placeholder scan:** No TBDs or "handle edge cases" — every step has exact code. + +**3. Type consistency:** `NoTextBlocksError` is defined in Task 10 Step 1 and imported in Task 10 Step 2. All method names match existing codebase patterns. diff --git a/src/benchmark/init.ts b/src/benchmark/init.ts index 0b9464a..dd0fe2a 100644 --- a/src/benchmark/init.ts +++ b/src/benchmark/init.ts @@ -97,6 +97,9 @@ export function initBenchmark(targetDir: string = process.cwd(), surface: 'sdk' console.log(' target.discovery.sources → CLI entry file (for code-first discovery)'); console.log(' .skill-optimizer/cli-commands.json → replace template with your real commands'); console.log(' (cli-commands.json is used as a fallback if code-first discovery finds nothing)'); + } else if (surface === 'prompt') { + console.log(' target.skill → path to your SKILL.md or prompt document'); + console.log(' (no discovery sources needed — capabilities are read directly from the skill file)'); } else { console.log(' target.discovery.sources → MCP server file (for code-first discovery)'); console.log(' .skill-optimizer/tools.json → replace template with your real tools'); diff --git a/src/benchmark/runner.ts b/src/benchmark/runner.ts index 3976ed5..f14cb17 100644 --- a/src/benchmark/runner.ts +++ b/src/benchmark/runner.ts @@ -370,11 +370,13 @@ export async function runBenchmark(options: RunnerOptions = {}): Promise= 0.5; console.log(` [${slug}] Prompt score: ${promptResult.score.toFixed(3)} → ${taskResult.metrics.taskPassed ? 'PASS' : 'FAIL'}`); } @@ -382,6 +384,7 @@ export async function runBenchmark(options: RunnerOptions = {}): Promise { console.log(`Out of scope: ${outOfScope.length} — ${outOfScope.map((a) => a.name).join(', ')}`); const maxTasks = project.benchmark.taskGeneration.maxTasks; - if (project.benchmark.taskGeneration.enabled && inScope.length > 0 && maxTasks < inScope.length) { + if (project.target.surface !== 'prompt' && project.benchmark.taskGeneration.enabled && inScope.length > 0 && maxTasks < inScope.length) { console.error(`\nERROR: maxTasks (${maxTasks}) < in-scope action count (${inScope.length}).`); console.error(`Raise benchmark.taskGeneration.maxTasks in ${project.configPath}, or tighten target.scope.exclude.`); process.exit(1); diff --git a/src/doctor/checks.ts b/src/doctor/checks.ts index bae26f5..4c9ef70 100644 --- a/src/doctor/checks.ts +++ b/src/doctor/checks.ts @@ -13,10 +13,14 @@ export function checkDiscovery(project: ResolvedProjectConfig): Issue[] { try { discovered = discoverActionsOnly(project); } catch (err) { + const isPrompt = project.target.surface === 'prompt'; + const discoveryHint = isPrompt + ? `Check the skill file at target.skill — ensure it has parseable capability headings` + : `Check target.discovery.sources and your manifest file`; issues.push({ code: 'discovery-failed', severity: 'error', field: 'target.discovery', message: `Discovery threw an error: ${err instanceof Error ? err.message : String(err)}`, - hint: `Check target.discovery.sources and your manifest file`, + hint: discoveryHint, fixable: false, }); return issues; @@ -30,6 +34,8 @@ export function checkDiscovery(project: ResolvedProjectConfig): Issue[] { surfaceHint = `Add target.cli.commands pointing at a cli-commands.json manifest, or fix target.discovery.sources`; } else if (project.target.surface === 'mcp') { surfaceHint = `Add target.mcp.tools pointing at a tools.json manifest, or fix target.discovery.sources`; + } else if (project.target.surface === 'prompt') { + surfaceHint = `Ensure the skill file (target.skill) contains parseable capability headings`; } else { surfaceHint = `Fix target.discovery.sources to point at your SDK entry file`; } @@ -41,7 +47,7 @@ export function checkDiscovery(project: ResolvedProjectConfig): Issue[] { }); } else { const maxTasks = project.benchmark.taskGeneration?.maxTasks ?? 0; - if (project.benchmark.taskGeneration?.enabled && maxTasks < inScope.length) { + if (project.target.surface !== 'prompt' && project.benchmark.taskGeneration?.enabled && maxTasks < inScope.length) { issues.push({ code: 'max-tasks-too-low', severity: 'error', field: 'benchmark.taskGeneration.maxTasks', message: `maxTasks (${maxTasks}) is less than the number of in-scope actions (${inScope.length})`, diff --git a/src/init/wizard.ts b/src/init/wizard.ts index 154a452..e4c3e4b 100644 --- a/src/init/wizard.ts +++ b/src/init/wizard.ts @@ -126,7 +126,9 @@ export async function runWizard(cwd: string, preseed?: Partial): placeholder: 'openrouter/provider/model-name', validate: (v) => { if (!v || !v.trim()) return undefined; - if (!v.startsWith('openrouter/')) return 'Must start with openrouter/'; + if (!v.startsWith('openrouter/') && !v.startsWith('anthropic/') && !v.startsWith('openai/')) { + return 'Must start with openrouter/, anthropic/, or openai/'; + } return undefined; }, }) as string); diff --git a/src/optimizer/loop.ts b/src/optimizer/loop.ts index b53289b..2014461 100644 --- a/src/optimizer/loop.ts +++ b/src/optimizer/loop.ts @@ -142,7 +142,8 @@ export async function runOptimizeLoop( `[optimize] Changed files: ${changedFiles.length > 0 ? changedFiles.join(', ') : '(none)'}`, ); // In local-skill mode: restore the repo to undo any rogue writes the agent may have made - // (it runs with cwd=targetRepo), then skip scope validation (the local file is always in scope). + // (the agent cwd is dirname(localSkillPath), not the target repo, so rogue writes are unlikely but not impossible), + // then skip scope validation (the local file is always in scope). if (localSkillPath) { await deps.repo.restoreCheckpoint(resolvedManifest.targetRepo, acceptedCheckpoint); } diff --git a/src/project/schema.ts b/src/project/schema.ts index 5b2f8e4..2e6bf25 100644 --- a/src/project/schema.ts +++ b/src/project/schema.ts @@ -64,7 +64,7 @@ const VerdictConfigSchema = z.object({ const BenchmarkConfigSchema = z.object({ format: z.enum(['pi', 'openai', 'anthropic']).optional().describe('LLM transport format: "pi" routes through OpenRouter/Pi (use openrouter/* or openai/* model refs); "openai" calls the OpenAI API directly (supports Codex auth); "anthropic" calls the Anthropic API directly'), authMode: z.enum(['env', 'codex', 'auto']).optional().describe('How to resolve credentials: env var, ~/.codex/auth.json browser-login tokens, or env-then-codex fallback'), - apiKeyEnv: z.string().optional().describe('Env var name for the API key (default: OPENROUTER_API_KEY for format:pi, OPENAI_API_KEY for format:openai, ANTHROPIC_API_KEY for format:anthropic)'), + apiKeyEnv: z.string().optional().describe('Env var name for the API key (default is determined by the model provider prefix: openrouter/ → OPENROUTER_API_KEY, openai/ → OPENAI_API_KEY, anthropic/ → ANTHROPIC_API_KEY; leave unset to use the per-provider default)'), timeout: z.number().int().positive().optional().describe('Milliseconds per model call (default 240000)'), models: z.array(ModelConfigSchema).describe('Models to benchmark — at least one required'), taskGeneration: TaskGenerationConfigSchema.optional().describe('Automatic task generation config'), diff --git a/src/project/snapshot.ts b/src/project/snapshot.ts index db21ff8..e5ecd5f 100644 --- a/src/project/snapshot.ts +++ b/src/project/snapshot.ts @@ -69,7 +69,7 @@ export function loadSurfaceSnapshotFile(snapshotPath: string): SurfaceSnapshot { && 'actions' in parsed ) { throw new Error( - `Snapshot file format is not supported — delete .skill-optimizer/ and re-run the benchmark to regenerate.`, + `Snapshot file format is not supported: ${snapshotPath} — delete .skill-optimizer/ and re-run the benchmark to regenerate.`, ); } throw new Error(`Invalid surface snapshot file: ${snapshotPath}`); diff --git a/src/tasks/default-pi-critic.ts b/src/tasks/default-pi-critic.ts index 85675f7..ea0e6ab 100644 --- a/src/tasks/default-pi-critic.ts +++ b/src/tasks/default-pi-critic.ts @@ -1,5 +1,5 @@ import type { CriticDeps } from '../verdict/recommendations.js'; -import { piSimpleComplete } from './pi-simple-complete.js'; +import { piSimpleComplete, NoTextBlocksError } from './pi-simple-complete.js'; import type { PiAuthMode } from '../runtime/pi/auth.js'; export interface DefaultPiCriticOptions { @@ -31,7 +31,7 @@ export function createDefaultPiCritic(options: DefaultPiCriticOptions): CriticDe // A model that returns no text blocks is treated as "no recommendations" // rather than a hard failure — the verdict flow continues with an empty list. // Real provider errors (stopReason === 'error') are re-thrown. - if (err instanceof Error && err.message.startsWith('Model returned no text blocks')) { + if (err instanceof NoTextBlocksError) { return '[]'; } throw err; diff --git a/src/tasks/generate.ts b/src/tasks/generate.ts index 25f2586..7b3389b 100644 --- a/src/tasks/generate.ts +++ b/src/tasks/generate.ts @@ -225,6 +225,10 @@ function validateTask(task: unknown, index: number, knownCapabilityKeys?: string throw new Error(`Task ${taskId} must include a non-empty string prompt (received keys: ${received})`); } + if (!rawExpectedActions && knownCapabilityKeys !== undefined) { + rawExpectedActions = []; + } + if (!rawExpectedActions) { const received = JSON.stringify(Object.keys(candidate)); throw new Error(`Task ${taskId} must include an expected_actions array (received keys: ${received})`); @@ -273,6 +277,10 @@ export async function generateCandidateTasksWithCoverage( inScopeActions: ActionDefinition[], outOfScopeActions: ActionDefinition[] = [], ): Promise<{ tasks: GeneratedTask[]; coverage: CoverageReport }> { + if (surface.snapshot.surface === 'prompt') { + throw new Error('generateCandidateTasksWithCoverage must not be called for prompt surface — use generateCandidateTasks directly'); + } + // Iteration 1 — existing one-shot prompt const firstPass = await generateCandidateTasks(surface, config, deps); diff --git a/src/tasks/pi-simple-complete.ts b/src/tasks/pi-simple-complete.ts index 44d9f67..12c72e4 100644 --- a/src/tasks/pi-simple-complete.ts +++ b/src/tasks/pi-simple-complete.ts @@ -28,6 +28,15 @@ interface PiSimpleCompleteInput { prompt: string; } +export class NoTextBlocksError extends Error { + readonly contentTypes: string; + constructor(contentTypes: string) { + super(`Model returned no text blocks${contentTypes ? ` (content types: ${contentTypes})` : ''}`); + this.name = 'NoTextBlocksError'; + this.contentTypes = contentTypes; + } +} + /** * Resolve a Pi model, call completeSimple with a timeout, check for errors, * and return the concatenated text from all text blocks. @@ -74,7 +83,7 @@ export async function piSimpleComplete( if (!text) { const contentTypes = response.content.map((b) => b.type).join(', '); - throw new Error(`Model returned no text blocks${contentTypes ? ` (content types: ${contentTypes})` : ''}`); + throw new NoTextBlocksError(contentTypes); } return text; From efd5af73f72dc9e2299582077424f037cded5ded Mon Sep 17 00:00:00 2001 From: "OpenClaw Agent (basd)" Date: Fri, 17 Apr 2026 19:39:58 +0000 Subject: [PATCH 6/8] docs: regenerate config-schema.md with updated apiKeyEnv description MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reflects that the default API key env var is determined by model provider prefix (openrouter/ → OPENROUTER_API_KEY, etc.), not by benchmark.format. Co-Authored-By: Claude Sonnet 4.6 --- docs/reference/config-schema.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/config-schema.md b/docs/reference/config-schema.md index 0716edb..328a086 100644 --- a/docs/reference/config-schema.md +++ b/docs/reference/config-schema.md @@ -24,7 +24,7 @@ Paths in the config are relative to the config file location. | `target.scope.exclude` | `string[]` | — | Glob patterns for actions to exclude (default []) | | `benchmark.format` | `"pi" | "openai" | "anthropic"` | — | LLM transport format: "pi" routes through OpenRouter/Pi (use openrouter/* or openai/* model refs); "openai" calls the OpenAI API directly (supports Codex auth); "anthropic" calls the Anthropic API directly | | `benchmark.authMode` | `"env" | "codex" | "auto"` | — | How to resolve credentials: env var, ~/.codex/auth.json browser-login tokens, or env-then-codex fallback | -| `benchmark.apiKeyEnv` | `string` | — | Env var name for the API key (default: OPENROUTER_API_KEY for format:pi, OPENAI_API_KEY for format:openai, ANTHROPIC_API_KEY for format:anthropic) | +| `benchmark.apiKeyEnv` | `string` | — | Env var name for the API key (default is determined by the model provider prefix: openrouter/ → OPENROUTER_API_KEY, openai/ → OPENAI_API_KEY, anthropic/ → ANTHROPIC_API_KEY; leave unset to use the per-provider default) | | `benchmark.timeout` | `integer` | — | Milliseconds per model call (default 240000) | | `benchmark.models` | `object[]` | — | Models to benchmark — at least one required | | `benchmark.taskGeneration.enabled` | `boolean` | — | Whether to generate tasks automatically (default false) | From 5da06559f7daeea00c700915606b52cc7ef3d7e8 Mon Sep 17 00:00:00 2001 From: Bucur David <46384319+bucurdavid@users.noreply.github.com> Date: Fri, 1 May 2026 23:04:40 -0700 Subject: [PATCH 7/8] Add Docker skill eval workbench (#40) * feat(workbench): replace benchmark stack with eval workbench * chore(workbench): untrack local eval corpora * feat(workbench): harden eval runner and add examples * feat(workbench): simplify eval outputs and skill docs * feat(workbench): pass through eval credentials * feat(skill): add cross-agent plugin distribution * chore: update package author metadata * fix(codex): add plugin marketplace metadata * docs: refresh agent context files * feat(workbench): add hidden MCP services * fix(plugin): align OpenCode entrypoint * refactor(workbench): remove reference solutions * docs(workbench): add optimization loop * docs(release): refresh Skill Optimizer positioning * style(plugin): normalize OpenCode plugin quotes * fix(workbench): harden env and MCP service handling * fix(workbench): align skill docs with current examples Clarify command-skill eval guidance and keep packaged examples from drifting from the supported workbench schema. * fix(workbench): validate models before runs Validate standalone run-case model refs early and keep partially-started MCP service containers visible to cleanup. --- .agents/plugins/marketplace.json | 22 + .claude-plugin/marketplace.json | 21 + .claude-plugin/plugin.json | 19 + .codex | 0 .codex-plugin/plugin.json | 35 + .codex/INSTALL.md | 31 + .cursor-plugin/plugin.json | 36 + .cursor/INSTALL.md | 15 + .gitignore | 7 +- .opencode/INSTALL.md | 21 + .opencode/plugins/skill-optimizer.js | 25 + AGENTS.md | 47 + CHANGELOG.md | 19 +- CLAUDE.md | 123 +- GEMINI.md | 3 + README.md | 236 +-- SKILL/SKILL.md | 160 -- SKILL/references/benchmark.md | 152 -- SKILL/references/config.md | 193 -- SKILL/references/optimize.md | 101 - SKILL/references/setup.md | 184 -- docker/workbench-runner.Dockerfile | 42 + docs/README.codex.md | 31 + docs/README.opencode.md | 35 + docs/reference/config-schema.md | 46 - docs/reference/errors.md | 226 --- docs/workbench.md | 270 +++ examples/workbench/README.md | 23 + examples/workbench/mcp/README.md | 11 + .../mcp/checks/calculator-answer.mjs | 61 + .../workbench/mcp/mcp/calculator-server.mjs | 120 ++ examples/workbench/mcp/references/README.md | 3 + examples/workbench/mcp/suite.yml | 30 + examples/workbench/pdf/README.md | 36 + examples/workbench/pdf/checks/_pdf.mjs | 305 +++ examples/workbench/pdf/checks/_trace.mjs | 60 + .../pdf/checks/build-briefing-pdf.mjs | 21 + .../pdf/checks/extract-pdf-facts.mjs | 33 + .../workbench/pdf/checks/no-pdf-skill.mjs | 28 + .../pdf/checks/split-customer-packet.mjs | 33 + .../pdf/references/pdf-skill/SKILL.md | 82 + examples/workbench/pdf/suite.yml | 57 + gemini-extension.json | 6 + mock-repos/README.md | 18 - mock-repos/cli-taskfile-demo/README.md | 28 - mock-repos/cli-taskfile-demo/SKILL.md | 37 - .../cli-taskfile-demo/skill-optimizer.json | 33 - mock-repos/cli-taskfile-demo/src/commands.ts | 49 - mock-repos/mcp-tracker-demo/.gitignore | 2 - mock-repos/mcp-tracker-demo/README.md | 31 - mock-repos/mcp-tracker-demo/SKILL.md | 8 - .../mcp-tracker-demo/skill-optimizer.json | 69 - mock-repos/mcp-tracker-demo/src/server.ts | 70 - mock-repos/mcp-tracker-demo/tools.json | 109 -- mock-repos/sdk-counter-demo/README.md | 27 - mock-repos/sdk-counter-demo/SKILL.md | 15 - .../sdk-counter-demo/skill-optimizer.json | 35 - mock-repos/sdk-counter-demo/src/counter.ts | 25 - package-lock.json | 1632 ++++++++++++++++- package.json | 72 +- scripts/gen-docs.ts | 161 -- skills/skill-optimizer/SKILL.md | 210 +++ .../skill-optimizer/references/workbench.md | 533 ++++++ src/actions/diff.ts | 61 - src/actions/discover.ts | 109 -- src/actions/index.ts | 25 - src/actions/loaders.ts | 82 - src/actions/readers/cli.ts | 21 - src/actions/readers/mcp.ts | 14 - src/actions/readers/sdk.ts | 14 - src/actions/snapshot.ts | 211 --- src/actions/types.ts | 30 - src/benchmark/compare.ts | 244 --- src/benchmark/config.ts | 160 -- src/benchmark/coverage.ts | 62 - src/benchmark/evaluator.ts | 549 ------ src/benchmark/extractors/cli-extractor.ts | 348 ---- src/benchmark/extractors/code-analyzer.ts | 342 ---- src/benchmark/extractors/code-extractor.ts | 29 - src/benchmark/extractors/index.ts | 58 - src/benchmark/extractors/mcp-extractor.ts | 18 - src/benchmark/extractors/sdk/parser.ts | 37 - src/benchmark/extractors/sdk/python.ts | 195 -- src/benchmark/extractors/sdk/registry.ts | 19 - src/benchmark/extractors/sdk/rust.ts | 239 --- src/benchmark/extractors/sdk/shared.ts | 28 - src/benchmark/extractors/sdk/types.ts | 12 - src/benchmark/extractors/sdk/typescript.ts | 13 - src/benchmark/index.ts | 28 - src/benchmark/init.ts | 212 --- src/benchmark/llm/anthropic-format.ts | 229 --- src/benchmark/llm/index.ts | 230 --- src/benchmark/llm/openai-format.ts | 206 --- src/benchmark/llm/pi-format.ts | 250 --- src/benchmark/llm/shared.ts | 54 - src/benchmark/llm/tool-name-aliases.ts | 65 - src/benchmark/prompt-criteria.ts | 41 - src/benchmark/prompt-evaluator.ts | 403 ---- src/benchmark/prompts.ts | 125 -- src/benchmark/reporter.ts | 255 --- src/benchmark/runner.ts | 590 ------ src/benchmark/scoring.ts | 89 - src/benchmark/skill-fetcher.ts | 220 --- src/benchmark/types.ts | 330 ---- src/cli.ts | 609 +----- src/discovery/cli.ts | 321 ---- src/discovery/mcp.ts | 335 ---- src/discovery/optique.ts | 315 ---- src/discovery/sdk.ts | 336 ---- src/discovery/types.ts | 30 - src/doctor/checks.ts | 146 -- src/doctor/format.ts | 50 - src/doctor/index.ts | 72 - src/errors.ts | 223 --- src/import/detect.ts | 58 - src/import/extractors/help-scraper.ts | 142 -- src/import/extractors/py-argparse.ts | 90 - src/import/extractors/py-click.ts | 110 -- src/import/extractors/rs-clap.ts | 76 - src/import/extractors/ts-commander.ts | 71 - src/import/extractors/ts-yargs.ts | 83 - src/import/index.ts | 84 - src/import/output.ts | 21 - src/import/types.ts | 26 - src/index.ts | 6 +- src/init/answers.ts | 57 - src/init/detect-project.ts | 159 -- src/init/scaffold.ts | 293 --- src/init/wizard.ts | 190 -- src/optimizer/benchmark-adapter.ts | 35 - src/optimizer/failure-analysis.ts | 43 - src/optimizer/feedback/failure-details.ts | 88 - src/optimizer/feedback/mutation-context.ts | 46 - .../feedback/passing-failing-diff.ts | 41 - src/optimizer/feedback/patterns.ts | 69 - src/optimizer/index.ts | 28 - src/optimizer/ledger.ts | 28 - src/optimizer/loop.ts | 543 ------ src/optimizer/main.ts | 171 -- src/optimizer/manifest.ts | 8 - src/optimizer/materialize-mock-repo.ts | 43 - src/optimizer/mock-repos.ts | 59 - src/optimizer/mutation/git-changes.ts | 23 - src/optimizer/mutation/pi-coding.ts | 216 --- src/optimizer/mutation/skill-writing-guide.ts | 117 -- src/optimizer/progress-table.ts | 107 -- src/optimizer/repo-state.ts | 104 -- src/optimizer/types.ts | 181 -- src/optimizer/validation.ts | 43 - src/project/adapters.ts | 120 -- src/project/discover-prompt.ts | 369 ---- src/project/fix.ts | 45 - src/project/index.ts | 28 - src/project/load.ts | 44 - src/project/resolve.ts | 141 -- src/project/schema.ts | 96 - src/project/snapshot.ts | 144 -- src/project/types.ts | 183 -- src/project/validate.ts | 495 ----- src/runtime/pi/auth.ts | 172 -- src/runtime/pi/coding-orchestrator.ts | 33 - src/runtime/pi/index.ts | 6 - src/runtime/pi/models.ts | 125 -- src/tasks/coverage.ts | 49 - src/tasks/default-pi-critic.ts | 41 - src/tasks/default-pi-generator.ts | 36 - src/tasks/discover.ts | 27 - src/tasks/freeze.ts | 80 - src/tasks/generate.ts | 325 ---- src/tasks/ground.ts | 82 - src/tasks/index.ts | 123 -- src/tasks/pi-simple-complete.ts | 90 - src/tasks/scope.ts | 37 - src/tasks/types.ts | 44 - src/verdict/recommendations.ts | 57 - src/verdict/render.ts | 93 - src/workbench/case-loader.ts | 489 +++++ src/workbench/check-runner.ts | 143 ++ src/workbench/cli-args.ts | 43 + src/workbench/container-runner.ts | 592 ++++++ src/workbench/docker-runner.ts | 620 +++++++ src/workbench/index.ts | 15 + src/workbench/mcp/config.ts | 33 + src/workbench/mcp/index.ts | 1 + src/workbench/metrics.ts | 135 ++ src/workbench/models.ts | 28 + src/workbench/pi-agent.ts | 156 ++ src/workbench/process.ts | 84 + src/workbench/run-case.ts | 222 +++ src/workbench/run-suite.ts | 194 ++ src/workbench/sandbox.ts | 13 + src/workbench/suite-loader.ts | 263 +++ src/workbench/trace.ts | 290 +++ src/workbench/trials.ts | 74 + src/workbench/types.ts | 233 +++ src/workbench/utils.ts | 56 + src/workbench/workspace.ts | 55 + .../import-commands/argparse-sample.py | 13 - tests/fixtures/import-commands/clap-sample.rs | 17 - .../fixtures/import-commands/click-sample.py | 23 - .../import-commands/commander-sample.ts | 23 - .../import-commands/help-output-account.txt | 9 - .../import-commands/help-output-sample.txt | 13 - .../fixtures/import-commands/yargs-sample.ts | 18 - tests/fixtures/sample-skill.md | 72 - tests/smoke-actions.ts | 544 ------ tests/smoke-changelog-coverage.ts | 87 - tests/smoke-cli-entry.ts | 50 - tests/smoke-cli.ts | 313 ---- tests/smoke-code.ts | 1021 ----------- tests/smoke-coverage.ts | 85 - tests/smoke-discovery-cli.ts | 385 ---- tests/smoke-discovery-mcp.ts | 269 --- tests/smoke-discovery-sdk.ts | 344 ---- tests/smoke-dry-run.ts | 77 - tests/smoke-e2e.ts | 221 --- tests/smoke-errors.ts | 144 -- tests/smoke-feedback.ts | 111 -- tests/smoke-gen-docs.ts | 77 - tests/smoke-generation.ts | 695 ------- tests/smoke-import.ts | 201 -- tests/smoke-init.ts | 375 ---- tests/smoke-llm.ts | 1434 --------------- tests/smoke-mcp.ts | 250 --- tests/smoke-mock-repos.ts | 154 -- tests/smoke-model-ids.ts | 85 - tests/smoke-optimize.ts | 1041 ----------- tests/smoke-prompt-criteria.ts | 104 -- tests/smoke-prompt-evaluator.ts | 357 ---- tests/smoke-release.ts | 72 - tests/smoke-scope.ts | 68 - tests/smoke-scoring.ts | 159 -- tests/smoke-sdk-python.ts | 135 -- tests/smoke-sdk-rust.ts | 138 -- tests/smoke-skill-distribution.ts | 168 ++ tests/smoke-snapshot-prompt.ts | 25 - tests/smoke-verdict-prompt.ts | 213 --- tests/smoke-verdict.ts | 117 -- tests/smoke-workbench-case.ts | 462 +++++ tests/smoke-workbench-checks.ts | 172 ++ tests/smoke-workbench-container.ts | 241 +++ tests/smoke-workbench-docker-runner.ts | 433 +++++ tests/smoke-workbench-metrics.ts | 75 + tests/smoke-workbench-models.ts | 201 ++ tests/smoke-workbench-pi-agent.ts | 159 ++ tests/smoke-workbench-run-case.ts | 60 + tests/smoke-workbench-suite.ts | 403 ++++ tests/smoke-workbench-trace.ts | 176 ++ tests/smoke-workbench-trials.ts | 109 ++ 249 files changed, 10463 insertions(+), 26875 deletions(-) create mode 100644 .agents/plugins/marketplace.json create mode 100644 .claude-plugin/marketplace.json create mode 100644 .claude-plugin/plugin.json delete mode 100644 .codex create mode 100644 .codex-plugin/plugin.json create mode 100644 .codex/INSTALL.md create mode 100644 .cursor-plugin/plugin.json create mode 100644 .cursor/INSTALL.md create mode 100644 .opencode/INSTALL.md create mode 100644 .opencode/plugins/skill-optimizer.js create mode 100644 AGENTS.md create mode 100644 GEMINI.md delete mode 100644 SKILL/SKILL.md delete mode 100644 SKILL/references/benchmark.md delete mode 100644 SKILL/references/config.md delete mode 100644 SKILL/references/optimize.md delete mode 100644 SKILL/references/setup.md create mode 100644 docker/workbench-runner.Dockerfile create mode 100644 docs/README.codex.md create mode 100644 docs/README.opencode.md delete mode 100644 docs/reference/config-schema.md delete mode 100644 docs/reference/errors.md create mode 100644 docs/workbench.md create mode 100644 examples/workbench/README.md create mode 100644 examples/workbench/mcp/README.md create mode 100644 examples/workbench/mcp/checks/calculator-answer.mjs create mode 100644 examples/workbench/mcp/mcp/calculator-server.mjs create mode 100644 examples/workbench/mcp/references/README.md create mode 100644 examples/workbench/mcp/suite.yml create mode 100644 examples/workbench/pdf/README.md create mode 100644 examples/workbench/pdf/checks/_pdf.mjs create mode 100644 examples/workbench/pdf/checks/_trace.mjs create mode 100644 examples/workbench/pdf/checks/build-briefing-pdf.mjs create mode 100644 examples/workbench/pdf/checks/extract-pdf-facts.mjs create mode 100644 examples/workbench/pdf/checks/no-pdf-skill.mjs create mode 100644 examples/workbench/pdf/checks/split-customer-packet.mjs create mode 100644 examples/workbench/pdf/references/pdf-skill/SKILL.md create mode 100644 examples/workbench/pdf/suite.yml create mode 100644 gemini-extension.json delete mode 100644 mock-repos/README.md delete mode 100644 mock-repos/cli-taskfile-demo/README.md delete mode 100644 mock-repos/cli-taskfile-demo/SKILL.md delete mode 100644 mock-repos/cli-taskfile-demo/skill-optimizer.json delete mode 100644 mock-repos/cli-taskfile-demo/src/commands.ts delete mode 100644 mock-repos/mcp-tracker-demo/.gitignore delete mode 100644 mock-repos/mcp-tracker-demo/README.md delete mode 100644 mock-repos/mcp-tracker-demo/SKILL.md delete mode 100644 mock-repos/mcp-tracker-demo/skill-optimizer.json delete mode 100644 mock-repos/mcp-tracker-demo/src/server.ts delete mode 100644 mock-repos/mcp-tracker-demo/tools.json delete mode 100644 mock-repos/sdk-counter-demo/README.md delete mode 100644 mock-repos/sdk-counter-demo/SKILL.md delete mode 100644 mock-repos/sdk-counter-demo/skill-optimizer.json delete mode 100644 mock-repos/sdk-counter-demo/src/counter.ts delete mode 100644 scripts/gen-docs.ts create mode 100644 skills/skill-optimizer/SKILL.md create mode 100644 skills/skill-optimizer/references/workbench.md delete mode 100644 src/actions/diff.ts delete mode 100644 src/actions/discover.ts delete mode 100644 src/actions/index.ts delete mode 100644 src/actions/loaders.ts delete mode 100644 src/actions/readers/cli.ts delete mode 100644 src/actions/readers/mcp.ts delete mode 100644 src/actions/readers/sdk.ts delete mode 100644 src/actions/snapshot.ts delete mode 100644 src/actions/types.ts delete mode 100644 src/benchmark/compare.ts delete mode 100644 src/benchmark/config.ts delete mode 100644 src/benchmark/coverage.ts delete mode 100644 src/benchmark/evaluator.ts delete mode 100644 src/benchmark/extractors/cli-extractor.ts delete mode 100644 src/benchmark/extractors/code-analyzer.ts delete mode 100644 src/benchmark/extractors/code-extractor.ts delete mode 100644 src/benchmark/extractors/index.ts delete mode 100644 src/benchmark/extractors/mcp-extractor.ts delete mode 100644 src/benchmark/extractors/sdk/parser.ts delete mode 100644 src/benchmark/extractors/sdk/python.ts delete mode 100644 src/benchmark/extractors/sdk/registry.ts delete mode 100644 src/benchmark/extractors/sdk/rust.ts delete mode 100644 src/benchmark/extractors/sdk/shared.ts delete mode 100644 src/benchmark/extractors/sdk/types.ts delete mode 100644 src/benchmark/extractors/sdk/typescript.ts delete mode 100644 src/benchmark/index.ts delete mode 100644 src/benchmark/init.ts delete mode 100644 src/benchmark/llm/anthropic-format.ts delete mode 100644 src/benchmark/llm/index.ts delete mode 100644 src/benchmark/llm/openai-format.ts delete mode 100644 src/benchmark/llm/pi-format.ts delete mode 100644 src/benchmark/llm/shared.ts delete mode 100644 src/benchmark/llm/tool-name-aliases.ts delete mode 100644 src/benchmark/prompt-criteria.ts delete mode 100644 src/benchmark/prompt-evaluator.ts delete mode 100644 src/benchmark/prompts.ts delete mode 100644 src/benchmark/reporter.ts delete mode 100644 src/benchmark/runner.ts delete mode 100644 src/benchmark/scoring.ts delete mode 100644 src/benchmark/skill-fetcher.ts delete mode 100644 src/benchmark/types.ts delete mode 100644 src/discovery/cli.ts delete mode 100644 src/discovery/mcp.ts delete mode 100644 src/discovery/optique.ts delete mode 100644 src/discovery/sdk.ts delete mode 100644 src/discovery/types.ts delete mode 100644 src/doctor/checks.ts delete mode 100644 src/doctor/format.ts delete mode 100644 src/doctor/index.ts delete mode 100644 src/errors.ts delete mode 100644 src/import/detect.ts delete mode 100644 src/import/extractors/help-scraper.ts delete mode 100644 src/import/extractors/py-argparse.ts delete mode 100644 src/import/extractors/py-click.ts delete mode 100644 src/import/extractors/rs-clap.ts delete mode 100644 src/import/extractors/ts-commander.ts delete mode 100644 src/import/extractors/ts-yargs.ts delete mode 100644 src/import/index.ts delete mode 100644 src/import/output.ts delete mode 100644 src/import/types.ts delete mode 100644 src/init/answers.ts delete mode 100644 src/init/detect-project.ts delete mode 100644 src/init/scaffold.ts delete mode 100644 src/init/wizard.ts delete mode 100644 src/optimizer/benchmark-adapter.ts delete mode 100644 src/optimizer/failure-analysis.ts delete mode 100644 src/optimizer/feedback/failure-details.ts delete mode 100644 src/optimizer/feedback/mutation-context.ts delete mode 100644 src/optimizer/feedback/passing-failing-diff.ts delete mode 100644 src/optimizer/feedback/patterns.ts delete mode 100644 src/optimizer/index.ts delete mode 100644 src/optimizer/ledger.ts delete mode 100644 src/optimizer/loop.ts delete mode 100644 src/optimizer/main.ts delete mode 100644 src/optimizer/manifest.ts delete mode 100644 src/optimizer/materialize-mock-repo.ts delete mode 100644 src/optimizer/mock-repos.ts delete mode 100644 src/optimizer/mutation/git-changes.ts delete mode 100644 src/optimizer/mutation/pi-coding.ts delete mode 100644 src/optimizer/mutation/skill-writing-guide.ts delete mode 100644 src/optimizer/progress-table.ts delete mode 100644 src/optimizer/repo-state.ts delete mode 100644 src/optimizer/types.ts delete mode 100644 src/optimizer/validation.ts delete mode 100644 src/project/adapters.ts delete mode 100644 src/project/discover-prompt.ts delete mode 100644 src/project/fix.ts delete mode 100644 src/project/index.ts delete mode 100644 src/project/load.ts delete mode 100644 src/project/resolve.ts delete mode 100644 src/project/schema.ts delete mode 100644 src/project/snapshot.ts delete mode 100644 src/project/types.ts delete mode 100644 src/project/validate.ts delete mode 100644 src/runtime/pi/auth.ts delete mode 100644 src/runtime/pi/coding-orchestrator.ts delete mode 100644 src/runtime/pi/index.ts delete mode 100644 src/runtime/pi/models.ts delete mode 100644 src/tasks/coverage.ts delete mode 100644 src/tasks/default-pi-critic.ts delete mode 100644 src/tasks/default-pi-generator.ts delete mode 100644 src/tasks/discover.ts delete mode 100644 src/tasks/freeze.ts delete mode 100644 src/tasks/generate.ts delete mode 100644 src/tasks/ground.ts delete mode 100644 src/tasks/index.ts delete mode 100644 src/tasks/pi-simple-complete.ts delete mode 100644 src/tasks/scope.ts delete mode 100644 src/tasks/types.ts delete mode 100644 src/verdict/recommendations.ts delete mode 100644 src/verdict/render.ts create mode 100644 src/workbench/case-loader.ts create mode 100644 src/workbench/check-runner.ts create mode 100644 src/workbench/cli-args.ts create mode 100644 src/workbench/container-runner.ts create mode 100644 src/workbench/docker-runner.ts create mode 100644 src/workbench/index.ts create mode 100644 src/workbench/mcp/config.ts create mode 100644 src/workbench/mcp/index.ts create mode 100644 src/workbench/metrics.ts create mode 100644 src/workbench/models.ts create mode 100644 src/workbench/pi-agent.ts create mode 100644 src/workbench/process.ts create mode 100644 src/workbench/run-case.ts create mode 100644 src/workbench/run-suite.ts create mode 100644 src/workbench/sandbox.ts create mode 100644 src/workbench/suite-loader.ts create mode 100644 src/workbench/trace.ts create mode 100644 src/workbench/trials.ts create mode 100644 src/workbench/types.ts create mode 100644 src/workbench/utils.ts create mode 100644 src/workbench/workspace.ts delete mode 100644 tests/fixtures/import-commands/argparse-sample.py delete mode 100644 tests/fixtures/import-commands/clap-sample.rs delete mode 100644 tests/fixtures/import-commands/click-sample.py delete mode 100644 tests/fixtures/import-commands/commander-sample.ts delete mode 100644 tests/fixtures/import-commands/help-output-account.txt delete mode 100644 tests/fixtures/import-commands/help-output-sample.txt delete mode 100644 tests/fixtures/import-commands/yargs-sample.ts delete mode 100644 tests/fixtures/sample-skill.md delete mode 100644 tests/smoke-actions.ts delete mode 100644 tests/smoke-changelog-coverage.ts delete mode 100644 tests/smoke-cli-entry.ts delete mode 100644 tests/smoke-cli.ts delete mode 100644 tests/smoke-code.ts delete mode 100644 tests/smoke-coverage.ts delete mode 100644 tests/smoke-discovery-cli.ts delete mode 100644 tests/smoke-discovery-mcp.ts delete mode 100644 tests/smoke-discovery-sdk.ts delete mode 100644 tests/smoke-dry-run.ts delete mode 100644 tests/smoke-e2e.ts delete mode 100644 tests/smoke-errors.ts delete mode 100644 tests/smoke-feedback.ts delete mode 100644 tests/smoke-gen-docs.ts delete mode 100644 tests/smoke-generation.ts delete mode 100644 tests/smoke-import.ts delete mode 100644 tests/smoke-init.ts delete mode 100644 tests/smoke-llm.ts delete mode 100644 tests/smoke-mcp.ts delete mode 100644 tests/smoke-mock-repos.ts delete mode 100644 tests/smoke-model-ids.ts delete mode 100644 tests/smoke-optimize.ts delete mode 100644 tests/smoke-prompt-criteria.ts delete mode 100644 tests/smoke-prompt-evaluator.ts delete mode 100644 tests/smoke-release.ts delete mode 100644 tests/smoke-scope.ts delete mode 100644 tests/smoke-scoring.ts delete mode 100644 tests/smoke-sdk-python.ts delete mode 100644 tests/smoke-sdk-rust.ts create mode 100644 tests/smoke-skill-distribution.ts delete mode 100644 tests/smoke-snapshot-prompt.ts delete mode 100644 tests/smoke-verdict-prompt.ts delete mode 100644 tests/smoke-verdict.ts create mode 100644 tests/smoke-workbench-case.ts create mode 100644 tests/smoke-workbench-checks.ts create mode 100644 tests/smoke-workbench-container.ts create mode 100644 tests/smoke-workbench-docker-runner.ts create mode 100644 tests/smoke-workbench-metrics.ts create mode 100644 tests/smoke-workbench-models.ts create mode 100644 tests/smoke-workbench-pi-agent.ts create mode 100644 tests/smoke-workbench-run-case.ts create mode 100644 tests/smoke-workbench-suite.ts create mode 100644 tests/smoke-workbench-trace.ts create mode 100644 tests/smoke-workbench-trials.ts diff --git a/.agents/plugins/marketplace.json b/.agents/plugins/marketplace.json new file mode 100644 index 0000000..133fe03 --- /dev/null +++ b/.agents/plugins/marketplace.json @@ -0,0 +1,22 @@ +{ + "name": "skill-optimizer", + "interface": { + "displayName": "Skill Optimizer", + "shortDescription": "Build, run, and improve evals for agent skills.", + "longDescription": "Create realistic skill eval suites, run them across models, inspect traces and grader evidence, then iterate on the skill or supporting code until behavior improves." + }, + "plugins": [ + { + "name": "skill-optimizer", + "source": { + "source": "local", + "path": "./" + }, + "policy": { + "installation": "AVAILABLE", + "authentication": "ON_INSTALL" + }, + "category": "Coding" + } + ] +} diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 0000000..ad43fec --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,21 @@ +{ + "name": "skill-optimizer", + "description": "Skill eval lab for testing and improving agent skills.", + "owner": { + "name": "Fast" + }, + "plugins": [ + { + "name": "skill-optimizer", + "description": "Build, run, and improve evals for agent skills.", + "version": "2.0.0", + "source": "./", + "author": { + "name": "Fast" + }, + "skills": [ + "./skills/skill-optimizer" + ] + } + ] +} diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 0000000..f8c5110 --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,19 @@ +{ + "name": "skill-optimizer", + "description": "Build, run, and improve evals for agent skills.", + "version": "2.0.0", + "author": { + "name": "Fast" + }, + "homepage": "https://github.com/fastxyz/skill-optimizer#readme", + "repository": "https://github.com/fastxyz/skill-optimizer", + "license": "MIT", + "keywords": [ + "agent-skills", + "evals", + "skill-testing", + "model-evaluation", + "optimization" + ], + "skills": "./skills/" +} diff --git a/.codex b/.codex deleted file mode 100644 index e69de29..0000000 diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json new file mode 100644 index 0000000..a43e05b --- /dev/null +++ b/.codex-plugin/plugin.json @@ -0,0 +1,35 @@ +{ + "name": "skill-optimizer", + "version": "2.0.0", + "description": "Build, run, and improve evals for agent skills.", + "author": { + "name": "Fast" + }, + "homepage": "https://github.com/fastxyz/skill-optimizer#readme", + "repository": "https://github.com/fastxyz/skill-optimizer", + "license": "MIT", + "keywords": [ + "agent-skills", + "evals", + "skill-testing", + "model-evaluation", + "optimization" + ], + "skills": "./skills/", + "interface": { + "displayName": "Skill Optimizer", + "shortDescription": "Build, run, and improve evals for agent skills.", + "longDescription": "Create realistic skill eval suites, run them across models, inspect traces and grader evidence, then iterate on the skill or supporting code until behavior improves.", + "developerName": "Fast", + "category": "Coding", + "capabilities": [ + "Read", + "Write" + ], + "defaultPrompt": [ + "Use Skill Optimizer to design and run an eval suite for this agent skill.", + "Use Skill Optimizer to inspect failing traces and improve the skill." + ], + "brandColor": "#3B82F6" + } +} diff --git a/.codex/INSTALL.md b/.codex/INSTALL.md new file mode 100644 index 0000000..489e8ee --- /dev/null +++ b/.codex/INSTALL.md @@ -0,0 +1,31 @@ +# Installing skill-optimizer for Codex + +Use `skill-optimizer` in Codex as either a plugin or a native skill. + +## Plugin Install + +Register this repository as a plugin marketplace: + +```bash +codex plugin marketplace add fastxyz/skill-optimizer +``` + +Open `/plugins`, select the `skill-optimizer` marketplace, and install the `skill-optimizer` plugin. + +The marketplace file is `.agents/plugins/marketplace.json`. It exposes the repository root as the plugin source so Codex can load `.codex-plugin/plugin.json` and the bundled `skills/` directory. + +To pin a Git ref while installing the marketplace: + +```bash +codex plugin marketplace add fastxyz/skill-optimizer --ref main +``` + +## Skill-Only Install + +Install the canonical skill with the open skills CLI: + +```bash +npx skills add fastxyz/skill-optimizer --skill skill-optimizer -a codex -y +``` + +Restart Codex if the skill does not appear immediately. diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json new file mode 100644 index 0000000..8a619ef --- /dev/null +++ b/.cursor-plugin/plugin.json @@ -0,0 +1,36 @@ +{ + "name": "skill-optimizer", + "displayName": "Skill Optimizer", + "description": "Build, run, and improve evals for agent skills.", + "version": "2.0.0", + "author": { + "name": "Fast" + }, + "homepage": "https://github.com/fastxyz/skill-optimizer#readme", + "repository": "https://github.com/fastxyz/skill-optimizer", + "license": "MIT", + "keywords": [ + "agent-skills", + "evals", + "skill-testing", + "model-evaluation", + "optimization" + ], + "skills": "./skills/", + "interface": { + "displayName": "Skill Optimizer", + "shortDescription": "Build, run, and improve evals for agent skills.", + "longDescription": "Create realistic skill eval suites, run them across models, inspect traces and grader evidence, then iterate on the skill or supporting code until behavior improves.", + "developerName": "Fast", + "category": "Coding", + "capabilities": [ + "Read", + "Write" + ], + "defaultPrompt": [ + "Use Skill Optimizer to design and run an eval suite for this agent skill.", + "Use Skill Optimizer to inspect failing traces and improve the skill." + ], + "brandColor": "#3B82F6" + } +} diff --git a/.cursor/INSTALL.md b/.cursor/INSTALL.md new file mode 100644 index 0000000..c6d8de6 --- /dev/null +++ b/.cursor/INSTALL.md @@ -0,0 +1,15 @@ +# Installing skill-optimizer for Cursor + +## Skill install + +Install the skill into Cursor's project or global skill directory through the open skills CLI: + +```bash +npx skills add fastxyz/skill-optimizer --skill skill-optimizer -a cursor -y +``` + +Cursor can also import remote skills from GitHub in Settings -> Rules -> Project Rules -> Add Rule -> Remote Rule (Github). + +## Plugin metadata + +This repository includes `.cursor-plugin/plugin.json` for Cursor-compatible plugin metadata. The canonical skill remains `skills/skill-optimizer/SKILL.md`. diff --git a/.gitignore b/.gitignore index 1a93e1c..955a362 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,10 @@ dist/ # Cache directories .cache/ -.opencode/ +.opencode/* +!.opencode/INSTALL.md +!.opencode/plugins/ +!.opencode/plugins/skill-optimizer.js # Test results and benchmarks results/ @@ -55,6 +58,8 @@ docs/specs/ # Skill-optimizer generated artifacts .skill-optimizer/ +.skill-eval/ +.results/ # Local user config (personal paths, model choices — not repo artifacts) skill-optimizer.json diff --git a/.opencode/INSTALL.md b/.opencode/INSTALL.md new file mode 100644 index 0000000..c93409a --- /dev/null +++ b/.opencode/INSTALL.md @@ -0,0 +1,21 @@ +# Installing skill-optimizer for OpenCode + +Add the plugin to `opencode.json` at user or project scope: + +```json +{ + "plugin": ["skill-optimizer@git+https://github.com/fastxyz/skill-optimizer.git"] +} +``` + +Restart OpenCode. The plugin registers the repository `skills/` directory so the native `skill` tool can load `skill-optimizer`. + +Verify with the skill tool by listing skills or loading `skill-optimizer`. + +To pin a version, append a tag or commit ref: + +```json +{ + "plugin": ["skill-optimizer@git+https://github.com/fastxyz/skill-optimizer.git#v2.0.0"] +} +``` diff --git a/.opencode/plugins/skill-optimizer.js b/.opencode/plugins/skill-optimizer.js new file mode 100644 index 0000000..f733c67 --- /dev/null +++ b/.opencode/plugins/skill-optimizer.js @@ -0,0 +1,25 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const pluginDir = path.dirname(fileURLToPath(import.meta.url)); +const skillsDir = path.resolve(pluginDir, "..", "..", "skills"); + +function registerSkillsDir(config) { + config.skills = config.skills || {}; + config.skills.paths = config.skills.paths || []; + + if (!config.skills.paths.includes(skillsDir)) { + config.skills.paths.push(skillsDir); + } +} + +export const SkillOptimizerPlugin = async () => ({ + config: async (config) => { + registerSkillsDir(config); + }, +}); + +export default { + id: "skill-optimizer", + server: SkillOptimizerPlugin, +}; diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..3b4c530 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,47 @@ +# AGENTS.md + +## Project Overview + +`skill-optimizer` is a Docker workbench for running and grading agent skill eval cases. The current public CLI centers on `run-case` and `run-suite`. + +## Key Commands + +```bash +npm run build +npm run typecheck +npm test +npx tsx src/cli.ts --help +npx tsx src/cli.ts run-case --help +npx tsx src/cli.ts run-suite --help +``` + +## Important Files + +- `src/cli.ts`: public CLI entrypoint +- `src/workbench/`: workbench case loading, suite loading, Docker runner, Pi agent, graders, and traces +- `docker/workbench-runner.Dockerfile`: generic non-root container image for setup, agent, grade, and cleanup phases +- `skills/skill-optimizer/SKILL.md`: canonical distributable Agent Skill +- `.claude-plugin/`, `.codex-plugin/`, `.cursor-plugin/`, `.opencode/`: cross-agent plugin manifests and install support +- `.agents/plugins/marketplace.json`: Codex repo marketplace entry for the root plugin +- `gemini-extension.json`, `GEMINI.md`: Gemini extension metadata and context file +- `examples/workbench/`: tracked example eval suites + +## Invariants + +- Keep evaluation static: extraction and matching are allowed; do not execute model-produced code outside the Docker workbench as part of evaluation. +- `run-suite` uses models from `suite.yml`; do not add a `run-suite --models` override. +- Keep OpenRouter model refs as `openrouter/...`; real model runs require `OPENROUTER_API_KEY`. +- Cases use `graders: [{ name, command }]`; legacy `check:` and `artifacts:` are invalid. +- Graders are the acceptance contract; evaluate outputs from `/work`, generated artifacts, `answer.json`, `trace.jsonl`, and result state. +- The agent phase sees only `/work`, not `/case` or `/results`. +- Keep plugin metadata pointed at the canonical `skills/skill-optimizer/SKILL.md`; do not create divergent skill copies. +- Codex plugin metadata lives in `.codex-plugin/plugin.json`; the repo marketplace lives in `.agents/plugins/marketplace.json` and points at `./`. +- Do not commit `.skill-eval/`, `.results/`, `.env`, or credentials. + +## Testing Guidance + +- Run `npm run typecheck` after TypeScript changes. +- Run `npm test` before finishing behavior changes. +- For Docker runner or image changes, also run `docker build -t skill-optimizer-workbench:local -f docker/workbench-runner.Dockerfile .`. +- For CLI/docs changes, verify `npx tsx src/cli.ts --help` if touched docs mention CLI behavior. +- For plugin/package metadata changes, run `npx tsx tests/smoke-skill-distribution.ts` and verify `npm pack --dry-run --json` includes required plugin files without result/cache directories. diff --git a/CHANGELOG.md b/CHANGELOG.md index d52754a..5dc20ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## 2.0.0 — 2026-05-01 + +### Changed + +- Rebuilt Skill Optimizer around the eval workbench: realistic skill cases, model matrices, isolated agent workspaces, trace inspection, deterministic grader evidence, and iterative skill improvement. +- Repositioned package and plugin metadata around the skill eval lab workflow instead of implementation mechanics. + +### Breaking Changes + +- Removed the legacy reference-solution preflight flow and `verify-suite`; graders are now the sole acceptance contract. +- Removed reference-solution SDK exports and packaged example solution scripts. + +### Added + +- Hidden MCP services for eval cases, exposed to agents through the workbench `mcp` command. +- Post-run optimization guidance for inspecting failures, updating skills or supporting code, and re-running evals. + ## 1.1.0 — 2026-04-16 ### Breaking Changes @@ -25,7 +42,7 @@ The config file `skill-benchmark.json` is no longer auto-detected. Rename it to ### Added - **prompt surface type** — benchmark and optimize prompt templates, Claude Code skills, and agent instructions. Discovers phases and capabilities from markdown, evaluates output quality with content-based criteria. - **Codex auth** — direct OpenAI model runs can use browser-login tokens stored by Codex (`~/.codex/auth.json`) instead of requiring `OPENAI_API_KEY`. Set `benchmark.authMode: "codex"` and use `openai/` IDs. -- **SKILL folder** — bundled AI-agent guidance (`SKILL/SKILL.md`) so agents can use skill-optimizer reliably without extra setup. +- **skills folder** — bundled AI-agent guidance (`skills/skill-optimizer/SKILL.md`) so agents can use skill-optimizer reliably without extra setup. - **Optimizer loop diagram** — README now includes a visual workflow diagram of the optimizer loop. - **Stable task IDs** — task IDs are now derived from a SHA-1 hash of the action names (SDK/CLI/MCP surfaces) or prompt text (prompt surface). For SDK/CLI/MCP surfaces, where action names come from discovered code rather than LLM output, IDs are stable across regenerations and the `--task ` filter works reliably. For the prompt surface, IDs are stable when the LLM produces identical wording; if it rephrases a task the ID changes (fixes [#17](https://github.com/fastxyz/skill-optimizer/issues/17)). diff --git a/CLAUDE.md b/CLAUDE.md index f5fd890..c917490 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,17 +2,9 @@ ## Project Overview -`skill-optimizer` measures whether LLMs pick the right SDK methods, CLI commands, or MCP tools from docs and task prompts, and can run a benchmark-driven optimization loop over an allowed target repo. +`skill-optimizer` is a Docker workbench for running and grading agent skill eval cases. The current public CLI centers on `run-case` and `run-suite`. -The repo has five layers: - -- `src/project/`: unified `skill-optimizer.json` config loading, validation, and path resolution -- `src/runtime/pi/`: shared Pi auth/model/runtime helpers -- `src/tasks/`: shared task generation, grounding, and artifact freezing from discovered surfaces -- `src/benchmark/`: loads tasks and surface definitions, builds prompts, calls models, extracts actions, evaluates them, and writes reports -- `src/optimizer/`: runs a benchmark-driven optimization loop against a constrained target repo - -The benchmark is static. Do not change behavior in ways that execute model-produced code or shell commands as part of evaluation. +The workbench gives an agent an isolated Docker `/work` directory, captures traces, and grades deterministic local outcomes from files, command logs, generated artifacts, or other workspace state. ## Key Commands @@ -21,101 +13,38 @@ npm run build npm run typecheck npm test npx tsx src/cli.ts --help -npx tsx src/cli.ts generate-tasks --help -npx tsx src/cli.ts optimize --help -``` - -Typical benchmark run: - -```bash -export OPENROUTER_API_KEY=... -npx tsx src/cli.ts run --config ./skill-optimizer.json -``` - -Generate tasks only: - -```bash -npx tsx src/cli.ts generate-tasks --config ./skill-optimizer.json -``` - -Typical optimizer run: - -```bash -tsx src/optimizer/materialize-mock-repo.ts mcp-tracker-demo ./.tmp/mock-repos -npx tsx src/cli.ts optimize --config ./.tmp/mock-repos/mcp-tracker-demo/skill-optimizer.json +npx tsx src/cli.ts run-case --help +npx tsx src/cli.ts run-suite --help ``` ## Important Files -- `src/cli.ts`: public CLI entrypoint (`init`, `run`, `optimize`, `compare`) -- `src/project/types.ts`: unified public project config types -- `src/project/load.ts`: unified `skill-optimizer.json` loader -- `src/runtime/pi/models.ts`: shared Pi model/auth resolution -- `src/tasks/index.ts`: shared task generation entrypoint over discovered surfaces -- `src/benchmark/runner.ts`: orchestration for benchmark execution -- `src/benchmark/types.ts`: benchmark report, metric, and extraction types -- `src/benchmark/init.ts`: scaffolded starter `skill-optimizer.json` -- `src/optimizer/loop.ts`: accept/reject iteration loop -- `src/optimizer/manifest.ts`: adapter from unified project config into the current optimizer loop -- `src/optimizer/mock-repos.ts`: tracked template materialization and isolated git init -- `mock-repos/mcp-tracker-demo/`: current richer demo target for optimizer testing +- `src/cli.ts`: public CLI entrypoint +- `src/workbench/`: workbench case loading, suite loading, Docker runner, Pi agent, graders, and traces +- `docker/workbench-runner.Dockerfile`: generic non-root container image for setup, agent, grade, and cleanup phases +- `skills/skill-optimizer/SKILL.md`: canonical distributable Agent Skill +- `skills/skill-optimizer/references/workbench.md`: detailed workbench schema and usage reference +- `.claude-plugin/`, `.codex-plugin/`, `.cursor-plugin/`, `.opencode/`: cross-agent plugin manifests and install support +- `.agents/plugins/marketplace.json`: Codex repo marketplace entry for the root plugin +- `gemini-extension.json`, `GEMINI.md`: Gemini extension metadata and context file +- `examples/workbench/`: tracked example eval suites ## Invariants -- Keep benchmark evaluation static. Extraction and matching are allowed; executing generated code is not. -- Keep path resolution relative to the unified config file being loaded. -- `targetRepo.allowedPaths` is the optimizer safety boundary. Do not widen edits outside it during mutation. -- `requireCleanGit` must remain effectively enforced for optimizer targets. -- Optimizer-owned artifacts under the configured task-generation output dir must not be treated as target-repo mutations. -- **The target repo's skill file is never modified.** The optimizer copies it to `.skill-optimizer/skill-v0.md` on start and creates versioned copies per accepted iteration. The mutation agent writes to these local copies; `skillOverride` makes the benchmark read from them. -- Stable-surface optimize runs assume the callable surface is frozen for the duration of the run. If a change renames commands/tools/APIs, the surface must be rediscovered and the benchmark snapshot regenerated before further comparisons are meaningful. -- Materialized mock repos must stay isolated from tracked templates. -- Documentation examples should match the current CLI and config schema. - -## Editing Guidance - -- Prefer small changes in the existing architecture over broad refactors. -- When updating config or project types, also update the README examples and any scaffolding in `src/benchmark/init.ts` if needed. -- When changing optimizer behavior, verify both the loop and the unified project defaults still agree. -- Code-first surface discovery is the preferred mode for `sdk`, `cli`, and `mcp` via `target.discovery.sources`. Explicit manifest files (`target.cli.commands`, `target.mcp.tools`, `target.discovery.fallbackManifest`) are supported for projects that cannot use code-first discovery. -- Be careful around mock repo references: code may support template names that are not currently present in the working tree. +- Keep evaluation static: extraction and matching are allowed; do not execute model-produced code outside the Docker workbench as part of evaluation. +- `run-suite` uses models from `suite.yml`; do not add a `run-suite --models` override. +- Keep OpenRouter model refs as `openrouter/...`; real model runs require `OPENROUTER_API_KEY`. +- Cases use `graders: [{ name, command }]`; legacy `check:` and `artifacts:` are invalid. +- Graders are the acceptance contract; evaluate outputs from `/work`, generated artifacts, `answer.json`, `trace.jsonl`, and result state. +- The agent phase sees only `/work`, not `/case` or `/results`. +- Keep plugin metadata pointed at the canonical `skills/skill-optimizer/SKILL.md`; do not create divergent skill copies. +- Codex plugin metadata lives in `.codex-plugin/plugin.json`; the repo marketplace lives in `.agents/plugins/marketplace.json` and points at `./`. +- Do not commit `.skill-eval/`, `.results/`, `.env`, or credentials. ## Testing Guidance - Run `npm run typecheck` after TypeScript changes. -- Run `npm test` before finishing when behavior changes may affect extraction, evaluation, reporting, or optimizer flow. -- For CLI-only or docs-only changes, at minimum verify `npx tsx src/cli.ts --help` still works if the touched docs reference CLI behavior. - -## Model ID Convention - -Model IDs use a provider-prefixed format. The prefix determines how the request is routed: - -``` -openrouter// — routed through OpenRouter -anthropic/ — direct Anthropic API -openai/ — direct OpenAI API -``` - -**For `openrouter/` model IDs, preserve the exact slug from OpenRouter's catalog** — these are passed verbatim to OpenRouter's API and must match exactly, including dots in version numbers: -- `openrouter/anthropic/claude-sonnet-4.6` ✓ (dots — OpenRouter's catalog format) -- `openrouter/openai/gpt-5.4` ✓ (dots) -- `openrouter/deepseek/deepseek-v3.2` ✓ (dots) -- `openrouter/google/gemini-2.5-flash` ✓ - -**For `anthropic/` direct-API model IDs, use hyphens** — Anthropic's own API slugs use hyphens: -- `anthropic/claude-sonnet-4-6` ✓ (hyphens) -- `anthropic/claude-opus-4-6` ✓ (hyphens) - -**For `openai/` direct-API model IDs, use dots in version segments** — OpenAI's API slugs use dots: -- `openai/gpt-5.4` ✓ (dot) -- `openai/gpt-4.1` ✓ (dot) - -`src/project/validate.ts` warns on dot-notation for `anthropic/` model IDs only (`model-id-bad-format`) and `src/project/fix.ts` auto-corrects them. Both `openai/` and `openrouter/` are fully exempt from any dot→hyphen rewriting. When adding new model presets to `src/init/scaffold.ts`, `src/init/wizard.ts`, or `src/benchmark/init.ts`, copy the slug exactly from the OpenRouter catalog for `openrouter/` models. - -Display names (`name:` / `label:` fields) are human-readable and should keep dots (e.g. `'Claude Sonnet 4.6'`, `'Gemini 2.5 Flash'`). - -## Environment Notes - -- Do not commit `.env` or secrets. -- Pi-based examples use `benchmark.format: "pi"` and typically expect `OPENROUTER_API_KEY`. -- The current unified config also allows the optimizer model to use `OPENROUTER_API_KEY`. +- Run `npm test` before finishing behavior changes. +- For Docker runner or image changes, also run `docker build -t skill-optimizer-workbench:local -f docker/workbench-runner.Dockerfile .`. +- For CLI/docs changes, verify `npx tsx src/cli.ts --help` if touched docs mention CLI behavior. +- For plugin/package metadata changes, run `npx tsx tests/smoke-skill-distribution.ts` and verify `npm pack --dry-run --json` includes required plugin files without result/cache directories. diff --git a/GEMINI.md b/GEMINI.md new file mode 100644 index 0000000..0bf3b16 --- /dev/null +++ b/GEMINI.md @@ -0,0 +1,3 @@ +@./AGENTS.md +@./skills/skill-optimizer/SKILL.md +@./skills/skill-optimizer/references/workbench.md diff --git a/README.md b/README.md index 404058f..9454569 100644 --- a/README.md +++ b/README.md @@ -1,229 +1,127 @@ # skill-optimizer -Benchmark and self-optimize SDK, CLI, and MCP guidance so every agent model can use your tool reliably. +Docker workbench and Agent Skill for running deterministic evals against agent skills. -skill-optimizer runs your SDK / CLI / MCP docs against multiple LLMs, measures whether they call the right actions with the right arguments, and iteratively rewrites your `SKILL.md` / docs until a floor score is met across every model. +Use this repo in two ways: -Built by the team at [Fast](https://fast.xyz/) — payment infrastructure for AI agents. [Give your agent a wallet](https://github.com/fastxyz/fast-sdk) in 3 lines of code. +- Install the `skill-optimizer` skill/plugin into your agent so it can author and debug eval suites. +- Run the local CLI to execute cases and suites in Docker against OpenRouter models. -**Requirements:** Node.js 20+, plus either an [OpenRouter](https://openrouter.ai) API key or a local Codex login when using direct OpenAI models. +## Install For Agents -## How it works — at a glance +The canonical skill is `skills/skill-optimizer/SKILL.md`. Plugin metadata points at that same file for every supported agent. -![Optimizer Loop](https://raw.githubusercontent.com/fastxyz/skill-optimizer/main/docs/images/optimizer-loop.svg) - -`skill-optimizer run` benchmarks your callable surface against multiple LLMs — it discovers actions, generates tasks, calls each model, and statically evaluates action recall and argument accuracy to produce a PASS/FAIL verdict (exit 0/1) usable in CI. - -`skill-optimizer optimize` runs the benchmark as a feedback loop: it copies your SKILL.md, mutates it with an LLM agent, re-benchmarks, accepts only when scores improve, and repeats until stable. Your original SKILL.md is never modified. - -## Installation +Install the skill for common agents with the open skills CLI: ```bash -git clone https://github.com/fastxyz/skill-optimizer -cd skill-optimizer -npm install -npm run build -npm link # makes `skill-optimizer` available globally +npx skills add fastxyz/skill-optimizer --skill skill-optimizer -a claude-code -a opencode -a codex -a cursor ``` -## Quickstart +Claude Code plugin install: -```bash -export OPENROUTER_API_KEY=sk-or-... +```text +/plugin marketplace add fastxyz/skill-optimizer +/plugin install skill-optimizer@skill-optimizer ``` -For direct OpenAI API calls you can use your local Codex browser login instead of exporting `OPENAI_API_KEY` — set `format: "openai"` and `authMode: "codex"`: +OpenCode plugin install in `opencode.json`: ```json { - "benchmark": { - "format": "openai", - "authMode": "codex", - "models": [ - { "id": "openai/gpt-5.4", "name": "GPT-5.4", "tier": "flagship" } - ] - } + "plugin": ["skill-optimizer@git+https://github.com/fastxyz/skill-optimizer.git"] } ``` -Codex auth reads a browser-login JWT or a static `OPENAI_API_KEY` from `~/.codex/auth.json`. It only applies to `openai/` model refs; `openrouter/` models always use `OPENROUTER_API_KEY`. +See `docs/README.opencode.md` for OpenCode details. -**Step 1 — Scaffold config** (run from your project root): +Codex plugin install: ```bash -npx skill-optimizer init cli # or: init sdk, init mcp, init prompt +codex plugin marketplace add fastxyz/skill-optimizer ``` -The wizard asks for your repo path, models to benchmark, and where your `SKILL.md` lives. It creates a `.skill-optimizer/` directory: -- `.skill-optimizer/skill-optimizer.json` — the main config (commit this) -- `.skill-optimizer/cli-commands.json` — CLI surface manifest (template to edit, or auto-extracted) -- `.skill-optimizer/tools.json` — MCP surface manifest (template to edit) +Then open `/plugins` and install `skill-optimizer`. See `docs/README.codex.md` for the skill-only Codex path. -**Step 2 — (CLI/MCP only) Extract your surface** if code-first discovery yields nothing: +Cursor can install the skill through the skills CLI command above or from GitHub via Settings -> Rules -> Project Rules -> Add Rule -> Remote Rule (Github). The Cursor plugin metadata lives at `.cursor-plugin/plugin.json`. -```bash -npx skill-optimizer import-commands --from ./src/cli.ts -# or for a compiled binary: -npx skill-optimizer import-commands --from my-cli --scrape -``` +Gemini extension metadata is provided by `gemini-extension.json`; it loads `GEMINI.md`, which references the canonical skill and workbench reference. -**Step 3 — Run a benchmark:** +## Local CLI Setup -```bash -npx skill-optimizer run --config ./.skill-optimizer/skill-optimizer.json -``` +Requirements: + +- Node.js 20+ +- Docker +- `OPENROUTER_API_KEY` for real model runs -**Step 4 — Run the optimizer** (iteratively improves your `SKILL.md`): +Install and build: ```bash -npx skill-optimizer optimize --config ./.skill-optimizer/skill-optimizer.json +npm install +npm run build ``` -The optimizer never modifies your original `SKILL.md` — it works from versioned local copies in `.skill-optimizer/` and prints a progress table at the end showing per-model improvement. +Only `openrouter/...` model refs are supported. ---- +## Quick Start -**Non-interactive / CI mode:** +Run the suite against the models listed in `suite.yml`: ```bash -# Accept all wizard defaults without prompts -npx skill-optimizer init cli --yes - -# Load answers from a JSON file -npx skill-optimizer init --answers answers.json -``` - -`answers.json` format: -```json -{ - "surface": "cli", - "repoPath": "/absolute/path/to/your-repo", - "models": [ - "openrouter/anthropic/claude-sonnet-4.6", - "openrouter/deepseek/deepseek-v3.2", - "openrouter/google/gemini-2.5-flash", - "openrouter/qwen/qwen3.5-397b-a17b", - "openrouter/moonshotai/kimi-k2.5", - "openrouter/z-ai/glm-5.1", - "openrouter/minimax/minimax-m2.7", - "openrouter/google/gemma-4-31b-it", - "openrouter/meta-llama/llama-4-maverick" - ], - "maxTasks": 20, - "maxIterations": 5, - "entryFile": "src/cli.ts" -} +npx tsx src/cli.ts run-suite examples/workbench/pdf/suite.yml --trials 1 ``` -**Key config fields** in `.skill-optimizer/skill-optimizer.json`: - -| Field | What it does | Set it to | -|-------|-------------|-----------| -| `target.repoPath` | Root of the project being benchmarked | Absolute or relative path to your repo | -| `target.discovery.sources` | Source files to scan for callable methods/commands/tools | e.g. `["../src/index.ts"]` or `["../src/server.ts"]` | -| `target.skill` | Docs file the optimizer will edit | Path to your `SKILL.md` or equivalent guidance doc | -| `benchmark.models` | Models to benchmark | Model IDs with provider prefix: `openrouter//` (via OpenRouter), `anthropic/` (direct Anthropic), `openai/` (direct OpenAI) | -| `benchmark.authMode` | How model auth is resolved | `env` (default), `codex`, or `auto` | - -### Prompt templates / Claude Code skills - -Benchmark how well models follow your prompt templates: +Run one case directly: ```bash -skill-optimizer init prompt -skill-optimizer run +npx tsx src/cli.ts run-case ./case.yml --model openrouter/google/gemini-2.5-flash ``` -The prompt surface discovers phases and capabilities from your SKILL.md, -generates scenario-based tasks, and evaluates output quality — not just -tool calls. Each task is tagged with the specific capability it exercises -(`capabilityId`), and scoring is performed against that capability's -criteria — not the first discovered capability. It scores responses on -required sections, format patterns, forbidden keywords, and structural -elements (code blocks, numbered lists, tables). Coverage violations do -not hard-fail prompt runs; coverage is informational for the prompt -surface. This lets you optimize prompt templates the same way you -optimize SDK/CLI/MCP guidance. - -## How it works - -1. **Discover** callable surface (SDK methods / CLI commands / MCP tools / prompt phases) via tree-sitter, manifest, or markdown parsing. -2. **Scope** the surface with `target.scope.include` / `target.scope.exclude` globs. -3. **Generate tasks** — one prompt per in-scope action, coverage-guaranteed. -4. **Benchmark** — every configured model attempts every task; static evaluator checks action calls + args. -5. **Verdict** — PASS/FAIL against two gates (per-model floor, weighted average). -6. **Optimize** — create a local versioned copy of your `SKILL.md` (`skill-v{N}.md` in `.skill-optimizer/`), mutate it, re-benchmark, accept only if both gates hold, rollback if not. The target repo's original skill file is never modified. -7. **Recommendations** — on FAIL, one critic call summarizes what to improve manually. -8. **Progress table** — after the optimizer finishes, a per-model table shows Baseline → each iteration → Final → Δ so you can see exactly where each model improved. +CLI help: -## Configuration reference - -See [docs/reference/config-schema.md](https://github.com/fastxyz/skill-optimizer/blob/main/docs/reference/config-schema.md) for the full generated config reference — auto-updated at every build. - -See [docs/reference/errors.md](https://github.com/fastxyz/skill-optimizer/blob/main/docs/reference/errors.md) for all error codes, descriptions, and fix instructions. - -## Interpreting the verdict - -Every benchmark run produces one of two verdicts: **PASS** or **FAIL**. - -Two gates must both be satisfied for a PASS: - -- **`benchmark.verdict.perModelFloor`** (default `0.6`): every model must pass at least this fraction of tasks. A single model below the floor fails the run, regardless of the average. -- **`benchmark.verdict.targetWeightedAverage`** (default `0.7`): the weighted average score across all models must reach this threshold. - -**`benchmark.models[].weight`** (default `1.0`): heavier-weighted models count more toward the weighted average. Use higher weights for flagship models you care most about. - -The **optimizer** only accepts a mutation when: -1. the weighted average improves by at least `minImprovement`, AND -2. no model that was above the floor drops below it. - -**Exit codes**: `0` = PASS, `1` = FAIL — usable directly in CI pipelines. - -## Scope & coverage - -Control which actions are benchmarked with `target.scope`: - -- **`target.scope.include`** (default `["*"]`): glob patterns for actions to include. -- **`target.scope.exclude`** (default `[]`): glob patterns for actions to exclude. - -The `*` wildcard matches any sequence of characters including dots and slashes — it is not limited to a single path segment. - -Examples: -- `"Wallet.*"` — includes all Wallet methods -- `"*.internal*"` — excludes anything with "internal" anywhere in the name -- `"get_*"` — includes only getter actions - -Task generation is **coverage-guaranteed**: every in-scope action gets at least one task. If the first generation pass misses any, a targeted retry runs (max 2 iterations). If coverage still fails, an error names the uncovered actions and suggests either fixing SKILL.md guidance or adding them to `scope.exclude`. +```bash +npx tsx src/cli.ts --help +npx tsx src/cli.ts run-case --help +npx tsx src/cli.ts run-suite --help +``` -## Cost notes +## How The Workbench Works -Rough LLM spend per run: +The workbench gives an agent a skill/reference folder, an isolated `/work` directory, and deterministic graders. It is designed for evals where success can be verified from files, command logs, SQL, generated artifacts, or other local state. -- **Baseline benchmark**: N models × M tasks LLM calls. -- **Optimizer iteration**: 1 mutation call + N models × M tasks re-benchmark per iteration. -- **Recommendations**: 1 critic call, only on FAIL verdict. +Core concepts: -No per-failure LLM calls — feedback is deterministic (structured failure details + patterns + passing/failing diffs). +- A case is one user-like task plus one or more graders. +- A suite is a matrix of cases and OpenRouter models. +- `references/` is copied into `/work`; this is where the skill under test lives. +- The agent phase sees only `/work`, not graders, hidden answers, `/case`, or `/results`. +- Graders run after the agent with `$CASE`, `$WORK`, and `$RESULTS` available. +- Graders are the acceptance contract. They can inspect workspace files and artifacts, `answer.json`, `trace.jsonl`, and result state under `$RESULTS`. -## Dependencies +Read `docs/workbench.md` for the full model: directory layout, Docker phases, graders, outputs, and debugging. -The optimizer's coding agent is powered by `@mariozechner/pi-coding-agent`. OpenRouter-backed runs still use your configured API key env var. Direct OpenAI runs can use either `OPENAI_API_KEY` or the browser-login tokens that Codex stores in `~/.codex/auth.json`. +## Examples -## Troubleshooting +Tracked examples live under `examples/workbench/`. The PDF example includes positive PDF extraction/splitting/creation cases and a negative case that checks the agent did not read the PDF skill file for a non-PDF task. The MCP example shows a local calculator server started as a hidden Docker service and exposed through the workbench `mcp` command. -**Missing `OPENROUTER_API_KEY`**: Set it in your shell before running: ```bash -export OPENROUTER_API_KEY=sk-or-... +npx tsx src/cli.ts run-suite examples/workbench/pdf/suite.yml --trials 1 +npx tsx src/cli.ts run-suite examples/workbench/mcp/suite.yml --trials 1 ``` -**Using Codex auth**: Set `benchmark.authMode` (and optionally `optimize.authMode`) to `"codex"` or `"auto"` and use direct OpenAI model refs such as `openai/gpt-5.4`. Codex auth only applies to the `openai` provider and reads either a browser-login access token or `OPENAI_API_KEY` from `~/.codex/auth.json`. Alternatively, set `benchmark.format` to `"openai"` with `authMode: "codex"` and `openai/...` model IDs — the client bridges to the Pi/Codex path automatically. - -**Dirty git**: The optimizer requires a clean git state in the target repo (`requireCleanGit: true` by default). Commit or stash uncommitted changes before running. Note: the optimizer never writes to the target repo's skill file — it works from local versioned copies in `.skill-optimizer/`. +## Development -**`maxTasks < scope_size`**: `benchmark.taskGeneration.maxTasks` must be >= the number of in-scope actions. Run `npx skill-optimizer --dry-run --config .skill-optimizer/skill-optimizer.json` to see the count without making LLM calls. +```bash +npm run typecheck +npm test +npm run build +npx tsx src/cli.ts --help +``` -**Empty scope**: `target.scope.include` matched nothing. Check your glob patterns — remember `*` matches everything including dots. +For Docker runner or image changes: -## Contributing +```bash +docker build -t skill-optimizer-workbench:local -f docker/workbench-runner.Dockerfile . +``` -See [CONTRIBUTING.md](https://github.com/fastxyz/skill-optimizer/blob/main/CONTRIBUTING.md). +Do not commit `.skill-eval/`, `.results/`, `.env`, or credentials. diff --git a/SKILL/SKILL.md b/SKILL/SKILL.md deleted file mode 100644 index 38aa1ca..0000000 --- a/SKILL/SKILL.md +++ /dev/null @@ -1,160 +0,0 @@ ---- -name: skill-optimizer -description: > - Benchmark and optimize SDK, CLI, MCP, and prompt documentation so every LLM - model can reliably call the right actions with correct arguments. Use when - setting up skill-optimizer for a project, running benchmarks, interpreting - results, optimizing SKILL.md files, or diagnosing configuration issues. Also - use when working inside the skill-optimizer repository itself — for running - against mock repos, testing changes, or understanding the codebase. ---- - -# skill-optimizer - -Benchmark your SDK / CLI / MCP / prompt docs against multiple LLMs, measure whether they call the right actions with the right arguments, and iteratively rewrite your guidance until a quality floor is met across every model. - -## Context Detection - -Before doing anything, figure out where you are: - -1. **Look for `skill-optimizer.json`** (in CWD or parent directories). If found, you are in a **configured target project**. Use that file path as `` in all commands below. - -2. **Look for `src/cli.ts` and a `package.json` with `"name": "skill-optimizer"`**. If found, you are in the **optimizer repo itself**. You can use dev commands directly (`npm run build`, `npm test`, `npx tsx src/cli.ts`). To benchmark a target, either use the mock repos in `mock-repos/` or point `--config` at an external project's config. - -3. **Neither found** — you are in an **unconfigured target project**. Read `references/setup.md` to scaffold a config before proceeding. - -## Quick Reference - -| Task | Command | -|------|---------| -| Init config (interactive) | `npx skill-optimizer init cli\|sdk\|mcp\|prompt` | -| Init (non-interactive, explicit surface) | `npx skill-optimizer init cli --yes` | -| Init (auto-detect surface, non-interactive) | `npx skill-optimizer init --auto --yes` | -| Import CLI commands | `npx skill-optimizer import-commands --from ./src/cli.ts` | -| Import with output file | `npx skill-optimizer import-commands --from ./src/cli.ts --out ./commands.json` | -| Import (overwrite existing) | `npx skill-optimizer import-commands --from ./src/cli.ts --out ./commands.json --force` | -| Import (binary scrape) | `npx skill-optimizer import-commands --from my-cli --scrape --depth 3` | -| Diagnose config | `npx skill-optimizer doctor --config ` | -| Diagnose (skip code discovery) | `npx skill-optimizer doctor --config --static` | -| Diagnose (verify model access) | `npx skill-optimizer doctor --config --check-models` | -| Auto-fix config | `npx skill-optimizer doctor --fix --config ` | -| Dry run (no LLM calls) | `npx skill-optimizer run --dry-run --config ` | -| Run benchmark | `npx skill-optimizer run --config ` | -| Run (filter by model tier) | `npx skill-optimizer run --config --tier flagship` | -| Generate tasks only | `npx skill-optimizer generate-tasks --config ` | -| Run optimizer | `npx skill-optimizer optimize --config ` | -| Compare two runs | `npx skill-optimizer compare --baseline a.json --current b.json` | - -`` is the path to your `skill-optimizer.json` — typically `./.skill-optimizer/skill-optimizer.json` after running `init`, or wherever you placed it. - -## What Do You Need? - -Read the reference file that matches your current goal: - -| Goal | Reference | -|------|-----------| -| Set up skill-optimizer for a project (first time) | Read `references/setup.md` | -| Run a benchmark or understand results | Read `references/benchmark.md` | -| Automatically optimize a SKILL.md | Read `references/optimize.md` | -| Understand config options | Read `references/config.md` | - -If you are in an **unconfigured project** (context detection case 3), start with `references/setup.md`. - -## Command Details - -### `init` — scaffold a skill-optimizer config - -The `init` command has three modes: - -1. **Interactive wizard** (default): `npx skill-optimizer init [surface]` — prompts you through setup. Optionally pass `cli`, `sdk`, `mcp`, or `prompt` as a positional argument to pre-select the surface type. - -2. **Non-interactive with explicit surface**: `npx skill-optimizer init --yes` — accepts all defaults for the named surface without prompting. - -3. **Auto-detect + non-interactive** (fully automated, zero prompts): `npx skill-optimizer init --auto --yes` — inspects the current directory to detect the surface type, then applies defaults without prompting. This is the right choice when the task says "initialize without prompts", "fully automated setup", or "detect and scaffold" — especially when the surface type isn't stated. - -Key parameters: - -| Parameter | Meaning | Notes | -|-----------|---------|-------| -| `[surface]` | Positional: `cli`, `sdk`, `mcp`, or `prompt` | Optional; omit when using `--auto` or running the interactive wizard | -| `--auto` | Auto-detect surface type from CWD | Detects surface; still prompts unless combined with `--yes` | -| `--yes` | Accept all defaults without prompting | Alone: needs explicit surface. With `--auto`: fully non-interactive. | -| `--answers ` | Load answers from a JSON file | For CI pipelines with a pre-built answers file | - -**Critical:** `--auto` and `--yes` have independent effects. `--yes` alone still requires a surface name. `--auto` alone still opens the interactive wizard (pre-filled). Only `--auto --yes` together produces a completely non-interactive run. - -``` -# Fully automated: detect surface + accept defaults (no prompts at all) -npx skill-optimizer init --auto --yes - -# Explicit surface, no prompts -npx skill-optimizer init cli --yes - -# Interactive wizard for MCP surface -npx skill-optimizer init mcp -``` - -### `doctor` — diagnose your configuration - -The base command validates your `skill-optimizer.json` and checks that discovered surfaces are intact. Two optional flags activate additional checks that are *off by default*: - -- `--static` — skip live code discovery (tree-sitter analysis). Use this when you want to validate config and manifests without requiring the project source to be present, or to speed up CI checks. **Do not confuse with `--no-discovery` — the correct flag is `--static`.** -- `--check-models` — ping each configured model to verify API credentials and routing are working. Use this when you suspect auth issues or want to confirm model availability before a benchmark run. **The flag is `--check-models`, not `--ping` or `--verify-models`.** - -These flags are independent and can be combined: -``` -npx skill-optimizer doctor --config ./skill-optimizer.json --static -npx skill-optimizer doctor --config ./skill-optimizer.json --check-models -npx skill-optimizer doctor --config ./skill-optimizer.json --static --check-models -``` - -### `import-commands` — extract CLI surface from source or binary - -Discovery mode is determined by whether `--scrape` is present: - -- **Source mode** (default): `--from` points to a TypeScript/JavaScript file (e.g. `./src/cli.ts`). Tree-sitter parses commands statically. -- **Scrape mode**: Add `--scrape` to invoke the binary named in `--from` and walk its `--help` output. - -Key parameters: - -| Parameter | Meaning | Notes | -|-----------|---------|-------| -| `--from ` | File path or binary name to import from | Required | -| `--out ` | Write discovered commands to this JSON file | Optional; without it, output goes to stdout | -| `--force` | Overwrite `--out` file if it already exists | Required when the output file exists; without it the command refuses to overwrite | -| `--scrape` | Invoke as a binary and parse `--help` output | Enables scrape mode | -| `--depth ` | Max subcommand depth to explore during scrape | Only meaningful with `--scrape`; **the flag is `--depth`, not `--max-depth`** | - -Output goes to the `--out` file — **do not use shell redirection (`>`) to capture output** because the tool writes structured JSON with metadata that is not suitable for piping. - -``` -# Source import, write to file (safe to re-run with --force) -npx skill-optimizer import-commands --from ./src/cli.ts --out ./commands.json --force - -# Scrape a binary, limit depth to 3 levels -npx skill-optimizer import-commands --from my-app --scrape --depth 3 -``` - -### `run` — execute the benchmark - -Filterable via: -- `--tier ` — only run models whose tier matches. Valid values: `flagship`, `mid`, `budget`. **The flag is `--tier`, not `--model-tier`.** -- `--model ` — run a single specific model. -- `--dry-run` — generate prompts and tasks without making LLM calls. - -``` -npx skill-optimizer run --config ./skill-optimizer.json --tier flagship -npx skill-optimizer run --config ./skill-optimizer.json --tier mid -``` - -## Key Concepts - -**Surfaces** — The callable interface of your project: SDK methods, CLI commands, MCP tools, or prompt templates. Skill-optimizer discovers these via tree-sitter code analysis, manifest files, or markdown parsing. - -**Static evaluation** — Benchmark evaluation never executes generated code. Actions are extracted from model responses via pattern matching and compared structurally against expected calls. This makes benchmarks safe and repeatable. - -**Verdict gates** — Two thresholds must both pass for a benchmark to receive a PASS verdict: `perModelFloor` (each model individually meets a minimum score) and `targetWeightedAverage` (the weighted mean across all models meets a target). A single model below the floor fails the entire run. - -**Safety boundary** — The optimizer never modifies your original SKILL.md. It creates versioned copies in `.skill-optimizer/skill-v{N}.md` and only accepts mutations that improve scores without dropping any model below the floor. It does not modify tracked source files, but the generated artifacts appear under `.skill-optimizer/` — add that directory to your `.gitignore`. - -**LLM routing** — By default (`format: "pi"`), all benchmark calls route through [OpenRouter](https://openrouter.ai) and need `OPENROUTER_API_KEY`. You can also call providers directly: `format: "anthropic"` uses the Anthropic API directly (`ANTHROPIC_API_KEY`), and `format: "openai"` uses the OpenAI API directly (`OPENAI_API_KEY`), with optional Codex browser-login auth via `authMode: "codex"`. The model ID prefix must match the format — see `references/config.md` for the full mapping. diff --git a/SKILL/references/benchmark.md b/SKILL/references/benchmark.md deleted file mode 100644 index c7dfd20..0000000 --- a/SKILL/references/benchmark.md +++ /dev/null @@ -1,152 +0,0 @@ -# Running & Interpreting Benchmarks - -This guide covers running benchmarks, reading results, diagnosing failures, and comparing runs. - -## 1. Pre-flight Check - -Before running a benchmark, verify: - -```bash -# Config is valid -npx skill-optimizer doctor --config - -# API key is set -echo $OPENROUTER_API_KEY # should print sk-or-... - -# Git is clean (if requireCleanGit is true, which is the default) -git status # should show "nothing to commit, working tree clean" -``` - -## 2. Dry Run First - -Always start with a dry run to check scope and estimate cost: - -```bash -npx skill-optimizer run --dry-run --config -``` - -This shows: -- How many actions were discovered -- How many are in scope after filtering -- How many tasks would be generated -- Which models would be called - -No LLM calls are made. Use this to verify your scope and estimate cost (N models x M tasks = total calls). - -## 3. Run the Benchmark - -```bash -npx skill-optimizer run --config -``` - -Optional run flags: - -| Flag | Effect | Note | -|------|--------|------| -| `--tier ` | Only run models whose tier matches. | Valid values: `flagship`, `mid`, `budget`. Flag is `--tier`, not `--model-tier`. | -| `--model ` | Run a single specific model. | Pass the full model ID. | -| `--task ` | Run a single task by ID. | Stable IDs from `tasks.generated.json`. | -| `--no-cache` | Force fresh skill fetch. | | -| `--dry-run` | Preview scope without making LLM calls. | | - -```bash -# Run only flagship models -npx skill-optimizer run --config --tier flagship - -# Debug a single task -npx skill-optimizer run --config --task -``` - -What happens at each stage: - -1. **Discover** — find callable actions via tree-sitter or manifest -2. **Scope** — apply `include`/`exclude` filters -3. **Generate tasks** — create one prompt per in-scope action (coverage-guaranteed: every action gets at least one task) -4. **Call models** — each configured model attempts each task -5. **Extract** — pull action calls from model responses via pattern matching -6. **Evaluate** — compare extracted actions against expected actions -7. **Verdict** — PASS or FAIL based on two gates - -## 4. Reading the Output - -The benchmark produces: - -- **Per-model score table** — each model's pass rate as a fraction (e.g., `Claude Sonnet: 18/20 (0.90)`) -- **Weighted average** — computed from individual scores and model weights -- **Verdict** — `PASS` (both gates satisfied) or `FAIL` (at least one gate missed) -- **Exit code** — `0` for PASS, `1` for FAIL - -## 5. Verdict Gates - -Two gates must **both** pass for a PASS verdict: - -**`perModelFloor`** (default: `0.6`) -Every model must individually score at or above this threshold. If any single model scores below, the entire benchmark fails — regardless of how well other models did. This prevents one weak model from hiding behind a strong average. - -**`targetWeightedAverage`** (default: `0.7`) -The weighted mean across all models must reach this threshold. Models with higher `weight` values count more. This ensures overall quality, not just per-model minimums. - -**Model `weight`** (default: `1.0`) -Controls how much each model influences the weighted average. Set flagship models to `2.0` and budget models to `0.5` if you care more about flagship performance. - -## 6. Diagnosing Failures - -When a benchmark fails, look at the per-task breakdown to identify patterns: - -**Hallucinated actions** — the model calls functions that don't exist in your API. -- *Cause:* SKILL.md describes features ambiguously or mentions non-existent methods -- *Fix:* Tighten your docs. Remove references to deprecated methods. Be explicit about what exists. - -**Missing arguments** — the model calls the right action but with wrong or missing arguments. -- *Cause:* Documentation doesn't clearly specify required parameters or their types -- *Fix:* Add explicit parameter sections with types, defaults, and examples - -**Wrong tool selection** — the model calls a related but incorrect action (e.g., `deleteTask` instead of `removeTask`). -- *Cause:* Action names are ambiguous or the docs don't distinguish between similar actions -- *Fix:* Add disambiguation notes or rename actions to be more distinct - -**One model fails, others pass** — a specific model consistently underperforms. -- *Cause:* That model may need more explicit guidance or has known weaknesses with your API style -- *Fix:* Consider adjusting its `weight`, adding model-specific notes to your docs, or accepting the floor as-is - -## 7. Comparing Runs - -After making changes to your SKILL.md, compare before and after: - -```bash -npx skill-optimizer compare --baseline report-before.json --current report-after.json -``` - -This shows: -- Per-model score deltas (e.g. `Claude Sonnet: 0.75 → 0.90 (+0.15)`) -- Per-task deltas — which tasks improved, which regressed -- Overall weighted average change - -**Finding the report files:** The benchmark writes its report JSON to the `output.dir` configured in your `skill-optimizer.json` (default: `benchmark-results/`). Each run creates a timestamped file there. - -## 8. Cost Awareness - -Each benchmark run makes `N models x M tasks` LLM calls. To minimize cost while iterating: - -- **Start narrow** — use `scope.include` to benchmark only your most important actions first -- **Few models first** — start with 2-3 models, expand after the skill stabilizes -- **Dry run** — always check scope size with `--dry-run` before committing to a full run -- **Iterate on docs first** — fix obvious SKILL.md gaps before re-running. Each run costs real money. - -## 9. CI Integration - -The exit code (`0` = PASS, `1` = FAIL) makes skill-optimizer suitable for CI pipelines: - -```bash -# In a CI script or Makefile -npx skill-optimizer run --config -# Exits 0 on PASS, 1 on FAIL — use as a gate step -``` - -This lets you catch regressions in documentation quality as part of your CI workflow. - -## Next Steps - -If the benchmark fails and the issues are scattered (not one obvious fix), read `references/optimize.md` to run the automatic optimization loop. - -If you need to adjust config (models, scope, thresholds), read `references/config.md`. diff --git a/SKILL/references/config.md b/SKILL/references/config.md deleted file mode 100644 index 8a40364..0000000 --- a/SKILL/references/config.md +++ /dev/null @@ -1,193 +0,0 @@ -# Configuration Reference - -Complete reference for `skill-optimizer.json`. For auto-generated schema docs, see `docs/reference/config-schema.md` in the skill-optimizer repo. - -## Minimal Working Configs - -### CLI surface - -```json -{ - "name": "my-cli-tool", - "target": { - "surface": "cli", - "repoPath": "/path/to/my-project", - "skill": "./SKILL.md", - "discovery": { - "mode": "auto", - "sources": ["src/cli.ts"] - } - }, - "benchmark": { - "format": "pi", - "models": [ - { "id": "openrouter/anthropic/claude-sonnet-4.6", "name": "Claude Sonnet", "tier": "flagship" } - ] - } -} -``` - -### SDK surface - -```json -{ - "name": "my-sdk", - "target": { - "surface": "sdk", - "repoPath": "/path/to/my-sdk", - "skill": "./SKILL.md", - "discovery": { - "mode": "auto", - "sources": ["src/index.ts"], - "language": "typescript" - } - }, - "benchmark": { - "format": "pi", - "models": [ - { "id": "openrouter/anthropic/claude-sonnet-4.6", "name": "Claude Sonnet", "tier": "flagship" } - ] - } -} -``` - -### MCP surface - -```json -{ - "name": "my-mcp-server", - "target": { - "surface": "mcp", - "repoPath": "/path/to/my-mcp-server", - "skill": "./SKILL.md", - "discovery": { - "mode": "auto", - "sources": ["src/server.ts"] - } - }, - "benchmark": { - "format": "pi", - "models": [ - { "id": "openrouter/anthropic/claude-sonnet-4.6", "name": "Claude Sonnet", "tier": "flagship" } - ] - } -} -``` - -### Prompt surface - -```json -{ - "name": "my-skill-doc", - "target": { - "surface": "prompt", - "repoPath": "/path/to/my-project", - "skill": "./SKILL.md" - }, - "benchmark": { - "format": "pi", - "models": [ - { "id": "openrouter/anthropic/claude-sonnet-4.6", "name": "Claude Sonnet", "tier": "flagship" } - ] - } -} -``` - -## Field-by-Field Reference - -### `target` — What You're Benchmarking - -| Field | Required | Default | Description | -|-------|----------|---------|-------------| -| `surface` | Yes | — | `"cli"`, `"sdk"`, `"mcp"`, or `"prompt"` | -| `repoPath` | Yes | — | Absolute or config-relative path to your project root | -| `skill` | Yes | — | Path to your SKILL.md or guidance doc, relative to `repoPath` | -| `discovery.mode` | No | `"auto"` | `"auto"` (tree-sitter) or `"manifest"` (hand-written JSON) | -| `discovery.sources` | No | `[]` | Source files for tree-sitter to parse, relative to `repoPath` | -| `discovery.language` | No | — | SDK only: `"typescript"`, `"python"`, or `"rust"` | -| `scope.include` | No | `["*"]` | Glob patterns for actions to include | -| `scope.exclude` | No | `[]` | Glob patterns for actions to exclude | - -### `benchmark` — How to Test - -| Field | Required | Default | Description | -|-------|----------|---------|-------------| -| `format` | No | `"pi"` | `"pi"` — route through OpenRouter (default); `"openai"` — call OpenAI API directly; `"anthropic"` — call Anthropic API directly | -| `authMode` | No | `"env"` | `"env"` — read key from env var (default); `"codex"` — read from `~/.codex/auth.json` (OpenAI only); `"auto"` — try env first, fall back to codex for OpenAI | -| `apiKeyEnv` | No | provider default | Env var holding the API key. Defaults: `OPENROUTER_API_KEY` for `pi`, `OPENAI_API_KEY` for `openai`, `ANTHROPIC_API_KEY` for `anthropic` | -| `baseUrl` | No | — | Override the API base URL (e.g. for a custom OpenAI-compatible endpoint) | -| `models[].id` | Yes | — | Model ID with provider prefix: `openrouter/

/` (OpenRouter — dots in version segments, e.g. `openrouter/anthropic/claude-sonnet-4.6`), `anthropic/` (direct Anthropic — hyphens, e.g. `anthropic/claude-sonnet-4-6`), `openai/` (direct OpenAI — dots, e.g. `openai/gpt-5.4`). | -| `models[].name` | No | — | Human-readable label for output tables | -| `models[].tier` | No | — | `"flagship"`, `"mid"`, or `"low"` (informational only) | -| `models[].weight` | No | `1.0` | Influence on weighted average (higher = counts more) | -| `verdict.perModelFloor` | No | `0.6` | Minimum score each model must reach individually | -| `verdict.targetWeightedAverage` | No | `0.7` | Minimum weighted average across all models | -| `taskGeneration.enabled` | No | `true` | Whether to auto-generate tasks | -| `taskGeneration.maxTasks` | No | `20` | Upper bound on tasks (must be >= in-scope action count) | -| `taskGeneration.outputDir` | No | `".skill-optimizer"` | Where to write task artifacts | - -### `optimize` — How to Improve - -| Field | Required | Default | Description | -|-------|----------|---------|-------------| -| `enabled` | No | `true` | Whether optimization is allowed | -| `mode` | No | `"stable-surface"` | `"stable-surface"` (reuse tasks) or `"surface-changing"` (regenerate per iteration) | -| `model` | No | `"openrouter/anthropic/claude-sonnet-4.6"` | Which LLM writes mutations | -| `maxIterations` | No | `5` | Maximum optimization rounds | -| `minImprovement` | No | `0.02` | Minimum delta in weighted average required to accept a mutation | -| `allowedPaths` | No | `["SKILL.md"]` | Files the mutation agent may edit | -| `requireCleanGit` | No | `true` | Block optimizer if target repo has uncommitted changes | - -## Model ID and Auth Quick Guide - -The `benchmark.format` field controls which API receives requests. Pick the combination that matches your setup: - -| Format | Model ID prefix | Required env var | Notes | -|--------|----------------|------------------|-------| -| `"pi"` (default) | `openrouter//` | `OPENROUTER_API_KEY` | All providers via OpenRouter | -| `"openai"` | `openai/` | `OPENAI_API_KEY` | Direct OpenAI API | -| `"openai"` + `authMode: "codex"` | `openai/` | `~/.codex/auth.json` | Codex browser-login auth | -| `"anthropic"` | `anthropic/` | `ANTHROPIC_API_KEY` | Direct Anthropic API | - -**Codex auth** (`authMode: "codex"`) reads credentials from `~/.codex/auth.json` in this priority order: -1. `tokens.access_token` — browser-login JWT (checked for expiry) -2. `tokens.OPENAI_API_KEY` — static key nested under tokens -3. `OPENAI_API_KEY` — root-level static key - -Codex auth only works with the OpenAI provider. For `openrouter/` or `anthropic/` models, use `authMode: "env"`. - -**`authMode: "auto"`** tries the env var first; for OpenAI models, falls back to `~/.codex/auth.json` if the env var is unset. - -## Model Configuration Tips - -- Browse available models at [openrouter.ai/models](https://openrouter.ai/models) -- **Recommended starter set:** one flagship (Claude Sonnet or GPT-4o) + one budget model (Gemini Flash or Haiku) to test both capability ends -- **Weighting strategy:** set flagship models to `weight: 2.0` and budget to `weight: 0.5` if flagship performance matters most to you -- `tier` is informational only — it appears in output tables but doesn't affect scoring - -## Scope Patterns - -The `*` wildcard matches any sequence of characters, including dots and slashes. It is not limited to a single path segment like filesystem globs. - -| Pattern | Matches | -|---------|---------| -| `"Wallet.*"` | All Wallet methods (`Wallet.create`, `Wallet.balance`, etc.) | -| `"*.internal*"` | Anything with "internal" in the name | -| `"get_*"` | Only getter actions | -| `["create_*", "update_*", "delete_*"]` | Only mutation actions | - -Task generation is **coverage-guaranteed**: every in-scope action gets at least one task. If coverage fails after retries, an error names the uncovered actions and suggests either fixing SKILL.md guidance or excluding them. - -## Common Error Codes - -| Code | Meaning | Fix | -|------|---------|-----| -| `E_MISSING_SKILL` | `target.skill` file not found | Create the file or fix the path in config | -| `E_INVALID_SURFACE` | `target.surface` is not cli/sdk/mcp/prompt | Use one of the four valid values | -| `E_DIRTY_GIT` | Uncommitted changes in target repo | Commit or stash, or set `requireCleanGit: false` | -| `E_EMPTY_SCOPE` | Scope filters matched no actions | Check your `include`/`exclude` patterns | -| `E_MISSING_API_KEY` | API key env var not set | `export OPENROUTER_API_KEY=sk-or-...` (or `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` for direct formats) | - -Full error reference with detailed descriptions: `docs/reference/errors.md` - -Full config schema reference (auto-generated from Zod): `docs/reference/config-schema.md` diff --git a/SKILL/references/optimize.md b/SKILL/references/optimize.md deleted file mode 100644 index 12996e8..0000000 --- a/SKILL/references/optimize.md +++ /dev/null @@ -1,101 +0,0 @@ -# Optimization Loop - -This guide covers when and how to use the automatic optimizer, how to interpret its results, and what to do when it doesn't converge. - -## 1. When to Optimize vs. Fix Manually - -**Fix manually** when the benchmark reveals a clear, localized problem — a missing section, a wrong example, an outdated method name. Manual fixes are faster and more precise for known issues. - -**Run the optimizer** when failures are scattered across multiple models and tasks with no obvious single fix. The optimizer systematically tries mutations to your SKILL.md and keeps only changes that improve scores. - -A good workflow: run a benchmark, fix the obvious stuff by hand, re-benchmark, then let the optimizer handle whatever's left. - -## 2. How the Loop Works - -1. **Baseline benchmark** — establish starting scores for all models -2. **Copy** — your SKILL.md is copied to `.skill-optimizer/skill-v0.md` (original is never touched) -3. **Failure analysis** — identify patterns in what models get wrong -4. **Mutation** — a mutation agent (powered by `optimize.model`, defaults to Claude Opus via OpenRouter) proposes edits to the versioned copy -5. **Re-benchmark** — run all models against all tasks using the mutated skill -6. **Accept or reject** — the mutation is accepted only if: - - The weighted average improves by at least `minImprovement` - - No model that was above the floor drops below it -7. **Rollback** if rejected — revert to the previous version -8. **Repeat** up to `maxIterations` times -9. **Progress table** — final output shows Baseline -> each iteration -> Final -> delta per model - -## 3. Safety Guarantees - -The optimizer is designed to be safe to run: - -- **Your original SKILL.md is never modified.** All edits happen on versioned copies in `.skill-optimizer/skill-v0.md`, `skill-v1.md`, etc. -- **`requireCleanGit`** is enforced by default — the optimizer won't run if your target repo has uncommitted changes -- **`allowedPaths`** constrains which files the mutation agent can edit (defaults to just the skill file) -- **Stabilization window** prevents oscillation — if the same mutation keeps getting accepted and rejected, the optimizer exits early - -## 4. Running the Optimizer - -```bash -npx skill-optimizer optimize --config -``` - -Output during the run: -- Current iteration number and total -- Per-model scores after each mutation attempt -- Accept/reject decision with reasoning -- Running progress table - -The optimizer can take several minutes per iteration (it runs a full benchmark each time). - -## 5. Key Config Knobs - -| Setting | Default | What it controls | -|---------|---------|------------------| -| `optimize.maxIterations` | `5` | Upper bound on optimization rounds | -| `optimize.mode` | `"stable-surface"` | `"stable-surface"`: reuse tasks across iterations (faster, apples-to-apples). `"surface-changing"`: regenerate tasks each iteration (if skill changes might affect task phrasing) | -| `optimize.model` | `"openrouter/anthropic/claude-sonnet-4.6"` | Which LLM writes mutations | -| `optimize.enabled` | `true` | Set to `false` to skip optimization (useful in CI) | -| `optimize.requireCleanGit` | `true` | Block optimizer if target repo has uncommitted changes | - -## 6. Interpreting Results - -**Progress table** — rows are models, columns are iterations. Shows the score trajectory for each model across the optimization run. - -**Accepted iteration** — the mutation improved scores without violating either gate. The versioned copy advances to `skill-v{N+1}.md`. - -**Rejected iteration** — the mutation either didn't improve the weighted average enough, or it caused a model to drop below the floor. The previous version is kept and the optimizer tries a different mutation. - -**Early exit** — if scores plateau for consecutive iterations, the optimizer may stop before reaching `maxIterations`. This is normal and means further mutations aren't producing meaningful improvements. - -## 7. After Optimization - -The best version is the highest-numbered `skill-v{N}.md` in `.skill-optimizer/`. To apply it: - -```bash -# 1. See what changed -diff SKILL.md .skill-optimizer/skill-v3.md # adjust N to your highest version - -# 2. Review the diff — the optimizer is a tool, not an oracle -# Look for: overly specific examples, removed important context, awkward phrasing - -# 3. Copy it back -cp .skill-optimizer/skill-v3.md SKILL.md - -# 4. Commit -git add SKILL.md -git commit -m "docs: apply skill-optimizer improvements (v3)" -``` - -## 8. When It Doesn't Converge - -If the optimizer oscillates or plateaus without reaching your target scores: - -**Narrow the scope** — exclude actions that are inherently ambiguous or rarely used. A smaller, cleaner scope gives the optimizer more room to improve what matters. - -**Improve discovery** — make sure `discovery.sources` points at the right files. If the surface is incomplete (missing actions), the optimizer is working with bad data. - -**Manual intervention** — read the failure analysis output from the last iteration. It often reveals patterns that a targeted manual edit can fix more effectively than automated mutation. - -**Adjust gates** — if `perModelFloor` or `targetWeightedAverage` are set very high, lower them to something achievable first. Optimize to hit that floor, then ratchet up gradually. - -**Try different models** — change `optimize.model` to a different LLM. Different models have different strengths in rewriting documentation. diff --git a/SKILL/references/setup.md b/SKILL/references/setup.md deleted file mode 100644 index 159b80c..0000000 --- a/SKILL/references/setup.md +++ /dev/null @@ -1,184 +0,0 @@ -# Setup & Init - -This guide walks through setting up skill-optimizer for your project, from prerequisites to a verified configuration. - -## 1. Prerequisites - -Before starting, verify these three requirements: - -**Node.js 20+:** -```bash -node --version -# Expected: v20.x.x or higher -``` - -**API key** (which one depends on your `benchmark.format`): -```bash -# Default — OpenRouter (format: "pi"): -export OPENROUTER_API_KEY=sk-or-your-key-here - -# Direct OpenAI (format: "openai"): -export OPENAI_API_KEY=sk-your-key-here - -# Direct Anthropic (format: "anthropic"): -export ANTHROPIC_API_KEY=sk-ant-your-key-here -``` -If you're just getting started, use OpenRouter — one key covers all providers. - -**skill-optimizer available:** -```bash -npx skill-optimizer --help -# Expected: Usage information -# If not installed globally, install from the repo: -# cd /path/to/skill-optimizer && npm install && npm run build && npm link -``` - -## 2. Determine Your Surface Type - -skill-optimizer supports four surface types. Pick the one that matches your project: - -| Surface | Your project exposes... | Examples | -|---------|------------------------|----------| -| `cli` | CLI commands or a binary | Yargs, Commander, @optique/core, argparse, Click, Clap | -| `sdk` | Library methods users call in code | TypeScript/Python/Rust SDKs | -| `mcp` | MCP tool handlers | MCP servers with `server.tool()` definitions | -| `prompt` | Prompt templates or agent skill docs | SKILL.md files, Claude Code skills, agent instructions | - -If unsure: does your user run commands in a terminal (`cli`), import your package and call functions (`sdk`), connect an AI agent to your tool server (`mcp`), or follow a prompt template / skill document (`prompt`)? - -## 3. Run the Init Wizard - -From your project root: - -```bash -npx skill-optimizer init -# Example: npx skill-optimizer init cli -``` - -The wizard prompts for: - -- **Repo path** — absolute path to your project root (defaults to CWD) -- **Models** — model IDs to benchmark against (e.g., `openrouter/anthropic/claude-sonnet-4.6`) -- **SKILL.md location** — path to your existing documentation or guidance file -- **Discovery sources** — source files for tree-sitter to parse (e.g., `src/cli.ts`, `src/index.ts`) -- **Max tasks** — upper bound on generated benchmark tasks (default: 20) - -**Non-interactive mode** (for CI or scripting): - -The `--auto` and `--yes` flags are independent and serve different purposes: - -| Flag | Effect | -|------|--------| -| `--yes` | Accept all defaults without prompting. Still requires a surface name unless combined with `--auto`. | -| `--auto` | Auto-detect the surface type from the current directory. Still opens the interactive wizard (pre-filled) unless combined with `--yes`. | -| `--auto --yes` | **Fully non-interactive**: detect surface + accept all defaults. Use this for automated pipelines where the surface type isn't known in advance. | - -```bash -# Explicit surface, no prompts -npx skill-optimizer init cli --yes - -# Auto-detect surface + no prompts (fully automated, zero interaction) -npx skill-optimizer init --auto --yes - -# Load answers from a file -npx skill-optimizer init --answers answers.json -``` - -`answers.json` format: -```json -{ - "surface": "cli", - "repoPath": "/absolute/path/to/your-repo", - "models": ["openrouter/anthropic/claude-sonnet-4.6", "openrouter/openai/gpt-4o-mini"], - "maxTasks": 20, - "maxIterations": 5, - "entryFile": "src/cli.ts" -} -``` - -## 4. Surface Discovery - -After init, skill-optimizer needs to know what actions your project exposes. There are two discovery modes: - -**Code-first (auto)** — tree-sitter parses your source files automatically. This works for: -- TypeScript: Yargs, Commander, @optique/core CLI frameworks -- TypeScript/Python/Rust: SDK method extraction -- TypeScript: MCP `server.tool()` definitions - -If auto-discovery finds your actions, you're done. Check with: -```bash -npx skill-optimizer run --dry-run --config -# Look for "Discovered N actions" in the output -``` - -**Manual / import** — if auto-discovery yields nothing or misses actions: - -```bash -# Extract from TypeScript source, write to file -npx skill-optimizer import-commands --from ./src/cli.ts --out ./.skill-optimizer/cli-commands.json - -# Overwrite an existing output file (required when the file already exists) -npx skill-optimizer import-commands --from ./src/cli.ts --out ./.skill-optimizer/cli-commands.json --force - -# Extract from a compiled binary's help text, limit subcommand depth -npx skill-optimizer import-commands --from my-cli --scrape --depth 3 -``` - -Key `import-commands` flags: - -| Flag | Meaning | -|------|---------| -| `--from ` | Source file or binary name (required) | -| `--out ` | Write output to this file. Without `--out`, output goes to stdout. Do not use `>` shell redirection — it produces malformed output. | -| `--force` | Overwrite `--out` file if it already exists. Required on re-runs. | -| `--scrape` | Invoke as a binary and parse `--help` output instead of reading source | -| `--depth ` | Max subcommand depth during scrape. Flag is `--depth`, not `--max-depth`. | - -This populates `.skill-optimizer/cli-commands.json` (CLI) or `.skill-optimizer/tools.json` (MCP). You can also edit these manifest files by hand. - -## 5. Verify with Doctor - -Run the config diagnostics to catch problems early: - -```bash -npx skill-optimizer doctor --config -``` - -If issues are found, auto-fix what's fixable: - -```bash -npx skill-optimizer doctor --fix --config -``` - -Two optional flags activate additional checks that are off by default: - -| Flag | Effect | Note | -|------|--------|------| -| `--static` | Skip live code discovery (tree-sitter). Validates config and manifests only. | Flag is `--static`, not `--no-discovery`. | -| `--check-models` | Ping each configured model to verify API credentials and routing. | Flag is `--check-models`, not `--ping` or `--verify-models`. | - -```bash -# Validate config without running discovery (fast, works without project source) -npx skill-optimizer doctor --config --static - -# Verify model API keys are working -npx skill-optimizer doctor --config --check-models -``` - -## 6. What You Should Have Now - -After successful setup: - -- **`skill-optimizer.json`** — main config file (commit this); when created by `init`, the default location is `./.skill-optimizer/skill-optimizer.json` -- **`.skill-optimizer/`** — working directory for task artifacts, surface manifests, and versioned skill copies (gitignored) - -Your project is ready for benchmarking. Read `references/benchmark.md` for next steps. - -## 7. Common Pitfalls - -| Problem | Cause | Fix | -|---------|-------|-----| -| "Config not found" | Wrong path to `skill-optimizer.json` | Use `--config` with the full path | -| "No actions discovered" | `discovery.sources` points at wrong files | Check paths are relative to `repoPath` | -| "Skill file not found" | `target.skill` path is wrong | Path is relative to `repoPath` — verify it exists | -| "repoPath not found" | Relative path resolved wrong | Use absolute path, or make it relative to config file location | diff --git a/docker/workbench-runner.Dockerfile b/docker/workbench-runner.Dockerfile new file mode 100644 index 0000000..73591f7 --- /dev/null +++ b/docker/workbench-runner.Dockerfile @@ -0,0 +1,42 @@ +FROM node:22-bookworm + +ENV PATH="/app/node_modules/.bin:/work/.venv/bin:${PATH}" \ + PIP_REQUIRE_VIRTUALENV=1 + +WORKDIR /app + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + bash \ + ca-certificates \ + coreutils \ + curl \ + file \ + findutils \ + gawk \ + git \ + grep \ + jq \ + less \ + python-is-python3 \ + python3 \ + python3-pip \ + python3-venv \ + ripgrep \ + sed \ + unzip \ + wget \ + zip \ + && rm -rf /var/lib/apt/lists/* + +COPY package.json package-lock.json tsconfig.json ./ +COPY src ./src +COPY scripts ./scripts +COPY docs ./docs + +RUN npm ci \ + && npm run build \ + && useradd -m -u 10001 agent +USER agent + +ENTRYPOINT ["node", "/app/dist/workbench/container-runner.js"] diff --git a/docs/README.codex.md b/docs/README.codex.md new file mode 100644 index 0000000..6401f37 --- /dev/null +++ b/docs/README.codex.md @@ -0,0 +1,31 @@ +# Codex Install + +`skill-optimizer` can be used in Codex as either a plugin or a plain Agent Skill. + +## Plugin Install + +Register this repository as a plugin marketplace: + +```bash +codex plugin marketplace add fastxyz/skill-optimizer +``` + +Open `/plugins`, select the `skill-optimizer` marketplace, and install the `skill-optimizer` plugin. + +Codex reads the repo marketplace from `.agents/plugins/marketplace.json`. That marketplace points at the repository root, where the plugin manifest lives at `.codex-plugin/plugin.json`; bundled skills are read from `skills/`. + +To pin a Git ref while installing the marketplace: + +```bash +codex plugin marketplace add fastxyz/skill-optimizer --ref main +``` + +## Skill-Only Install + +Install only the skill files with the open skills CLI: + +```bash +npx skills add fastxyz/skill-optimizer --skill skill-optimizer -a codex -y +``` + +Restart Codex if the skill does not appear immediately. The canonical skill path is `skills/skill-optimizer/SKILL.md`. diff --git a/docs/README.opencode.md b/docs/README.opencode.md new file mode 100644 index 0000000..93ff893 --- /dev/null +++ b/docs/README.opencode.md @@ -0,0 +1,35 @@ +# skill-optimizer for OpenCode + +Use `skill-optimizer` in OpenCode through the bundled OpenCode plugin. + +## Installation + +Add the plugin to `opencode.json` at user or project scope: + +```json +{ + "plugin": ["skill-optimizer@git+https://github.com/fastxyz/skill-optimizer.git"] +} +``` + +Restart OpenCode. The plugin registers this repository's `skills/` directory so OpenCode can discover `skill-optimizer` without symlinks. + +## Verify + +Use OpenCode's native `skill` tool to list skills or load `skill-optimizer`. + +## Updating + +OpenCode reinstalls git plugins when it starts. To pin a tag or commit, append a ref: + +```json +{ + "plugin": ["skill-optimizer@git+https://github.com/fastxyz/skill-optimizer.git#v2.0.0"] +} +``` + +## How It Works + +The plugin exposes `.opencode/plugins/skill-optimizer.js` and adds the repository `skills/` directory to `config.skills.paths`. + +The canonical skill is `skills/skill-optimizer/SKILL.md`. diff --git a/docs/reference/config-schema.md b/docs/reference/config-schema.md deleted file mode 100644 index 328a086..0000000 --- a/docs/reference/config-schema.md +++ /dev/null @@ -1,46 +0,0 @@ - - - -# Config Schema Reference - -All configuration lives in a single `skill-optimizer.json` file. -Paths in the config are relative to the config file location. - -| Field | Type | Default | Description | -|---|---|---|---| -| `name` | `string` | — | Human-readable project name | -| `target.surface` | `"sdk" | "cli" | "mcp" | "prompt"` | — | Type of callable surface | -| `target.repoPath` | `string` | — | Path to the target repo (default ".") | -| `target.skill` | `string | object` | — | Path to SKILL.md or { source, cache } object | -| `target.discovery.mode` | `"auto" | "manifest"` | — | "auto" = code-first tree-sitter; "manifest" = use provided file only | -| `target.discovery.sources` | `string[]` | — | Source files to scan for callable methods/commands/tools | -| `target.discovery.fallbackManifest` | `string` | — | Path to manifest JSON when code-first discovery is incomplete | -| `target.discovery.language` | `"typescript" | "python" | "rust"` | — | Language for code-first discovery | -| `target.sdk.language` | `"typescript" | "python" | "rust"` | — | SDK language | -| `target.sdk.entrypoints` | `string[]` | — | SDK entry files for discovery | -| `target.cli.commands` | `string` | — | Path to CLI commands manifest JSON (CliCommandDefinition[]) | -| `target.mcp.tools` | `string` | — | Path to MCP tools manifest JSON (OpenAI function tool definitions) | -| `target.scope.include` | `string[]` | — | Glob patterns for actions to include (default ["*"]) | -| `target.scope.exclude` | `string[]` | — | Glob patterns for actions to exclude (default []) | -| `benchmark.format` | `"pi" | "openai" | "anthropic"` | — | LLM transport format: "pi" routes through OpenRouter/Pi (use openrouter/* or openai/* model refs); "openai" calls the OpenAI API directly (supports Codex auth); "anthropic" calls the Anthropic API directly | -| `benchmark.authMode` | `"env" | "codex" | "auto"` | — | How to resolve credentials: env var, ~/.codex/auth.json browser-login tokens, or env-then-codex fallback | -| `benchmark.apiKeyEnv` | `string` | — | Env var name for the API key (default is determined by the model provider prefix: openrouter/ → OPENROUTER_API_KEY, openai/ → OPENAI_API_KEY, anthropic/ → ANTHROPIC_API_KEY; leave unset to use the per-provider default) | -| `benchmark.timeout` | `integer` | — | Milliseconds per model call (default 240000) | -| `benchmark.models` | `object[]` | — | Models to benchmark — at least one required | -| `benchmark.taskGeneration.enabled` | `boolean` | — | Whether to generate tasks automatically (default false) | -| `benchmark.taskGeneration.maxTasks` | `integer` | — | Max tasks to generate — must be >= in-scope action count (default 10) | -| `benchmark.taskGeneration.seed` | `integer` | — | RNG seed for reproducible generation (default 1) | -| `benchmark.taskGeneration.outputDir` | `string` | — | Where to write generated task artifacts (default ".skill-optimizer") | -| `benchmark.output.dir` | `string` | — | Directory where reports are saved (default "benchmark-results/") | -| `benchmark.verdict.perModelFloor` | `number` | — | Minimum per-model pass fraction for PASS verdict (default 0.6) | -| `benchmark.verdict.targetWeightedAverage` | `number` | — | Minimum weighted average across all models for PASS (default 0.7) | -| `optimize.model` | `string` | — | Model for mutation, e.g. openrouter/anthropic/claude-sonnet-4.6 | -| `optimize.authMode` | `"env" | "codex" | "auto"` | — | How to resolve optimizer credentials: env var, ~/.codex/auth.json browser-login tokens, or env-then-codex fallback | -| `optimize.apiKeyEnv` | `string` | — | Env var for the optimizer API key | -| `optimize.thinkingLevel` | `"off" | "minimal" | "low" | "medium" | "high" | "xhigh"` | — | Reasoning depth for mutation calls (default "medium") | -| `optimize.allowedPaths` | `string[]` | — | Paths the optimizer may edit — safety boundary | -| `optimize.validation` | `string[]` | — | Shell commands to run to validate each mutation | -| `optimize.requireCleanGit` | `boolean` | — | Require clean git state before starting (default true) | -| `optimize.maxIterations` | `integer` | — | Maximum optimization iterations (default 5) | -| `optimize.minImprovement` | `number` | — | Minimum weighted-average gain per accepted iteration (default 0.02) | -| `optimize.reportContextMaxBytes` | `integer` | — | Byte budget for mutation context (default 16000) | diff --git a/docs/reference/errors.md b/docs/reference/errors.md deleted file mode 100644 index 4388373..0000000 --- a/docs/reference/errors.md +++ /dev/null @@ -1,226 +0,0 @@ - - - -# Error Reference - -Every `skill-optimizer` error has a code, a short message, and a fix list. -The catch-all `E_UNEXPECTED` appears if an error slips past the known list. - -## Summary - -| Code | Description | Quick fix | -|---|---|---| -| `E_INVALID_SURFACE` | Invalid surface value | Set target.surface to one of: sdk, cli, mcp, prompt | -| `E_MODELS_EMPTY` | benchmark.models is empty or missing | Add at least one model to benchmark.models, e.g.: | -| `E_MODEL_ID_FORMAT` | Model ID is missing a provider prefix | Prefix all model IDs with a supported provider prefix: | -| `E_VERDICT_OUT_OF_RANGE` | Verdict threshold is out of range | Set benchmark.verdict.perModelFloor and targetWeightedAverage to values between 0.0 and 1.0 | -| `E_MAX_ITERATIONS_ZERO` | optimize.maxIterations must be a positive integer | Set optimize.maxIterations to a positive integer, e.g. 5 | -| `E_INVALID_FORMAT` | Invalid benchmark.format value | Set benchmark.format to one of: pi, openai, anthropic | -| `E_REPO_NOT_FOUND` | target.repoPath does not exist or is not a directory | Fix target.repoPath in your skill-optimizer.json to point at an existing directory | -| `E_MISSING_SKILL` | target.skill file not found | Create a SKILL.md at the path specified in target.skill | -| `E_SOURCES_NOT_FOUND` | One or more target.discovery.sources files do not exist | Check that all paths in target.discovery.sources exist in your repo | -| `E_CLI_MANIFEST_NOT_FOUND` | target.cli.commands manifest file not found | Run: skill-optimizer import-commands --from to auto-extract | -| `E_MCP_MANIFEST_NOT_FOUND` | target.mcp.tools manifest file not found | Create the tools.json file at the path specified in target.mcp.tools | -| `E_ALLOWED_PATHS_ESCAPE` | optimize.allowedPaths contains a path outside target.repoPath | All paths in optimize.allowedPaths must be inside target.repoPath | -| `E_OUTPUT_DIR_NOT_WRITABLE` | benchmark.output.dir is not writable | Check directory permissions for the path set in benchmark.output.dir | -| `E_MISSING_API_KEY` | API key environment variable is not set | Export your OpenRouter API key before running: export OPENROUTER_API_KEY=sk-or-... | -| `E_DISCOVERY_EMPTY` | Discovery found zero callable actions | Check that target.discovery.sources points at the right entry file | -| `E_MAXTASKS_TOO_LOW` | benchmark.taskGeneration.maxTasks is less than the in-scope action count | Raise benchmark.taskGeneration.maxTasks to at least the number of in-scope actions | -| `E_COVERAGE_EXHAUSTED` | Task generation could not cover all in-scope actions after 2 retry passes | Add guidance for the uncovered actions to your SKILL.md | -| `E_DIRTY_GIT` | Target repo has uncommitted changes | Commit or stash changes in target.repoPath before running the optimizer | -| `E_GIT_CHECKPOINT_FAILED` | Git checkpoint creation failed | Check disk space and git permissions in target.repoPath | -| `E_VALIDATION_FAILED` | Configured validation command exited non-zero | Fix the issue flagged by the validation command before retrying | -| `E_INIT_AUTO_LOW_CONFIDENCE` | init --auto --yes requires high confidence detection | Run init interactively to review and confirm detection: skill-optimizer init --auto | -| `E_UNEXPECTED` | An unexpected error occurred | Check the full error message and stack trace above for details | - -## Details - -### `E_INVALID_SURFACE` - -**Invalid surface value** - -**How to fix:** -- Set target.surface to one of: sdk, cli, mcp, prompt -- sdk = TypeScript/Python/Rust library, cli = command-line tool, mcp = MCP server, prompt = prompt template / skill document - -### `E_MODELS_EMPTY` - -**benchmark.models is empty or missing** - -**How to fix:** -- Add at least one model to benchmark.models, e.g.: -- { "id": "openrouter/anthropic/claude-sonnet-4.6", "name": "Claude Sonnet", "tier": "flagship" } - -### `E_MODEL_ID_FORMAT` - -**Model ID is missing a provider prefix** - -**How to fix:** -- Prefix all model IDs with a supported provider prefix: -- openrouter// — routed via OpenRouter (e.g. openrouter/anthropic/claude-sonnet-4.6) -- anthropic/ — direct Anthropic API (e.g. anthropic/claude-sonnet-4-6) -- openai/ — direct OpenAI API (e.g. openai/gpt-4.1) -- Browse OpenRouter models at https://openrouter.ai/models - -### `E_VERDICT_OUT_OF_RANGE` - -**Verdict threshold is out of range** - -**How to fix:** -- Set benchmark.verdict.perModelFloor and targetWeightedAverage to values between 0.0 and 1.0 -- Typical values: perModelFloor=0.6, targetWeightedAverage=0.7 - -### `E_MAX_ITERATIONS_ZERO` - -**optimize.maxIterations must be a positive integer** - -**How to fix:** -- Set optimize.maxIterations to a positive integer, e.g. 5 - -### `E_INVALID_FORMAT` - -**Invalid benchmark.format value** - -**How to fix:** -- Set benchmark.format to one of: pi, openai, anthropic - -### `E_REPO_NOT_FOUND` - -**target.repoPath does not exist or is not a directory** - -**How to fix:** -- Fix target.repoPath in your skill-optimizer.json to point at an existing directory -- Paths in the config are relative to the config file location - -### `E_MISSING_SKILL` - -**target.skill file not found** - -**How to fix:** -- Create a SKILL.md at the path specified in target.skill -- Or update target.skill in your config to point at an existing file - -### `E_SOURCES_NOT_FOUND` - -**One or more target.discovery.sources files do not exist** - -**How to fix:** -- Check that all paths in target.discovery.sources exist in your repo -- Paths are relative to target.repoPath -- For CLI: point at your main entry file (e.g. src/cli.ts) -- For MCP: point at your server entry file (e.g. src/server.ts) - -### `E_CLI_MANIFEST_NOT_FOUND` - -**target.cli.commands manifest file not found** - -**How to fix:** -- Run: skill-optimizer import-commands --from to auto-extract -- Or create the file manually and populate it with your CLI commands -- Format: Array of { command, description, options[] } - -### `E_MCP_MANIFEST_NOT_FOUND` - -**target.mcp.tools manifest file not found** - -**How to fix:** -- Create the tools.json file at the path specified in target.mcp.tools -- Format: Array of OpenAI function tool definitions { type: "function", function: { name, description, parameters } } - -### `E_ALLOWED_PATHS_ESCAPE` - -**optimize.allowedPaths contains a path outside target.repoPath** - -**How to fix:** -- All paths in optimize.allowedPaths must be inside target.repoPath -- This is a safety boundary — the optimizer will only edit files within this list - -### `E_OUTPUT_DIR_NOT_WRITABLE` - -**benchmark.output.dir is not writable** - -**How to fix:** -- Check directory permissions for the path set in benchmark.output.dir -- Or change benchmark.output.dir to a path you have write access to - -### `E_MISSING_API_KEY` - -**API key environment variable is not set** - -**How to fix:** -- Export your OpenRouter API key before running: export OPENROUTER_API_KEY=sk-or-... -- Or add it to a .env file alongside your skill-optimizer.json -- Get a key at https://openrouter.ai/keys - -### `E_DISCOVERY_EMPTY` - -**Discovery found zero callable actions** - -**How to fix:** -- Check that target.discovery.sources points at the right entry file -- For SDK: should be your public API entry (e.g. src/index.ts) -- For CLI: should be the file that registers all subcommands -- For MCP: should be the file that registers all tools -- Add a fallback manifest: target.discovery.fallbackManifest or target.cli.commands / target.mcp.tools - -### `E_MAXTASKS_TOO_LOW` - -**benchmark.taskGeneration.maxTasks is less than the in-scope action count** - -**How to fix:** -- Raise benchmark.taskGeneration.maxTasks to at least the number of in-scope actions -- Run: skill-optimizer --dry-run --config ./skill-optimizer.json to see the action count -- Or narrow the scope with target.scope.exclude to reduce the action count - -### `E_COVERAGE_EXHAUSTED` - -**Task generation could not cover all in-scope actions after 2 retry passes** - -**How to fix:** -- Add guidance for the uncovered actions to your SKILL.md -- The error message above names the specific uncovered actions -- Or exclude them with target.scope.exclude if they should not be benchmarked - -### `E_DIRTY_GIT` - -**Target repo has uncommitted changes** - -**How to fix:** -- Commit or stash changes in target.repoPath before running the optimizer -- Run: git -C stash -- Or: git -C add -A && git -C commit -m "wip: before optimizer run" - -### `E_GIT_CHECKPOINT_FAILED` - -**Git checkpoint creation failed** - -**How to fix:** -- Check disk space and git permissions in target.repoPath -- Make sure the directory is a valid git repository -- Run: git -C status to verify git state - -### `E_VALIDATION_FAILED` - -**Configured validation command exited non-zero** - -**How to fix:** -- Fix the issue flagged by the validation command before retrying -- The failing command is listed in optimize.validation in your config -- Run the validation command manually to see the full error output - -### `E_INIT_AUTO_LOW_CONFIDENCE` - -**init --auto --yes requires high confidence detection** - -**How to fix:** -- Run init interactively to review and confirm detection: skill-optimizer init --auto -- Or supply a pre-filled answers file: skill-optimizer init --answers answers.json -- See README for the answers.json format - -### `E_UNEXPECTED` - -**An unexpected error occurred** - -**How to fix:** -- Check the full error message and stack trace above for details -- File an issue at https://github.com/fastxyz/skill-optimizer/issues with the full output diff --git a/docs/workbench.md b/docs/workbench.md new file mode 100644 index 0000000..2bb344d --- /dev/null +++ b/docs/workbench.md @@ -0,0 +1,270 @@ +# Workbench Guide + +`skill-optimizer` runs agent skill evals in a Docker workbench. A model receives a normal user task plus files under `/work`; deterministic graders inspect the final workspace and trace to decide whether the attempt passed. + +## Mental Model + +- A case is one user-like task plus one or more deterministic graders. +- A suite is a matrix of cases and OpenRouter models. +- `references/` is copied into `/work` before the agent starts. Put the skill under test here. +- `workspace/` is copied into `/work` after `references/`. Use it to seed starter files or repos. +- `checks/` and `bin/` are case support files. They are mounted for setup and grading, not for the agent. +- The agent phase sees only `/work`. It cannot see `/case`, `/results`, graders, hidden answers, or hidden metadata. +- Graders define acceptance. They inspect files, command logs, generated artifacts, `answer.json`, `trace.jsonl`, and result state. + +## Directory Layout + +```text +my-eval/ + suite.yml + references/ + my-skill/SKILL.md + my-skill/references/api.md + checks/ + create-inputs.mjs + grade-output.mjs + bin/ + fake-product-cli + workspace/ + starter-repo/ +``` + +Support directory behavior: + +| Directory | Visible To Agent | Purpose | +|-----------|------------------|---------| +| `references/` | yes, copied into `/work` | Skills and reference docs under test | +| `workspace/` | yes, copied into `/work` | Starter repos, input files, seed state | +| `checks/` | no during agent phase | Setup helpers and graders under `$CASE/checks` | +| `bin/` | yes, copied to `/work/bin` | Fake CLIs and command recorders; also mounted under `$CASE/bin` for setup/grading | + +## Suite And Case Files + +Minimal suite: + +```yaml +name: pdf-skill-eval +references: ./references +models: + - openrouter/google/gemini-2.5-flash +env: + - OPENROUTER_API_KEY +timeoutSeconds: 600 +setup: + - node $CASE/checks/create-inputs.mjs +appendSystemPrompt: | + Keep task outputs at the top level of /work unless the user asks otherwise. +cases: + - name: extract-pdf-facts + task: | + Read statement.pdf and write answer.json with the account, quarter, approval code, and risk flags. + graders: + - name: answer-json + command: node $CASE/checks/extract-pdf-facts.mjs +``` + +Case fields: + +- `name`: human-readable case name; inline suite cases use this to derive result slugs. +- `references`: directory copied into `/work`; required for standalone cases and defaulted by suites. +- `task`: natural user request sent to the agent. +- `graders`: shell commands run after the agent; every grader must pass for the case to pass. +- `setup`: shell commands run before the agent. +- `cleanup`: optional shell commands run after grading. +- `env`: host environment variable names forwarded into setup, agent, grading, and cleanup. +- `mcpServers`: optional MCP server map exposed through the agent's `mcp` tool. +- `mcpServices`: optional hidden Docker MCP services started beside the agent container. +- `model`: default model for `run-case`. +- `timeoutSeconds`: agent timeout, default `600`. + +Task prompts should not mention graders, hidden answers, `/case`, `/results`, or eval internals. Ask for the real deliverable just like a user would. + +## MCP Servers + +Cases and suite inline-case defaults may define `mcpServers`. The workbench writes a per-trial `/work/mcporter.json` with only those servers and `imports: []`, then exposes an `mcp` command on `PATH` for the agent. The command delegates to `mcporter` inside the workbench image. + +Example: + +```yaml +mcpServers: + calculator: + baseUrl: http://calculator:3000/mcp + + context7: + baseUrl: https://mcp.context7.com/mcp + headers: + Authorization: "Bearer ${CONTEXT7_API_KEY}" +env: + - OPENROUTER_API_KEY + - CONTEXT7_API_KEY + +mcpServices: + calculator: + command: node + args: + - calculator-server.mjs +``` + +Suite-level `mcpServers` apply to inline cases. Inline case definitions merge by server name, with the inline case winning. External case files do not inherit suite defaults. + +Use `mcpServices` for local MCP servers whose source should not be visible to the agent. Service files live under the case `mcp/` support directory. During `run-case` and `run-suite`, Docker mounts that directory read-only into separate service containers at `/mcp`, joins those containers to a private Docker network, and joins the agent container to the same network. The agent sees only the configured `mcpServers` URL such as `http://calculator:3000/mcp`; it does not mount `/case` or the `mcp/` source directory. Set service ports in the matching `mcpServers` URL rather than in `mcpServices`. + +Remote HTTP/SSE servers must be reachable from Docker. Host-local endpoints need Docker-reachable addresses such as `host.docker.internal`. Direct stdio `mcpServers.command` entries run inside the agent container and are only appropriate when the server implementation is intentionally agent-visible. + +OAuth/browser auth is not supported in v1. Use non-interactive headers, bearer tokens, or environment-variable placeholders. Only env names listed in `env` are forwarded into the containers. + +## Docker Execution Phases + +`run-case` and `run-suite` use Docker for model attempts. Each trial has a prepared case directory, work directory, and result directory on the host; Docker mounts them into phase containers. + +| Phase | Docker Mounts | Working Dir | What Happens | +|-------|---------------|-------------|--------------| +| setup | `/case:ro`, `/work:rw` | `/work` | Runs setup commands and prepares inputs | +| agent | `/work:rw` only | `/work` | Runs the agent/model with the user task | +| grade | `/case:ro`, `/work:rw`, `/results:rw` | `/work` | Runs graders and writes result files | +| cleanup | `/case:ro`, `/work:rw`, `/results:rw` | `/work` | Runs optional cleanup commands | + +Important agent-phase constraints: + +- The agent cannot see `/case` or `/results`. +- The Docker socket is not mounted. +- Global/user Pi skills are not mounted. +- Additional skills are discovered from `/work`. +- If configured, MCP servers are exposed through the `mcp` command using `/work/mcporter.json`. +- Python installs should use `/work/.venv`. +- Environment variables listed in `env` are available unchanged to the agent. + +Use dedicated test accounts and scoped credentials for live integration evals. Treat `trace.jsonl`, `result.json`, grader evidence, stdout/stderr, and preserved workspaces as potentially sensitive. + +## Graders + +Graders are shell commands. They run from `/work` with these environment variables: + +| Variable | Meaning | +|----------|---------| +| `$CASE` | Read-only case directory mounted at `/case` | +| `$WORK` | Mutable workspace used by the agent | +| `$RESULTS` | Result directory containing `trace.jsonl` | + +Preferred grader output is one JSON object on stdout: + +```json +{ "pass": false, "score": 0, "evidence": ["answer.json missing approvalCode"] } +``` + +If no JSON object is printed, exit code `0` passes and non-zero fails. Keep graders deterministic and local. Do not use an LLM judge unless the eval explicitly requires one. + +Good graders check one thing when practical: + +- Exact JSON shape and values. +- PDF, DOCX, PPTX, XLSX, image, ZIP, or database structure. +- Command calls recorded by a fake CLI. +- Static SQL, source code, diffs, or generated files. +- `trace.jsonl` for negative behavior, such as reading an irrelevant skill file. + +## Acceptance Contract + +Graders are the only source of truth for pass/fail. Design graders to inspect whatever local evidence the task should produce, including: + +- Workspace files and generated artifacts under `$WORK` +- Structured outputs such as `answer.json` +- Agent behavior captured in `$RESULTS/trace.jsonl` +- Any additional result-state files your setup/graders write under `$RESULTS` + +Keep graders deterministic and local so acceptance criteria stay stable across model runs. + +## Running Evals + +Run one case: + +```bash +npx tsx src/cli.ts run-case ./case.yml +``` + +Run a case across models: + +```bash +npx tsx src/cli.ts run-case ./case.yml \ + --models openrouter/google/gemini-2.5-flash,openrouter/openai/gpt-5.4 \ + --trials 3 \ + --concurrency 2 +``` + +Run a suite: + +```bash +npx tsx src/cli.ts run-suite ./suite.yml --trials 3 --concurrency 2 +``` + +Useful options: + +- `--out `: results root, default `/.results` or `/.results`. +- `--model `: single `run-case` model override. +- `--models `: comma-separated `run-case` model list. +- `--trials `: independent trials per model/case, default `1`. +- `--concurrency `: maximum concurrent trial containers, default `1`. +- `--image `: Docker image name, default `skill-optimizer-workbench:local`. +- `--keep-workspace`: copy final `/work` to results; failed trials are always preserved. + +`run-suite` always uses `models:` from `suite.yml`; it does not have a model override flag. + +## Outputs + +Single-trial `run-case` output: + +```text +case/.results// + trace.jsonl + result.json + summary.json + workspace/ # on failure or --keep-workspace +``` + +Matrix `run-case` output: + +```text +case/.results// + run-result.json + trials/--001/trace.jsonl + trials/--001/result.json +``` + +`run-suite` output: + +```text +suite/.results// + suite-result.json + trials/----001/trace.jsonl + trials/----001/result.json +``` + +`result.json` includes `pass`, `score`, `evidence`, per-grader results, duration, turns, tool counts, tokens, and cost. Aggregates include trial pass rate, pass@k, pass^k, mean score, and relative result/trace paths. + +`trace.jsonl` is the primary debugging source. It records assistant messages, tool calls, tool results, stop reasons, and errors. Use it to understand why a model failed or to grade negative cases. + +## Debugging Failed Runs + +1. Read the failing trial `result.json` evidence. +2. Inspect `graders[]` to identify the failed grader. +3. Open `summary.json` for final assistant text and commands. +4. Open `trace.jsonl` to inspect tool calls and file reads. +5. Inspect preserved `workspace/` for failed trials. +6. Classify the failure as unclear skill guidance, missing reference material, brittle grader, unrealistic input data, task ambiguity, or product/code bug. +7. Update the target skill, references, inputs, graders, or code according to that diagnosis. +8. Re-run the same case or suite and compare grader evidence across the target models/trials. + +## Example + +The tracked PDF demo is the best starting point: + +```bash +npx tsx src/cli.ts run-suite examples/workbench/pdf/suite.yml --trials 1 +npx tsx src/cli.ts run-suite examples/workbench/mcp/suite.yml --trials 1 +``` + +Useful files: + +- `examples/workbench/pdf/suite.yml`: inline suite with models, setup, graders, and append prompt. +- `examples/workbench/pdf/references/pdf-skill/SKILL.md`: skill under test copied into `/work`. +- `examples/workbench/pdf/checks/*.mjs`: deterministic graders and setup helpers. +- `examples/workbench/pdf/README.md`: demo walkthrough. diff --git a/examples/workbench/README.md b/examples/workbench/README.md new file mode 100644 index 0000000..47377ca --- /dev/null +++ b/examples/workbench/README.md @@ -0,0 +1,23 @@ +# Workbench Examples + +These examples are small, demoable suites that exercise the Docker workbench end to end. + +See `../../docs/workbench.md` for the full workbench model, including cases, suites, graders, Docker phases, and result files. + +## PDF Skill Demo + +```bash +npx tsx src/cli.ts run-suite examples/workbench/pdf/suite.yml --trials 1 +``` + +Graders are the acceptance contract. They evaluate agent outputs from `/work`, generated artifacts, `answer.json`, and behavior captured in `trace.jsonl`. + +`run-suite` runs the configured model matrix and writes `trace.jsonl`, `result.json`, and failed workspaces under `examples/workbench/pdf/.results//`. + +## MCP Calculator Demo + +```bash +npx tsx src/cli.ts run-suite examples/workbench/mcp/suite.yml --trials 1 +``` + +The MCP demo starts a local calculator MCP server as a separate hidden Docker service and exposes it through the workbench `mcp` command. The server has `add`, `subtract`, `multiply`, and `divide`; the grader checks both `answer.json` and `trace.jsonl` for MCP usage. diff --git a/examples/workbench/mcp/README.md b/examples/workbench/mcp/README.md new file mode 100644 index 0000000..5b84aca --- /dev/null +++ b/examples/workbench/mcp/README.md @@ -0,0 +1,11 @@ +# MCP Calculator Workbench Example + +This example shows a local MCP server started as a separate hidden Docker service beside the agent container. The agent sees the `calculator` MCP URL through the workbench `mcp` command, but it cannot read the server source file. The server exposes calculator tools: `add`, `subtract`, `multiply`, and `divide`. + +Run a model trial: + +```bash +npx tsx src/cli.ts run-suite examples/workbench/mcp/suite.yml --trials 1 +``` + +The case asks the agent to compute the expression and write `answer.json`. The grader checks the computed answer and verifies that the trace contains separate bash calls to `mcp call calculator.add`, `calculator.multiply`, `calculator.subtract`, and `calculator.divide`. diff --git a/examples/workbench/mcp/checks/calculator-answer.mjs b/examples/workbench/mcp/checks/calculator-answer.mjs new file mode 100644 index 0000000..07c30de --- /dev/null +++ b/examples/workbench/mcp/checks/calculator-answer.mjs @@ -0,0 +1,61 @@ +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const expectedExpression = '((17 + 25) * 3 - 18) / 6'; +const expectedResult = 18; +const failures = []; + +const answerPath = join(process.env.WORK, 'answer.json'); +if (!existsSync(answerPath)) { + failures.push('answer.json was not created'); +} else { + try { + const answer = JSON.parse(readFileSync(answerPath, 'utf-8')); + if (answer.expression !== expectedExpression) { + failures.push(`expression mismatch: ${JSON.stringify(answer.expression)}`); + } + if (answer.result !== expectedResult) { + failures.push(`result mismatch: ${JSON.stringify(answer.result)}`); + } + } catch (error) { + failures.push(`answer.json is not valid JSON: ${error.message}`); + } +} + +const tracePath = join(process.env.RESULTS, 'trace.jsonl'); +if (!existsSync(tracePath)) { + failures.push('trace.jsonl was not created'); +} else { + const trace = readFileSync(tracePath, 'utf-8').trim().split(/\r?\n/).flatMap((line) => { + try { + return [JSON.parse(line)]; + } catch { + return []; + } + }); + const requiredTools = [ + { tool: 'add', pattern: /\bmcp\s+call\s+calculator\.add\b/ }, + { tool: 'multiply', pattern: /\bmcp\s+call\s+calculator\.multiply\b/ }, + { tool: 'subtract', pattern: /\bmcp\s+call\s+calculator\.subtract\b/ }, + { tool: 'divide', pattern: /\bmcp\s+call\s+calculator\.divide\b/ }, + ]; + const bashCommands = trace.flatMap((entry) => { + if (entry.type !== 'tool_call' || entry.name !== 'bash') return []; + const args = entry.arguments ?? {}; + return typeof args.command === 'string' ? [args.command] : []; + }); + for (const { tool, pattern } of requiredTools) { + if (!bashCommands.some((command) => pattern.test(command))) { + failures.push(`trace does not contain calculator.${tool} MCP call`); + } + } +} + +const pass = failures.length === 0; +console.log(JSON.stringify({ + pass, + score: pass ? 1 : 0, + evidence: pass ? ['answer matched and all calculator MCP tools were used'] : failures, +})); + +process.exit(pass ? 0 : 1); diff --git a/examples/workbench/mcp/mcp/calculator-server.mjs b/examples/workbench/mcp/mcp/calculator-server.mjs new file mode 100644 index 0000000..5b12ebb --- /dev/null +++ b/examples/workbench/mcp/mcp/calculator-server.mjs @@ -0,0 +1,120 @@ +import { randomUUID } from 'node:crypto'; +import { createRequire } from 'node:module'; + +const requireFromApp = createRequire('/app/package.json'); +const { createMcpExpressApp } = requireFromApp('@modelcontextprotocol/sdk/server/express.js'); +const { Server } = requireFromApp('@modelcontextprotocol/sdk/server/index.js'); +const { StreamableHTTPServerTransport } = requireFromApp('@modelcontextprotocol/sdk/server/streamableHttp.js'); +const { CallToolRequestSchema, isInitializeRequest, ListToolsRequestSchema } = requireFromApp('@modelcontextprotocol/sdk/types.js'); + +const tools = [ + { name: 'add', description: 'Add two numbers.', inputSchema: binaryNumberSchema('a', 'b') }, + { name: 'subtract', description: 'Subtract b from a.', inputSchema: binaryNumberSchema('a', 'b') }, + { name: 'multiply', description: 'Multiply two numbers.', inputSchema: binaryNumberSchema('a', 'b') }, + { name: 'divide', description: 'Divide a by b.', inputSchema: binaryNumberSchema('a', 'b') }, +]; + +const transports = {}; +const app = createMcpExpressApp({ host: '0.0.0.0' }); + +app.post('/mcp', async (req, res) => { + try { + const sessionId = req.headers['mcp-session-id']; + let transport = typeof sessionId === 'string' ? transports[sessionId] : undefined; + + if (!transport && isInitializeRequest(req.body)) { + transport = new StreamableHTTPServerTransport({ + sessionIdGenerator: () => randomUUID(), + onsessioninitialized: (newSessionId) => { + transports[newSessionId] = transport; + }, + }); + await createCalculatorServer().connect(transport); + } + + if (!transport) { + res.status(400).json({ jsonrpc: '2.0', error: { code: -32000, message: 'Bad Request' }, id: null }); + return; + } + + await transport.handleRequest(req, res, req.body); + } catch (error) { + if (!res.headersSent) { + res.status(500).json({ + jsonrpc: '2.0', + error: { code: -32603, message: error instanceof Error ? error.message : String(error) }, + id: null, + }); + } + } +}); + +app.get('/mcp', async (req, res) => { + const sessionId = req.headers['mcp-session-id']; + const transport = typeof sessionId === 'string' ? transports[sessionId] : undefined; + if (!transport) { + res.status(400).send('Invalid or missing session ID'); + return; + } + await transport.handleRequest(req, res); +}); + +app.listen(3000, (error) => { + if (error) { + console.error(error); + process.exit(1); + } + console.error('calculator MCP server listening on :3000'); +}); + +function createCalculatorServer() { + const server = new Server( + { name: 'calculator', version: '1.0.0' }, + { capabilities: { tools: {} } }, + ); + + server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools })); + server.setRequestHandler(CallToolRequestSchema, async (request) => { + const name = request.params.name; + const args = request.params.arguments ?? {}; + const a = readNumber(args.a, 'a'); + const b = readNumber(args.b, 'b'); + + if (name === 'add') return toolResult(a + b); + if (name === 'subtract') return toolResult(a - b); + if (name === 'multiply') return toolResult(a * b); + if (name === 'divide') { + if (b === 0) throw new Error('Cannot divide by zero'); + return toolResult(a / b); + } + throw new Error(`Unknown calculator tool: ${name}`); + }); + + return server; +} + +function toolResult(result) { + return { + content: [{ type: 'text', text: String(result) }], + structuredContent: { result }, + }; +} + +function binaryNumberSchema(left, right) { + return { + type: 'object', + properties: { + [left]: { type: 'number' }, + [right]: { type: 'number' }, + }, + required: [left, right], + additionalProperties: false, + }; +} + +function readNumber(value, name) { + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new Error(`Argument ${name} must be a finite number`); + } + return value; +} diff --git a/examples/workbench/mcp/references/README.md b/examples/workbench/mcp/references/README.md new file mode 100644 index 0000000..6832bfa --- /dev/null +++ b/examples/workbench/mcp/references/README.md @@ -0,0 +1,3 @@ +This directory is copied into `/work` for the agent. + +The calculator MCP server source is intentionally not here. It lives under the case's hidden `mcp/` support directory and is started by the Docker workbench as a separate service container. diff --git a/examples/workbench/mcp/suite.yml b/examples/workbench/mcp/suite.yml new file mode 100644 index 0000000..5ac689a --- /dev/null +++ b/examples/workbench/mcp/suite.yml @@ -0,0 +1,30 @@ +name: mcp-calculator-example +references: ./references +models: + - openrouter/google/gemini-2.5-flash +env: + - OPENROUTER_API_KEY +timeoutSeconds: 600 +mcpServers: + calculator: + baseUrl: http://calculator:3000/mcp +mcpServices: + calculator: + command: node + args: + - calculator-server.mjs +cases: + - name: use-calculator-mcp + task: | + Compute this expression: + + ((17 + 25) * 3 - 18) / 6 + + Write answer.json with this exact shape: + { + "expression": "((17 + 25) * 3 - 18) / 6", + "result": + } + graders: + - name: calculator-answer + command: node $CASE/checks/calculator-answer.mjs diff --git a/examples/workbench/pdf/README.md b/examples/workbench/pdf/README.md new file mode 100644 index 0000000..3fdf617 --- /dev/null +++ b/examples/workbench/pdf/README.md @@ -0,0 +1,36 @@ +# PDF Workbench Demo + +This suite demonstrates the main workbench features with a PDF skill: + +- `models`: suite-owned model matrix +- `env`: API key forwarding into the agent container +- `appendSystemPrompt`: suite-wide prompt additions +- `setup`: input generation before the agent starts +- `graders`: deterministic post-run checks +- trace grading: the negative case checks `trace.jsonl` for forbidden skill reads + +## Run The Demo + +```bash +npx tsx src/cli.ts run-suite examples/workbench/pdf/suite.yml --trials 1 +``` + +`run-suite` runs each case against the suite models. Results are written to: + +```text +examples/workbench/pdf/.results// + suite-result.json + trials/----001/result.json + trials/----001/trace.jsonl +``` + +Failed trials also preserve `workspace/` so you can inspect exactly what the agent wrote. + +## Cases + +- `extract-pdf-facts`: reads `statement.pdf` and writes exact structured JSON. +- `split-customer-packet`: keeps only customer-copy pages from a packet PDF. +- `build-briefing-pdf`: creates a valid one-page briefing PDF. +- `no-pdf-skill-needed`: writes a text file and fails if the agent reads `/work/pdf-skill/SKILL.md`. + +The example skill under `references/pdf-skill/` is intentionally small and demo-safe. Replace it with a real skill to evaluate production PDF guidance. diff --git a/examples/workbench/pdf/checks/_pdf.mjs b/examples/workbench/pdf/checks/_pdf.mjs new file mode 100644 index 0000000..2a263cb --- /dev/null +++ b/examples/workbench/pdf/checks/_pdf.mjs @@ -0,0 +1,305 @@ +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { inflateSync } from 'node:zlib'; + +function escapePdfLiteral(value) { + return String(value) + .replace(/\\/g, '\\\\') + .replace(/\(/g, '\\(') + .replace(/\)/g, '\\)'); +} + +function unescapePdfLiteral(value) { + let output = ''; + for (let index = 0; index < value.length; index += 1) { + const char = value[index]; + if (char !== '\\') { + output += char; + continue; + } + + const next = value[index + 1]; + if (next === undefined) { + output += '\\'; + continue; + } + + if (/[0-7]/.test(next)) { + const match = value.slice(index + 1).match(/^[0-7]{1,3}/)?.[0] ?? next; + output += String.fromCharCode(Number.parseInt(match, 8)); + index += match.length; + continue; + } + + index += 1; + if (next === 'n') output += '\n'; + else if (next === 'r') output += '\r'; + else if (next === 't') output += '\t'; + else if (next === 'b') output += '\b'; + else if (next === 'f') output += '\f'; + else if (next === '\n' || next === '\r') output += ''; + else output += next; + } + return output; +} + +function contentStreamForPage(pageText) { + const lines = String(pageText).split(/\r?\n/); + return [ + 'BT', + '/F1 12 Tf', + '72 720 Td', + ...lines.flatMap((line, index) => [ + index === 0 ? null : '0 -18 Td', + `(${escapePdfLiteral(line)}) Tj`, + ]).filter(Boolean), + 'ET', + ].join('\n'); +} + +export function createPdf(filePath, pages) { + if (!Array.isArray(pages) || pages.length === 0) { + throw new Error('createPdf requires at least one page'); + } + + const pageObjectIds = pages.map((_, index) => 4 + index * 2); + const contentObjectIds = pages.map((_, index) => 5 + index * 2); + const objects = new Map(); + + objects.set(1, '<< /Type /Catalog /Pages 2 0 R >>'); + objects.set(2, `<< /Type /Pages /Kids [${pageObjectIds.map((id) => `${id} 0 R`).join(' ')}] /Count ${pages.length} >>`); + objects.set(3, '<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>'); + + for (let index = 0; index < pages.length; index += 1) { + const pageObjectId = pageObjectIds[index]; + const contentObjectId = contentObjectIds[index]; + const stream = contentStreamForPage(pages[index]); + objects.set(pageObjectId, `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >> /Contents ${contentObjectId} 0 R >>`); + objects.set(contentObjectId, `<< /Length ${Buffer.byteLength(stream, 'ascii')} >>\nstream\n${stream}\nendstream`); + } + + const maxObjectId = Math.max(...objects.keys()); + let pdf = '%PDF-1.4\n'; + const offsets = [0]; + for (let objectId = 1; objectId <= maxObjectId; objectId += 1) { + const body = objects.get(objectId); + if (!body) { + throw new Error(`missing PDF object ${objectId}`); + } + offsets[objectId] = Buffer.byteLength(pdf, 'ascii'); + pdf += `${objectId} 0 obj\n${body}\nendobj\n`; + } + const xrefOffset = Buffer.byteLength(pdf, 'ascii'); + pdf += `xref\n0 ${maxObjectId + 1}\n`; + pdf += '0000000000 65535 f \n'; + for (let objectId = 1; objectId <= maxObjectId; objectId += 1) { + pdf += `${String(offsets[objectId]).padStart(10, '0')} 00000 n \n`; + } + pdf += `trailer\n<< /Size ${maxObjectId + 1} /Root 1 0 R >>\nstartxref\n${xrefOffset}\n%%EOF\n`; + + mkdirSync(dirname(filePath), { recursive: true }); + writeFileSync(filePath, pdf, 'ascii'); +} + +export function readTextFile(filePath) { + return readFileSync(filePath, 'latin1'); +} + +export function isPdfFile(filePath) { + const raw = readTextFile(filePath); + return raw.startsWith('%PDF-') && raw.includes('%%EOF'); +} + +export function countPdfPages(filePath) { + const raw = readTextFile(filePath); + return [...raw.matchAll(/\/Type\s*\/Page\b/g)].length; +} + +function readFilters(dictionary) { + const match = dictionary.match(/\/Filter\s*(\[[^\]]+\]|\/\w+)/); + if (!match) { + return []; + } + return [...match[1].matchAll(/\/(\w+)/g)].map((filter) => filter[1]); +} + +function ascii85Decode(buffer) { + const input = buffer.toString('latin1').replace(/\s+/g, '').replace(/^<~/, '').replace(/~>$/, ''); + const bytes = []; + let group = []; + + const flush = (values, outputLength) => { + let value = 0; + for (const digit of values) { + value = value * 85 + digit; + } + const decoded = [ + (value >>> 24) & 0xff, + (value >>> 16) & 0xff, + (value >>> 8) & 0xff, + value & 0xff, + ]; + bytes.push(...decoded.slice(0, outputLength)); + }; + + for (const char of input) { + if (char === 'z' && group.length === 0) { + bytes.push(0, 0, 0, 0); + continue; + } + group.push(char.charCodeAt(0) - 33); + if (group.length === 5) { + flush(group, 4); + group = []; + } + } + + if (group.length > 0) { + const outputLength = group.length - 1; + while (group.length < 5) group.push(84); + flush(group, outputLength); + } + + return Buffer.from(bytes); +} + +function decodeStream(filters, streamContent) { + let buffer = Buffer.from(streamContent, 'latin1'); + for (const filter of filters) { + if (filter === 'ASCII85Decode' || filter === 'A85') { + buffer = ascii85Decode(buffer); + } else if (filter === 'FlateDecode' || filter === 'Fl') { + buffer = inflateSync(buffer); + } + } + return buffer.toString('latin1'); +} + +function decodedContentStreams(raw) { + const streams = [raw]; + const pattern = /(<<[\s\S]*?>>)\s*stream\r?\n([\s\S]*?)\r?\nendstream/g; + let match; + while ((match = pattern.exec(raw)) !== null) { + const filters = readFilters(match[1]); + try { + streams.push(decodeStream(filters, match[2])); + } catch { + streams.push(match[2]); + } + } + return streams; +} + +function extractPdfStringLiterals(value) { + const texts = []; + const literalPattern = /\(((?:\\.|[^\\)])*)\)/g; + let match; + while ((match = literalPattern.exec(value)) !== null) { + texts.push(unescapePdfLiteral(match[1])); + } + return texts; +} + +export function extractSimplePdfText(filePath) { + const raw = readTextFile(filePath); + const texts = []; + + for (const stream of decodedContentStreams(raw)) { + const tjPattern = /\(((?:\\.|[^\\)])*)\)\s*Tj/g; + let tjMatch; + while ((tjMatch = tjPattern.exec(stream)) !== null) { + texts.push(unescapePdfLiteral(tjMatch[1])); + } + + const tjArrayPattern = /\[([\s\S]*?)\]\s*TJ/g; + let arrayMatch; + while ((arrayMatch = tjArrayPattern.exec(stream)) !== null) { + texts.push(extractPdfStringLiterals(arrayMatch[1]).join('')); + } + } + + return texts.join('\n'); +} + +export function readJson(filePath) { + return JSON.parse(readFileSync(filePath, 'utf-8')); +} + +export function result(pass, evidence, score = pass ? 1 : 0) { + return { + pass, + score, + evidence: Array.isArray(evidence) ? evidence : [String(evidence)], + }; +} + +export function printResult(passOrResult, evidence, score) { + const output = typeof passOrResult === 'object' && passOrResult !== null + ? passOrResult + : result(passOrResult, evidence, score); + console.log(JSON.stringify(output)); + process.exit(output.pass ? 0 : 1); +} + +export function requireEnv(name) { + const value = process.env[name]; + if (!value) { + printResult(false, `${name} env var is required`); + } + return value; +} + +export function missingStrings(text, expected) { + return expected.filter((value) => !text.includes(value)); +} + +export function writeInputPdfs(rootDir) { + createPdf(join(rootDir, 'statement.pdf'), [ + [ + 'Quarterly Statement', + 'Account: Delta Orchard Cooperative', + 'Quarter: Q4 2025', + 'Total Revenue: $128,430.00', + 'Risk Flag: inventory write-down', + 'Risk Flag: late supplier audit', + 'Approval Code: PDF-7429', + ].join('\n'), + ]); + + createPdf(join(rootDir, 'customer-packet.pdf'), [ + [ + 'CUSTOMER COPY', + 'Invoice: C-204', + 'Status: PAID', + 'Customer: Northwind Labs', + ].join('\n'), + [ + 'INTERNAL NOTES', + 'Do not share with customer.', + 'Margin review pending.', + ].join('\n'), + [ + 'CUSTOMER COPY', + 'Warranty Code: W-8832', + 'Support Tier: Priority', + ].join('\n'), + ]); + + createPdf(join(rootDir, 'briefing-source.pdf'), [ + [ + 'Renewal Source Notes', + 'Source: Alpine Sensors', + 'Decision: approve expedited renewal', + 'Deadline: 2026-05-14', + 'draft-only note: internal discount floor is 18 percent', + ].join('\n'), + ]); +} + +if (process.argv[1] === new URL(import.meta.url).pathname && process.argv[2] === 'write-inputs') { + const outputDir = process.argv[3]; + if (!outputDir) { + throw new Error('Usage: node _pdf.mjs write-inputs '); + } + writeInputPdfs(outputDir); +} diff --git a/examples/workbench/pdf/checks/_trace.mjs b/examples/workbench/pdf/checks/_trace.mjs new file mode 100644 index 0000000..60aea18 --- /dev/null +++ b/examples/workbench/pdf/checks/_trace.mjs @@ -0,0 +1,60 @@ +import { readFileSync } from 'node:fs'; + +export function readTraceJsonl(tracePath) { + return readFileSync(tracePath, 'utf-8') + .trim() + .split(/\r?\n/) + .filter(Boolean) + .flatMap((line) => { + try { + return [JSON.parse(line)]; + } catch { + return []; + } + }); +} + +function readPathFromToolCall(entry) { + if (entry?.type !== 'tool_call' || entry.name !== 'read') { + return undefined; + } + const args = entry.arguments; + if (!args || typeof args !== 'object') { + return undefined; + } + if (typeof args.path === 'string') return args.path; + if (typeof args.filePath === 'string') return args.filePath; + return undefined; +} + +function matchesPath(path, pattern) { + if (pattern instanceof RegExp) { + return pattern.test(path); + } + return path === String(pattern); +} + +export function noReadPath(tracePath, forbiddenPath) { + const forbidden = readTraceJsonl(tracePath) + .map(readPathFromToolCall) + .filter((path) => typeof path === 'string' && matchesPath(path, forbiddenPath)); + + if (forbidden.length > 0) { + return { + pass: false, + score: 0, + evidence: forbidden.map((path) => `forbidden read path: ${path}`), + }; + } + + return { + pass: true, + score: 1, + evidence: ['no forbidden read paths found'], + }; +} + +export function printResult(result) { + console.log(JSON.stringify(result)); + process.exit(result.pass ? 0 : 1); +} diff --git a/examples/workbench/pdf/checks/build-briefing-pdf.mjs b/examples/workbench/pdf/checks/build-briefing-pdf.mjs new file mode 100644 index 0000000..5c4e137 --- /dev/null +++ b/examples/workbench/pdf/checks/build-briefing-pdf.mjs @@ -0,0 +1,21 @@ +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; + +import { countPdfPages, isPdfFile, printResult, requireEnv } from './_pdf.mjs'; + +const outputPath = join(requireEnv('WORK'), 'briefing.pdf'); + +if (!existsSync(outputPath)) { + printResult(false, 'briefing.pdf was not created'); +} +if (!isPdfFile(outputPath)) { + printResult(false, 'briefing.pdf is not a valid-looking PDF with header and EOF marker'); +} + +const failures = []; +const pageCount = countPdfPages(outputPath); +if (pageCount !== 1) { + failures.push(`expected 1 page, found ${pageCount}`); +} + +printResult(failures.length === 0, failures.length === 0 ? 'briefing.pdf is a valid one-page PDF' : failures); diff --git a/examples/workbench/pdf/checks/extract-pdf-facts.mjs b/examples/workbench/pdf/checks/extract-pdf-facts.mjs new file mode 100644 index 0000000..83af938 --- /dev/null +++ b/examples/workbench/pdf/checks/extract-pdf-facts.mjs @@ -0,0 +1,33 @@ +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; + +import { printResult, readJson, requireEnv } from './_pdf.mjs'; + +const answerPath = join(requireEnv('WORK'), 'answer.json'); + +if (!existsSync(answerPath)) { + printResult(false, 'answer.json was not created'); +} + +let answer; +try { + answer = readJson(answerPath); +} catch (error) { + printResult(false, `answer.json is not valid JSON: ${error instanceof Error ? error.message : String(error)}`); +} + +const failures = []; +if (answer.account !== 'Delta Orchard Cooperative') failures.push('account mismatch'); +if (answer.quarter !== 'Q4 2025') failures.push('quarter mismatch'); +if (answer.totalRevenue !== 128430) failures.push('totalRevenue must be numeric 128430'); +if (!Array.isArray(answer.riskFlags)) { + failures.push('riskFlags must be an array'); +} else { + for (const expected of ['inventory write-down', 'late supplier audit']) { + if (!answer.riskFlags.includes(expected)) failures.push(`missing risk flag: ${expected}`); + } + if (answer.riskFlags.length !== 2) failures.push('riskFlags should contain exactly the two source risk flags'); +} +if (answer.approvalCode !== 'PDF-7429') failures.push('approvalCode mismatch'); + +printResult(failures.length === 0, failures.length === 0 ? 'answer.json matched expected PDF facts' : failures); diff --git a/examples/workbench/pdf/checks/no-pdf-skill.mjs b/examples/workbench/pdf/checks/no-pdf-skill.mjs new file mode 100644 index 0000000..efd6bed --- /dev/null +++ b/examples/workbench/pdf/checks/no-pdf-skill.mjs @@ -0,0 +1,28 @@ +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +import { printResult as printPdfResult, requireEnv, result } from './_pdf.mjs'; +import { noReadPath } from './_trace.mjs'; + +const workDir = requireEnv('WORK'); +const resultsDir = requireEnv('RESULTS'); +const notePath = join(workDir, 'note.txt'); +const tracePath = join(resultsDir, 'trace.jsonl'); +const failures = []; + +if (!existsSync(notePath)) { + failures.push('note.txt was not created'); +} else if (readFileSync(notePath, 'utf-8').trim() !== 'done') { + failures.push('note.txt did not contain exactly: done'); +} + +if (existsSync(tracePath)) { + const traceResult = noReadPath(tracePath, /\/pdf-skill\/SKILL\.md$/); + if (!traceResult.pass) { + failures.push(...traceResult.evidence); + } +} + +printPdfResult(failures.length === 0 + ? result(true, 'note.txt was created without reading the PDF skill') + : result(false, failures)); diff --git a/examples/workbench/pdf/checks/split-customer-packet.mjs b/examples/workbench/pdf/checks/split-customer-packet.mjs new file mode 100644 index 0000000..c6e1182 --- /dev/null +++ b/examples/workbench/pdf/checks/split-customer-packet.mjs @@ -0,0 +1,33 @@ +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; + +import { countPdfPages, extractSimplePdfText, isPdfFile, missingStrings, printResult, requireEnv } from './_pdf.mjs'; + +const outputPath = join(requireEnv('WORK'), 'customer-copy.pdf'); + +if (!existsSync(outputPath)) { + printResult(false, 'customer-copy.pdf was not created'); +} +if (!isPdfFile(outputPath)) { + printResult(false, 'customer-copy.pdf is not a valid-looking PDF with header and EOF marker'); +} + +const text = extractSimplePdfText(outputPath); +const missing = missingStrings(text, [ + 'CUSTOMER COPY', + 'Invoice: C-204', + 'Status: PAID', + 'Warranty Code: W-8832', + 'Support Tier: Priority', +]); +const failures = [...missing.map((value) => `missing expected text: ${value}`)]; + +if (text.includes('INTERNAL NOTES') || text.includes('Margin review pending')) { + failures.push('customer-copy.pdf includes internal-only page text'); +} +const pageCount = countPdfPages(outputPath); +if (pageCount !== 2) { + failures.push(`expected 2 pages, found ${pageCount}`); +} + +printResult(failures.length === 0, failures.length === 0 ? 'customer-copy.pdf contains only customer pages' : failures); diff --git a/examples/workbench/pdf/references/pdf-skill/SKILL.md b/examples/workbench/pdf/references/pdf-skill/SKILL.md new file mode 100644 index 0000000..d8f9a45 --- /dev/null +++ b/examples/workbench/pdf/references/pdf-skill/SKILL.md @@ -0,0 +1,82 @@ +--- +name: pdf +description: Use this skill when a task requires reading, creating, splitting, merging, or otherwise manipulating PDF files. +--- + +# PDF Skill Demo + +Use Python packages installed in `/work/.venv` for PDF work. Common choices: + +- `pypdf` for reading, splitting, and writing pages +- `pdfplumber` for extracting text from PDFs +- `reportlab` for creating new PDFs + +Always inspect the extracted text before writing parsing regexes. Do not guess labels or field formats from the task prompt. + +Example text extraction: + +```python +from pypdf import PdfReader + +reader = PdfReader("input.pdf") +text = "\n".join(page.extract_text() or "" for page in reader.pages) +print(text) +``` + +Example structured extraction after inspecting text: + +```python +from pypdf import PdfReader +import json + +reader = PdfReader("statement.pdf") +text = "\n".join(page.extract_text() or "" for page in reader.pages) +lines = [line.strip() for line in text.splitlines() if line.strip()] + +answer = {"riskFlags": []} +for line in lines: + if line.startswith("Account:"): + answer["account"] = line.split(":", 1)[1].strip() + elif line.startswith("Quarter:"): + answer["quarter"] = line.split(":", 1)[1].strip() + elif line.startswith("Total Revenue:"): + raw = line.split(":", 1)[1].strip().replace("$", "").replace(",", "") + answer["totalRevenue"] = float(raw) + elif line.startswith("Risk Flag:"): + answer["riskFlags"].append(line.split(":", 1)[1].strip()) + elif line.startswith("Approval Code:"): + answer["approvalCode"] = line.split(":", 1)[1].strip() + +with open("answer.json", "w") as output: + json.dump(answer, output, indent=2) +``` + +Example page filtering: + +```python +from pypdf import PdfReader, PdfWriter + +reader = PdfReader("input.pdf") +writer = PdfWriter() +writer.add_page(reader.pages[0]) + +with open("output.pdf", "wb") as output: + writer.write(output) +``` + +Example page filtering by extracted page text: + +```python +from pypdf import PdfReader, PdfWriter + +reader = PdfReader("customer-packet.pdf") +writer = PdfWriter() + +for page in reader.pages: + text = page.extract_text() or "" + if "CUSTOMER COPY" in text and "INTERNAL NOTES" not in text: + writer.add_page(page) + +with open("customer-copy.pdf", "wb") as output: + writer.write(output) +``` diff --git a/examples/workbench/pdf/suite.yml b/examples/workbench/pdf/suite.yml new file mode 100644 index 0000000..0dc13de --- /dev/null +++ b/examples/workbench/pdf/suite.yml @@ -0,0 +1,57 @@ +name: pdf-workbench-example +references: ./references +appendSystemPrompt: | + Keep task outputs at the top level of /work unless the user asks for a different path. +models: + - openrouter/google/gemini-2.5-flash +env: + - OPENROUTER_API_KEY +timeoutSeconds: 600 +setup: + - mkdir -p input + - node $CASE/checks/_pdf.mjs write-inputs input + - cp input/statement.pdf statement.pdf + - cp input/customer-packet.pdf customer-packet.pdf + - cp input/briefing-source.pdf briefing-source.pdf +cases: + - name: extract-pdf-facts + task: | + Extract the key facts from statement.pdf and write answer.json with this exact schema: + { + "account": string, + "quarter": string, + "totalRevenue": number, + "riskFlags": string[], + "approvalCode": string + } + Use numeric dollars without commas or a currency symbol for totalRevenue. + graders: + - name: answer-json + command: node $CASE/checks/extract-pdf-facts.mjs + + - name: split-customer-packet + task: | + Create customer-copy.pdf from customer-packet.pdf. Include only the pages marked CUSTOMER COPY, in their original order. Exclude the page marked INTERNAL NOTES. The output must be a PDF, not a text file. + graders: + - name: customer-copy-pdf + command: node $CASE/checks/split-customer-packet.mjs + + - name: build-briefing-pdf + task: | + Create briefing.pdf as a one-page PDF briefing based on briefing-source.pdf. It must include these exact lines: + PDF Skill Briefing + Source: Alpine Sensors + Decision: approve expedited renewal + Deadline: 2026-05-14 + Do not include the draft-only note from the source document. + graders: + - name: briefing-pdf + command: node $CASE/checks/build-briefing-pdf.mjs + + - name: no-pdf-skill-needed + task: | + Write note.txt with exactly this text: + done + graders: + - name: note-without-pdf-skill + command: node $CASE/checks/no-pdf-skill.mjs diff --git a/gemini-extension.json b/gemini-extension.json new file mode 100644 index 0000000..fbac849 --- /dev/null +++ b/gemini-extension.json @@ -0,0 +1,6 @@ +{ + "name": "skill-optimizer", + "description": "Build, run, and improve evals for agent skills.", + "version": "2.0.0", + "contextFileName": "GEMINI.md" +} diff --git a/mock-repos/README.md b/mock-repos/README.md deleted file mode 100644 index 264fee3..0000000 --- a/mock-repos/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# Mock Repos - -`mock-repos/` contains tracked end-to-end demo templates for manual benchmark and optimizer testing: - -- `mcp-tracker-demo` — MCP surface, `surface-changing` optimize mode -- `sdk-counter-demo` — SDK surface, intentionally lossy SKILL.md -- `cli-taskfile-demo` — CLI surface, intentionally lossy SKILL.md - -Use a tracked template directly for read-only benchmark runs. - -Materialize a standalone copy before running the optimizer so git checkpointing stays isolated: - -```bash -tsx src/optimizer/materialize-mock-repo.ts mcp-tracker-demo ./.tmp/mock-repos -npx skill-optimizer optimize --config ./.tmp/mock-repos/mcp-tracker-demo/skill-optimizer.json -``` - -Each demo repo's `skill-optimizer.json` is the unified config entry point for both benchmarking and optimization. diff --git a/mock-repos/cli-taskfile-demo/README.md b/mock-repos/cli-taskfile-demo/README.md deleted file mode 100644 index 6d54be6..0000000 --- a/mock-repos/cli-taskfile-demo/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# cli-taskfile-demo - -A CLI surface demo for skill-optimizer. - -This mock repo demonstrates optimizing a CLI tool's `SKILL.md` so that LLMs can use all five commands (`add`, `list`, `done`, `delete`, `update`) with the right flags. The included `SKILL.md` intentionally omits `delete`, `update`, and flag details like `--priority` and `--due` — leaving the optimizer room to improve coverage. - -## What's here - -| File | Purpose | -|------|---------| -| `skill-optimizer.json` | Unified benchmark + optimizer config | -| `SKILL.md` | Intentionally incomplete docs — the optimizer rewrites this | -| `src/commands.ts` | CLI command definitions (the surface being benchmarked) | - -## Quickstart - -Materialize an isolated copy before running the optimizer (required for git checkpointing): - -```bash -tsx src/optimizer/materialize-mock-repo.ts cli-taskfile-demo ./.tmp/mock-repos -npx tsx src/cli.ts optimize --config ./.tmp/mock-repos/cli-taskfile-demo/skill-optimizer.json -``` - -Or run a benchmark-only pass against the tracked template: - -```bash -npx tsx src/cli.ts run --config mock-repos/cli-taskfile-demo/skill-optimizer.json -``` diff --git a/mock-repos/cli-taskfile-demo/SKILL.md b/mock-repos/cli-taskfile-demo/SKILL.md deleted file mode 100644 index 7bf84c4..0000000 --- a/mock-repos/cli-taskfile-demo/SKILL.md +++ /dev/null @@ -1,37 +0,0 @@ -# taskfile CLI - -A simple command-line task manager. - -## Usage - -```bash -taskfile add --title "Buy groceries" -taskfile list -taskfile done --id abc123 -``` - -## Commands - -### add - -Add a new task. - -```bash -taskfile add --title "Task title" -``` - -### list - -List your current tasks. - -```bash -taskfile list -``` - -### done - -Mark a task as done using its ID. - -```bash -taskfile done --id -``` diff --git a/mock-repos/cli-taskfile-demo/skill-optimizer.json b/mock-repos/cli-taskfile-demo/skill-optimizer.json deleted file mode 100644 index 002bd21..0000000 --- a/mock-repos/cli-taskfile-demo/skill-optimizer.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "name": "cli-taskfile-demo", - "target": { - "surface": "cli", - "repoPath": ".", - "skill": "./SKILL.md", - "discovery": { - "mode": "auto", - "sources": ["./src/commands.ts"] - }, - "scope": { "include": ["*"], "exclude": [] } - }, - "benchmark": { - "format": "pi", - "apiKeyEnv": "OPENROUTER_API_KEY", - "models": [ - { "id": "openrouter/openai/gpt-4o-mini", "name": "GPT-4o mini", "tier": "low" }, - { "id": "openrouter/anthropic/claude-sonnet-4.6", "name": "Claude 4.6", "tier": "mid" } - ], - "verdict": { "perModelFloor": 0.6, "targetWeightedAverage": 0.7 }, - "taskGeneration": { "enabled": true, "maxTasks": 10, "seed": 1, "outputDir": "./.skill-optimizer" } - }, - "optimize": { - "enabled": true, - "model": "openrouter/anthropic/claude-sonnet-4.6", - "apiKeyEnv": "OPENROUTER_API_KEY", - "allowedPaths": ["SKILL.md"], - "validation": [], - "maxIterations": 3, - "minImprovement": 0.02, - "reportContextMaxBytes": 16000 - } -} diff --git a/mock-repos/cli-taskfile-demo/src/commands.ts b/mock-repos/cli-taskfile-demo/src/commands.ts deleted file mode 100644 index 2f825bf..0000000 --- a/mock-repos/cli-taskfile-demo/src/commands.ts +++ /dev/null @@ -1,49 +0,0 @@ -/** - * CLI command definitions for the taskfile demo. - * Exported as a literal array so the skill-optimizer can discover the surface - * via static analysis — no runtime evaluation needed. - */ -export const COMMANDS = [ - { - command: "add", - description: "Add a new task to the list", - options: [ - { name: "title", takesValue: true, description: "Task title (required)" }, - { name: "priority", takesValue: true, description: "Priority level: low, medium, or high (default: medium)" }, - { name: "due", takesValue: true, description: "Due date in YYYY-MM-DD format" }, - ], - }, - { - command: "list", - description: "List tasks, optionally filtered", - options: [ - { name: "status", takesValue: true, description: "Filter by status: pending, done, or all (default: pending)" }, - { name: "priority", takesValue: true, description: "Filter by priority level" }, - ], - }, - { - command: "done", - description: "Mark a task as completed", - options: [ - { name: "id", takesValue: true, description: "Task ID to mark as done (required)" }, - ], - }, - { - command: "delete", - description: "Permanently delete a task", - options: [ - { name: "id", takesValue: true, description: "Task ID to delete (required)" }, - { name: "force", takesValue: false, description: "Skip the confirmation prompt" }, - ], - }, - { - command: "update", - description: "Update one or more fields of an existing task", - options: [ - { name: "id", takesValue: true, description: "Task ID to update (required)" }, - { name: "title", takesValue: true, description: "New task title" }, - { name: "priority", takesValue: true, description: "New priority level" }, - { name: "due", takesValue: true, description: "New due date in YYYY-MM-DD format" }, - ], - }, -]; diff --git a/mock-repos/mcp-tracker-demo/.gitignore b/mock-repos/mcp-tracker-demo/.gitignore deleted file mode 100644 index d36689c..0000000 --- a/mock-repos/mcp-tracker-demo/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -.skill-benchmark-optimize/ -.skill-optimizer/ diff --git a/mock-repos/mcp-tracker-demo/README.md b/mock-repos/mcp-tracker-demo/README.md deleted file mode 100644 index 15a26da..0000000 --- a/mock-repos/mcp-tracker-demo/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# mcp-tracker-demo - -A minimal MCP server used to demonstrate `skill-optimizer` end-to-end. - -## What this shows - -- How to configure `skill-optimizer.json` for an MCP surface -- Task generation, benchmarking, and optimization against a small tool set - -## Quickstart - -```bash -# From the skill-optimizer repo root: -export OPENROUTER_API_KEY=sk-or-... - -# Preview the surface without any LLM calls: -npx skill-optimizer --dry-run --config mock-repos/mcp-tracker-demo/skill-optimizer.json - -# Run the benchmark only: -npx skill-optimizer run --config mock-repos/mcp-tracker-demo/skill-optimizer.json - -# Run the full optimization loop: -npx skill-optimizer optimize --config mock-repos/mcp-tracker-demo/skill-optimizer.json -``` - -## Files - -- `SKILL.md` — the guidance document being evaluated and improved -- `tools.json` — MCP tool definitions (used for manifest discovery) -- `src/server.ts` — the actual server implementation (used for code-first discovery) -- `skill-optimizer.json` — benchmark + optimizer config diff --git a/mock-repos/mcp-tracker-demo/SKILL.md b/mock-repos/mcp-tracker-demo/SKILL.md deleted file mode 100644 index 63edb6f..0000000 --- a/mock-repos/mcp-tracker-demo/SKILL.md +++ /dev/null @@ -1,8 +0,0 @@ -# Tracker MCP Notes - -There are tracker tools for creating and reading tickets. - -Use `tkt_new` when a fresh issue is needed. -Use `get_tkt` when checking an existing issue. - -Keep state and comments in sync during longer work. diff --git a/mock-repos/mcp-tracker-demo/skill-optimizer.json b/mock-repos/mcp-tracker-demo/skill-optimizer.json deleted file mode 100644 index a11ebab..0000000 --- a/mock-repos/mcp-tracker-demo/skill-optimizer.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "name": "mcp-tracker-demo", - "target": { - "surface": "mcp", - "repoPath": ".", - "skill": "./SKILL.md", - "scope": { - "include": ["*"], - "exclude": [] - }, - "discovery": { - "mode": "auto", - "sources": [ - "./src/server.ts" - ] - } - }, - "benchmark": { - "format": "pi", - "apiKeyEnv": "OPENROUTER_API_KEY", - "models": [ - { - "id": "openrouter/openai/gpt-5.4", - "name": "GPT-5.4", - "tier": "flagship" - }, - { - "id": "openrouter/anthropic/claude-sonnet-4.6", - "name": "Claude Sonnet 4.6", - "tier": "mid" - }, - { - "id": "openrouter/google/gemini-3-flash-preview", - "name": "Gemini 3 Flash Preview", - "tier": "low" - } - ], - "taskGeneration": { - "enabled": true, - "maxTasks": 8, - "seed": 1, - "outputDir": "./.skill-optimizer" - }, - "output": { - "dir": "./benchmark-results" - }, - "verdict": { - "perModelFloor": 0.6, - "targetWeightedAverage": 0.7 - } - }, - "optimize": { - "enabled": true, - "mode": "surface-changing", - "model": "openrouter/openai/gpt-5.4", - "apiKeyEnv": "OPENROUTER_API_KEY", - "thinkingLevel": "medium", - "allowedPaths": [ - "src", - "SKILL.md", - "README.md" - ], - "validation": [], - "maxIterations": 5, - "stabilityWindow": 2, - "minImprovement": 0.02, - "reportContextMaxBytes": 16000 - } -} diff --git a/mock-repos/mcp-tracker-demo/src/server.ts b/mock-repos/mcp-tracker-demo/src/server.ts deleted file mode 100644 index 0759bc9..0000000 --- a/mock-repos/mcp-tracker-demo/src/server.ts +++ /dev/null @@ -1,70 +0,0 @@ -export const TRACKER_TOOLS = [ - { - type: 'function', - function: { - name: 'tkt_new', - description: 'make ticket row', - parameters: { - type: 'object', - properties: { - t: { type: 'string', description: 'title' }, - d: { type: 'string', description: 'desc text' }, - p: { type: 'string', description: 'priority code' }, - usr: { type: 'string', description: 'owner handle' }, - }, - required: ['t', 'd', 'p'], - additionalProperties: false, - }, - }, - }, - { - type: 'function', - function: { - name: 'get_tkt', - description: 'pull one ticket by id', - parameters: { - type: 'object', - properties: { - id: { type: 'string', description: 'ticket key' }, - }, - required: ['id'], - additionalProperties: false, - }, - }, - }, - { - type: 'function', - function: { - name: 'update_tkt_state', - description: 'change state to another', - parameters: { - type: 'object', - properties: { - id: { type: 'string' }, - to: { type: 'string', description: 'new state label' }, - }, - required: ['id', 'to'], - additionalProperties: false, - }, - }, - }, - { - type: 'function', - function: { - name: 'add_cmnt', - description: 'append comment on ticket', - parameters: { - type: 'object', - properties: { - tkt: { type: 'string', description: 'ticket key' }, - body: { type: 'string', description: 'comment body' }, - author: { type: 'string', description: 'user handle' }, - }, - required: ['tkt', 'body'], - additionalProperties: false, - }, - }, - }, -] as const; - -export default TRACKER_TOOLS; diff --git a/mock-repos/mcp-tracker-demo/tools.json b/mock-repos/mcp-tracker-demo/tools.json deleted file mode 100644 index 87011a2..0000000 --- a/mock-repos/mcp-tracker-demo/tools.json +++ /dev/null @@ -1,109 +0,0 @@ -[ - { - "type": "function", - "function": { - "name": "tkt_new", - "description": "make ticket row", - "parameters": { - "type": "object", - "properties": { - "t": { - "type": "string", - "description": "title" - }, - "d": { - "type": "string", - "description": "desc text" - }, - "p": { - "type": "string", - "description": "priority code" - }, - "usr": { - "type": "string", - "description": "owner handle" - } - }, - "required": [ - "t", - "d", - "p" - ], - "additionalProperties": false - } - } - }, - { - "type": "function", - "function": { - "name": "get_tkt", - "description": "pull one ticket by id", - "parameters": { - "type": "object", - "properties": { - "id": { - "type": "string", - "description": "ticket key" - } - }, - "required": [ - "id" - ], - "additionalProperties": false - } - } - }, - { - "type": "function", - "function": { - "name": "update_tkt_state", - "description": "change state to another", - "parameters": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "to": { - "type": "string", - "description": "new state label" - } - }, - "required": [ - "id", - "to" - ], - "additionalProperties": false - } - } - }, - { - "type": "function", - "function": { - "name": "add_cmnt", - "description": "append comment on ticket", - "parameters": { - "type": "object", - "properties": { - "tkt": { - "type": "string", - "description": "ticket key" - }, - "body": { - "type": "string", - "description": "comment body" - }, - "author": { - "type": "string", - "description": "user handle" - } - }, - "required": [ - "tkt", - "body" - ], - "additionalProperties": false - } - } - } -] diff --git a/mock-repos/sdk-counter-demo/README.md b/mock-repos/sdk-counter-demo/README.md deleted file mode 100644 index f1da732..0000000 --- a/mock-repos/sdk-counter-demo/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# sdk-counter-demo - -A minimal TypeScript SDK used to demonstrate `skill-optimizer` end-to-end. - -The bundled `SKILL.md` intentionally omits the `amount` parameter, the `reset` method, and the `start` option — so the first benchmark run fails, then the optimizer proposes improvements. - -## Quickstart - -```bash -# From the skill-optimizer repo root: -export OPENROUTER_API_KEY=sk-or-... - -# Preview the surface without any LLM calls: -npx skill-optimizer --dry-run --config mock-repos/sdk-counter-demo/skill-optimizer.json - -# Run the benchmark only: -npx skill-optimizer run --config mock-repos/sdk-counter-demo/skill-optimizer.json - -# Run the full optimization loop: -npx skill-optimizer optimize --config mock-repos/sdk-counter-demo/skill-optimizer.json -``` - -## Files - -- `SKILL.md` — the guidance document being evaluated and improved -- `src/counter.ts` — the SDK source (used for code-first discovery) -- `skill-optimizer.json` — benchmark + optimizer config diff --git a/mock-repos/sdk-counter-demo/SKILL.md b/mock-repos/sdk-counter-demo/SKILL.md deleted file mode 100644 index 519578c..0000000 --- a/mock-repos/sdk-counter-demo/SKILL.md +++ /dev/null @@ -1,15 +0,0 @@ -# Counter SDK - -A small counter utility. - -## Usage - -Import from `./counter.ts` and build a counter. - -```ts -import { createCounter } from './counter'; -const c = createCounter(); -c.increment(); -``` - -That's it. Use `.value()` for the current value. diff --git a/mock-repos/sdk-counter-demo/skill-optimizer.json b/mock-repos/sdk-counter-demo/skill-optimizer.json deleted file mode 100644 index 17bd952..0000000 --- a/mock-repos/sdk-counter-demo/skill-optimizer.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "name": "sdk-counter-demo", - "target": { - "surface": "sdk", - "repoPath": ".", - "skill": "./SKILL.md", - "discovery": { - "mode": "auto", - "sources": ["./src/counter.ts"], - "language": "typescript" - }, - "sdk": { "language": "typescript" }, - "scope": { "include": ["*"], "exclude": [] } - }, - "benchmark": { - "format": "pi", - "apiKeyEnv": "OPENROUTER_API_KEY", - "models": [ - { "id": "openrouter/openai/gpt-4o-mini", "name": "GPT-4o mini", "tier": "low" }, - { "id": "openrouter/anthropic/claude-sonnet-4.6", "name": "Claude 4.6", "tier": "mid" } - ], - "verdict": { "perModelFloor": 0.6, "targetWeightedAverage": 0.7 }, - "taskGeneration": { "enabled": true, "maxTasks": 8, "seed": 1, "outputDir": "./.skill-optimizer" } - }, - "optimize": { - "enabled": true, - "model": "openrouter/anthropic/claude-sonnet-4.6", - "apiKeyEnv": "OPENROUTER_API_KEY", - "allowedPaths": ["SKILL.md"], - "validation": [], - "maxIterations": 3, - "minImprovement": 0.02, - "reportContextMaxBytes": 16000 - } -} diff --git a/mock-repos/sdk-counter-demo/src/counter.ts b/mock-repos/sdk-counter-demo/src/counter.ts deleted file mode 100644 index 3147073..0000000 --- a/mock-repos/sdk-counter-demo/src/counter.ts +++ /dev/null @@ -1,25 +0,0 @@ -// src/counter.ts - -/** Creates a new counter, optionally starting at a given value. */ -export function createCounter(options?: { start?: number }): Counter { - return new Counter(options?.start ?? 0); -} - -export class Counter { - #value: number; - constructor(start: number) { this.#value = start; } - - /** Advances the counter and returns the new value. */ - increment(amount?: number): number { - this.#value += amount ?? 1; - return this.#value; - } - - /** Resets the counter to 0 (or the given value). */ - reset(to?: number): number { - this.#value = to ?? 0; - return this.#value; - } - - value(): number { return this.#value; } -} diff --git a/package-lock.json b/package-lock.json index 373185c..d985d28 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,23 +1,21 @@ { "name": "skill-optimizer", - "version": "1.1.0", + "version": "2.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "skill-optimizer", - "version": "1.1.0", + "version": "2.0.0", "license": "MIT", + "main": ".opencode/plugins/skill-optimizer.js", "dependencies": { - "@clack/prompts": "^1.2.0", "@mariozechner/pi-agent-core": "^0.66.1", "@mariozechner/pi-ai": "^0.66.1", "@mariozechner/pi-coding-agent": "^0.66.1", "dotenv": "^17.4.1", - "tree-sitter-wasms": "^0.1.13", - "web-tree-sitter": "^0.24.7", - "zod": "^4.3.6", - "zod-to-json-schema": "^3.25.2" + "mcporter": "^0.9.0", + "yaml": "^2.8.2" }, "bin": { "skill-optimizer": "dist/cli.js" @@ -765,26 +763,35 @@ "url": "https://github.com/sponsors/Borewit" } }, - "node_modules/@clack/core": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@clack/core/-/core-1.2.0.tgz", - "integrity": "sha512-qfxof/3T3t9DPU/Rj3OmcFyZInceqj/NVtO9rwIuJqCUgh32gwPjpFQQp/ben07qKlhpwq7GzfWpST4qdJ5Drg==", + "node_modules/@emnapi/core": { + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.9.2.tgz", + "integrity": "sha512-UC+ZhH3XtczQYfOlu3lNEkdW/p4dsJ1r/bP7H8+rhao3TTTMO1ATq/4DdIi23XuGoFY+Cz0JmCbdVl0hz9jZcA==", "license": "MIT", + "optional": true, "dependencies": { - "fast-wrap-ansi": "^0.1.3", - "sisteransi": "^1.0.5" + "@emnapi/wasi-threads": "1.2.1", + "tslib": "^2.4.0" } }, - "node_modules/@clack/prompts": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@clack/prompts/-/prompts-1.2.0.tgz", - "integrity": "sha512-4jmztR9fMqPMjz6H/UZXj0zEmE43ha1euENwkckKKel4XpSfokExPo5AiVStdHSAlHekz4d0CA/r45Ok1E4D3w==", + "node_modules/@emnapi/runtime": { + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.9.2.tgz", + "integrity": "sha512-3U4+MIWHImeyu1wnmVygh5WlgfYDtyf0k8AbLhMFxOipihf6nrWC4syIm/SwEeec0mNSafiiNnMJwbza/Is6Lw==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", + "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", "license": "MIT", + "optional": true, "dependencies": { - "@clack/core": "1.2.0", - "fast-string-width": "^1.1.0", - "fast-wrap-ansi": "^0.1.3", - "sisteransi": "^1.0.5" + "tslib": "^2.4.0" } }, "node_modules/@esbuild/aix-ppc64": { @@ -1252,6 +1259,24 @@ } } }, + "node_modules/@hono/node-server": { + "version": "1.19.14", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.14.tgz", + "integrity": "sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw==", + "license": "MIT", + "engines": { + "node": ">=18.14.1" + }, + "peerDependencies": { + "hono": "^4" + } + }, + "node_modules/@iarna/toml": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/@iarna/toml/-/toml-2.2.5.tgz", + "integrity": "sha512-trnsAYxU3xnS1gPHPyU961coFyLkh4gAD/0zQ5mymY4yOZ+CYvsPqUbOFSw0aDM4y0tV7tiFxL/1XfXPNC6IPg==", + "license": "ISC" + }, "node_modules/@mariozechner/clipboard": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/@mariozechner/clipboard/-/clipboard-0.3.2.tgz", @@ -1549,6 +1574,73 @@ "zod-to-json-schema": "^3.24.1" } }, + "node_modules/@modelcontextprotocol/sdk": { + "version": "1.29.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.29.0.tgz", + "integrity": "sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ==", + "license": "MIT", + "dependencies": { + "@hono/node-server": "^1.19.9", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "content-type": "^1.0.5", + "cors": "^2.8.5", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "express": "^5.2.1", + "express-rate-limit": "^8.2.1", + "hono": "^4.11.4", + "jose": "^6.1.3", + "json-schema-typed": "^8.0.2", + "pkce-challenge": "^5.0.0", + "raw-body": "^3.0.0", + "zod": "^3.25 || ^4.0", + "zod-to-json-schema": "^3.25.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cfworker/json-schema": "^4.1.1", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@cfworker/json-schema": { + "optional": true + }, + "zod": { + "optional": false + } + } + }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.4.tgz", + "integrity": "sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow==", + "license": "MIT", + "optional": true, + "dependencies": { + "@tybys/wasm-util": "^0.10.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.126.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.126.0.tgz", + "integrity": "sha512-oGfVtjAgwQVVpfBrbtk4e1XDyWHRFta6BS3GWVzrF8xYBT2VGQAk39yJS/wFSMrZqoiCU4oghT3Ch0HaHGIHcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -1613,6 +1705,254 @@ "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", "license": "BSD-3-Clause" }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-rc.16.tgz", + "integrity": "sha512-rhY3k7Bsae9qQfOtph2Pm2jZEA+s8Gmjoz4hhmx70K9iMQ/ddeae+xhRQcM5IuVx5ry1+bGfkvMn7D6MJggVSA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.0-rc.16.tgz", + "integrity": "sha512-rNz0yK078yrNn3DrdgN+PKiMOW8HfQ92jQiXxwX8yW899ayV00MLVdaCNeVBhG/TbH3ouYVObo8/yrkiectkcQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.0-rc.16.tgz", + "integrity": "sha512-r/OmdR00HmD4i79Z//xO06uEPOq5hRXdhw7nzkxQxwSavs3PSHa1ijntdpOiZ2mzOQ3fVVu8C1M19FoNM+dMUQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.0-rc.16.tgz", + "integrity": "sha512-KcRE5w8h0OnjUatG8pldyD14/CQ5Phs1oxfR+3pKDjboHRo9+MkqQaiIZlZRpsxC15paeXme/I127tUa9TXJ6g==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.0-rc.16.tgz", + "integrity": "sha512-bT0guA1bpxEJ/ZhTRniQf7rNF8ybvXOuWbNIeLABaV5NGjx4EtOWBTSRGWFU9ZWVkPOZ+HNFP8RMcBokBiZ0Kg==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.0-rc.16.tgz", + "integrity": "sha512-+tHktCHWV8BDQSjemUqm/Jl/TPk3QObCTIjmdDy/nlupcujZghmKK2962LYrqFpWu+ai01AN/REOH3NEpqvYQg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.0-rc.16.tgz", + "integrity": "sha512-3fPzdREH806oRLxpTWW1Gt4tQHs0TitZFOECB2xzCFLPKnSOy90gwA7P29cksYilFO6XVRY1kzga0cL2nRjKPg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.0-rc.16.tgz", + "integrity": "sha512-EKwI1tSrLs7YVw+JPJT/G2dJQ1jl9qlTTTEG0V2Ok/RdOenRfBw2PQdLPyjhIu58ocdBfP7vIRN/pvMsPxs/AQ==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.0-rc.16.tgz", + "integrity": "sha512-Uknladnb3Sxqu6SEcqBldQyJUpk8NleooZEc0MbRBJ4inEhRYWZX0NJu12vNf2mqAq7gsofAxHrGghiUYjhaLQ==", + "cpu": [ + "s390x" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.0-rc.16.tgz", + "integrity": "sha512-FIb8+uG49sZBtLTn+zt1AJ20TqVcqWeSIyoVt0or7uAWesgKaHbiBh6OpA/k9v0LTt+PTrb1Lao133kP4uVxkg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.0-rc.16.tgz", + "integrity": "sha512-RuERhF9/EgWxZEXYWCOaViUWHIboceK4/ivdtQ3R0T44NjLkIIlGIAVAuCddFxsZ7vnRHtNQUrt2vR2n2slB2w==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.0-rc.16.tgz", + "integrity": "sha512-mXcXnvd9GpazCxeUCCnZ2+YF7nut+ZOEbE4GtaiPtyY6AkhZWbK70y1KK3j+RDhjVq5+U8FySkKRb/+w0EeUwA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-wasm32-wasi": { + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.0-rc.16.tgz", + "integrity": "sha512-3Q2KQxnC8IJOLqXmUMoYwyIPZU9hzRbnHaoV3Euz+VVnjZKcY8ktnNP8T9R4/GGQtb27C/UYKABxesKWb8lsvQ==", + "cpu": [ + "wasm32" + ], + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "1.9.2", + "@emnapi/runtime": "1.9.2", + "@napi-rs/wasm-runtime": "^1.1.4" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.0-rc.16.tgz", + "integrity": "sha512-tj7XRemQcOcFwv7qhpUxMTBbI5mWMlE4c1Omhg5+h8GuLXzyj8HviYgR+bB2DMDgRqUE+jiDleqSCRjx4aYk/Q==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.0-rc.16.tgz", + "integrity": "sha512-PH5DRZT+F4f2PTXRXR8uJxnBq2po/xFtddyabTJVJs/ZYVHqXPEgNIr35IHTEa6bpa0Q8Awg+ymkTaGnKITw4g==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.16.tgz", + "integrity": "sha512-45+YtqxLYKDWQouLKCrpIZhke+nXxhsw+qAHVzHDVwttyBlHNBVs2K25rDXrZzhpTp9w1FlAlvweV1H++fdZoA==", + "license": "MIT" + }, "node_modules/@silvia-odwyer/photon-node": { "version": "0.3.4", "resolved": "https://registry.npmjs.org/@silvia-odwyer/photon-node/-/photon-node-0.3.4.tgz", @@ -2292,6 +2632,16 @@ "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==", "license": "MIT" }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz", + "integrity": "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@types/mime-types": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/@types/mime-types/-/mime-types-2.1.4.tgz", @@ -2323,6 +2673,31 @@ "@types/node": "*" } }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/acorn": { + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", + "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/agent-base": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", @@ -2457,6 +2832,30 @@ "node": "*" } }, + "node_modules/body-parser": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.2.tgz", + "integrity": "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^1.0.5", + "debug": "^4.4.3", + "http-errors": "^2.0.0", + "iconv-lite": "^0.7.0", + "on-finished": "^2.4.1", + "qs": "^6.14.1", + "raw-body": "^3.0.1", + "type-is": "^2.0.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/bowser": { "version": "2.14.1", "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", @@ -2490,18 +2889,71 @@ "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", "license": "BSD-3-Clause" }, - "node_modules/chalk": { - "version": "5.6.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", - "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", "license": "MIT", "engines": { - "node": "^12.17.0 || ^14.13 || >=16.0.0" - }, + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/chalk": { + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", + "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", + "license": "MIT", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, "funding": { "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/cli-cursor": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-5.0.0.tgz", + "integrity": "sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==", + "license": "MIT", + "dependencies": { + "restore-cursor": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/cli-highlight": { "version": "2.1.11", "resolved": "https://registry.npmjs.org/cli-highlight/-/cli-highlight-2.1.11.tgz", @@ -2539,6 +2991,18 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/cli-spinners": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-3.4.0.tgz", + "integrity": "sha512-bXfOC4QcT1tKXGorxL3wbJm6XJPDqEnij2gQ2m7ESQuE+/z9YFIWnl/5RpTiKWbMq3EVKR4fRLJGn6DVfu0mpw==", + "license": "MIT", + "engines": { + "node": ">=18.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/cliui": { "version": "7.0.4", "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", @@ -2589,6 +3053,86 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "license": "MIT" }, + "node_modules/commander": { + "version": "14.0.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.3.tgz", + "integrity": "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==", + "license": "MIT", + "engines": { + "node": ">=20" + } + }, + "node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/cors": { + "version": "2.8.6", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.6.tgz", + "integrity": "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==", + "license": "MIT", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/data-uri-to-buffer": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", @@ -2629,6 +3173,15 @@ "node": ">= 14" } }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/diff": { "version": "8.0.4", "resolved": "https://registry.npmjs.org/diff/-/diff-8.0.4.tgz", @@ -2650,6 +3203,20 @@ "url": "https://dotenvx.com" } }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/ecdsa-sig-formatter": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", @@ -2659,12 +3226,27 @@ "safe-buffer": "^5.0.1" } }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, "node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "license": "MIT" }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/end-of-stream": { "version": "1.4.5", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", @@ -2674,6 +3256,46 @@ "once": "^1.4.0" } }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-toolkit": { + "version": "1.46.1", + "resolved": "https://registry.npmjs.org/es-toolkit/-/es-toolkit-1.46.1.tgz", + "integrity": "sha512-5eNtXOs3tbfxXOj04tjjseeWkRWaoCjdEI+96DgwzZoe6c9juL49pXlzAFTI72aWC9Y8p7168g6XIKjh7k6pyQ==", + "license": "MIT", + "workspaces": [ + "docs", + "benchmarks" + ] + }, "node_modules/esbuild": { "version": "0.27.7", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz", @@ -2725,6 +3347,12 @@ "node": ">=6" } }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, "node_modules/escodegen": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", @@ -2777,6 +3405,97 @@ "node": ">=0.10.0" } }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/eventsource": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", + "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.8.tgz", + "integrity": "sha512-70QWGkr4snxr0OXLRWsFLeRBIRPuQOvt4s8QYjmUlmlkyTZkRqS7EDVRZtzU3TiyDbXSzaOeF0XUKy8PchzukQ==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express-rate-limit": { + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.4.1.tgz", + "integrity": "sha512-NGVYwQSAyEQgzxX1iCM978PP9AdO/hW93gMcF6ZwQCm+rFvLsBH6w4xcXWTcliS8La5EPRN3p9wzItqBwJrfNw==", + "license": "MIT", + "dependencies": { + "ip-address": "10.1.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, "node_modules/extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -2809,21 +3528,6 @@ "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", "license": "MIT" }, - "node_modules/fast-string-truncated-width": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/fast-string-truncated-width/-/fast-string-truncated-width-1.2.1.tgz", - "integrity": "sha512-Q9acT/+Uu3GwGj+5w/zsGuQjh9O1TyywhIwAxHudtWrgF09nHOPrvTLhQevPbttcxjr/SNN7mJmfOw/B1bXgow==", - "license": "MIT" - }, - "node_modules/fast-string-width": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fast-string-width/-/fast-string-width-1.1.0.tgz", - "integrity": "sha512-O3fwIVIH5gKB38QNbdg+3760ZmGz0SZMgvwJbA1b2TGXceKE6A2cOlfogh1iw8lr049zPyd7YADHy+B7U4W9bQ==", - "license": "MIT", - "dependencies": { - "fast-string-truncated-width": "^1.2.0" - } - }, "node_modules/fast-uri": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", @@ -2840,15 +3544,6 @@ ], "license": "BSD-3-Clause" }, - "node_modules/fast-wrap-ansi": { - "version": "0.1.6", - "resolved": "https://registry.npmjs.org/fast-wrap-ansi/-/fast-wrap-ansi-0.1.6.tgz", - "integrity": "sha512-HlUwET7a5gqjURj70D5jl7aC3Zmy4weA1SHUfM0JFI0Ptq987NH2TwbBFLoERhfwk+E+eaq4EK3jXoT+R3yp3w==", - "license": "MIT", - "dependencies": { - "fast-string-width": "^1.1.0" - } - }, "node_modules/fast-xml-builder": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.1.5.tgz", @@ -2934,6 +3629,27 @@ "url": "https://github.com/sindresorhus/file-type?sponsor=1" } }, + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/formdata-polyfill": { "version": "4.0.10", "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", @@ -2946,6 +3662,24 @@ "node": ">=12.20.0" } }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -2961,6 +3695,15 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/gaxios": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.4.tgz", @@ -3010,6 +3753,43 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/get-stream": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", @@ -3104,6 +3884,18 @@ "node": ">=14" } }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", @@ -3119,6 +3911,30 @@ "node": ">=8" } }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.3.tgz", + "integrity": "sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/highlight.js": { "version": "10.7.3", "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz", @@ -3128,6 +3944,15 @@ "node": "*" } }, + "node_modules/hono": { + "version": "4.12.16", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.16.tgz", + "integrity": "sha512-jN0ZewiNAWSe5khM3EyCmBb250+b40wWbwNILNfEvq84VREWwOIkuUsFONk/3i3nqkz7Oe1PcpM2mwQEK2L9Kg==", + "license": "MIT", + "engines": { + "node": ">=16.9.0" + } + }, "node_modules/hosted-git-info": { "version": "9.0.2", "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-9.0.2.tgz", @@ -3140,19 +3965,39 @@ "node": "^20.17.0 || >=22.9.0" } }, - "node_modules/http-proxy-agent": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", - "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", "license": "MIT", "dependencies": { - "agent-base": "^7.1.0", - "debug": "^4.3.4" + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" }, "engines": { - "node": ">= 14" - } - }, + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/https-proxy-agent": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", @@ -3166,6 +4011,22 @@ "node": ">= 14" } }, + "node_modules/iconv-lite": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", + "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -3195,6 +4056,12 @@ "node": ">= 4" } }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, "node_modules/ip-address": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.1.0.tgz", @@ -3204,6 +4071,15 @@ "node": ">= 12" } }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, "node_modules/is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", @@ -3213,6 +4089,51 @@ "node": ">=8" } }, + "node_modules/is-interactive": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-2.0.0.tgz", + "integrity": "sha512-qP1vozQRI+BMOPcjFzrjXuQvdak2pHNUMZoeG2eRbiSqyvbEf/wQtEOTOX1guk6E3t36RkaqiSt8A/6YElNxLQ==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/is-unicode-supported": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-2.1.0.tgz", + "integrity": "sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jose": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.3.tgz", + "integrity": "sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/json-bigint": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", @@ -3241,6 +4162,18 @@ "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", "license": "MIT" }, + "node_modules/json-schema-typed": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.2.tgz", + "integrity": "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==", + "license": "BSD-2-Clause" + }, + "node_modules/jsonc-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.3.1.tgz", + "integrity": "sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==", + "license": "MIT" + }, "node_modules/jwa": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", @@ -3273,6 +4206,22 @@ "url": "https://liberapay.com/Koromix" } }, + "node_modules/log-symbols": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-7.0.1.tgz", + "integrity": "sha512-ja1E3yCr9i/0hmBVaM0bfwDjnGy8I/s6PP4DFp+yP+a+mrHO4Rm7DtmnqROTUkHIkqffC84YY7AeqX6oFk0WFg==", + "license": "MIT", + "dependencies": { + "is-unicode-supported": "^2.0.0", + "yoctocolors": "^2.1.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/long": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", @@ -3300,6 +4249,59 @@ "node": ">= 18" } }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/mcporter": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/mcporter/-/mcporter-0.9.0.tgz", + "integrity": "sha512-zbvhQpBUL0DME8H0cYlNDQDLjvdhk1Lpy0QLGxJ6T2eCPyfs71GnP3uFP4u60vO5p/jdDsWpkZsmAdlA7OZ61w==", + "license": "MIT", + "dependencies": { + "@iarna/toml": "^2.2.5", + "@modelcontextprotocol/sdk": "^1.29.0", + "acorn": "^8.16.0", + "commander": "^14.0.3", + "es-toolkit": "^1.45.1", + "jsonc-parser": "^3.3.1", + "ora": "^9.3.0", + "rolldown": "1.0.0-rc.16", + "zod": "^4.3.6" + }, + "bin": { + "mcporter": "dist/cli.js" + }, + "engines": { + "node": ">=20.11.0" + } + }, + "node_modules/media-typer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/mime-db": { "version": "1.54.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", @@ -3325,6 +4327,18 @@ "url": "https://opencollective.com/express" } }, + "node_modules/mimic-function": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/mimic-function/-/mimic-function-5.0.1.tgz", + "integrity": "sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/minimatch": { "version": "10.2.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", @@ -3366,6 +4380,15 @@ "thenify-all": "^1.0.0" } }, + "node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/netmask": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.1.1.tgz", @@ -3422,6 +4445,30 @@ "node": ">=0.10.0" } }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -3431,6 +4478,21 @@ "wrappy": "1" } }, + "node_modules/onetime": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-7.0.0.tgz", + "integrity": "sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==", + "license": "MIT", + "dependencies": { + "mimic-function": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/openai": { "version": "6.26.0", "resolved": "https://registry.npmjs.org/openai/-/openai-6.26.0.tgz", @@ -3452,6 +4514,44 @@ } } }, + "node_modules/ora": { + "version": "9.4.0", + "resolved": "https://registry.npmjs.org/ora/-/ora-9.4.0.tgz", + "integrity": "sha512-84cglkRILFxdtA8hAvLNdMrtBpPNBTrQ9/ulg0FA7xLMnD6mifv+enAIeRmvtv+WgdCE+LPGOfQmtJRrVaIVhQ==", + "license": "MIT", + "dependencies": { + "chalk": "^5.6.2", + "cli-cursor": "^5.0.0", + "cli-spinners": "^3.2.0", + "is-interactive": "^2.0.0", + "is-unicode-supported": "^2.1.0", + "log-symbols": "^7.0.1", + "stdin-discarder": "^0.3.2", + "string-width": "^8.1.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ora/node_modules/string-width": { + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.2.1.tgz", + "integrity": "sha512-IIaP0g3iy9Cyy18w3M9YcaDudujEAVHKt3a3QJg1+sr/oX96TbaGUubG0hJyCjCBThFH+tFpcIyoUHUn1ogaLA==", + "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.5.0", + "strip-ansi": "^7.1.2" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-retry": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", @@ -3518,6 +4618,15 @@ "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==", "license": "MIT" }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/partial-json": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/partial-json/-/partial-json-0.1.7.tgz", @@ -3539,6 +4648,15 @@ "node": ">=14.0.0" } }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/path-scurry": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", @@ -3555,12 +4673,31 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/pend": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", "license": "MIT" }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, "node_modules/proper-lockfile": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", @@ -3605,6 +4742,19 @@ "node": ">=12.0.0" } }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/proxy-agent": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.5.0.tgz", @@ -3649,6 +4799,45 @@ "once": "^1.3.1" } }, + "node_modules/qs": { + "version": "6.15.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.1.tgz", + "integrity": "sha512-6YHEFRL9mfgcAvql/XhwTvf5jKcOiiupt2FiJxHkiX1z4j7WL8J/jRHYLluORvc1XxB5rV20KoeK00gVJamspg==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -3677,6 +4866,34 @@ "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" } }, + "node_modules/restore-cursor": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-5.1.0.tgz", + "integrity": "sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==", + "license": "MIT", + "dependencies": { + "onetime": "^7.0.0", + "signal-exit": "^4.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/restore-cursor/node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/retry": { "version": "0.13.1", "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", @@ -3686,6 +4903,55 @@ "node": ">= 4" } }, + "node_modules/rolldown": { + "version": "1.0.0-rc.16", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.0-rc.16.tgz", + "integrity": "sha512-rzi5WqKzEZw3SooTt7cgm4eqIoujPIyGcJNGFL7iPEuajQw7vxMHUkXylu4/vhCkJGXsgRmxqMKXUpT6FEgl0g==", + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.126.0", + "@rolldown/pluginutils": "1.0.0-rc.16" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm64": "1.0.0-rc.16", + "@rolldown/binding-darwin-arm64": "1.0.0-rc.16", + "@rolldown/binding-darwin-x64": "1.0.0-rc.16", + "@rolldown/binding-freebsd-x64": "1.0.0-rc.16", + "@rolldown/binding-linux-arm-gnueabihf": "1.0.0-rc.16", + "@rolldown/binding-linux-arm64-gnu": "1.0.0-rc.16", + "@rolldown/binding-linux-arm64-musl": "1.0.0-rc.16", + "@rolldown/binding-linux-ppc64-gnu": "1.0.0-rc.16", + "@rolldown/binding-linux-s390x-gnu": "1.0.0-rc.16", + "@rolldown/binding-linux-x64-gnu": "1.0.0-rc.16", + "@rolldown/binding-linux-x64-musl": "1.0.0-rc.16", + "@rolldown/binding-openharmony-arm64": "1.0.0-rc.16", + "@rolldown/binding-wasm32-wasi": "1.0.0-rc.16", + "@rolldown/binding-win32-arm64-msvc": "1.0.0-rc.16", + "@rolldown/binding-win32-x64-msvc": "1.0.0-rc.16" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -3706,18 +4972,162 @@ ], "license": "MIT" }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/side-channel": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", "license": "ISC" }, - "node_modules/sisteransi": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", - "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", - "license": "MIT" - }, "node_modules/smart-buffer": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", @@ -3766,12 +5176,33 @@ "node": ">=0.10.0" } }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/std-env": { "version": "3.10.0", "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", "integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==", "license": "MIT" }, + "node_modules/stdin-discarder": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/stdin-discarder/-/stdin-discarder-0.3.2.tgz", + "integrity": "sha512-eCPu1qRxPVkl5605OTWF8Wz40b4Mf45NY5LQmVPQ599knfs5QhASUm9GbJ5BDMDOXgrnh0wyEdvzmL//YMlw0A==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", @@ -3883,6 +5314,15 @@ "node": ">=0.8" } }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, "node_modules/token-types": { "version": "6.1.2", "resolved": "https://registry.npmjs.org/token-types/-/token-types-6.1.2.tgz", @@ -3901,15 +5341,6 @@ "url": "https://github.com/sponsors/Borewit" } }, - "node_modules/tree-sitter-wasms": { - "version": "0.1.13", - "resolved": "https://registry.npmjs.org/tree-sitter-wasms/-/tree-sitter-wasms-0.1.13.tgz", - "integrity": "sha512-wT+cR6DwaIz80/vho3AvSF0N4txuNx/5bcRKoXouOfClpxh/qqrF4URNLQXbbt8MaAxeksZcZd1j8gcGjc+QxQ==", - "license": "Unlicense", - "dependencies": { - "tree-sitter-wasms": "^0.1.11" - } - }, "node_modules/ts-algebra": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", @@ -3942,6 +5373,20 @@ "fsevents": "~2.3.3" } }, + "node_modules/type-is": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.1.tgz", + "integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==", + "license": "MIT", + "dependencies": { + "content-type": "^1.0.5", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", @@ -3983,6 +5428,24 @@ "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", "license": "MIT" }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/web-streams-polyfill": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", @@ -3992,11 +5455,20 @@ "node": ">= 8" } }, - "node_modules/web-tree-sitter": { - "version": "0.24.7", - "resolved": "https://registry.npmjs.org/web-tree-sitter/-/web-tree-sitter-0.24.7.tgz", - "integrity": "sha512-CdC/TqVFbXqR+C51v38hv6wOPatKEUGxa39scAeFSm98wIhZxAYonhRQPSMmfZ2w7JDI0zQDdzdmgtNk06/krQ==", - "license": "MIT" + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } }, "node_modules/wrap-ansi": { "version": "7.0.0", diff --git a/package.json b/package.json index 8422dea..8554d2f 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,9 @@ { "name": "skill-optimizer", - "version": "1.1.0", - "description": "Benchmark and optimizer for evaluating SDK, CLI, and MCP guidance with static action matching.", + "version": "2.0.0", + "description": "Build, run, and improve evals for agent skills.", "license": "MIT", - "author": "Pi2 Labs", + "author": "Fast", "homepage": "https://github.com/fastxyz/skill-optimizer#readme", "bugs": { "url": "https://github.com/fastxyz/skill-optimizer/issues" @@ -13,21 +13,51 @@ "url": "git+https://github.com/fastxyz/skill-optimizer.git" }, "keywords": [ - "benchmark", - "optimizer", - "mcp", - "sdk", - "cli", - "llm", - "tool-calling", - "evaluation" + "agent", + "skills", + "agent-skills", + "evals", + "skill-testing", + "model-evaluation" ], "type": "module", + "main": "./dist/index.js", "files": [ "dist/", + "docker/", + "src/", + "scripts/", + "docs/", + "docs/README.codex.md", + "docs/README.opencode.md", + "examples/workbench/README.md", + "examples/workbench/pdf/README.md", + "examples/workbench/pdf/suite.yml", + "examples/workbench/pdf/checks/", + "examples/workbench/pdf/references/", + "examples/workbench/mcp/README.md", + "examples/workbench/mcp/suite.yml", + "examples/workbench/mcp/checks/", + "examples/workbench/mcp/mcp/", + "examples/workbench/mcp/references/", + "skills/", + ".agents/plugins/marketplace.json", + ".claude-plugin/", + ".codex-plugin/", + ".cursor-plugin/", + ".opencode/plugins/skill-optimizer.js", + ".opencode/INSTALL.md", + ".codex/INSTALL.md", + ".cursor/INSTALL.md", + "AGENTS.md", + "CLAUDE.md", "README.md", + "GEMINI.md", + "gemini-extension.json", "LICENSE", - "CHANGELOG.md" + "CHANGELOG.md", + "package-lock.json", + "tsconfig.json" ], "bin": { "skill-optimizer": "./dist/cli.js" @@ -36,31 +66,27 @@ ".": { "import": "./dist/index.js", "types": "./dist/index.d.ts" + }, + "./server": { + "import": "./.opencode/plugins/skill-optimizer.js" } }, "scripts": { - "benchmark": "tsx src/cli.ts run", "prepack": "npm run build", "clean": "node --eval \"import { rmSync } from 'node:fs'; rmSync('dist', { recursive: true, force: true });\"", "dev": "tsx src/cli.ts", - "optimize": "tsx src/cli.ts optimize", - "materialize:mock": "tsx src/optimizer/materialize-mock-repo.ts", - "gen-docs": "tsx scripts/gen-docs.ts", - "build": "tsc && npm run gen-docs && chmod +x dist/cli.js", + "build": "tsc && chmod +x dist/cli.js", "typecheck": "tsc --noEmit", "lint": "tsc --noUnusedLocals --noEmit", - "test": "tsx tests/smoke-code.ts && tsx tests/smoke-sdk-python.ts && tsx tests/smoke-sdk-rust.ts && tsx tests/smoke-cli.ts && tsx tests/smoke-cli-entry.ts && tsx tests/smoke-mcp.ts && tsx tests/smoke-llm.ts && tsx tests/smoke-discovery-sdk.ts && tsx tests/smoke-discovery-cli.ts && tsx tests/smoke-discovery-mcp.ts && tsx tests/smoke-prompt-evaluator.ts && tsx tests/smoke-prompt-criteria.ts && tsx tests/smoke-snapshot-prompt.ts && tsx tests/smoke-generation.ts && tsx tests/smoke-optimize.ts && tsx tests/smoke-mock-repos.ts && tsx tests/smoke-release.ts && tsx tests/smoke-changelog-coverage.ts && tsx tests/smoke-scoring.ts && tsx tests/smoke-scope.ts && tsx tests/smoke-coverage.ts && tsx tests/smoke-feedback.ts && tsx tests/smoke-verdict.ts && tsx tests/smoke-verdict-prompt.ts && tsx tests/smoke-dry-run.ts && tsx tests/smoke-errors.ts && tsx tests/smoke-model-ids.ts && tsx tests/smoke-e2e.ts && tsx tests/smoke-import.ts && tsx tests/smoke-init.ts && tsx tests/smoke-gen-docs.ts && tsx tests/smoke-actions.ts" + "test": "tsx tests/smoke-workbench-case.ts && tsx tests/smoke-workbench-checks.ts && tsx tests/smoke-workbench-trace.ts && tsx tests/smoke-workbench-container.ts && tsx tests/smoke-workbench-docker-runner.ts && tsx tests/smoke-workbench-pi-agent.ts && tsx tests/smoke-workbench-run-case.ts && tsx tests/smoke-workbench-models.ts && tsx tests/smoke-workbench-suite.ts && tsx tests/smoke-workbench-trials.ts && tsx tests/smoke-workbench-metrics.ts && tsx tests/smoke-skill-distribution.ts" }, "dependencies": { - "@clack/prompts": "^1.2.0", "@mariozechner/pi-agent-core": "^0.66.1", "@mariozechner/pi-ai": "^0.66.1", "@mariozechner/pi-coding-agent": "^0.66.1", "dotenv": "^17.4.1", - "tree-sitter-wasms": "^0.1.13", - "web-tree-sitter": "^0.24.7", - "zod": "^4.3.6", - "zod-to-json-schema": "^3.25.2" + "mcporter": "^0.9.0", + "yaml": "^2.8.2" }, "devDependencies": { "@types/node": "^22.12.0", diff --git a/scripts/gen-docs.ts b/scripts/gen-docs.ts deleted file mode 100644 index adfd6eb..0000000 --- a/scripts/gen-docs.ts +++ /dev/null @@ -1,161 +0,0 @@ -#!/usr/bin/env tsx -// scripts/gen-docs.ts — auto-generates docs/reference/ from code artifacts. -// Run via: npm run gen-docs -// Hooked into: npm run build - -import { writeFileSync, mkdirSync } from 'node:fs'; -import { resolve, dirname } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { zodToJsonSchema } from 'zod-to-json-schema'; -import { ERRORS } from '../src/errors.js'; -import { ProjectConfigSchema } from '../src/project/schema.js'; - -const __dirname = dirname(fileURLToPath(import.meta.url)); -const refDir = resolve(__dirname, '../docs/reference'); -mkdirSync(refDir, { recursive: true }); - -const GENERATED_HEADER = '\n\n'; - -// ── errors.md ───────────────────────────────────────────────────────────────── - -function generateErrorsMd(): string { - const entries = Object.values(ERRORS); - const lines: string[] = [ - GENERATED_HEADER, - '# Error Reference', - '', - 'Every `skill-optimizer` error has a code, a short message, and a fix list.', - 'The catch-all `E_UNEXPECTED` appears if an error slips past the known list.', - '', - '## Summary', - '', - '| Code | Description | Quick fix |', - '|---|---|---|', - ]; - - for (const def of entries) { - const msg = def.message.replace(/\|/g, '\\|'); - const quickFix = (def.fix[0] ?? '').replace(/\|/g, '\\|'); - lines.push(`| \`${def.code}\` | ${msg} | ${quickFix} |`); - } - - lines.push('', '## Details', ''); - - for (const def of entries) { - lines.push(`### \`${def.code}\``); - lines.push(''); - lines.push(`**${def.message}**`); - lines.push(''); - lines.push('**How to fix:**'); - for (const step of def.fix) { - lines.push(`- ${step}`); - } - lines.push(''); - } - - return lines.join('\n'); -} - -// ── config-schema.md ────────────────────────────────────────────────────────── - -interface JsonSchemaNode { - type?: string; - description?: string; - default?: unknown; - enum?: unknown[]; - properties?: Record; - items?: JsonSchemaNode; - anyOf?: JsonSchemaNode[]; - $ref?: string; - $defs?: Record; - definitions?: Record; -} - -function typeLabel(node: JsonSchemaNode): string { - if (node.enum) return node.enum.map(v => `"${v}"`).join(' | '); - if (node.anyOf) return node.anyOf.map(typeLabel).filter(Boolean).join(' | '); - if (node.type === 'array') { - const itemLabel = node.items ? typeLabel(node.items) : 'any'; - return `${itemLabel}[]`; - } - return node.type ?? ''; -} - -function resolveRef(node: JsonSchemaNode, defs: Record): JsonSchemaNode { - if (!node.$ref) return node; - const refKey = node.$ref.replace(/^#\/\$defs\//, '').replace(/^#\/definitions\//, ''); - return defs[refKey] ?? node; -} - -function flattenSchema( - node: JsonSchemaNode, - prefix: string, - rows: Array<{ path: string; type: string; default: string; description: string }>, - defs: Record, -): void { - if (!node.properties) return; - - for (const [key, child] of Object.entries(node.properties)) { - const path = prefix ? `${prefix}.${key}` : key; - const resolved = resolveRef(child, defs); - - // If it has nested properties, recurse without adding a row for the parent - if (resolved.properties) { - flattenSchema(resolved, path, rows, defs); - } else { - rows.push({ - path, - type: typeLabel(resolved), - default: resolved.default !== undefined ? JSON.stringify(resolved.default) : '—', - description: resolved.description ?? '', - }); - } - } -} - -function generateConfigSchemaMd(): string { - const jsonSchema = zodToJsonSchema(ProjectConfigSchema, { - name: 'ProjectConfig', - $refStrategy: 'none', - }) as JsonSchemaNode; - - const defs: Record = { - ...(jsonSchema.$defs ?? {}), - ...(jsonSchema.definitions ?? {}), - }; - - // Resolve a top-level $ref if the named schema strategy wrapped everything in one - const root = jsonSchema.properties ? jsonSchema : resolveRef(jsonSchema, defs); - - const rows: Array<{ path: string; type: string; default: string; description: string }> = []; - flattenSchema(root, '', rows, defs); - - const lines: string[] = [ - GENERATED_HEADER, - '# Config Schema Reference', - '', - 'All configuration lives in a single `skill-optimizer.json` file.', - 'Paths in the config are relative to the config file location.', - '', - '| Field | Type | Default | Description |', - '|---|---|---|---|', - ]; - - for (const row of rows) { - const desc = row.description.replace(/\|/g, '\\|'); - lines.push(`| \`${row.path}\` | \`${row.type}\` | ${row.default} | ${desc} |`); - } - - lines.push(''); - return lines.join('\n'); -} - -// ── Write files ──────────────────────────────────────────────────────────────── - -const errorsPath = resolve(refDir, 'errors.md'); -writeFileSync(errorsPath, generateErrorsMd(), 'utf-8'); -console.log(`[gen-docs] Written: ${errorsPath}`); - -const schemaPath = resolve(refDir, 'config-schema.md'); -writeFileSync(schemaPath, generateConfigSchemaMd(), 'utf-8'); -console.log(`[gen-docs] Written: ${schemaPath}`); diff --git a/skills/skill-optimizer/SKILL.md b/skills/skill-optimizer/SKILL.md new file mode 100644 index 0000000..b376e89 --- /dev/null +++ b/skills/skill-optimizer/SKILL.md @@ -0,0 +1,210 @@ +--- +name: skill-optimizer +description: Use when creating, running, debugging, or documenting skill-optimizer workbench evals; working with agent skill cases, suites, graders, traces, Docker workspaces, OpenRouter model matrices, or the skill-optimizer SDK/CLI. +--- + +# skill-optimizer + +`skill-optimizer` is an eval workbench for agent skills. It runs a model in an isolated Docker `/work` directory, provides skills/references as normal workspace files, captures an agent trace, and grades deterministic local outcomes. + +Use this skill as the source of truth for authoring eval suites in this repo. Detailed schema and patterns are in `references/workbench.md`. + +## Core Model + +- A case is one user-like task plus one or more deterministic graders. +- A suite is a set of cases and OpenRouter models to run as a matrix. +- `references` are copied into `/work` before the agent starts; this is where eval skills live. +- The agent phase sees `/work` only. It cannot see `/case`, `/results`, graders, hidden answers, or hidden metadata. +- Cases can define `mcpServers`; these are exposed through a workbench `mcp` command during the agent phase. +- Graders run after the agent with `/case`, `/work`, and `/results` mounted. +- `trace.jsonl` is the debugging source for what the agent saw, said, and did. + +## Commands + +| Goal | Command | +|------|---------| +| Install deps | `npm install` | +| Build CLI | `npm run build` | +| Run one case | `npx tsx src/cli.ts run-case ` | +| Run one case across models | `npx tsx src/cli.ts run-case --models openrouter/google/gemini-2.5-flash,openrouter/openai/gpt-5.4` | +| Run a suite | `npx tsx src/cli.ts run-suite ` | +| CLI help | `npx tsx src/cli.ts --help` | + +Rules: + +- Use only `openrouter/...` model refs. +- `OPENROUTER_API_KEY` is required for real model runs. +- `run-suite` uses `models:` from `suite.yml`; it has no model override flag. +- `run-case` can use its case `model:` or `--model` / `--models`. +- Docker image default is `skill-optimizer-workbench:local`. + +## Install This Skill + +This repository ships one canonical skill at `skills/skill-optimizer/SKILL.md` plus plugin metadata for Claude Code, OpenCode, Codex, Cursor, and Gemini. + +Install the skill for common agents with: + +```bash +npx skills add fastxyz/skill-optimizer --skill skill-optimizer -a claude-code -a opencode -a codex -a cursor +``` + +Plugin entrypoints: + +- Claude Code: `.claude-plugin/plugin.json` and `.claude-plugin/marketplace.json` +- OpenCode: `.opencode/plugins/skill-optimizer.js` +- Codex: `.codex-plugin/plugin.json` +- Cursor: `.cursor-plugin/plugin.json` +- Gemini: `gemini-extension.json` and `GEMINI.md` + +## Authoring Workflow + +1. Create `suite.yml` with `models`, shared defaults, and inline cases or case paths. +2. Put the skill/reference material under `references/`; it will be copied into `/work`. +3. Write natural user tasks. Do not mention graders, hidden answers, `/case`, or eval internals. +4. Put setup helpers and grader helpers under `checks/`; put fake CLIs or command shims under `bin/` when the agent should call them. +5. Add one or more `graders` per case. Prefer small deterministic graders over one broad grader. +6. Run `run-suite --trials ` and inspect `suite-result.json`, failing `result.json`, `summary.json`, and `trace.jsonl`. + +Variables listed in `env` are forwarded unchanged into setup, agent, grading, and cleanup containers. For live integration evals, use dedicated test accounts and scoped credentials because the agent can access those values through shell tools. Treat `trace.jsonl`, `result.json`, grader evidence, stdout/stderr, and preserved `workspace/` directories as potentially sensitive if an agent or grader prints or writes secret values. + +Use `mcpServers` when the task should interact with MCP tools. For local servers whose source should stay hidden from the agent, put server files under the case `mcp/` support directory and define `mcpServices`; Docker starts those as separate service containers and the agent only sees their HTTP MCP URL. Direct stdio `mcpServers.command` entries run inside the agent container and are only appropriate when the server implementation is intentionally agent-visible. Remote HTTP/SSE servers must be reachable from Docker. The workbench generates `/work/mcporter.json` with `imports: []`, so host/user MCP configs are not imported. OAuth/browser auth is not supported; use env/header credentials listed in `env`. + +Prefer the real CLI/API/service when you do not know its internal behavior well enough to mock it faithfully. Mock only when you are sure the mock matches the real command surface, validation, outputs, and failure modes; otherwise the eval will measure the mock, not the skill. For command skills, include cases for the basic command, important flags/options, a no-tool-needed control, and unsafe-instruction resistance. + +## Minimal Suite + +```yaml +name: pdf-skill-eval +references: ./references +models: + - openrouter/google/gemini-2.5-flash +env: + - OPENROUTER_API_KEY +timeoutSeconds: 600 +setup: + - node $CASE/checks/create-inputs.mjs +appendSystemPrompt: | + Keep task outputs at the top level of /work unless the user asks otherwise. +cases: + - name: extract-pdf-facts + task: | + Read statement.pdf and write answer.json with the account, quarter, approval code, and risk flags. + graders: + - name: answer-json + command: node $CASE/checks/extract-pdf-facts.mjs +``` + +## Directory Layout + +```text +my-eval/ + suite.yml + references/ + my-skill/SKILL.md + checks/ + create-inputs.mjs + extract-pdf-facts.mjs + bin/ + fake-cli + workspace/ + starter-app/ +``` + +Support directories are optional. `checks/` is mounted read-only at `/case/checks` for setup/grading. `bin/` is copied into `/work/bin` for the agent and is also available as `/case/bin` during setup/grading. `workspace/` is copied into `/work` after `references/`. + +## Grader Contract + +Graders are shell commands. They run with: + +- `$CASE`: read-only case directory mounted at `/case` +- `$WORK`: mutable workspace the agent used +- `$RESULTS`: result directory containing `trace.jsonl` + +Preferred grader output: + +```json +{ "pass": true, "score": 1, "evidence": ["answer matched"] } +``` + +If no JSON object is printed, exit code `0` passes and non-zero fails. Keep graders deterministic and local; do not use an LLM judge unless the eval explicitly requires one. + +Graders are the acceptance contract. They should evaluate evidence in `/work`, generated artifacts, `answer.json`, `trace.jsonl`, and any relevant result-state files under `$RESULTS`. + +## Outputs + +```text +.results// + suite-result.json # run-suite aggregate + run-result.json # run-case matrix aggregate + trials/----001/ + trace.jsonl # agent messages and tool calls + result.json # pass, score, evidence, graders, metrics + summary.json # final text, failed graders, commands + workspace/ # failures or --keep-workspace +``` + +Use `trace.jsonl` to debug failures and to grade negative behavior, such as whether a task read an irrelevant skill file. + +## Optimization Loop + +After a run, inspect failing `result.json`, `summary.json`, `trace.jsonl`, and preserved `workspace/` evidence. Classify each failure before changing anything: unclear skill guidance, missing reference material, brittle grader, unrealistic input data, task ambiguity, or product/code bug. Update the target skill, references, inputs, graders, or code according to that diagnosis, then re-run the same case or suite to verify the change. Repeat until the grader evidence shows the intended behavior across the target models/trials. + +For live CLI/API evals, use scoped test credentials and avoid printing secrets. Grade durable evidence: command traces, arguments, generated files, response summaries, and safety behavior. Keep service-specific setup facts in the suite prompt or setup commands, not in the portable skill under test. + +## Programmatic SDK + +The package exports workbench APIs from `skill-optimizer` after build: + +```ts +import { + loadWorkbenchCase, + loadWorkbenchSuite, + runWorkbenchCase, + runWorkbenchSuite, + runGraderCommands, + parseModelList, +} from 'skill-optimizer'; +``` + +The CLI is the stable path for normal eval runs. Use SDK functions for tests, wrappers, and internal automation. + +## Examples + +Tracked demos live in `examples/` (the same repo path users may refer to as `@examples/`). Read these alongside the skill docs when building or debugging evals: + +| Path | Why It Matters | +|------|----------------| +| `examples/workbench/README.md` | Short command walkthrough for demos | +| `examples/workbench/pdf/README.md` | Explains the PDF demo cases and expected outputs | +| `examples/workbench/pdf/suite.yml` | Concrete suite using models, setup, env, graders, and append prompt | +| `examples/workbench/pdf/references/pdf-skill/SKILL.md` | Example skill copied into `/work` for the agent | +| `examples/workbench/pdf/checks/*.mjs` | Deterministic grader and setup helper patterns | +| `examples/workbench/mcp/suite.yml` | Hidden-service MCP calculator example | +| `examples/workbench/mcp/mcp/calculator-server.mjs` | Example MCP server with add/subtract/multiply/divide tools | + +```bash +npx tsx src/cli.ts run-suite examples/workbench/pdf/suite.yml --trials 1 +npx tsx src/cli.ts run-suite examples/workbench/mcp/suite.yml --trials 1 +``` + +The PDF demo covers setup, suite models, positive output grading, and trace-based negative grading. + +## Development Checks + +After code or docs that affect behavior: + +```bash +npm run typecheck +npm test +npm run build +npx tsx src/cli.ts --help +node dist/cli.js --help +``` + +After Dockerfile/container-runner changes: + +```bash +docker build -t skill-optimizer-workbench:local -f docker/workbench-runner.Dockerfile . +``` + +Do not commit `.skill-eval/`; it is local ignored eval data. diff --git a/skills/skill-optimizer/references/workbench.md b/skills/skill-optimizer/references/workbench.md new file mode 100644 index 0000000..e0c5a0f --- /dev/null +++ b/skills/skill-optimizer/references/workbench.md @@ -0,0 +1,533 @@ +# Workbench Reference + +This reference is for humans and agents authoring evals with the `skill-optimizer` CLI or SDK. + +## What The Workbench Evaluates + +The workbench is for tasks that can be graded from local evidence: + +- Files the agent creates or edits in `/work` +- Command invocations recorded by fake CLIs +- Generated files such as PDF, DOCX, PPTX, XLSX, images, JSON, or code +- Static SQL, shell scripts, config, or source code +- Agent behavior captured in `trace.jsonl` + +Avoid evals that require running model-produced arbitrary production code outside the container or using a second LLM as the default judge. + +## CLI Surface + +```bash +npx tsx src/cli.ts run-case +npx tsx src/cli.ts run-case --model openrouter/google/gemini-2.5-flash +npx tsx src/cli.ts run-case --models openrouter/google/gemini-2.5-flash,openrouter/openai/gpt-5.4 --trials 3 --concurrency 2 +npx tsx src/cli.ts run-suite --trials 3 --concurrency 2 +``` + +Options: + +| Command | Option | Meaning | +|---------|--------|---------| +| `run-case` | `--out ` | Results root, default `/.results` | +| `run-case` | `--model ` | Single OpenRouter model override | +| `run-case` | `--models ` | Comma-separated OpenRouter model refs | +| `run-case` | `--trials ` | Independent trials per model | +| `run-suite` | `--out ` | Results root, default `/.results` | +| `run-suite` | `--trials ` | Independent trials per case/model | +| both | `--concurrency ` | Maximum concurrent trial containers | +| both | `--image ` | Docker image, default `skill-optimizer-workbench:local` | +| both | `--keep-workspace` | Preserve successful workspaces too; failures are always preserved | + +Only `openrouter/...` model refs are accepted. `run-suite` uses the `models:` array in the suite file. + +## Case Schema + +Case files may be `.yml`, `.yaml`, or `.json`. + +```yaml +name: extract-pdf-facts +references: ./references +task: | + Read statement.pdf and write answer.json with the account, quarter, approval code, and risk flags. +graders: + - name: answer-json + command: node $CASE/checks/extract-pdf-facts.mjs +setup: + - node $CASE/checks/create-inputs.mjs +cleanup: [] +env: + - OPENROUTER_API_KEY +mcpServers: + calculator: + baseUrl: http://calculator:3000/mcp +mcpServices: + calculator: + command: node + args: + - calculator-server.mjs +model: openrouter/google/gemini-2.5-flash +timeoutSeconds: 600 +``` + +Required fields: + +| Field | Type | Meaning | +|-------|------|---------| +| `name` | string | Human-readable case name; suite inline cases slug this for result dirs | +| `references` | string | Directory copied into `/work` before the agent starts | +| `task` | string | User-like task sent to the agent | +| `graders` | array | Non-empty list of `{ name, command }` grader commands | + +Optional fields: + +| Field | Type | Meaning | +|-------|------|---------| +| `setup` | string[] | Commands run in `/work` before the agent phase | +| `cleanup` | string[] | Commands run after grading | +| `env` | string[] | Host environment variable names forwarded into setup, agent, grading, and cleanup containers | +| `mcpServers` | object | MCP servers exposed through the agent `mcp` tool | +| `mcpServices` | object | Hidden local MCP services started as separate Docker containers | +| `model` | string | Default model for `run-case`; defaults to `openrouter/google/gemini-2.5-flash` | +| `timeoutSeconds` | number | Agent timeout; defaults to `600` | + +All relative paths resolve from the case file directory. + +## Suite Schema + +Suites may contain inline case objects or paths to external case files. + +```yaml +name: pdf-workbench-example +references: ./references +models: + - openrouter/google/gemini-2.5-flash +env: + - OPENROUTER_API_KEY +timeoutSeconds: 600 +setup: + - node $CASE/checks/_pdf.mjs write-inputs input +appendSystemPrompt: | + Keep task outputs at the top level of /work unless the user asks otherwise. +cases: + - name: extract-pdf-facts + task: | + Read statement.pdf and write answer.json with the account, quarter, approval code, and risk flags. + graders: + - name: answer-json + command: node $CASE/checks/extract-pdf-facts.mjs + - cases/external-case/case.yml +``` + +Suite fields: + +| Field | Required | Meaning | +|-------|----------|---------| +| `name` | yes | Suite name in aggregate output | +| `models` | yes | OpenRouter model refs for the case/model matrix | +| `cases` | yes | Inline case objects or paths to case files | +| `references` | no | Default references dir for inline cases; defaults to `./references` | +| `env` | no | Default env allowlist for inline cases | +| `setup` | no | Default setup commands for inline cases | +| `cleanup` | no | Default cleanup commands for inline cases | +| `mcpServers` | no | Default MCP servers for inline cases, merged by server name | +| `mcpServices` | no | Default hidden MCP service containers for inline cases, merged by service name | +| `timeoutSeconds` | no | Default agent timeout for inline cases | +| `appendSystemPrompt` | no | Extra suite-wide system prompt appended after the workbench prompt | + +Inline case fields override suite defaults. External case files are loaded from their own file directory and do not inherit suite defaults. + +Environment variables listed in `env` are forwarded unchanged. This intentionally supports live integration evals such as authenticated CLI calls, but it also means the agent can read or print those values through shell tools. Use dedicated test accounts, least-privilege credentials, and cleanup routines for live systems. Treat `trace.jsonl`, `result.json`, grader evidence, stdout/stderr, and preserved `workspace/` directories as potentially sensitive if an agent or grader prints or writes secret values. + +## MCP Servers + +`mcpServers` uses mcporter-compatible server entries. During each Docker trial, the workbench writes `/work/mcporter.json` with `imports: []` and exposes an `mcp` command on `PATH`. + +The `mcp` command delegates to `mcporter`: + +```bash +mcp list calculator +mcp call calculator.add a=17 b=25 +``` + +Example suite default: + +```yaml +mcpServers: + calculator: + baseUrl: http://calculator:3000/mcp + context7: + baseUrl: https://mcp.context7.com/mcp + headers: + Authorization: "Bearer ${CONTEXT7_API_KEY}" +env: + - OPENROUTER_API_KEY + - CONTEXT7_API_KEY +mcpServices: + calculator: + command: node + args: + - calculator-server.mjs +``` + +Suite-level `mcpServers` apply only to inline cases. Inline cases merge by server name and win on conflicts. External case files define their own MCP servers and do not inherit suite defaults. + +Use `mcpServices` for local MCP servers whose source should not be visible to the agent. Service files live under the case `mcp/` support directory. During Docker runs, the workbench mounts that directory read-only into separate service containers at `/mcp`, joins those containers to a private Docker network, and joins the agent container to the same network. The agent sees only the configured `mcpServers` URL such as `http://calculator:3000/mcp`; it does not mount `/case` or the `mcp/` source directory. Set service ports in the matching `mcpServers` URL rather than in `mcpServices`. + +Remote HTTP/SSE servers must be reachable from Docker. `localhost` means the container, not the host, so use `host.docker.internal` or Docker networking for host-local services. Direct stdio `mcpServers.command` entries run inside the agent container and are only appropriate when the server implementation is intentionally agent-visible. + +OAuth/browser auth is not supported. Use non-interactive headers, bearer tokens, or env placeholders. Only variables listed in `env` are forwarded. + +## Directory Layout + +```text +eval-root/ + suite.yml + references/ + product-skill/SKILL.md + product-skill/references/api.md + checks/ + create-inputs.mjs + grade-output.mjs + trace-guards.mjs + bin/ + fake-product-cli + workspace/ + starter-repo/ +``` + +Directory behavior: + +| Directory | Visible To Agent | Purpose | +|-----------|------------------|---------| +| `references/` | yes, copied into `/work` | Skills, docs, examples, starter reference files | +| `workspace/` | yes, copied into `/work` | Seed app repo or starter files the agent may edit | +| `checks/` | no during agent phase | Graders and setup helpers under `/case/checks` | +| `bin/` | yes, copied into `/work/bin` and mounted as `/case/bin` during setup and grading | Fake CLIs and command shims on `PATH` | + +## Execution Phases + +`run-case` and `run-suite` use Docker for model attempts. Each trial is prepared on the host, then mounted into phase containers. + +| Phase | Docker Mounts | Working Dir | What Happens | +|-------|---------------|-------------|--------------| +| setup | `/case:ro`, `/work:rw` | `/work` | Run `setup` commands and prepare inputs | +| agent | `/work:rw` only | `/work` | Pi agent receives task and uses tools | +| grade | `/case:ro`, `/work:rw`, `/results:rw` | `/work` | Run grader commands and write result files | +| cleanup | `/case:ro`, `/work:rw`, `/results:rw` | `/work` | Run optional cleanup commands | + +Agent phase constraints: + +- No `/case` mount +- No `/results` mount +- No Docker socket +- No global/user Pi skills +- Additional skills are discovered from `/work` +- Configured MCP servers are exposed through the `mcp` command using `/work/mcporter.json` +- Python installs should use `/work/.venv` +- Internet is available unless Docker environment blocks it +- `env` allowlisted credentials are available unchanged to agent shell commands + +## Task Writing Rules + +Write tasks like normal user requests: + +- Ask for the actual deliverable and path. +- Include enough business detail to complete the task. +- Keep hidden expected answers in graders or hidden case support files, not in the task. +- Do not mention graders, answer keys, trace checks, `/case`, `/results`, or benchmark metadata. +- Do not instruct the agent to read or not read a skill unless that is the real user behavior being evaluated. + +Good task: + +```text +Read statement.pdf and write answer.json with the account, quarter, approval code, and risk flags. +``` + +Poor task: + +```text +Use the PDF skill and satisfy the grader in /case/checks/extract-pdf-facts.mjs. +``` + +## Grader Contract + +Each grader is a shell command run in `/work`. + +Environment variables: + +| Var | Meaning | +|-----|---------| +| `$CASE` | Read-only case directory mounted at `/case` | +| `$WORK` | Mutable workspace from the agent run | +| `$RESULTS` | Trial result directory with `trace.jsonl` | + +Preferred output is one JSON object on stdout: + +```json +{ "pass": false, "score": 0, "evidence": ["answer.json missing approvalCode"] } +``` + +Accepted fields: + +| Field | Type | Meaning | +|-------|------|---------| +| `pass` | boolean | Whether the grader passed | +| `score` | number | Optional score clamped to 0..1; defaults to 1 for pass and 0 for fail | +| `evidence` | string or string[] | Human-readable details surfaced in result files | + +If stdout does not contain a JSON object, exit code `0` passes and non-zero fails. JSON can be surrounded by logs; the runner parses the first object-shaped span from stdout. + +Grader principles: + +- Check one concept per grader when practical. +- Prefer exact structural checks over brittle prose matching. +- Print useful evidence for failure triage. +- Keep all grading deterministic and local. +- Graders should inspect `/work`, command logs, generated outputs, or `trace.jsonl`. + +## Grader Examples + +JSON output grader: + +```js +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const path = join(process.env.WORK, 'answer.json'); +const failures = []; + +if (!existsSync(path)) { + failures.push('answer.json was not created'); +} else { + const answer = JSON.parse(readFileSync(path, 'utf-8')); + if (answer.approvalCode !== 'PDF-7429') failures.push('approvalCode mismatch'); +} + +console.log(JSON.stringify({ + pass: failures.length === 0, + score: failures.length === 0 ? 1 : 0, + evidence: failures.length === 0 ? ['answer.json matched'] : failures, +})); +process.exit(failures.length === 0 ? 0 : 1); +``` + +Trace guard grader: + +```js +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const tracePath = join(process.env.RESULTS, 'trace.jsonl'); +const lines = existsSync(tracePath) ? readFileSync(tracePath, 'utf-8').trim().split(/\r?\n/) : []; +const readForbiddenSkill = lines.some((line) => { + try { + const entry = JSON.parse(line); + const path = entry?.arguments?.path ?? entry?.arguments?.filePath; + return entry.type === 'tool_call' && entry.name === 'read' && /\/pdf-skill\/SKILL\.md$/.test(path); + } catch { + return false; + } +}); + +console.log(JSON.stringify({ + pass: !readForbiddenSkill, + score: readForbiddenSkill ? 0 : 1, + evidence: readForbiddenSkill ? ['agent read the PDF skill'] : ['no forbidden skill read'], +})); +process.exit(readForbiddenSkill ? 1 : 0); +``` + +## Acceptance Contract + +Graders are the source of truth for pass/fail. They can evaluate: + +- Files and generated artifacts in `/work` +- Structured outputs such as `answer.json` +- Behavior traces in `$RESULTS/trace.jsonl` +- Any additional result-state files your checks create under `$RESULTS` + +Keep grading deterministic and local so results stay stable and reproducible. + +## Results And Metrics + +Single-trial `run-case` output: + +```text +case/.results// + trace.jsonl + result.json + summary.json + workspace/ # on failure or --keep-workspace +``` + +Matrix `run-case` output: + +```text +case/.results// + run-result.json + trials/--001/trace.jsonl + trials/--001/result.json +``` + +`run-suite` output: + +```text +suite/.results// + suite-result.json + trials/----001/trace.jsonl + trials/----001/result.json +``` + +`result.json` includes: + +- `pass`, `score`, and `evidence` +- Per-grader results under `graders` +- `metrics.durationMs`, turns, tool counts, tokens, and cost + +Aggregate files include: + +- `trialPassRate`: passed trials / total trials +- `meanScore`: mean top-level score +- `passAtK`: at least one trial passed +- `passHatK`: all trials passed +- Relative `tracePath`, `resultPath`, and `summaryPath` entries + +## Trace JSONL + +`trace.jsonl` is newline-delimited JSON. Useful entry shapes: + +```json +{ "type": "trace_start", "caseName": "extract-pdf-facts", "model": "openrouter/google/gemini-2.5-flash" } +{ "type": "message", "role": "assistant", "text": "..." } +{ "type": "tool_call", "name": "bash", "arguments": { "command": "node script.mjs" } } +{ "type": "tool_call", "name": "read", "arguments": { "path": "/work/pdf-skill/SKILL.md" } } +{ "type": "tool_result", "name": "bash", "text": "...", "isError": false } +``` + +Use trace evidence to debug why a model failed, verify tool usage, or enforce negative cases. + +## SDK Surface + +After `npm run build`, the package exports these workbench APIs from `skill-optimizer`: + +| API | Purpose | +|-----|---------| +| `loadWorkbenchCase(path)` | Parse and validate a case file | +| `loadWorkbenchSuite(path)` | Parse and validate a suite file | +| `runWorkbenchCase(params)` | Run one case or a model/trial matrix | +| `runWorkbenchSuite(params)` | Run a suite matrix | +| `runDockerWorkbenchCase(params)` | Lower-level Docker case runner | +| `runGraderCommands(graders, opts)` | Execute grader commands and normalize results | +| `normalizeCheckResult(result)` | Normalize shell output into a grade | +| `parseModelList(raw)` | Parse comma-separated OpenRouter refs | +| `aggregateTrials(results)` | Compute pass@k/pass^k/trial metrics | + +Example: + +```ts +import { runWorkbenchSuite } from 'skill-optimizer'; + +await runWorkbenchSuite({ + suitePath: 'examples/workbench/pdf/suite.yml', + trials: 3, + concurrency: 2, +}); +``` + +Use CLI commands for normal human workflows. Use SDK functions for tests, wrappers, and automation inside this repo. + +## Eval Patterns + +Live CLI/API Skills: + +- Prefer the real CLI/API/service when you are not certain how to mock its internals. +- Mock only when you know the real command surface, validation, outputs, and failure modes well enough to reproduce them faithfully. +- Use dedicated test credentials with least privilege, allowlist only the needed env vars, and avoid printing secrets into trace or grader evidence. +- If mocking is justified, put a fake executable in `bin/` and record calls to `$WORK/calls.jsonl`. Grade command names, flags, output files, and trace behavior. +- If the real tool is safe to call with setup/cleanup and scoped test credentials, install it in `setup` and grade its real dry-run or live request output. +- Include a basic-command case and a flag/options case for command-selection coverage. +- Include a no-tool-needed control case to catch unnecessary skill or CLI use. +- Include a prompt-injection or unsafe-instruction case when external content, fetched pages, or third-party responses can influence the agent. + +File-output skills: + +- Ask for a concrete output file. +- Grade structure directly, such as PDF page count, ZIP members, JSON schema, image dimensions, or file hash. +- Inspect failed workspaces or rerun with `--keep-workspace` when you need output files for triage. + +Code/editing skills: + +- Seed `workspace/` with a small repo. +- Ask for a normal change. +- Grade diff, tests, generated files, or static properties. + +Negative/control cases: + +- Ask for a task that should not require the target skill. +- Grade `trace.jsonl` for forbidden reads, tool calls, or commands. +- For trace-based negative cases, ensure graders handle missing or empty trace entries defensively. + +## Debugging Failed Runs + +1. Open the failing trial `result.json` and read top-level `evidence`. +2. Open `graders[]` to see which grader failed. +3. Open `summary.json` for final assistant text and bash commands. +4. Open `trace.jsonl` to inspect tool calls and file reads. +5. Inspect preserved `workspace/` for failed trials. +6. Classify the failure as unclear skill guidance, missing reference material, brittle grader, unrealistic input data, task ambiguity, or product/code bug. +7. Update the target skill, references, inputs, graders, or code according to that diagnosis. +8. Re-run the same case or suite and compare grader evidence across the target models/trials. + +## Example Suite + +The `examples/` tree (often referenced as `@examples/` in path-aware prompts) is part of the packaged skill-optimizer reference material. Use it as the concrete companion to this document. + +Start here: + +```text +examples/ + workbench/ + README.md + pdf/ + README.md + suite.yml + references/pdf-skill/SKILL.md + checks/*.mjs + mcp/ + mcp/calculator-server.mjs +``` + +The tracked PDF demo is the best starting point: + +```bash +npx tsx src/cli.ts run-suite examples/workbench/pdf/suite.yml --trials 1 +``` + +Files to inspect: + +| File | Purpose | +|------|---------| +| `examples/workbench/README.md` | Top-level example command walkthrough | +| `examples/workbench/pdf/suite.yml` | Inline suite using models, setup, graders, and append prompt | +| `examples/workbench/pdf/references/pdf-skill/SKILL.md` | Skill under test copied into `/work` | +| `examples/workbench/pdf/checks/*.mjs` | Deterministic graders and setup helpers | +| `examples/workbench/pdf/README.md` | Demo walkthrough | +| `examples/workbench/mcp/suite.yml` | Hidden-service MCP calculator demo | +| `examples/workbench/mcp/mcp/calculator-server.mjs` | Calculator MCP server with add/subtract/multiply/divide | + +## Repository Verification + +Use these before claiming repo changes are complete: + +```bash +npm run typecheck +npm test +npm run build +npx tsx src/cli.ts --help +node dist/cli.js --help +``` + +For runner/Docker changes, rebuild the image: + +```bash +docker build -t skill-optimizer-workbench:local -f docker/workbench-runner.Dockerfile . +``` diff --git a/src/actions/diff.ts b/src/actions/diff.ts deleted file mode 100644 index a1763b2..0000000 --- a/src/actions/diff.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { normalizeActionArgSchema } from './snapshot.js'; -import type { ActionCatalog, ActionDefinition } from './types.js'; - -export interface ChangedAction { - before: ActionDefinition; - after: ActionDefinition; -} - -export interface ActionCatalogDiff { - added: ActionDefinition[]; - removed: ActionDefinition[]; - changed: ChangedAction[]; -} - -function schemaFingerprint(action: ActionDefinition): string { - return JSON.stringify(normalizeActionArgSchema(action.args)); -} - -function indexByKey(actions: ActionDefinition[], side: 'before' | 'after'): Map { - const indexed = new Map(); - for (const action of actions) { - const canonicalKey = action.key.trim(); - if (indexed.has(canonicalKey)) { - throw new Error(`Duplicate action key in ${side} catalog: ${canonicalKey}`); - } - indexed.set(canonicalKey, { - ...action, - key: canonicalKey, - }); - } - return indexed; -} - -export function diffActionCatalog(before: ActionCatalog, after: ActionCatalog): ActionCatalogDiff { - const beforeByKey = indexByKey(before.actions, 'before'); - const afterByKey = indexByKey(after.actions, 'after'); - - const added: ActionDefinition[] = []; - const removed: ActionDefinition[] = []; - const changed: ChangedAction[] = []; - - for (const [key, afterAction] of afterByKey.entries()) { - const beforeAction = beforeByKey.get(key); - if (!beforeAction) { - added.push(afterAction); - continue; - } - - if (schemaFingerprint(beforeAction) !== schemaFingerprint(afterAction)) { - changed.push({ before: beforeAction, after: afterAction }); - } - } - - for (const [key, beforeAction] of beforeByKey.entries()) { - if (!afterByKey.has(key)) { - removed.push(beforeAction); - } - } - - return { added, removed, changed }; -} diff --git a/src/actions/discover.ts b/src/actions/discover.ts deleted file mode 100644 index 4e9d86e..0000000 --- a/src/actions/discover.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { loadCliCommands, loadMcpTools } from './loaders.js'; -import type { ResolvedProjectConfig } from '../project/types.js'; -import type { ActionCatalog } from './types.js'; -import { readCliActionsFromSources } from './readers/cli.js'; -import { readMcpActionsFromSources } from './readers/mcp.js'; -import { readSdkActionsFromSources } from './readers/sdk.js'; - -export function discoverActions(project: ResolvedProjectConfig): ActionCatalog { - const discoveryMode = project.target.discovery?.mode ?? 'auto'; - const discoverySources = project.target.discovery?.sources ?? []; - const shouldUseDiscovery = discoveryMode !== 'manifest' && discoverySources.length > 0; - - if (project.target.surface === 'sdk') { - if (shouldUseDiscovery) { - const actions = readSdkActionsFromSources(discoverySources); - if (actions.length > 0) { - return { - surface: 'sdk', - actions, - }; - } - - if ((project.target.sdk?.apiSurface?.length ?? 0) === 0) { - throw new Error(`SDK discovery found 0 actions from configured sources: ${discoverySources.join(', ')}`); - } - } - - return { - surface: 'sdk', - actions: (project.target.sdk?.apiSurface ?? []).map((name) => ({ - key: name, - name, - args: [], - source: 'sdk.apiSurface', - })), - }; - } - - if (project.target.surface === 'cli') { - if (shouldUseDiscovery) { - const actions = readCliActionsFromSources(discoverySources); - if (actions.length > 0) { - return { - surface: 'cli', - actions, - }; - } - - if (!project.target.cli?.commands) { - throw new Error(`CLI discovery found 0 actions from configured sources: ${discoverySources.join(', ')}`); - } - } - - const commands = project.target.cli ? loadCliCommands(project.target.cli.commands) : []; - return { - surface: 'cli', - actions: commands.map((command) => ({ - key: command.command, - name: command.command, - description: command.description, - args: (command.options ?? []).map((option) => ({ - name: normalizeCliArgName(option.name), - required: false, - type: option.takesValue ? 'string' : 'boolean', - description: option.description, - })), - source: 'cli.commands', - })), - }; - } - - if (shouldUseDiscovery) { - const actions = readMcpActionsFromSources(discoverySources); - if (actions.length > 0) { - return { - surface: 'mcp', - actions, - }; - } - - if (!project.target.mcp?.tools) { - throw new Error(`MCP discovery found 0 actions from configured sources: ${discoverySources.join(', ')}`); - } - } - - const tools = project.target.mcp ? loadMcpTools(project.target.mcp.tools) : []; - return { - surface: 'mcp', - actions: tools.map((tool) => ({ - key: tool.function.name, - name: tool.function.name, - description: tool.function.description, - args: Object.entries(tool.function.parameters?.properties ?? {}).map(([name, schema]) => ({ - name, - required: (tool.function.parameters?.required ?? []).includes(name), - type: typeof schema === 'object' && schema && 'type' in schema ? String((schema as { type?: unknown }).type ?? '') || undefined : undefined, - description: typeof schema === 'object' && schema && 'description' in schema ? String((schema as { description?: unknown }).description ?? '') || undefined : undefined, - schema: typeof schema === 'object' && schema && !Array.isArray(schema) - ? schema as Record - : undefined, - })), - source: 'mcp.tools', - })), - }; -} - -function normalizeCliArgName(name: string): string { - return name.replace(/^-+/, ''); -} diff --git a/src/actions/index.ts b/src/actions/index.ts deleted file mode 100644 index dda1e60..0000000 --- a/src/actions/index.ts +++ /dev/null @@ -1,25 +0,0 @@ -export { diffActionCatalog, type ActionCatalogDiff, type ChangedAction } from './diff.js'; -export { loadCliCommands, loadMcpTools } from './loaders.js'; -export { discoverActions } from './discover.js'; -export { readCliActionsFromSources } from './readers/cli.js'; -export { readMcpActionsFromSources } from './readers/mcp.js'; -export { readSdkActionsFromSources } from './readers/sdk.js'; -export { - ACTION_SNAPSHOT_VERSION, - fromSurfaceSnapshot, - loadActionSnapshotFile, - normalizeActionArgSchema, - normalizeActionCatalog, - normalizeActionDefinition, - toSurfaceSnapshot, - writeActionSnapshotFile, - type ActionSnapshotArtifact, -} from './snapshot.js'; - -export type { - ActionArgSchema, - ActionAttempt, - ActionCatalog, - ActionDefinition, - ActionSurface, -} from './types.js'; diff --git a/src/actions/loaders.ts b/src/actions/loaders.ts deleted file mode 100644 index f4088f2..0000000 --- a/src/actions/loaders.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { readFileSync, existsSync } from 'node:fs'; -import { resolve } from 'node:path'; -import type { McpToolDefinition, CliCommandDefinition } from '../benchmark/types.js'; - -/** - * Load MCP tool definitions from the tools.json path specified in config. - */ -export function loadMcpTools(toolsPath: string, baseDir?: string): McpToolDefinition[] { - const resolved = resolve(baseDir ?? process.cwd(), toolsPath); - if (!existsSync(resolved)) { - throw new Error(`MCP tools file not found: ${resolved}`); - } - - let raw: string; - try { - raw = readFileSync(resolved, 'utf-8'); - } catch (err) { - throw new Error(`Failed to read MCP tools: ${resolved}: ${err instanceof Error ? err.message : err}`); - } - - let tools: McpToolDefinition[]; - try { - tools = JSON.parse(raw) as McpToolDefinition[]; - } catch (err) { - throw new Error(`Invalid JSON in MCP tools file: ${resolved}: ${err instanceof Error ? err.message : err}`); - } - - if (!Array.isArray(tools)) { - throw new Error(`MCP tools file ${resolved}: must be a JSON array of tool definitions`); - } - - return tools; -} - -/** - * Load CLI command definitions from the commands.json path specified in config. - */ -export function loadCliCommands(commandsPath: string, baseDir?: string): CliCommandDefinition[] { - const resolved = resolve(baseDir ?? process.cwd(), commandsPath); - if (!existsSync(resolved)) { - throw new Error(`CLI commands file not found: ${resolved}`); - } - - let raw: string; - try { - raw = readFileSync(resolved, 'utf-8'); - } catch (err) { - throw new Error(`Failed to read CLI commands: ${resolved}: ${err instanceof Error ? err.message : err}`); - } - - let commands: CliCommandDefinition[]; - try { - commands = JSON.parse(raw) as CliCommandDefinition[]; - } catch (err) { - throw new Error(`Invalid JSON in CLI commands file: ${resolved}: ${err instanceof Error ? err.message : err}`); - } - - if (!Array.isArray(commands)) { - throw new Error(`CLI commands file ${resolved}: must be a JSON array of command definitions`); - } - - for (const [index, command] of commands.entries()) { - if (!command || typeof command !== 'object') { - throw new Error(`CLI commands file ${resolved}: entry ${index} must be an object`); - } - if (typeof command.command !== 'string' || command.command.trim() === '') { - throw new Error(`CLI commands file ${resolved}: entry ${index} must include a non-empty "command" string`); - } - if (command.options !== undefined && !Array.isArray(command.options)) { - throw new Error(`CLI commands file ${resolved}: entry ${index} options must be an array when present`); - } - if (Array.isArray(command.options)) { - for (const [optionIndex, option] of command.options.entries()) { - if (!option || typeof option !== 'object' || typeof option.name !== 'string' || option.name.trim() === '') { - throw new Error(`CLI commands file ${resolved}: entry ${index} option ${optionIndex} must include a non-empty "name" string`); - } - } - } - } - - return commands; -} diff --git a/src/actions/readers/cli.ts b/src/actions/readers/cli.ts deleted file mode 100644 index a1de900..0000000 --- a/src/actions/readers/cli.ts +++ /dev/null @@ -1,21 +0,0 @@ -import type { DiscoveryOptions } from '../../discovery/types.js'; -import { discoverCliSurfaceFromSources } from '../../discovery/cli.js'; -import type { ActionDefinition } from '../types.js'; - -export function readCliActionsFromSources(sources: string[], options: DiscoveryOptions = {}): ActionDefinition[] { - const snapshot = discoverCliSurfaceFromSources(sources, options); - return snapshot.actions.map((action) => ({ - key: action.name, - name: action.name, - description: action.description, - args: action.args.map((arg) => ({ - ...arg, - name: normalizeCliArgName(arg.name), - })), - source: action.source, - })); -} - -function normalizeCliArgName(name: string): string { - return name.replace(/^-+/, ''); -} diff --git a/src/actions/readers/mcp.ts b/src/actions/readers/mcp.ts deleted file mode 100644 index 08a40d9..0000000 --- a/src/actions/readers/mcp.ts +++ /dev/null @@ -1,14 +0,0 @@ -import type { DiscoveryOptions } from '../../discovery/types.js'; -import { discoverMcpSurfaceFromSources } from '../../discovery/mcp.js'; -import type { ActionDefinition } from '../types.js'; - -export function readMcpActionsFromSources(sources: string[], options: DiscoveryOptions = {}): ActionDefinition[] { - const snapshot = discoverMcpSurfaceFromSources(sources, options); - return snapshot.actions.map((action) => ({ - key: action.name, - name: action.name, - description: action.description, - args: action.args, - source: action.source, - })); -} diff --git a/src/actions/readers/sdk.ts b/src/actions/readers/sdk.ts deleted file mode 100644 index e7eb12c..0000000 --- a/src/actions/readers/sdk.ts +++ /dev/null @@ -1,14 +0,0 @@ -import type { DiscoveryOptions } from '../../discovery/types.js'; -import { discoverSdkSurfaceFromSources } from '../../discovery/sdk.js'; -import type { ActionDefinition } from '../types.js'; - -export function readSdkActionsFromSources(sources: string[], options: DiscoveryOptions = {}): ActionDefinition[] { - const snapshot = discoverSdkSurfaceFromSources(sources, options); - return snapshot.actions.map((action) => ({ - key: action.name, - name: action.name, - description: action.description, - args: action.args, - source: action.source, - })); -} diff --git a/src/actions/snapshot.ts b/src/actions/snapshot.ts deleted file mode 100644 index c265a7b..0000000 --- a/src/actions/snapshot.ts +++ /dev/null @@ -1,211 +0,0 @@ -import { existsSync, readFileSync, writeFileSync } from 'node:fs'; - -import type { SurfaceSnapshot } from '../project/types.js'; -import type { ActionArgSchema, ActionCatalog, ActionDefinition } from './types.js'; - -export const ACTION_SNAPSHOT_VERSION = 1; - -export interface ActionSnapshotArtifact { - version: typeof ACTION_SNAPSHOT_VERSION; - catalog: ActionCatalog; -} - -function invalidSnapshot(snapshotPath: string, detail: string): never { - throw new Error(`Invalid action snapshot file: ${snapshotPath} (${detail})`); -} - -function validateActionArgs(snapshotPath: string, actionIndex: number, args: unknown): ActionArgSchema[] { - if (!Array.isArray(args)) { - invalidSnapshot(snapshotPath, `catalog.actions[${actionIndex}].args must be an array`); - } - - return args.map((arg, argIndex) => { - const path = `catalog.actions[${actionIndex}].args[${argIndex}]`; - if (!arg || typeof arg !== 'object') { - invalidSnapshot(snapshotPath, `${path} must be an object`); - } - const candidate = arg as Partial; - if (typeof candidate.name !== 'string') { - invalidSnapshot(snapshotPath, `${path}.name must be a string`); - } - if (typeof candidate.required !== 'boolean') { - invalidSnapshot(snapshotPath, `${path}.required must be a boolean`); - } - if (candidate.type !== undefined && typeof candidate.type !== 'string') { - invalidSnapshot(snapshotPath, `${path}.type must be a string when provided`); - } - if (candidate.description !== undefined && typeof candidate.description !== 'string') { - invalidSnapshot(snapshotPath, `${path}.description must be a string when provided`); - } - if (candidate.schema !== undefined && (!candidate.schema || typeof candidate.schema !== 'object' || Array.isArray(candidate.schema))) { - invalidSnapshot(snapshotPath, `${path}.schema must be an object when provided`); - } - - return { - name: candidate.name, - required: candidate.required, - type: candidate.type, - description: candidate.description, - schema: candidate.schema as Record | undefined, - }; - }); -} - -function validateCatalogActions(snapshotPath: string, actions: unknown): ActionDefinition[] { - if (!Array.isArray(actions)) { - invalidSnapshot(snapshotPath, 'catalog.actions must be an array'); - } - - return actions.map((action, actionIndex) => { - const path = `catalog.actions[${actionIndex}]`; - if (!action || typeof action !== 'object') { - invalidSnapshot(snapshotPath, `${path} must be an object`); - } - const candidate = action as Partial; - if (typeof candidate.key !== 'string') { - invalidSnapshot(snapshotPath, `${path}.key must be a string`); - } - if (typeof candidate.name !== 'string') { - invalidSnapshot(snapshotPath, `${path}.name must be a string`); - } - if (candidate.description !== undefined && typeof candidate.description !== 'string') { - invalidSnapshot(snapshotPath, `${path}.description must be a string when provided`); - } - if (candidate.source !== undefined && typeof candidate.source !== 'string') { - invalidSnapshot(snapshotPath, `${path}.source must be a string when provided`); - } - - return { - key: candidate.key, - name: candidate.name, - description: candidate.description, - args: validateActionArgs(snapshotPath, actionIndex, candidate.args), - source: candidate.source, - }; - }); -} - -function normalizeSchemaValue(value: unknown): unknown { - if (Array.isArray(value)) { - return value.map(normalizeSchemaValue); - } - - if (!value || typeof value !== 'object') { - return value; - } - - const normalizedEntries = Object.entries(value as Record) - .sort(([left], [right]) => left.localeCompare(right)) - .map(([key, childValue]) => { - if (key === 'required' && Array.isArray(childValue) && childValue.every((entry) => typeof entry === 'string')) { - return [key, [...childValue].sort()] as const; - } - - return [key, normalizeSchemaValue(childValue)] as const; - }); - - return Object.fromEntries(normalizedEntries); -} - -export function normalizeActionArgSchema(args: ActionArgSchema[]): ActionArgSchema[] { - return [...args] - .map((arg) => ({ - name: arg.name, - required: Boolean(arg.required), - type: arg.type, - description: arg.description, - schema: arg.schema ? normalizeSchemaValue(arg.schema) as Record : undefined, - })) - .sort((a, b) => a.name.localeCompare(b.name)); -} - -export function normalizeActionDefinition(action: ActionDefinition): ActionDefinition { - return { - ...action, - key: action.key.trim(), - args: normalizeActionArgSchema(action.args), - }; -} - -export function normalizeActionCatalog(catalog: ActionCatalog): ActionCatalog { - return { - surface: catalog.surface, - actions: catalog.actions.map(normalizeActionDefinition), - }; -} - -export function writeActionSnapshotFile(snapshotPath: string, catalog: ActionCatalog): void { - const artifact: ActionSnapshotArtifact = { - version: ACTION_SNAPSHOT_VERSION, - catalog: normalizeActionCatalog(catalog), - }; - writeFileSync(snapshotPath, JSON.stringify(artifact, null, 2), 'utf-8'); -} - -export function loadActionSnapshotFile(snapshotPath: string): ActionSnapshotArtifact { - if (!existsSync(snapshotPath)) { - throw new Error(`Action snapshot file not found: ${snapshotPath}`); - } - - const raw = readFileSync(snapshotPath, 'utf-8'); - let parsed: unknown; - try { - parsed = JSON.parse(raw) as unknown; - } catch (error) { - invalidSnapshot(snapshotPath, `invalid JSON: ${error instanceof Error ? error.message : String(error)}`); - } - if (!parsed || typeof parsed !== 'object') { - invalidSnapshot(snapshotPath, 'expected object root'); - } - - const candidate = parsed as Partial; - if (typeof candidate.version !== 'number') { - invalidSnapshot(snapshotPath, 'version must be a number'); - } - if (candidate.version !== ACTION_SNAPSHOT_VERSION) { - throw new Error(`Unsupported action snapshot version ${candidate.version}; expected ${ACTION_SNAPSHOT_VERSION}`); - } - - if (!candidate.catalog || typeof candidate.catalog !== 'object') { - invalidSnapshot(snapshotPath, 'catalog must be an object'); - } - - const catalog = candidate.catalog as Partial; - if (catalog.surface !== 'sdk' && catalog.surface !== 'cli' && catalog.surface !== 'mcp' && catalog.surface !== 'prompt') { - invalidSnapshot(snapshotPath, 'catalog.surface must be one of sdk|cli|mcp|prompt'); - } - const validatedActions = validateCatalogActions(snapshotPath, catalog.actions); - - return { - version: ACTION_SNAPSHOT_VERSION, - catalog: normalizeActionCatalog({ - surface: catalog.surface, - actions: validatedActions, - }), - }; -} - -export function fromSurfaceSnapshot(snapshot: SurfaceSnapshot): ActionCatalog { - return normalizeActionCatalog({ - surface: snapshot.surface, - actions: snapshot.actions.map((action) => ({ - key: action.name, - name: action.name, - description: action.description, - args: normalizeActionArgSchema(action.args), - source: action.source, - })), - }); -} - -export function toSurfaceSnapshot(catalog: ActionCatalog): SurfaceSnapshot { - return { - surface: catalog.surface, - actions: catalog.actions.map((action) => ({ - name: action.name, - description: action.description, - args: normalizeActionArgSchema(action.args), - source: action.source, - })), - }; -} diff --git a/src/actions/types.ts b/src/actions/types.ts deleted file mode 100644 index 1ed78c4..0000000 --- a/src/actions/types.ts +++ /dev/null @@ -1,30 +0,0 @@ -export type ActionSurface = 'sdk' | 'cli' | 'mcp' | 'prompt'; - -export interface ActionArgSchema { - name: string; - required: boolean; - type?: string; - description?: string; - schema?: Record; -} - -export interface ActionDefinition { - key: string; - name: string; - description?: string; - args: ActionArgSchema[]; - source?: string; -} - -export interface ActionCatalog { - surface: ActionSurface; - actions: ActionDefinition[]; -} - -export interface ActionAttempt { - method: string; - key?: string; - args: Record; - line: number; - raw: string; -} diff --git a/src/benchmark/compare.ts b/src/benchmark/compare.ts deleted file mode 100644 index 980ec8f..0000000 --- a/src/benchmark/compare.ts +++ /dev/null @@ -1,244 +0,0 @@ -import { readFileSync } from 'node:fs'; -import type { BenchmarkReport, ComparisonReport, TaskDelta, Delta } from './types.js'; - -/** - * Load a benchmark report from a JSON file. - */ -export function loadReport(path: string): BenchmarkReport { - return JSON.parse(readFileSync(path, 'utf-8')) as BenchmarkReport; -} - -/** - * Compare baseline and current reports. Compute deltas for each task×model pair. - */ -export function compareReports( - baseline: BenchmarkReport, - current: BenchmarkReport, -): ComparisonReport { - const baselineMap = new Map(); - for (const r of baseline.results) { - const key = `${r.task.id}:${r.model.id}`; - baselineMap.set(key, { - passed: r.metrics.taskPassed, - recall: r.metrics.toolRecall, - toolSelection: r.metrics.toolSelectionAccuracy, - }); - } - - const currentMap = new Map(); - for (const r of current.results) { - const key = `${r.task.id}:${r.model.id}`; - currentMap.set(key, { - passed: r.metrics.taskPassed, - recall: r.metrics.toolRecall, - toolSelection: r.metrics.toolSelectionAccuracy, - taskId: r.task.id, - modelId: r.model.id, - }); - } - - const allKeys = new Set([...baselineMap.keys(), ...currentMap.keys()]); - - const taskDeltas: TaskDelta[] = []; - let improved = 0; - let regressed = 0; - let unchanged = 0; - - for (const key of allKeys) { - const [taskId, ...modelParts] = key.split(':'); - const modelId = modelParts.join(':'); // model IDs may contain ':' - - const inBaseline = baselineMap.get(key); - const inCurrent = currentMap.get(key); - - let delta: Delta; - let passedBefore = false; - let passedNow = false; - let recallBefore = 0; - let recallNow = 0; - - if (inBaseline && inCurrent) { - passedBefore = inBaseline.passed; - passedNow = inCurrent.passed; - recallBefore = inBaseline.recall; - recallNow = inCurrent.recall; - - if (!passedBefore && passedNow) { - delta = 'improved'; - improved++; - } else if (passedBefore && !passedNow) { - delta = 'regressed'; - regressed++; - } else { - delta = 'unchanged'; - unchanged++; - } - } else if (inCurrent && !inBaseline) { - passedNow = inCurrent.passed; - recallNow = inCurrent.recall; - delta = 'new'; - } else { - passedBefore = inBaseline!.passed; - recallBefore = inBaseline!.recall; - delta = 'removed'; - } - - taskDeltas.push({ - taskId: taskId ?? key, - modelId, - passedBefore, - passedNow, - delta, - recallBefore, - recallNow, - toolSelectionBefore: inBaseline?.toolSelection ?? 0, - toolSelectionNow: inCurrent?.toolSelection ?? 0, - }); - } - - const deltaOrder: Record = { - regressed: 0, - improved: 1, - new: 2, - removed: 3, - unchanged: 4, - }; - taskDeltas.sort((a, b) => { - const orderDiff = deltaOrder[a.delta] - deltaOrder[b.delta]; - if (orderDiff !== 0) return orderDiff; - if (a.taskId < b.taskId) return -1; - if (a.taskId > b.taskId) return 1; - return a.modelId.localeCompare(b.modelId); - }); - - const coverageBefore = baseline.summary.methodCoveragePercent; - const coverageNow = current.summary.methodCoveragePercent; - const accuracyBefore = baseline.summary.overallPassRate; - const accuracyNow = current.summary.overallPassRate; - - return { - baseline: { - timestamp: baseline.timestamp, - skillVersion: baseline.skillVersion, - }, - current: { - timestamp: current.timestamp, - skillVersion: current.skillVersion, - }, - taskDeltas, - summary: { - improved, - regressed, - unchanged, - coverageBefore, - coverageNow, - accuracyBefore, - accuracyNow, - }, - }; -} - -/** - * Print comparison to console. - */ -export function printComparison(comparison: ComparisonReport): void { - const { baseline, current, taskDeltas, summary } = comparison; - - const baseSha = baseline.skillVersion.commitSha.slice(0, 8); - const curSha = current.skillVersion.commitSha.slice(0, 8); - - console.log(''); - console.log(`Skill Version: ${baseSha} → ${curSha}`); - console.log(`Baseline: ${new Date(baseline.timestamp).toUTCString()}`); - console.log(`Current: ${new Date(current.timestamp).toUTCString()}`); - console.log(''); - - // Column widths - const COL_TASK = 26; - const COL_MODEL = 20; - const COL_BEFORE = 10; - const COL_AFTER = 10; - const COL_DELTA = 12; - - // Header - const header = - padR('Task', COL_TASK) + - ' ' + - padR('Model', COL_MODEL) + - ' ' + - padR('Baseline', COL_BEFORE) + - ' ' + - padR('Current', COL_AFTER) + - ' ' + - 'Delta'; - console.log(header); - console.log('─'.repeat(header.length + COL_DELTA)); - - const interesting = taskDeltas.filter(d => d.delta !== 'unchanged'); - const unchangedDeltas = taskDeltas.filter(d => d.delta === 'unchanged'); - - const printRow = (d: TaskDelta): void => { - const before = d.delta === 'new' ? '—' : d.passedBefore ? '✅' : '❌'; - const after = d.delta === 'removed' ? '—' : d.passedNow ? '✅' : '❌'; - - let deltaLabel: string; - switch (d.delta) { - case 'improved': - deltaLabel = 'IMPROVED ↑'; - break; - case 'regressed': - deltaLabel = 'REGRESSED ↓'; - break; - case 'new': - deltaLabel = 'new'; - break; - case 'removed': - deltaLabel = 'removed'; - break; - default: - deltaLabel = 'unchanged'; - } - - const modelDisplay = d.modelId.includes('/') ? d.modelId.split('/').pop()! : d.modelId; - - console.log( - padR(d.taskId, COL_TASK) + - ' ' + - padR(modelDisplay.slice(0, COL_MODEL), COL_MODEL) + - ' ' + - padR(before, COL_BEFORE) + - ' ' + - padR(after, COL_AFTER) + - ' ' + - deltaLabel, - ); - }; - - for (const d of interesting) { - printRow(d); - } - - if (interesting.length > 0 && unchangedDeltas.length > 0) { - console.log(` ... and ${unchangedDeltas.length} unchanged result(s)`); - } else if (unchangedDeltas.length > 0 && interesting.length === 0) { - console.log(` All ${unchangedDeltas.length} result(s) unchanged.`); - } - - console.log(''); - console.log( - `Summary: ${summary.improved} improved, ${summary.regressed} regressed, ${summary.unchanged} unchanged`, - ); - console.log( - `Coverage: ${(summary.coverageBefore * 100).toFixed(1)}% → ${(summary.coverageNow * 100).toFixed(1)}%`, - ); - console.log( - `Accuracy: ${(summary.accuracyBefore * 100).toFixed(1)}% → ${(summary.accuracyNow * 100).toFixed(1)}%`, - ); - console.log(''); -} - -// ── Internal helpers ────────────────────────────────────────────────────────── - -function padR(s: string, w: number): string { - return s.length >= w ? s.slice(0, w) : s + ' '.repeat(w - s.length); -} diff --git a/src/benchmark/config.ts b/src/benchmark/config.ts deleted file mode 100644 index b5f9923..0000000 --- a/src/benchmark/config.ts +++ /dev/null @@ -1,160 +0,0 @@ -import { readFileSync, existsSync } from 'node:fs'; -import { resolve } from 'node:path'; -import type { - BenchmarkConfig, - TaskDefinition, - ModelConfig, - ExpectedAction, -} from './types.js'; -import { DEFAULT_PROJECT_CONFIG_NAME, loadProjectConfig, toBenchmarkConfig } from '../project/index.js'; -export { loadMcpTools, loadCliCommands } from '../actions/loaders.js'; - -const DEFAULT_CONFIG_NAME = DEFAULT_PROJECT_CONFIG_NAME; -const SAFE_TASK_ID = /^[A-Za-z0-9._-]+$/; - -function isSafeTaskId(taskId: string): boolean { - return SAFE_TASK_ID.test(taskId) && taskId !== '.' && taskId !== '..'; -} - -/** - * Load benchmark config from the given path or search for benchmark.config.json - * in the current working directory. - */ -export async function loadConfig(configPath?: string): Promise<{ config: BenchmarkConfig; configDir: string }> { - // Skip the dirty-git check here — it runs on every benchmark invocation (baseline, - // each iteration) which causes false failures when the mutation agent operates in - // the target repo between iterations. The optimizer manages git state via ensureReady - // (run once before the loop); the standalone `run` command validates via its own - // loadProjectConfig call in cli.ts before generating tasks. - const project = await loadProjectConfig(configPath ?? DEFAULT_CONFIG_NAME, { skipDirtyGitCheck: true }); - return { - config: toBenchmarkConfig(project), - configDir: project.configDir, - }; -} - -/** - * Load task definitions from the tasks.json path specified in config. - * Resolves the path relative to the config file's directory or CWD. - */ -export function loadTasks(tasksPath: string, baseDir?: string): TaskDefinition[] { - const resolved = resolve(baseDir ?? process.cwd(), tasksPath); - if (!existsSync(resolved)) { - throw new Error(`Tasks file not found: ${resolved}`); - } - - let raw: string; - try { - raw = readFileSync(resolved, 'utf-8'); - } catch (err) { - throw new Error(`Failed to read tasks: ${resolved}: ${err instanceof Error ? err.message : err}`); - } - - let parsed: { tasks: Array<{ id?: unknown; prompt?: unknown; expected_actions?: unknown; verify?: unknown; expected_fetches?: unknown; capabilityId?: unknown }> }; - try { - parsed = JSON.parse(raw) as typeof parsed; - } catch (err) { - throw new Error(`Invalid JSON in tasks file: ${resolved}: ${err instanceof Error ? err.message : err}`); - } - - if (!parsed.tasks || !Array.isArray(parsed.tasks)) { - throw new Error(`Tasks file ${resolved}: must have a "tasks" array at the root`); - } - - return parsed.tasks.map((task, index) => normalizeTaskDefinition(task, resolved, index)); -} - -function normalizeTaskDefinition( - task: { id?: unknown; prompt?: unknown; expected_actions?: unknown; verify?: unknown; expected_fetches?: unknown; capabilityId?: unknown }, - resolvedPath: string, - index: number, -): TaskDefinition { - if (typeof task.id !== 'string' || task.id.trim() === '') { - throw new Error(`Tasks file ${resolvedPath}: task at index ${index} must include a non-empty string id`); - } - if (!isSafeTaskId(task.id)) { - throw new Error(`Tasks file ${resolvedPath}: task id "${task.id}" must match ${SAFE_TASK_ID.toString()} and cannot be . or ..`); - } - if (typeof task.prompt !== 'string' || task.prompt.trim() === '') { - throw new Error(`Tasks file ${resolvedPath}: task ${task.id} must include a non-empty string prompt`); - } - - const rawExpectedActions = Array.isArray(task.expected_actions) ? task.expected_actions : null; - - if (!rawExpectedActions) { - throw new Error(`Tasks file ${resolvedPath}: task at index ${index} must include an expected_actions array`); - } - - const expected_actions = rawExpectedActions.map((rawAction, actionIndex) => normalizeExpectedAction(rawAction, resolvedPath, index, actionIndex)); - - const rawVerify = Array.isArray(task.verify) ? task.verify : undefined; - if (rawVerify !== undefined) { - for (let i = 0; i < rawVerify.length; i++) { - if (!rawVerify[i] || typeof rawVerify[i] !== 'object') { - throw new Error(`Tasks file ${resolvedPath}: task ${task.id} verify[${i}] must be an object`); - } - } - } - - const rawFetches = Array.isArray(task.expected_fetches) ? task.expected_fetches : undefined; - if (rawFetches !== undefined) { - for (let i = 0; i < rawFetches.length; i++) { - if (typeof rawFetches[i] !== 'string' || !(rawFetches[i] as string).trim()) { - throw new Error(`Tasks file ${resolvedPath}: task ${task.id} expected_fetches[${i}] must be a non-empty string`); - } - } - } - - const capabilityId = typeof task.capabilityId === 'string' ? task.capabilityId : undefined; - - return { - id: task.id, - prompt: task.prompt, - expected_actions, - verify: rawVerify as TaskDefinition['verify'] | undefined, - expected_fetches: rawFetches as string[] | undefined, - ...(capabilityId !== undefined ? { capabilityId } : {}), - }; -} - -function normalizeExpectedAction( - rawAction: unknown, - resolvedPath: string, - taskIndex: number, - actionIndex: number, -): ExpectedAction { - if (!rawAction || typeof rawAction !== 'object') { - throw new Error(`Tasks file ${resolvedPath}: task ${taskIndex} action ${actionIndex} must be an object`); - } - - const candidate = rawAction as { name?: unknown; args?: unknown }; - const name = typeof candidate.name === 'string' ? candidate.name : null; - - if (!name || name.trim() === '') { - throw new Error(`Tasks file ${resolvedPath}: task ${taskIndex} action ${actionIndex} must include a non-empty name`); - } - - if (candidate.args !== undefined && (!candidate.args || typeof candidate.args !== 'object' || Array.isArray(candidate.args))) { - throw new Error(`Tasks file ${resolvedPath}: task ${taskIndex} action ${actionIndex} args must be an object when present`); - } - - return { - name, - args: (candidate.args as Record | undefined) ?? {}, - }; -} - -/** - * Helper to get a model by slug (lowercased name with non-alphanumeric replaced by hyphens). - */ -export function slugify(name: string): string { - return name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, ''); -} - -export function getModelBySlug(config: BenchmarkConfig, slug: string): ModelConfig | undefined { - return config.llm.models.find(m => slugify(m.name) === slug || m.id === slug); -} - -export function getModelsByTier(config: BenchmarkConfig, tier: string): ModelConfig[] { - return config.llm.models.filter(m => m.tier === tier); -} diff --git a/src/benchmark/coverage.ts b/src/benchmark/coverage.ts deleted file mode 100644 index f7bce4e..0000000 --- a/src/benchmark/coverage.ts +++ /dev/null @@ -1,62 +0,0 @@ -import type { TaskDefinition, MethodCoverage } from './types.js'; -import { getExpectedActions, getExpectedActionName } from './types.js'; - -// ── Coverage computation ─────────────────────────────────────────────────── - -/** - * Compute which methods are covered by at least one task in the task suite. - * - * @param tasks - The task definitions to check coverage against - * @param allMethods - The full list of known methods (from config.code.methods or MCP tool names) - */ -export function computeCoverage(tasks: TaskDefinition[], allMethods: string[]): MethodCoverage[] { - return allMethods.map((method) => { - const tasksCovering: string[] = []; - - for (const task of tasks) { - const covers = getExpectedActions(task).some((tool) => getExpectedActionName(tool) === method); - if (covers) { - tasksCovering.push(task.id); - } - } - - return { - method, - tasksCovering, - covered: tasksCovering.length > 0, - }; - }); -} - -// ── Coverage report ──────────────────────────────────────────────────────── - -/** - * Print a coverage report to console. - * - * Example output: - * SDK Method Coverage: - * ✔ MyClient.constructor (3 tasks) - * ✘ MyClient.submit (0 tasks) - * - * Coverage: 14/17 methods (82%) - */ -export function printCoverage(coverage: MethodCoverage[]): void { - console.log('SDK Method Coverage:'); - - const maxMethodLen = Math.max(...coverage.map((c) => c.method.length)); - - for (const entry of coverage) { - const icon = entry.covered ? '✔' : '✘'; - const padded = entry.method.padEnd(maxMethodLen); - const taskCount = entry.tasksCovering.length; - const taskLabel = taskCount === 1 ? 'task' : 'tasks'; - console.log(`${icon} ${padded} (${taskCount} ${taskLabel})`); - } - - const coveredCount = coverage.filter((c) => c.covered).length; - const totalCount = coverage.length; - const percent = totalCount === 0 ? 0 : Math.round((coveredCount / totalCount) * 100); - - console.log(''); - console.log(`Coverage: ${coveredCount}/${totalCount} methods (${percent}%)`); -} diff --git a/src/benchmark/evaluator.ts b/src/benchmark/evaluator.ts deleted file mode 100644 index c1bf64e..0000000 --- a/src/benchmark/evaluator.ts +++ /dev/null @@ -1,549 +0,0 @@ -import type { - ExpectedAction, - ExtractedCall, - ActionMatch, - TaskDefinition, - TaskResult, - ModelConfig, - TokenUsage, -} from './types.js'; -import { getExpectedActionName, getExpectedActions } from './types.js'; - -// ── Argument matching ────────────────────────────────────────────────────── - -/** - * Resolve an argument value from an extracted call's args, with positional fallback. - * - * CLI commands often take positional arguments (e.g. `fast account set-default myname`) - * which the extractor records as `_positional_0`, `_positional_1`, etc. When the expected - * args use the semantic name (e.g. `{name: "myname"}`), we look for the value in positionals - * as a fallback so that positional and named-flag invocations both match. - */ -function resolveArgValue(args: Record, key: string, expectedValue: unknown): unknown { - const direct = args[key]; - if (direct !== undefined) return direct; - // Positional fallback: if expected is a plain string, check _positional_N entries - if (typeof expectedValue === 'string') { - for (const [k, v] of Object.entries(args)) { - if (k.startsWith('_positional_') && v === expectedValue) { - return v; - } - } - } - return undefined; -} - -/** - * Compare an extracted argument value against an expected string value. - * - * Rules: - * 1. If got is a sentinel (,