diff --git a/README.md b/README.md index 8470249..16fa43c 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,15 @@ pnpm run dev ``` Once the server is running, you can utilize the MCP server within Visual Studio Code or other MCP client. +## Running evals + +The evals package loads an mcp client that then runs the index.ts file, so there is no need to rebuild between tests. You can load environment variables by prefixing the npx command. Full documentation can be found [here](https://www.mcpevals.io/docs). + +```bash +OPENAI_API_KEY=your-key npx mcp-eval src/evals/evals.ts src/index.ts +``` + + ## Upgrading ArgoCD Types To update the TypeScript type definitions based on the latest Argo CD API specification: diff --git a/package.json b/package.json index 1165530..a535d41 100644 --- a/package.json +++ b/package.json @@ -50,11 +50,13 @@ "@modelcontextprotocol/sdk": "^1.10.1", "dotenv": "^16.5.0", "express": "^5.1.0", + "mcp-evals": "^1.0.18", "pino": "^9.6.0", "yargs": "^17.7.2", "zod": "^3.24.3" }, "devDependencies": { + "@ai-sdk/openai": "^1.3.22", "@dtsgenerator/replace-namespace": "^1.7.0", "@eslint/js": "^9.25.0", "@types/express": "^5.0.1", @@ -70,4 +72,4 @@ "typescript": "^5.8.3", "typescript-eslint": "^8.30.1" } -} +} \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 16655a7..f23c222 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -17,6 +17,9 @@ importers: express: specifier: ^5.1.0 version: 5.1.0 + mcp-evals: + specifier: ^1.0.18 + version: 1.0.18(react@19.1.0)(zod@3.24.3) pino: specifier: ^9.6.0 version: 9.6.0 @@ -27,6 +30,9 @@ importers: specifier: ^3.24.3 version: 3.24.3 devDependencies: + '@ai-sdk/openai': + specifier: ^1.3.22 + version: 1.3.22(zod@3.24.3) '@dtsgenerator/replace-namespace': specifier: ^1.7.0 version: 1.7.0(dtsgenerator@3.19.2)(tslib@2.8.1) @@ -72,6 +78,59 @@ importers: packages: + '@actions/core@1.11.1': + resolution: {integrity: sha512-hXJCSrkwfA46Vd9Z3q4cpEpHB1rL5NG04+/rbqW9d3+CSvtB1tYe8UTpAlixa1vj0m/ULglfEK2UKxMGxCxv5A==} + + '@actions/exec@1.1.1': + resolution: {integrity: sha512-+sCcHHbVdk93a0XT19ECtO/gIXoxvdsgQLzb2fE2/5sIZmWQuluYyjPQtrtTHdU1YzTZ7bAPN4sITq2xi1679w==} + + '@actions/http-client@2.2.3': + resolution: {integrity: sha512-mx8hyJi/hjFvbPokCg4uRd4ZX78t+YyRPtnKWwIl+RzNaVuFpQHfmlGVfsKEJN8LwTCvL+DfVgAM04XaHkm6bA==} + + '@actions/io@1.1.3': + resolution: {integrity: sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q==} + + '@ai-sdk/openai@1.3.22': + resolution: {integrity: sha512-QwA+2EkG0QyjVR+7h6FE7iOu2ivNqAVMm9UJZkVxxTk5OIq5fFJDTEI/zICEMuHImTTXR2JjsL6EirJ28Jc4cw==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.0.0 + + '@ai-sdk/provider-utils@2.2.7': + resolution: {integrity: sha512-kM0xS3GWg3aMChh9zfeM+80vEZfXzR3JEUBdycZLtbRZ2TRT8xOj3WodGHPb06sUK5yD7pAXC/P7ctsi2fvUGQ==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.23.8 + + '@ai-sdk/provider-utils@2.2.8': + resolution: {integrity: sha512-fqhG+4sCVv8x7nFzYnFo19ryhAa3w096Kmc3hWxMQfW/TubPOmt3A6tYZhl4mUfQWWQMsuSkLrtjlWuXBVSGQA==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.23.8 + + '@ai-sdk/provider@1.1.3': + resolution: {integrity: sha512-qZMxYJ0qqX/RfnuIaab+zp8UAeJn/ygXXAffR5I4N0n1IrvA6qBsjc8hXLmBiMV2zoXlifkacF7sEFnYnjBcqg==} + engines: {node: '>=18'} + + '@ai-sdk/react@1.2.11': + resolution: {integrity: sha512-+kPqLkJ3TWP6czaJPV+vzAKSUcKQ1598BUrcLHt56sH99+LhmIIW3ylZp0OfC3O6TR3eO1Lt0Yzw4R0mK6g9Gw==} + engines: {node: '>=18'} + peerDependencies: + react: ^18 || ^19 || ^19.0.0-rc + zod: ^3.23.8 + peerDependenciesMeta: + zod: + optional: true + + '@ai-sdk/ui-utils@1.2.10': + resolution: {integrity: sha512-GUj+LBoAlRQF1dL/M49jtufGqtLOMApxTpCmVjoRpIPt/dFALVL9RfqfvxwztyIwbK+IxGzcYjSGRsrWrj+86g==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.23.8 + + '@anthropic-ai/sdk@0.8.1': + resolution: {integrity: sha512-59etePenCizVx1O8Qhi1T1ruE04ISfNzCnyhZNcsss1QljsLmYS83jttarMNEvGYcsUF7rwxw2lzcC3Zbxao7g==} + '@dtsgenerator/replace-namespace@1.7.0': resolution: {integrity: sha512-fOS8AP/Vz4u/+etAZDFPBpsHK3JASlVAId/cf+SH2Hcv5V96wsuvihXuKUomV08dPEUQhv98pgx/o1NiFUBzmA==} peerDependencies: @@ -266,6 +325,10 @@ packages: resolution: {integrity: sha512-ZAoA40rNMPwSm+AeHpCq8STiNAwzWLJuP8Xv4CHIc9wv/PSuExjMrmjfYNj682vW0OOiZ1HKxzvjQr9XZIisQA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@fastify/busboy@2.1.1': + resolution: {integrity: sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==} + engines: {node: '>=14'} + '@humanfs/core@0.19.1': resolution: {integrity: sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==} engines: {node: '>=18.18.0'} @@ -312,6 +375,10 @@ packages: resolution: {integrity: sha512-xNYdFdkJqEfIaTVP1gPKoEvluACHZsHZegIoICX8DM1o6Qf3G5u2BQJHmgd0n4YgRPqqK/u1ujQvrgAxxSJT9w==} engines: {node: '>=18'} + '@modelcontextprotocol/sdk@1.11.0': + resolution: {integrity: sha512-k/1pb70eD638anoi0e8wUGAlbMJXyvdV4p62Ko+EZ7eBe1xMx8Uhak1R5DgfoofsK5IBBnRwsYGTaLZl+6/+RQ==} + engines: {node: '>=18'} + '@nodelib/fs.scandir@2.1.5': resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} @@ -324,6 +391,10 @@ packages: resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} engines: {node: '>= 8'} + '@opentelemetry/api@1.9.0': + resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} + engines: {node: '>=8.0.0'} + '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -438,6 +509,9 @@ packages: '@types/connect@3.4.38': resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==} + '@types/diff-match-patch@1.0.36': + resolution: {integrity: sha512-xFdR6tkm0MWvBfO8xXCSsinYxHcqkQUlcHeSpMC2ukzOb6lwQAfDmW+Qt0AvlGd8HpsS28qKsB+oPeJn9I39jg==} + '@types/estree@1.0.7': resolution: {integrity: sha512-w28IoSUCJpidD/TGviZwwMJckNESJZXFu7NBZ5YJ4mEUnNraUn9Pm8HSZm/jDF1pDWYKspWE7oVphigUPRakIQ==} @@ -456,6 +530,12 @@ packages: '@types/mime@1.3.5': resolution: {integrity: sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==} + '@types/node-fetch@2.6.12': + resolution: {integrity: sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==} + + '@types/node@18.19.87': + resolution: {integrity: sha512-OIAAu6ypnVZHmsHCeJ+7CCSub38QNBS9uceMQeg7K5Ur0Jr+wG9wEOEvvMbhp09pxD5czIUy/jND7s7Tb6Nw7A==} + '@types/node@22.14.1': resolution: {integrity: sha512-u0HuPQwe/dHrItgHHpmw3N2fYCR6x4ivMNbPHRkBVP4CvN+kiRrKHWk3i8tXiO/joPwXLMYvF9TTF0eqgHIuOw==} @@ -524,6 +604,10 @@ packages: resolution: {integrity: sha512-aEhgas7aJ6vZnNFC7K4/vMGDGyOiqWcYZPpIWrTKuTAlsvDNKy2GFDqh9smL+iq069ZvR0YzEeq0B8NJlLzjFA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + abort-controller@3.0.0: + resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} + engines: {node: '>=6.5'} + accepts@2.0.0: resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} engines: {node: '>= 0.6'} @@ -542,6 +626,20 @@ packages: resolution: {integrity: sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==} engines: {node: '>= 14'} + agentkeepalive@4.6.0: + resolution: {integrity: sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==} + engines: {node: '>= 8.0.0'} + + ai@4.3.13: + resolution: {integrity: sha512-cC5HXItuOwGykSMacCPzNp6+NMTxeuTjOenztVgSJhdC9Z4OrzBxwkyeDAf4h1QP938ZFi7IBdq3u4lxVoVmvw==} + engines: {node: '>=18'} + peerDependencies: + react: ^18 || ^19 || ^19.0.0-rc + zod: ^3.23.8 + peerDependenciesMeta: + react: + optional: true + ajv@6.12.6: resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} @@ -567,6 +665,9 @@ packages: argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + atomic-sleep@1.0.0: resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==} engines: {node: '>=8.0.0'} @@ -574,6 +675,9 @@ packages: balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + base-64@0.1.0: + resolution: {integrity: sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==} + body-parser@2.2.0: resolution: {integrity: sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg==} engines: {node: '>=18'} @@ -618,6 +722,13 @@ packages: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} + chalk@5.4.1: + resolution: {integrity: sha512-zgVZuo2WcZgfUEmsn6eO3kINexW8RAE4maiQ8QNs8CtpPCSyMiYsULR3HQYkm3w8FIA3SberyMJMSldGsW+U3w==} + engines: {node: ^12.17.0 || ^14.13 || >=16.0.0} + + charenc@0.0.2: + resolution: {integrity: sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==} + chokidar@4.0.3: resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} engines: {node: '>= 14.16.0'} @@ -633,6 +744,10 @@ packages: color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + commander@12.1.0: resolution: {integrity: sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==} engines: {node: '>=18'} @@ -675,6 +790,9 @@ packages: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} + crypt@0.0.2: + resolution: {integrity: sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==} + debug@4.4.0: resolution: {integrity: sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==} engines: {node: '>=6.0'} @@ -687,10 +805,24 @@ packages: deep-is@0.1.4: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} + delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + depd@2.0.0: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} engines: {node: '>= 0.8'} + dequal@2.0.3: + resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} + engines: {node: '>=6'} + + diff-match-patch@1.0.5: + resolution: {integrity: sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw==} + + digest-fetch@1.3.0: + resolution: {integrity: sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==} + dotenv@16.5.0: resolution: {integrity: sha512-m/C+AwOAr9/W1UOIZUo232ejMNnJAJtYQjUbHoNTBNTJSvqzzDh7vnrei3o3r3m9blf6ZoDkvcw0VmozNRFJxg==} engines: {node: '>=12'} @@ -732,6 +864,10 @@ packages: resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} engines: {node: '>= 0.4'} + es-set-tostringtag@2.1.0: + resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==} + engines: {node: '>= 0.4'} + esbuild@0.25.2: resolution: {integrity: sha512-16854zccKPnC+toMywC+uKNeYSv+/eXkevRAfwRD/G9Cleq66m8XFIrigkbvauLLlCfDL45Q2cWegSg53gGBnQ==} engines: {node: '>=18'} @@ -814,6 +950,10 @@ packages: resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} engines: {node: '>= 0.6'} + event-target-shim@5.0.1: + resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} + engines: {node: '>=6'} + eventsource-parser@3.0.1: resolution: {integrity: sha512-VARTJ9CYeuQYb0pZEPbzi740OWFgpHe7AYJ2WFZVnUDUQp5Dk2yJUgF36YsZ81cOyxT0QxmXD2EQpapAouzWVA==} engines: {node: '>=18.0.0'} @@ -890,6 +1030,17 @@ packages: resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} engines: {node: '>=14'} + form-data-encoder@1.7.2: + resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==} + + form-data@4.0.2: + resolution: {integrity: sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w==} + engines: {node: '>= 6'} + + formdata-node@4.4.1: + resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==} + engines: {node: '>= 12.20'} + forwarded@0.2.0: resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} engines: {node: '>= 0.6'} @@ -952,6 +1103,10 @@ packages: resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} engines: {node: '>= 0.4'} + has-tostringtag@1.0.2: + resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==} + engines: {node: '>= 0.4'} + hasown@2.0.2: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} @@ -968,6 +1123,9 @@ packages: resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} engines: {node: '>= 14'} + humanize-ms@1.2.1: + resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==} + iconv-lite@0.6.3: resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==} engines: {node: '>=0.10.0'} @@ -991,6 +1149,9 @@ packages: resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} engines: {node: '>= 0.10'} + is-buffer@1.1.6: + resolution: {integrity: sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==} + is-extglob@2.1.1: resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} engines: {node: '>=0.10.0'} @@ -1030,9 +1191,17 @@ packages: json-schema-traverse@0.4.1: resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} + json-schema@0.4.0: + resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==} + json-stable-stringify-without-jsonify@1.0.1: resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} + jsondiffpatch@0.6.0: + resolution: {integrity: sha512-3QItJOXp2AP1uv7waBkao5nCvhEv+QmJAd38Ybq7wNI74Q+BBmnLn4EDKz6yI9xGAIQoUF87qHt+kc1IVxB4zQ==} + engines: {node: ^18.0.0 || >=20.0.0} + hasBin: true + keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} @@ -1068,6 +1237,15 @@ packages: resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} engines: {node: '>= 0.4'} + mcp-evals@1.0.18: + resolution: {integrity: sha512-khDcEG0XWshdCRirqLXogNoDLmzFA86QyuKoi5ioXsbeRZ3XQra8Zsg7vD+C0K5vwkFIoB1vTuPjHEHMhdLFtQ==} + hasBin: true + peerDependencies: + react: ^19.1.0 + + md5@2.3.0: + resolution: {integrity: sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==} + media-typer@1.1.0: resolution: {integrity: sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==} engines: {node: '>= 0.8'} @@ -1084,10 +1262,18 @@ packages: resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} engines: {node: '>=8.6'} + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + mime-db@1.54.0: resolution: {integrity: sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==} engines: {node: '>= 0.6'} + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + mime-types@3.0.1: resolution: {integrity: sha512-xRc4oEhT6eaBpU1XF7AjpOFD+xQmXNB5OVKwp4tqCuBpHLS/ZbBDrc07mYTDqVMg6PfxUjjNp85O6Cd2Z/5HWA==} engines: {node: '>= 0.6'} @@ -1109,6 +1295,11 @@ packages: mz@2.7.0: resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==} + nanoid@3.3.11: + resolution: {integrity: sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} @@ -1116,6 +1307,11 @@ packages: resolution: {integrity: sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==} engines: {node: '>= 0.6'} + node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + deprecated: Use your platform's native DOMException instead + node-fetch@2.7.0: resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} engines: {node: 4.x || >=6.0.0} @@ -1144,6 +1340,18 @@ packages: once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + openai@4.97.0: + resolution: {integrity: sha512-LRoiy0zvEf819ZUEJhgfV8PfsE8G5WpQi4AwA1uCV8SKvvtXQkoWUFkepD6plqyJQRghy2+AEPQ07FrJFKHZ9Q==} + hasBin: true + peerDependencies: + ws: ^8.18.0 + zod: ^3.23.8 + peerDependenciesMeta: + ws: + optional: true + zod: + optional: true + optionator@0.9.4: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} @@ -1272,6 +1480,10 @@ packages: resolution: {integrity: sha512-RmkhL8CAyCRPXCE28MMH0z2PNWQBNk2Q09ZdxM9IOOXwxwZbN+qbWaatPkdkWIKL2ZVDImrN/pK5HTRz2PcS4g==} engines: {node: '>= 0.8'} + react@19.1.0: + resolution: {integrity: sha512-FS+XFBNvn3GTAWq26joslQgWNoFu08F4kl0J4CgdNKADkdSGXQyTCnKteIAJy96Br6YbpEU1LSzV5dYtjMkMDg==} + engines: {node: '>=0.10.0'} + readdirp@4.1.2: resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} engines: {node: '>= 14.18.0'} @@ -1321,6 +1533,9 @@ packages: safer-buffer@2.1.2: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + secure-json-parse@2.7.0: + resolution: {integrity: sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==} + semver@7.7.1: resolution: {integrity: sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==} engines: {node: '>=10'} @@ -1409,6 +1624,11 @@ packages: resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} engines: {node: '>=8'} + swr@2.3.3: + resolution: {integrity: sha512-dshNvs3ExOqtZ6kJBaAsabhPdHyeY4P2cKwRCniDVifBMoG/SVI7tfLWqPXriVspf2Rg4tPzXJTnwaihIeFw2A==} + peerDependencies: + react: ^16.11.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + synckit@0.11.4: resolution: {integrity: sha512-Q/XQKRaJiLiFIBNN+mndW7S/RHxvwzuZS6ZwmRzUBqJBv/5QIKCEwkBC8GBf8EQJKYnaFs0wOZbKTXBPj8L9oQ==} engines: {node: ^14.18.0 || >=16.0.0} @@ -1423,6 +1643,10 @@ packages: thread-stream@3.1.0: resolution: {integrity: sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A==} + throttleit@2.1.0: + resolution: {integrity: sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==} + engines: {node: '>=18'} + tinyexec@0.3.2: resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} @@ -1484,6 +1708,10 @@ packages: engines: {node: '>=18.0.0'} hasBin: true + tunnel@0.0.6: + resolution: {integrity: sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==} + engines: {node: '>=0.6.11 <=0.7.0 || >=0.7.3'} + type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} @@ -1504,9 +1732,16 @@ packages: engines: {node: '>=14.17'} hasBin: true + undici-types@5.26.5: + resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + undici@5.29.0: + resolution: {integrity: sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==} + engines: {node: '>=14.0'} + unpipe@1.0.0: resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} engines: {node: '>= 0.8'} @@ -1514,10 +1749,23 @@ packages: uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + use-sync-external-store@1.5.0: + resolution: {integrity: sha512-Rb46I4cGGVBmjamjphe8L/UnvJD+uPPtTkNvX5mZgqdbavhI4EbgIWJiIHXJ8bc/i9EQGPRh4DwEURJ552Do0A==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + vary@1.1.2: resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} engines: {node: '>= 0.8'} + web-streams-polyfill@3.3.3: + resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} + engines: {node: '>= 8'} + + web-streams-polyfill@4.0.0-beta.3: + resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==} + engines: {node: '>= 14'} + webidl-conversions@3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} @@ -1576,6 +1824,77 @@ packages: snapshots: + '@actions/core@1.11.1': + dependencies: + '@actions/exec': 1.1.1 + '@actions/http-client': 2.2.3 + + '@actions/exec@1.1.1': + dependencies: + '@actions/io': 1.1.3 + + '@actions/http-client@2.2.3': + dependencies: + tunnel: 0.0.6 + undici: 5.29.0 + + '@actions/io@1.1.3': {} + + '@ai-sdk/openai@1.3.22(zod@3.24.3)': + dependencies: + '@ai-sdk/provider': 1.1.3 + '@ai-sdk/provider-utils': 2.2.8(zod@3.24.3) + zod: 3.24.3 + + '@ai-sdk/provider-utils@2.2.7(zod@3.24.3)': + dependencies: + '@ai-sdk/provider': 1.1.3 + nanoid: 3.3.11 + secure-json-parse: 2.7.0 + zod: 3.24.3 + + '@ai-sdk/provider-utils@2.2.8(zod@3.24.3)': + dependencies: + '@ai-sdk/provider': 1.1.3 + nanoid: 3.3.11 + secure-json-parse: 2.7.0 + zod: 3.24.3 + + '@ai-sdk/provider@1.1.3': + dependencies: + json-schema: 0.4.0 + + '@ai-sdk/react@1.2.11(react@19.1.0)(zod@3.24.3)': + dependencies: + '@ai-sdk/provider-utils': 2.2.7(zod@3.24.3) + '@ai-sdk/ui-utils': 1.2.10(zod@3.24.3) + react: 19.1.0 + swr: 2.3.3(react@19.1.0) + throttleit: 2.1.0 + optionalDependencies: + zod: 3.24.3 + + '@ai-sdk/ui-utils@1.2.10(zod@3.24.3)': + dependencies: + '@ai-sdk/provider': 1.1.3 + '@ai-sdk/provider-utils': 2.2.7(zod@3.24.3) + zod: 3.24.3 + zod-to-json-schema: 3.24.5(zod@3.24.3) + + '@anthropic-ai/sdk@0.8.1': + dependencies: + '@types/node': 18.19.87 + '@types/node-fetch': 2.6.12 + abort-controller: 3.0.0 + agentkeepalive: 4.6.0 + digest-fetch: 1.3.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0 + web-streams-polyfill: 3.3.3 + transitivePeerDependencies: + - encoding + '@dtsgenerator/replace-namespace@1.7.0(dtsgenerator@3.19.2)(tslib@2.8.1)': dependencies: dtsgenerator: 3.19.2 @@ -1700,6 +2019,8 @@ snapshots: '@eslint/core': 0.13.0 levn: 0.4.1 + '@fastify/busboy@2.1.1': {} + '@humanfs/core@0.19.1': {} '@humanfs/node@0.16.6': @@ -1754,6 +2075,21 @@ snapshots: transitivePeerDependencies: - supports-color + '@modelcontextprotocol/sdk@1.11.0': + dependencies: + content-type: 1.0.5 + cors: 2.8.5 + cross-spawn: 7.0.6 + eventsource: 3.0.6 + express: 5.1.0 + express-rate-limit: 7.5.0(express@5.1.0) + pkce-challenge: 5.0.0 + raw-body: 3.0.0 + zod: 3.24.3 + zod-to-json-schema: 3.24.5(zod@3.24.3) + transitivePeerDependencies: + - supports-color + '@nodelib/fs.scandir@2.1.5': dependencies: '@nodelib/fs.stat': 2.0.5 @@ -1766,6 +2102,8 @@ snapshots: '@nodelib/fs.scandir': 2.1.5 fastq: 1.19.1 + '@opentelemetry/api@1.9.0': {} + '@pkgjs/parseargs@0.11.0': optional: true @@ -1840,6 +2178,8 @@ snapshots: dependencies: '@types/node': 22.14.1 + '@types/diff-match-patch@1.0.36': {} + '@types/estree@1.0.7': {} '@types/express-serve-static-core@5.0.6': @@ -1861,6 +2201,15 @@ snapshots: '@types/mime@1.3.5': {} + '@types/node-fetch@2.6.12': + dependencies: + '@types/node': 22.14.1 + form-data: 4.0.2 + + '@types/node@18.19.87': + dependencies: + undici-types: 5.26.5 + '@types/node@22.14.1': dependencies: undici-types: 6.21.0 @@ -1963,6 +2312,10 @@ snapshots: '@typescript-eslint/types': 8.30.1 eslint-visitor-keys: 4.2.0 + abort-controller@3.0.0: + dependencies: + event-target-shim: 5.0.1 + accepts@2.0.0: dependencies: mime-types: 3.0.1 @@ -1976,6 +2329,22 @@ snapshots: agent-base@7.1.3: {} + agentkeepalive@4.6.0: + dependencies: + humanize-ms: 1.2.1 + + ai@4.3.13(react@19.1.0)(zod@3.24.3): + dependencies: + '@ai-sdk/provider': 1.1.3 + '@ai-sdk/provider-utils': 2.2.7(zod@3.24.3) + '@ai-sdk/react': 1.2.11(react@19.1.0)(zod@3.24.3) + '@ai-sdk/ui-utils': 1.2.10(zod@3.24.3) + '@opentelemetry/api': 1.9.0 + jsondiffpatch: 0.6.0 + zod: 3.24.3 + optionalDependencies: + react: 19.1.0 + ajv@6.12.6: dependencies: fast-deep-equal: 3.1.3 @@ -1997,10 +2366,14 @@ snapshots: argparse@2.0.1: {} + asynckit@0.4.0: {} + atomic-sleep@1.0.0: {} balanced-match@1.0.2: {} + base-64@0.1.0: {} + body-parser@2.2.0: dependencies: bytes: 3.1.2 @@ -2054,6 +2427,10 @@ snapshots: ansi-styles: 4.3.0 supports-color: 7.2.0 + chalk@5.4.1: {} + + charenc@0.0.2: {} + chokidar@4.0.3: dependencies: readdirp: 4.1.2 @@ -2070,6 +2447,10 @@ snapshots: color-name@1.1.4: {} + combined-stream@1.0.8: + dependencies: + delayed-stream: 1.0.0 + commander@12.1.0: {} commander@4.1.1: {} @@ -2105,14 +2486,27 @@ snapshots: shebang-command: 2.0.0 which: 2.0.2 + crypt@0.0.2: {} + debug@4.4.0: dependencies: ms: 2.1.3 deep-is@0.1.4: {} + delayed-stream@1.0.0: {} + depd@2.0.0: {} + dequal@2.0.3: {} + + diff-match-patch@1.0.5: {} + + digest-fetch@1.3.0: + dependencies: + base-64: 0.1.0 + md5: 2.3.0 + dotenv@16.5.0: {} dtsgenerator@3.19.2: @@ -2154,6 +2548,13 @@ snapshots: dependencies: es-errors: 1.3.0 + es-set-tostringtag@2.1.0: + dependencies: + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + has-tostringtag: 1.0.2 + hasown: 2.0.2 + esbuild@0.25.2: optionalDependencies: '@esbuild/aix-ppc64': 0.25.2 @@ -2270,6 +2671,8 @@ snapshots: etag@1.8.1: {} + event-target-shim@5.0.1: {} + eventsource-parser@3.0.1: {} eventsource@3.0.6: @@ -2374,6 +2777,20 @@ snapshots: cross-spawn: 7.0.6 signal-exit: 4.1.0 + form-data-encoder@1.7.2: {} + + form-data@4.0.2: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 + mime-types: 2.1.35 + + formdata-node@4.4.1: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 4.0.0-beta.3 + forwarded@0.2.0: {} fresh@2.0.0: {} @@ -2434,6 +2851,10 @@ snapshots: has-symbols@1.1.0: {} + has-tostringtag@1.0.2: + dependencies: + has-symbols: 1.1.0 + hasown@2.0.2: dependencies: function-bind: 1.1.2 @@ -2460,6 +2881,10 @@ snapshots: transitivePeerDependencies: - supports-color + humanize-ms@1.2.1: + dependencies: + ms: 2.1.3 + iconv-lite@0.6.3: dependencies: safer-buffer: 2.1.2 @@ -2477,6 +2902,8 @@ snapshots: ipaddr.js@1.9.1: {} + is-buffer@1.1.6: {} + is-extglob@2.1.1: {} is-fullwidth-code-point@3.0.0: {} @@ -2507,8 +2934,16 @@ snapshots: json-schema-traverse@0.4.1: {} + json-schema@0.4.0: {} + json-stable-stringify-without-jsonify@1.0.1: {} + jsondiffpatch@0.6.0: + dependencies: + '@types/diff-match-patch': 1.0.36 + chalk: 5.4.1 + diff-match-patch: 1.0.5 + keyv@4.5.4: dependencies: json-buffer: 3.0.1 @@ -2536,6 +2971,30 @@ snapshots: math-intrinsics@1.1.0: {} + mcp-evals@1.0.18(react@19.1.0)(zod@3.24.3): + dependencies: + '@actions/core': 1.11.1 + '@ai-sdk/openai': 1.3.22(zod@3.24.3) + '@anthropic-ai/sdk': 0.8.1 + '@modelcontextprotocol/sdk': 1.11.0 + ai: 4.3.13(react@19.1.0)(zod@3.24.3) + chalk: 4.1.2 + dotenv: 16.5.0 + openai: 4.97.0(zod@3.24.3) + react: 19.1.0 + tsx: 4.19.3 + transitivePeerDependencies: + - encoding + - supports-color + - ws + - zod + + md5@2.3.0: + dependencies: + charenc: 0.0.2 + crypt: 0.0.2 + is-buffer: 1.1.6 + media-typer@1.1.0: {} merge-descriptors@2.0.0: {} @@ -2547,8 +3006,14 @@ snapshots: braces: 3.0.3 picomatch: 2.3.1 + mime-db@1.52.0: {} + mime-db@1.54.0: {} + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + mime-types@3.0.1: dependencies: mime-db: 1.54.0 @@ -2571,10 +3036,14 @@ snapshots: object-assign: 4.1.1 thenify-all: 1.6.0 + nanoid@3.3.11: {} + natural-compare@1.4.0: {} negotiator@1.0.0: {} + node-domexception@1.0.0: {} + node-fetch@2.7.0: dependencies: whatwg-url: 5.0.0 @@ -2593,6 +3062,20 @@ snapshots: dependencies: wrappy: 1.0.2 + openai@4.97.0(zod@3.24.3): + dependencies: + '@types/node': 18.19.87 + '@types/node-fetch': 2.6.12 + abort-controller: 3.0.0 + agentkeepalive: 4.6.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0 + optionalDependencies: + zod: 3.24.3 + transitivePeerDependencies: + - encoding + optionator@0.9.4: dependencies: deep-is: 0.1.4 @@ -2699,6 +3182,8 @@ snapshots: iconv-lite: 0.6.3 unpipe: 1.0.0 + react@19.1.0: {} + readdirp@4.1.2: {} real-require@0.2.0: {} @@ -2759,6 +3244,8 @@ snapshots: safer-buffer@2.1.2: {} + secure-json-parse@2.7.0: {} + semver@7.7.1: {} send@1.2.0: @@ -2872,6 +3359,12 @@ snapshots: dependencies: has-flag: 4.0.0 + swr@2.3.3(react@19.1.0): + dependencies: + dequal: 2.0.3 + react: 19.1.0 + use-sync-external-store: 1.5.0(react@19.1.0) + synckit@0.11.4: dependencies: '@pkgr/core': 0.2.4 @@ -2889,6 +3382,8 @@ snapshots: dependencies: real-require: 0.2.0 + throttleit@2.1.0: {} + tinyexec@0.3.2: {} tinyglobby@0.2.12: @@ -2951,6 +3446,8 @@ snapshots: optionalDependencies: fsevents: 2.3.3 + tunnel@0.0.6: {} + type-check@0.4.0: dependencies: prelude-ls: 1.2.1 @@ -2973,16 +3470,30 @@ snapshots: typescript@5.8.3: {} + undici-types@5.26.5: {} + undici-types@6.21.0: {} + undici@5.29.0: + dependencies: + '@fastify/busboy': 2.1.1 + unpipe@1.0.0: {} uri-js@4.4.1: dependencies: punycode: 2.3.1 + use-sync-external-store@1.5.0(react@19.1.0): + dependencies: + react: 19.1.0 + vary@1.1.2: {} + web-streams-polyfill@3.3.3: {} + + web-streams-polyfill@4.0.0-beta.3: {} + webidl-conversions@3.0.1: {} webidl-conversions@4.0.2: {} diff --git a/src/evals/evals.ts b/src/evals/evals.ts new file mode 100644 index 0000000..4b12b4f --- /dev/null +++ b/src/evals/evals.ts @@ -0,0 +1,174 @@ +import { openai } from '@ai-sdk/openai'; +import { grade, EvalFunction } from 'mcp-evals'; + +const listApplicationsEval: EvalFunction = { + name: 'List Applications Evaluation', + description: "Evaluates the model's ability to list ArgoCD applications", + run: async () => { + const result = await grade( + openai('gpt-4'), + 'Show me all the ArgoCD applications in the cluster.' + ); + return JSON.parse(result); + } +}; + +const getApplicationEval: EvalFunction = { + name: 'Get Application Evaluation', + description: "Evaluates the model's ability to retrieve a specific ArgoCD application", + run: async () => { + const result = await grade( + openai('gpt-4'), + "Get details for the 'my-app' application in ArgoCD." + ); + return JSON.parse(result); + } +}; + +const createApplicationEval: EvalFunction = { + name: 'Create Application Evaluation', + description: "Evaluates the model's ability to create a new ArgoCD application", + run: async () => { + const result = await grade( + openai('gpt-4'), + "Create a new ArgoCD application named 'test-app' pointing to 'https://github.com/test/repo' in the 'default' namespace." + ); + return JSON.parse(result); + } +}; + +const updateApplicationEval: EvalFunction = { + name: 'Update Application Evaluation', + description: "Evaluates the model's ability to update an existing ArgoCD application", + run: async () => { + const result = await grade( + openai('gpt-4'), + "Update the 'test-app' ArgoCD application to point to the 'main' branch instead of 'master'." + ); + return JSON.parse(result); + } +}; + +const deleteApplicationEval: EvalFunction = { + name: 'Delete Application Evaluation', + description: "Evaluates the model's ability to delete an ArgoCD application", + run: async () => { + const result = await grade(openai('gpt-4'), "Delete the 'test-app' application from ArgoCD."); + return JSON.parse(result); + } +}; + +const syncApplicationEval: EvalFunction = { + name: 'Sync Application Evaluation', + description: "Evaluates the model's ability to sync an ArgoCD application", + run: async () => { + const result = await grade(openai('gpt-4'), "Sync the 'my-app' application in ArgoCD."); + return JSON.parse(result); + } +}; + +const getApplicationResourceTreeEval: EvalFunction = { + name: 'Get Application Resource Tree Evaluation', + description: + "Evaluates the model's ability to retrieve the resource tree for an ArgoCD application", + run: async () => { + const result = await grade( + openai('gpt-4'), + "Show me the resource tree for the 'my-app' application in ArgoCD." + ); + return JSON.parse(result); + } +}; + +const getApplicationManagedResourcesEval: EvalFunction = { + name: 'Get Application Managed Resources Evaluation', + description: + "Evaluates the model's ability to retrieve managed resources for an ArgoCD application", + run: async () => { + const result = await grade( + openai('gpt-4'), + "What resources are managed by the 'my-app' application in ArgoCD?" + ); + return JSON.parse(result); + } +}; + +const getApplicationWorkloadLogsEval: EvalFunction = { + name: 'Get Application Workload Logs Evaluation', + description: "Evaluates the model's ability to retrieve logs for an ArgoCD application workload", + run: async () => { + const result = await grade( + openai('gpt-4'), + "Show me the logs for the 'web' deployment in the 'my-app' application." + ); + return JSON.parse(result); + } +}; + +const getApplicationEventsEval: EvalFunction = { + name: 'Get Application Events Evaluation', + description: "Evaluates the model's ability to retrieve events for an ArgoCD application", + run: async () => { + const result = await grade( + openai('gpt-4'), + "What events have occurred for the 'my-app' application in ArgoCD?" + ); + return JSON.parse(result); + } +}; + +const getResourceEventsEval: EvalFunction = { + name: 'Get Resource Events Evaluation', + description: + "Evaluates the model's ability to retrieve events for a resource within an ArgoCD application", + run: async () => { + const result = await grade( + openai('gpt-4'), + "Show me the events for the 'web' deployment in the 'my-app' application." + ); + return JSON.parse(result); + } +}; + +const getResourceActionsEval: EvalFunction = { + name: 'Get Resource Actions Evaluation', + description: "Evaluates the model's ability to retrieve available actions for a resource", + run: async () => { + const result = await grade( + openai('gpt-4'), + "What actions can I perform on the 'database' StatefulSet in the 'my-app' application?" + ); + return JSON.parse(result); + } +}; + +const runResourceActionEval: EvalFunction = { + name: 'Run Resource Action Evaluation', + description: "Evaluates the model's ability to run an action on a resource", + run: async () => { + const result = await grade( + openai('gpt-4'), + "Restart the 'web' deployment in the 'my-app' application." + ); + return JSON.parse(result); + } +}; + +export const evalConfig = { + model: openai('gpt-4'), + evals: [ + listApplicationsEval, + getApplicationEval, + createApplicationEval, + updateApplicationEval, + deleteApplicationEval, + syncApplicationEval, + getApplicationResourceTreeEval, + getApplicationManagedResourcesEval, + getApplicationWorkloadLogsEval, + getApplicationEventsEval, + getResourceEventsEval, + getResourceActionsEval, + runResourceActionEval + ] +};