diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 44a6b2986e022..a41840404d7d2 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -156,6 +156,7 @@ impl AIRequestConfig { let is_azure = matches!(provider, AIProvider::OpenAI) && base_url != OPENAI_BASE_URL || matches!(provider, AIProvider::AzureOpenAI); let is_anthropic = matches!(provider, AIProvider::Anthropic); + let is_anthropic_sdk = headers.get("X-Anthropic-SDK").is_some(); let url = if is_azure && method != Method::GET { if base_url.ends_with("/deployments") { @@ -167,6 +168,9 @@ impl AIRequestConfig { } else { format!("{}/{}", base_url, path) } + } else if is_anthropic_sdk { + let truncated_base_url = base_url.trim_end_matches("/v1"); + format!("{}/{}", truncated_base_url, path) } else { format!("{}/{}", base_url, path) }; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 6401b0b9250f3..f7bff3ed3282d 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -10,6 +10,7 @@ "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { + "@anthropic-ai/sdk": "^0.60.0", "@aws-crypto/sha256-js": "^4.0.0", "@codingame/monaco-vscode-configuration-service-override": "~20.2.1", "@codingame/monaco-vscode-editor-api": "~20.2.1", @@ -56,7 +57,7 @@ "monaco-languageclient": "9.11.0", "monaco-vim": "^0.4.1", "ol": "^7.4.0", - "openai": "^4.87.1", + "openai": "^5.16.0", "openapi-types": "^12.1.3", "p-limit": "^6.1.0", "panzoom": "^9.4.3", @@ -186,6 +187,15 @@ "node": ">=6.0.0" } }, + "node_modules/@anthropic-ai/sdk": { + "version": "0.60.0", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.60.0.tgz", + "integrity": "sha512-9zu/TXaUy8BZhXedDtt1wT3H4LOlpKDO1/ftiFpeR3N1PCr3KJFKkxxlQWWt1NNp08xSwUNJ3JNY8yhl8av6eQ==", + "license": "MIT", + "bin": { + "anthropic-ai-sdk": "bin/cli" + } + }, "node_modules/@apidevtools/json-schema-ref-parser": { "version": "11.6.1", "resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-11.6.1.tgz", @@ -3640,20 +3650,12 @@ "version": "20.19.10", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.10.tgz", "integrity": "sha512-iAFpG6DokED3roLSP0K+ybeDdIX6Bc0Vd3mLW5uDqThPWtNos3E+EqOM11mPQHKzfWHqEBuLjIlsBQQ8CsISmQ==", + "dev": true, "license": "MIT", "dependencies": { "undici-types": "~6.21.0" } }, - "node_modules/@types/node-fetch": { - "version": "2.6.12", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.12.tgz", - "integrity": "sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==", - "dependencies": { - "@types/node": "*", - "form-data": "^4.0.0" - } - }, "node_modules/@types/normalize-package-data": { "version": "2.4.4", "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.4.tgz", @@ -3953,17 +3955,6 @@ "svelte": "^3.57.0 || ^4.0.0 || ^5.0.0" } }, - "node_modules/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", - "dependencies": { - "event-target-shim": "^5.0.0" - }, - "engines": { - "node": ">=6.5" - } - }, "node_modules/abstract-leveldown": { "version": "6.2.3", "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.2.3.tgz", @@ -4028,17 +4019,6 @@ "ag-grid-community": "31.3.4" } }, - "node_modules/agentkeepalive": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", - "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", - "dependencies": { - "humanize-ms": "^1.2.1" - }, - "engines": { - "node": ">= 8.0.0" - } - }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -4210,7 +4190,8 @@ "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "dev": true }, "node_modules/autoprefixer": { "version": "10.4.21", @@ -4802,6 +4783,7 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, "dependencies": { "delayed-stream": "~1.0.0" }, @@ -5426,6 +5408,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "dev": true, "engines": { "node": ">=0.4.0" } @@ -5742,6 +5725,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "dev": true, "dependencies": { "es-errors": "^1.3.0", "get-intrinsic": "^1.2.6", @@ -6171,14 +6155,6 @@ "node": ">=0.10.0" } }, - "node_modules/event-target-shim": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", - "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", - "engines": { - "node": ">=6" - } - }, "node_modules/eventemitter3": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-2.0.3.tgz", @@ -6405,6 +6381,7 @@ "version": "4.0.2", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.2.tgz", "integrity": "sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w==", + "dev": true, "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", @@ -6415,23 +6392,6 @@ "node": ">= 6" } }, - "node_modules/form-data-encoder": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", - "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==" - }, - "node_modules/formdata-node": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", - "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", - "dependencies": { - "node-domexception": "1.0.0", - "web-streams-polyfill": "4.0.0-beta.3" - }, - "engines": { - "node": ">= 12.20" - } - }, "node_modules/fraction.js": { "version": "4.3.7", "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.3.7.tgz", @@ -7129,14 +7089,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/humanize-ms": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", - "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", - "dependencies": { - "ms": "^2.0.0" - } - }, "node_modules/idb": { "version": "8.0.3", "resolved": "https://registry.npmjs.org/idb/-/idb-8.0.3.tgz", @@ -8784,6 +8736,7 @@ "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "dev": true, "engines": { "node": ">= 0.6" } @@ -8792,6 +8745,7 @@ "version": "2.1.35", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dev": true, "dependencies": { "mime-db": "1.52.0" }, @@ -9141,44 +9095,6 @@ "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==", "optional": true }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "deprecated": "Use your platform's native DOMException instead", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "engines": { - "node": ">=10.5.0" - } - }, - "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, "node_modules/node-fetch-native": { "version": "1.6.6", "resolved": "https://registry.npmjs.org/node-fetch-native/-/node-fetch-native-1.6.6.tgz", @@ -9363,18 +9279,10 @@ } }, "node_modules/openai": { - "version": "4.100.0", - "resolved": "https://registry.npmjs.org/openai/-/openai-4.100.0.tgz", - "integrity": "sha512-9soq/wukv3utxcuD7TWFqKdKp0INWdeyhUCvxwrne5KwnxaCp4eHL4GdT/tMFhYolxgNhxFzg5GFwM331Z5CZg==", - "dependencies": { - "@types/node": "^18.11.18", - "@types/node-fetch": "^2.6.4", - "abort-controller": "^3.0.0", - "agentkeepalive": "^4.2.1", - "form-data-encoder": "1.7.2", - "formdata-node": "^4.3.2", - "node-fetch": "^2.6.7" - }, + "version": "5.16.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-5.16.0.tgz", + "integrity": "sha512-hoEH8ZNvg1HXjU9mp88L/ZH8O082Z8r6FHCXGiWAzVRrEv443aI57qhch4snu07yQydj+AUAWLenAiBXhu89Tw==", + "license": "Apache-2.0", "bin": { "openai": "bin/cli" }, @@ -9391,19 +9299,6 @@ } } }, - "node_modules/openai/node_modules/@types/node": { - "version": "18.19.101", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.101.tgz", - "integrity": "sha512-Ykg7fcE3+cOQlLUv2Ds3zil6DVjriGQaSN/kEpl5HQ3DIGM6W0F2n9+GkWV4bRt7KjLymgzNdTnSKCbFUUJ7Kw==", - "dependencies": { - "undici-types": "~5.26.4" - } - }, - "node_modules/openai/node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" - }, "node_modules/openapi-types": { "version": "12.1.3", "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-12.1.3.tgz", @@ -12381,11 +12276,6 @@ "node": ">=6" } }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, "node_modules/trim-lines": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz", @@ -12521,6 +12411,7 @@ "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, "license": "MIT" }, "node_modules/unified": { @@ -12956,33 +12847,11 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/web-streams-polyfill": { - "version": "4.0.0-beta.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", - "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", - "engines": { - "node": ">= 14" - } - }, "node_modules/web-worker": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/web-worker/-/web-worker-1.5.0.tgz", "integrity": "sha512-RiMReJrTAiA+mBjGONMnjVDP2u3p9R1vkcGz6gDIrOMT3oGuYwX2WRMYI9ipkphSuE5XKEhydbhNEJh4NY9mlw==" }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, "node_modules/wheel": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/wheel/-/wheel-1.0.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index 1f0841eea9f5f..f3b4617c443b6 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -77,6 +77,7 @@ }, "type": "module", "dependencies": { + "@anthropic-ai/sdk": "^0.60.0", "@aws-crypto/sha256-js": "^4.0.0", "@codingame/monaco-vscode-configuration-service-override": "~20.2.1", "@codingame/monaco-vscode-editor-api": "~20.2.1", @@ -123,7 +124,7 @@ "monaco-languageclient": "9.11.0", "monaco-vim": "^0.4.1", "ol": "^7.4.0", - "openai": "^4.87.1", + "openai": "^5.16.0", "openapi-types": "^12.1.3", "p-limit": "^6.1.0", "panzoom": "^9.4.3", @@ -544,4 +545,4 @@ "@rollup/rollup-linux-x64-gnu": "^4.35.0", "fsevents": "^2.3.3" } -} \ No newline at end of file +} diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index b1f02f00b4fe4..dcd91e1d1f642 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -11,16 +11,13 @@ import HistoryManager from './HistoryManager.svelte' import { extractCodeFromMarkdown, getLatestAssistantMessage, - processToolCall, type DisplayMessage, type Tool, type ToolCallbacks, type ToolDisplayMessage } from './shared' import type { - ChatCompletionChunk, ChatCompletionMessageParam, - ChatCompletionMessageToolCall, ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources/chat/completions.mjs' @@ -34,7 +31,7 @@ import { loadApiTools } from './api/apiTools' import { prepareScriptUserMessage } from './script/core' import { prepareNavigatorUserMessage } from './navigator/core' import { sendUserToast } from '$lib/toast' -import { getCompletion, getModelContextWindow } from '../lib' +import { getCompletion, getModelContextWindow, parseOpenAICompletion } from '../lib' import { dfs } from '$lib/components/flows/previousResults' import { getStringError } from './utils' import type { FlowModuleState, FlowState } from '$lib/components/flows/flowState' @@ -47,6 +44,7 @@ import type { ContextElement } from './context' import type { Selection } from 'monaco-editor' import type AIChatInput from './AIChatInput.svelte' import { prepareApiSystemMessage, prepareApiUserMessage } from './api/core' +import { getAnthropicCompletion, parseAnthropicCompletion } from './anthropic' // If the estimated token usage is greater than the model context window - the threshold, we delete the oldest message const MAX_TOKENS_THRESHOLD_PERCENTAGE = 0.05 @@ -380,10 +378,8 @@ class AIChatManager { } systemMessage?: ChatCompletionSystemMessageParam }) => { - let addedMessages: ChatCompletionMessageParam[] = [] try { - let completion: any = null - + let addedMessages: ChatCompletionMessageParam[] = [] while (true) { const systemMessage = systemMessageOverride ?? this.systemMessage const helpers = this.helpers @@ -413,99 +409,28 @@ class AIChatManager { } this.pendingPrompt = '' } - completion = await getCompletion( + + const model = getCurrentModel() + const completionFn = model.provider === 'anthropic' ? getAnthropicCompletion : getCompletion + const parseFn = + model.provider === 'anthropic' ? parseAnthropicCompletion : parseOpenAICompletion + + const completion = await completionFn( [systemMessage, ...messages, ...(pendingUserMessage ? [pendingUserMessage] : [])], abortController, tools.map((t) => t.def) ) if (completion) { - const finalToolCalls: Record = {} - - let answer = '' - for await (const chunk of completion) { - if (!('choices' in chunk && chunk.choices.length > 0 && 'delta' in chunk.choices[0])) { - continue - } - const c = chunk as ChatCompletionChunk - const delta = c.choices[0].delta.content - if (delta) { - answer += delta - callbacks.onNewToken(delta) - } - const toolCalls = c.choices[0].delta.tool_calls || [] - if (toolCalls.length > 0 && answer) { - // if tool calls are present but we have some textual content already, we need to display it to the user first - callbacks.onMessageEnd() - answer = '' - } - for (const toolCall of toolCalls) { - const { index } = toolCall - let finalToolCall = finalToolCalls[index] - if (!finalToolCall) { - finalToolCalls[index] = toolCall - } else { - if (toolCall.function?.arguments) { - if (!finalToolCall.function) { - finalToolCall.function = toolCall.function - } else { - finalToolCall.function.arguments = - (finalToolCall.function.arguments ?? '') + toolCall.function.arguments - } - } - } - finalToolCall = finalToolCalls[index] - if (finalToolCall?.function) { - const { - function: { name: funcName }, - id: toolCallId - } = finalToolCall - if (funcName && toolCallId) { - const tool = tools.find((t) => t.def.function.name === funcName) - if (tool && tool.preAction) { - tool.preAction({ toolCallbacks: callbacks, toolId: toolCallId }) - } - } - } - } - } - - if (answer) { - const toAdd = { role: 'assistant' as const, content: answer } - addedMessages.push(toAdd) - messages.push(toAdd) - } - - callbacks.onMessageEnd() - - const toolCalls = Object.values(finalToolCalls).filter( - (toolCall) => toolCall.id !== undefined && toolCall.function?.arguments !== undefined - ) as ChatCompletionMessageToolCall[] - - if (toolCalls.length > 0) { - const toAdd = { - role: 'assistant' as const, - tool_calls: toolCalls.map((t) => ({ - ...t, - function: { - ...t.function, - arguments: t.function.arguments || '{}' - } - })) - } - messages.push(toAdd) - addedMessages.push(toAdd) - for (const toolCall of toolCalls) { - const messageToAdd = await processToolCall({ - tools, - toolCall, - helpers, - toolCallbacks: callbacks - }) - messages.push(messageToAdd) - addedMessages.push(messageToAdd) - } - } else { + const continueCompletion = await parseFn( + completion as any, + callbacks, + messages, + addedMessages, + tools, + helpers + ) + if (!continueCompletion) { break } } diff --git a/frontend/src/lib/components/copilot/chat/anthropic.ts b/frontend/src/lib/components/copilot/chat/anthropic.ts new file mode 100644 index 0000000000000..645cc40644321 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/anthropic.ts @@ -0,0 +1,264 @@ +import { OpenAI } from 'openai' +import type { + ChatCompletionMessageParam, + ChatCompletionMessageFunctionToolCall +} from 'openai/resources/index.mjs' +import type { + MessageParam, + TextBlockParam, + ToolUnion, + ToolUseBlockParam, + Tool as AnthropicTool, + Message +} from '@anthropic-ai/sdk/resources' +import type { MessageStream } from '@anthropic-ai/sdk/lib/MessageStream' +import { getProviderAndCompletionConfig, workspaceAIClients } from '../lib' +import { processToolCall, type Tool, type ToolCallbacks } from './shared' + +export async function getAnthropicCompletion( + messages: ChatCompletionMessageParam[], + abortController: AbortController, + tools?: OpenAI.Chat.Completions.ChatCompletionFunctionTool[] +): Promise { + const { provider, config } = getProviderAndCompletionConfig({ messages, stream: true }) + const { system, messages: anthropicMessages } = convertOpenAIToAnthropicMessages(messages) + const anthropicTools = convertOpenAIToolsToAnthropic(tools) + + const anthropicClient = workspaceAIClients.getAnthropicClient() + + const anthropicParams = { + model: config.model, + max_tokens: config.max_tokens as number, + messages: anthropicMessages, + ...(system && { system }), + ...(anthropicTools && { tools: anthropicTools }), + ...(typeof config.temperature === 'number' && { temperature: config.temperature }) + } + + const stream = anthropicClient.messages.stream(anthropicParams, { + signal: abortController.signal, + headers: { + 'X-Provider': provider, + 'anthropic-version': '2023-06-01', + 'X-Anthropic-SDK': 'true' + } + }) + + return stream +} + +export async function parseAnthropicCompletion( + completion: MessageStream, + callbacks: ToolCallbacks & { + onNewToken: (token: string) => void + onMessageEnd: () => void + }, + messages: ChatCompletionMessageParam[], + addedMessages: ChatCompletionMessageParam[], + tools: Tool[], + helpers: any +): Promise { + let toolCallsToProcess: ChatCompletionMessageFunctionToolCall[] = [] + let error = null + + // Handle text streaming + completion.on('text', (textDelta: string, _textSnapshot: string) => { + callbacks.onNewToken(textDelta) + }) + + completion.on('message', (message: Message) => { + for (const block of message.content) { + if (block.type === 'text') { + const text = block.text + const assistantMessage = { role: 'assistant' as const, content: text } + messages.push(assistantMessage) + addedMessages.push(assistantMessage) + callbacks.onMessageEnd() + } else if (block.type === 'tool_use') { + // Convert Anthropic tool calls to OpenAI format for compatibility + toolCallsToProcess.push({ + id: block.id, + type: 'function' as const, + function: { + name: block.name, + arguments: JSON.stringify(block.input) + } + }) + // Preprocess tool if it has a preAction + const tool = tools.find((t) => t.def.function.name === block.name) + if (tool && tool.preAction) { + tool.preAction({ toolCallbacks: callbacks, toolId: block.id }) + } + } + } + }) + + // Handle errors + completion.on('error', (e: any) => { + console.error('Anthropic stream error:', e) + error = e + }) + + // Wait for completion + await completion.done() + + callbacks.onMessageEnd() + + if (error) { + throw error + } + + // Process tool calls if any + if (toolCallsToProcess.length > 0) { + const assistantWithTools = { + role: 'assistant' as const, + tool_calls: toolCallsToProcess + } + messages.push(assistantWithTools) + addedMessages.push(assistantWithTools) + + // Process each tool call + for (const toolCall of toolCallsToProcess) { + const messageToAdd = await processToolCall({ + tools, + toolCall, + helpers, + toolCallbacks: callbacks + }) + messages.push(messageToAdd) + addedMessages.push(messageToAdd) + } + return true // Continue the conversation loop + } + + return false // End the conversation +} + +export function convertOpenAIToAnthropicMessages(messages: ChatCompletionMessageParam[]): { + system: TextBlockParam[] | undefined + messages: MessageParam[] +} { + let system: TextBlockParam[] | undefined + const anthropicMessages: MessageParam[] = [] + + for (const message of messages) { + if (message.role === 'system') { + const systemText = + typeof message.content === 'string' ? message.content : JSON.stringify(message.content) + // Convert system to array format with cache_control for caching + system = [ + { + type: 'text', + text: systemText, + cache_control: { type: 'ephemeral' } + } + ] + continue + } + + if (message.role === 'user') { + anthropicMessages.push({ + role: 'user', + content: + typeof message.content === 'string' ? message.content : JSON.stringify(message.content) + }) + } else if (message.role === 'assistant') { + const content: (TextBlockParam | ToolUseBlockParam)[] = [] + + if (message.content) { + content.push({ + type: 'text', + text: + typeof message.content === 'string' ? message.content : JSON.stringify(message.content) + }) + } + + if (message.tool_calls) { + for (const toolCall of message.tool_calls) { + if (toolCall.type !== 'function') continue + let input = {} + try { + input = JSON.parse(toolCall.function.arguments || '{}') + } catch (e) { + console.error('Failed to parse tool call arguments', e) + } + content.push({ + type: 'tool_use', + id: toolCall.id, + name: toolCall.function.name, + input + }) + } + } + + if (content.length > 0) { + anthropicMessages.push({ + role: 'assistant', + content: content.length === 1 && content[0].type === 'text' ? content[0].text : content + }) + } + } else if (message.role === 'tool') { + // Tool results must be in user messages in Anthropic format + anthropicMessages.push({ + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: message.tool_call_id, + content: + typeof message.content === 'string' + ? message.content + : JSON.stringify(message.content) + } + ] + }) + } + } + + // Add cache_control to the last message content blocks + if (anthropicMessages.length > 0) { + const lastMessage = anthropicMessages[anthropicMessages.length - 1] + if (Array.isArray(lastMessage.content)) { + // Add cache_control to the last content block + if (lastMessage.content.length > 0) { + const lastBlock = lastMessage.content[lastMessage.content.length - 1] + if (lastBlock.type === 'text') { + lastBlock.cache_control = { type: 'ephemeral' } + } + } + } else if (typeof lastMessage.content === 'string') { + // Convert string content to array format with cache_control + lastMessage.content = [ + { + type: 'text', + text: lastMessage.content, + cache_control: { type: 'ephemeral' } + } + ] + } + } + + return { system, messages: anthropicMessages } +} + +export function convertOpenAIToolsToAnthropic( + tools?: OpenAI.Chat.Completions.ChatCompletionFunctionTool[] +): ToolUnion[] | undefined { + if (!tools || tools.length === 0) return undefined + + const anthropicTools: ToolUnion[] = tools.map((tool) => ({ + name: tool.function.name, + description: tool.function.description, + input_schema: (tool.function.parameters || { + type: 'object', + properties: {} + }) as AnthropicTool.InputSchema + })) + + // Add cache_control to the last tool to cache all tool definitions + if (anthropicTools.length > 0) { + anthropicTools[anthropicTools.length - 1].cache_control = { type: 'ephemeral' } + } + + return anthropicTools +} diff --git a/frontend/src/lib/components/copilot/chat/api/apiTools.ts b/frontend/src/lib/components/copilot/chat/api/apiTools.ts index 4964201443170..1fa644603c202 100644 --- a/frontend/src/lib/components/copilot/chat/api/apiTools.ts +++ b/frontend/src/lib/components/copilot/chat/api/apiTools.ts @@ -1,11 +1,11 @@ -import type { ChatCompletionTool } from 'openai/resources/index.mjs' +import type { ChatCompletionFunctionTool } from 'openai/resources/index.mjs' import type { Tool } from '../shared' import { get } from 'svelte/store' import { workspaceStore } from '$lib/stores' import type { EndpointTool } from '$lib/gen/types.gen' import { McpService } from '$lib/gen/services.gen' -function buildApiCallTool(endpointTool: EndpointTool): ChatCompletionTool { +function buildApiCallTool(endpointTool: EndpointTool): ChatCompletionFunctionTool { // Build the parameters schema for OpenAI function calling const parameters: Record = { type: 'object', @@ -18,10 +18,13 @@ function buildApiCallTool(endpointTool: EndpointTool): ChatCompletionTool { for (const [key, schema] of Object.entries(endpointTool.path_params_schema.properties)) { // Skip workspace parameter as it's auto-filled if (key === 'workspace') continue - + parameters.properties[key] = schema - - if (Array.isArray(endpointTool.path_params_schema.required) && endpointTool.path_params_schema.required.includes(key)) { + + if ( + Array.isArray(endpointTool.path_params_schema.required) && + endpointTool.path_params_schema.required.includes(key) + ) { parameters.required.push(key) } } @@ -31,8 +34,11 @@ function buildApiCallTool(endpointTool: EndpointTool): ChatCompletionTool { if (endpointTool.query_params_schema?.properties) { for (const [key, schema] of Object.entries(endpointTool.query_params_schema.properties)) { parameters.properties[key] = schema - - if (Array.isArray(endpointTool.query_params_schema.required) && endpointTool.query_params_schema.required.includes(key)) { + + if ( + Array.isArray(endpointTool.query_params_schema.required) && + endpointTool.query_params_schema.required.includes(key) + ) { parameters.required.push(key) } } @@ -47,8 +53,11 @@ function buildApiCallTool(endpointTool: EndpointTool): ChatCompletionTool { properties: endpointTool.body_schema.properties, required: endpointTool.body_schema.required || [] } - - if (Array.isArray(endpointTool.body_schema.required) && endpointTool.body_schema.required.length > 0) { + + if ( + Array.isArray(endpointTool.body_schema.required) && + endpointTool.body_schema.required.length > 0 + ) { parameters.required.push('body') } } @@ -63,16 +72,17 @@ function buildApiCallTool(endpointTool: EndpointTool): ChatCompletionTool { } } -function buildToolsFromEndpoints( - endpointTools: EndpointTool[] -): { tools: ChatCompletionTool[]; endpointMap: Record } { - const tools: ChatCompletionTool[] = [] +function buildToolsFromEndpoints(endpointTools: EndpointTool[]): { + tools: ChatCompletionFunctionTool[] + endpointMap: Record +} { + const tools: ChatCompletionFunctionTool[] = [] const endpointMap: Record = {} for (const endpointTool of endpointTools) { const tool = buildApiCallTool(endpointTool) tools.push(tool) - + // Store the endpoint info in the map endpointMap[endpointTool.name] = { method: endpointTool.method, @@ -84,17 +94,17 @@ function buildToolsFromEndpoints( } export function createApiTools( - chatTools: ChatCompletionTool[], + chatTools: ChatCompletionFunctionTool[], endpointMap: Record = {} ): Tool<{}>[] { return chatTools.map((chatTool) => { const toolName = chatTool.function.name const endpoint = endpointMap[toolName] const method = endpoint?.method?.toUpperCase() || 'GET' - + // Determine if tool needs confirmation based on method const needsConfirmation = ['DELETE', 'POST', 'PUT', 'PATCH'].includes(method) - + return { def: chatTool, requiresConfirmation: needsConfirmation, @@ -103,7 +113,7 @@ export function createApiTools( fn: async ({ args, toolId, toolCallbacks }) => { const toolName = chatTool.function.name const endpoint = endpointMap[toolName] - + if (!endpoint) { throw new Error(`No endpoint mapping found for tool ${toolName}`) } @@ -111,7 +121,7 @@ export function createApiTools( try { const workspace = get(workspaceStore) as string let path = endpoint.path.replace('{workspace}', workspace) - + // Build URL with path parameters let url = `/api${path}` const queryParams: Record = {} @@ -143,7 +153,7 @@ export function createApiTools( } toolCallbacks.setToolStatus(toolId, { - content: `Calling ${toolName}...`, + content: `Calling ${toolName}...` }) const fetchOptions: RequestInit = { @@ -173,7 +183,7 @@ export function createApiTools( }) toolCallbacks.setToolStatus(toolId, { content: `Call to ${toolName} completed`, - result: jsonResult, + result: jsonResult }) return jsonResult } else { @@ -186,7 +196,7 @@ export function createApiTools( toolCallbacks.setToolStatus(toolId, { content: `Call to ${toolName} failed`, result: jsonResult, - error: `HTTP ${response.status}: ${text}`, + error: `HTTP ${response.status}: ${text}` }) return jsonResult } @@ -194,7 +204,7 @@ export function createApiTools( const errorMessage = `Error calling API: ${error instanceof Error ? error.message : String(error)}` toolCallbacks.setToolStatus(toolId, { content: `Call to ${toolName} failed`, - error: errorMessage, + error: errorMessage }) console.error(`Error calling API:`, error) return errorMessage @@ -210,10 +220,10 @@ export async function loadApiTools(): Promise[]> { const endpointTools = await McpService.listMcpTools({ workspace: get(workspaceStore) as string }) - + // Build tools from the endpoint definitions const { tools: apiTools, endpointMap } = buildToolsFromEndpoints(endpointTools) - + // Create executable tools const executableApiTools = createApiTools(apiTools, endpointMap) return executableApiTools @@ -221,4 +231,4 @@ export async function loadApiTools(): Promise[]> { console.error('Failed to load API tools:', error) return [] } -} \ No newline at end of file +} diff --git a/frontend/src/lib/components/copilot/chat/flow/core.ts b/frontend/src/lib/components/copilot/chat/flow/core.ts index d0a908ea7bd07..dae637f156ce0 100644 --- a/frontend/src/lib/components/copilot/chat/flow/core.ts +++ b/frontend/src/lib/components/copilot/chat/flow/core.ts @@ -638,7 +638,10 @@ export const flowTools: Tool[] = [ } }, { - def: setForLoopOptionsToolDef, + def: { + ...setForLoopOptionsToolDef, + function: { ...setForLoopOptionsToolDef.function, strict: false } + }, fn: async ({ args, helpers, toolId, toolCallbacks }) => { const parsedArgs = setForLoopOptionsSchema.parse(args) await helpers.setForLoopOptions(parsedArgs.id, { @@ -656,7 +659,10 @@ export const flowTools: Tool[] = [ } }, { - def: setModuleControlOptionsToolDef, + def: { + ...setModuleControlOptionsToolDef, + function: { ...setModuleControlOptionsToolDef.function, strict: false } + }, fn: async ({ args, helpers, toolId, toolCallbacks }) => { const parsedArgs = setModuleControlOptionsSchema.parse(args) await helpers.setModuleControlOptions(parsedArgs.id, { diff --git a/frontend/src/lib/components/copilot/chat/script/core.ts b/frontend/src/lib/components/copilot/chat/script/core.ts index 1154623f3af54..9deb4f0412d0c 100644 --- a/frontend/src/lib/components/copilot/chat/script/core.ts +++ b/frontend/src/lib/components/copilot/chat/script/core.ts @@ -5,7 +5,7 @@ import { get } from 'svelte/store' import { compile, phpCompile, pythonCompile } from '../../utils' import type { ChatCompletionSystemMessageParam, - ChatCompletionTool, + ChatCompletionFunctionTool, ChatCompletionUserMessageParam } from 'openai/resources/index.mjs' import { type DBSchema, dbSchemas, getCurrentModel } from '$lib/stores' @@ -498,7 +498,7 @@ export function prepareScriptUserMessage( } } -const RESOURCE_TYPE_FUNCTION_DEF: ChatCompletionTool = { +const RESOURCE_TYPE_FUNCTION_DEF: ChatCompletionFunctionTool = { type: 'function', function: { name: 'search_resource_types', @@ -519,7 +519,7 @@ const RESOURCE_TYPE_FUNCTION_DEF: ChatCompletionTool = { } } -const DB_SCHEMA_FUNCTION_DEF: ChatCompletionTool = { +const DB_SCHEMA_FUNCTION_DEF: ChatCompletionFunctionTool = { type: 'function', function: { name: 'get_db_schema', @@ -693,7 +693,7 @@ export async function searchExternalIntegrationResources(args: { query: string } } } -const SEARCH_NPM_PACKAGES_TOOL: ChatCompletionTool = { +const SEARCH_NPM_PACKAGES_TOOL: ChatCompletionFunctionTool = { type: 'function', function: { name: 'search_npm_packages', @@ -778,7 +778,7 @@ export async function fetchNpmPackageTypes( } } -const TEST_RUN_SCRIPT_TOOL: ChatCompletionTool = { +const TEST_RUN_SCRIPT_TOOL: ChatCompletionFunctionTool = { type: 'function', function: { name: 'test_run_script', diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index d3589df323cfe..39e5a6035900e 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -1,7 +1,7 @@ import type { - ChatCompletionMessageParam, - ChatCompletionMessageToolCall, - ChatCompletionTool + ChatCompletionFunctionTool, + ChatCompletionMessageFunctionToolCall, + ChatCompletionMessageParam } from 'openai/resources/chat/completions.mjs' import { get } from 'svelte/store' import type { CodePieceElement, ContextElement, FlowModuleCodePieceElement } from './context' @@ -257,7 +257,7 @@ export async function processToolCall({ toolCallbacks }: { tools: Tool[] - toolCall: ChatCompletionMessageToolCall + toolCall: ChatCompletionMessageFunctionToolCall helpers: T toolCallbacks: ToolCallbacks }): Promise { @@ -345,7 +345,7 @@ export async function processToolCall({ } export interface Tool { - def: ChatCompletionTool + def: ChatCompletionFunctionTool fn: (p: { args: any workspace: string @@ -369,7 +369,7 @@ export function createToolDef( zodSchema: z.ZodSchema, name: string, description: string -): ChatCompletionTool { +): ChatCompletionFunctionTool { const schema = zodToJsonSchema(zodSchema, { name, target: 'openAi' @@ -438,7 +438,7 @@ export const createSearchHubScriptsTool = (withContent: boolean = false) => ({ }) export async function buildSchemaForTool( - toolDef: ChatCompletionTool, + toolDef: ChatCompletionFunctionTool, schemaBuilder: () => Promise ): Promise { try { @@ -586,7 +586,10 @@ function getErrorMessage(result: unknown): string { } // Build test run args based on the tool definition, if it contains a fallback schema -export async function buildTestRunArgs(args: any, toolDef: ChatCompletionTool): Promise { +export async function buildTestRunArgs( + args: any, + toolDef: ChatCompletionFunctionTool +): Promise { let parsedArgs = args // if the schema is the fallback schema, parse the args as a JSON string if ( diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index 45e2f6982df6d..5cacb53894390 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -7,18 +7,23 @@ import { type SQLSchema } from '$lib/stores' import { buildClientSchema, printSchema } from 'graphql' -import { OpenAI } from 'openai' +import OpenAI from 'openai' import type { + ChatCompletionChunk, ChatCompletionCreateParams, ChatCompletionCreateParamsNonStreaming, ChatCompletionCreateParamsStreaming, + ChatCompletionMessageFunctionToolCall, ChatCompletionMessageParam } from 'openai/resources/index.mjs' +import Anthropic from '@anthropic-ai/sdk' import { get, type Writable } from 'svelte/store' import { OpenAPI, ResourceService, type Script } from '../../gen' import { EDIT_CONFIG, FIX_CONFIG, GEN_CONFIG } from './prompts' import { formatResourceTypes } from './utils' import { z } from 'zod' +import { processToolCall, type Tool, type ToolCallbacks } from './chat/shared' +import type { Stream } from 'openai/core/streaming.mjs' export const SUPPORTED_LANGUAGES = new Set(Object.keys(GEN_CONFIG.prompts)) @@ -219,9 +224,11 @@ export const PROVIDER_COMPLETION_CONFIG_MAP: Record({ +export function getProviderAndCompletionConfig({ messages, stream, tools, @@ -622,10 +645,11 @@ export async function getCompletion( messages: ChatCompletionMessageParam[], abortController: AbortController, tools?: OpenAI.Chat.Completions.ChatCompletionTool[] -) { +): Promise> { const { provider, config } = getProviderAndCompletionConfig({ messages, stream: true, tools }) + const openaiClient = workspaceAIClients.getOpenaiClient() - const completion = await openaiClient.chat.completions.create(config, { + const completion = openaiClient.chat.completions.create(config, { signal: abortController.signal, headers: { 'X-Provider': provider @@ -634,6 +658,108 @@ export async function getCompletion( return completion } +export async function parseOpenAICompletion( + completion: Stream, + callbacks: ToolCallbacks & { + onNewToken: (token: string) => void + onMessageEnd: () => void + }, + messages: ChatCompletionMessageParam[], + addedMessages: ChatCompletionMessageParam[], + tools: Tool[], + helpers: any +): Promise { + const finalToolCalls: Record = {} + + let answer = '' + for await (const chunk of completion) { + if (!('choices' in chunk && chunk.choices.length > 0 && 'delta' in chunk.choices[0])) { + continue + } + const c = chunk as ChatCompletionChunk + const delta = c.choices[0].delta.content + if (delta) { + answer += delta + callbacks.onNewToken(delta) + } + const toolCalls = c.choices[0].delta.tool_calls || [] + if (toolCalls.length > 0 && answer) { + // if tool calls are present but we have some textual content already, we need to display it to the user first + callbacks.onMessageEnd() + answer = '' + } + for (const toolCall of toolCalls) { + const { index } = toolCall + let finalToolCall = finalToolCalls[index] + if (!finalToolCall) { + finalToolCalls[index] = toolCall + } else { + if (toolCall.function?.arguments) { + if (!finalToolCall.function) { + finalToolCall.function = toolCall.function + } else { + finalToolCall.function.arguments = + (finalToolCall.function.arguments ?? '') + toolCall.function.arguments + } + } + } + finalToolCall = finalToolCalls[index] + if (finalToolCall?.function) { + const { + function: { name: funcName }, + id: toolCallId + } = finalToolCall + if (funcName && toolCallId) { + const tool = tools.find((t) => t.def.function.name === funcName) + if (tool && tool.preAction) { + tool.preAction({ toolCallbacks: callbacks, toolId: toolCallId }) + } + } + } + } + } + + if (answer) { + const toAdd = { role: 'assistant' as const, content: answer } + addedMessages.push(toAdd) + messages.push(toAdd) + } + + callbacks.onMessageEnd() + + const toolCalls = Object.values(finalToolCalls).filter( + (toolCall) => toolCall.id !== undefined && toolCall.function?.arguments !== undefined + ) as ChatCompletionMessageFunctionToolCall[] + + if (toolCalls.length > 0) { + const toAdd = { + role: 'assistant' as const, + tool_calls: toolCalls.map((t) => ({ + ...t, + function: { + ...t.function, + arguments: t.function.arguments || '{}' + } + })) + } + messages.push(toAdd) + addedMessages.push(toAdd) + for (const toolCall of toolCalls) { + const messageToAdd = await processToolCall({ + tools, + toolCall, + helpers, + toolCallbacks: callbacks + }) + messages.push(messageToAdd) + addedMessages.push(messageToAdd) + } + } else { + return false + } + return true +} + export function getResponseFromEvent(part: OpenAI.Chat.Completions.ChatCompletionChunk): string { return part.choices?.[0]?.delta?.content || '' } @@ -757,7 +883,7 @@ export async function deltaCodeCompletion( } if (!match[1].endsWith('`')) { - // skip udpating if possible that part of three ticks (end of code block)s + // skip updating if possible that part of three ticks (end of code block)s delta = getStringEndDelta(code, match[1]) generatedCodeDelta.set(delta) code = match[1]