diff --git a/.vscode/settings.json b/.vscode/settings.json index a0e5a25044..a5c65d4d69 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,11 +1,17 @@ { - "editor.rulers": [100], + "editor.rulers": [ + 100 + ], "editor.formatOnSave": true, "editor.defaultFormatter": "esbenp.prettier-vscode", "jest.virtualFolders": [ { "name": "cli", "rootPath": "packages/cli" + }, + { + "name": "navie", + "rootPath": "packages/navie" } - ], -} + ] +} \ No newline at end of file diff --git a/package.json b/package.json index 5fe01e8a65..e4c4e06e57 100644 --- a/package.json +++ b/package.json @@ -43,5 +43,8 @@ "packageManager": "yarn@3.2.1", "dependencies": { "puppeteer": "^19.7.2" + }, + "resolutions": { + "web-auth-library": "getappmap/web-auth-library#v1.0.3-cjs" } } diff --git a/packages/cli/src/cmds/index/aiEnvVar.ts b/packages/cli/src/cmds/index/aiEnvVar.ts index e1ef56651e..f8484d9b53 100644 --- a/packages/cli/src/cmds/index/aiEnvVar.ts +++ b/packages/cli/src/cmds/index/aiEnvVar.ts @@ -1,4 +1,9 @@ -export const AI_KEY_ENV_VARS = ['OPENAI_API_KEY', 'AZURE_OPENAI_API_KEY', 'ANTHROPIC_API_KEY']; +export const AI_KEY_ENV_VARS = [ + 'GOOGLE_WEB_CREDENTIALS', + 'OPENAI_API_KEY', + 'AZURE_OPENAI_API_KEY', + 'ANTHROPIC_API_KEY', +]; export default function detectAIEnvVar(): string | undefined { return Object.keys(process.env).find((key) => AI_KEY_ENV_VARS.includes(key)); diff --git a/packages/cli/src/cmds/index/index.ts b/packages/cli/src/cmds/index/index.ts index d9ff89d2f2..0185477994 100644 --- a/packages/cli/src/cmds/index/index.ts +++ b/packages/cli/src/cmds/index/index.ts @@ -24,8 +24,7 @@ import LocalNavie from '../../rpc/explain/navie/navie-local'; import RemoteNavie from '../../rpc/explain/navie/navie-remote'; import { InteractionEvent } from '@appland/navie/dist/interaction-history'; import { update } from '../../rpc/file/update'; - -const AI_KEY_ENV_VARS = ['OPENAI_API_KEY', 'AZURE_OPENAI_API_KEY']; +import { AI_KEY_ENV_VARS } from './aiEnvVar'; export const command = 'index'; export const describe = diff --git a/packages/cli/src/rpc/llmConfiguration.ts b/packages/cli/src/rpc/llmConfiguration.ts index 7e6637e8d3..4cb7b4b7ed 100644 --- a/packages/cli/src/rpc/llmConfiguration.ts +++ b/packages/cli/src/rpc/llmConfiguration.ts @@ -47,9 +47,14 @@ function openAIBaseURL(): string | undefined { return baseUrl; } +const DEFAULT_BASE_URLS = { + anthropic: 'https://api.anthropic.com/v1/', + 'vertex-ai': 'https://googleapis.com', + openai: undefined, +} as const; + export function getLLMConfiguration(): LLMConfiguration { - const baseUrl = - SELECTED_BACKEND === 'anthropic' ? 'https://api.anthropic.com/v1/' : openAIBaseURL(); + const baseUrl = (SELECTED_BACKEND && DEFAULT_BASE_URLS[SELECTED_BACKEND]) ?? openAIBaseURL(); return { baseUrl, diff --git a/packages/navie/.eslintrc.js b/packages/navie/.eslintrc.js index d98123a73e..2132a5aeaf 100644 --- a/packages/navie/.eslintrc.js +++ b/packages/navie/.eslintrc.js @@ -22,6 +22,7 @@ module.exports = { }, root: true, rules: { + 'eslint-comments/disable-enable-pair': ['error', { allowWholeFile: true }], 'no-param-reassign': ['error', { props: false }], 'no-console': process.env.NODE_ENV === 'production' ? 'warn' : 'off', 'no-debugger': process.env.NODE_ENV === 'production' ? 'warn' : 'off', diff --git a/packages/navie/package.json b/packages/navie/package.json index dad5f16133..7142ad626e 100644 --- a/packages/navie/package.json +++ b/packages/navie/package.json @@ -41,6 +41,7 @@ "dependencies": { "@langchain/anthropic": "^0.3.1", "@langchain/core": "^0.2.27", + "@langchain/google-vertexai-web": "^0.1.0", "@langchain/openai": "^0.2.7", "fast-xml-parser": "^4.4.0", "js-yaml": "^4.1.0", diff --git a/packages/navie/src/interaction-history.ts b/packages/navie/src/interaction-history.ts index b6470c44b1..d71c8f2756 100644 --- a/packages/navie/src/interaction-history.ts +++ b/packages/navie/src/interaction-history.ts @@ -1,5 +1,5 @@ +/* eslint-disable @typescript-eslint/no-unsafe-declaration-merging */ import EventEmitter from 'events'; -import * as path from 'path'; import InteractionState from './interaction-state'; import { ContextV2 } from './context'; import { PROMPTS, PromptType } from './prompt'; @@ -283,11 +283,7 @@ export class TechStackEvent extends InteractionEvent { } } -export interface InteractionHistoryEvents { - on(event: 'event', listener: (event: InteractionEvent) => void): void; -} - -export default class InteractionHistory extends EventEmitter implements InteractionHistoryEvents { +class InteractionHistory extends EventEmitter { public readonly events: InteractionEvent[] = []; // eslint-disable-next-line class-methods-use-this @@ -333,3 +329,10 @@ export default class InteractionHistory extends EventEmitter implements Interact return state; } } + +interface InteractionHistory { + on(event: 'event', listener: (event: InteractionEvent) => void): this; + on(event: string, listener: (...args: unknown[]) => void): this; +} + +export default InteractionHistory; diff --git a/packages/navie/src/lib/trajectory.ts b/packages/navie/src/lib/trajectory.ts index dcf21f87a7..39c496a963 100644 --- a/packages/navie/src/lib/trajectory.ts +++ b/packages/navie/src/lib/trajectory.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-unsafe-declaration-merging */ import EventEmitter from 'events'; import Message from '../message'; @@ -7,7 +8,7 @@ export interface TrajectoryEvent { timestamp: Date; } -export default class Trajectory extends EventEmitter { +class Trajectory extends EventEmitter { logSentMessage(message: Message) { const event: TrajectoryEvent = { type: 'sent', @@ -26,3 +27,10 @@ export default class Trajectory extends EventEmitter { this.emit('event', event); } } + +interface Trajectory { + on(event: 'event', listener: (event: TrajectoryEvent) => void): this; + on(event: string, listener: (...args: unknown[]) => void): this; +} + +export default Trajectory; diff --git a/packages/navie/src/navie.ts b/packages/navie/src/navie.ts index 8d41767e51..44d299bd7b 100644 --- a/packages/navie/src/navie.ts +++ b/packages/navie/src/navie.ts @@ -8,7 +8,6 @@ import InteractionHistory, { AgentSelectionEvent, ClassificationEvent, InteractionEvent, - InteractionHistoryEvents, } from './interaction-history'; import { ContextV2 } from './context'; import ProjectInfoService from './services/project-info-service'; @@ -48,7 +47,7 @@ export interface ClientRequest { prompt?: string; } -export interface INavie extends InteractionHistoryEvents { +export interface INavie { on(event: 'event', listener: (event: InteractionEvent) => void): void; on(event: 'agent', listener: (agent: string) => void): void; diff --git a/packages/navie/src/services/completion-service-factory.ts b/packages/navie/src/services/completion-service-factory.ts index 19a3239d1d..13395147e5 100644 --- a/packages/navie/src/services/completion-service-factory.ts +++ b/packages/navie/src/services/completion-service-factory.ts @@ -1,5 +1,6 @@ import { warn } from 'node:console'; +import GoogleVertexAICompletionService from './google-vertexai-completion-service'; import OpenAICompletionService from './openai-completion-service'; import AnthropicCompletionService from './anthropic-completion-service'; import CompletionService from './completion-service'; @@ -10,47 +11,41 @@ interface Options { modelName: string; temperature: number; trajectory: Trajectory; + backend?: Backend; } -type Backend = 'anthropic' | 'openai'; +const BACKENDS = { + anthropic: AnthropicCompletionService, + openai: OpenAICompletionService, + 'vertex-ai': GoogleVertexAICompletionService, +} as const; -function defaultBackend(): Backend { - return 'ANTHROPIC_API_KEY' in process.env ? 'anthropic' : 'openai'; -} +type Backend = keyof typeof BACKENDS; -function environmentBackend(): Backend | undefined { +function determineCompletionBackend(): Backend { switch (process.env.APPMAP_NAVIE_COMPLETION_BACKEND) { case 'anthropic': case 'openai': + case 'vertex-ai': return process.env.APPMAP_NAVIE_COMPLETION_BACKEND; default: - return undefined; + // pass } + if ('ANTHROPIC_API_KEY' in process.env) return 'anthropic'; + if ('GOOGLE_WEB_CREDENTIALS' in process.env) return 'vertex-ai'; + if ('OPENAI_API_KEY' in process.env) return 'openai'; + return 'openai'; // fallback } -export const SELECTED_BACKEND: Backend = environmentBackend() ?? defaultBackend(); +export const SELECTED_BACKEND: Backend = determineCompletionBackend(); export default function createCompletionService({ modelName, temperature, trajectory, + backend = determineCompletionBackend(), }: Options): CompletionService { - const backend = environmentBackend() ?? defaultBackend(); const messageTokenReducerService = new MessageTokenReducerService(); - if (backend === 'anthropic') { - warn('Using Anthropic AI backend'); - return new AnthropicCompletionService( - modelName, - temperature, - trajectory, - messageTokenReducerService - ); - } - warn('Using OpenAI backend'); - return new OpenAICompletionService( - modelName, - temperature, - trajectory, - messageTokenReducerService - ); + warn(`Using completion service ${backend}`); + return new BACKENDS[backend](modelName, temperature, trajectory, messageTokenReducerService); } diff --git a/packages/navie/src/services/google-vertexai-completion-service.ts b/packages/navie/src/services/google-vertexai-completion-service.ts new file mode 100644 index 0000000000..9d66ae5cd3 --- /dev/null +++ b/packages/navie/src/services/google-vertexai-completion-service.ts @@ -0,0 +1,137 @@ +import { warn } from 'node:console'; +import { isNativeError } from 'node:util/types'; + +import { ChatVertexAI, type ChatVertexAIInput } from '@langchain/google-vertexai-web'; +import { zodResponseFormat } from 'openai/helpers/zod'; +import { z } from 'zod'; + +import Trajectory from '../lib/trajectory'; +import Message from '../message'; +import CompletionService, { + CompleteOptions, + Completion, + CompletionRetries, + CompletionRetryDelay, + convertToMessage, + mergeSystemMessages, + Usage, +} from './completion-service'; + +export default class GoogleVertexAICompletionService implements CompletionService { + constructor( + public readonly modelName: string, + public readonly temperature: number, + private trajectory: Trajectory + ) {} + + // Construct a model with non-default options. There doesn't seem to be a way to configure + // the model parameters at invocation time like with OpenAI. + private buildModel(options?: ChatVertexAIInput): ChatVertexAI { + return new ChatVertexAI({ + model: this.modelName, + temperature: this.temperature, + streaming: true, + maxOutputTokens: 8192, + ...options, + }); + } + + get miniModelName(): string { + const miniModel = process.env.APPMAP_NAVIE_MINI_MODEL; + return miniModel ?? 'gemini-1.5-flash-002'; + } + + // Request a JSON object with a given JSON schema. + async json( + messages: Message[], + schema: Schema, + options?: CompleteOptions + ): Promise | undefined> { + const model = this.buildModel({ + ...options, + streaming: false, + responseMimeType: 'application/json', + }); + const sentMessages = mergeSystemMessages([ + ...messages, + { + role: 'system', + content: `Use the following JSON schema for your response:\n\n${JSON.stringify( + zodResponseFormat(schema, 'requestedObject').json_schema.schema, + null, + 2 + )}`, + }, + ]); + + for (const message of sentMessages) this.trajectory.logSentMessage(message); + + const response = await model.invoke(sentMessages.map(convertToMessage)); + + this.trajectory.logReceivedMessage({ + role: 'assistant', + content: JSON.stringify(response), + }); + + const sanitizedContent = response.content.toString().replace(/^`{3,}[^\s]*?$/gm, ''); + const parsed = JSON.parse(sanitizedContent) as unknown; + schema.parse(parsed); + return parsed; + } + + async *complete(messages: readonly Message[], options?: { temperature?: number }): Completion { + const usage = new Usage(); + const model = this.buildModel(options); + const sentMessages: Message[] = mergeSystemMessages(messages); + const tokens = new Array(); + for (const message of sentMessages) this.trajectory.logSentMessage(message); + + const maxAttempts = CompletionRetries; + for (let attempt = 0; attempt < maxAttempts; attempt += 1) { + try { + // eslint-disable-next-line no-await-in-loop + const response = await model.stream(sentMessages.map(convertToMessage)); + + // eslint-disable-next-line @typescript-eslint/naming-convention, no-await-in-loop + for await (const { content, usage_metadata } of response) { + yield content.toString(); + tokens.push(content.toString()); + if (usage_metadata) { + usage.promptTokens += usage_metadata.input_tokens; + usage.completionTokens += usage_metadata.output_tokens; + } + } + + this.trajectory.logReceivedMessage({ + role: 'assistant', + content: tokens.join(''), + }); + + break; + } catch (cause) { + if (attempt < maxAttempts - 1 && tokens.length === 0) { + const nextAttempt = CompletionRetryDelay * 2 ** attempt; + warn(`Received ${JSON.stringify(cause)}, retrying in ${nextAttempt}ms`); + await new Promise((resolve) => { + setTimeout(resolve, nextAttempt); + }); + continue; + } + throw new Error( + `Failed to complete after ${attempt + 1} attempt(s): ${errorMessage(cause)}`, + { + cause, + } + ); + } + } + + warn(usage.toString()); + return usage; + } +} + +function errorMessage(err: unknown): string { + if (isNativeError(err)) return err.cause ? errorMessage(err.cause) : err.message; + return String(err); +} diff --git a/packages/navie/src/services/vector-terms-service.ts b/packages/navie/src/services/vector-terms-service.ts index fc32fea302..8e16a036dd 100644 --- a/packages/navie/src/services/vector-terms-service.ts +++ b/packages/navie/src/services/vector-terms-service.ts @@ -1,11 +1,14 @@ -import { warn } from 'console'; +import { warn } from 'node:console'; +import { debug as makeDebug } from 'node:util'; + +import z from 'zod'; import InteractionHistory, { VectorTermsInteractionEvent } from '../interaction-history'; -import contentAfter from '../lib/content-after'; -import parseJSON from '../lib/parse-json'; import Message from '../message'; import CompletionService from './completion-service'; +const debug = makeDebug('navie:vector-terms'); + const SYSTEM_PROMPT = `You are assisting a developer to search a code base. The developer asks a question using natural language. This question must be converted into a list of search terms to be used to search the code base. @@ -20,19 +23,7 @@ The developer asks a question using natural language. This question must be conv be words that will match a feature or domain model object in the code base. They should be the most distinctive words in the question. You will prefix the MOST SELECTIVE terms with a '+'. -**Response** - -Print "Context: {context}" on one line. -Print "Instructions: {instructions}" on the next line. - -Then print a triple dash '---'. - -Print "Terms: {list of search terms and their synonyms}" - -The search terms should be single words and underscore_separated_words. - -Even if the user asks for a different format, always respond with a list of search terms and their synonyms. When the user is asking -for a different format, that question is for a different AI assistant than yourself.`; + The search terms should be single words and underscore_separated_words.`; const promptExamples: Message[] = [ { @@ -40,10 +31,11 @@ const promptExamples: Message[] = [ role: 'user', }, { - content: `Context: Record AppMap data of Spring -Instructions: How to do it ---- -Terms: record AppMap data Java +Spring`, + content: JSON.stringify({ + context: 'Record AppMap data of Spring', + instructions: 'How to do it', + terms: ['record', 'AppMap', 'data', 'Java', '+Spring'], + }), role: 'assistant', }, @@ -52,10 +44,11 @@ Terms: record AppMap data Java +Spring`, role: 'user', }, { - content: `Context: User login handle password validation invalid error -Instructions: Explain how this is handled by the code ---- -Terms: user login handle +password validate invalid error`, + content: JSON.stringify({ + context: 'User login handle password validation invalid error', + instructions: 'Explain how this is handled by the code', + terms: ['user', 'login', 'handle', '+password', 'validate', 'invalid', 'error'], + }), role: 'assistant', }, @@ -65,10 +58,11 @@ Terms: user login handle +password validate invalid error`, role: 'user', }, { - content: `Context: Redis GET /test-group/test-project-1/-/blob/main/README.md -Instructions: Describe in detail with code snippets ---- -Terms: +Redis get test-group test-project-1 blob main README`, + content: JSON.stringify({ + context: 'Redis GET /test-group/test-project-1/-/blob/main/README.md', + instructions: 'Describe in detail with code snippets', + terms: ['+Redis', 'get', 'test-group', 'test-project-1', 'blob', 'main', 'README'], + }), role: 'assistant', }, @@ -78,10 +72,11 @@ Terms: +Redis get test-group test-project-1 blob main README`, role: 'user', }, { - content: `Context: logContext jest test case -Instructions: Create test cases, following established patterns for mocking with jest. ---- -Terms: test cases +logContext jest`, + content: JSON.stringify({ + context: 'logContext jest test case', + instructions: 'Create test cases, following established patterns for mocking with jest.', + terms: ['test', 'cases', '+logContext', 'jest'], + }), role: 'assistant', }, @@ -90,15 +85,20 @@ Terms: test cases +logContext jest`, role: 'user', }, { - content: `Context: auth authentication authorization -Instructions: Describe the authentication and authorization process ---- -Terms: +auth authentication authorization token strategy provider`, + content: JSON.stringify({ + context: 'auth authentication authorization', + instructions: 'Describe the authentication and authorization process', + terms: ['+auth', 'authentication', 'authorization', 'token', 'strategy', 'provider'], + }), role: 'assistant', }, ]; -const parseText = (text: string): string[] => text.split(/\s+/); +const schema = z.object({ + context: z.string(), + instructions: z.string(), + terms: z.array(z.string()), +}); export default class VectorTermsService { constructor( @@ -119,45 +119,15 @@ export default class VectorTermsService { }, ]; - const response = this.completionsService.complete(messages, { + const response = await this.completionsService.json(messages, schema, { model: this.completionsService.miniModelName, }); - const tokens = Array(); - for await (const token of response) { - tokens.push(token); - } - const rawResponse = tokens.join(''); - warn(`Vector terms response:\n${rawResponse}`); - - let searchTermsObject: Record | string | string[] | undefined; - { - let responseText = rawResponse; - responseText = contentAfter(responseText, 'Terms:'); - searchTermsObject = - parseJSON | string | string[]>(responseText, false) || - parseText(responseText); - } - - const terms = new Set(); - { - const collectTerms = (obj: unknown) => { - if (!obj) return; - - if (typeof obj === 'string') { - terms.add(obj); - } else if (Array.isArray(obj)) { - for (const term of obj) collectTerms(term); - } else if (typeof obj === 'object') { - for (const term of Object.values(obj)) { - collectTerms(term); - } - } - }; - collectTerms(searchTermsObject); - } - - const result = [...terms]; - this.interactionHistory.addEvent(new VectorTermsInteractionEvent(result)); - return result; + + debug(`Vector terms response: ${JSON.stringify(response, undefined, 2)}`); + + const terms = response?.terms ?? []; + if (terms.length === 0) warn('No terms suggested'); + this.interactionHistory.addEvent(new VectorTermsInteractionEvent(terms)); + return terms; } } diff --git a/packages/navie/test/fixture.ts b/packages/navie/test/fixture.ts index 6004fba6eb..e3bd4403c6 100644 --- a/packages/navie/test/fixture.ts +++ b/packages/navie/test/fixture.ts @@ -98,23 +98,3 @@ export function predictsSummary(): (messages: Message[]) => void { return Promise.resolve([]); }); } - -export function mockAIResponse(completionWithRetry: jest.Mock, responses: string[]): void { - completionWithRetry.mockResolvedValueOnce( - responses.map((response, index) => ({ - id: 'cmpl-3Z5z9J5Z5Z5Z5Z5Z5Z5Z5Z5Z5Z5', - choices: [ - { - delta: { - content: response, - }, - index, - finish_reason: index === responses.length - 1 ? 'stop' : null, - }, - ], - created: 1635989729, - model: 'gpt-3.5', - object: 'chat.completion.chunk', - })) - ); -} diff --git a/packages/navie/test/services/classification-service.spec.ts b/packages/navie/test/services/classification-service.spec.ts index 00c34f891c..471f2cf5a1 100644 --- a/packages/navie/test/services/classification-service.spec.ts +++ b/packages/navie/test/services/classification-service.spec.ts @@ -1,31 +1,20 @@ -import { ChatOpenAI } from '@langchain/openai'; - import InteractionHistory from '../../src/interaction-history'; import ClassificationService from '../../src/services/classification-service'; -import { mockAIResponse } from '../fixture'; -import OpenAICompletionService from '../../src/services/openai-completion-service'; -import Trajectory from '../../src/lib/trajectory'; -import { TrajectoryEvent } from '../../dist/lib/trajectory'; -import MessageTokenReducerService from '../../src/services/message-token-reducer-service'; -jest.mock('@langchain/openai'); -const completionWithRetry = jest.mocked(ChatOpenAI.prototype.completionWithRetry); +import MockCompletionService from './mock-completion-service'; describe('ClassificationService', () => { let interactionHistory: InteractionHistory; - let trajectory: Trajectory; let service: ClassificationService; + const completion = new MockCompletionService(); + const completeSpy = jest.spyOn(completion, 'complete'); beforeEach(() => { interactionHistory = new InteractionHistory(); interactionHistory.on('event', (event) => console.log(event.message)); - trajectory = new Trajectory(); - service = new ClassificationService( - interactionHistory, - new OpenAICompletionService('gpt-4', 0.5, trajectory, new MessageTokenReducerService()) - ); + service = new ClassificationService(interactionHistory, completion); }); - afterEach(() => jest.resetAllMocks()); + afterEach(() => jest.restoreAllMocks()); describe('when LLM responds', () => { const classification = ` @@ -33,7 +22,7 @@ describe('ClassificationService', () => { - troubleshoot: medium `; - beforeEach(() => mockAIResponse(completionWithRetry, [classification])); + beforeEach(() => completion.mock(classification)); it('returns the response', async () => { const response = await service.classifyQuestion('user management'); @@ -47,7 +36,7 @@ describe('ClassificationService', () => { weight: 'medium', }, ]); - expect(completionWithRetry).toHaveBeenCalledTimes(1); + expect(completeSpy).toHaveBeenCalledTimes(1); }); it('emits classification event', async () => { @@ -60,16 +49,5 @@ describe('ClassificationService', () => { classification: ['architecture=high', 'troubleshoot=medium'], }); }); - - it('emits trajectory events', async () => { - const trajectoryEvents = new Array(); - - trajectory.on('event', (event) => trajectoryEvents.push(event)); - - await service.classifyQuestion('user management'); - - expect(trajectoryEvents.map((e) => e.message.role)).toEqual(['system', 'user', 'assistant']); - expect(trajectoryEvents.map((e) => e.type)).toEqual(['sent', 'sent', 'received']); - }); }); }); diff --git a/packages/navie/test/services/compute-update-service.spec.ts b/packages/navie/test/services/compute-update-service.spec.ts index 7763ee355c..f3772ffa25 100644 --- a/packages/navie/test/services/compute-update-service.spec.ts +++ b/packages/navie/test/services/compute-update-service.spec.ts @@ -1,31 +1,20 @@ -import { ChatOpenAI } from '@langchain/openai'; - import InteractionHistory from '../../src/interaction-history'; -import ClassificationService from '../../src/services/classification-service'; -import { mockAIResponse } from '../fixture'; -import OpenAICompletionService from '../../src/services/openai-completion-service'; import ComputeUpdateService from '../../src/services/compute-update-service'; -import Trajectory from '../../src/lib/trajectory'; -import MessageTokenReducerService from '../../src/services/message-token-reducer-service'; -jest.mock('@langchain/openai'); -const completionWithRetry = jest.mocked(ChatOpenAI.prototype.completionWithRetry); +import MockCompletionService from './mock-completion-service'; describe('ComputeUpdateService', () => { let interactionHistory: InteractionHistory; - let trajectory: Trajectory; let service: ComputeUpdateService; + const completion = new MockCompletionService(); + const complete = jest.spyOn(completion, 'complete'); beforeEach(() => { interactionHistory = new InteractionHistory(); interactionHistory.on('event', (event) => console.log(event.message)); - trajectory = new Trajectory(); - service = new ComputeUpdateService( - interactionHistory, - new OpenAICompletionService('gpt-4', 0.5, trajectory, new MessageTokenReducerService()) - ); + service = new ComputeUpdateService(interactionHistory, completion); }); - afterEach(() => jest.resetAllMocks()); + afterEach(() => jest.restoreAllMocks()); describe('when LLM responds', () => { const existingContent = `class User < ApplicationRecord @@ -50,12 +39,12 @@ end `; - beforeEach(() => mockAIResponse(completionWithRetry, [changeStr])); + beforeEach(() => completion.mock(changeStr)); it('computes the update', async () => { const response = await service.computeUpdate(existingContent, newContent); expect(response).toStrictEqual(change); - expect(completionWithRetry).toHaveBeenCalledTimes(1); + expect(complete).toHaveBeenCalledTimes(1); }); }); }); diff --git a/packages/navie/test/services/mock-completion-service.ts b/packages/navie/test/services/mock-completion-service.ts new file mode 100644 index 0000000000..6eb4ba93f9 --- /dev/null +++ b/packages/navie/test/services/mock-completion-service.ts @@ -0,0 +1,55 @@ +import assert from 'node:assert'; + +import { ZodType } from 'zod'; + +import type { Message } from '../../src'; +import CompletionService, { type Completion, Usage } from '../../src/services/completion-service'; + +export default class MockCompletionService implements CompletionService { + // eslint-disable-next-line @typescript-eslint/require-await + async *complete(messages: readonly Message[]): Completion { + const completion = this.completion(messages); + for (const c of completion) { + yield c; + } + return new Usage(); + } + + /** + * The mock completion function. This function can be used to mock the completion result. + * By default, it returns a hardcoded string split on spaces. It's a normal Jest mock, so you can manipulate the result + * further using eg. the `mockReturnValue` function. mock() method is provided as a shorthand. + * @param messages The messages to complete. + * @returns The mocked completion. + */ + // eslint-disable-next-line @typescript-eslint/no-unused-vars + completion = jest.fn(function (this: MockCompletionService, messages: readonly Message[]) { + return ['Example ', 'response ', 'from ', 'the ', 'LLM', '.']; + }); + + /** + * A shorthand to mock the completion result. + * @param response The response to give to callers. This can be text or a JSON object. + * @returns The mock function that can be used to manipulate the results further. + */ + mock(...response: string[]): typeof this.completion; + mock(response: string): typeof this.completion; + mock(response: unknown): typeof this.completion; + mock(...response: unknown[]): typeof this.completion { + if (response.length > 1 && response.every((x) => typeof x === 'string')) + return this.completion.mockReturnValue(response as string[]); + assert(response.length === 1, 'Only one response is supported'); + const cpl = typeof response[0] === 'string' ? response[0] : JSON.stringify(response[0]); + return this.completion.mockReturnValue(cpl.split(/(?= )/)); + } + + // eslint-disable-next-line @typescript-eslint/require-await + async json(messages: Message[], schema: ZodType): Promise { + const completion = this.completion(messages).join(''); + return schema.parse(JSON.parse(completion)); + } + + modelName = 'mock-model'; + miniModelName = 'mock-mini-model'; + temperature = 0.7; +} diff --git a/packages/navie/test/services/vector-terms-service.spec.ts b/packages/navie/test/services/vector-terms-service.spec.ts index 76ff90eb79..7012e6862c 100644 --- a/packages/navie/test/services/vector-terms-service.spec.ts +++ b/packages/navie/test/services/vector-terms-service.spec.ts @@ -1,115 +1,33 @@ -/* eslint-disable @typescript-eslint/no-unsafe-return */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ - -import { ChatOpenAI } from '@langchain/openai'; - import VectorTermsService from '../../src/services/vector-terms-service'; import InteractionHistory from '../../src/interaction-history'; -import { mockAIResponse } from '../fixture'; -import OpenAICompletionService from '../../src/services/openai-completion-service'; -import Trajectory from '../../src/lib/trajectory'; -import MessageTokenReducerService from '../../src/services/message-token-reducer-service'; -jest.mock('@langchain/openai'); -const completionWithRetry = jest.mocked(ChatOpenAI.prototype.completionWithRetry); +import MockCompletionService from './mock-completion-service'; describe('VectorTermsService', () => { + describe('when LLM suggested terms', () => { + it('is recorded in the interaction history', async () => { + completion.mock({ + context: 'ctx', + instructions: 'insns', + terms: ['user', 'management', '+provider'], + }); + await service.suggestTerms('user management'); + expect(interactionHistory.events).toEqual([ + expect.objectContaining({ + type: 'vectorTerms', + terms: ['user', 'management', '+provider'], + }), + ]); + }); + }); + let interactionHistory: InteractionHistory; let service: VectorTermsService; - let trajectory: Trajectory; beforeEach(() => { interactionHistory = new InteractionHistory(); - interactionHistory.on('event', (event) => console.log(event.message)); - trajectory = new Trajectory(); - service = new VectorTermsService( - interactionHistory, - new OpenAICompletionService('gpt-4', 0.5, trajectory, new MessageTokenReducerService()) - ); + service = new VectorTermsService(interactionHistory, completion); }); afterEach(() => jest.resetAllMocks()); - - describe('when LLM suggested terms', () => { - describe('is a valid JSON object', () => { - it('is recorded in the interaction history', async () => { - mockAIResponse(completionWithRetry, [`{"terms": ["user", "management"]}`]); - await service.suggestTerms('user management'); - expect(interactionHistory.events.map((e) => ({ ...e }))).toEqual([ - { - type: 'vectorTerms', - terms: ['user', 'management'], - }, - ]); - }); - it('should return the terms', async () => { - mockAIResponse(completionWithRetry, [`{"terms": ["user", "management"]}`]); - const terms = await service.suggestTerms('user management'); - expect(terms).toEqual(['user', 'management']); - expect(completionWithRetry).toHaveBeenCalledTimes(1); - }); - it('removes very short terms', async () => { - mockAIResponse(completionWithRetry, [`["user", "management", "a"]`]); - const terms = await service.suggestTerms('user management'); - expect(terms).toEqual(['user', 'management', 'a']); - expect(completionWithRetry).toHaveBeenCalledTimes(1); - }); - it('converts underscore_words to distinct words', async () => { - mockAIResponse(completionWithRetry, [`["user_management"]`]); - const terms = await service.suggestTerms('user management'); - expect(terms).toEqual(['user_management']); - expect(completionWithRetry).toHaveBeenCalledTimes(1); - }); - }); - - describe('are a valid JSON list', () => { - it('should return the terms', async () => { - mockAIResponse(completionWithRetry, ['["user", "management"]']); - const terms = await service.suggestTerms('user management'); - expect(terms).toEqual(['user', 'management']); - }); - }); - - describe('are valid JSON wrapped in fences', () => { - it('should return the terms', async () => { - mockAIResponse(completionWithRetry, ['```json\n', '["user", "management"]\n', '```\n']); - const terms = await service.suggestTerms('user management'); - expect(terms).toEqual(['user', 'management']); - }); - }); - - describe('is YAML', () => { - it('parses the terms', async () => { - mockAIResponse(completionWithRetry, ['response_key:\n', ' - user\n', ' - management\n']); - const terms = await service.suggestTerms('user management'); - expect(terms).toEqual(['response_key:', '-', 'user', 'management']); - }); - }); - - describe('is prefixed by "Terms:"', () => { - it('is accepted and processed', async () => { - mockAIResponse(completionWithRetry, ['Terms: ["user", "management"]']); - const terms = await service.suggestTerms('user management'); - expect(terms).toEqual(['user', 'management']); - expect(completionWithRetry).toHaveBeenCalledTimes(1); - }); - }); - - describe('includes terms with "+" prefix', () => { - it('is accepted and processed', async () => { - mockAIResponse(completionWithRetry, ['Terms: +user management']); - const terms = await service.suggestTerms('user management'); - expect(terms).toEqual(['+user', 'management']); - expect(completionWithRetry).toHaveBeenCalledTimes(1); - }); - }); - - describe('is list-ish ', () => { - it('is accepted and processed', async () => { - mockAIResponse(completionWithRetry, ['-user -mgmt']); - const terms = await service.suggestTerms('user management'); - expect(terms).toEqual(['-user', '-mgmt']); - expect(completionWithRetry).toHaveBeenCalledTimes(1); - }); - }); - }); + const completion = new MockCompletionService(); }); diff --git a/yarn.lock b/yarn.lock index edb801a4e8..635f429c92 100644 --- a/yarn.lock +++ b/yarn.lock @@ -457,6 +457,7 @@ __metadata: dependencies: "@langchain/anthropic": ^0.3.1 "@langchain/core": ^0.2.27 + "@langchain/google-vertexai-web": ^0.1.0 "@langchain/openai": ^0.2.7 "@tsconfig/node-lts": ^20.1.3 "@types/jest": ^29.4.1 @@ -7039,6 +7040,41 @@ __metadata: languageName: node linkType: hard +"@langchain/google-common@npm:~0.1.0": + version: 0.1.1 + resolution: "@langchain/google-common@npm:0.1.1" + dependencies: + uuid: ^10.0.0 + zod-to-json-schema: ^3.22.4 + peerDependencies: + "@langchain/core": ">=0.2.21 <0.4.0" + checksum: e460a08eaf5e6902c3cb7e8deb9edddcdb46c6bc38657ee1050d05ab5f17bf864bf298a9f00cc41e2824f8c072d79c1dca9b84a7ce64ebcf5a5357af14f5b9d9 + languageName: node + linkType: hard + +"@langchain/google-vertexai-web@npm:^0.1.0": + version: 0.1.0 + resolution: "@langchain/google-vertexai-web@npm:0.1.0" + dependencies: + "@langchain/google-webauth": ~0.1.0 + peerDependencies: + "@langchain/core": ">=0.2.21 <0.4.0" + checksum: 8c32499e4070ddf28de26e3e4354c60303921e0be84aa68bbcbbeecd5e79e78354fb940708dcfc94efbc67f51893e51039288d78418ec00ec3f64a6cb1e5b20e + languageName: node + linkType: hard + +"@langchain/google-webauth@npm:~0.1.0": + version: 0.1.0 + resolution: "@langchain/google-webauth@npm:0.1.0" + dependencies: + "@langchain/google-common": ~0.1.0 + web-auth-library: ^1.0.3 + peerDependencies: + "@langchain/core": ">=0.2.21 <0.4.0" + checksum: 90d7c04f95e9950ec5fb39a779352f145efa319d2003564b82a183809ef92d64f8f878999e5cb9c75b1bfda83e38c9b650946c928b1d137dd8bf0bebbaddca74 + languageName: node + linkType: hard + "@langchain/openai@npm:>=0.1.0 <0.3.0, @langchain/openai@npm:^0.2.7": version: 0.2.7 resolution: "@langchain/openai@npm:0.2.7" @@ -28417,6 +28453,13 @@ __metadata: languageName: node linkType: hard +"jose@npm:>= 4.12.0 < 5.0.0": + version: 4.15.9 + resolution: "jose@npm:4.15.9" + checksum: 41abe1c99baa3cf8a78ebbf93da8f8e50e417b7a26754c4afa21865d87527b8ac2baf66de2c5f6accc3f7d7158658dae7364043677236ea1d07895b040097f15 + languageName: node + linkType: hard + "joycon@npm:^3.0.1": version: 3.1.1 resolution: "joycon@npm:3.1.1" @@ -36936,6 +36979,13 @@ resolve@1.1.7: languageName: node linkType: hard +"rfc4648@npm:^1.5.2": + version: 1.5.3 + resolution: "rfc4648@npm:1.5.3" + checksum: 19c81d502582e377125b00fbd7a5cdb0e351f9a1e40182fa9f608b48e1ab852d211b75facb2f4f3fa17f7c6ebc2ef4acca61ae7eb7fbcfa4768f11d2db678116 + languageName: node + linkType: hard + "rfdc@npm:^1.3.0": version: 1.3.0 resolution: "rfdc@npm:1.3.0" @@ -41849,6 +41899,16 @@ typescript@~4.4.3: languageName: node linkType: hard +"web-auth-library@getappmap/web-auth-library#v1.0.3-cjs": + version: 1.0.3 + resolution: "web-auth-library@https://github.com/getappmap/web-auth-library.git#commit=f60401541b00795465224c5da786fd273dbc9459" + dependencies: + jose: ">= 4.12.0 < 5.0.0" + rfc4648: ^1.5.2 + checksum: 8031779036fdba6f7eba1638e73368a11b7745a26cab2401998878e767e83fcd88c5b0270ddce140fd1f1822e177b09619a856368295a644af256a2d2d721b88 + languageName: node + linkType: hard + "web-streams-polyfill@npm:4.0.0-beta.3": version: 4.0.0-beta.3 resolution: "web-streams-polyfill@npm:4.0.0-beta.3"