diff --git a/js/core/src/tracing.ts b/js/core/src/tracing.ts index 87b69283da..df71d04d9f 100644 --- a/js/core/src/tracing.ts +++ b/js/core/src/tracing.ts @@ -42,6 +42,7 @@ const instrumentationKey = '__GENKIT_TELEMETRY_INSTRUMENTED'; * @hidden */ export async function ensureBasicTelemetryInstrumentation() { + console.log("enabling basic telemetry"); if (global[instrumentationKey]) { return await global[instrumentationKey]; } @@ -54,6 +55,7 @@ export async function ensureBasicTelemetryInstrumentation() { export async function enableTelemetry( telemetryConfig: TelemetryConfig | Promise ) { + console.log("enabling telemetry"); if (process.env.GENKIT_TELEMETRY_SERVER) { setTelemetryServerUrl(process.env.GENKIT_TELEMETRY_SERVER); } @@ -102,6 +104,7 @@ export async function cleanUpTracing(): Promise { * Creates a new SpanProcessor for exporting data to the telemetry server. */ function createTelemetryServerProcessor(): SpanProcessor { + console.log("creating processor telemetry"); const exporter = new TraceServerExporter(); return isDevEnv() ? new SimpleSpanProcessor(exporter) @@ -110,6 +113,7 @@ function createTelemetryServerProcessor(): SpanProcessor { /** Flush metrics if present. */ function maybeFlushMetrics(): Promise { + console.log("maybe flushing telemetry"); if (nodeOtelConfig?.metricReader) { return nodeOtelConfig.metricReader.forceFlush(); } @@ -122,6 +126,7 @@ function maybeFlushMetrics(): Promise { * @hidden */ export async function flushTracing() { + console.log("flushing tracins"); if (nodeOtelConfig?.spanProcessors) { await Promise.all(nodeOtelConfig.spanProcessors.map((p) => p.forceFlush())); } diff --git a/js/core/src/tracing/exporter.ts b/js/core/src/tracing/exporter.ts index c9092317bd..f4f3fb6fb1 100644 --- a/js/core/src/tracing/exporter.ts +++ b/js/core/src/tracing/exporter.ts @@ -47,6 +47,7 @@ export class TraceServerExporter implements SpanExporter { spans: ReadableSpan[], resultCallback: (result: ExportResult) => void ): void { + console.log("export telemetry"); this._sendSpans(spans, resultCallback); } @@ -106,6 +107,7 @@ export class TraceServerExporter implements SpanExporter { * Exports any pending spans in exporter */ forceFlush(): Promise { + console.log("force flush 2 telemetry"); return Promise.resolve(); } @@ -137,6 +139,7 @@ export class TraceServerExporter implements SpanExporter { } private async save(traceId, spans: ReadableSpan[]): Promise { + console.log("fsaving 2 telemetry"); if (!telemetryServerUrl) { logger.debug( `Telemetry server is not configured, trace ${traceId} not saved!` diff --git a/js/core/src/tracing/instrumentation.ts b/js/core/src/tracing/instrumentation.ts index 34612ee2f7..d5f0c96d65 100644 --- a/js/core/src/tracing/instrumentation.ts +++ b/js/core/src/tracing/instrumentation.ts @@ -47,6 +47,7 @@ export async function newTrace( }, fn: (metadata: SpanMetadata, rootSpan: ApiSpan) => Promise ) { + console.log("newTrace") registry = (registry as HasRegistry).registry ? (registry as HasRegistry).registry : (registry as Registry); @@ -90,6 +91,8 @@ export async function runInNewSpan( }, fn: (metadata: SpanMetadata, otSpan: ApiSpan, isRoot: boolean) => Promise ): Promise { + + console.log("runINnew span") await ensureBasicTelemetryInstrumentation(); const resolvedRegistry = (registry as HasRegistry).registry ? (registry as HasRegistry).registry @@ -154,6 +157,7 @@ export async function appendSpan( metadata: SpanMetadata, labels?: Record ) { + console.log("append span") await ensureBasicTelemetryInstrumentation(); const tracer = trace.getTracer(TRACER_NAME, TRACER_VERSION); diff --git a/js/core/src/tracing/multiSpanProcessor.ts b/js/core/src/tracing/multiSpanProcessor.ts index 9693a4ec83..8ef455e739 100644 --- a/js/core/src/tracing/multiSpanProcessor.ts +++ b/js/core/src/tracing/multiSpanProcessor.ts @@ -28,10 +28,13 @@ export class MultiSpanProcessor implements SpanProcessor { constructor(private processors: SpanProcessor[]) {} forceFlush(): Promise { + console.log("multispan ") return Promise.all(this.processors.map((p) => p.forceFlush())).then(); } onStart(span: Span, parentContext: Context): void { + + console.log("multispan ") this.processors.map((p) => p.onStart(span, parentContext)); } diff --git a/js/genkit/src/genkit.ts b/js/genkit/src/genkit.ts index 2bda1cb8eb..43cfec8eb5 100644 --- a/js/genkit/src/genkit.ts +++ b/js/genkit/src/genkit.ts @@ -840,7 +840,7 @@ export function genkit(options: GenkitOptions): Genkit { return new Genkit(options); } -const shutdown = async () => { +export const shutdown = async () => { logger.info('Shutting down all Genkit servers...'); await ReflectionServer.stopAll(); process.exit(0); @@ -852,5 +852,6 @@ process.on('SIGINT', shutdown); let disableReflectionApi = false; export function __disableReflectionApi() { + console.log("disabling reflection"); disableReflectionApi = true; } diff --git a/js/genkit/src/index.ts b/js/genkit/src/index.ts index 937eb37ed7..1dbf6950f7 100644 --- a/js/genkit/src/index.ts +++ b/js/genkit/src/index.ts @@ -26,4 +26,10 @@ * @module / */ export * from './common.js'; -export { Genkit, genkit, type GenkitOptions } from './genkit.js'; +export { + Genkit, + __disableReflectionApi, + genkit, + shutdown, + type GenkitOptions, +} from './genkit.js'; diff --git a/js/pnpm-lock.yaml b/js/pnpm-lock.yaml index a19d3e0108..694b5a209a 100644 --- a/js/pnpm-lock.yaml +++ b/js/pnpm-lock.yaml @@ -1133,12 +1133,24 @@ importers: specifier: ^1.0.1 version: 1.0.1 devDependencies: + '@jest/globals': + specifier: ^29.7.0 + version: 29.7.0 + '@types/jest': + specifier: ^29.5.12 + version: 29.5.13 cross-env: specifier: ^7.0.3 version: 7.0.3 + jest: + specifier: ^29.7.0 + version: 29.7.0(@types/node@20.16.9)(ts-node@10.9.2(@types/node@20.16.9)(typescript@5.6.3)) rimraf: specifier: ^6.0.1 version: 6.0.1 + ts-jest: + specifier: ^29.1.2 + version: 29.2.5(@babel/core@7.25.7)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.25.7))(jest@29.7.0(@types/node@20.16.9)(ts-node@10.9.2(@types/node@20.16.9)(typescript@5.6.3)))(typescript@5.6.3) tsx: specifier: ^4.19.2 version: 4.19.2 @@ -1388,7 +1400,7 @@ importers: version: link:../../plugins/ollama genkitx-openai: specifier: ^0.10.1 - version: 0.10.1(@genkit-ai/ai@1.4.0)(@genkit-ai/core@1.4.0) + version: 0.10.1(@genkit-ai/ai@1.8.0-rc.1)(@genkit-ai/core@1.8.0-rc.1) devDependencies: rimraf: specifier: ^6.0.1 @@ -2520,11 +2532,11 @@ packages: '@firebase/webchannel-wrapper@1.0.3': resolution: {integrity: sha512-2xCRM9q9FlzGZCdgDMJwc0gyUkWFtkosy7Xxr6sFgQwn+wMNIWd7xIvYNauU1r64B5L5rsGKy/n9TKJ0aAFeqQ==} - '@genkit-ai/ai@1.4.0': - resolution: {integrity: sha512-s0YZ7quoYF4LYFFVnJz/3GvBmXPl8Ty9a5ZMOCB8k0xmAopiFwKEpaCMFbpIyF04EmB2U8x5/k3bjliD32eZXQ==} + '@genkit-ai/ai@1.8.0-rc.1': + resolution: {integrity: sha512-m2tqut6G5P7Q+L9gFmf2BMraUatUyA8AAJHHOEw1Hwz7fpkZwN+GexH17NGy61PYaCysDYptPrmpvFvLbsZAGw==} - '@genkit-ai/core@1.4.0': - resolution: {integrity: sha512-Y85RsvXfejH7vQOH/O8/GgaKqDeqiDDMnWNKa2Cy2ugwlsy1P5jSHkQ5wUPgCCTSwQG4eOfdmmwGpFVvNi0QXw==} + '@genkit-ai/core@1.8.0-rc.1': + resolution: {integrity: sha512-tif/+/i6fwcnn94nQGE4xS64xmoA4B8nkw0w0sIlO0afYKf31NWFbY7laELwdGyeZ5MMEUZCt/YEZ46k/m03iQ==} '@gerrit0/mini-shiki@1.24.4': resolution: {integrity: sha512-YEHW1QeAg6UmxEmswiQbOVEg1CW22b1XUD/lNTliOsu0LD0wqoyleFMnmbTp697QE0pcadQiR5cVtbbAPncvpw==} @@ -6111,6 +6123,7 @@ packages: node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} engines: {node: '>=10.5.0'} + deprecated: Use your platform's native DOMException instead node-ensure@0.0.0: resolution: {integrity: sha512-DRI60hzo2oKN1ma0ckc6nQWlHU69RH6xN0sjQTjMpChPfTYvKZdcQFfdYK2RWbJcKyUizSIy/l8OTGxMAM1QDw==} @@ -8088,9 +8101,9 @@ snapshots: '@firebase/webchannel-wrapper@1.0.3': {} - '@genkit-ai/ai@1.4.0': + '@genkit-ai/ai@1.8.0-rc.1': dependencies: - '@genkit-ai/core': 1.4.0 + '@genkit-ai/core': 1.8.0-rc.1 '@opentelemetry/api': 1.9.0 '@types/node': 20.17.17 colorette: 2.0.20 @@ -8102,7 +8115,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@genkit-ai/core@1.4.0': + '@genkit-ai/core@1.8.0-rc.1': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/context-async-hooks': 1.30.1(@opentelemetry/api@1.9.0) @@ -11096,10 +11109,10 @@ snapshots: - encoding - supports-color - genkitx-openai@0.10.1(@genkit-ai/ai@1.4.0)(@genkit-ai/core@1.4.0): + genkitx-openai@0.10.1(@genkit-ai/ai@1.8.0-rc.1)(@genkit-ai/core@1.8.0-rc.1): dependencies: - '@genkit-ai/ai': 1.4.0 - '@genkit-ai/core': 1.4.0 + '@genkit-ai/ai': 1.8.0-rc.1 + '@genkit-ai/core': 1.8.0-rc.1 openai: 4.53.0(encoding@0.1.13) zod: 3.24.1 transitivePeerDependencies: diff --git a/js/testapps/evals/jest.config.json b/js/testapps/evals/jest.config.json new file mode 100644 index 0000000000..9b9bbd5682 --- /dev/null +++ b/js/testapps/evals/jest.config.json @@ -0,0 +1,21 @@ +{ + "testEnvironment": "node", + "coveragePathIgnorePatterns": [ + "/node_modules/" + ], + "moduleFileExtensions": ["ts", "tsx", "js", "json"], + "testMatch": ["**/?(*.)+(spec|test).ts"], + "verbose": true, + "preset": "ts-jest/presets/default-esm", + "transform": { + "^.+\\.tsx?$": [ + "ts-jest", + { + "useESM": true + } + ] + }, + "moduleNameMapper": { + "^(\\.{1,2}/.*)\\.js$": "$1" + } +} diff --git a/js/testapps/evals/package.json b/js/testapps/evals/package.json index b0d002c162..a2597b5bb9 100644 --- a/js/testapps/evals/package.json +++ b/js/testapps/evals/package.json @@ -10,7 +10,9 @@ "build:clean": "rimraf ./lib", "build:watch": "tsc --watch", "dev": "tsx --watch src/index.ts", - "genkit:dev": "genkit start -- tsx --watch src/index.ts" + "genkit:dev": "genkit start -- tsx --watch src/index.ts", + "test": "cross-env NODE_OPTIONS=--experimental-vm-modules jest --detectOpenHandles", + "test:watch": "cross-env NODE_OPTIONS=--experimental-vm-modules jest --detectOpenHandles --watch" }, "keywords": [], "author": "", @@ -28,8 +30,12 @@ "pdfjs-dist-legacy": "^1.0.1" }, "devDependencies": { + "@jest/globals": "^29.7.0", + "@types/jest": "^29.5.12", "cross-env": "^7.0.3", + "jest": "^29.7.0", "rimraf": "^6.0.1", + "ts-jest": "^29.1.2", "tsx": "^4.19.2", "typescript": "^5.3.3" } diff --git a/js/testapps/evals/src/genkit.ts b/js/testapps/evals/src/genkit.ts index d36a95530e..a4c12e1e9e 100644 --- a/js/testapps/evals/src/genkit.ts +++ b/js/testapps/evals/src/genkit.ts @@ -17,18 +17,11 @@ import { devLocalVectorstore } from '@genkit-ai/dev-local-vectorstore'; import { genkitEval, GenkitMetric } from '@genkit-ai/evaluator'; import { - gemini15Flash, gemini15Pro, googleAI, textEmbeddingGecko001, } from '@genkit-ai/googleai'; -import { vertexAI } from '@genkit-ai/vertexai'; -import { - vertexAIEvaluation, - VertexAIEvaluationMetricType, -} from '@genkit-ai/vertexai/evaluation'; import { genkit } from 'genkit'; -import { langchain } from 'genkitx-langchain'; // Turn off safety checks for evaluation so that the LLM as an evaluator can // respond appropriately to potentially harmful content without error. @@ -70,37 +63,11 @@ export const ai = genkit({ }, ], }), - vertexAI({ - location: 'us-central1', - }), - vertexAIEvaluation({ - location: 'us-central1', - metrics: [ - VertexAIEvaluationMetricType.BLEU, - VertexAIEvaluationMetricType.GROUNDEDNESS, - VertexAIEvaluationMetricType.SAFETY, - { - type: VertexAIEvaluationMetricType.ROUGE, - metricSpec: { - rougeType: 'rougeLsum', - useStemmer: true, - splitSummaries: 'true', - }, - }, - ], - }), devLocalVectorstore([ { indexName: 'pdfQA', embedder: textEmbeddingGecko001, }, ]), - langchain({ - evaluators: { - criteria: ['coherence'], - labeledCriteria: ['correctness'], - judge: gemini15Flash, - }, - }), ], }); diff --git a/js/testapps/evals/src/pdf-rag.ts b/js/testapps/evals/src/pdf-rag.ts index baa2f87474..8211884a4a 100644 --- a/js/testapps/evals/src/pdf-rag.ts +++ b/js/testapps/evals/src/pdf-rag.ts @@ -23,7 +23,7 @@ import { z } from 'genkit'; import { Document } from 'genkit/retriever'; import { chunk } from 'llm-chunk'; import path from 'path'; -import { getDocument } from 'pdfjs-dist-legacy'; +import * as pdfjsDistLegacy from 'pdfjs-dist-legacy'; import { ai } from './genkit.js'; export const pdfChatRetriever = devLocalRetrieverRef('pdfQA'); @@ -100,11 +100,7 @@ export const simpleEcho = ai.defineFlow( outputSchema: z.string(), }, async (i) => { - const llmResponse = await ai.generate({ - model: gemini15Flash, - prompt: i, - }); - return llmResponse.text; + return i; } ); @@ -143,7 +139,7 @@ export const indexPdf = ai.defineFlow( ); async function extractText(filePath: string): Promise { - let doc = await getDocument(filePath).promise; + let doc = await pdfjsDistLegacy.getDocument(filePath).promise; let pdfTxt = ''; const numPages = doc.numPages; diff --git a/js/testapps/evals/src/sample.test.ts b/js/testapps/evals/src/sample.test.ts new file mode 100644 index 0000000000..7d1e73862e --- /dev/null +++ b/js/testapps/evals/src/sample.test.ts @@ -0,0 +1,28 @@ +import { __disableReflectionApi, shutdown } from 'genkit'; +import { pdfQA, simpleEcho } from './pdf-rag.js'; + +describe('Math functions', () => { + beforeAll(() => { + __disableReflectionApi(); + }); + + // afterAll(async () => { + // await shutdown(); + // }); + // test('adds 1 + 2 to equal 3', () => { + // expect(1 + 2).toBe(3); + // }); + + // test('multiplies 3 * 4 to equal 12', () => { + // expect(3 * 4).toBe(12); + // }); + + test('simpleEcho', async () => { + + expect(await simpleEcho('Hellozzz world')).toBe('Hellozzz world'); + }); + + // test('pdfQA', async () => { + // expect(await pdfQA('Can I feed milk to my cats?')).toContain('No'); + // }); +});