Skip to content

Commit 68fe9e8

Browse files
mydeamydea
authored
feat(server-utils): Implement orchestrion-based instrumentation for vercel-ai v6 (#21658)
When using orchestrion, this replaces the regular `vercelAiIntegration` with the channel-variant. This today supports vercel ai v6 and v7 (v5/v4 support TBD). Test coverage exists and everything passes. This does not add any event processors etc. anymore, everything works inside of the integrations. OTEL Spans emitted directly by vercel-ai v6 or below are ignored to avoid double instrumentation.
1 parent 3e5d2ae commit 68fe9e8

15 files changed

Lines changed: 800 additions & 48 deletions

File tree

‎.github/workflows/build.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -781,7 +781,7 @@ jobs:
781781
needs: [job_get_metadata, job_build]
782782
if: needs.job_build.outputs.changed_node_integration == 'true' || github.event_name != 'pull_request'
783783
runs-on: ubuntu-24.04
784-
timeout-minutes: 15
784+
timeout-minutes: 20
785785
strategy:
786786
fail-fast: false
787787
matrix:

‎dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import {
1919
GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE,
2020
} from '../../../../../../packages/core/src/tracing/ai/gen-ai-attributes';
2121
import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner';
22+
import { isOrchestrionEnabled } from '../../../../utils';
2223

2324
/**
2425
* Helper to match a typed attribute value in a SerializedStreamedSpan.
@@ -29,7 +30,7 @@ function attr(value: unknown) {
2930
return expect.objectContaining({ value });
3031
}
3132

32-
describe('Vercel AI integration (streaming)', () => {
33+
describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)', () => {
3334
afterAll(() => {
3435
cleanupChildProcesses();
3536
});

‎dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts‎

Lines changed: 19 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import {
1919
GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE,
2020
} from '../../../../../../packages/core/src/tracing/ai/gen-ai-attributes';
2121
import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner';
22+
import { isOrchestrionEnabled } from '../../../../utils';
2223

2324
/**
2425
* Helper to match a typed attribute value in a SerializedStreamedSpan.
@@ -29,11 +30,13 @@ function attr(value: unknown) {
2930
return expect.objectContaining({ value });
3031
}
3132

32-
describe('Vercel AI integration (streaming, V6)', () => {
33+
describe('Vercel AI integration (streaming, v6)', () => {
3334
afterAll(() => {
3435
cleanupChildProcesses();
3536
});
3637

38+
const origin = isOrchestrionEnabled() ? 'auto.vercelai.channel' : 'auto.vercelai.otel';
39+
3740
const EXPECTED_SPANS_DEFAULT_PII_FALSE = {
3841
items: expect.arrayContaining([
3942
// First span - invoke_agent for simple generateText
@@ -48,10 +51,7 @@ describe('Vercel AI integration (streaming, V6)', () => {
4851
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30),
4952
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'),
5053
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'),
51-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
52-
'vercel.ai.pipeline.name': attr('generateText'),
53-
'vercel.ai.streaming': attr(false),
54-
'vercel.ai.request.headers.user-agent': expect.objectContaining({ value: expect.any(String) }),
54+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(origin),
5555
}),
5656
}),
5757
// Second span - generate_content for simple generateText
@@ -66,9 +66,7 @@ describe('Vercel AI integration (streaming, V6)', () => {
6666
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30),
6767
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'),
6868
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'),
69-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
70-
'vercel.ai.pipeline.name': attr('generateText.doGenerate'),
71-
'vercel.ai.streaming': attr(false),
69+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(origin),
7270
}),
7371
}),
7472
// Third span - invoke_agent for explicit telemetry generateText
@@ -82,7 +80,7 @@ describe('Vercel AI integration (streaming, V6)', () => {
8280
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30),
8381
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'),
8482
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'),
85-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
83+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(origin),
8684
}),
8785
}),
8886
// Fourth span - tool call invoke_agent
@@ -96,7 +94,7 @@ describe('Vercel AI integration (streaming, V6)', () => {
9694
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40),
9795
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'),
9896
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'),
99-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
97+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(origin),
10098
}),
10199
}),
102100
// Fifth span - tool call generate_content
@@ -110,7 +108,7 @@ describe('Vercel AI integration (streaming, V6)', () => {
110108
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40),
111109
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'),
112110
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'),
113-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
111+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(origin),
114112
}),
115113
}),
116114
// Sixth span - execute_tool
@@ -124,7 +122,7 @@ describe('Vercel AI integration (streaming, V6)', () => {
124122
[GEN_AI_TOOL_TYPE_ATTRIBUTE]: attr('function'),
125123
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('execute_tool'),
126124
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.execute_tool'),
127-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
125+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(origin),
128126
}),
129127
}),
130128
]),
@@ -149,9 +147,7 @@ describe('Vercel AI integration (streaming, V6)', () => {
149147
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30),
150148
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'),
151149
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'),
152-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
153-
'vercel.ai.pipeline.name': attr('generateText'),
154-
'vercel.ai.streaming': attr(false),
150+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(origin),
155151
}),
156152
}),
157153
// Second span - generate_content with input/output messages
@@ -170,7 +166,7 @@ describe('Vercel AI integration (streaming, V6)', () => {
170166
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30),
171167
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'),
172168
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'),
173-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
169+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(origin),
174170
}),
175171
}),
176172
// Third span - explicit telemetry invoke_agent with messages
@@ -188,7 +184,7 @@ describe('Vercel AI integration (streaming, V6)', () => {
188184
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30),
189185
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'),
190186
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'),
191-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
187+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(origin),
192188
}),
193189
}),
194190
// Fourth span - tool call invoke_agent with messages (V6: no text part, only tool_call)
@@ -208,7 +204,7 @@ describe('Vercel AI integration (streaming, V6)', () => {
208204
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40),
209205
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'),
210206
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'),
211-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
207+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(origin),
212208
}),
213209
}),
214210
// Fifth span - tool call generate_content with available_tools
@@ -225,7 +221,7 @@ describe('Vercel AI integration (streaming, V6)', () => {
225221
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40),
226222
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'),
227223
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'),
228-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
224+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(origin),
229225
}),
230226
}),
231227
// Sixth span - execute_tool with description and input/output
@@ -241,7 +237,7 @@ describe('Vercel AI integration (streaming, V6)', () => {
241237
[GEN_AI_TOOL_TYPE_ATTRIBUTE]: attr('function'),
242238
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('execute_tool'),
243239
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.execute_tool'),
244-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
240+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(origin),
245241
}),
246242
}),
247243
]),
@@ -254,7 +250,7 @@ describe('Vercel AI integration (streaming, V6)', () => {
254250
attributes: expect.objectContaining({
255251
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'),
256252
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'),
257-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
253+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(origin),
258254
}),
259255
}),
260256
expect.objectContaining({
@@ -267,7 +263,7 @@ describe('Vercel AI integration (streaming, V6)', () => {
267263
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40),
268264
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'),
269265
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'),
270-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
266+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(origin),
271267
}),
272268
}),
273269
expect.objectContaining({
@@ -279,7 +275,7 @@ describe('Vercel AI integration (streaming, V6)', () => {
279275
[GEN_AI_TOOL_TYPE_ATTRIBUTE]: attr('function'),
280276
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('execute_tool'),
281277
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.execute_tool'),
282-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
278+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(origin),
283279
}),
284280
}),
285281
]),

‎dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,9 @@ import {
2222
GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE,
2323
} from '../../../../../packages/core/src/tracing/ai/gen-ai-attributes';
2424
import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner';
25+
import { isOrchestrionEnabled } from '../../../utils';
2526

26-
describe('Vercel AI integration', () => {
27+
describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (v4)', () => {
2728
afterAll(() => {
2829
cleanupChildProcesses();
2930
});

‎dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,9 @@ import {
1818
GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE,
1919
} from '../../../../../../packages/core/src/tracing/ai/gen-ai-attributes';
2020
import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner';
21+
import { isOrchestrionEnabled } from '../../../../utils';
2122

22-
describe('Vercel AI integration (V5)', () => {
23+
describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (v5)', () => {
2324
afterAll(() => {
2425
cleanupChildProcesses();
2526
});

‎dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-concurrent-stream.mjs‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
import * as Sentry from '@sentry/node';
2+
import { streamText } from 'ai';
3+
import { MockLanguageModelV3, simulateReadableStream } from 'ai/test';
4+
5+
function makeStreamModel(text) {
6+
return new MockLanguageModelV3({
7+
doStream: async () => ({
8+
stream: simulateReadableStream({
9+
chunks: [
10+
{ type: 'stream-start', warnings: [] },
11+
{ type: 'text-start', id: '0' },
12+
{ type: 'text-delta', id: '0', delta: text },
13+
{ type: 'text-end', id: '0' },
14+
{
15+
type: 'finish',
16+
finishReason: { unified: 'stop', raw: 'stop' },
17+
usage: {
18+
inputTokens: { total: 10, noCache: 10, cached: 0 },
19+
outputTokens: { total: 20, noCache: 20, cached: 0 },
20+
totalTokens: { total: 30, noCache: 30, cached: 0 },
21+
},
22+
},
23+
],
24+
}),
25+
}),
26+
});
27+
}
28+
29+
async function consume(result) {
30+
for await (const _part of result.fullStream) {
31+
void _part;
32+
}
33+
}
34+
35+
async function run() {
36+
// A single model instance shared by two *concurrent* streamText calls. The shared model carries
37+
// only a single captured-parent slot, so naive parent tracking could attribute a model call to
38+
// whichever operation resolved the model last. Each `generate_content` (doStream) must land under
39+
// its own `invoke_agent`, and both invoke_agents under `main`.
40+
const sharedModel = makeStreamModel('shared!');
41+
42+
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
43+
// Start both operations before consuming either, so both resolve the shared model first.
44+
const streams = [
45+
streamText({
46+
experimental_telemetry: { isEnabled: true },
47+
model: sharedModel,
48+
prompt: 'Concurrent stream A?',
49+
}),
50+
streamText({
51+
experimental_telemetry: { isEnabled: true },
52+
model: sharedModel,
53+
prompt: 'Concurrent stream B?',
54+
}),
55+
];
56+
57+
await Promise.all(streams.map(consume));
58+
});
59+
}
60+
61+
run();

‎dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-embeddings.mjs‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
import * as Sentry from '@sentry/node';
2+
import { embed, embedMany } from 'ai';
3+
import { MockEmbeddingModelV3 } from 'ai/test';
4+
5+
async function run() {
6+
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
7+
// Single embedding
8+
await embed({
9+
model: new MockEmbeddingModelV3({
10+
doEmbed: async () => ({
11+
embeddings: [[0.1, 0.2, 0.3]],
12+
usage: { tokens: 10 },
13+
}),
14+
}),
15+
value: 'Embedding test!',
16+
});
17+
18+
// Multiple embeddings
19+
await embedMany({
20+
model: new MockEmbeddingModelV3({
21+
maxEmbeddingsPerCall: 5,
22+
doEmbed: async () => ({
23+
embeddings: [
24+
[0.1, 0.2, 0.3],
25+
[0.4, 0.5, 0.6],
26+
],
27+
usage: { tokens: 20 },
28+
}),
29+
}),
30+
values: ['First input', 'Second input'],
31+
});
32+
});
33+
}
34+
35+
run();

‎dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario.mjs‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,9 @@ async function run() {
2121
prompt: 'Where is the first span?',
2222
});
2323

24-
// This span should have input and output prompts attached because telemetry is explicitly enabled.
25-
// Vercel AI treats this as recording inputs and outputs by default
26-
// but in v7, we do not have access to this, so we do not override data collection settings for this
24+
// Recording and inputs explicitly enabled
2725
await generateText({
28-
experimental_telemetry: { isEnabled: true },
26+
experimental_telemetry: { isEnabled: true, recordInputs: true, recordOutputs: true },
2927
model: new MockLanguageModelV3({
3028
doGenerate: async () => ({
3129
finishReason: { unified: 'stop', raw: 'stop' },

0 commit comments

Comments
 (0)