Skip to content

fix(e2e): stabilize canary tests for langchain version drift and anthropic timeout #1707

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Luca Forstner (lforst) merged 3 commits into from
Mar 31, 2026
Merged

fix(e2e): stabilize canary tests for langchain version drift and anthropic timeout #1707

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1b2e62c
fix(e2e): stabilize canary tests for langchain version drift and anth…
Qard Mar 30, 2026
386d195
fix(e2e): Fix remaining canary snapshot failures
Qard Mar 31, 2026
a528a7b
fix(e2e): Sort ai-sdk-otel-export spans before snapshotting
Qard Mar 31, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v5.otel-spans.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,54 +1,54 @@
[
{
"hasParent": false,
"name": "ai.generateText.doGenerate"
"name": "ai.generateText"
},
{
"hasParent": false,
"name": "ai.generateText"
},
{
"hasParent": false,
"name": "ai.streamText.doStream"
"name": "ai.generateText.doGenerate"
},
{
"hasParent": false,
"name": "ai.streamText"
"name": "ai.generateText.doGenerate"
},
{
"hasParent": false,
"name": "ai.generateText.doGenerate"
},
{
"hasParent": false,
"name": "ai.toolCall"
"name": "ai.generateText.doGenerate"
},
{
"hasParent": false,
"name": "ai.generateText.doGenerate"
},
{
"hasParent": false,
"name": "ai.toolCall"
"name": "ai.streamText"
},
{
"hasParent": false,
"name": "ai.generateText.doGenerate"
"name": "ai.streamText.doStream"
},
{
"hasParent": false,
"name": "ai.toolCall"
},
{
"hasParent": false,
"name": "ai.generateText.doGenerate"
"name": "ai.toolCall"
},
{
"hasParent": false,
"name": "ai.toolCall"
},
{
"hasParent": false,
"name": "ai.generateText"
"name": "ai.toolCall"
}
]
16 changes: 8 additions & 8 deletions e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v6.otel-spans.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,54 +1,54 @@
[
{
"hasParent": false,
"name": "ai.generateText.doGenerate"
"name": "ai.generateText"
},
{
"hasParent": false,
"name": "ai.generateText"
},
{
"hasParent": false,
"name": "ai.streamText.doStream"
"name": "ai.generateText.doGenerate"
},
{
"hasParent": false,
"name": "ai.streamText"
"name": "ai.generateText.doGenerate"
},
{
"hasParent": false,
"name": "ai.generateText.doGenerate"
},
{
"hasParent": false,
"name": "ai.toolCall"
"name": "ai.generateText.doGenerate"
},
{
"hasParent": false,
"name": "ai.generateText.doGenerate"
},
{
"hasParent": false,
"name": "ai.toolCall"
"name": "ai.streamText"
},
{
"hasParent": false,
"name": "ai.generateText.doGenerate"
"name": "ai.streamText.doStream"
},
{
"hasParent": false,
"name": "ai.toolCall"
},
{
"hasParent": false,
"name": "ai.generateText.doGenerate"
"name": "ai.toolCall"
},
{
"hasParent": false,
"name": "ai.toolCall"
},
{
"hasParent": false,
"name": "ai.generateText"
"name": "ai.toolCall"
}
]
15 changes: 11 additions & 4 deletions e2e/scenarios/ai-sdk-otel-export/scenario.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,10 +119,17 @@ for (const scenario of scenarios) {

// Snapshot the span names and key structure (not full payloads, since
// response content and token counts are non-deterministic).
const spanSummary = allSpans.map((span) => ({
hasParent: !!span.parentSpanId,
name: span.name,
}));
// Sort for determinism — OTel spans arrive in non-deterministic order.
const spanSummary = allSpans
.map((span) => ({
hasParent: !!span.parentSpanId,
name: span.name,
}))
.sort((a, b) =>
a.name !== b.name
? a.name.localeCompare(b.name)
: Number(a.hasParent) - Number(b.hasParent),
);
await expect(formatJsonFileSnapshot(spanSummary)).toMatchFileSnapshot(
resolveFileSnapshotPath(
import.meta.url,
Expand Down
2 changes: 1 addition & 1 deletion e2e/scenarios/anthropic-instrumentation/scenario.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import { defineAnthropicInstrumentationAssertions } from "./assertions";
const scenarioDir = await prepareScenarioDir({
scenarioDir: resolveScenarioDir(import.meta.url),
});
const TIMEOUT_MS = 90_000;
const TIMEOUT_MS = 150_000;
const anthropicScenarios = await Promise.all(
[
{
Expand Down
27 changes: 0 additions & 27 deletions e2e/scenarios/wrap-langchain-js-traces/__snapshots__/log-payloads.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,10 +126,6 @@
"ls_model_type": "chat",
"ls_provider": "openai",
"ls_temperature": 0,
"max_tokens": 16,
"model": "gpt-4o-mini",
"stream": false,
"temperature": 0,
"versions": {
"@langchain/core": "<langchain-version>",
"@langchain/openai": "<langchain-version>"
Expand Down Expand Up @@ -704,10 +700,6 @@
"ls_model_type": "chat",
"ls_provider": "openai",
"ls_temperature": 0,
"max_tokens": 32,
"model": "gpt-4o-mini",
"stream": false,
"temperature": 0,
"versions": {
"@langchain/core": "<langchain-version>",
"@langchain/openai": "<langchain-version>"
Expand Down Expand Up @@ -973,13 +965,6 @@
"ls_model_type": "chat",
"ls_provider": "openai",
"ls_temperature": 0,
"max_tokens": 32,
"model": "gpt-4o-mini",
"stream": true,
"stream_options": {
"include_usage": true
},
"temperature": 0,
"versions": {
"@langchain/core": "<langchain-version>",
"@langchain/openai": "<langchain-version>"
Expand Down Expand Up @@ -1265,10 +1250,6 @@
"ls_model_type": "chat",
"ls_provider": "openai",
"ls_temperature": 0,
"max_tokens": 128,
"model": "gpt-4o-mini",
"stream": false,
"temperature": 0,
"versions": {
"@langchain/core": "<langchain-version>",
"@langchain/openai": "<langchain-version>"
Expand Down Expand Up @@ -1606,10 +1587,6 @@
"ls_model_type": "chat",
"ls_provider": "openai",
"ls_temperature": 0,
"max_tokens": 128,
"model": "gpt-4o-mini",
"stream": false,
"temperature": 0,
"versions": {
"@langchain/core": "<langchain-version>",
"@langchain/openai": "<langchain-version>"
Expand Down Expand Up @@ -1962,10 +1939,6 @@
"ls_model_type": "chat",
"ls_provider": "openai",
"ls_temperature": 0,
"max_tokens": 128,
"model": "gpt-4o-mini",
"stream": false,
"temperature": 0,
"versions": {
"@langchain/core": "<langchain-version>",
"@langchain/openai": "<langchain-version>"
Expand Down
20 changes: 20 additions & 0 deletions e2e/scenarios/wrap-langchain-js-traces/assertions.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,17 @@ function normalizeToolCallIds(obj: unknown): void {
}
}

// Fields that older @langchain/openai included in the ls_* metadata block but
// newer versions removed. Normalize them out so the snapshot is stable across
// both locked and canary (latest) langchain versions.
const LANGCHAIN_LS_VOLATILE_KEYS = new Set([
"max_tokens",
"model",
"stream",
"stream_options",
"temperature",
]);

function normalizeLangchainVersions(obj: unknown): void {
if (!obj || typeof obj !== "object") return;

Expand All @@ -161,6 +172,15 @@ function normalizeLangchainVersions(obj: unknown): void {
}
}

// If this object is the ls_* metadata block (identified by the presence of
// any `ls_` key), remove volatile keys that newer langchain drops.
const hasLsKey = Object.keys(record).some((k) => k.startsWith("ls_"));
if (hasLsKey) {
for (const key of LANGCHAIN_LS_VOLATILE_KEYS) {
delete record[key];
}
}

for (const value of Object.values(record)) {
if (typeof value === "object" && value !== null) {
normalizeLangchainVersions(value);
Expand Down
Loading