Skip to content

Commit d09d65d

Browse files
brendan-kellambrendan-kellam
authored
fix(ask): Extract reasoning tokens for openai compatible models (#582)
1 parent 727a6da commit d09d65d

File tree

10 files changed

+113
-2
lines changed

10 files changed

+113
-2
lines changed

‎CHANGELOG.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
77

88
## [Unreleased]
99

10+
### Fixed
11+
- [ask sb] Fixed issue where reasoning tokens would appear in `text` content for openai compatible models. [#582](https://github.com/sourcebot-dev/sourcebot/pull/582)
12+
1013
## [4.8.1] - 2025-10-29
1114

1215
### Fixed

‎docs/docs/configuration/language-model-providers.mdx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -292,6 +292,7 @@ The OpenAI compatible provider allows you to use any model that is compatible wi
292292

293293
<Accordion title="Troubleshooting">
294294
- When using [llama.cpp](https://github.com/ggml-org/llama.cpp), if you hit "Failed after 3 attempts. Last error: tools param requires --jinja flag", add the `--jinja` flag to your `llama-server` command.
295+
- If you're seeing the LLM outputing reasoning tokens wrapped in XML tags (e.g., `<reasoning>`, `<thinking>`, etc.), you can configure the `reasoningTag` parameter to the name of the tag (without angle brackets). This parameter defaults to `think`.
295296
</Accordion>
296297

297298
### OpenRouter

‎docs/snippets/schemas/v3/index.schema.mdx‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2633,6 +2633,16 @@
26332633
}
26342634
},
26352635
"additionalProperties": false
2636+
},
2637+
"reasoningTag": {
2638+
"type": "string",
2639+
"description": "The name of the XML tag to extract reasoning from (without angle brackets). Defaults to `think`.",
2640+
"default": "think",
2641+
"examples": [
2642+
"think",
2643+
"thinking",
2644+
"reasoning"
2645+
]
26362646
}
26372647
},
26382648
"required": [
@@ -4052,6 +4062,16 @@
40524062
}
40534063
},
40544064
"additionalProperties": false
4065+
},
4066+
"reasoningTag": {
4067+
"type": "string",
4068+
"description": "The name of the XML tag to extract reasoning from (without angle brackets). Defaults to `think`.",
4069+
"default": "think",
4070+
"examples": [
4071+
"think",
4072+
"thinking",
4073+
"reasoning"
4074+
]
40554075
}
40564076
},
40574077
"required": [

‎docs/snippets/schemas/v3/languageModel.schema.mdx‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1202,6 +1202,16 @@
12021202
}
12031203
},
12041204
"additionalProperties": false
1205+
},
1206+
"reasoningTag": {
1207+
"type": "string",
1208+
"description": "The name of the XML tag to extract reasoning from (without angle brackets). Defaults to `think`.",
1209+
"default": "think",
1210+
"examples": [
1211+
"think",
1212+
"thinking",
1213+
"reasoning"
1214+
]
12051215
}
12061216
},
12071217
"required": [
@@ -2621,6 +2631,16 @@
26212631
}
26222632
},
26232633
"additionalProperties": false
2634+
},
2635+
"reasoningTag": {
2636+
"type": "string",
2637+
"description": "The name of the XML tag to extract reasoning from (without angle brackets). Defaults to `think`.",
2638+
"default": "think",
2639+
"examples": [
2640+
"think",
2641+
"thinking",
2642+
"reasoning"
2643+
]
26242644
}
26252645
},
26262646
"required": [

‎packages/schemas/src/v3/index.schema.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2632,6 +2632,16 @@ const schema = {
26322632
}
26332633
},
26342634
"additionalProperties": false
2635+
},
2636+
"reasoningTag": {
2637+
"type": "string",
2638+
"description": "The name of the XML tag to extract reasoning from (without angle brackets). Defaults to `think`.",
2639+
"default": "think",
2640+
"examples": [
2641+
"think",
2642+
"thinking",
2643+
"reasoning"
2644+
]
26352645
}
26362646
},
26372647
"required": [
@@ -4051,6 +4061,16 @@ const schema = {
40514061
}
40524062
},
40534063
"additionalProperties": false
4064+
},
4065+
"reasoningTag": {
4066+
"type": "string",
4067+
"description": "The name of the XML tag to extract reasoning from (without angle brackets). Defaults to `think`.",
4068+
"default": "think",
4069+
"examples": [
4070+
"think",
4071+
"thinking",
4072+
"reasoning"
4073+
]
40544074
}
40554075
},
40564076
"required": [

‎packages/schemas/src/v3/index.type.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -974,6 +974,10 @@ export interface OpenAICompatibleLanguageModel {
974974
baseUrl: string;
975975
headers?: LanguageModelHeaders;
976976
queryParams?: LanguageModelQueryParams;
977+
/**
978+
* The name of the XML tag to extract reasoning from (without angle brackets). Defaults to `think`.
979+
*/
980+
reasoningTag?: string;
977981
}
978982
/**
979983
* Optional query parameters to include in the request url.

‎packages/schemas/src/v3/languageModel.schema.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1201,6 +1201,16 @@ const schema = {
12011201
}
12021202
},
12031203
"additionalProperties": false
1204+
},
1205+
"reasoningTag": {
1206+
"type": "string",
1207+
"description": "The name of the XML tag to extract reasoning from (without angle brackets). Defaults to `think`.",
1208+
"default": "think",
1209+
"examples": [
1210+
"think",
1211+
"thinking",
1212+
"reasoning"
1213+
]
12041214
}
12051215
},
12061216
"required": [
@@ -2620,6 +2630,16 @@ const schema = {
26202630
}
26212631
},
26222632
"additionalProperties": false
2633+
},
2634+
"reasoningTag": {
2635+
"type": "string",
2636+
"description": "The name of the XML tag to extract reasoning from (without angle brackets). Defaults to `think`.",
2637+
"default": "think",
2638+
"examples": [
2639+
"think",
2640+
"thinking",
2641+
"reasoning"
2642+
]
26232643
}
26242644
},
26252645
"required": [

‎packages/schemas/src/v3/languageModel.type.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -453,6 +453,10 @@ export interface OpenAICompatibleLanguageModel {
453453
baseUrl: string;
454454
headers?: LanguageModelHeaders;
455455
queryParams?: LanguageModelQueryParams;
456+
/**
457+
* The name of the XML tag to extract reasoning from (without angle brackets). Defaults to `think`.
458+
*/
459+
reasoningTag?: string;
456460
}
457461
/**
458462
* Optional query parameters to include in the request url.

‎packages/web/src/features/chat/actions.ts‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ import { ChatVisibility, OrgRole, Prisma, PrismaClient } from "@sourcebot/db";
2525
import { LanguageModel } from "@sourcebot/schemas/v3/languageModel.type";
2626
import { Token } from "@sourcebot/schemas/v3/shared.type";
2727
import { loadConfig } from "@sourcebot/shared";
28-
import { generateText, JSONValue } from "ai";
28+
import { generateText, JSONValue, extractReasoningMiddleware, wrapLanguageModel } from "ai";
2929
import fs from 'fs';
3030
import { StatusCodes } from "http-status-codes";
3131
import path from 'path';
@@ -568,8 +568,17 @@ export const _getAISDKLanguageModelAndOptions = async (config: LanguageModel, or
568568
: undefined,
569569
});
570570

571-
return {
571+
const model = wrapLanguageModel({
572572
model: openai.chatModel(modelId),
573+
middleware: [
574+
extractReasoningMiddleware({
575+
tagName: config.reasoningTag ?? 'think',
576+
}),
577+
]
578+
});
579+
580+
return {
581+
model,
573582
}
574583
}
575584
case 'openrouter': {

‎schemas/v3/languageModel.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -425,6 +425,16 @@
425425
},
426426
"queryParams": {
427427
"$ref": "./shared.json#/definitions/LanguageModelQueryParams"
428+
},
429+
"reasoningTag": {
430+
"type": "string",
431+
"description": "The name of the XML tag to extract reasoning from (without angle brackets). Defaults to `think`.",
432+
"default": "think",
433+
"examples": [
434+
"think",
435+
"thinking",
436+
"reasoning"
437+
]
428438
}
429439
},
430440
"required": [

0 commit comments

Comments
 (0)