|
| 1 | +import litellm |
| 2 | +import pytest |
| 3 | +from litellm.types.utils import ChatCompletionMessageToolCall, Choices, Function, Message, ModelResponse, Usage |
| 4 | + |
| 5 | +from agents.extensions.models.litellm_model import LitellmModel |
| 6 | +from agents.model_settings import ModelSettings |
| 7 | +from agents.models.chatcmpl_converter import Converter |
| 8 | +from agents.models.interface import ModelTracing |
| 9 | + |
| 10 | + |
| 11 | +@pytest.mark.allow_call_model_methods |
| 12 | +@pytest.mark.asyncio |
| 13 | +async def test_deepseek_reasoning_content_preserved_in_tool_calls(monkeypatch): |
| 14 | + """ |
| 15 | + Ensure DeepSeek reasoning_content is preserved when converting items to messages. |
| 16 | +
|
| 17 | + DeepSeek requires reasoning_content field in assistant messages with tool_calls. |
| 18 | + This test verifies that reasoning content from reasoning items is correctly |
| 19 | + extracted and added to assistant messages during conversion. |
| 20 | + """ |
| 21 | + # Capture the messages sent to the model |
| 22 | + captured_calls: list[dict] = [] |
| 23 | + |
| 24 | + async def fake_acompletion(model, messages=None, **kwargs): |
| 25 | + captured_calls.append({"model": model, "messages": messages, **kwargs}) |
| 26 | + |
| 27 | + # First call: model returns reasoning_content + tool_call |
| 28 | + if len(captured_calls) == 1: |
| 29 | + tool_call = ChatCompletionMessageToolCall( |
| 30 | + id="call_123", |
| 31 | + type="function", |
| 32 | + function=Function(name="get_weather", arguments='{"city": "Tokyo"}'), |
| 33 | + ) |
| 34 | + msg = Message( |
| 35 | + role="assistant", |
| 36 | + content=None, |
| 37 | + tool_calls=[tool_call], |
| 38 | + ) |
| 39 | + # DeepSeek adds reasoning_content to the message |
| 40 | + msg.reasoning_content = "Let me think about getting the weather for Tokyo..." |
| 41 | + |
| 42 | + choice = Choices(index=0, message=msg) |
| 43 | + return ModelResponse(choices=[choice], usage=Usage(100, 50, 150)) |
| 44 | + |
| 45 | + # Second call: model returns final response |
| 46 | + msg = Message(role="assistant", content="The weather in Tokyo is sunny.") |
| 47 | + choice = Choices(index=0, message=msg) |
| 48 | + return ModelResponse(choices=[choice], usage=Usage(100, 50, 150)) |
| 49 | + |
| 50 | + monkeypatch.setattr(litellm, "acompletion", fake_acompletion) |
| 51 | + |
| 52 | + model = LitellmModel(model="deepseek/deepseek-reasoner") |
| 53 | + |
| 54 | + # First call: get the tool call response |
| 55 | + first_response = await model.get_response( |
| 56 | + system_instructions="You are a helpful assistant.", |
| 57 | + input="What's the weather in Tokyo?", |
| 58 | + model_settings=ModelSettings(), |
| 59 | + tools=[], # We'll simulate the tool response manually |
| 60 | + output_schema=None, |
| 61 | + handoffs=[], |
| 62 | + tracing=ModelTracing.DISABLED, |
| 63 | + ) |
| 64 | + |
| 65 | + assert len(first_response.output) >= 1 |
| 66 | + |
| 67 | + input_items = [] |
| 68 | + input_items.append({"role": "user", "content": "What's the weather in Tokyo?"}) |
| 69 | + |
| 70 | + for item in first_response.output: |
| 71 | + if hasattr(item, "model_dump"): |
| 72 | + input_items.append(item.model_dump()) |
| 73 | + else: |
| 74 | + input_items.append(item) |
| 75 | + |
| 76 | + input_items.append({ |
| 77 | + "type": "function_call_output", |
| 78 | + "call_id": "call_123", |
| 79 | + "output": "The weather in Tokyo is sunny.", |
| 80 | + }) |
| 81 | + |
| 82 | + messages = Converter.items_to_messages( |
| 83 | + input_items, |
| 84 | + model="deepseek/deepseek-reasoner", |
| 85 | + ) |
| 86 | + |
| 87 | + assistant_messages_with_tool_calls = [ |
| 88 | + m for m in messages |
| 89 | + if isinstance(m, dict) and m.get("role") == "assistant" and m.get("tool_calls") |
| 90 | + ] |
| 91 | + |
| 92 | + assert len(assistant_messages_with_tool_calls) > 0 |
| 93 | + assistant_msg = assistant_messages_with_tool_calls[0] |
| 94 | + assert "reasoning_content" in assistant_msg |
| 95 | + |
| 96 | + |
| 97 | +@pytest.mark.allow_call_model_methods |
| 98 | +@pytest.mark.asyncio |
| 99 | +async def test_deepseek_reasoning_content_in_multi_turn_conversation(monkeypatch): |
| 100 | + """ |
| 101 | + Verify reasoning_content is included in assistant messages during multi-turn conversations. |
| 102 | +
|
| 103 | + When DeepSeek returns reasoning_content with tool_calls, subsequent API calls must |
| 104 | + include the reasoning_content field in the assistant message to avoid 400 errors. |
| 105 | + """ |
| 106 | + captured_calls: list[dict] = [] |
| 107 | + |
| 108 | + async def fake_acompletion(model, messages=None, **kwargs): |
| 109 | + captured_calls.append({"model": model, "messages": messages, **kwargs}) |
| 110 | + |
| 111 | + # First call: model returns reasoning_content + tool_call |
| 112 | + if len(captured_calls) == 1: |
| 113 | + tool_call = ChatCompletionMessageToolCall( |
| 114 | + id="call_weather_123", |
| 115 | + type="function", |
| 116 | + function=Function(name="get_weather", arguments='{"city": "Tokyo"}'), |
| 117 | + ) |
| 118 | + msg = Message( |
| 119 | + role="assistant", |
| 120 | + content=None, |
| 121 | + tool_calls=[tool_call], |
| 122 | + ) |
| 123 | + # DeepSeek adds reasoning_content |
| 124 | + msg.reasoning_content = "I need to get the weather for Tokyo first." |
| 125 | + choice = Choices(index=0, message=msg) |
| 126 | + return ModelResponse(choices=[choice], usage=Usage(100, 50, 150)) |
| 127 | + |
| 128 | + # Second call: check if reasoning_content was in the request |
| 129 | + # In real DeepSeek API, this would fail with 400 if reasoning_content is missing |
| 130 | + msg = Message(role="assistant", content="Based on my findings, the weather in Tokyo is sunny.") |
| 131 | + choice = Choices(index=0, message=msg) |
| 132 | + return ModelResponse(choices=[choice], usage=Usage(100, 50, 150)) |
| 133 | + |
| 134 | + monkeypatch.setattr(litellm, "acompletion", fake_acompletion) |
| 135 | + |
| 136 | + model = LitellmModel(model="deepseek/deepseek-reasoner") |
| 137 | + |
| 138 | + # First call |
| 139 | + first_response = await model.get_response( |
| 140 | + system_instructions="You are a helpful assistant.", |
| 141 | + input="What's the weather in Tokyo?", |
| 142 | + model_settings=ModelSettings(), |
| 143 | + tools=[], |
| 144 | + output_schema=None, |
| 145 | + handoffs=[], |
| 146 | + tracing=ModelTracing.DISABLED, |
| 147 | + ) |
| 148 | + |
| 149 | + input_items = [] |
| 150 | + input_items.append({"role": "user", "content": "What's the weather in Tokyo?"}) |
| 151 | + |
| 152 | + for item in first_response.output: |
| 153 | + if hasattr(item, "model_dump"): |
| 154 | + input_items.append(item.model_dump()) |
| 155 | + else: |
| 156 | + input_items.append(item) |
| 157 | + |
| 158 | + input_items.append({ |
| 159 | + "type": "function_call_output", |
| 160 | + "call_id": "call_weather_123", |
| 161 | + "output": "The weather in Tokyo is sunny and 22°C.", |
| 162 | + }) |
| 163 | + |
| 164 | + await model.get_response( |
| 165 | + system_instructions="You are a helpful assistant.", |
| 166 | + input=input_items, |
| 167 | + model_settings=ModelSettings(), |
| 168 | + tools=[], |
| 169 | + output_schema=None, |
| 170 | + handoffs=[], |
| 171 | + tracing=ModelTracing.DISABLED, |
| 172 | + ) |
| 173 | + |
| 174 | + assert len(captured_calls) == 2 |
| 175 | + |
| 176 | + second_call_messages = captured_calls[1]["messages"] |
| 177 | + |
| 178 | + assistant_with_tools = None |
| 179 | + for msg in second_call_messages: |
| 180 | + if isinstance(msg, dict) and msg.get("role") == "assistant" and msg.get("tool_calls"): |
| 181 | + assistant_with_tools = msg |
| 182 | + break |
| 183 | + |
| 184 | + assert assistant_with_tools is not None |
| 185 | + assert "reasoning_content" in assistant_with_tools |
| 186 | + |
| 187 | + |
| 188 | +def test_deepseek_reasoning_content_with_openai_chatcompletions_path(): |
| 189 | + """ |
| 190 | + Verify reasoning_content works when using OpenAIChatCompletionsModel. |
| 191 | +
|
| 192 | + This ensures the fix works for both LiteLLM and OpenAI ChatCompletions code paths. |
| 193 | + """ |
| 194 | + from agents.models.chatcmpl_converter import Converter |
| 195 | + |
| 196 | + input_items = [ |
| 197 | + {"role": "user", "content": "What's the weather in Paris?"}, |
| 198 | + { |
| 199 | + "id": "__fake_id__", |
| 200 | + "summary": [{"text": "I need to check the weather in Paris.", "type": "summary_text"}], |
| 201 | + "type": "reasoning", |
| 202 | + "content": None, |
| 203 | + "encrypted_content": None, |
| 204 | + "status": None, |
| 205 | + "provider_data": {"model": "deepseek-reasoner", "response_id": "chatcmpl-test"}, |
| 206 | + }, |
| 207 | + { |
| 208 | + "arguments": '{"city": "Paris"}', |
| 209 | + "call_id": "call_weather_456", |
| 210 | + "name": "get_weather", |
| 211 | + "type": "function_call", |
| 212 | + "id": "__fake_id__", |
| 213 | + "status": None, |
| 214 | + "provider_data": {"model": "deepseek-reasoner"}, |
| 215 | + }, |
| 216 | + { |
| 217 | + "type": "function_call_output", |
| 218 | + "call_id": "call_weather_456", |
| 219 | + "output": "The weather in Paris is cloudy and 15°C.", |
| 220 | + }, |
| 221 | + ] |
| 222 | + |
| 223 | + messages = Converter.items_to_messages( |
| 224 | + input_items, |
| 225 | + model="deepseek-reasoner", |
| 226 | + ) |
| 227 | + |
| 228 | + assistant_with_tools = None |
| 229 | + for msg in messages: |
| 230 | + if isinstance(msg, dict) and msg.get("role") == "assistant" and msg.get("tool_calls"): |
| 231 | + assistant_with_tools = msg |
| 232 | + break |
| 233 | + |
| 234 | + assert assistant_with_tools is not None |
| 235 | + assert "reasoning_content" in assistant_with_tools |
| 236 | + assert assistant_with_tools["reasoning_content"] == "I need to check the weather in Paris." |
0 commit comments