Reasoning support for evaluators

sdk/evaluation/azure-ai-evaluation/CHANGELOG.md

@@ -5,11 +5,19 @@
 ### Breaking Changes
 
 ### Features Added
+
 - Added support for user-supplied tags in the `evaluate` function. Tags are key-value pairs that can be used for experiment tracking, A/B testing, filtering, and organizing evaluation runs. The function accepts a `tags` parameter.
 - Enhanced `GroundednessEvaluator` to support AI agent evaluation with tool calls. The evaluator now accepts agent response data containing tool calls and can extract context from `file_search` tool results for groundedness assessment. This enables evaluation of AI agents that use tools to retrieve information and generate responses. Note: Agent groundedness evaluation is currently supported only when the `file_search` tool is used.
 
 ### Bugs Fixed
 
+- [Bug](https://github.com/Azure/azure-sdk-for-python/issues/39909): Added `is_reasoning_model` keyword parameter to all evaluators
+    (`SimilarityEvaluator`, `RelevanceEvaluator`, `CoherenceEvaluator`, `FluencyEvaluator`,
+    `RetrievalEvaluator`, `GroundednessEvaluator`, `IntentResolutionEvaluator`,
+    `ResponseCompletenessEvaluator`, `TaskAdherenceEvaluator`, `ToolCallAccuracyEvaluator`).
+    When set, evaluator configuration is adjusted appropriately for reasoning models.
+    `QAEvaluator` now propagates this parameter to its child evaluators.
+
 ### Other Changes
 
 ## 1.10.0 (2025-07-31)

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/code_client.py

@@ -178,8 +178,6 @@ def get_metrics(self, client_run: BatchClientRun) -> Dict[str, Any]:
         run = self._get_result(client_run)
         try:
             aggregated_metrics = run.get_aggregated_metrics()
-            print("Aggregated metrics")
-            print(aggregated_metrics)
         except Exception as ex:  # pylint: disable=broad-exception-caught
             LOGGER.debug("Error calculating metrics for evaluator %s, failed with error %s", run.evaluator_name, ex)
             return {}

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py

@@ -1028,36 +1028,36 @@ def _preprocess_data(
     batch_run_data: Union[str, os.PathLike, pd.DataFrame] = data
 
     def get_client_type(evaluate_kwargs: Dict[str, Any]) -> Literal["run_submitter", "pf_client", "code_client"]:
-        """Determines the BatchClient to use from provided kwargs (_use_run_submitter_client and _use_pf_client)"""
-        _use_run_submitter_client = cast(Optional[bool], kwargs.pop("_use_run_submitter_client", None))
-        _use_pf_client = cast(Optional[bool], kwargs.pop("_use_pf_client", None))
+        """Pick batch client from kwargs: _use_run_submitter_client and _use_pf_client."""
+        _use_run = cast(Optional[bool], evaluate_kwargs.pop("_use_run_submitter_client", None))
+        _use_pf = cast(Optional[bool], evaluate_kwargs.pop("_use_pf_client", None))
 
-        if _use_run_submitter_client is None and _use_pf_client is None:
-            # If both are unset, return default
+        if _use_run is None and _use_pf is None:
             return "run_submitter"
 
-        if _use_run_submitter_client and _use_pf_client:
+        if _use_run and _use_pf:
             raise EvaluationException(
-                message="Only one of _use_pf_client and _use_run_submitter_client should be set to True.",
+                message=("Only one of _use_pf_client and _use_run_submitter_client " "should be set to True."),
                 target=ErrorTarget.EVALUATE,
                 category=ErrorCategory.INVALID_VALUE,
                 blame=ErrorBlame.USER_ERROR,
             )
 
-        if _use_run_submitter_client == False and _use_pf_client == False:
+        if _use_run is False and _use_pf is False:
             return "code_client"
 
-        if _use_run_submitter_client:
+        if _use_run:
             return "run_submitter"
-        if _use_pf_client:
+        if _use_pf:
             return "pf_client"
 
-        if _use_run_submitter_client is None and _use_pf_client == False:
+        if _use_run is None and _use_pf is False:
             return "run_submitter"
-        if _use_run_submitter_client == False and _use_pf_client is None:
+        if _use_run is False and _use_pf is None:
             return "pf_client"
 
-        assert False, "This should be impossible"
+        # Defensive default
+        return "run_submitter"
 
     client_type: Literal["run_submitter", "pf_client", "code_client"] = get_client_type(kwargs)
 

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_coherence/_coherence.py

@@ -12,17 +12,22 @@
 
 class CoherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     """
-    Evaluates coherence score for a given query and response or a multi-turn conversation, including reasoning.
+    Evaluates coherence for a given query and response or a multi-turn
+    conversation, including reasoning.
 
-    The coherence measure assesses the ability of the language model to generate text that reads naturally,
-    flows smoothly, and resembles human-like language in its responses. Use it when assessing the readability
-    and user-friendliness of a model's generated responses in real-world applications.
+    The coherence measure assesses the model's ability to generate text that
+    reads naturally, flows smoothly, and resembles human-like language. Use it
+    when assessing the readability and user-friendliness of responses.
 
     :param model_config: Configuration for the Azure OpenAI model.
-    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+    :type model_config:
+        Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
         ~azure.ai.evaluation.OpenAIModelConfiguration]
     :param threshold: The threshold for the coherence evaluator. Default is 3.
     :type threshold: int
+    :keyword is_reasoning_model: (Preview) config for chat completions is
+        updated to use reasoning models
+    :type is_reasoning_model: bool
 
     .. admonition:: Example:
 
@@ -31,7 +36,8 @@ class CoherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             :end-before: [END coherence_evaluator]
             :language: python
             :dedent: 8
-            :caption: Initialize and call CoherenceEvaluator using azure.ai.evaluation.AzureAIProject
+            :caption: Initialize and call CoherenceEvaluator using
+                azure.ai.evaluation.AzureAIProject
 
     .. admonition:: Example using Azure AI Project URL:
 
@@ -40,7 +46,8 @@ class CoherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             :end-before: [END coherence_evaluator]
             :language: python
             :dedent: 8
-            :caption: Initialize and call CoherenceEvaluator using Azure AI Project URL in following format
+            :caption: Initialize and call CoherenceEvaluator using Azure AI
+                Project URL in following format
                 https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
 
     .. admonition:: Example with Threshold:
@@ -50,23 +57,24 @@ class CoherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             :end-before: [END threshold_coherence_evaluator]
             :language: python
             :dedent: 8
-            :caption: Initialize with threshold and call a CoherenceEvaluator with a query and response.
+            :caption: Initialize with threshold and call a CoherenceEvaluator
+                with a query and response.
 
     .. note::
 
-        To align with our support of a diverse set of models, an output key without the `gpt_` prefix has been added.
-        To maintain backwards compatibility, the old key with the `gpt_` prefix is still be present in the output;
-        however, it is recommended to use the new key moving forward as the old key will be deprecated in the future.
+    To align with support of diverse models, an output key without the
+    `gpt_` prefix has been added. The old key with the `gpt_` prefix is
+    still present for compatibility; however, it will be deprecated.
     """
 
     _PROMPTY_FILE = "coherence.prompty"
     _RESULT_KEY = "coherence"
 
     id = "azureai://built-in/evaluators/coherence"
-    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+    """Evaluator identifier, experimental to be used only with cloud evaluation"""
 
     @override
-    def __init__(self, model_config, *, threshold=3):
+    def __init__(self, model_config, *, threshold=3, **kwargs):
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
         self._threshold = threshold
@@ -77,6 +85,7 @@ def __init__(self, model_config, *, threshold=3):
             result_key=self._RESULT_KEY,
             threshold=threshold,
             _higher_is_better=self._higher_is_better,
+            **kwargs,
         )
 
     @overload
@@ -104,9 +113,11 @@ def __call__(
     ) -> Dict[str, Union[float, Dict[str, List[Union[str, float]]]]]:
         """Evaluate coherence for a conversation
 
-        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
-            key "messages", and potentially a global context under the key "context". Conversation turns are expected
-            to be dictionaries with keys "content", "role", and possibly "context".
+        :keyword conversation: The conversation to evaluate. Expected to
+            contain a list of conversation turns under the key "messages",
+            and optionally a global context under the key "context". Turns are
+            dictionaries with keys "content", "role", and possibly
+            "context".
         :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
         :return: The coherence score.
         :rtype: Dict[str, Union[float, Dict[str, List[float]]]]
@@ -118,19 +129,22 @@ def __call__(  # pylint: disable=docstring-missing-param
         *args,
         **kwargs,
     ):
-        """Evaluate coherence. Accepts either a query and response for a single evaluation,
-        or a conversation for a potentially multi-turn evaluation. If the conversation has more than one pair of
-        turns, the evaluator will aggregate the results of each turn.
+        """Evaluate coherence.
+
+        Accepts a query/response for a single evaluation, or a conversation
+        for a multi-turn evaluation. If the conversation has more than one
+        pair of turns, results are aggregated.
 
         :keyword query: The query to be evaluated.
         :paramtype query: str
         :keyword response: The response to be evaluated.
         :paramtype response: Optional[str]
-        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
-            key "messages". Conversation turns are expected
-            to be dictionaries with keys "content" and "role".
+        :keyword conversation: The conversation to evaluate. Expected to
+            contain conversation turns under the key "messages" as
+            dictionaries with keys "content" and "role".
         :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
         :return: The relevance score.
-        :rtype: Union[Dict[str, float], Dict[str, Union[float, Dict[str, List[float]]]]]
+        :rtype: Union[Dict[str, float], Dict[str, Union[float, Dict[str,
+            List[float]]]]]
         """
         return super().__call__(*args, **kwargs)

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_multi_eval.py

@@ -4,7 +4,9 @@
 from concurrent.futures import as_completed
 from typing import TypeVar, Dict, List
 
-from azure.ai.evaluation._legacy._adapters.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
+from azure.ai.evaluation._legacy._adapters.tracing import (
+    ThreadPoolExecutorWithContext as ThreadPoolExecutor,
+)
 from typing_extensions import override
 
 from azure.ai.evaluation._evaluators._common import EvaluatorBase

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py

@@ -3,22 +3,35 @@
 # ---------------------------------------------------------
 
 import math
-import re
 import os
+import re
 from typing import Dict, TypeVar, Union
 
-if os.getenv("AI_EVALS_USE_PF_PROMPTY", "false").lower() == "true":
-    from promptflow.core._flow import AsyncPrompty
-else:
-    from azure.ai.evaluation._legacy.prompty import AsyncPrompty
+from azure.ai.evaluation._legacy.prompty import (
+    AsyncPrompty as _LegacyAsyncPrompty,
+)
 from typing_extensions import override
 
-from azure.ai.evaluation._common.constants import PROMPT_BASED_REASON_EVALUATORS
+from azure.ai.evaluation._common.constants import (
+    PROMPT_BASED_REASON_EVALUATORS,
+)
 from azure.ai.evaluation._constants import EVALUATION_PASS_FAIL_MAPPING
-from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
-from ..._common.utils import construct_prompty_model_config, validate_model_config, parse_quality_evaluator_reason_score
+from azure.ai.evaluation._exceptions import (
+    EvaluationException,
+    ErrorBlame,
+    ErrorCategory,
+    ErrorTarget,
+)
+from ..._common.utils import (
+    construct_prompty_model_config,
+    validate_model_config,
+    parse_quality_evaluator_reason_score,
+)
 from . import EvaluatorBase
 
+_PFAsyncPrompty = None  # type: ignore[assignment]
+
+
 try:
     from ..._user_agent import UserAgentSingleton
 except ImportError:
@@ -71,7 +84,11 @@ def __init__(
         self._prompty_file = prompty_file
         self._threshold = threshold
         self._higher_is_better = _higher_is_better
-        super().__init__(eval_last_turn=eval_last_turn, threshold=threshold, _higher_is_better=_higher_is_better)
+        super().__init__(
+            eval_last_turn=eval_last_turn,
+            threshold=threshold,
+            _higher_is_better=_higher_is_better,
+        )
 
         subclass_name = self.__class__.__name__
         user_agent = f"{UserAgentSingleton().value} (type=evaluator subtype={subclass_name})"
@@ -80,9 +97,28 @@ def __init__(
             self._DEFAULT_OPEN_API_VERSION,
             user_agent,
         )
-
-        self._flow = AsyncPrompty.load(
-            source=self._prompty_file, model=prompty_model_config, is_reasoning_model=self._is_reasoning_model
+        # Choose backend: force legacy prompty for reasoning models so
+        # parameter adaptation applies
+        use_pf = os.getenv("AI_EVALS_USE_PF_PROMPTY", "false").lower() == "true"
+        if self._is_reasoning_model:
+            AsyncPromptyClass = _LegacyAsyncPrompty
+        else:
+            if use_pf and _PFAsyncPrompty is None:
+                try:
+                    from promptflow.core._flow import (  # type: ignore
+                        AsyncPrompty as _PFAsyncPrompty_import,
+                    )
+
+                    # assign to module-level for reuse
+                    globals()["_PFAsyncPrompty"] = _PFAsyncPrompty_import
+                except Exception:  # pragma: no cover - PF not available
+                    globals()["_PFAsyncPrompty"] = None
+            AsyncPromptyClass = _PFAsyncPrompty if (use_pf and _PFAsyncPrompty is not None) else _LegacyAsyncPrompty
+
+        self._flow = AsyncPromptyClass.load(  # type: ignore[call-arg]
+            source=self._prompty_file,
+            model=prompty_model_config,
+            is_reasoning_model=self._is_reasoning_model,
         )
 
     # __call__ not overridden here because child classes have such varied signatures that there's no point