feat(InputMocking): read input schema from entrypoint to pass to mocker

Author: bai-uipath

samples/calculator/evals/eval-sets/default.json

@@ -64,6 +64,25 @@
       "evalSetId": "default-eval-set-id",
       "createdAt": "2025-09-04T18:54:58.378Z",
       "updatedAt": "2025-09-04T18:55:55.416Z"
+    },
+    {
+      "id": "test-with-llm-input-mocking",
+      "name": "Test with LLM input mocking",
+      "inputs": {},
+      "expectedOutput": {
+        "result": 35
+      },
+      "expectedAgentBehavior": "",
+      "inputMockingStrategy": {
+        "prompt": "Generate a multiplication calculation where the first number is 5 and the second number is 7",
+        "model": {
+          "model": "gpt-4o-mini-2024-07-18",
+          "temperature": 0.3
+        }
+      },
+      "evalSetId": "default-eval-set-id",
+      "createdAt": "2025-09-04T18:54:58.378Z",
+      "updatedAt": "2025-09-04T18:55:55.416Z"
     }
   ],
   "modelSettings": [],

src/uipath/_cli/_evals/_runtime.py

@@ -1,6 +1,7 @@
 import asyncio
 import json
 import logging
+import os
 import uuid
 from collections import defaultdict
 from pathlib import Path
@@ -140,6 +141,7 @@ class UiPathEvalContext(UiPathRuntimeContext):
     workers: Optional[int] = 1
     eval_set: Optional[str] = None
     eval_ids: Optional[List[str]] = None
+    input_schema: Optional[Dict[str, Any]] = None
 
 
 class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
@@ -156,6 +158,7 @@ def __init__(
         self.factory: UiPathRuntimeFactory[T, C] = factory
         self.event_bus: EventBus = event_bus
 
+        self._initialize_input_schema()
         self.span_exporter: ExecutionSpanExporter = ExecutionSpanExporter()
         self.span_collector: ExecutionSpanCollector = ExecutionSpanCollector()
 
@@ -167,6 +170,18 @@ def __init__(
         self.logs_exporter: ExecutionLogsExporter = ExecutionLogsExporter()
         self.execution_id = str(uuid.uuid4())
 
+    def _initialize_input_schema(self) -> None:
+        """Initialize the input schema using a temporary runtime to get the entrypoint."""
+        temp_context = self.factory.new_context(
+            entrypoint=self.context.entrypoint, runtime_dir=os.getcwd()
+        )
+        temp_runtime = self.factory.from_context(temp_context)
+        self.context.input_schema = temp_runtime.get_entrypoint.input
+
+        # Ensure additionalProperties is set for OpenAI strict mode compatibility
+        if "additionalProperties" not in self.context.input_schema:
+            self.context.input_schema["additionalProperties"] = False
+
     @classmethod
     def from_eval_context(
         cls,
@@ -430,9 +445,7 @@ async def _generate_input_for_eval(
         self, eval_item: EvaluationItem
     ) -> EvaluationItem:
         """Use LLM to generate a mock input for an evaluation item."""
-        # TODO(bai): get the input schema from agent definition, once it is available there.
-        input_schema: dict[str, Any] = {}
-        generated_input = await generate_llm_input(eval_item, input_schema)
+        generated_input = await generate_llm_input(eval_item, self.context.input_schema)
         updated_eval_item = eval_item.model_copy(update={"inputs": generated_input})
         return updated_eval_item
 

tests/cli/eval/test_evaluate.py

@@ -5,6 +5,7 @@
 from uipath._cli._evals._runtime import UiPathEvalContext
 from uipath._cli._runtime._contracts import UiPathRuntimeContext, UiPathRuntimeFactory
 from uipath._cli._runtime._runtime import UiPathRuntime
+from uipath._cli.models.runtime_schema import Entrypoint
 from uipath._events._event_bus import EventBus
 
 
@@ -18,12 +19,23 @@ async def test_evaluate():
     async def identity(input: Any) -> Any:
         return input
 
-    class MyFactory(UiPathRuntimeFactory[UiPathRuntime, UiPathRuntimeContext]):
+    class TestRuntime(UiPathRuntime):
+        @property
+        def get_entrypoint(self) -> Entrypoint:
+            return Entrypoint(
+                file_path="test.py",
+                unique_id="test",
+                type="workflow",
+                input={"type": "object", "properties": {}},
+                output={"type": "object", "properties": {}},
+            )
+
+    class MyFactory(UiPathRuntimeFactory[TestRuntime, UiPathRuntimeContext]):
         def __init__(self):
             super().__init__(
-                UiPathRuntime,
+                TestRuntime,
                 UiPathRuntimeContext,
-                runtime_generator=lambda context: UiPathRuntime(
+                runtime_generator=lambda context: TestRuntime(
                     context, executor=identity
                 ),
             )