remove evaluate sync test

Author: m7mdrevest

tests/evals/test_dataset.py

@@ -1662,84 +1662,3 @@ async def mock_async_task(inputs: TaskInput) -> TaskOutput:
             ),
         ]
     )
-
-
-@pytest.mark.skipif(not tenacity_import_successful(), reason='tenacity not installed')
-def test_evaluate_sync_with_retried_task_and_evaluator(
-    example_dataset: Dataset[TaskInput, TaskOutput, TaskMetadata],
-):
-    task_attempt = 0
-
-    def mock_sync_task(inputs: TaskInput) -> TaskOutput:
-        nonlocal task_attempt
-        if task_attempt < 3:
-            task_attempt += 1
-            raise RuntimeError(f'task failure {task_attempt}')
-        if inputs.query == 'What is 2+2?':
-            return TaskOutput(answer='4')
-        elif inputs.query == 'What is the capital of France?':
-            return TaskOutput(answer='Paris')
-        return TaskOutput(answer='Unknown')  # pragma: no cover
-
-    evaluator_attempt = 0
-
-    @dataclass
-    class RetryEvaluator(Evaluator[TaskInput, TaskOutput, TaskMetadata]):
-        def evaluate(self, ctx: EvaluatorContext[TaskInput, TaskOutput, TaskMetadata]):
-            nonlocal evaluator_attempt
-            if evaluator_attempt < 3:
-                evaluator_attempt += 1
-                raise RuntimeError(f'evaluator failure {evaluator_attempt}')
-            if ctx.expected_output is None:  # pragma: no cover
-                return {'result': 'no_expected_output'}
-            return {
-                'correct': ctx.output.answer == ctx.expected_output.answer,
-                'confidence': ctx.output.confidence,
-            }
-
-    example_dataset.add_evaluator(RetryEvaluator())
-
-    report = example_dataset.evaluate_sync(
-        mock_sync_task,
-        retry_task=RetryConfig(stop=stop_after_attempt(3)),
-        retry_evaluators=RetryConfig(stop=stop_after_attempt(3)),
-    )
-
-    assert task_attempt == 3
-    assert evaluator_attempt == 3
-
-    assert report is not None
-    assert len(report.cases) == 2
-    assert ReportCaseAdapter.dump_python(report.cases[0]) == snapshot(
-        {
-            'assertions': {
-                'correct': {
-                    'name': 'correct',
-                    'reason': None,
-                    'source': {'name': 'RetryEvaluator', 'arguments': None},
-                    'value': True,
-                }
-            },
-            'attributes': {},
-            'evaluator_failures': [],
-            'expected_output': {'answer': '4', 'confidence': 1.0},
-            'inputs': {'query': 'What is 2+2?'},
-            'labels': {},
-            'metadata': {'category': 'general', 'difficulty': 'easy'},
-            'metrics': {},
-            'name': 'case1',
-            'output': {'answer': '4', 'confidence': 1.0},
-            'scores': {
-                'confidence': {
-                    'name': 'confidence',
-                    'reason': None,
-                    'source': {'name': 'RetryEvaluator', 'arguments': None},
-                    'value': 1.0,
-                }
-            },
-            'span_id': '0000000000000003',
-            'task_duration': IsNumber(),
-            'total_duration': IsNumber(),
-            'trace_id': '00000000000000000000000000000001',
-        }
-    )