chore(weave): test flake fix for windows (#6272)

Author: gtarpenning

tests/trace/test_evaluate.py

@@ -1,5 +1,6 @@
 import asyncio
 import os
+import sys
 import time
 from unittest.mock import patch
 
@@ -9,14 +10,16 @@
 import weave
 from weave import Dataset, Evaluation, Model
 
+_LATENCY_TOL = 10 if sys.platform == "win32" else 1
+
 dataset_rows = [{"input": "1 + 2", "target": 3}, {"input": "2**4", "target": 15}]
 dataset = Dataset(rows=dataset_rows)
 
 
 expected_eval_result = {
     "output": {"mean": 9.5},
     "score": {"true_count": 1, "true_fraction": 0.5},
-    "model_latency": {"mean": pytest.approx(0, abs=1)},
+    "model_latency": {"mean": pytest.approx(0, abs=_LATENCY_TOL)},
 }
 
 
@@ -239,7 +242,7 @@ def model(col_a, col_b):
         return col_a + col_b
 
     result = await evaluation.evaluate(model)
-    assert result.pop("model_latency").get("mean") == pytest.approx(0, abs=1)
+    assert result.pop("model_latency").get("mean") == pytest.approx(0, abs=_LATENCY_TOL)
 
     # Build expected result dynamically
     expected_result = {
@@ -260,7 +263,9 @@ def model(col_a, col_b):
     predict_and_score_calls = list(evaluation.predict_and_score.calls())
     assert len(predict_and_score_calls) == 3
     outputs = [c.output for c in predict_and_score_calls]
-    assert all(o.pop("model_latency") == pytest.approx(0, abs=1) for o in outputs)
+    assert all(
+        o.pop("model_latency") == pytest.approx(0, abs=_LATENCY_TOL) for o in outputs
+    )
 
     # Build expected output dynamically
     expected_output = {
@@ -324,9 +329,9 @@ def score(output):
     assert result == {
         "output": {"mean": 5.5},
         "score": {"mean": 1.0},
-        "model_latency": {"mean": pytest.approx(1, abs=1)},
+        "model_latency": {"mean": pytest.approx(1, abs=_LATENCY_TOL)},
     }
-    assert time.time() - now < 5
+    assert time.time() - now < (15 if sys.platform == "win32" else 5)
 
 
 def test_evaluation_from_weaveobject_missing_evaluation_name(client):

tests/trace/test_evaluate_oldstyle.py

@@ -1,19 +1,22 @@
 import asyncio
+import sys
 
 import pytest
 
 import weave
 from weave import Dataset, Evaluation, Model
 from weave.scorers import MultiTaskBinaryClassificationF1
 
+_LATENCY_TOL = 10 if sys.platform == "win32" else 1
+
 dataset_rows = [{"input": "1 + 2", "target": 3}, {"input": "2**4", "target": 15}]
 dataset = Dataset(rows=dataset_rows)
 
 
 expected_eval_result = {
     "model_output": {"mean": 9.5},
     "score_oldstyle": {"true_count": 1, "true_fraction": 0.5},
-    "model_latency": {"mean": pytest.approx(0, abs=1)},
+    "model_latency": {"mean": pytest.approx(0, abs=_LATENCY_TOL)},
 }
 
 
@@ -109,7 +112,7 @@ def test_evaluate_both_styles(client):
         "model_output": {"mean": 9.5},
         "score_oldstyle": {"true_count": 1, "true_fraction": 0.5},
         "score_newstyle": {"true_count": 1, "true_fraction": 0.5},
-        "model_latency": {"mean": pytest.approx(0, abs=1)},
+        "model_latency": {"mean": pytest.approx(0, abs=_LATENCY_TOL)},
     }
 
 

tests/trace/test_evaluations.py

@@ -1,5 +1,6 @@
 import dataclasses
 import random
+import sys
 from typing import Any
 
 import pydantic
@@ -13,6 +14,8 @@
 from weave.trace.refs import CallRef
 from weave.trace_server import trace_server_interface as tsi
 
+_LATENCY_TOL = 10 if sys.platform == "win32" else 1
+
 
 def flatten_calls(
     calls: list[tsi.CallSchema], parent_id: str | None = None, depth: int = 0
@@ -535,7 +538,7 @@ async def test_evaluation_data_topology(client):
         "nested": {"bool_avg": 0.5},
         "reason": "This is a custom test reason",
     }
-    model_latency = {"mean": pytest.approx(0, abs=1)}
+    model_latency = {"mean": pytest.approx(0, abs=_LATENCY_TOL)}
     predict_usage_summary = {
         "usage": {
             "gpt-4o-2024-05-13": {
@@ -733,7 +736,7 @@ def function_score(scorer_res, output) -> dict:
     assert res == {
         "output": {"a": {"mean": 3.0}, "b": {"c": {"mean": 2.0}}},
         "function_score": {"a": {"mean": 3.0}, "b": {"c": {"mean": 2.0}}},
-        "model_latency": {"mean": pytest.approx(0, abs=2)},
+        "model_latency": {"mean": pytest.approx(0, abs=max(2, _LATENCY_TOL))},
     }
 
 

tests/trace_server/test_trace_server_evaluation_apis.py

@@ -25,6 +25,8 @@
 from weave.trace_server.workers.evaluate_model_worker import evaluate_model_worker
 from weave.utils.project_id import from_project_id, to_project_id
 
+_LATENCY_TOL = 10 if sys.platform == "win32" else 1
+
 
 @pytest.mark.asyncio
 async def test_evaluation_status(client):
@@ -84,7 +86,7 @@ def generate_id_side_effect():
         output={
             "output": {"mean": 3.0},
             "scorer": {"mean": 1.0},
-            "model_latency": {"mean": pytest.approx(0, abs=1)},
+            "model_latency": {"mean": pytest.approx(0, abs=_LATENCY_TOL)},
         }
     )
 
@@ -374,9 +376,7 @@ def evaluate_model_wrapped(req: EvaluateModelReq):
         assert eval_call.summary["weave"]["status"] == TraceStatus.DESCENDANT_ERROR
         assert eval_call.output == {
             "LLMAsAJudgeScorer": None,
-            "model_latency": {"mean": pytest.approx(0, abs=2)}
-            if sys.platform != "win32"
-            else {"mean": pytest.approx(0, abs=10)},
+            "model_latency": {"mean": pytest.approx(0, abs=max(2, _LATENCY_TOL))},
         }
     else:
         assert eval_call.summary["status_counts"] == {
@@ -387,5 +387,5 @@ def evaluate_model_wrapped(req: EvaluateModelReq):
         assert eval_call.output == {
             "output": {"score": {"mean": 9.0}},
             "LLMAsAJudgeScorer": {"score": {"mean": 9.0}},
-            "model_latency": {"mean": pytest.approx(0, abs=1)},
+            "model_latency": {"mean": pytest.approx(0, abs=_LATENCY_TOL)},
         }