Fix silently incorrect scores in parea/evals (two inverted metrics, eval() on model output)

parea/evals/dataset_level/balanced_acc.py

@@ -11,7 +11,9 @@ def balanced_acc(logs: List[EvaluatedLog]) -> Union[EvaluationResult, None]:
         total = defaultdict(int)
         for log in logs:
             if (eval_result := log.get_score(score_name)) is not None:
-                correct[log.target] += int(eval_result.score)
+                # Threshold instead of truncating: int(0.9) is 0, which would report a class as
+                # entirely wrong even though every one of its scores was almost perfect.
+                correct[log.target] += int(eval_result.score >= 0.5)
                 total[log.target] += 1
         recalls = [correct[key] / total[key] for key in correct]
 

parea/evals/rag/answer_context_faithfulness_statement_level.py

@@ -96,9 +96,10 @@ def answer_context_faithfulness_statement_level(log: Log) -> float:
         final_answer = "Final verdict for each statement in order:".lower()
         if final_answer in verdicts:
             verdicts = verdicts[verdicts.find(final_answer) + len(final_answer) :]
-            yes_count = sum(0 if "yes" in answer else 1 for answer in verdicts.strip().split(".") if answer != "")
-            return yes_count / len(statements_formatted)
+            yes_count = sum(1 if "yes" in answer else 0 for answer in verdicts.strip().split(".") if answer.strip())
         else:
-            return max(0, output.count("verdict: no")) / len(statements_formatted)
+            # No summary line: fall back to counting the per-statement verdicts in the grader's response.
+            yes_count = max(0, len(statements_formatted) - verdicts.count("verdict: no"))
+        return min(1.0, yes_count / len(statements_formatted))
 
     return answer_context_faithfulness_statement_level

parea/evals/rag/context_ranking_listwise.py

@@ -1,5 +1,7 @@
 from typing import Callable, List, Optional
 
+import re
+
 from parea.evals.utils import call_openai, get_context, ndcg
 from parea.schemas.log import Log
 
@@ -64,17 +66,26 @@ def listwise_reranking(query: str, contexts: List[str]) -> List[int]:
             is_azure=is_azure,
         )
 
-        s = sorted_list.strip("[] ").replace(" ", "")
-        number_strings = s.split(",")
-        return [int(num) for num in number_strings if num.isdigit()]
+        # The prompt numbers the passages starting at 1, and models answer either with bare numbers
+        # ("3, 1, 2") or with the passage names ("Passage3, Passage1, Passage2"), so pull out every
+        # integer and shift it back to a 0-based index.
+        ranked = []
+        for match in re.findall(r"\d+", sorted_list):
+            index = int(match) - 1
+            if 0 <= index < len(contexts) and index not in ranked:
+                ranked.append(index)
+        # Passages the model dropped or mangled keep their original relative order at the end, so
+        # that the result is always a permutation of the contexts.
+        ranked += [i for i in range(len(contexts)) if i not in ranked]
+        return ranked
 
     def progressive_reranking(query: str, contexts: List[str]) -> List[int]:
         """Returns the indices of the contexts in the order of their relevance (most relevant to least relevant)."""
         if len(contexts) <= n_contexts_to_rank:
             return listwise_reranking(query, contexts)
 
         window_size = n_contexts_to_rank
-        window_step = n_contexts_to_rank // 2
+        window_step = max(1, n_contexts_to_rank // 2)
         offset = len(contexts) - window_size
 
         indices = list(range(len(contexts)))
@@ -101,10 +112,20 @@ def context_ranking(log: Log) -> float:
         question = log.inputs[question_field]
         contexts = get_context(log, context_fields, True)
 
+        if not contexts:
+            return 0.0
+
         reranked_indices = progressive_reranking(question, contexts)
 
         if ranking_measurement == "ndcg":
-            return ndcg(reranked_indices, list(range(len(contexts))))
+            # `reranked_indices[j]` is the index of the j-th most relevant context, so the context the
+            # reranker put first gets the highest relevance grade. The retriever proposed the contexts
+            # in the order 0, 1, ..., n-1, and NDCG measures how well that order agrees with the grades.
+            n_contexts = len(contexts)
+            relevance = [0] * n_contexts
+            for rank, context_index in enumerate(reranked_indices):
+                relevance[context_index] = n_contexts - rank
+            return ndcg(relevance, list(range(n_contexts)))
         else:
             raise NotImplementedError
 

parea/evals/rag/context_ranking_pointwise.py

@@ -28,14 +28,7 @@ def context_ranking_pointwise_factory(
     Returns:
         Callable[[Log], float]: A function that takes a log as input and returns a score between 0 and 1 indicating
         how well the retrieved context is ranked by their relevancy.
-
-    Raises:
-        ImportError: If numpy is not installed.
     """
-    try:
-        import numpy as np
-    except ImportError:
-        raise ImportError("Please install numpy to use this metric.")
 
     def context_ranking_pointwise(log: Log) -> float:
         """Quantifies if the retrieved context is ranked by their relevancy"""
@@ -76,8 +69,13 @@ def context_ranking_pointwise(log: Log) -> float:
             verifications.append(response)
 
         if ranking_measurement == "average_precision":
-            response = [safe_json_loads(item) for item in verifications]
-            response = [int("yes" in resp.get("verdict", " ").lower()) if resp.get("verdict") else np.nan for resp in response]
+            # A missing or unparseable verdict counts as "not relevant"; using NaN here would poison
+            # the sums below and silently turn the whole score into NaN.
+            response = []
+            for item in verifications:
+                parsed = safe_json_loads(item)
+                verdict = parsed.get("verdict") if isinstance(parsed, dict) else None
+                response.append(int("yes" in str(verdict).lower()) if verdict else 0)
             denominator = sum(response) + 1e-10
             numerator = sum([(sum(response[: i + 1]) / (i + 1)) * response[i] for i in range(len(response))])
             return numerator / denominator

parea/evals/rag/percent_target_supported_by_context.py

@@ -1,8 +1,9 @@
 from typing import Callable, List, Optional, Union
 
 import re
+import warnings
 
-from parea.evals.utils import call_openai, get_context
+from parea.evals.utils import call_openai, get_context, safe_json_loads
 from parea.schemas.log import Log
 
 
@@ -78,13 +79,17 @@ def percent_target_supported_by_context(log: Log) -> Union[float, None]:
             temperature=0.0,
             is_azure=is_azure,
         )
-        pattern = r"\[\s*\{.*?\}(\s*,\s*\{.*?\})*\s*\]"
-        match = re.search(pattern, classification.replace("\n", ""))
-        if match:
-            response = eval(classification)
-            numerator = sum(item.get("Attributed").lower() == "yes" for item in response)
-            return numerator / len(response)
-        else:
-            return 0.0
+        pattern = r"\[\s*\{.*?\}(?:\s*,\s*\{.*?\})*\s*\]"
+        match = re.search(pattern, classification, flags=re.DOTALL)
+        if not match:
+            warnings.warn(f"Could not find a classification in the model response: {classification}")
+            return None
+
+        response = safe_json_loads(match.group(0))
+        if not isinstance(response, list) or not response:
+            return None
+
+        numerator = sum(1 for item in response if isinstance(item, dict) and str(item.get("Attributed", "")).lower() == "yes")
+        return numerator / len(response)
 
     return percent_target_supported_by_context

parea/evals/utils.py

@@ -113,8 +113,9 @@ def dcg(y_true, ranking):
     """Discounted cumulative gain (DCG) at rank k."""
     import numpy as np
 
-    y_true = np.asarray(y_true)
-    ranking = np.asarray(ranking)
+    # float, so that 2**rel doesn't silently overflow int64 once there are >62 relevance grades
+    y_true = np.asarray(y_true, dtype=float)
+    ranking = np.asarray(ranking, dtype=int)
     rel = y_true[ranking]
     gains = 2**rel - 1
     discounts = np.log2(np.arange(len(ranking)) + 2)
@@ -128,6 +129,8 @@ def ndcg(y_true, ranking):
     k = len(ranking)
     best_ranking = np.argsort(y_true)[::-1]
     best = dcg(y_true, best_ranking[:k])
+    if best == 0:
+        return 0.0
     return dcg(y_true, ranking) / best