fix: complete the eval metrics Truth_Ratio calculation mentioned in the paper

configs/eval/tofu.yaml

@@ -4,6 +4,7 @@
 defaults: # include all defined metrics files
   - tofu_metrics: # When you import a metric here, its configuration automatically populates the 
   # metric key below, enabled by the @package directive at the top of each configuration file.
+    - forget_Truth_Ratio
     - forget_quality
     - forget_Q_A_Prob
     - forget_Q_A_ROUGE

src/evals/metrics/memorization.py

@@ -119,10 +119,16 @@ def closer_to_1_better(arr):
     def true_better(arr):
         return np.mean(np.maximum(0, 1 - arr))
 
+    # Extent of knowledge (as used in OpenUnlearning paper's meta-evaluation) uses tr=true/(true+false)
+    def prob_mean(arr):
+        return np.mean(arr)
+
     if kwargs["aggregator"] == "closer_to_1_better":
         aggregator = closer_to_1_better
     elif kwargs["aggregator"] == "true_better":
         aggregator = true_better
+    elif kwargs["aggregator"] == "prob_mean":
+        aggregator = prob_mean
     else:
         raise ValueError(f"Invalid truth ratio aggregator: {kwargs['aggregator']}")
 
@@ -153,7 +159,13 @@ def true_better(arr):
     correct_prob = np.exp(-correct_avg_losses)
     wrong_prob = np.exp(-wrong_avg_losses)
 
-    truth_ratios = wrong_prob / (correct_prob + 1e-10)
+    if kwargs["aggregator"] != "prob_mean":
+        # Original definition from TOFU: wrong / correct
+        truth_ratios = wrong_prob / (correct_prob + 1e-10)
+    else:
+        # New definition from OpenUnlearning: correct / (correct + wrong)
+        truth_ratios = correct_prob / (correct_prob + wrong_prob + 1e-10)
+
     value_by_index = dict(
         zip(correct_indices, [{"score": val} for val in truth_ratios])
     )