Merge pull request #138 from alimaredia/mtbench-branch-judgement-return-overall-score

return overall_score from MTBenchBranch.generate_judgement()

Author: danmcp

.github/workflows/e2e-nvidia-t4-x1.yml

@@ -142,7 +142,7 @@ jobs:
         working-directory: ./instructlab
         run: |
           . venv/bin/activate
-          ./scripts/basic-workflow-tests.sh -m
+          ./scripts/basic-workflow-tests.sh -msq
 
   stop-runner:
     name: Stop external EC2 runner

src/instructlab/eval/mt_bench.py

@@ -246,10 +246,12 @@ def judge_answers(
             serving_gpus    Number of gpus allocated for serving.  Used to tune with max_workers=auto.  None indicates to use value specified in constructor.
 
         Returns:
+            overall_score   overall score from the evaluation
             qa_pairs        Question and answer pairs (with scores) from the evaluation
+            error_rate      percentage of questions dropped due to errors during evaluation
         """
         logger.debug(locals())
-        _, qa_pairs, _, error_rate = mt_bench_judgment.generate_judgment(
+        overall_score, qa_pairs, _, error_rate = mt_bench_judgment.generate_judgment(
             self.model_name,
             self.judge_model_name,
             server_url,
@@ -261,4 +263,4 @@ def judge_answers(
             bench_name="mt_bench_branch",
             merge_system_user_message=self.merge_system_user_message,
         )
-        return qa_pairs, error_rate
+        return overall_score, qa_pairs, error_rate

tests/test_branch_judge_answers.py

@@ -10,7 +10,11 @@
     "../taxonomy",
     "main",
 )
-qa_pairs, error_rate = mt_bench_branch.judge_answers("http://localhost:8000/v1")
+overall_score, qa_pairs, error_rate = mt_bench_branch.judge_answers(
+    "http://localhost:8000/v1"
+)
+
+print(f"Overall Score: {overall_score}")
 print(f"Error Rate: {error_rate}")
 print(f"QA Pair 0:")
 pprint.pprint(qa_pairs[0])