adds mmlu-pro

Author: NathanHB

src/lighteval/tasks/tasks/mmlu_pro.py

@@ -0,0 +1,77 @@
+"""
+name:
+MMLU Pro
+
+dataset:
+TIGER-Lab/MMLU-Pro
+
+abstract:
+
+languages:
+english
+
+tags:
+general-knowledge
+
+paper:
+
+"""
+from string import ascii_uppercase
+
+from lighteval.metrics.dynamic_metrics import (
+    LogLikelihoodAccMetric,
+)
+from lighteval.metrics.metrics import Metrics
+from lighteval.metrics.normalizations import LogProbCharNorm, LogProbPMINorm, LogProbTokenNorm
+from lighteval.tasks.lighteval_task import LightevalTaskConfig
+from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation
+from lighteval.tasks.requests import Doc
+from lighteval.tasks.templates.multichoice import get_mcq_prompt_function
+from lighteval.tasks.templates.utils.formulation import (
+    CFFormulation,
+    HybridFormulation,
+    MCFFormulation,
+)
+from lighteval.utils.language import Language
+
+
+TEMPLATE = """
+Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.
+
+{question}
+
+{choices}
+
+Answer:""".strip()
+
+
+def mmlu_pro_prompt_function(line, task_name: str = None):
+    choices = "\n".join([f"{letter}: {choice}" for letter, choice in zip(ascii_uppercase, line["options"])])
+
+    query = TEMPLATE.format(
+        question=line["question"],
+        choices=choices,
+    )
+
+    return Doc(
+        task_name=task_name,
+        query=query,
+        choices=ascii_uppercase[: len(choices)],
+        gold_index=line["answer_index"],
+        instruction=query,
+    )
+
+
+mmlu_pro = LightevalTaskConfig(
+        name="mmlu_pro",
+        prompt_function=mmlu_pro_prompt_function,
+        suite=("lighteval",),
+        hf_repo="TIGER-Lab/MMLU-Pro",
+        hf_subset="default",
+        hf_revision="3373e0b32277875b8db2aa555a333b78a08477ea",
+        evaluation_splits=("test",),
+        few_shots_split="validation",
+        metrics=[Metrics.gpqa_instruct_metric],
+    )
+
+TASKS_TABLE = [mmlu_pro]