Skip to content

Commit 78a5c23

Browse files
clefourrierclefourrier
committed
updated prompts/instruction management
1 parent bae9544 commit 78a5c23

File tree

2 files changed

+103
-11
lines changed

2 files changed

+103
-11
lines changed

‎examples/tasks/instruct_multilingual.txt

Lines changed: 97 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,100 @@
1-
extended|belebele_instruct_deu_Latn|0|0
2-
extended|belebele_instruct_fra_Latn|0|0
3-
extended|belebele_instruct_ita_Latn|0|0
4-
extended|belebele_instruct_por_Latn|0|0
5-
extended|belebele_instruct_spa_Latn|0|0
1+
extended|belebele_native_instruct_deu_Latn|0|0
2+
extended|belebele_native_instruct_fra_Latn|0|0
3+
extended|belebele_native_instruct_ita_Latn|0|0
4+
extended|belebele_native_instruct_por_Latn|0|0
5+
extended|belebele_native_instruct_spa_Latn|0|0
6+
extended|belebele_en_instruct_acm_Arab|0|0
7+
extended|belebele_en_instruct_arz_Arab|0|0
8+
extended|belebele_en_instruct_ceb_Latn|0|0
9+
extended|belebele_en_instruct_fin_Latn|0|0
10+
extended|belebele_en_instruct_hin_Deva|0|0
11+
extended|belebele_en_instruct_ita_Latn|0|0
12+
extended|belebele_en_instruct_khm_Khmr|0|0
13+
extended|belebele_en_instruct_lvs_Latn|0|0
14+
extended|belebele_en_instruct_npi_Deva|0|0
15+
extended|belebele_en_instruct_pol_Latn|0|0
16+
extended|belebele_en_instruct_slv_Latn|0|0
17+
extended|belebele_en_instruct_swe_Latn|0|0
18+
extended|belebele_en_instruct_afr_Latn|0|0
19+
extended|belebele_en_instruct_asm_Beng|0|0
20+
extended|belebele_en_instruct_ces_Latn|0|0
21+
extended|belebele_en_instruct_fra_Latn|0|0
22+
extended|belebele_en_instruct_hin_Latn|0|0
23+
extended|belebele_en_instruct_jav_Latn|0|0
24+
extended|belebele_en_instruct_mal_Mlym|0|0
25+
extended|belebele_en_instruct_npi_Latn|0|0
26+
extended|belebele_en_instruct_por_Latn|0|0
27+
extended|belebele_en_instruct_swh_Latn|0|0
28+
extended|belebele_en_instruct_tur_Latn|0|0
29+
extended|belebele_en_instruct_yor_Latn|0|0
30+
extended|belebele_en_instruct_als_Latn|0|0
31+
extended|belebele_en_instruct_azj_Latn|0|0
32+
extended|belebele_en_instruct_ckb_Arab|0|0
33+
extended|belebele_en_instruct_hrv_Latn|0|0
34+
extended|belebele_en_instruct_jpn_Jpan|0|0
35+
extended|belebele_en_instruct_kir_Cyrl|0|0
36+
extended|belebele_en_instruct_mar_Deva|0|0
37+
extended|belebele_en_instruct_snd_Arab|0|0
38+
extended|belebele_en_instruct_tam_Taml|0|0
39+
extended|belebele_en_instruct_ukr_Cyrl|0|0
40+
extended|belebele_en_instruct_zho_Hans|0|0
41+
extended|belebele_en_instruct_amh_Ethi|0|0
42+
extended|belebele_en_instruct_dan_Latn|0|0
43+
extended|belebele_en_instruct_hun_Latn|0|0
44+
extended|belebele_en_instruct_kor_Hang|0|0
45+
extended|belebele_en_instruct_mkd_Cyrl|0|0
46+
extended|belebele_en_instruct_ron_Latn|0|0
47+
extended|belebele_en_instruct_som_Latn|0|0
48+
extended|belebele_en_instruct_tel_Telu|0|0
49+
extended|belebele_en_instruct_urd_Arab|0|0
50+
extended|belebele_en_instruct_zho_Hant|0|0
51+
extended|belebele_en_instruct_apc_Arab|0|0
52+
extended|belebele_en_instruct_ben_Beng|0|0
53+
extended|belebele_en_instruct_deu_Latn|0|0
54+
extended|belebele_en_instruct_hye_Armn|0|0
55+
extended|belebele_en_instruct_kan_Knda|0|0
56+
extended|belebele_en_instruct_lao_Laoo|0|0
57+
extended|belebele_en_instruct_mlt_Latn|0|0
58+
extended|belebele_en_instruct_ory_Orya|0|0
59+
extended|belebele_en_instruct_rus_Cyrl|0|0
60+
extended|belebele_en_instruct_tgk_Cyrl|0|0
61+
extended|belebele_en_instruct_urd_Latn|0|0
62+
extended|belebele_en_instruct_zsm_Latn|0|0
63+
extended|belebele_en_instruct_arb_Arab|0|0
64+
extended|belebele_en_instruct_ben_Latn|0|0
65+
extended|belebele_en_instruct_ell_Grek|0|0
66+
extended|belebele_en_instruct_guj_Gujr|0|0
67+
extended|belebele_en_instruct_kat_Geor|0|0
68+
extended|belebele_en_instruct_pan_Guru|0|0
69+
extended|belebele_en_instruct_spa_Latn|0|0
70+
extended|belebele_en_instruct_tgl_Latn|0|0
71+
extended|belebele_en_instruct_uzn_Latn|0|0
72+
extended|belebele_en_instruct_arb_Latn|0|0
73+
extended|belebele_en_instruct_eng_Latn|0|0
74+
extended|belebele_en_instruct_kaz_Cyrl|0|0
75+
extended|belebele_en_instruct_lit_Latn|0|0
76+
extended|belebele_en_instruct_mya_Mymr|0|0
77+
extended|belebele_en_instruct_pbt_Arab|0|0
78+
extended|belebele_en_instruct_sin_Latn|0|0
79+
extended|belebele_en_instruct_srp_Cyrl|0|0
80+
extended|belebele_en_instruct_tha_Thai|0|0
81+
extended|belebele_en_instruct_vie_Latn|0|0
82+
extended|belebele_en_instruct_ars_Arab|0|0
83+
extended|belebele_en_instruct_bul_Cyrl|0|0
84+
extended|belebele_en_instruct_est_Latn|0|0
85+
extended|belebele_en_instruct_ind_Latn|0|0
86+
extended|belebele_en_instruct_nld_Latn|0|0
87+
extended|belebele_en_instruct_pes_Arab|0|0
88+
extended|belebele_en_instruct_sin_Sinh|0|0
89+
extended|belebele_en_instruct_war_Latn|0|0
90+
extended|belebele_en_instruct_ary_Arab|0|0
91+
extended|belebele_en_instruct_cat_Latn|0|0
92+
extended|belebele_en_instruct_eus_Latn|0|0
93+
extended|belebele_en_instruct_heb_Hebr|0|0
94+
extended|belebele_en_instruct_isl_Latn|0|0
95+
extended|belebele_en_instruct_nob_Latn|0|0
96+
extended|belebele_en_instruct_plt_Latn|0|0
97+
extended|belebele_en_instruct_slk_Latn|0|0
698
extended|global_mmlu_instruct_amh|0|0
799
extended|global_mmlu_instruct_ara|0|0
8100
extended|global_mmlu_instruct_ben|0|0

‎src/lighteval/tasks/extended/misc/instruct.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@
5050

5151

5252
def belebele_prompt_en_instruct(line, task_name: str = None):
53-
line["dialect"] == "eng_Latn"
53+
line["dialect"] = "eng_Latn"
5454
return belebele_prompt(line, task_name)
5555

5656

@@ -75,7 +75,7 @@ def belebele_prompt(line, task_name: str = None):
7575
Passage=line["flores_passage"],
7676
Question=line["question"],
7777
)
78-
instruction = query_template.split("\n\n###")[0]
78+
instruction = query_template.split("###\n")[0]
7979

8080
return Doc(
8181
task_name=task_name,
@@ -136,7 +136,7 @@ def belebele_prompt(line, task_name: str = None):
136136
prompt_function=belebele_prompt_en_instruct,
137137
suite=["extended"],
138138
hf_repo="facebook/belebele",
139-
hf_subset=f"{lang}_Latn",
139+
hf_subset=lang,
140140
evaluation_splits=["test"],
141141
hf_avail_splits=["test"],
142142
few_shots_split=None,
@@ -302,7 +302,7 @@ def __init__(self, lang):
302302
"fra": "A partir de la question et des choix de réponses suivants, indiquez la lettre correspondant à la bonne réponse. La dernière ligne de votre réponse doit avoir le format suivant : 'Réponse: '$LETTRE' (sans les guillemets) où LETTRE est l'une des lettres: A, B, C ou D. Réfléchissez étape par étape avant de répondre.\n\n###\nRequête:\n{Question}\n###\nChoix:\nA) {A}\nB) {B}\nC) {C}\nD) {D}",
303303
"ita": "Dato il seguente quesito e le diverse opzioni per una risposta, indicare la lettera corrispondente alla risposta corretta. L'ultima riga della risposta deve avere il seguente formato: 'Risposta: $LETTERA' (senza virgolette), e LETTERA è necessariamente una tra A, B, C, D. Prima di rispondere, è importante che si ragioni passo per passo.\n\n###\nQuesito:\n{Question}\n###\nOpzioni:\nA) {A}\nB) {B}\nC) {C}\nD) {D}",
304304
"por": "Tendo em conta a seguinte pergunta e opções de resposta, indique a letra correspondente à resposta correta. A última linha da sua resposta deve ter o seguinte formato: 'Resposta: $LETRA' (sem aspas) em que LETRA é uma de A, B, C ou D. Pense passo a passo antes de responder.\n\n###\nPergunta:\n{Question}\n###\nOpções:\nA) {A}\nB) {B}\nC) {C}\nD) {D}",
305-
"spa": "Dado el siguiente pregunta y opciones para la respuesta, escriba la letra correspondiente a la respuesta correcta. La última línea de su respuesta debe seguir el siguiente formato: 'Respuesta: $LETTER' (sin comillas) donde LETTER es A, B, C o D. Piense paso a paso antes de responder.\n\\###\nPregunta:\n{Question}\n###\nOpciones:\nA) {A}\nB) {B}\nC) {C}\nD) {D}",
305+
"spa": "Dado el siguiente pregunta y opciones para la respuesta, escriba la letra correspondiente a la respuesta correcta. La última línea de su respuesta debe seguir el siguiente formato: 'Respuesta: $LETTER' (sin comillas) donde LETTER es A, B, C o D. Piense paso a paso antes de responder.\n\n###\nPregunta:\n{Question}\n###\nOpciones:\nA) {A}\nB) {B}\nC) {C}\nD) {D}",
306306
}
307307

308308
def prompt(self, line, task_name: str = None):
@@ -316,7 +316,7 @@ def prompt(self, line, task_name: str = None):
316316
D=choices[3],
317317
Question=line["question"],
318318
)
319-
instruction = query_template.split("\n\n###")[0]
319+
instruction = query_template.split("###\n")[0]
320320

321321
return Doc(
322322
task_name=task_name,
@@ -457,4 +457,4 @@ def mmlu_pro(line, task_name: str = None):
457457
)
458458

459459
TASKS_TABLE.append(mmlu_pro)
460-
print(TASKS_TABLE)
460+
# print("\n".join([f"extended|{task.name}|0|0" for task in TASKS_TABLE]))

0 commit comments

Comments
 (0)