diff --git a/dataset_builder/humaneval_to_tcl.py b/dataset_builder/humaneval_to_tcl.py new file mode 100644 index 0000000000..99d9011737 --- /dev/null +++ b/dataset_builder/humaneval_to_tcl.py @@ -0,0 +1,68 @@ +# This script translates problems from the OpenAI HumanEval dataset into Tcl. +import re +import ast +from typing import List + + +class Translator: + + stop = ["\nproc ", "\n#", "\n\n"] + + def file_ext(self): + return "tcl" + + def translate_prompt(self, name: str, args: List[ast.arg], _returns, description: str): + tcl_description = ( + "# " + re.sub(r"\n(\s*)", "\n# ", description.strip()) + "\n" if description else "" + ) + arg_names = [arg.arg for arg in args] + arg_list = " ".join(arg_names) + return f"{tcl_description}proc {name} {{{arg_list}}} {{\n" + + def test_suite_prefix_lines(self, entry_point: str) -> List[str]: + return [ + "", + "proc assert_equal {actual expected} {", + " if {$actual ne $expected} {", + " error \"Expected: $expected | Actual: $actual\"", + " }", + "}", + "", + f"interp alias {{}} candidate {{}} {entry_point}", + "", + ] + + def test_suite_suffix_lines(self) -> List[str]: + return [] + + def deep_equality(self, left: str, right: str) -> str: + return f"assert_equal {left} {right}" + + def gen_literal(self, c: bool | str | int | float | None): + if type(c) == bool: + return str(c).lower() + elif type(c) == str: + escaped = c.replace("\\", "\\\\").replace('"', '\\"').replace("$", "\\$").replace("[", "\\[").replace("\n", "\\n") + return f'"{escaped}"' + elif c is None: + return '""' + return repr(c) + + def gen_var(self, v: str): + return v + + def gen_list(self, l: List[str]): + return "[list " + " ".join(l) + "]" + + def gen_tuple(self, t: List[str]): + return "[list " + " ".join(t) + "]" + + def gen_dict(self, keys: List[str], values: List[str]): + pairs = " ".join(f"{k} {v}" for k, v in zip(keys, values)) + return "[dict create " + pairs + "]" + + def gen_call(self, func: str, args: List[str]): + return "[" + func + " " + " ".join(args) + "]" + + def no_completion_prompt_stub(self): + return " return 0\n}\n" diff --git a/dataset_builder/terms.csv b/dataset_builder/terms.csv index 8b089d40fe..1324b624eb 100644 --- a/dataset_builder/terms.csv +++ b/dataset_builder/terms.csv @@ -26,3 +26,4 @@ Matlab,m,array,array,array,dictionary,,true,false Haskell,hs,list,list,tuple,association list,Nothing,True,False Clojure,clj,vector,list,vector,map,nil,true,false Dart,dart,list,list,record,map,null,true,false +Tcl,tcl,list,list,list,dictionary,empty string,true,false diff --git a/evaluation/Dockerfile b/evaluation/Dockerfile index 0808bfb027..651204ae98 100644 --- a/evaluation/Dockerfile +++ b/evaluation/Dockerfile @@ -106,6 +106,9 @@ RUN apt-get update -yqq && apt-get install -yqq dart # Lean # RUN wget https://github.com/leanprover/lean4/releases/download/v4.6.0-rc1/lean-4.6.0-rc1-linux.zip -O /tmp/lean.zip && unzip /tmp/lean.zip -d /root/lean/ && ln -s /root/lean/bin/lean /bin/lean +# Tcl +RUN apt-get install -yqq tcl + # install numpy for humanevalplus RUN python3 -m pip install numpy diff --git a/evaluation/src/containerized_eval.py b/evaluation/src/containerized_eval.py index 533fb68274..72a03d9fdc 100644 --- a/evaluation/src/containerized_eval.py +++ b/evaluation/src/containerized_eval.py @@ -29,6 +29,7 @@ import eval_v import eval_lean import eval_dart +import eval_tcl import tempfile @@ -65,6 +66,7 @@ "coq": (eval_v.eval_script, ".v"), "lean": (eval_lean.eval_script, ".lean"), "dart": (eval_dart.eval_script, ".dart"), + "tcl": (eval_tcl.eval_script, ".tcl"), } def eval_string_script(language, program): diff --git a/evaluation/src/eval_tcl.py b/evaluation/src/eval_tcl.py new file mode 100644 index 0000000000..f11d03477f --- /dev/null +++ b/evaluation/src/eval_tcl.py @@ -0,0 +1,17 @@ +from pathlib import Path +from safe_subprocess import run + +def eval_script(path: Path): + r = run(["tclsh", str(path)]) + if r.timeout: + status = "Timeout" + elif r.exit_code == 0: + status = "OK" + else: + status = "Exception" + return { + "status": status, + "exit_code": r.exit_code, + "stdout": r.stdout, + "stderr": r.stderr, + } diff --git a/results/humaneval-tcl-Qwen2.5_Coder_7B-0.2-reworded.csv b/results/humaneval-tcl-Qwen2.5_Coder_7B-0.2-reworded.csv new file mode 100644 index 0000000000..b14dec9b8f --- /dev/null +++ b/results/humaneval-tcl-Qwen2.5_Coder_7B-0.2-reworded.csv @@ -0,0 +1,162 @@ +problem,pass,total +HumanEval_0_has_close_elements,186,200 +HumanEval_1_separate_paren_groups,61,200 +HumanEval_10_make_palindrome,1,200 +HumanEval_100_make_a_pile,17,200 +HumanEval_101_words_string,10,200 +HumanEval_102_choose_num,120,200 +HumanEval_103_rounded_avg,0,200 +HumanEval_104_unique_digits,125,200 +HumanEval_105_by_length,0,200 +HumanEval_106_f,83,200 +HumanEval_107_even_odd_palindrome,145,200 +HumanEval_108_count_nums,0,200 +HumanEval_109_move_one_ball,27,200 +HumanEval_11_string_xor,200,200 +HumanEval_110_exchange,7,200 +HumanEval_111_histogram,152,200 +HumanEval_112_reverse_delete,0,200 +HumanEval_113_odd_count,0,200 +HumanEval_114_minSubArraySum,197,200 +HumanEval_115_max_fill,27,200 +HumanEval_116_sort_array,0,200 +HumanEval_117_select_words,169,200 +HumanEval_118_get_closest_vowel,2,200 +HumanEval_119_match_parens,0,200 +HumanEval_12_longest,200,200 +HumanEval_120_maximum,0,200 +HumanEval_121_solution,200,200 +HumanEval_122_add_elements,163,200 +HumanEval_123_get_odd_collatz,84,200 +HumanEval_124_valid_date,52,200 +HumanEval_125_split_words,8,200 +HumanEval_126_is_sorted,51,200 +HumanEval_127_intersection,0,200 +HumanEval_128_prod_signs,23,200 +HumanEval_129_minPath,0,200 +HumanEval_13_greatest_common_divisor,200,200 +HumanEval_130_tri,0,200 +HumanEval_131_digits,52,200 +HumanEval_132_is_nested,0,200 +HumanEval_133_sum_squares,130,200 +HumanEval_134_check_if_last_char_is_a_letter,41,200 +HumanEval_135_can_arrange,150,200 +HumanEval_136_largest_smallest_integers,115,200 +HumanEval_137_compare_one,20,200 +HumanEval_138_is_equal_to_sum_even,164,200 +HumanEval_139_special_factorial,147,200 +HumanEval_14_all_prefixes,200,200 +HumanEval_140_fix_spaces,46,200 +HumanEval_141_file_name_check,19,200 +HumanEval_142_sum_squares,200,200 +HumanEval_143_words_in_sentence,0,200 +HumanEval_144_simplify,16,200 +HumanEval_145_order_by_points,0,200 +HumanEval_146_specialFilter,36,200 +HumanEval_147_get_max_triples,9,200 +HumanEval_148_bf,151,200 +HumanEval_149_sorted_list_sum,0,200 +HumanEval_15_string_sequence,200,200 +HumanEval_150_x_or_y,0,200 +HumanEval_151_double_the_difference,0,200 +HumanEval_152_compare,200,200 +HumanEval_153_Strongest_Extension,58,200 +HumanEval_154_cycpattern_check,7,200 +HumanEval_155_even_odd_count,0,200 +HumanEval_156_int_to_mini_roman,14,200 +HumanEval_157_right_angle_triangle,144,200 +HumanEval_158_find_max,48,200 +HumanEval_159_eat,0,200 +HumanEval_16_count_distinct_characters,192,200 +HumanEval_160_do_algebra,0,200 +HumanEval_161_solve,171,200 +HumanEval_162_string_to_md5,0,200 +HumanEval_163_generate_integers,0,200 +HumanEval_17_parse_music,199,200 +HumanEval_18_how_many_times,178,200 +HumanEval_19_sort_numbers,4,200 +HumanEval_2_truncate_number,200,200 +HumanEval_20_find_closest_elements,0,200 +HumanEval_21_rescale_to_unit,200,200 +HumanEval_22_filter_integers,0,200 +HumanEval_23_strlen,198,200 +HumanEval_24_largest_divisor,200,200 +HumanEval_25_factorize,171,200 +HumanEval_26_remove_duplicates,0,200 +HumanEval_27_flip_case,170,200 +HumanEval_28_concatenate,200,200 +HumanEval_29_filter_by_prefix,200,200 +HumanEval_3_below_zero,200,200 +HumanEval_30_get_positive,56,200 +HumanEval_31_is_prime,200,200 +HumanEval_33_sort_third,0,200 +HumanEval_34_unique,65,200 +HumanEval_35_max_element,200,200 +HumanEval_36_fizz_buzz,185,200 +HumanEval_37_sort_even,29,200 +HumanEval_39_prime_fib,0,200 +HumanEval_4_mean_absolute_deviation,75,200 +HumanEval_40_triples_sum_to_zero,100,200 +HumanEval_41_car_race_collision,133,200 +HumanEval_42_incr_list,200,200 +HumanEval_43_pairs_sum_to_zero,200,200 +HumanEval_44_change_base,200,200 +HumanEval_45_triangle_area,0,200 +HumanEval_46_fib4,89,200 +HumanEval_47_median,191,200 +HumanEval_48_is_palindrome,102,200 +HumanEval_49_modp,111,200 +HumanEval_5_intersperse,180,200 +HumanEval_51_remove_vowels,157,200 +HumanEval_52_below_threshold,200,200 +HumanEval_53_add,200,200 +HumanEval_54_same_chars,7,200 +HumanEval_55_fib,200,200 +HumanEval_56_correct_bracketing,0,200 +HumanEval_57_monotonic,59,200 +HumanEval_58_common,199,200 +HumanEval_59_largest_prime_factor,192,200 +HumanEval_6_parse_nested_parens,200,200 +HumanEval_60_sum_to_n,200,200 +HumanEval_61_correct_bracketing,0,200 +HumanEval_62_derivative,160,200 +HumanEval_63_fibfib,200,200 +HumanEval_64_vowels_count,28,200 +HumanEval_65_circular_shift,9,200 +HumanEval_66_digitSum,200,200 +HumanEval_67_fruit_distribution,15,200 +HumanEval_68_pluck,170,200 +HumanEval_69_search,101,200 +HumanEval_7_filter_by_substring,200,200 +HumanEval_70_strange_sort_list,135,200 +HumanEval_71_triangle_area,0,200 +HumanEval_72_will_it_fly,173,200 +HumanEval_73_smallest_change,125,200 +HumanEval_74_total_match,24,200 +HumanEval_75_is_multiply_prime,21,200 +HumanEval_76_is_simple_power,68,200 +HumanEval_77_iscube,0,200 +HumanEval_78_hex_key,84,200 +HumanEval_79_decimal_to_binary,31,200 +HumanEval_8_sum_product,200,200 +HumanEval_80_is_happy,200,200 +HumanEval_81_numerical_letter_grade,200,200 +HumanEval_82_prime_length,135,200 +HumanEval_83_starts_one_ends,0,200 +HumanEval_84_solve,28,200 +HumanEval_85_add,199,200 +HumanEval_86_anti_shuffle,18,200 +HumanEval_87_get_row,0,200 +HumanEval_88_sort_array,15,200 +HumanEval_89_encrypt,7,200 +HumanEval_9_rolling_max,188,200 +HumanEval_90_next_smallest,17,200 +HumanEval_91_is_bored,0,200 +HumanEval_92_any_int,0,200 +HumanEval_93_encode,0,200 +HumanEval_94_skjkasdkd,15,200 +HumanEval_95_check_dict_case,4,200 +HumanEval_96_count_up_to,48,200 +HumanEval_97_multiply,200,200 +HumanEval_98_count_upper,68,200 +HumanEval_99_closest_integer,71,200 diff --git a/results/humaneval-tcl-Qwen2.5_Coder_7B-0.8-reworded.csv b/results/humaneval-tcl-Qwen2.5_Coder_7B-0.8-reworded.csv new file mode 100644 index 0000000000..299f6da735 --- /dev/null +++ b/results/humaneval-tcl-Qwen2.5_Coder_7B-0.8-reworded.csv @@ -0,0 +1,162 @@ +problem,pass,total +HumanEval_0_has_close_elements,67,200 +HumanEval_1_separate_paren_groups,26,200 +HumanEval_10_make_palindrome,0,200 +HumanEval_100_make_a_pile,14,200 +HumanEval_101_words_string,22,200 +HumanEval_102_choose_num,59,200 +HumanEval_103_rounded_avg,0,200 +HumanEval_104_unique_digits,49,200 +HumanEval_105_by_length,23,200 +HumanEval_106_f,55,200 +HumanEval_107_even_odd_palindrome,27,200 +HumanEval_108_count_nums,3,200 +HumanEval_109_move_one_ball,15,200 +HumanEval_11_string_xor,134,200 +HumanEval_110_exchange,19,200 +HumanEval_111_histogram,40,200 +HumanEval_112_reverse_delete,9,200 +HumanEval_113_odd_count,1,200 +HumanEval_114_minSubArraySum,63,200 +HumanEval_115_max_fill,23,200 +HumanEval_116_sort_array,0,200 +HumanEval_117_select_words,41,200 +HumanEval_118_get_closest_vowel,4,200 +HumanEval_119_match_parens,0,200 +HumanEval_12_longest,120,200 +HumanEval_120_maximum,5,200 +HumanEval_121_solution,122,200 +HumanEval_122_add_elements,108,200 +HumanEval_123_get_odd_collatz,48,200 +HumanEval_124_valid_date,21,200 +HumanEval_125_split_words,3,200 +HumanEval_126_is_sorted,12,200 +HumanEval_127_intersection,1,200 +HumanEval_128_prod_signs,34,200 +HumanEval_129_minPath,0,200 +HumanEval_13_greatest_common_divisor,155,200 +HumanEval_130_tri,0,200 +HumanEval_131_digits,26,200 +HumanEval_132_is_nested,0,200 +HumanEval_133_sum_squares,49,200 +HumanEval_134_check_if_last_char_is_a_letter,8,200 +HumanEval_135_can_arrange,33,200 +HumanEval_136_largest_smallest_integers,49,200 +HumanEval_137_compare_one,29,200 +HumanEval_138_is_equal_to_sum_even,31,200 +HumanEval_139_special_factorial,61,200 +HumanEval_14_all_prefixes,156,200 +HumanEval_140_fix_spaces,7,200 +HumanEval_141_file_name_check,1,200 +HumanEval_142_sum_squares,78,200 +HumanEval_143_words_in_sentence,18,200 +HumanEval_144_simplify,8,200 +HumanEval_145_order_by_points,0,200 +HumanEval_146_specialFilter,71,200 +HumanEval_147_get_max_triples,42,200 +HumanEval_148_bf,45,200 +HumanEval_149_sorted_list_sum,1,200 +HumanEval_15_string_sequence,129,200 +HumanEval_150_x_or_y,3,200 +HumanEval_151_double_the_difference,5,200 +HumanEval_152_compare,160,200 +HumanEval_153_Strongest_Extension,23,200 +HumanEval_154_cycpattern_check,9,200 +HumanEval_155_even_odd_count,30,200 +HumanEval_156_int_to_mini_roman,23,200 +HumanEval_157_right_angle_triangle,17,200 +HumanEval_158_find_max,22,200 +HumanEval_159_eat,29,200 +HumanEval_16_count_distinct_characters,67,200 +HumanEval_160_do_algebra,0,200 +HumanEval_161_solve,14,200 +HumanEval_162_string_to_md5,0,200 +HumanEval_163_generate_integers,0,200 +HumanEval_17_parse_music,88,200 +HumanEval_18_how_many_times,66,200 +HumanEval_19_sort_numbers,16,200 +HumanEval_2_truncate_number,145,200 +HumanEval_20_find_closest_elements,10,200 +HumanEval_21_rescale_to_unit,110,200 +HumanEval_22_filter_integers,14,200 +HumanEval_23_strlen,171,200 +HumanEval_24_largest_divisor,124,200 +HumanEval_25_factorize,97,200 +HumanEval_26_remove_duplicates,12,200 +HumanEval_27_flip_case,55,200 +HumanEval_28_concatenate,176,200 +HumanEval_29_filter_by_prefix,124,200 +HumanEval_3_below_zero,170,200 +HumanEval_30_get_positive,58,200 +HumanEval_31_is_prime,83,200 +HumanEval_33_sort_third,13,200 +HumanEval_34_unique,48,200 +HumanEval_35_max_element,162,200 +HumanEval_36_fizz_buzz,91,200 +HumanEval_37_sort_even,15,200 +HumanEval_39_prime_fib,2,200 +HumanEval_4_mean_absolute_deviation,50,200 +HumanEval_40_triples_sum_to_zero,61,200 +HumanEval_41_car_race_collision,35,200 +HumanEval_42_incr_list,149,200 +HumanEval_43_pairs_sum_to_zero,96,200 +HumanEval_44_change_base,111,200 +HumanEval_45_triangle_area,74,200 +HumanEval_46_fib4,51,200 +HumanEval_47_median,67,200 +HumanEval_48_is_palindrome,70,200 +HumanEval_49_modp,35,200 +HumanEval_5_intersperse,110,200 +HumanEval_51_remove_vowels,81,200 +HumanEval_52_below_threshold,117,200 +HumanEval_53_add,200,200 +HumanEval_54_same_chars,3,200 +HumanEval_55_fib,142,200 +HumanEval_56_correct_bracketing,8,200 +HumanEval_57_monotonic,22,200 +HumanEval_58_common,76,200 +HumanEval_59_largest_prime_factor,67,200 +HumanEval_6_parse_nested_parens,86,200 +HumanEval_60_sum_to_n,176,200 +HumanEval_61_correct_bracketing,13,200 +HumanEval_62_derivative,60,200 +HumanEval_63_fibfib,95,200 +HumanEval_64_vowels_count,14,200 +HumanEval_65_circular_shift,1,200 +HumanEval_66_digitSum,86,200 +HumanEval_67_fruit_distribution,58,200 +HumanEval_68_pluck,58,200 +HumanEval_69_search,51,200 +HumanEval_7_filter_by_substring,155,200 +HumanEval_70_strange_sort_list,40,200 +HumanEval_71_triangle_area,2,200 +HumanEval_72_will_it_fly,26,200 +HumanEval_73_smallest_change,111,200 +HumanEval_74_total_match,55,200 +HumanEval_75_is_multiply_prime,8,200 +HumanEval_76_is_simple_power,14,200 +HumanEval_77_iscube,9,200 +HumanEval_78_hex_key,76,200 +HumanEval_79_decimal_to_binary,39,200 +HumanEval_8_sum_product,136,200 +HumanEval_80_is_happy,64,200 +HumanEval_81_numerical_letter_grade,100,200 +HumanEval_82_prime_length,28,200 +HumanEval_83_starts_one_ends,0,200 +HumanEval_84_solve,7,200 +HumanEval_85_add,75,200 +HumanEval_86_anti_shuffle,45,200 +HumanEval_87_get_row,16,200 +HumanEval_88_sort_array,33,200 +HumanEval_89_encrypt,24,200 +HumanEval_9_rolling_max,106,200 +HumanEval_90_next_smallest,33,200 +HumanEval_91_is_bored,3,200 +HumanEval_92_any_int,7,200 +HumanEval_93_encode,0,200 +HumanEval_94_skjkasdkd,15,200 +HumanEval_95_check_dict_case,10,200 +HumanEval_96_count_up_to,39,200 +HumanEval_97_multiply,165,200 +HumanEval_98_count_upper,27,200 +HumanEval_99_closest_integer,39,200 diff --git a/results/humaneval-tcl-Qwen2.5_Coder_7B_Instruct-0.2-reworded.csv b/results/humaneval-tcl-Qwen2.5_Coder_7B_Instruct-0.2-reworded.csv new file mode 100644 index 0000000000..c8e605bea6 --- /dev/null +++ b/results/humaneval-tcl-Qwen2.5_Coder_7B_Instruct-0.2-reworded.csv @@ -0,0 +1,162 @@ +problem,pass,total +HumanEval_0_has_close_elements,200,200 +HumanEval_1_separate_paren_groups,0,200 +HumanEval_10_make_palindrome,12,200 +HumanEval_100_make_a_pile,198,200 +HumanEval_101_words_string,0,200 +HumanEval_102_choose_num,200,200 +HumanEval_103_rounded_avg,0,200 +HumanEval_104_unique_digits,189,200 +HumanEval_105_by_length,59,200 +HumanEval_106_f,190,200 +HumanEval_107_even_odd_palindrome,89,200 +HumanEval_108_count_nums,12,200 +HumanEval_109_move_one_ball,65,200 +HumanEval_11_string_xor,199,200 +HumanEval_110_exchange,35,200 +HumanEval_111_histogram,0,200 +HumanEval_112_reverse_delete,0,200 +HumanEval_113_odd_count,0,200 +HumanEval_114_minSubArraySum,198,200 +HumanEval_115_max_fill,0,200 +HumanEval_116_sort_array,0,200 +HumanEval_117_select_words,21,200 +HumanEval_118_get_closest_vowel,167,200 +HumanEval_119_match_parens,52,200 +HumanEval_12_longest,187,200 +HumanEval_120_maximum,0,200 +HumanEval_121_solution,5,200 +HumanEval_122_add_elements,200,200 +HumanEval_123_get_odd_collatz,181,200 +HumanEval_124_valid_date,3,200 +HumanEval_125_split_words,55,200 +HumanEval_126_is_sorted,74,200 +HumanEval_127_intersection,0,200 +HumanEval_128_prod_signs,84,200 +HumanEval_129_minPath,0,200 +HumanEval_13_greatest_common_divisor,200,200 +HumanEval_130_tri,0,200 +HumanEval_131_digits,95,200 +HumanEval_132_is_nested,0,200 +HumanEval_133_sum_squares,3,200 +HumanEval_134_check_if_last_char_is_a_letter,0,200 +HumanEval_135_can_arrange,195,200 +HumanEval_136_largest_smallest_integers,99,200 +HumanEval_137_compare_one,111,200 +HumanEval_138_is_equal_to_sum_even,200,200 +HumanEval_139_special_factorial,0,200 +HumanEval_14_all_prefixes,187,200 +HumanEval_140_fix_spaces,116,200 +HumanEval_141_file_name_check,12,200 +HumanEval_142_sum_squares,0,200 +HumanEval_143_words_in_sentence,44,200 +HumanEval_144_simplify,9,200 +HumanEval_145_order_by_points,0,200 +HumanEval_146_specialFilter,128,200 +HumanEval_147_get_max_triples,1,200 +HumanEval_148_bf,196,200 +HumanEval_149_sorted_list_sum,0,200 +HumanEval_15_string_sequence,200,200 +HumanEval_150_x_or_y,60,200 +HumanEval_151_double_the_difference,0,200 +HumanEval_152_compare,170,200 +HumanEval_153_Strongest_Extension,67,200 +HumanEval_154_cycpattern_check,0,200 +HumanEval_155_even_odd_count,91,200 +HumanEval_156_int_to_mini_roman,19,200 +HumanEval_157_right_angle_triangle,0,200 +HumanEval_158_find_max,178,200 +HumanEval_159_eat,121,200 +HumanEval_16_count_distinct_characters,200,200 +HumanEval_160_do_algebra,0,200 +HumanEval_161_solve,0,200 +HumanEval_162_string_to_md5,0,200 +HumanEval_163_generate_integers,0,200 +HumanEval_17_parse_music,200,200 +HumanEval_18_how_many_times,45,200 +HumanEval_19_sort_numbers,15,200 +HumanEval_2_truncate_number,140,200 +HumanEval_20_find_closest_elements,1,200 +HumanEval_21_rescale_to_unit,92,200 +HumanEval_22_filter_integers,160,200 +HumanEval_23_strlen,200,200 +HumanEval_24_largest_divisor,200,200 +HumanEval_25_factorize,188,200 +HumanEval_26_remove_duplicates,172,200 +HumanEval_27_flip_case,200,200 +HumanEval_28_concatenate,200,200 +HumanEval_29_filter_by_prefix,87,200 +HumanEval_3_below_zero,200,200 +HumanEval_30_get_positive,200,200 +HumanEval_31_is_prime,200,200 +HumanEval_33_sort_third,1,200 +HumanEval_34_unique,60,200 +HumanEval_35_max_element,200,200 +HumanEval_36_fizz_buzz,69,200 +HumanEval_37_sort_even,0,200 +HumanEval_39_prime_fib,0,200 +HumanEval_4_mean_absolute_deviation,200,200 +HumanEval_40_triples_sum_to_zero,192,200 +HumanEval_41_car_race_collision,200,200 +HumanEval_42_incr_list,200,200 +HumanEval_43_pairs_sum_to_zero,200,200 +HumanEval_44_change_base,198,200 +HumanEval_45_triangle_area,138,200 +HumanEval_46_fib4,87,200 +HumanEval_47_median,0,200 +HumanEval_48_is_palindrome,1,200 +HumanEval_49_modp,99,200 +HumanEval_5_intersperse,200,200 +HumanEval_51_remove_vowels,29,200 +HumanEval_52_below_threshold,200,200 +HumanEval_53_add,200,200 +HumanEval_54_same_chars,5,200 +HumanEval_55_fib,199,200 +HumanEval_56_correct_bracketing,0,200 +HumanEval_57_monotonic,7,200 +HumanEval_58_common,110,200 +HumanEval_59_largest_prime_factor,170,200 +HumanEval_6_parse_nested_parens,69,200 +HumanEval_60_sum_to_n,200,200 +HumanEval_61_correct_bracketing,0,200 +HumanEval_62_derivative,198,200 +HumanEval_63_fibfib,138,200 +HumanEval_64_vowels_count,14,200 +HumanEval_65_circular_shift,0,200 +HumanEval_66_digitSum,200,200 +HumanEval_67_fruit_distribution,136,200 +HumanEval_68_pluck,145,200 +HumanEval_69_search,200,200 +HumanEval_7_filter_by_substring,187,200 +HumanEval_70_strange_sort_list,115,200 +HumanEval_71_triangle_area,0,200 +HumanEval_72_will_it_fly,113,200 +HumanEval_73_smallest_change,21,200 +HumanEval_74_total_match,200,200 +HumanEval_75_is_multiply_prime,25,200 +HumanEval_76_is_simple_power,56,200 +HumanEval_77_iscube,15,200 +HumanEval_78_hex_key,47,200 +HumanEval_79_decimal_to_binary,197,200 +HumanEval_8_sum_product,200,200 +HumanEval_80_is_happy,200,200 +HumanEval_81_numerical_letter_grade,200,200 +HumanEval_82_prime_length,77,200 +HumanEval_83_starts_one_ends,0,200 +HumanEval_84_solve,127,200 +HumanEval_85_add,9,200 +HumanEval_86_anti_shuffle,0,200 +HumanEval_87_get_row,0,200 +HumanEval_88_sort_array,169,200 +HumanEval_89_encrypt,123,200 +HumanEval_9_rolling_max,200,200 +HumanEval_90_next_smallest,0,200 +HumanEval_91_is_bored,0,200 +HumanEval_92_any_int,165,200 +HumanEval_93_encode,9,200 +HumanEval_94_skjkasdkd,0,200 +HumanEval_95_check_dict_case,25,200 +HumanEval_96_count_up_to,200,200 +HumanEval_97_multiply,200,200 +HumanEval_98_count_upper,45,200 +HumanEval_99_closest_integer,184,200 diff --git a/results/humaneval-tcl-Qwen2.5_Coder_7B_Instruct-0.8-reworded.csv b/results/humaneval-tcl-Qwen2.5_Coder_7B_Instruct-0.8-reworded.csv new file mode 100644 index 0000000000..eb97bbac2c --- /dev/null +++ b/results/humaneval-tcl-Qwen2.5_Coder_7B_Instruct-0.8-reworded.csv @@ -0,0 +1,162 @@ +problem,pass,total +HumanEval_0_has_close_elements,190,200 +HumanEval_1_separate_paren_groups,5,200 +HumanEval_10_make_palindrome,17,200 +HumanEval_100_make_a_pile,191,200 +HumanEval_101_words_string,0,200 +HumanEval_102_choose_num,198,200 +HumanEval_103_rounded_avg,0,200 +HumanEval_104_unique_digits,174,200 +HumanEval_105_by_length,53,200 +HumanEval_106_f,111,200 +HumanEval_107_even_odd_palindrome,106,200 +HumanEval_108_count_nums,27,200 +HumanEval_109_move_one_ball,72,200 +HumanEval_11_string_xor,186,200 +HumanEval_110_exchange,95,200 +HumanEval_111_histogram,16,200 +HumanEval_112_reverse_delete,1,200 +HumanEval_113_odd_count,5,200 +HumanEval_114_minSubArraySum,99,200 +HumanEval_115_max_fill,1,200 +HumanEval_116_sort_array,0,200 +HumanEval_117_select_words,42,200 +HumanEval_118_get_closest_vowel,139,200 +HumanEval_119_match_parens,61,200 +HumanEval_12_longest,163,200 +HumanEval_120_maximum,26,200 +HumanEval_121_solution,28,200 +HumanEval_122_add_elements,193,200 +HumanEval_123_get_odd_collatz,160,200 +HumanEval_124_valid_date,11,200 +HumanEval_125_split_words,93,200 +HumanEval_126_is_sorted,60,200 +HumanEval_127_intersection,0,200 +HumanEval_128_prod_signs,71,200 +HumanEval_129_minPath,0,200 +HumanEval_13_greatest_common_divisor,200,200 +HumanEval_130_tri,4,200 +HumanEval_131_digits,130,200 +HumanEval_132_is_nested,0,200 +HumanEval_133_sum_squares,8,200 +HumanEval_134_check_if_last_char_is_a_letter,11,200 +HumanEval_135_can_arrange,136,200 +HumanEval_136_largest_smallest_integers,118,200 +HumanEval_137_compare_one,80,200 +HumanEval_138_is_equal_to_sum_even,187,200 +HumanEval_139_special_factorial,67,200 +HumanEval_14_all_prefixes,181,200 +HumanEval_140_fix_spaces,103,200 +HumanEval_141_file_name_check,30,200 +HumanEval_142_sum_squares,5,200 +HumanEval_143_words_in_sentence,89,200 +HumanEval_144_simplify,91,200 +HumanEval_145_order_by_points,0,200 +HumanEval_146_specialFilter,111,200 +HumanEval_147_get_max_triples,33,200 +HumanEval_148_bf,175,200 +HumanEval_149_sorted_list_sum,0,200 +HumanEval_15_string_sequence,200,200 +HumanEval_150_x_or_y,108,200 +HumanEval_151_double_the_difference,0,200 +HumanEval_152_compare,171,200 +HumanEval_153_Strongest_Extension,127,200 +HumanEval_154_cycpattern_check,3,200 +HumanEval_155_even_odd_count,80,200 +HumanEval_156_int_to_mini_roman,50,200 +HumanEval_157_right_angle_triangle,1,200 +HumanEval_158_find_max,143,200 +HumanEval_159_eat,176,200 +HumanEval_16_count_distinct_characters,198,200 +HumanEval_160_do_algebra,0,200 +HumanEval_161_solve,19,200 +HumanEval_162_string_to_md5,0,200 +HumanEval_163_generate_integers,0,200 +HumanEval_17_parse_music,194,200 +HumanEval_18_how_many_times,89,200 +HumanEval_19_sort_numbers,17,200 +HumanEval_2_truncate_number,169,200 +HumanEval_20_find_closest_elements,9,200 +HumanEval_21_rescale_to_unit,122,200 +HumanEval_22_filter_integers,186,200 +HumanEval_23_strlen,185,200 +HumanEval_24_largest_divisor,200,200 +HumanEval_25_factorize,171,200 +HumanEval_26_remove_duplicates,106,200 +HumanEval_27_flip_case,183,200 +HumanEval_28_concatenate,199,200 +HumanEval_29_filter_by_prefix,88,200 +HumanEval_3_below_zero,198,200 +HumanEval_30_get_positive,200,200 +HumanEval_31_is_prime,200,200 +HumanEval_33_sort_third,7,200 +HumanEval_34_unique,51,200 +HumanEval_35_max_element,200,200 +HumanEval_36_fizz_buzz,48,200 +HumanEval_37_sort_even,4,200 +HumanEval_39_prime_fib,1,200 +HumanEval_4_mean_absolute_deviation,119,200 +HumanEval_40_triples_sum_to_zero,125,200 +HumanEval_41_car_race_collision,200,200 +HumanEval_42_incr_list,200,200 +HumanEval_43_pairs_sum_to_zero,199,200 +HumanEval_44_change_base,180,200 +HumanEval_45_triangle_area,122,200 +HumanEval_46_fib4,96,200 +HumanEval_47_median,27,200 +HumanEval_48_is_palindrome,45,200 +HumanEval_49_modp,114,200 +HumanEval_5_intersperse,165,200 +HumanEval_51_remove_vowels,61,200 +HumanEval_52_below_threshold,199,200 +HumanEval_53_add,200,200 +HumanEval_54_same_chars,3,200 +HumanEval_55_fib,191,200 +HumanEval_56_correct_bracketing,0,200 +HumanEval_57_monotonic,41,200 +HumanEval_58_common,66,200 +HumanEval_59_largest_prime_factor,191,200 +HumanEval_6_parse_nested_parens,148,200 +HumanEval_60_sum_to_n,200,200 +HumanEval_61_correct_bracketing,0,200 +HumanEval_62_derivative,179,200 +HumanEval_63_fibfib,134,200 +HumanEval_64_vowels_count,53,200 +HumanEval_65_circular_shift,3,200 +HumanEval_66_digitSum,176,200 +HumanEval_67_fruit_distribution,115,200 +HumanEval_68_pluck,76,200 +HumanEval_69_search,177,200 +HumanEval_7_filter_by_substring,181,200 +HumanEval_70_strange_sort_list,108,200 +HumanEval_71_triangle_area,0,200 +HumanEval_72_will_it_fly,97,200 +HumanEval_73_smallest_change,84,200 +HumanEval_74_total_match,175,200 +HumanEval_75_is_multiply_prime,15,200 +HumanEval_76_is_simple_power,67,200 +HumanEval_77_iscube,29,200 +HumanEval_78_hex_key,61,200 +HumanEval_79_decimal_to_binary,155,200 +HumanEval_8_sum_product,181,200 +HumanEval_80_is_happy,172,200 +HumanEval_81_numerical_letter_grade,199,200 +HumanEval_82_prime_length,109,200 +HumanEval_83_starts_one_ends,2,200 +HumanEval_84_solve,151,200 +HumanEval_85_add,63,200 +HumanEval_86_anti_shuffle,17,200 +HumanEval_87_get_row,3,200 +HumanEval_88_sort_array,125,200 +HumanEval_89_encrypt,47,200 +HumanEval_9_rolling_max,199,200 +HumanEval_90_next_smallest,3,200 +HumanEval_91_is_bored,3,200 +HumanEval_92_any_int,117,200 +HumanEval_93_encode,12,200 +HumanEval_94_skjkasdkd,37,200 +HumanEval_95_check_dict_case,33,200 +HumanEval_96_count_up_to,173,200 +HumanEval_97_multiply,197,200 +HumanEval_98_count_upper,40,200 +HumanEval_99_closest_integer,148,200