Add support for the Tcl programming language

dataset_builder/humaneval_to_tcl.py

@@ -0,0 +1,68 @@
+# This script translates problems from the OpenAI HumanEval dataset into Tcl.
+import re
+import ast
+from typing import List
+
+
+class Translator:
+
+    stop = ["\nproc ", "\n#", "\n\n"]
+
+    def file_ext(self):
+        return "tcl"
+
+    def translate_prompt(self, name: str, args: List[ast.arg], _returns, description: str):
+        tcl_description = (
+            "# " + re.sub(r"\n(\s*)", "\n# ", description.strip()) + "\n" if description else ""
+        )
+        arg_names = [arg.arg for arg in args]
+        arg_list = " ".join(arg_names)
+        return f"{tcl_description}proc {name} {{{arg_list}}} {{\n"
+
+    def test_suite_prefix_lines(self, entry_point: str) -> List[str]:
+        return [
+            "",
+            "proc assert_equal {actual expected} {",
+            "    if {$actual ne $expected} {",
+            "        error \"Expected: $expected | Actual: $actual\"",
+            "    }",
+            "}",
+            "",
+            f"interp alias {{}} candidate {{}} {entry_point}",
+            "",
+        ]
+
+    def test_suite_suffix_lines(self) -> List[str]:
+        return []
+
+    def deep_equality(self, left: str, right: str) -> str:
+        return f"assert_equal {left} {right}"
+
+    def gen_literal(self, c: bool | str | int | float | None):
+        if type(c) == bool:
+            return str(c).lower()
+        elif type(c) == str:
+            escaped = c.replace("\\", "\\\\").replace('"', '\\"').replace("$", "\\$").replace("[", "\\[").replace("\n", "\\n")
+            return f'"{escaped}"'
+        elif c is None:
+            return '""'
+        return repr(c)
+
+    def gen_var(self, v: str):
+        return v
+
+    def gen_list(self, l: List[str]):
+        return "[list " + " ".join(l) + "]"
+
+    def gen_tuple(self, t: List[str]):
+        return "[list " + " ".join(t) + "]"
+
+    def gen_dict(self, keys: List[str], values: List[str]):
+        pairs = " ".join(f"{k} {v}" for k, v in zip(keys, values))
+        return "[dict create " + pairs + "]"
+
+    def gen_call(self, func: str, args: List[str]):
+        return "[" + func + " " + " ".join(args) + "]"
+
+    def no_completion_prompt_stub(self):
+        return "    return 0\n}\n"

dataset_builder/terms.csv

@@ -26,3 +26,4 @@ Matlab,m,array,array,array,dictionary,<missing>,true,false
 Haskell,hs,list,list,tuple,association list,Nothing,True,False
 Clojure,clj,vector,list,vector,map,nil,true,false
 Dart,dart,list,list,record,map,null,true,false
+Tcl,tcl,list,list,list,dictionary,empty string,true,false

evaluation/Dockerfile

@@ -106,6 +106,9 @@ RUN apt-get update -yqq && apt-get install -yqq dart
 # Lean
 # RUN wget https://github.com/leanprover/lean4/releases/download/v4.6.0-rc1/lean-4.6.0-rc1-linux.zip -O /tmp/lean.zip && unzip /tmp/lean.zip -d /root/lean/ && ln -s /root/lean/bin/lean /bin/lean
 
+# Tcl
+RUN apt-get install -yqq tcl
+
 # install numpy for humanevalplus
 RUN python3 -m pip install numpy
 

evaluation/src/containerized_eval.py

@@ -29,6 +29,7 @@
 import eval_v
 import eval_lean
 import eval_dart
+import eval_tcl
 import tempfile
 
 
@@ -65,6 +66,7 @@
     "coq": (eval_v.eval_script, ".v"),
     "lean": (eval_lean.eval_script, ".lean"),
     "dart": (eval_dart.eval_script, ".dart"),
+    "tcl": (eval_tcl.eval_script, ".tcl"),
 }
 
 def eval_string_script(language, program):

evaluation/src/eval_tcl.py

@@ -0,0 +1,17 @@
+from pathlib import Path
+from safe_subprocess import run
+
+def eval_script(path: Path):
+    r = run(["tclsh", str(path)])
+    if r.timeout:
+        status = "Timeout"
+    elif r.exit_code == 0:
+        status = "OK"
+    else:
+        status = "Exception"
+    return {
+        "status": status,
+        "exit_code": r.exit_code,
+        "stdout": r.stdout,
+        "stderr": r.stderr,
+    }

results/humaneval-tcl-Qwen2.5_Coder_7B-0.2-reworded.csv

@@ -0,0 +1,162 @@
+problem,pass,total
+HumanEval_0_has_close_elements,186,200
+HumanEval_1_separate_paren_groups,61,200
+HumanEval_10_make_palindrome,1,200
+HumanEval_100_make_a_pile,17,200
+HumanEval_101_words_string,10,200
+HumanEval_102_choose_num,120,200
+HumanEval_103_rounded_avg,0,200
+HumanEval_104_unique_digits,125,200
+HumanEval_105_by_length,0,200
+HumanEval_106_f,83,200
+HumanEval_107_even_odd_palindrome,145,200
+HumanEval_108_count_nums,0,200
+HumanEval_109_move_one_ball,27,200
+HumanEval_11_string_xor,200,200
+HumanEval_110_exchange,7,200
+HumanEval_111_histogram,152,200
+HumanEval_112_reverse_delete,0,200
+HumanEval_113_odd_count,0,200
+HumanEval_114_minSubArraySum,197,200
+HumanEval_115_max_fill,27,200
+HumanEval_116_sort_array,0,200
+HumanEval_117_select_words,169,200
+HumanEval_118_get_closest_vowel,2,200
+HumanEval_119_match_parens,0,200
+HumanEval_12_longest,200,200
+HumanEval_120_maximum,0,200
+HumanEval_121_solution,200,200
+HumanEval_122_add_elements,163,200
+HumanEval_123_get_odd_collatz,84,200
+HumanEval_124_valid_date,52,200
+HumanEval_125_split_words,8,200
+HumanEval_126_is_sorted,51,200
+HumanEval_127_intersection,0,200
+HumanEval_128_prod_signs,23,200
+HumanEval_129_minPath,0,200
+HumanEval_13_greatest_common_divisor,200,200
+HumanEval_130_tri,0,200
+HumanEval_131_digits,52,200
+HumanEval_132_is_nested,0,200
+HumanEval_133_sum_squares,130,200
+HumanEval_134_check_if_last_char_is_a_letter,41,200
+HumanEval_135_can_arrange,150,200
+HumanEval_136_largest_smallest_integers,115,200
+HumanEval_137_compare_one,20,200
+HumanEval_138_is_equal_to_sum_even,164,200
+HumanEval_139_special_factorial,147,200
+HumanEval_14_all_prefixes,200,200
+HumanEval_140_fix_spaces,46,200
+HumanEval_141_file_name_check,19,200
+HumanEval_142_sum_squares,200,200
+HumanEval_143_words_in_sentence,0,200
+HumanEval_144_simplify,16,200
+HumanEval_145_order_by_points,0,200
+HumanEval_146_specialFilter,36,200
+HumanEval_147_get_max_triples,9,200
+HumanEval_148_bf,151,200
+HumanEval_149_sorted_list_sum,0,200
+HumanEval_15_string_sequence,200,200
+HumanEval_150_x_or_y,0,200
+HumanEval_151_double_the_difference,0,200
+HumanEval_152_compare,200,200
+HumanEval_153_Strongest_Extension,58,200
+HumanEval_154_cycpattern_check,7,200
+HumanEval_155_even_odd_count,0,200
+HumanEval_156_int_to_mini_roman,14,200
+HumanEval_157_right_angle_triangle,144,200
+HumanEval_158_find_max,48,200
+HumanEval_159_eat,0,200
+HumanEval_16_count_distinct_characters,192,200
+HumanEval_160_do_algebra,0,200
+HumanEval_161_solve,171,200
+HumanEval_162_string_to_md5,0,200
+HumanEval_163_generate_integers,0,200
+HumanEval_17_parse_music,199,200
+HumanEval_18_how_many_times,178,200
+HumanEval_19_sort_numbers,4,200
+HumanEval_2_truncate_number,200,200
+HumanEval_20_find_closest_elements,0,200
+HumanEval_21_rescale_to_unit,200,200
+HumanEval_22_filter_integers,0,200
+HumanEval_23_strlen,198,200
+HumanEval_24_largest_divisor,200,200
+HumanEval_25_factorize,171,200
+HumanEval_26_remove_duplicates,0,200
+HumanEval_27_flip_case,170,200
+HumanEval_28_concatenate,200,200
+HumanEval_29_filter_by_prefix,200,200
+HumanEval_3_below_zero,200,200
+HumanEval_30_get_positive,56,200
+HumanEval_31_is_prime,200,200
+HumanEval_33_sort_third,0,200
+HumanEval_34_unique,65,200
+HumanEval_35_max_element,200,200
+HumanEval_36_fizz_buzz,185,200
+HumanEval_37_sort_even,29,200
+HumanEval_39_prime_fib,0,200
+HumanEval_4_mean_absolute_deviation,75,200
+HumanEval_40_triples_sum_to_zero,100,200
+HumanEval_41_car_race_collision,133,200
+HumanEval_42_incr_list,200,200
+HumanEval_43_pairs_sum_to_zero,200,200
+HumanEval_44_change_base,200,200
+HumanEval_45_triangle_area,0,200
+HumanEval_46_fib4,89,200
+HumanEval_47_median,191,200
+HumanEval_48_is_palindrome,102,200
+HumanEval_49_modp,111,200
+HumanEval_5_intersperse,180,200
+HumanEval_51_remove_vowels,157,200
+HumanEval_52_below_threshold,200,200
+HumanEval_53_add,200,200
+HumanEval_54_same_chars,7,200
+HumanEval_55_fib,200,200
+HumanEval_56_correct_bracketing,0,200
+HumanEval_57_monotonic,59,200
+HumanEval_58_common,199,200
+HumanEval_59_largest_prime_factor,192,200
+HumanEval_6_parse_nested_parens,200,200
+HumanEval_60_sum_to_n,200,200
+HumanEval_61_correct_bracketing,0,200
+HumanEval_62_derivative,160,200
+HumanEval_63_fibfib,200,200
+HumanEval_64_vowels_count,28,200
+HumanEval_65_circular_shift,9,200
+HumanEval_66_digitSum,200,200
+HumanEval_67_fruit_distribution,15,200
+HumanEval_68_pluck,170,200
+HumanEval_69_search,101,200
+HumanEval_7_filter_by_substring,200,200
+HumanEval_70_strange_sort_list,135,200
+HumanEval_71_triangle_area,0,200
+HumanEval_72_will_it_fly,173,200
+HumanEval_73_smallest_change,125,200
+HumanEval_74_total_match,24,200
+HumanEval_75_is_multiply_prime,21,200
+HumanEval_76_is_simple_power,68,200
+HumanEval_77_iscube,0,200
+HumanEval_78_hex_key,84,200
+HumanEval_79_decimal_to_binary,31,200
+HumanEval_8_sum_product,200,200
+HumanEval_80_is_happy,200,200
+HumanEval_81_numerical_letter_grade,200,200
+HumanEval_82_prime_length,135,200
+HumanEval_83_starts_one_ends,0,200
+HumanEval_84_solve,28,200
+HumanEval_85_add,199,200
+HumanEval_86_anti_shuffle,18,200
+HumanEval_87_get_row,0,200
+HumanEval_88_sort_array,15,200
+HumanEval_89_encrypt,7,200
+HumanEval_9_rolling_max,188,200
+HumanEval_90_next_smallest,17,200
+HumanEval_91_is_bored,0,200
+HumanEval_92_any_int,0,200
+HumanEval_93_encode,0,200
+HumanEval_94_skjkasdkd,15,200
+HumanEval_95_check_dict_case,4,200
+HumanEval_96_count_up_to,48,200
+HumanEval_97_multiply,200,200
+HumanEval_98_count_upper,68,200
+HumanEval_99_closest_integer,71,200