fixmeup

ghukill · ghukill · commit 973e14e9d883 · 2025-06-13T09:05:05.000-04:00
diff --git a/timdex_dataset_api/dataset.py b/timdex_dataset_api/dataset.py
@@ -489,6 +489,7 @@ def read_batches_iter(
             if not columns:
                 columns = TIMDEX_DATASET_SCHEMA.names
             columns.extend(["__fragment_index", "timdex_record_id"])
+            columns = list(set(columns))
 
         batches = dataset.to_batches(
             columns=columns,
