Add fast splitter to plotter

Author: johnramsden

eval/generate_all_plots.sh

@@ -85,8 +85,8 @@ split_data_if_needed() {
 
     if ! check_split_data "$split_dir" "$name"; then
         echo "📊 Splitting $name data..."
-        python split_data.py "$source_dir"
-        
+        python3 split_data_fast.py "$source_dir"
+
         if check_split_data "$split_dir" "$name"; then
             echo "✅ $name data splitting completed"
         else
@@ -111,51 +111,51 @@ echo "===================================="
 
 echo "📈 Generating BLOCK-PROMO individual plots..."
 
-## Throughput plots
-#python plot_throughput.py "$BLOCK_SPLIT_DIR" \
-#    --bucket-seconds $BUCKET_SECONDS \
-#    --output-dir "${OUTPUT_DIR}/block_individual" \
-#    --metrics bytes_total # written_bytes_total
-
-## Raw latency plots
-#python plot_latency.py "$BLOCK_SPLIT_DIR" \
-#    --output-dir "${OUTPUT_DIR}/block_individual" \
-#    --metrics get_total_latency_ms # device_write_latency_ms disk_write_latency_ms get_miss_latency_ms
-#
-## Smoothed latency plots
-#python plot_latency_smoothed.py "$BLOCK_SPLIT_DIR" \
-#    --window-seconds $WINDOW_SECONDS \
-#    --output-dir "${OUTPUT_DIR}/block_individual" \
-#    --metrics get_total_latency_ms # device_write_latency_ms disk_write_latency_ms get_miss_latency_ms
-
-## Hit ratio plots
-#python plot_hitratio.py "$BLOCK_SPLIT_DIR" \
-#    --output-dir "${OUTPUT_DIR}/block_individual"
+# Throughput plots
+python plot_throughput.py "$BLOCK_SPLIT_DIR" \
+    --bucket-seconds $BUCKET_SECONDS \
+    --output-dir "${OUTPUT_DIR}/block_individual" \
+    --metrics bytes_total written_bytes_total
+
+# Raw latency plots
+python plot_latency.py "$BLOCK_SPLIT_DIR" \
+    --output-dir "${OUTPUT_DIR}/block_individual" \
+    --metrics get_total_latency_ms device_write_latency_ms disk_write_latency_ms get_miss_latency_ms
+
+# Smoothed latency plots
+python plot_latency_smoothed.py "$BLOCK_SPLIT_DIR" \
+    --window-seconds $WINDOW_SECONDS \
+    --output-dir "${OUTPUT_DIR}/block_individual" \
+    --metrics get_total_latency_ms device_write_latency_ms disk_write_latency_ms get_miss_latency_ms
+
+# Hit ratio plots
+python plot_hitratio.py "$BLOCK_SPLIT_DIR" \
+    --output-dir "${OUTPUT_DIR}/block_individual"
 
 echo "✅ BLOCK-PROMO individual plots completed"
 
 echo "📈 Generating ZONED-PROMO individual plots..."
 
-## Throughput plots
-#python plot_throughput.py "$ZONED_SPLIT_DIR" \
-#    --bucket-seconds $BUCKET_SECONDS \
-#    --output-dir "${OUTPUT_DIR}/zoned_individual" \
-#    --metrics bytes_total # written_bytes_total
-
-## Raw latency plots
-#python plot_latency.py "$ZONED_SPLIT_DIR" \
-#    --output-dir "${OUTPUT_DIR}/zoned_individual" \
-#    --metrics get_total_latency_ms # device_write_latency_ms disk_write_latency_ms get_miss_latency_ms
-#
-## Smoothed latency plots
-#python plot_latency_smoothed.py "$ZONED_SPLIT_DIR" \
-#    --window-seconds $WINDOW_SECONDS \
-#    --output-dir "${OUTPUT_DIR}/zoned_individual" \
-#    --metrics get_total_latency_ms # device_write_latency_ms disk_write_latency_ms get_miss_latency_ms
-#
-## Hit ratio plots
-#python plot_hitratio.py "$ZONED_SPLIT_DIR" \
-#    --output-dir "${OUTPUT_DIR}/zoned_individual"
+# Throughput plots
+python plot_throughput.py "$ZONED_SPLIT_DIR" \
+    --bucket-seconds $BUCKET_SECONDS \
+    --output-dir "${OUTPUT_DIR}/zoned_individual" \
+    --metrics bytes_total  written_bytes_total
+
+# Raw latency plots
+python plot_latency.py "$ZONED_SPLIT_DIR" \
+    --output-dir "${OUTPUT_DIR}/zoned_individual" \
+    --metrics get_total_latency_ms device_write_latency_ms disk_write_latency_ms get_miss_latency_ms
+
+# Smoothed latency plots
+python plot_latency_smoothed.py "$ZONED_SPLIT_DIR" \
+    --window-seconds $WINDOW_SECONDS \
+    --output-dir "${OUTPUT_DIR}/zoned_individual" \
+    --metrics get_total_latency_ms device_write_latency_ms disk_write_latency_ms get_miss_latency_ms
+
+# Hit ratio plots
+python plot_hitratio.py "$ZONED_SPLIT_DIR" \
+    --output-dir "${OUTPUT_DIR}/zoned_individual"
 
 echo "✅ ZONED-PROMO individual plots completed"
 
@@ -167,31 +167,31 @@ echo "===================================="
 
 echo "📊 Generating comparison plots (Block vs ZNS)..."
 
-## Throughput comparison plots
-#python plot_throughput.py "$ZONED_SPLIT_DIR" "$BLOCK_SPLIT_DIR" \
-#    --labels "$ZONED_LABEL" "$BLOCK_LABEL" \
-#    --bucket-seconds $BUCKET_SECONDS \
-#    --output-dir "${OUTPUT_DIR}/comparison" \
-#    --metrics bytes_total # written_bytes_total read_bytes_total
-
-## Smoothed latency comparison plots
-#python plot_latency_smoothed.py "$ZONED_SPLIT_DIR" "$BLOCK_SPLIT_DIR" \
-#    --labels "$ZONED_LABEL" "$BLOCK_LABEL" \
-#    --window-seconds $WINDOW_SECONDS \
-#    --output-dir "${OUTPUT_DIR}/comparison" \
-#    --metrics get_total_latency_ms # device_write_latency_ms device_read_latency_ms disk_write_latency_ms disk_read_latency_ms get_miss_latency_ms get_hit_latency_ms
-#
-## Hit ratio comparison plots
-#python plot_hitratio.py "$ZONED_SPLIT_DIR" "$BLOCK_SPLIT_DIR" \
-#    --labels "$ZONED_LABEL" "$BLOCK_LABEL" \
-#    --output-dir "${OUTPUT_DIR}/comparison"
-#
-## Distribution comparison boxplots
-#python distribution_comparison_boxplots.py \
-#    --block-dir "$BLOCK_DIR" \
-#    --zns-dir "$ZONED_DIR" \
-#    --output-dir "${OUTPUT_DIR}/comparison"
-#
+# Throughput comparison plots
+python plot_throughput.py "$ZONED_SPLIT_DIR" "$BLOCK_SPLIT_DIR" \
+    --labels "$ZONED_LABEL" "$BLOCK_LABEL" \
+    --bucket-seconds $BUCKET_SECONDS \
+    --output-dir "${OUTPUT_DIR}/comparison" \
+    --metrics bytes_total written_bytes_total read_bytes_total
+
+# Smoothed latency comparison plots
+python plot_latency_smoothed.py "$ZONED_SPLIT_DIR" "$BLOCK_SPLIT_DIR" \
+    --labels "$ZONED_LABEL" "$BLOCK_LABEL" \
+    --window-seconds $WINDOW_SECONDS \
+    --output-dir "${OUTPUT_DIR}/comparison" \
+    --metrics get_total_latency_ms device_write_latency_ms device_read_latency_ms disk_write_latency_ms disk_read_latency_ms get_miss_latency_ms get_hit_latency_ms
+
+# Hit ratio comparison plots
+python plot_hitratio.py "$ZONED_SPLIT_DIR" "$BLOCK_SPLIT_DIR" \
+    --labels "$ZONED_LABEL" "$BLOCK_LABEL" \
+    --output-dir "${OUTPUT_DIR}/comparison"
+
+# Distribution comparison boxplots
+python distribution_comparison_boxplots.py \
+    --block-dir "$BLOCK_DIR" \
+    --zns-dir "$ZONED_DIR" \
+    --output-dir "${OUTPUT_DIR}/comparison"
+
 # Hit ratio horizontal bar charts
 python hitratio_horizontal_bars.py \
     --block-dir "$BLOCK_DIR" \