diff --git a/duckdb-vortex-partitioned/benchmark.sh b/duckdb-vortex-partitioned/benchmark.sh index 065968c47..8bd613314 100755 --- a/duckdb-vortex-partitioned/benchmark.sh +++ b/duckdb-vortex-partitioned/benchmark.sh @@ -8,14 +8,14 @@ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --defaul export CC=clang export CXX=clang++ -git clone https://github.com/vortex-data/vortex --recursive -cd vortex -git fetch --tags -git checkout 0.35.0 +git clone https://github.com/vortex-data/duckdb-vortex --recursive cd duckdb-vortex +git fetch --tags +git checkout v0.44.0 +git submodule update --init --recursive GEN=ninja NATIVE_ARCH=1 LTO=thin make export PATH="`pwd`/build/release/:$PATH" -cd ../.. +cd .. # Load the data seq 0 99 | xargs -P100 -I{} bash -c 'wget --continue --progress=dot:giga https://datasets.clickhouse.com/hits_compatible/athena_partitioned/hits_{}.parquet' @@ -24,7 +24,17 @@ seq 0 99 | xargs -P100 -I{} bash -c 'wget --continue --progress=dot:giga https:/ echo -n "Load time: " seq 0 99 | command time -f '%e' xargs -P"$(nproc)" -I{} bash -c ' if [ ! -f "hits_{}.vortex" ]; then - duckdb -c "COPY 'hits_{}.parquet' TO hits_{}.vortex (FORMAT vortex)" + duckdb -c " + COPY ( + SELECT * + REPLACE ( + make_date(EventDate) AS EventDate, + epoch_ms(EventTime * 1000) as EventTime + ) + FROM read_parquet('"'"'hits_{}.parquet'"'"', binary_as_string=True) + ) + TO '"'"'hits_{}.vortex'"'"' (FORMAT VORTEX) + " fi ' diff --git a/duckdb-vortex-partitioned/results/c6a.4xlarge.json b/duckdb-vortex-partitioned/results/c6a.4xlarge.json index a37892a2f..67a27fe4c 100644 --- a/duckdb-vortex-partitioned/results/c6a.4xlarge.json +++ b/duckdb-vortex-partitioned/results/c6a.4xlarge.json @@ -1,6 +1,6 @@ { "system": "DuckDB (Vortex, partitioned)", - "date": "2025-05-21", + "date": "2025-08-06", "machine": "c6a.4xlarge", "cluster_size": 1, "proprietary": "no", @@ -9,52 +9,52 @@ "tags": ["Rust", "column-oriented", "embedded", "stateless"], - "load_time": 0, - "data_size": 16028112632, + "load_time": 742.26, + "data_size": 15961049404, "result": [ - [0.239,0.011,0.010], - [0.498,0.027,0.027], - [1.586,0.150,0.145], - [3.163,0.108,0.108], - [3.321,0.381,0.380], - [3.327,0.417,0.415], - [0.247,0.050,0.052], - [0.446,0.033,0.030], - [4.517,0.568,0.565], - [4.495,0.819,0.829], - [2.772,0.147,0.144], - [3.467,0.175,0.168], - [3.809,0.388,0.387], - [6.402,0.789,0.773], - [3.615,0.409,0.440], - [2.741,0.449,0.454], - [6.105,1.176,1.135], - [5.895,0.987,0.993], - [8.413,2.137,2.134], - [1.890,0.256,0.254], - [27.029,0.593,0.593], - [28.245,0.619,0.659], - [35.089,1.163,1.177], - [98.719,2.680,2.845], - [5.261,0.259,0.254], - [3.447,0.187,0.179], - [5.643,0.258,0.247], - [25.593,1.212,0.654], - [21.055,9.490,9.472], - [0.773,0.106,0.107], - [6.830,0.357,0.345], - [12.732,0.531,0.486], - [10.444,2.272,2.167], - [26.346,1.819,1.804], - [26.248,2.339,2.319], - [1.834,0.622,0.627], - [0.247,0.032,0.033], - [0.619,0.028,0.027], - [0.884,0.022,0.037], - [1.011,0.065,0.062], - [0.718,0.025,0.027], - [0.819,0.028,0.023], - [0.765,0.024,0.020] + [0.184,0.013,0.003], + [0.523,0.014,0.014], + [1.610,0.035,0.035], + [3.435,0.049,0.052], + [3.466,0.329,0.332], + [4.172,0.297,0.292], + [0.181,0.022,0.020], + [0.567,0.019,0.018], + [4.334,0.415,0.405], + [4.319,0.561,0.558], + [2.784,0.097,0.091], + [3.485,0.124,0.107], + [4.721,0.307,0.316], + [7.135,0.672,0.675], + [4.478,0.343,0.342], + [2.783,0.392,0.387], + [7.061,0.852,0.847], + [6.856,0.740,0.628], + [9.200,1.517,1.505], + [1.971,0.038,0.031], + [33.849,0.556,0.530], + [36.486,0.679,0.636], + [40.129,1.065,1.072], + [7.566,0.392,0.382], + [1.880,0.122,0.062], + [4.896,0.096,0.098], + [1.791,0.126,0.031], + [34.787,0.863,0.790], + [28.059,9.317,9.314], + [0.717,0.033,0.033], + [7.835,0.279,0.317], + [13.900,0.420,0.403], + [10.751,1.919,1.892], + [34.222,2.047,1.980], + [34.208,2.288,2.131], + [1.861,0.511,0.506], + [0.258,0.025,0.024], + [0.840,0.012,0.021], + [1.098,0.024,0.018], + [1.265,0.063,0.053], + [0.815,0.022,0.008], + [0.889,0.010,0.010], + [0.833,0.032,0.012] ] }