Skip to content

Commit 339a48f

Browse files
normanrznormanrz
authored
Merge pull request #12 from scalableminds/fix-http-store
2 parents 8342e1c + a90fb68 commit 339a48f

File tree

6 files changed

+106
-30
lines changed

6 files changed

+106
-30
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,17 +19,17 @@ jobs:
1919
shell: bash
2020

2121
steps:
22-
- uses: actions/checkout@v3
22+
- uses: actions/checkout@v4
2323

2424
- name: Set up JDK
25-
uses: actions/setup-java@v3
25+
uses: actions/setup-java@v4
2626
with:
2727
java-version: '11'
2828
distribution: 'temurin'
2929
cache: maven
3030

3131
- name: Set up Python
32-
uses: actions/setup-python@v4
32+
uses: actions/setup-python@v5
3333
with:
3434
python-version: '3.11'
3535

@@ -56,7 +56,7 @@ jobs:
5656
- name: Assemble JAR
5757
run: mvn package -DskipTests
5858

59-
- uses: actions/upload-artifact@v3
59+
- uses: actions/upload-artifact@v4
6060
with:
61-
name: jar
61+
name: jar-${{ runner.os }}
6262
path: target/*.jar

‎.github/workflows/deploy.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,10 @@ jobs:
77
runs-on: ubuntu-latest
88
steps:
99

10-
- uses: actions/checkout@v3
10+
- uses: actions/checkout@v4
1111

1212
- name: Set up Java 11
13-
uses: actions/setup-java@v3
13+
uses: actions/setup-java@v4
1414
with:
1515
java-version: '11'
1616
distribution: 'adopt'
@@ -32,7 +32,7 @@ jobs:
3232
# Begin copy from ci.yml. Refactor?
3333

3434
- name: Set up Python
35-
uses: actions/setup-python@v4
35+
uses: actions/setup-python@v5
3636
with:
3737
python-version: '3.11'
3838

‎README.md‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@ Group hierarchy = Group.open(
1717
new HttpStore("https://static.webknossos.org/data/zarr_v3")
1818
.resolve("l4_sample")
1919
);
20-
Array array = hierarchy.get("color").get("1");
20+
Group color = (Group) hierarchy.get("color");
21+
Array array = (Array) color.get("1");
2122
ucar.ma2.Array outArray = array.read(
2223
new long[]{0, 3073, 3073, 513}, // offset
2324
new int[]{1, 64, 64, 64} // shape
@@ -31,11 +32,12 @@ Array array = Array.create(
3132
.withChunkShape(1, 1024, 1024, 1024)
3233
.withFillValue(0)
3334
.withCodecs(c -> c.withSharding(new int[]{1, 32, 32, 32}, c1 -> c1.withBlosc()))
34-
.build();
35+
.build()
3536
);
37+
ucar.ma2.Array data = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1, 1024, 1024, 1024});
3638
array.write(
3739
new long[]{0, 0, 0, 0}, // offset
38-
ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1, 1024, 1024, 1024})
40+
data
3941
);
4042
```
4143
## Development Start-Guide

‎src/main/java/dev/zarr/zarrjava/store/HttpStore.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ public ByteBuffer get(String[] keys, long start, long end) {
7979
throw new IllegalArgumentException("Argument 'start' needs to be non-negative.");
8080
}
8181
Request request = new Request.Builder().url(resolveKeys(keys)).header(
82-
"Range", String.format("Bytes=%d-%d", start, end + 1)).build();
82+
"Range", String.format("Bytes=%d-%d", start, end - 1)).build();
8383
return get(request);
8484
}
8585

‎src/main/java/dev/zarr/zarrjava/v3/Array.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,11 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape) throws ZarrEx
142142
if (shape.length != metadata.ndim()) {
143143
throw new IllegalArgumentException("'shape' needs to have rank '" + metadata.ndim() + "'.");
144144
}
145+
for (int dimIdx = 0; dimIdx < metadata.ndim(); dimIdx++) {
146+
if (offset[dimIdx] < 0 || offset[dimIdx] + shape[dimIdx] > metadata.shape[dimIdx]) {
147+
throw new ZarrException("Requested data is outside of the array's domain.");
148+
}
149+
}
145150

146151
final int[] chunkShape = metadata.chunkShape();
147152
if (IndexingUtils.isSingleFullChunk(offset, shape, chunkShape)) {
@@ -168,7 +173,9 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape) throws ZarrEx
168173

169174
final String[] chunkKeys = metadata.chunkKeyEncoding.encodeChunkKey(chunkCoords);
170175
final StoreHandle chunkHandle = storeHandle.resolve(chunkKeys);
171-
176+
if (!chunkHandle.exists()) {
177+
return;
178+
}
172179
if (codecPipeline.supportsPartialDecode()) {
173180
final ucar.ma2.Array chunkArray = codecPipeline.decodePartial(chunkHandle,
174181
Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape);

‎src/test/java/dev/zarr/zarrjava/ZarrTest.java‎

Lines changed: 84 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,10 @@
66
import com.fasterxml.jackson.databind.ObjectMapper;
77
import com.github.luben.zstd.Zstd;
88
import com.github.luben.zstd.ZstdCompressCtx;
9-
import dev.zarr.zarrjava.store.FilesystemStore;
10-
import dev.zarr.zarrjava.store.HttpStore;
11-
import dev.zarr.zarrjava.store.S3Store;
12-
import dev.zarr.zarrjava.store.StoreHandle;
9+
import dev.zarr.zarrjava.store.*;
1310
import dev.zarr.zarrjava.utils.MultiArrayUtils;
1411
import dev.zarr.zarrjava.v3.*;
12+
import dev.zarr.zarrjava.v3.codec.Codec;
1513
import dev.zarr.zarrjava.v3.codec.CodecBuilder;
1614
import dev.zarr.zarrjava.v3.codec.core.BytesCodec;
1715
import dev.zarr.zarrjava.v3.codec.core.TransposeCodec;
@@ -65,13 +63,15 @@ public static void clearTestoutputFolder() throws IOException {
6563
@CsvSource({
6664
"blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9",
6765
"gzip,0", "gzip,5",
68-
"zstd,0_true", "zstd,5_true","zstd,0_false", "zstd,5_false",
66+
"zstd,0_true", "zstd,5_true", "zstd,0_false", "zstd,5_false",
6967
"bytes,BIG", "bytes,LITTLE",
7068
"transpose,_",
7169
"sharding,start", "sharding,end",
7270
"sharding_nested,_",
7371
"crc32c,_",
74-
}) public void testReadFromZarrita(String codec, String codecParam) throws IOException, ZarrException, InterruptedException {
72+
})
73+
74+
public void testReadFromZarrita(String codec, String codecParam) throws IOException, ZarrException, InterruptedException {
7575
String command = pythonPath();
7676
ProcessBuilder pb = new ProcessBuilder(command, PYTHON_TEST_PATH.resolve("zarrita_write.py").toString(), codec, codecParam, TESTOUTPUT.toString());
7777
Process process = pb.start();
@@ -142,7 +142,7 @@ public void testZstdLibrary(int clevel, boolean checksumFlag) throws IOException
142142
@CsvSource({
143143
"blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9",
144144
"gzip,0", "gzip,5",
145-
"zstd,0_true", "zstd,5_true","zstd,0_false", "zstd,5_false",
145+
"zstd,0_true", "zstd,5_true", "zstd,0_false", "zstd,5_false",
146146
"bytes,BIG", "bytes,LITTLE",
147147
"transpose,_",
148148
"sharding,start", "sharding,end",
@@ -233,12 +233,12 @@ public void testWriteReadWithZarrita(String codec, String codecParam) throws Exc
233233
assert exitCode == 0;
234234
}
235235

236-
static Stream<Function<CodecBuilder, CodecBuilder>> invalidCodecBuilder(){
236+
static Stream<Function<CodecBuilder, CodecBuilder>> invalidCodecBuilder() {
237237
return Stream.of(
238238
c -> c.withBytes(BytesCodec.Endian.LITTLE).withBytes(BytesCodec.Endian.LITTLE),
239239
c -> c.withBlosc().withBytes(BytesCodec.Endian.LITTLE),
240-
c -> c.withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1,0}),
241-
c -> c.withTranspose(new int[]{1,0}).withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1,0})
240+
c -> c.withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0}),
241+
c -> c.withTranspose(new int[]{1, 0}).withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0})
242242
);
243243
}
244244

@@ -247,9 +247,9 @@ static Stream<Function<CodecBuilder, CodecBuilder>> invalidCodecBuilder(){
247247
public void testCheckInvalidCodecConfiguration(Function<CodecBuilder, CodecBuilder> codecBuilder) throws Exception {
248248
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("invalid_codec_config", String.valueOf(codecBuilder.hashCode()));
249249
ArrayMetadataBuilder builder = Array.metadataBuilder()
250-
.withShape(new long[] {4, 4})
250+
.withShape(new long[]{4, 4})
251251
.withDataType(DataType.UINT32)
252-
.withChunkShape(new int[]{2,2})
252+
.withChunkShape(new int[]{2, 2})
253253
.withCodecs(codecBuilder);
254254

255255
assertThrows(ZarrException.class, () -> Array.create(storeHandle, builder.build()));
@@ -287,7 +287,7 @@ static Stream<int[]> invalidChunkSizes() {
287287
@ParameterizedTest
288288
@MethodSource("invalidChunkSizes")
289289
public void testCheckInvalidChunkDimensions(int[] chunkSize) {
290-
long[] shape = new long[] {4, 4};
290+
long[] shape = new long[]{4, 4};
291291

292292
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("invalid_chunksize");
293293
ArrayMetadataBuilder builder = Array.metadataBuilder()
@@ -312,8 +312,8 @@ static Stream<int[]> invalidShardSizes() {
312312
@ParameterizedTest
313313
@MethodSource("invalidShardSizes")
314314
public void testCheckShardingBounds(int[] shardSize) throws Exception {
315-
long[] shape = new long[] {10, 10};
316-
int[] innerChunkSize = new int[] {2, 2};
315+
long[] shape = new long[]{10, 10};
316+
int[] innerChunkSize = new int[]{2, 2};
317317

318318
ArrayMetadataBuilder builder = Array.metadataBuilder()
319319
.withShape(shape)
@@ -371,8 +371,8 @@ static Stream<int[]> invalidTransposeOrder() {
371371
return Stream.of(
372372
new int[]{1, 0, 0},
373373
new int[]{1, 2, 3},
374-
new int[]{1,2,3,0},
375-
new int[]{1,2}
374+
new int[]{1, 2, 3, 0},
375+
new int[]{1, 2}
376376
);
377377
}
378378

@@ -392,6 +392,7 @@ public void testCheckInvalidTransposeOrder(int[] transposeOrder) throws Exceptio
392392
ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, shapeInt);
393393
assertThrows(ZarrException.class, () -> transposeCodec.encode(testData));
394394
}
395+
395396
@Test
396397
public void testFileSystemStores() throws IOException, ZarrException {
397398
FilesystemStore fsStore = new FilesystemStore(TESTDATA);
@@ -564,4 +565,70 @@ public void testV2() throws IOException {
564565
}
565566

566567

568+
@Test
569+
public void testReadme1() throws IOException, ZarrException {
570+
Group hierarchy = Group.open(
571+
new HttpStore("https://static.webknossos.org/data/zarr_v3")
572+
.resolve("l4_sample")
573+
);
574+
Group color = (Group) hierarchy.get("color");
575+
Array array = (Array) color.get("1");
576+
ucar.ma2.Array outArray = array.read(
577+
new long[]{0, 3073, 3073, 513}, // offset
578+
new int[]{1, 64, 64, 64} // shape
579+
);
580+
}
581+
582+
@Test
583+
public void testReadme2() throws IOException, ZarrException {
584+
Array array = Array.create(
585+
new FilesystemStore(TESTOUTPUT).resolve("testoutput", "color", "1"),
586+
Array.metadataBuilder()
587+
.withShape(1, 4096, 4096, 1536)
588+
.withDataType(DataType.UINT32)
589+
.withChunkShape(1, 1024, 1024, 1024)
590+
.withFillValue(0)
591+
.withCodecs(c -> c.withSharding(new int[]{1, 32, 32, 32}, c1 -> c1.withBlosc()))
592+
.build()
593+
);
594+
ucar.ma2.Array data = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1, 1, 2, 2}, new int[]{1, 2, 3, 4});
595+
array.write(
596+
new long[]{0, 0, 0, 0}, // offset
597+
data
598+
);
599+
ucar.ma2.Array output = array.read(new long[]{0, 0, 0, 0}, new int[]{1, 1, 2, 2});
600+
assert MultiArrayUtils.allValuesEqual(data, output);
601+
602+
}
603+
604+
@ParameterizedTest
605+
@ValueSource(strings = {"1", "2-2-1", "4-4-1", "16-16-4"})
606+
public void testReadL4Sample(String mag) throws IOException, ZarrException {
607+
StoreHandle httpStoreHandle = new HttpStore("https://static.webknossos.org/data/zarr_v3/").resolve("l4_sample", "color", mag);
608+
StoreHandle localStoreHandle = new FilesystemStore(TESTDATA).resolve("l4_sample", "color", mag);
609+
610+
Array httpArray = Array.open(httpStoreHandle);
611+
Array localArray = Array.open(localStoreHandle);
612+
System.out.println(httpArray);
613+
System.out.println(localArray);
614+
615+
ucar.ma2.Array httpData1 = httpArray.read(new long[]{0, 0, 0, 0}, new int[]{1, 64, 64, 64});
616+
ucar.ma2.Array localData1 = localArray.read(new long[]{0, 0, 0, 0}, new int[]{1, 64, 64, 64});
617+
618+
assert MultiArrayUtils.allValuesEqual(httpData1, localData1);
619+
620+
//offset to where l4_sample contains non-zero values
621+
long[] offset = new long[4];
622+
long[] originalOffset = new long[]{0, 3073, 3073, 513};
623+
long[] originalShape = new long[]{1, 4096, 4096, 2048};
624+
long[] arrayShape = httpArray.metadata.shape;
625+
for (int i = 0; i < 4; i++) {
626+
offset[i] = originalOffset[i] / (originalShape[i] / arrayShape[i]);
627+
}
628+
629+
ucar.ma2.Array httpData2 = httpArray.read(offset, new int[]{1, 64, 64, 64});
630+
ucar.ma2.Array localData2 = localArray.read(offset, new int[]{1, 64, 64, 64});
631+
632+
assert MultiArrayUtils.allValuesEqual(httpData2, localData2);
633+
}
567634
}

0 commit comments

Comments
 (0)