Move manifest to s3

Author: thodson-usgs

examples/virtual-rechunk/create-virtualzarr.py

@@ -40,7 +40,7 @@ def reduce_references(results):
         coords="minimal",
         compat="override",
     )
-
+    
     return combined_vds
 
 
@@ -55,5 +55,9 @@ def reduce_references(results):
 
 ds = futures.get_result()
 
-# Save the virtual zarr manifest to s3
-ds.virtualize.to_kerchunk(f"{bucket_url}/combined.json", format="json")
+# Save the virtual zarr manifest
+ds.virtualize.to_kerchunk(f"combined.json", format="json")
+
+# Upload manifest to s3
+fs_write = fsspec.filesystem("s3", anon=False, skip_instance_cache=True)
+fs_write.put("combined.json", f"{bucket_url}/combined.json")

examples/virtual-rechunk/virtual-rechunk.py

@@ -7,10 +7,14 @@
 # aren't propogated until the kernel is restarted.
 
 import os
+import fsspec
 import xarray as xr
 
 bucket_url = os.getenv("BUCKET_URL")
 
+target = fsspec.get_mapper(f"{bucket_url}/rechunked.zarr")
+                           # client_kwargs={'region_name':'us-west-2'})
+
 combined_ds = xr.open_dataset(
     f"{bucket_url}/combined.json", # location must be accessible to workers
     engine="kerchunk",
@@ -26,7 +30,7 @@
 )
 
 rechunked_ds.to_zarr(
-    f"{bucket_url}/rechunked.zarr",
+    target,
     mode="w",
     encoding={},  # TODO
     consolidated=True,