Pip install (#53)

* Clean up

* Add workflow for publishing on PyPI

* Add workflow for publishing on PyPI

* Dask test update

Author: constantinpape

.github/workflows/environment.yaml .github/environment.yaml.github/workflows/environment.yaml renamed to .github/environment.yaml

@@ -2,10 +2,9 @@ name: pybdv-build-env
 channels:
   - conda-forge
 dependencies:
+  - pip
   - imageio
   - h5py
   - scikit-image
   - tqdm
   - z5py
-  # zarr doesn't work yet, because it doesn't allow writing 'dataType' as group attribute
-  # - zarr

.github/workflows/build.yml

@@ -24,7 +24,7 @@ jobs:
           activate-environment: pybdv-build-env
           auto-update-conda: true
           channels: conda-forge
-          environment-file: .github/workflows/environment.yaml
+          environment-file: .github/environment.yaml
           python-version: ${{ matrix.python-version }}
           auto-activate-base: false
         env:
@@ -33,7 +33,7 @@ jobs:
       - name: Install pybdv
         if: matrix.os == 'ubuntu-latest'
         shell: bash -l {0}
-        run: pip install -e .[dask]
+        run: python -m pip install -e .[dask]
 
       - name: Run tests
         shell: bash -l {0}

.github/workflows/publish.yml

@@ -0,0 +1,81 @@
+---
+name: publish
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Build distribution
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install build
+        run: python -m pip install build
+
+      - name: Build sdist and wheel
+        run: python -m build
+
+      - name: Upload distribution
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+  publish-pypi:
+    name: Publish to PyPI
+    if: github.event_name == 'release'
+    needs: build
+    runs-on: ubuntu-latest
+
+    environment:
+      name: pypi
+      url: https://pypi.org/p/pybdv
+
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Download distribution
+        uses: actions/download-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+  publish-testpypi:
+    name: Publish to TestPyPI
+    if: github.event_name == 'workflow_dispatch'
+    needs: build
+    runs-on: ubuntu-latest
+
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/pybdv
+
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Download distribution
+        uses: actions/download-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+      - name: Publish to TestPyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/

conda-recipe/meta.yaml

@@ -97,7 +97,7 @@ def _scale_and_add_to_dataset(
     scale_factors = scales[1: 2].tolist()
 
     for scale in scales[2:]:
-        scale_factors.append((scale / np.product(scale_factors, axis=0)).astype(int).tolist())
+        scale_factors.append((scale / np.prod(scale_factors, axis=0)).astype(int).tolist())
 
     # Scale the data
     downscaled_vols = [target_vol]

pybdv/bdv_datasets.py

@@ -16,8 +16,6 @@
 
 try:
     import dask.array as da
-    import zarr
-    from numcodecs import GZip
     has_dask = True
 except ImportError:
     has_dask = False
@@ -589,11 +587,7 @@ def make_scales_dask(data, data_path, is_n5,
     elif downscale_factors is not None:
         downsample_chunks = tuple(((64, 64, 64) for _ in range(len(downscale_factors))))
 
-    if is_n5:
-        store = zarr.N5FSStore(data_path, overwrite=overwrite)
-    else:
-        store = zarr.DirectoryStore(data_path)
-    with zarr.open(store, mode='a') as group:
+    with open_file(data_path, mode='a') as group:
         # run all downsample steps for better data locality with dask
         base_key = get_key(is_h5=False, timepoint=timepoint, setup_id=setup_id, scale=0)
         have_data = base_key in group
@@ -621,8 +615,11 @@ def make_scales_dask(data, data_path, is_n5,
             save_chunks_all = [data.chunksize]
         arrays = []
         for (k, v), save_chunks in zip(pyramid.items(), save_chunks_all):
-            arrays.append(group.zeros(
-                name=k, shape=v.shape, dtype=v.dtype, chunks=save_chunks, compressor=GZip(), overwrite=overwrite
+            # deleting is required to over-write, because the shape may change
+            if k in group and overwrite:
+                del group[k]
+            arrays.append(group.create_dataset(
+                k, shape=v.shape, dtype=v.dtype, chunks=tuple(save_chunks), compression='gzip'
             ))
         array = da.store(pyramid.values(), arrays, lock=None, compute=compute, return_stored=return_stored)
 

pybdv/converter.py

@@ -19,30 +19,21 @@
 if open_file is None:
     import h5py
 
+    # n5 and zarr files are always opened with z5py;
+    # zarr >= 3 has dropped support for the n5 format
     try:
         import z5py
         n5_file = z5py.File
     except ImportError:
         n5_file = None
 
-    if n5_file is None:
-        try:
-            import zarr
-            n5_file = zarr.open
-        except ImportError:
-            n5_file = None
-
     def open_file(path, mode='r'):
         ext = os.path.splitext(path)[1].lower()
         if ext in HDF5_EXTENSIONS:
             return h5py.File(path, mode=mode)
-        elif ext in N5_EXTENSIONS:
-            if n5_file is None:
-                raise ValueError("Need zarr or z5py to open n5 files")
-            return n5_file(path, mode=mode)
-        elif ext in ZARR_EXTENSIONS:
+        elif ext in N5_EXTENSIONS + ZARR_EXTENSIONS:
             if n5_file is None:
-                raise ValueError("Need zarr or z5py to open zarr files")
+                raise ValueError("Need z5py to open n5 or zarr files")
             return n5_file(path, mode=mode)
         else:
             raise ValueError(f"Invalid extension: {ext}")

pybdv/util.py

@@ -8,10 +8,11 @@
     "numpy",
     "h5py",
     "scikit-image",
-    "tqdm"
+    "tqdm",
+    "z5py"
 ]
 
-extras = {"elf": ["elf"], "dask": ["dask", "zarr"]}
+extras = {"elf": ["elf"], "dask": ["dask"]}
 extras["all"] = list(itertools.chain.from_iterable(extras.values()))
 
 setup(

setup.py

@@ -344,7 +344,7 @@ def test_dask_lazy(self):
         if self.is_dask:
             data = np.random.rand(10,10,10).astype(np.float32)
             array = self._make_bdv(data, self.out_path,  compute=False, return_stored=True )
-            self.assertTrue('store-map' in array.name)
+            self.assertIsInstance(array, dask.array.Array)
             key = get_key(self.is_h5, timepoint=0, setup_id=0, scale=0)
             self.assertTrue(os.path.exists(self.out_path))
             with open_file(self.out_path, 'r') as f: