Improve reading and writing of NetCDF files to/from bytes or file-like objects.

* Allows use of h5netcdf engine when writing to file-like objects (such as BytesIO), stop forcing use of scipy backend in this case (which is incompatible with groups and DataTree). Makes h5netcdf the default engine for DataTree.to_netcdf rather than leaving the choice of default up to Dataset.to_netcdf.
* Allows use of h5netcdf engine to read from a bytes object.
* Allows DataTree.to_netcdf to return bytes when filepath argument is omitted (similar to Dataset.to_netcdf.

Author: mjwillson

doc/whats-new.rst

@@ -13,6 +13,8 @@ v2025.07.2 (unreleased)
 New Features
 ~~~~~~~~~~~~
 
+- :py:meth:`DataTree.to_netcdf` can now write to a file-like object, or return bytes if called without a filepath. (:issue:`10570`)
+  By `Matthew Willson <https://github.com/mjwillson>`_.
 
 Breaking changes
 ~~~~~~~~~~~~~~~~

xarray/backends/api.py

@@ -43,7 +43,7 @@
 from xarray.core.indexes import Index
 from xarray.core.treenode import group_subtrees
 from xarray.core.types import NetcdfWriteModes, ZarrWriteModes
-from xarray.core.utils import is_remote_uri
+from xarray.core.utils import emit_user_level_warning, is_remote_uri
 from xarray.namedarray.daskmanager import DaskManager
 from xarray.namedarray.parallelcompat import guess_chunkmanager
 from xarray.structure.chunks import _get_chunk, _maybe_chunk
@@ -1911,11 +1911,11 @@ def to_netcdf(
     if path_or_file is None:
         if engine is None:
             engine = "scipy"
-        elif engine != "scipy":
+        elif engine not in ("scipy", "h5netcdf"):
             raise ValueError(
                 "invalid engine for creating bytes with "
-                f"to_netcdf: {engine!r}. Only the default engine "
-                "or engine='scipy' is supported"
+                f"to_netcdf: {engine!r}. Only the default engine, "
+                "engine='scipy' or engine='h5netcdf' is supported."
             )
         if not compute:
             raise NotImplementedError(
@@ -1927,7 +1927,13 @@ def to_netcdf(
             engine = _get_default_engine(path_or_file)
         path_or_file = _normalize_path(path_or_file)
     else:  # file-like object
-        engine = "scipy"
+        if engine not in ("scipy", "h5netcdf"):
+            emit_user_level_warning(
+                f"Requested {engine=} is not compatible with writing to a file-like object. "
+                "This will raise an error in the future, for now defaulting to engine='scipy'.",
+                FutureWarning,
+            )
+            engine = "scipy"
 
     # validate Dataset keys, DataArray names, and attr keys/values
     _validate_dataset_names(dataset)

xarray/backends/h5netcdf_.py

@@ -159,11 +159,9 @@ def open(
             )
 
         if isinstance(filename, bytes):
-            raise ValueError(
-                "can't open netCDF4/HDF5 as bytes "
-                "try passing a path or file-like object"
-            )
-        elif isinstance(filename, io.IOBase):
+            filename = io.BytesIO(filename)
+
+        if isinstance(filename, io.IOBase) and mode == "r":
             magic_number = read_magic_number_from_file(filename)
             if not magic_number.startswith(b"\211HDF\r\n\032\n"):
                 raise ValueError(

xarray/core/datatree.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
 
 import functools
+import io
 import itertools
+from os import PathLike
 import textwrap
 from collections import ChainMap
 from collections.abc import (
@@ -1659,9 +1661,11 @@ def _inplace_binary_op(self, other, f) -> Self:
     def __eq__(self, other: DtCompatible) -> Self:  # type: ignore[override]
         return super().__eq__(other)
 
+    # filepath=None writes to bytes
+    @overload
     def to_netcdf(
         self,
-        filepath,
+        filepath: None = None,
         mode: NetcdfWriteModes = "w",
         encoding=None,
         unlimited_dims=None,
@@ -1671,14 +1675,45 @@ def to_netcdf(
         write_inherited_coords: bool = False,
         compute: bool = True,
         **kwargs,
-    ):
+    ) -> bytes: ...
+
+    @overload
+    def to_netcdf(
+        self,
+        filepath: str | PathLike,
+        mode: NetcdfWriteModes = "w",
+        encoding=None,
+        unlimited_dims=None,
+        format: T_DataTreeNetcdfTypes | None = None,
+        engine: T_DataTreeNetcdfEngine | None = None,
+        group: str | None = None,
+        write_inherited_coords: bool = False,
+        compute: bool = True,
+        **kwargs,
+    ) -> None: ...
+
+    def to_netcdf(
+        self,
+        filepath: str | PathLike | io.IOBase | None = None,
+        mode: NetcdfWriteModes = "w",
+        encoding=None,
+        unlimited_dims=None,
+        format: T_DataTreeNetcdfTypes | None = None,
+        engine: T_DataTreeNetcdfEngine | None = None,
+        group: str | None = None,
+        write_inherited_coords: bool = False,
+        compute: bool = True,
+        **kwargs,
+    ) -> None | bytes:
         """
         Write datatree contents to a netCDF file.
 
         Parameters
         ----------
-        filepath : str or Path
-            Path to which to save this datatree.
+        filepath : str or PathLike or file-like object or None
+            Path to which to save this datatree, or a file-like object to write
+            it to (which must support read and write and be seekable) or None
+            to return in-memory bytes.
         mode : {"w", "a"}, default: "w"
             Write ('w') or append ('a') mode. If mode='w', any existing file at
             this location will be overwritten. If mode='a', existing variables
@@ -1717,14 +1752,18 @@ def to_netcdf(
         kwargs :
             Additional keyword arguments to be passed to ``xarray.Dataset.to_netcdf``
 
+        Returns
+        -------
+            A bytes object with the byte content of the netCDF file, if filepath was None.
+
         Note
         ----
             Due to file format specifications the on-disk root group name
             is always ``"/"`` overriding any given ``DataTree`` root node name.
         """
         from xarray.core.datatree_io import _datatree_to_netcdf
 
-        _datatree_to_netcdf(
+        return _datatree_to_netcdf(
             self,
             filepath,
             mode=mode,

xarray/core/datatree_io.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
 from collections.abc import Mapping
+import io
 from os import PathLike
-from typing import TYPE_CHECKING, Any, Literal, get_args
+from typing import TYPE_CHECKING, Any, Literal, get_args, overload
 
 from xarray.core.datatree import DataTree
 from xarray.core.types import NetcdfWriteModes, ZarrWriteModes
@@ -13,10 +14,9 @@
 if TYPE_CHECKING:
     from xarray.core.types import ZarrStoreLike
 
-
 def _datatree_to_netcdf(
     dt: DataTree,
-    filepath: str | PathLike,
+    filepath: str | PathLike | io.IOBase | None = None,
     mode: NetcdfWriteModes = "w",
     encoding: Mapping[str, Any] | None = None,
     unlimited_dims: Mapping | None = None,
@@ -26,18 +26,21 @@ def _datatree_to_netcdf(
     write_inherited_coords: bool = False,
     compute: bool = True,
     **kwargs,
-) -> None:
+) -> None | bytes:
     """This function creates an appropriate datastore for writing a datatree to
     disk as a netCDF file.
 
     See `DataTree.to_netcdf` for full API docs.
     """
 
     if format not in [None, *get_args(T_DataTreeNetcdfTypes)]:
-        raise ValueError("to_netcdf only supports the NETCDF4 format")
+        raise ValueError("DataTree.to_netcdf only supports the NETCDF4 format")
 
     if engine not in [None, *get_args(T_DataTreeNetcdfEngine)]:
-        raise ValueError("to_netcdf only supports the netcdf4 and h5netcdf engines")
+        raise ValueError("DataTree.to_netcdf only supports the netcdf4 and h5netcdf engines")
+
+    if engine is None:
+        engine = "h5netcdf"
 
     if group is not None:
         raise NotImplementedError(
@@ -58,6 +61,9 @@ def _datatree_to_netcdf(
             f"unexpected encoding group name(s) provided: {set(encoding) - set(dt.groups)}"
         )
 
+    if filepath is None:
+        filepath = io.BytesIO()
+
     if unlimited_dims is None:
         unlimited_dims = {}
 
@@ -78,6 +84,8 @@ def _datatree_to_netcdf(
         )
         mode = "a"
 
+    return filepath.getvalue() if isinstance(filepath, io.BytesIO) else None
+
 
 def _datatree_to_zarr(
     dt: DataTree,

xarray/tests/test_backends.py

@@ -4543,9 +4543,6 @@ class TestH5NetCDFFileObject(TestH5NetCDFData):
     engine: T_NetcdfEngine = "h5netcdf"
 
     def test_open_badbytes(self) -> None:
-        with pytest.raises(ValueError, match=r"HDF5 as bytes"):
-            with open_dataset(b"\211HDF\r\n\032\n", engine="h5netcdf"):  # type: ignore[arg-type]
-                pass
         with pytest.raises(
             ValueError, match=r"match in any of xarray's currently installed IO"
         ):

xarray/tests/test_backends_datatree.py

@@ -539,6 +539,28 @@ def test_phony_dims_warning(self, tmpdir) -> None:
                     "phony_dim_3": 25,
                 }
 
+    def test_roundtrip_via_bytes(self, simple_datatree):
+        original_dt = simple_datatree
+        roundtrip_dt = open_datatree(original_dt.to_netcdf())
+        assert_equal(original_dt, roundtrip_dt)
+
+    def test_roundtrip_via_bytes_engine_specified(self, simple_datatree):
+        original_dt = simple_datatree
+        roundtrip_dt = open_datatree(original_dt.to_netcdf(engine=self.engine))
+        assert_equal(original_dt, roundtrip_dt)
+
+    def test_roundtrip_using_filelike_object(self, tmpdir, simple_datatree):
+        original_dt = simple_datatree
+        filepath = tmpdir + '/test.nc'
+        # h5py requires both read and write access when writing, it will
+        # work with file-like objects provided they support both, and are
+        # seekable.
+        with open(filepath, 'wb+') as file:
+            original_dt.to_netcdf(file, engine=self.engine)
+        with open(filepath, 'rb') as file:
+            roundtrip_dt = open_datatree(file, engine=self.engine)
+            assert_equal(original_dt, roundtrip_dt)
+
 
 @requires_zarr
 @parametrize_zarr_format