feat: refactor dataset create

.gitignore

@@ -141,3 +141,13 @@ _version.py
 *.to_upload
 tempCodeRunnerFile.python
 Untitled-*.py
+*.zip
+*.json
+*.db
+*.tgz
+_api/
+trace.txt
+?/
+*.prof
+prof/
+*.gz

pyproject.toml

@@ -51,9 +51,12 @@ dependencies = [
   "anemoi-transform>=0.1.10",
   "anemoi-utils[provenance]>=0.4.32",
   "cfunits",
+  "glom",
+  "jsonschema",
   "numcodecs<0.16",                   # Until we move to zarr3
   "numpy",
   "pyyaml",
+  "ruamel-yaml",
   "semantic-version",
   "tqdm",
   "zarr<=2.18.4",

src/anemoi/datasets/commands/recipe/__init__.py

@@ -0,0 +1,93 @@
+# (C) Copyright 2024 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+import argparse
+import logging
+import sys
+from typing import Any
+
+import yaml
+
+from anemoi.datasets.create import validate_config
+
+from .. import Command
+from .format import format_recipe
+from .migrate import migrate_recipe
+
+LOG = logging.getLogger(__name__)
+
+
+class Recipe(Command):
+    def add_arguments(self, command_parser: Any) -> None:
+        """Add arguments to the command parser.
+
+        Parameters
+        ----------
+        command_parser : Any
+            Command parser object.
+        """
+
+        command_parser.add_argument("--validate", action="store_true", help="Validate recipe.")
+        command_parser.add_argument("--format", action="store_true", help="Format the recipe.")
+        command_parser.add_argument("--migrate", action="store_true", help="Migrate the recipe to the latest version.")
+
+        group = command_parser.add_mutually_exclusive_group()
+        group.add_argument("--inplace", action="store_true", help="Overwrite the recipe file in place.")
+        group.add_argument("--output", type=str, help="Output file path for the converted recipe.")
+
+        command_parser.add_argument(
+            "path",
+            help="Path to recipe.",
+        )
+
+    def run(self, args: Any) -> None:
+
+        if not args.validate and not args.format and not args.migrate:
+            args.validate = True
+
+        with open(args.path) as file:
+            config = yaml.safe_load(file)
+
+        assert isinstance(config, dict)
+
+        if args.validate:
+            if args.inplace and (not args.format and not args.migrate):
+                argparse.ArgumentError(None, "--inplace is not supported with --validate.")
+
+            if args.output and (not args.format and not args.migrate):
+                argparse.ArgumentError(None, "--output is not supported with --validate.")
+
+            validate_config(config)
+            LOG.info(f"{args.path}: Recipe is valid.")
+            return
+
+        if args.migrate:
+            config = migrate_recipe(args, config)
+            if config is None:
+                LOG.info(f"{args.path}: No changes needed.")
+                return
+
+            args.format = True
+
+        if args.format:
+            formatted = format_recipe(args, config)
+            assert "dates" in formatted
+            f = sys.stdout
+            if args.output:
+                f = open(args.output, "w")
+
+            if args.inplace:
+                f = open(args.path, "w")
+
+            print(formatted, file=f)
+            f.close()
+
+
+command = Recipe

src/anemoi/datasets/commands/recipe/format.py

@@ -0,0 +1,55 @@
+# (C) Copyright 2025 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+import datetime
+import logging
+
+from ...dumper import yaml_dump
+
+LOG = logging.getLogger(__name__)
+
+
+def make_dates(config):
+    if isinstance(config, dict):
+        return {k: make_dates(v) for k, v in config.items()}
+    if isinstance(config, list):
+        return [make_dates(v) for v in config]
+    if isinstance(config, str):
+        try:
+            return datetime.datetime.fromisoformat(config)
+        except ValueError:
+            return config
+    return config
+
+
+ORDER = (
+    "name",
+    "description",
+    "dataset_status",
+    "licence",
+    "attribution",
+    "env",
+    "dates",
+    "common",
+    "data_sources",
+    "input",
+    "output",
+    "statistics",
+    "build",
+    "platform",
+)
+
+
+def format_recipe(args, config: dict) -> str:
+
+    config = make_dates(config)
+    assert config
+
+    return yaml_dump(config, order=ORDER)