feat: log convert metrics to benchmark local runs

Author: wietzesuijker

.gitignore

@@ -211,3 +211,9 @@ tests-output/
 # uv
 uv.lock
 .vscode/settings.json
+
+*.zarr
+.DS_§tore
+out
+runs
+*.gz

Dockerfile

@@ -0,0 +1,76 @@
+# ========= stage: builder =========
+FROM python:3.11-slim AS builder
+
+ENV PYTHONUNBUFFERED="1" \
+    PIP_NO_CACHE_DIR="1"
+
+# Build arg: 0=wheel-only (small; build as linux/amd64), 1=portable (adds GDAL/PROJ & toolchain)
+ARG PORTABLE_BUILD=0
+
+# Base OS deps
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+      ca-certificates curl git \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# uv + modern pip
+COPY pyproject.toml uv.lock ./
+RUN pip install --no-cache-dir -U pip \
+ && pip install --no-cache-dir uv \
+ && uv --version
+
+# ---- Export ONLY third-party runtime deps (no hashes). We filter out any editable/self lines.
+# NOTE: We export BEFORE copying src/ to avoid uv deciding this is a local edit; still filter to be safe.
+RUN set -euo pipefail; \
+    uv export --no-group dev --no-group test --format=requirements-txt --no-hashes -o /tmp/req.raw.txt; \
+    awk ' \
+      BEGIN{IGNORECASE=1} \
+      # drop editable flags
+      /^-e[[:space:]]/ || /^--editable[[:space:]]/ {next} \
+      # drop local/self refs
+      /@ file:/ || /file:\/\// {next} \
+      # drop our own package if present
+      /^eopf-geozarr([[:space:]]|==|$)/ {next} \
+      # drop comments/blank lines
+      /^[[:space:]]*#/ || /^[[:space:]]*$/ {next} \
+      {print} \
+    ' /tmp/req.raw.txt > /tmp/requirements.txt; \
+    echo "----- filtered requirements (head) -----"; \
+    sed -n '1,80p' /tmp/requirements.txt
+
+# ---- If PORTABLE, add toolchain + GDAL/PROJ for sdist builds (arm64 etc.)
+RUN if [ "$PORTABLE_BUILD" = "1" ]; then \
+      apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+        build-essential gdal-bin libgdal-dev proj-bin libproj-dev \
+      && rm -rf /var/lib/apt/lists/* ; \
+    fi
+
+# ---- Install third-party deps, then your package (no re-resolve of deps)
+RUN if [ "$PORTABLE_BUILD" = "1" ]; then \
+      pip install --no-cache-dir -r /tmp/requirements.txt ; \
+    else \
+      PIP_ONLY_BINARY=":all:" pip install --no-cache-dir --prefer-binary -r /tmp/requirements.txt ; \
+    fi
+
+# Now copy source and install the project itself without deps (pure python install)
+COPY src ./src
+RUN pip install --no-cache-dir --no-deps .
+
+# Optional: byte-compile
+RUN python -m compileall -q /usr/local/lib/python3.11/site-packages || true
+
+# ========= stage: runtime =========
+FROM python:3.11-slim AS runtime
+ENV PYTHONUNBUFFERED="1"
+
+# Tiny libs manylinux wheels often need
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+      libstdc++6 libgomp1 \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+COPY --from=builder /usr/local /usr/local
+
+# Argo Script template supplies the command
+CMD ["python", "-V"]

Makefile

@@ -0,0 +1,156 @@
+# ===== Config =====
+IMAGE     ?= eopf-geozarr:dev
+NAMESPACE ?= argo                  # Kubernetes namespace where Argo runs
+TPL       ?= geozarr-convert-template.yaml
+PARAMS    ?= params.json
+CLUSTER   ?= k3s-default
+
+# Runtime param overrides (env > PARAMS file)
+STAC_URL    ?=
+OUTPUT_ZARR ?=
+GROUPS      ?=
+
+# Abbrev: WF = Workflow name; PVC = PersistentVolumeClaim (<WF>-outpvc)
+
+.PHONY: build load-k3d load-minikube argo-install template apply \
+        submit submit-cli submit-api status latest logs-save clean \
+        _ensure-dirs fetch-tar run clean-pvc
+
+# Build the image locally
+# make build                 -> WHEEL mode (small), builds linux/amd64
+# make build PORTABLE=1      -> PORTABLE mode (bigger), builds for native arch
+build:
+	@if [ "$(PORTABLE)" = "1" ]; then \
+	  echo "==> Building PORTABLE image for native platform (allows source builds)"; \
+	  docker build \
+	    --build-arg PORTABLE_BUILD=1 \
+	    -t $(IMAGE) . ; \
+	else \
+	  echo "==> Building WHEEL image for linux/amd64 (prebuilt wheels)"; \
+	  docker buildx build --platform=linux/amd64 \
+	    --build-arg PORTABLE_BUILD=0 \
+	    -t $(IMAGE) --load . ; \
+	fi
+
+# Load image into k3d’s containerd (dev clusters)
+load-k3d:
+	k3d image import $(IMAGE) --cluster $(CLUSTER) || \
+	  (docker save $(IMAGE) | docker exec -i $$(docker ps --format '{{.Names}}' | grep $(CLUSTER)-server-0) ctr -n k8s.io images import -)
+
+# Build the image inside minikube’s Docker
+load-minikube:
+	eval "$$(minikube docker-env)"; docker build -t $(IMAGE) .
+
+# Install Argo Workflows (v3.7.1) into $(NAMESPACE)
+argo-install:
+	kubectl create ns $(NAMESPACE) 2>/dev/null || true
+	kubectl apply -n $(NAMESPACE) -f https://github.com/argoproj/argo-workflows/releases/download/v3.7.1/install.yaml
+	kubectl -n $(NAMESPACE) rollout status deploy/workflow-controller
+	kubectl -n $(NAMESPACE) rollout status deploy/argo-server
+
+# Apply (or update) the WorkflowTemplate
+template:
+	kubectl -n $(NAMESPACE) apply -f $(TPL)
+	kubectl -n $(NAMESPACE) get workflowtemplate geozarr-convert
+
+# Build + load + install + template (one shot)
+apply: build load-k3d argo-install template
+
+# Submit via CLI (uses env overrides, else PARAMS file)
+submit: _ensure-dirs
+	@STAC="$${STAC_URL:-$$(jq -r '.arguments.parameters[] | select(.name=="stac_url").value' $(PARAMS))}"; \
+	OUT="$${OUTPUT_ZARR:-$$(jq -r '.arguments.parameters[] | select(.name=="output_zarr").value' $(PARAMS))}"; \
+	GRP="$${GROUPS:-$$(jq -r '.arguments.parameters[] | select(.name=="groups").value' $(PARAMS))}"; \
+	echo "Submitting:"; echo "  stac_url=$$STAC"; echo "  output_zarr=$$OUT"; echo "  groups=$$GRP"; \
+	WF=$$(argo submit -n $(NAMESPACE) --from workflowtemplate/geozarr-convert \
+	  -p stac_url="$$STAC" -p output_zarr="$$OUT" -p groups="$$GRP" -o name); \
+	TSTAMP=$$(date +%Y%m%d-%H%M%S); \
+	argo get -n $(NAMESPACE) $$WF -o json > runs/$${TSTAMP}-$${WF##*/}.json; \
+	argo get -n $(NAMESPACE) $$WF --output wide | tee runs/$${TSTAMP}-$${WF##*/}.summary.txt; \
+	echo "Workflow: $$WF"
+
+# Submit via CLI (PARAMS file only, no env overrides)
+submit-cli: _ensure-dirs
+	@WF=$$(argo submit -n $(NAMESPACE) --from workflowtemplate/geozarr-convert \
+	  -p stac_url="$$(jq -r '.arguments.parameters[] | select(.name=="stac_url").value' $(PARAMS))" \
+	  -p output_zarr="$$(jq -r '.arguments.parameters[] | select(.name=="output_zarr").value' $(PARAMS))" \
+	  -p groups="$$(jq -r '.arguments.parameters[] | select(.name=="groups").value' $(PARAMS))" \
+	  -o name); \
+	TSTAMP=$$(date +%Y%m%d-%H%M%S); \
+	argo get -n $(NAMESPACE) $$WF -o json > runs/$${TSTAMP}-$${WF##*/}.json; \
+	argo get -n $(NAMESPACE) $$WF --output wide | tee runs/$${TSTAMP}-$${WF##*/}.summary.txt; \
+	echo "Workflow: $$WF"
+
+# Submit via Argo Server HTTP (dev port-forward, no token)
+submit-api: _ensure-dirs
+	kubectl -n $(NAMESPACE) port-forward svc/argo-server 2746:2746 >/dev/null 2>&1 & echo $$! > .pf.pid
+	sleep 1
+	curl -s -H 'Content-Type: application/json' \
+	  --data-binary @$(PARAMS) \
+	  http://localhost:2746/api/v1/workflows/$(NAMESPACE)/submit \
+	  | tee runs/submit-response.json | jq . >/dev/null || \
+	  (echo "Non-JSON response (see runs/submit-response.json)"; exit 1)
+	-@[ -f .pf.pid ] && kill $$(cat .pf.pid) 2>/dev/null || true
+	-@rm -f .pf.pid
+
+# Inspect
+status:
+	argo list -n $(NAMESPACE); echo; kubectl -n $(NAMESPACE) get wf
+
+latest:
+	argo get -n $(NAMESPACE) @latest --output wide
+
+logs-save: _ensure-dirs
+	@WF=$$(argo list -n $(NAMESPACE) --output name | tail -1); \
+	TSTAMP=$$(date +%Y%m%d-%H%M%S); \
+	argo logs -n $(NAMESPACE) $$WF -c main > logs/$${TSTAMP}-$${WF##*/}.log; \
+	echo "Wrote logs/$${TSTAMP}-$${WF##*/}.log"
+
+# Delete all workflows + completed pods
+clean:
+	argo delete -n $(NAMESPACE) --all || true
+	kubectl -n $(NAMESPACE) delete pod -l workflows.argoproj.io/completed=true --force --grace-period=0 || true
+
+_ensure-dirs:
+	@mkdir -p runs logs
+
+# Fetch from PVC: copy tarball, unpack into runs/<WF>/, pull any extra files on /outputs
+fetch-tar: _ensure-dirs
+	@WF=$$(argo list -n $(NAMESPACE) --output name | tail -1 | sed 's#.*/##'); \
+	PVC="$$WF-outpvc"; OUTDIR="runs/$$WF"; \
+	echo "Workflow: $$WF"; echo "PVC: $$PVC"; mkdir -p $$OUTDIR; \
+	kubectl -n $(NAMESPACE) delete pod fetch-$$WF --ignore-not-found >/dev/null 2>&1 || true; \
+	cat <<'YAML' | sed "s/{{WF}}/$$WF/g" | sed "s/{{PVC}}/$$PVC/g" | kubectl -n $(NAMESPACE) apply -f -
+	apiVersion: v1
+	kind: Pod
+	metadata:
+	  name: fetch-{{WF}}
+	spec:
+	  restartPolicy: Never
+	  containers:
+	  - name: fetch
+	    image: busybox:1.36
+	    command: ["sh","-lc","sleep 600"]
+	    volumeMounts:
+	    - name: out
+	      mountPath: /mnt/out
+	  volumes:
+	  - name: out
+	    persistentVolumeClaim:
+	      claimName: {{PVC}}
+	YAML
+	kubectl -n $(NAMESPACE) wait --for=condition=Ready pod/fetch-$$WF --timeout=60s
+	# Main artifact
+	kubectl -n $(NAMESPACE) cp fetch-$$WF:/mnt/out/geozarr.tar.gz $$OUTDIR/geozarr.tar.gz
+	tar -xzf $$OUTDIR/geozarr.tar.gz -C $$OUTDIR
+	# Copy any other files (e.g., dask-report.html)
+	kubectl -n $(NAMESPACE) cp fetch-$$WF:/mnt/out/. $$OUTDIR/ || true
+	kubectl -n $(NAMESPACE) delete pod fetch-$$WF --wait=false
+	@echo "Unpacked into $$OUTDIR/"
+
+# Convenience: build + load + template + submit + fetch
+run: apply submit fetch-tar
+
+# Cleanup stray per-run PVCs (removes stored artifacts)
+clean-pvc:
+	kubectl -n $(NAMESPACE) delete pvc -l workflows.argoproj.io/workflow 2>/dev/null || true

docs/pipeline-integration-recommendations.md

@@ -0,0 +1,74 @@
+# Pipeline Integration Recommendations
+
+This document lists enhancements observed while integrating `eopf-geozarr` into an Argo-based batch conversion pipeline (data-model-pipeline). They are candidates for upstream inclusion or API refinement.
+
+## 1. Output Prefix Expansion
+**Current (pipeline)**: Wrapper detects `output_zarr` ending with `/` and appends `<item_id>_geozarr.zarr` derived from the input STAC/Zarr URL.
+**Recommendation**: Support `--output-prefix` OR accept a trailing slash on positional `output_path` and perform the expansion internally (emitting the resolved final path). Add a log line: `Resolved output store: s3://.../S2A_..._geozarr.zarr`.
+
+## 2. Group Existence Validation (Pre-flight)
+**Current (pipeline)**: `validate_params_groups.py` inspects filesystem structure (presence of `.zarray` / `.zgroup`).
+**Recommendation**: Native CLI flag `--validate-groups` to prune or fail fast when groups don’t exist. Modes:
+- `--validate-groups=warn` (default): drop missing, report.
+- `--validate-groups=error`: abort if any missing.
+Emit JSON or structured summary when `--verbose`.
+
+## 3. Profiles (WOZ Profiles)
+**Current (pipeline)**: External JSON profile expansion before calling CLI.
+**Recommendation**: Provide `--profile <name>` in CLI mapping to preset groups + chunk params. Add `eopf-geozarr profile list` / `profile show <name>` subcommands. Keep external mechanism as fallback.
+
+## 4. Compressor Handling
+**Current**: Template attempted to pass `--compressor`; CLI does not expose codec choice.
+**Recommendation**: If codec selection is desired, add `--compressor <name>` now (zstd, lz4, blosc) with validation; else document fixed default explicitly to avoid confusion.
+
+## 5. CRS Groups Convenience
+**Current**: `--crs-groups` optional list.
+**Recommendation**: Discover candidate groups automatically (search for geometry-like datasets) unless `--crs-groups` provided (override). Provide `eopf-geozarr info --crs-scan` to preview.
+
+## 6. Dask Cluster Ergonomics
+**Current**: `--dask-cluster` toggles local cluster with no feedback.
+**Recommendation**: Print cluster dashboard URL (if available) and add `--dask-workers N` for quick scaling.
+
+## 7. Structured Logging / Run Metadata
+**Current**: Plain prints; pipeline scrapes logs.
+**Recommendation**: Optional `--run-metadata <path.json>` to write machine-readable summary: inputs, resolved groups, timings, warnings. Eases automation and reproducibility.
+
+## 8. Validation Command Enhancements
+**Current**: `validate` skeleton present but incomplete.
+**Recommendation**:
+- Implement spec checks (multiscales, attributes, chunk shape policy).
+- Exit code non-zero on *hard* failures, zero with warnings for soft issues.
+- `--format json` for programmatic consumption.
+
+## 9. HTML Tree Generation
+**Current**: `_generate_html_output` scaffold incomplete.
+**Recommendation**: Finish implementation; integrate with `info --html-output`. Provide minimal inline CSS (already drafted) and optional `--open-browser` flag.
+
+## 10. Progress Reporting
+**Recommendation**: Emit periodic progress per group: `group=/measurements/r10m scale=2 written=...MB elapsed=...s` to assist monitoring in batch workflows.
+
+## 11. Retry / Resumability
+**Recommendation**: Add `--resume` to skip already existing multiscale levels if output store partially present.
+
+## 12. Exit Codes (Contract)
+Document exit code meanings:
+- 0 success
+- 2 validation (input) error
+- 3 group resolution failure
+- 4 conversion runtime error
+
+## 13. Environment Variable Overrides
+Allow `EOZ_DEFAULT_PROFILE`, `EOZ_OUTPUT_PREFIX` env vars as implicit defaults (still overridden by flags).
+
+## 14. Example Invocation Block in README
+Provide ready-to-copy examples for Sentinel-2 & Sentinel-1 including polarization groups when Phase 1 logic is public.
+
+---
+**Next Steps (Suggested Order)**
+1. Implement output prefix expansion (low risk, high UX win)
+2. Group validation flag (prevents silent empty writes)
+3. Finish validate + info HTML features
+4. Add structured run metadata output
+5. Introduce profiles subcommands then deprecate external expansion path over time
+
+Feedback welcome—pipeline experience will continue surfacing actionable deltas.