refactor: update storage tier handling to v2.0 format in STAC items

lhoupert · lhoupert · commit 88b0e2be58ea · 2026-01-26T11:47:46.000Z
- Changed storage tier fields to use a nested 'storage:scheme' object.
- Updated tests to reflect new structure and ensure correct assertions.
- Adjusted functions to create and manage the new storage scheme format.
diff --git a/scripts/register_v1.py b/scripts/register_v1.py
@@ -375,7 +375,7 @@ def add_alternate_s3_assets(item: Item, s3_endpoint: str) -> None:
         # Query storage class for this asset
         storage_tier = get_s3_storage_class(s3_url, s3_endpoint)
 
-        # Add alternate with storage extension fields
+        # Add alternate with storage extension fields (v2.0 format)
         if not hasattr(asset, "extra_fields"):
             asset.extra_fields = {}
 
@@ -389,20 +389,27 @@ def add_alternate_s3_assets(item: Item, s3_endpoint: str) -> None:
         if not isinstance(existing_s3, dict):
             existing_s3 = {}
 
+        # Get or create storage:scheme object (v2.0 format)
+        storage_scheme = existing_s3.get("storage:scheme", {})
+        if not isinstance(storage_scheme, dict):
+            storage_scheme = {}
+
+        # Update scheme fields
+        storage_scheme["platform"] = "OVHcloud"
+        storage_scheme["region"] = region
+        storage_scheme["requester_pays"] = False
+
+        # Add tier to scheme (standard field in v2.0)
+        if storage_tier:
+            storage_scheme["tier"] = storage_tier
+
         # Update s3 alternate (preserving any existing fields)
         s3_alternate = {
             **existing_s3,  # Preserve existing fields
             "href": s3_url,
-            "storage:platform": "OVHcloud",
-            "storage:region": region,
-            "storage:requester_pays": False,
+            "storage:scheme": storage_scheme,
         }
 
-        # Add storage tier as a custom field (not part of storage extension spec)
-        # Using ovh: prefix to indicate vendor-specific extension
-        if storage_tier:
-            s3_alternate["ovh:storage_tier"] = storage_tier
-
         # Preserve other alternate formats (e.g., alternate.xarray if it exists)
         existing_alternate["s3"] = s3_alternate
         asset.extra_fields["alternate"] = existing_alternate
diff --git a/scripts/test_complete_workflow.py b/scripts/test_complete_workflow.py
@@ -104,12 +104,16 @@ def process_item_with_gateway(item_dict: dict, s3_endpoint: str) -> dict:
         # Add alternate S3 URL
         s3_url = https_to_s3(asset["href"])
         if s3_url:
+            # Create storage scheme object following v2.0 spec
+            storage_scheme = {
+                "platform": "OVHcloud",
+                "region": region,
+                "requester_pays": False,
+            }
             asset["alternate"] = {
                 "s3": {
                     "href": s3_url,
-                    "storage:platform": "OVHcloud",
-                    "storage:region": region,
-                    "storage:requester_pays": False,
+                    "storage:scheme": storage_scheme,
                 }
             }
             processed_count += 1
@@ -176,8 +180,10 @@ def main() -> int:
             if "alternate" in asset:
                 s3_alt = asset["alternate"]["s3"]
                 print(f"  S3 alternate href: {s3_alt['href']}")
-                print(f"  Storage platform: {s3_alt['storage:platform']}")
-                print(f"  Storage region: {s3_alt['storage:region']}")
+                if "storage:scheme" in s3_alt:
+                    scheme = s3_alt["storage:scheme"]
+                    print(f"  Storage platform: {scheme.get('platform', 'N/A')}")
+                    print(f"  Storage region: {scheme.get('region', 'N/A')}")
             break
 
     # Show store link
diff --git a/scripts/update_stac_storage_tier.py b/scripts/update_stac_storage_tier.py
@@ -124,13 +124,21 @@ def update_item_storage_tiers(
             if not isinstance(existing_alternate, dict):
                 existing_alternate = {}
 
+            # Create storage scheme object following v2.0 spec
+            storage_scheme = {
+                "platform": "OVHcloud",
+                "region": region,
+                "requester_pays": False,
+            }
+
+            # Add tier to scheme (standard field in v2.0)
+            if tier:
+                storage_scheme["tier"] = tier
+
             # Create alternate.s3 object
             s3_alternate = {
                 "href": s3_url,
-                "storage:platform": "OVHcloud",
-                "storage:region": region,
-                "storage:requester_pays": False,
-                "ovh:storage_tier": tier,
+                "storage:scheme": storage_scheme,
             }
 
             # Add distribution if storage is mixed or multiple files sampled
@@ -174,24 +182,32 @@ def update_item_storage_tiers(
         else:
             storage_tier = storage_info["tier"]
 
+        # Get or create storage:scheme object (v2.0 format)
+        storage_scheme = s3_info.get("storage:scheme", {})
+        if not isinstance(storage_scheme, dict):
+            storage_scheme = {}
+
         # Track if anything changed
         asset_changed = False
-        old_tier = s3_info.get("ovh:storage_tier")
-
-        # Update or add storage extension fields (only if missing)
-        if "storage:platform" not in s3_info:
-            s3_info["storage:platform"] = "OVHcloud"
-            asset_changed = True
-        if "storage:region" not in s3_info:
-            s3_info["storage:region"] = region
-            asset_changed = True
-        if "storage:requester_pays" not in s3_info:
-            s3_info["storage:requester_pays"] = False
-            asset_changed = True
-
-        # Add/update storage tier if available
+        old_tier = storage_scheme.get("tier")
+
+        # Update scheme fields (only if missing)
+        scheme_changed = False
+        if "platform" not in storage_scheme:
+            storage_scheme["platform"] = "OVHcloud"
+            scheme_changed = True
+        if "region" not in storage_scheme:
+            storage_scheme["region"] = region
+            scheme_changed = True
+        if "requester_pays" not in storage_scheme:
+            storage_scheme["requester_pays"] = False
+            scheme_changed = True
+
+        # Add/update tier in scheme (standard v2.0 field)
         if storage_tier:
-            s3_info["ovh:storage_tier"] = storage_tier
+            if storage_scheme.get("tier") != storage_tier:
+                storage_scheme["tier"] = storage_tier
+                scheme_changed = True
             assets_with_tier += 1
 
             # Add or update distribution if available
@@ -212,15 +228,20 @@ def update_item_storage_tiers(
                 logger.debug(f"  {asset_key}: {old_tier or 'none'} -> {storage_tier}")
         else:
             # Remove tier if it cannot be determined
-            if "ovh:storage_tier" in s3_info:
-                del s3_info["ovh:storage_tier"]
-                asset_changed = True
+            if "tier" in storage_scheme:
+                del storage_scheme["tier"]
+                scheme_changed = True
                 logger.debug(f"  {asset_key}: removed tier (not available)")
             # Also remove distribution if present
             if "ovh:storage_tier_distribution" in s3_info:
                 del s3_info["ovh:storage_tier_distribution"]
                 asset_changed = True
 
+        # Update storage:scheme in s3_info if changed
+        if scheme_changed:
+            s3_info["storage:scheme"] = storage_scheme
+            asset_changed = True
+
         if asset_changed:
             assets_updated += 1
 
diff --git a/tests/fixtures/update_storage_tier/stac_item_after_tier_change.json b/tests/fixtures/update_storage_tier/stac_item_after_tier_change.json
@@ -15,10 +15,12 @@
       "alternate": {
         "s3": {
           "href": "s3://bucket/data.zarr/measurements/reflectance",
-          "storage:platform": "OVHcloud",
-          "storage:region": "de",
-          "storage:requester_pays": false,
-          "ovh:storage_tier": "STANDARD_IA"
+          "storage:scheme": {
+            "platform": "OVHcloud",
+            "region": "de",
+            "requester_pays": false,
+            "tier": "STANDARD_IA"
+          }
         }
       }
     },
diff --git a/tests/fixtures/update_storage_tier/stac_item_before.json b/tests/fixtures/update_storage_tier/stac_item_before.json
@@ -15,10 +15,12 @@
       "alternate": {
         "s3": {
           "href": "s3://bucket/data.zarr/measurements/reflectance",
-          "storage:platform": "OVHcloud",
-          "storage:region": "de",
-          "storage:requester_pays": false,
-          "ovh:storage_tier": "STANDARD"
+          "storage:scheme": {
+            "platform": "OVHcloud",
+            "region": "de",
+            "requester_pays": false,
+            "tier": "STANDARD"
+          }
         }
       }
     },
diff --git a/tests/fixtures/update_storage_tier/stac_item_mixed_storage.json b/tests/fixtures/update_storage_tier/stac_item_mixed_storage.json
@@ -15,10 +15,12 @@
       "alternate": {
         "s3": {
           "href": "s3://bucket/data.zarr/measurements/reflectance",
-          "storage:platform": "OVHcloud",
-          "storage:region": "de",
-          "storage:requester_pays": false,
-          "ovh:storage_tier": "MIXED",
+          "storage:scheme": {
+            "platform": "OVHcloud",
+            "region": "de",
+            "requester_pays": false,
+            "tier": "MIXED"
+          },
           "ovh:storage_tier_distribution": {
             "STANDARD": 450,
             "STANDARD_IA": 608
diff --git a/tests/test_alternate_extension.py b/tests/test_alternate_extension.py
@@ -65,13 +65,16 @@ def add_alternate_s3_to_item(item_dict: dict, s3_endpoint: str) -> dict:
         if not s3_url:
             continue
 
-        # Add alternate
+        # Add alternate (v2.0 format)
+        storage_scheme = {
+            "platform": "OVHcloud",
+            "region": region,
+            "requester_pays": False,
+        }
         asset["alternate"] = {
             "s3": {
                 "href": s3_url,
-                "storage:platform": "OVHcloud",
-                "storage:region": region,
-                "storage:requester_pays": False,
+                "storage:scheme": storage_scheme,
             }
         }
         modified_count += 1
@@ -110,8 +113,10 @@ def main():
             if "alternate" in asset:
                 s3_alt = asset["alternate"]["s3"]
                 print(f"  S3 href: {s3_alt['href']}")
-                print(f"  Storage platform: {s3_alt['storage:platform']}")
-                print(f"  Storage region: {s3_alt['storage:region']}")
+                if "storage:scheme" in s3_alt:
+                    scheme = s3_alt["storage:scheme"]
+                    print(f"  Storage platform: {scheme.get('platform', 'N/A')}")
+                    print(f"  Storage region: {scheme.get('region', 'N/A')}")
             break
 
     # Show extensions
diff --git a/tests/test_s3_gateway.py b/tests/test_s3_gateway.py
@@ -308,9 +308,11 @@ def test_add_alternate_to_gateway_url(self):
 
         s3_alt = asset.extra_fields["alternate"]["s3"]
         assert s3_alt["href"] == "s3://my-bucket/path/file.zarr"
-        assert s3_alt["storage:platform"] == "OVHcloud"
-        assert s3_alt["storage:region"] == "de"
-        assert s3_alt["storage:requester_pays"] is False
+        assert "storage:scheme" in s3_alt
+        scheme = s3_alt["storage:scheme"]
+        assert scheme["platform"] == "OVHcloud"
+        assert scheme["region"] == "de"
+        assert scheme["requester_pays"] is False
 
     def test_add_alternate_to_old_s3_url(self):
         """Test adding alternate S3 URL to asset with old S3 format URL."""
@@ -336,7 +338,8 @@ def test_add_alternate_to_old_s3_url(self):
         asset = item.assets["data"]
         s3_alt = asset.extra_fields["alternate"]["s3"]
         assert s3_alt["href"] == "s3://my-bucket/path/file.zarr"
-        assert s3_alt["storage:region"] == "gra"
+        assert "storage:scheme" in s3_alt
+        assert s3_alt["storage:scheme"]["region"] == "gra"
 
     def test_skip_thumbnail_asset(self):
         """Test that thumbnail assets are skipped."""
@@ -426,8 +429,14 @@ def test_multiple_assets(self):
         # Data assets should have alternates
         assert "alternate" in item.assets["data1"].extra_fields
         assert "alternate" in item.assets["data2"].extra_fields
-        assert item.assets["data1"].extra_fields["alternate"]["s3"]["storage:region"] == "sbg"
-        assert item.assets["data2"].extra_fields["alternate"]["s3"]["storage:region"] == "sbg"
+        assert (
+            item.assets["data1"].extra_fields["alternate"]["s3"]["storage:scheme"]["region"]
+            == "sbg"
+        )
+        assert (
+            item.assets["data2"].extra_fields["alternate"]["s3"]["storage:scheme"]["region"]
+            == "sbg"
+        )
 
         # Thumbnail should not
         assert "alternate" not in item.assets["thumbnail"].extra_fields
@@ -451,7 +460,9 @@ def test_region_detection_de(self):
         )
 
         add_alternate_s3_assets(item, "https://s3.de.io.cloud.ovh.net")
-        assert item.assets["data"].extra_fields["alternate"]["s3"]["storage:region"] == "de"
+        assert (
+            item.assets["data"].extra_fields["alternate"]["s3"]["storage:scheme"]["region"] == "de"
+        )
 
     def test_region_detection_gra(self):
         """Test region detection for GRA region."""
@@ -472,7 +483,9 @@ def test_region_detection_gra(self):
         )
 
         add_alternate_s3_assets(item, "https://s3.gra.io.cloud.ovh.net")
-        assert item.assets["data"].extra_fields["alternate"]["s3"]["storage:region"] == "gra"
+        assert (
+            item.assets["data"].extra_fields["alternate"]["s3"]["storage:scheme"]["region"] == "gra"
+        )
 
     def test_region_detection_unknown(self):
         """Test region detection for unknown region."""
@@ -493,7 +506,10 @@ def test_region_detection_unknown(self):
         )
 
         add_alternate_s3_assets(item, "https://s3.amazonaws.com")
-        assert item.assets["data"].extra_fields["alternate"]["s3"]["storage:region"] == "unknown"
+        assert (
+            item.assets["data"].extra_fields["alternate"]["s3"]["storage:scheme"]["region"]
+            == "unknown"
+        )
 
     def test_extensions_not_duplicated(self):
         """Test that extensions are not duplicated if already present."""
diff --git a/tests/unit/test_update_stac_storage_tier.py b/tests/unit/test_update_stac_storage_tier.py
