chore: tier distribution into the storage scheme

lhoupert · lhoupert · commit 97534d921b4c · 2026-01-26T17:27:04.000Z
diff --git a/scripts/README_update_storage_tier.md b/scripts/README_update_storage_tier.md
@@ -4,7 +4,7 @@ Updates existing STAC items with current S3 storage tier metadata.
 
 ## Modes
 
-**Update (default)**: Updates `ovh:storage_tier` for assets with existing `alternate.s3`
+**Update (default)**: Updates `storage:scheme.tier` for assets with existing `alternate.s3`
 
 **Add Missing (`--add-missing`)**: Creates `alternate.s3` structure for legacy items without it
 
@@ -19,19 +19,24 @@ Distribution shows file counts per tier, based on sample of up to 100 files.
 ### Example: Mixed Storage
 ```json
 {
-  "ovh:storage_tier": "MIXED",
-  "ovh:storage_tier_distribution": {
-    "STANDARD": 450,
-    "STANDARD_IA": 608
+  "storage:scheme": {
+    "platform": "OVHcloud",
+    "region": "de",
+    "requester_pays": false,
+    "tier": "MIXED",
+    "tier_distribution": {
+      "STANDARD": 450,
+      "STANDARD_IA": 608
+    }
   }
 }
 ```
 
 ## Notes
 
 - Thumbnail assets automatically skipped
-- Failed S3 queries remove existing `ovh:storage_tier` field
-- Distribution metadata only for Zarr directories
+- Failed S3 queries remove existing `storage:scheme.tier` field
+- Distribution metadata only for Zarr directories (stored in `storage:scheme.tier_distribution`)
 
 ## Setup
 
diff --git a/scripts/storage_tier_utils.py b/scripts/storage_tier_utils.py
@@ -103,7 +103,8 @@ def get_s3_storage_class(s3_url: str, s3_endpoint: str) -> str | None:
                 # Extract storage classes from list response (no need for additional head_object calls)
                 storage_classes = []
                 for obj in list_response["Contents"]:
-                    # StorageClass field is included in list_objects_v2 response
+                    # StorageClass field is included in list_objects_v2 response.
+                    # If not present for STANDARD tier, default to STANDARD
                     obj_class = obj.get("StorageClass", "STANDARD")
                     storage_classes.append(obj_class)
 
diff --git a/scripts/update_stac_storage_tier.py b/scripts/update_stac_storage_tier.py
@@ -135,16 +135,16 @@ def update_item_storage_tiers(
             if tier:
                 storage_scheme["tier"] = tier
 
+            # Add distribution to scheme if storage is mixed or multiple files sampled
+            if storage_info["distribution"] is not None:
+                storage_scheme["tier_distribution"] = storage_info["distribution"]
+
             # Create alternate.s3 object
             s3_alternate = {
                 "href": s3_url,
                 "storage:scheme": storage_scheme,
             }
 
-            # Add distribution if storage is mixed or multiple files sampled
-            if storage_info["distribution"] is not None:
-                s3_alternate["ovh:storage_tier_distribution"] = storage_info["distribution"]
-
             # Preserve other alternate formats (e.g., alternate.xarray if it exists)
             existing_alternate["s3"] = s3_alternate
             asset.extra_fields["alternate"] = existing_alternate
@@ -210,18 +210,20 @@ def update_item_storage_tiers(
                 scheme_changed = True
             assets_with_tier += 1
 
-            # Add or update distribution if available
+            # Add or update distribution in scheme if available
             if storage_info and storage_info.get("distribution") is not None:
-                s3_info["ovh:storage_tier_distribution"] = storage_info["distribution"]
+                if storage_scheme.get("tier_distribution") != storage_info["distribution"]:
+                    storage_scheme["tier_distribution"] = storage_info["distribution"]
+                    scheme_changed = True
                 if storage_tier == "MIXED":
                     logger.info(
                         f"  {asset_key}: Mixed storage detected - {storage_info['distribution']}"
                     )
             else:
-                # Remove distribution if no longer mixed
-                if "ovh:storage_tier_distribution" in s3_info:
-                    del s3_info["ovh:storage_tier_distribution"]
-                    asset_changed = True
+                # Remove distribution from scheme if no longer mixed
+                if "tier_distribution" in storage_scheme:
+                    del storage_scheme["tier_distribution"]
+                    scheme_changed = True
 
             if old_tier != storage_tier:
                 asset_changed = True
@@ -232,10 +234,10 @@ def update_item_storage_tiers(
                 del storage_scheme["tier"]
                 scheme_changed = True
                 logger.debug(f"  {asset_key}: removed tier (not available)")
-            # Also remove distribution if present
-            if "ovh:storage_tier_distribution" in s3_info:
-                del s3_info["ovh:storage_tier_distribution"]
-                asset_changed = True
+            # Also remove distribution from scheme if present
+            if "tier_distribution" in storage_scheme:
+                del storage_scheme["tier_distribution"]
+                scheme_changed = True
 
         # Update storage:scheme in s3_info if changed
         if scheme_changed:
diff --git a/tests/fixtures/update_storage_tier/stac_item_mixed_storage.json b/tests/fixtures/update_storage_tier/stac_item_mixed_storage.json
@@ -19,11 +19,11 @@
             "platform": "OVHcloud",
             "region": "de",
             "requester_pays": false,
-            "tier": "MIXED"
-          },
-          "ovh:storage_tier_distribution": {
-            "STANDARD": 450,
-            "STANDARD_IA": 608
+            "tier": "MIXED",
+            "tier_distribution": {
+              "STANDARD": 450,
+              "STANDARD_IA": 608
+            }
           }
         }
       }
diff --git a/tests/unit/test_update_stac_storage_tier.py b/tests/unit/test_update_stac_storage_tier.py
@@ -148,7 +148,7 @@ def test_mixed_storage_adds_distribution(self, mock_get_info, stac_item_before):
         assert updated == 1
         s3_info = stac_item_before.assets["reflectance"].extra_fields["alternate"]["s3"]
         assert s3_info["storage:scheme"]["tier"] == "MIXED"
-        assert s3_info["ovh:storage_tier_distribution"] == {"STANDARD": 450, "GLACIER": 608}
+        assert s3_info["storage:scheme"]["tier_distribution"] == {"STANDARD": 450, "GLACIER": 608}
 
     @patch("update_stac_storage_tier.get_s3_storage_info")
     def test_uniform_zarr_adds_distribution(self, mock_get_info, stac_item_before):
@@ -164,7 +164,7 @@ def test_uniform_zarr_adds_distribution(self, mock_get_info, stac_item_before):
         assert updated == 1
         s3_info = stac_item_before.assets["reflectance"].extra_fields["alternate"]["s3"]
         assert s3_info["storage:scheme"]["tier"] == "GLACIER"
-        assert s3_info["ovh:storage_tier_distribution"] == {"GLACIER": 100}
+        assert s3_info["storage:scheme"]["tier_distribution"] == {"GLACIER": 100}
 
     @patch("update_stac_storage_tier.get_s3_storage_info")
     def test_single_file_no_distribution(self, mock_get_info, stac_item_before):
@@ -180,7 +180,7 @@ def test_single_file_no_distribution(self, mock_get_info, stac_item_before):
         assert updated == 1
         s3_info = stac_item_before.assets["reflectance"].extra_fields["alternate"]["s3"]
         assert s3_info["storage:scheme"]["tier"] == "GLACIER"
-        assert "ovh:storage_tier_distribution" not in s3_info
+        assert "tier_distribution" not in s3_info["storage:scheme"]
 
     @patch("update_stac_storage_tier.get_s3_storage_info")
     def test_removes_distribution_when_becomes_single_file(self, mock_get_info, stac_item_before):
@@ -192,7 +192,7 @@ def test_removes_distribution_when_becomes_single_file(self, mock_get_info, stac
         if "storage:scheme" not in s3_info:
             s3_info["storage:scheme"] = {}
         s3_info["storage:scheme"]["tier"] = "MIXED"
-        s3_info["ovh:storage_tier_distribution"] = {"STANDARD": 450, "GLACIER": 608}
+        s3_info["storage:scheme"]["tier_distribution"] = {"STANDARD": 450, "GLACIER": 608}
 
         mock_get_info.return_value = {"tier": "GLACIER", "distribution": None}
 
@@ -203,7 +203,7 @@ def test_removes_distribution_when_becomes_single_file(self, mock_get_info, stac
         assert updated == 1
         s3_info = stac_item_before.assets["reflectance"].extra_fields["alternate"]["s3"]
         assert s3_info["storage:scheme"]["tier"] == "GLACIER"
-        assert "ovh:storage_tier_distribution" not in s3_info
+        assert "tier_distribution" not in s3_info["storage:scheme"]
 
     @patch("update_stac_storage_tier.get_s3_storage_info")
     def test_skips_thumbnail_assets(self, mock_get_info, stac_item_before):

Original file line number	Diff line number	Diff line change
`@@ -19,11 +19,11 @@`
`19`	`19`	`"platform": "OVHcloud",`
`20`	`20`	`"region": "de",`
`21`	`21`	`"requester_pays": false,`
`22`		`- "tier": "MIXED"`
`23`		`- },`
`24`		`- "ovh:storage_tier_distribution": {`
`25`		`- "STANDARD": 450,`
`26`		`- "STANDARD_IA": 608`
	`22`	`+ "tier": "MIXED",`
	`23`	`+ "tier_distribution": {`
	`24`	`+ "STANDARD": 450,`
	`25`	`+ "STANDARD_IA": 608`
	`26`	`+ }`
`27`	`27`	`}`
`28`	`28`	`}`
`29`	`29`	`}`