Merge pull request #29 from cBioPortal/codex/beta-16m-contract

perf(wsi): unify 16M decode and metadata contract

Author: raylim

README.md

@@ -100,7 +100,8 @@ optional Redis cache:
 | `THUMBNAIL_PREWARM_URI` | — | Optional stable thumbnail object used to prewarm each worker |
 | `TILE_SIZE` | `256` | Tile edge length |
 | `JPEG_QUALITY` | `85` | JPEG encoding quality |
-| `MAX_DECODE_PIXELS` | `4194304` | Maximum on-demand tile decode |
+| `MAX_DECODE_PIXELS` | `16777216` | Maximum on-demand tile decode |
+| `THUMBNAIL_MAX_DECODE_PIXELS` | `16777216` | Maximum thumbnail decode |
 | `MAX_OPEN_SLIDES` | `64` | Open-slide LRU capacity |
 | `MAX_IMAGE_OPERATIONS` | `2` | Concurrent pixel operations per worker |
 | `N_WORKERS` | `4` | Gunicorn worker count |

app/config.py

@@ -72,9 +72,9 @@ class Settings:
     # Tile settings
     tile_size: int = field(default_factory=lambda: _env_int("TILE_SIZE", 256))
     jpeg_quality: int = field(default_factory=lambda: _env_int("JPEG_QUALITY", 85))
-    max_decode_pixels: int = field(default_factory=lambda: _env_int("MAX_DECODE_PIXELS", 4_194_304))
+    max_decode_pixels: int = field(default_factory=lambda: _env_int("MAX_DECODE_PIXELS", 16_777_216))
     thumbnail_max_decode_pixels: int = field(
-        default_factory=lambda: _env_int("THUMBNAIL_MAX_DECODE_PIXELS", 16_000_000)
+        default_factory=lambda: _env_int("THUMBNAIL_MAX_DECODE_PIXELS", 16_777_216)
     )
     thumbnail_timeout_sec: int = field(
         default_factory=lambda: _env_int("THUMBNAIL_TIMEOUT_SEC", 8)

app/identity.py

@@ -8,6 +8,7 @@
 
 
 IDENTITY_VERSION = "v2"
+TILE_METADATA_SCHEMA_VERSION = 2
 
 
 def timestamp_to_epoch_ms(value: Any) -> int | None:
@@ -39,7 +40,7 @@ def source_fingerprint(path: Any, size: Any, last_modified: Any) -> str | None:
 
 
 def decode_policy_version() -> str:
-    return f"geometry-v1;tile-max={settings.max_decode_pixels};thumbnail-max={settings.thumbnail_max_decode_pixels}"
+    return f"geometry-v2;tile-max={settings.max_decode_pixels};thumbnail-max={settings.thumbnail_max_decode_pixels}"
 
 
 from .config import settings  # noqa: E402  (settings is needed by policy helper)

app/metadata_contract.py

@@ -0,0 +1,113 @@
+"""Validation for the source-bound WSI tile metadata contract."""
+from __future__ import annotations
+
+import math
+from typing import Any
+
+from .config import settings
+from .identity import TILE_METADATA_SCHEMA_VERSION, decode_policy_version
+
+
+def validate_tile_metadata(
+    metadata: Any,
+    *,
+    allow_legacy: bool = True,
+) -> tuple[bool, str | None]:
+    """Return whether metadata is usable by the tile server.
+
+    Schema-null records are retained for the migration window, but an
+    explicitly versioned record must satisfy the complete v2 contract.
+    """
+    if not isinstance(metadata, dict):
+        return False, "metadata_not_object"
+
+    schema = metadata.get("tile_metadata_schema_version")
+    if schema is None:
+        if not allow_legacy:
+            return False, "legacy_metadata_not_allowed"
+        return _validate_common(metadata, require_v2=False)
+    if (
+        not isinstance(schema, int)
+        or isinstance(schema, bool)
+        or schema != TILE_METADATA_SCHEMA_VERSION
+    ):
+        return False, "unsupported_metadata_schema"
+
+    valid, reason = _validate_common(metadata, require_v2=True)
+    if not valid:
+        return valid, reason
+    if metadata.get("safe_min_level") is None:
+        return False, "missing_safe_min_level"
+    if metadata.get("decode_policy_version") != decode_policy_version():
+        return False, "stale_decode_policy"
+    if (
+        not isinstance(metadata.get("max_decode_pixels"), int)
+        or isinstance(metadata.get("max_decode_pixels"), bool)
+        or metadata.get("max_decode_pixels") != settings.max_decode_pixels
+    ):
+        return False, "stale_tile_decode_limit"
+    if (
+        not isinstance(metadata.get("thumbnail_max_decode_pixels"), int)
+        or isinstance(metadata.get("thumbnail_max_decode_pixels"), bool)
+        or metadata.get("thumbnail_max_decode_pixels") != settings.thumbnail_max_decode_pixels
+    ):
+        return False, "stale_thumbnail_decode_limit"
+    return True, None
+
+
+def _validate_common(metadata: dict[str, Any], *, require_v2: bool) -> tuple[bool, str | None]:
+    dimensions = metadata.get("dimensions")
+    if not isinstance(dimensions, dict):
+        return False, "invalid_dimensions"
+    width = dimensions.get("width")
+    height = dimensions.get("height")
+    if not _positive_int(width) or not _positive_int(height):
+        return False, "invalid_dimensions"
+
+    levels = metadata.get("levels")
+    if not _positive_int(levels):
+        return False, "invalid_levels"
+    level_dimensions = metadata.get("level_dimensions")
+    if not isinstance(level_dimensions, list) or len(level_dimensions) != levels:
+        return False, "invalid_level_dimensions"
+    for level in level_dimensions:
+        if (
+            not isinstance(level, dict)
+            or not _positive_int(level.get("width"))
+            or not _positive_int(level.get("height"))
+        ):
+            return False, "invalid_level_dimensions"
+
+    downsample_values = metadata.get("level_downsamples")
+    if require_v2:
+        if not isinstance(downsample_values, list) or len(downsample_values) != levels:
+            return False, "invalid_level_downsamples"
+        if any(
+            not isinstance(value, (int, float))
+            or isinstance(value, bool)
+            or not math.isfinite(float(value))
+            or float(value) <= 0
+            for value in downsample_values
+        ):
+            return False, "invalid_level_downsamples"
+
+    max_zoom = metadata.get("max_zoom")
+    tile_size = metadata.get("tile_size")
+    if not isinstance(max_zoom, int) or isinstance(max_zoom, bool) or max_zoom < 0:
+        return False, "invalid_max_zoom"
+    if not _positive_int(tile_size):
+        return False, "invalid_tile_size"
+
+    safe_min_level = metadata.get("safe_min_level")
+    if safe_min_level is not None and (
+        not isinstance(safe_min_level, int)
+        or isinstance(safe_min_level, bool)
+        or safe_min_level < 0
+        or safe_min_level > max_zoom
+    ):
+        return False, "invalid_safe_min_level"
+    return True, None
+
+
+def _positive_int(value: Any) -> bool:
+    return isinstance(value, int) and not isinstance(value, bool) and value > 0

app/metrics.py

@@ -16,7 +16,7 @@
 DECODE_SOURCE_PIXELS = Histogram(
     "tile_server_decode_source_pixels",
     "Source pixels decoded for a tile or thumbnail request.",
-    buckets=(0, 65_536, 262_144, 1_048_576, 4_194_304, 8_388_608, float("inf")),
+    buckets=(0, 65_536, 262_144, 1_048_576, 4_194_304, 8_388_608, 16_777_216, float("inf")),
 )
 OVERSIZED_DECODE_REJECTIONS = Counter(
     "tile_server_oversized_decode_rejections_total",

app/tiles.py

@@ -17,13 +17,11 @@
 from tiffslide import TiffSlide
 
 from .config import settings
+from .identity import IDENTITY_VERSION, TILE_METADATA_SCHEMA_VERSION, decode_policy_version
 from .metrics import DECODE_SOURCE_PIXELS
 
 TILE_SIZE = settings.tile_size
-DECODE_POLICY_VERSION = (
-    f"geometry-v1;tile-max={settings.max_decode_pixels};"
-    f"thumbnail-max={settings.thumbnail_max_decode_pixels}"
-)
+DECODE_POLICY_VERSION = decode_policy_version()
 
 
 class OverviewTooLarge(RuntimeError):
@@ -115,7 +113,65 @@ def max_zoom(slide: TiffSlide) -> int:
     rounding to the nearest power-of-two pyramid level).
     """
     w, h = slide.dimensions
-    return math.ceil(math.log2(max(w, h) / TILE_SIZE))
+    return max(0, math.ceil(math.log2(max(w, h) / TILE_SIZE)))
+
+
+def _best_level_for_downsample(level_downsamples: list[float], downsample: float) -> int:
+    """Match tiffslide's level selection without opening or decoding a slide."""
+    if downsample <= 1.0:
+        return 0
+    for level, level_downsample in enumerate(level_downsamples):
+        if level_downsample > downsample:
+            return max(0, level - 1)
+    return len(level_downsamples) - 1
+
+
+def safe_min_level_from_geometry(
+    *,
+    width: int,
+    height: int,
+    level_dimensions: list[tuple[int, int]],
+    level_downsamples: list[float],
+    tile_size: int,
+    max_decode_pixels: int,
+) -> int | None:
+    """Compute the first safe ZXY level using only intrinsic pyramid geometry."""
+    if (
+        width <= 0
+        or height <= 0
+        or not level_dimensions
+        or len(level_dimensions) != len(level_downsamples)
+        or tile_size <= 0
+        or max_decode_pixels <= 0
+    ):
+        return None
+    pyramid_max_zoom = max(0, math.ceil(math.log2(max(width, height) / tile_size)))
+    for z in range(pyramid_max_zoom + 1):
+        target_ds = 2 ** (pyramid_max_zoom - z)
+        tiles_x = max(1, math.ceil(width / (tile_size * target_ds)))
+        tiles_y = max(1, math.ceil(height / (tile_size * target_ds)))
+        worst_pixels = 0
+        best_level = _best_level_for_downsample(level_downsamples, target_ds)
+        level_w, level_h = level_dimensions[best_level]
+        level_ds = max(1.0, float(level_downsamples[best_level]))
+        for x in {0, tiles_x - 1}:
+            for y in {0, tiles_y - 1}:
+                x0 = x * tile_size * target_ds
+                y0 = y * tile_size * target_ds
+                src_w = min(tile_size * target_ds, width - x0)
+                src_h = min(tile_size * target_ds, height - y0)
+                read_w = min(
+                    math.ceil(src_w / level_ds),
+                    level_w - math.floor(x0 / level_ds),
+                )
+                read_h = min(
+                    math.ceil(src_h / level_ds),
+                    level_h - math.floor(y0 / level_ds),
+                )
+                worst_pixels = max(worst_pixels, max(0, read_w) * max(0, read_h))
+        if worst_pixels <= max_decode_pixels:
+            return z
+    return None
 
 
 def safe_min_level(slide: TiffSlide) -> int | None:
@@ -125,33 +181,17 @@ def safe_min_level(slide: TiffSlide) -> int | None:
     result is emitted into offline tile metadata and remains advisory because
     the request path independently enforces the same pixel budget.
     """
-    slide_w, slide_h = slide.dimensions
-    for z in range(max_zoom(slide) + 1):
-        target_ds = 2 ** (max_zoom(slide) - z)
-        tiles_x = max(1, math.ceil(slide_w / (TILE_SIZE * target_ds)))
-        tiles_y = max(1, math.ceil(slide_h / (TILE_SIZE * target_ds)))
-        worst_pixels = 0
-        for x in {0, tiles_x - 1}:
-            for y in {0, tiles_y - 1}:
-                try:
-                    _, _, x0, y0, src_w, src_h, _, _ = _tile_geometry(slide, z, x, y)
-                    best_level = slide.get_best_level_for_downsample(target_ds)
-                    level_ds = max(1.0, float(slide.level_downsamples[best_level]))
-                    level_w, level_h = slide.level_dimensions[best_level]
-                    read_w = min(
-                        math.ceil(src_w / level_ds),
-                        level_w - math.floor(x0 / level_ds),
-                    )
-                    read_h = min(
-                        math.ceil(src_h / level_ds),
-                        level_h - math.floor(y0 / level_ds),
-                    )
-                    worst_pixels = max(worst_pixels, max(0, read_w) * max(0, read_h))
-                except (IndexError, ValueError, TypeError):
-                    return None
-        if worst_pixels <= settings.max_decode_pixels:
-            return z
-    return None
+    try:
+        return safe_min_level_from_geometry(
+            width=int(slide.dimensions[0]),
+            height=int(slide.dimensions[1]),
+            level_dimensions=[(int(width), int(height)) for width, height in slide.level_dimensions],
+            level_downsamples=[float(value) for value in slide.level_downsamples],
+            tile_size=TILE_SIZE,
+            max_decode_pixels=settings.max_decode_pixels,
+        )
+    except (IndexError, ValueError, TypeError, ZeroDivisionError):
+        return None
 
 
 def _slide_properties_metadata(slide: TiffSlide) -> tuple[float, float, str, int | None]:
@@ -187,7 +227,8 @@ def slide_metadata(slide: TiffSlide) -> dict:
         "objective_power": objective_power,
         "vendor": vendor,
         "safe_min_level": safe_min_level(slide),
-        "identity_version": "v2",
+        "identity_version": IDENTITY_VERSION,
+        "tile_metadata_schema_version": TILE_METADATA_SCHEMA_VERSION,
         "decode_policy_version": DECODE_POLICY_VERSION,
         "max_decode_pixels": settings.max_decode_pixels,
         "thumbnail_max_decode_pixels": settings.thumbnail_max_decode_pixels,

docker-compose.yml

@@ -34,7 +34,8 @@ services:
       # ── Tile settings ─────────────────────────────────────────────────────
       TILE_SIZE: ${TILE_SIZE:-256}
       JPEG_QUALITY: ${JPEG_QUALITY:-85}
-      MAX_DECODE_PIXELS: ${MAX_DECODE_PIXELS:-4194304}
+      MAX_DECODE_PIXELS: ${MAX_DECODE_PIXELS:-16777216}
+      THUMBNAIL_MAX_DECODE_PIXELS: ${THUMBNAIL_MAX_DECODE_PIXELS:-16777216}
 
       # ── Redis cache ───────────────────────────────────────────────────────
       REDIS_URL: redis://:${REDIS_PASSWORD:?REDIS_PASSWORD must be set}@redis:6379

tests/test_config.py

@@ -65,11 +65,11 @@ def test_profile_used_when_env_absent(self):
 class TestOtherSettings:
     def test_max_decode_pixels_default(self):
         s = make_settings()
-        assert s.max_decode_pixels == 4_194_304
+        assert s.max_decode_pixels == 16_777_216
 
     def test_thumbnail_max_decode_pixels_default(self):
         s = make_settings()
-        assert s.thumbnail_max_decode_pixels == 16_000_000
+        assert s.thumbnail_max_decode_pixels == 16_777_216
 
     def test_thumbnail_cache_ttl_default(self):
         s = make_settings()