fix(wsi): fall back to fsspec for ECS thumbnails

Merged after container smoke, security, and 194 local tests passed.

Author: raylim

app/thumbnail_store.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import json
+import logging
 import os
 import threading
 import time
@@ -20,6 +21,9 @@
 from .slide_store import s3_opts
 
 
+logger = logging.getLogger(__name__)
+
+
 @dataclass(frozen=True)
 class ThumbnailRecord:
     image_id: str
@@ -253,12 +257,26 @@ def store_generated_thumbnail(image_id: str, payload: bytes) -> ThumbnailRecord
 def read_thumbnail_bytes(record: ThumbnailRecord) -> bytes:
     if record.uri.startswith("s3://"):
         bucket, key = _s3_location(record.uri)
-        response = _runtime_s3().get_object(Bucket=bucket, Key=key)
-        body = response["Body"]
         try:
-            return body.read()
-        finally:
-            body.close()
+            response = _runtime_s3().get_object(Bucket=bucket, Key=key)
+            body = response["Body"]
+            try:
+                return body.read()
+            finally:
+                body.close()
+        except Exception as exc:
+            # ECS accepts the s3fs/fsspec path used by the slide reader even
+            # when the pooled boto3 client cannot complete a direct object
+            # read from a worker.  Keep the exact published variant URI and
+            # fall back to that known-good transport before reporting the
+            # thumbnail as unavailable.
+            logger.warning(
+                "Direct thumbnail object read failed; retrying through fsspec; error_type=%s",
+                type(exc).__name__,
+            )
+            fs = _filesystem_for_uri(record.uri)
+            with fs.open(_filesystem_path(record.uri), "rb") as handle:
+                return handle.read()
     fs = _filesystem_for_uri(record.uri)
     with fs.open(_filesystem_path(record.uri), "rb") as handle:
         return handle.read()

tests/test_thumbnail_store.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from io import BytesIO
+from unittest.mock import MagicMock
 
 from PIL import Image
 
@@ -15,6 +16,32 @@ def _jpeg_bytes(size: tuple[int, int]) -> bytes:
 
 
 class TestThumbnailStore:
+    def test_s3_read_falls_back_to_fsspec_after_direct_client_failure(self, monkeypatch):
+        record = thumbnail_store.ThumbnailRecord(
+            image_id="2907269",
+            uri="s3://bucket/2907269.jpg",
+            width=128,
+            height=88,
+        )
+        client = MagicMock()
+        client.get_object.side_effect = RuntimeError("direct ECS read failed")
+        handle = MagicMock()
+        handle.__enter__.return_value = handle
+        handle.read.return_value = b"fallback-jpeg"
+        filesystem = MagicMock()
+        filesystem.open.return_value = handle
+
+        monkeypatch.setattr(thumbnail_store, "_runtime_s3", lambda: client)
+        monkeypatch.setattr(
+            thumbnail_store, "_filesystem_for_uri", lambda uri: filesystem
+        )
+
+        assert thumbnail_store.read_thumbnail_bytes(record) == b"fallback-jpeg"
+        client.get_object.assert_called_once_with(
+            Bucket="bucket", Key="2907269.jpg"
+        )
+        filesystem.open.assert_called_once_with("bucket/2907269.jpg", "rb")
+
     def test_payload_passthrough_for_display_sized_variant(self):
         thumb_bytes = _jpeg_bytes((128, 79))
         record = thumbnail_store.ThumbnailRecord(