initial refactor to remove mediastore client in favor of BucketStore (untested)

.github/workflows/publish-image-api.yml

@@ -3,7 +3,7 @@ on:
   push:
     branches:
       - "main"
-      - "geolocation-fix"
+      - "remove-mediastore-client"
 
 env:
   IMAGE_NAME: nes-lter-api-2

api/chl/management/commands/importchl.py

@@ -24,9 +24,6 @@ class Command(BaseCommand):
 
     def __init__(self):
         super().__init__()
-        self.URL = os.getenv("URL")
-        self.TOKEN = os.getenv("TOKEN")
-        self.MEDIASTORE_PREFIX = os.getenv("MEDIASTORE_PREFIX")
         self.logger = logging.getLogger('management')
 
     def add_arguments(self, parser):
@@ -35,7 +32,7 @@ def add_arguments(self, parser):
     def read_btl_summary(self, cruise, chl):
 
         object_key = f"{cruise}{BTLSUM_SUFFIX}"
-        with get_store(self.URL, self.TOKEN, self.MEDIASTORE_PREFIX) as store:
+        with get_store() as store:
             try:
                 data = store.get(object_key)
             except Exception as e:
@@ -148,7 +145,7 @@ def handle(self, *args, **options):
                     csv_binary = csv_buffer.getvalue().encode("utf-8")
 
                     object_key = f"{cruise_name}{CHL_SUFFIX}"
-                    with get_store(self.URL, self.TOKEN, self.MEDIASTORE_PREFIX) as store:
+                    with get_store() as store:
                         try:
                             store.put(object_key, csv_binary)
                             self.stdout.write(self.style.SUCCESS(f'{cruise_name}{CHL_SUFFIX} successfully created.'))

api/chl/services.py

@@ -7,9 +7,6 @@
 from core.models import Cruise
 
 class ChlService:
-    URL = os.getenv("URL")
-    TOKEN = os.getenv("TOKEN")
-    MEDIASTORE_PREFIX = os.getenv("MEDIASTORE_PREFIX")
     FILE_SUFFIX = '_chl.csv'
 
     @classmethod
@@ -18,7 +15,7 @@ def get(cls, cruise_name: str) -> FileResponse:
         try:
             Cruise.objects.get(name__iexact=cruise_name) 
             object_key = f"{cruise_name.lower()}{cls.FILE_SUFFIX}"
-            with get_store(cls.URL, cls.TOKEN, cls.MEDIASTORE_PREFIX) as store:
+            with get_store() as store:
                 try:
                     data = store.get(object_key)
                 except Exception as e:
@@ -41,7 +38,7 @@ def getall(cls) -> FileResponse:
 
         for cruise in cruises:
             object_key = f"{cruise.name}{cls.FILE_SUFFIX}"
-            with get_store(cls.URL, cls.TOKEN, cls.MEDIASTORE_PREFIX) as store:
+            with get_store() as store:
                 try:
                     data = store.get(object_key)
                 except Exception as e:

api/core/utils.py

@@ -7,7 +7,8 @@
 import numpy as np
 import logging
 from contextlib import contextmanager
-from storage.mediastore import MediaStore
+import boto3
+from storage.s3 import BucketStore
 from storage.utils import PrefixStore
 from storage.object import DictStore
 from storage.fs import FilesystemStore
@@ -167,20 +168,28 @@ def _use_dictstore() -> bool:
     return os.getenv("USE_DICTSTORE", "FALSE").upper() == "TRUE"
 
 @contextmanager
-def get_store( url, token, prefix):
+def get_store():
+
+    prefix = os.getenv("S3_PREFIX", "")
 
     if _use_dictstore():
         # In-memory store for CI/tests; no network
         root = "/data/.store"
         os.makedirs(root, exist_ok=True)
         base_store = FilesystemStore(root)
-        prefixed = PrefixStore(base_store, prefix or "")
+        prefixed = PrefixStore(base_store, prefix)
         yield prefixed
     else:
-        # Real vast store
-        with MediaStore(url, token=token) as base_store:
-            prefixed = PrefixStore(base_store, prefix or "")
-            yield prefixed
+        # VAST S3 store
+        s3_client = boto3.client(
+            "s3",
+            endpoint_url=os.getenv("S3_ENDPOINT_URL"),
+            aws_access_key_id=os.getenv("S3_ACCESS_KEY_ID"),
+            aws_secret_access_key=os.getenv("S3_SECRET_ACCESS_KEY"),
+        )
+        base_store = BucketStore(os.getenv("S3_BUCKET_NAME"), s3_client)
+        prefixed = PrefixStore(base_store, prefix)
+        yield prefixed
 
 def date_time_to_datetime(date, time):
     try:

api/core/views.py

@@ -1,6 +1,5 @@
 from django.shortcuts import render
 from django.http import JsonResponse, HttpResponse, Http404, FileResponse
-import os
 import io
 import json
 import pandas as pd
@@ -308,12 +307,8 @@ def cruise_track_view(request, cruise_name):
             for cast in casts
         ]
 
-        URL = os.getenv("URL")
-        TOKEN = os.getenv("TOKEN")
-        MEDIASTORE_PREFIX = os.getenv("MEDIASTORE_PREFIX")
-
         object_key = f"{cruise_name}{UNDERWAY_SUFFIX}"
-        with get_store(URL, TOKEN, MEDIASTORE_PREFIX) as store:
+        with get_store() as store:
             try:
                 data = store.get(object_key)
             except Exception as e:
@@ -391,13 +386,9 @@ def ctd_plot_view(request, cruise_name, cast_number):
         "sbeox0ml_l": "Oxygen (mL/L)"
     }
 
-    URL = os.getenv("URL")
-    TOKEN = os.getenv("TOKEN")
-    MEDIASTORE_PREFIX = os.getenv("MEDIASTORE_PREFIX")
-
     object_key = f"{cruise_name}_ctd_cast_{cast_number}.csv"
 
-    with get_store(URL, TOKEN, MEDIASTORE_PREFIX) as store:
+    with get_store() as store:
             try:
                 data = store.get(object_key)
             except Exception as e:

api/ctd/management/commands/importcast.py

@@ -26,9 +26,6 @@ class Command(BaseCommand):
 
     def __init__(self):
         super().__init__()
-        self.URL = os.getenv("URL")
-        self.TOKEN = os.getenv("TOKEN")
-        self.MEDIASTORE_PREFIX = os.getenv("MEDIASTORE_PREFIX")
         self.logger = logging.getLogger('management')
 
     def add_arguments(self, parser):
@@ -101,7 +98,7 @@ def create_cast_file(self, file, cruise, cast, time):
             csv_binary = csv_buffer.getvalue().encode("utf-8")
 
             object_key = f"{cruise}{"_ctd_cast_"}{cast}{".csv"}"
-            with get_store(self.URL, self.TOKEN, self.MEDIASTORE_PREFIX) as store:
+            with get_store() as store:
                 try:
                     store.put(object_key, csv_binary)
                 except Exception as e:
@@ -219,7 +216,7 @@ def handle(self, *args, **options):
                 csv_binary = csv_buffer.getvalue().encode("utf-8")
 
                 object_key = f"{cruise.name}{METADATA_SUFFIX}"
-                with get_store(self.URL, self.TOKEN, self.MEDIASTORE_PREFIX) as store:
+                with get_store() as store:
                     try:
                         store.put(object_key, csv_binary)
                     except Exception as e:

api/ctd/management/commands/importcruise.py

@@ -10,8 +10,6 @@
 class Command(BaseCommand):
     help = 'Create Cruise Model. If Cruise Name is not supplied, all Cruises will be created.'
 
-    URL = os.getenv("URL")
-    TOKEN = os.getenv("TOKEN")
     logger = logging.getLogger('management')
 
     def add_arguments(self, parser):

api/ctd/management/commands/importniskin.py

@@ -141,12 +141,6 @@ def to_dataframe(cruise_name, cast, in_lines):
 class Command(BaseCommand):
     help = 'Create Ninkin Models. If Cruise Name is not supplied, all Niskins for all Cruises and Casts will be created.'
 
-    def __init__(self):
-        super().__init__()
-        self.URL = os.getenv("URL")
-        self.TOKEN = os.getenv("TOKEN")
-        self.MEDIASTORE_PREFIX = os.getenv("MEDIASTORE_PREFIX")
-
     def add_arguments(self, parser):
         parser.add_argument('--cruise_name', type=str, help='Optional name of the cruise.', default=None)
 
@@ -291,7 +285,7 @@ def handle(self, *args, **options):
                     csv_binary = csv_buffer.getvalue().encode("utf-8")
 
                     object_key = f"{cruise_name}{BOTTLES_SUFFIX}"
-                    with get_store(self.URL, self.TOKEN, self.MEDIASTORE_PREFIX) as store:
+                    with get_store() as store:
                         try:
                             store.put(object_key, csv_binary)
                         except Exception as e:
@@ -308,7 +302,7 @@ def handle(self, *args, **options):
                     csv_binary = csv_buffer.getvalue().encode("utf-8")
 
                     object_key = f"{cruise_name}{SUMMARY_SUFFIX}"
-                    with get_store(self.URL, self.TOKEN, self.MEDIASTORE_PREFIX) as store:
+                    with get_store() as store:
                         try:
                             store.put(object_key, csv_binary)
                             self.stdout.write(self.style.SUCCESS(f'Niskins for Cruise {cruise_name} successfully imported.'))

api/ctd/services.py

@@ -390,17 +390,15 @@ def get_casts_csv(cruise_name: str) -> FileResponse:
 
     @classmethod
     def get_cast_csv(cls, cruise_name: str, cast_number: str) -> FileResponse:
-        URL = os.getenv("URL")
-        TOKEN = os.getenv("TOKEN")
-        MEDIASTORE_PREFIX = os.getenv("MEDIASTORE_PREFIX")
+
 
         try:
             cruise = Cruise.objects.get(name__iexact=cruise_name)
             cruise_name = cruise.name.lower()
             cast_number = cast_number.lstrip("0")
             cast = Cast.objects.get(cruise=cruise, number__iexact=cast_number)
             object_key = f"{cruise_name}_ctd_cast_{cast.number}.csv"
-            with get_store(URL, TOKEN, MEDIASTORE_PREFIX) as store:
+            with get_store() as store:
                 try:
                     data = store.get(object_key)
                 except Exception as e:
@@ -592,14 +590,12 @@ def delete_niskin(cruise_name: str, cast_number: str, niskin_number: int):
 
     @classmethod
     def get_bottles(cls, cruise_name: str) -> FileResponse:
-        URL = os.getenv("URL")
-        TOKEN = os.getenv("TOKEN")
-        MEDIASTORE_PREFIX = os.getenv("MEDIASTORE_PREFIX")
+
         FILE_SUFFIX = '_ctd_bottles.csv'
         try:
             Cruise.objects.get(name__iexact=cruise_name) 
             object_key = f"{cruise_name.lower()}{FILE_SUFFIX}"
-            with get_store(URL, TOKEN, MEDIASTORE_PREFIX) as store:
+            with get_store() as store:
                 try:
                     data = store.get(object_key)
                 except Exception as e:
@@ -614,14 +610,12 @@ def get_bottles(cls, cruise_name: str) -> FileResponse:
 
     @classmethod
     def get_bottle_summary(cls, cruise_name: str) -> FileResponse:
-        URL = os.getenv("URL")
-        TOKEN = os.getenv("TOKEN")
-        MEDIASTORE_PREFIX = os.getenv("MEDIASTORE_PREFIX")
+
         FILE_SUFFIX = '_ctd_bottle_summary.csv'
         try:
             Cruise.objects.get(name__iexact=cruise_name) 
             object_key = f"{cruise_name.lower()}{FILE_SUFFIX}"
-            with get_store(URL, TOKEN, MEDIASTORE_PREFIX) as store:
+            with get_store() as store:
                 try:
                     data = store.get(object_key)
                 except Exception as e:
@@ -636,14 +630,12 @@ def get_bottle_summary(cls, cruise_name: str) -> FileResponse:
 
     @classmethod
     def get_metadata(cls, cruise_name: str) -> FileResponse:
-        URL = os.getenv("URL")
-        TOKEN = os.getenv("TOKEN")
-        MEDIASTORE_PREFIX = os.getenv("MEDIASTORE_PREFIX")
+
         FILE_SUFFIX = '_ctd_metadata.csv'
         try:
             Cruise.objects.get(name__iexact=cruise_name) 
             object_key = f"{cruise_name.lower()}{FILE_SUFFIX}"
-            with get_store(URL, TOKEN, MEDIASTORE_PREFIX) as store:
+            with get_store() as store:
                 try:
                     data = store.get(object_key)
                 except Exception as e:

api/events/management/commands/importevent.py

@@ -52,9 +52,6 @@ class Command(BaseCommand):
 
     def __init__(self):
         super().__init__()
-        self.URL = os.getenv("URL")
-        self.TOKEN = os.getenv("TOKEN")
-        self.MEDIASTORE_PREFIX = os.getenv("MEDIASTORE_PREFIX")
         self.logger = logging.getLogger('management')
 
     def add_arguments(self, parser):
@@ -69,7 +66,7 @@ def store_csv_file(self, cruise_name, csv_data):
         csv_binary = csv_buffer.getvalue().encode("utf-8")
         # Use the put method to store the CSV in the vast media store
         object_key = f"{cruise_name}{FILE_SUFFIX}"
-        with get_store(self.URL, self.TOKEN, self.MEDIASTORE_PREFIX) as store:
+        with get_store() as store:
             try:
                 store.put(object_key, csv_binary)
             except Exception as e:

api/events/services.py

@@ -1,4 +1,3 @@
-import os
 import io
 import pandas as pd
 import numpy as np
@@ -74,9 +73,7 @@ def serialize_event(event: Event) -> EventOutput:
         )
 
     def store_csv_file(self, cruise_name, csv_data):
-        URL = os.getenv("URL")
-        TOKEN = os.getenv("TOKEN")
-        MEDIASTORE_PREFIX = os.getenv("MEDIASTORE_PREFIX")
+
 
         df = pd.DataFrame(csv_data)
         df[DATETIME] = pd.to_datetime(df[DATETIME])
@@ -88,7 +85,7 @@ def store_csv_file(self, cruise_name, csv_data):
         csv_binary = csv_buffer.getvalue().encode("utf-8")
         # Use the put method to store the CSV in the vast media store
         object_key = f"{cruise_name}{FILE_SUFFIX}"
-        with get_store(URL, TOKEN, MEDIASTORE_PREFIX) as store:
+        with get_store() as store:
             try:
                 store.put(object_key, csv_binary)
             except Exception as e:
@@ -114,15 +111,13 @@ def apply_additions(self, addns_path):
 
     @classmethod
     def get_events(cls, cruise_name: str) -> FileResponse:
-        URL = os.getenv("URL")
-        TOKEN = os.getenv("TOKEN")
-        MEDIASTORE_PREFIX = os.getenv("MEDIASTORE_PREFIX")
+
 
         try:
             cruise = Cruise.objects.get(name__iexact=cruise_name) 
             if Event.objects.filter(cruise=cruise).exists():
                 object_key = f"{cruise_name.lower()}{FILE_SUFFIX}"
-                with get_store(URL, TOKEN, MEDIASTORE_PREFIX) as store:
+                with get_store() as store:
                     try:
                         data = store.get(object_key)
                     except Exception as e:

api/hplc/management/commands/importhplc.py

@@ -24,9 +24,6 @@ class Command(BaseCommand):
 
     def __init__(self):
         super().__init__()
-        self.URL = os.getenv("URL")
-        self.TOKEN = os.getenv("TOKEN")
-        self.MEDIASTORE_PREFIX = os.getenv("MEDIASTORE_PREFIX")
         self.logger = logging.getLogger('management')
 
     def add_arguments(self, parser):
@@ -130,7 +127,7 @@ def handle(self, *args, **options):
                 csv_binary = csv_buffer.getvalue().encode("utf-8")
 
                 object_key = f"{cruise_name}{HPLC_SUFFIX}"
-                with get_store(self.URL, self.TOKEN, self.MEDIASTORE_PREFIX) as store:
+                with get_store() as store:
                     try:
                         store.put(object_key, csv_binary)
                         self.stdout.write(self.style.SUCCESS(f'{cruise_name}{HPLC_SUFFIX} successfully created.'))

api/hplc/services.py

@@ -10,16 +10,13 @@ class HplcService:
 
     @classmethod
     def get(cls, cruise_name: str) -> FileResponse:
-        URL = os.getenv("URL")
-        TOKEN = os.getenv("TOKEN")
-        MEDIASTORE_PREFIX = os.getenv("MEDIASTORE_PREFIX")
         FILE_SUFFIX = '_hplc.csv'
 
         try:
             if cruise_name.lower() != "mvco":
                 Cruise.objects.get(name__iexact=cruise_name) 
             object_key = f"{cruise_name.lower()}{FILE_SUFFIX}"
-            with get_store(URL, TOKEN, MEDIASTORE_PREFIX) as store:
+            with get_store() as store:
                 try:
                     data = store.get(object_key)
                 except Exception as e: