feat: stage-out MAAP use case (#114)

* feat: add AUDIT log level for upload

* chore: update outdated tests

* fix: allow empty str as RESULT_PATH_PREFIX & replace w/ default val

* fix: allowing optional original stac item

* feat: add archive method

* feat: adding stac fast api client

* feat: fast api catalog code

* fix: add missing file

* fix: fix fast api catalog with test case

* feat: add update item

* fix: minor updates

Author: wphyojpl

.gitignore

@@ -13,4 +13,5 @@ __pycache__
 .terraform.lock.hcl
 terraform.tf
 terraform.tfvars
-cumulus_lambda_functions_deployment.zip
+cumulus_lambda_functions_deployment.zip
+examples/**/*

mdps_ds_lib/ds_client/auth_token/token_factory.py

@@ -5,6 +5,9 @@ class TokenFactory(FactoryAbstract):
     DUMMY = 'DUMMY'
     COGNITO = 'COGNITO'
 
+    def get_instance_from_env(self, **kwargs):
+        raise NotImplementedError('not yet')
+
     def get_instance(self, class_type, **kwargs):
         ct = class_type.upper()
         if ct == self.DUMMY:

mdps_ds_lib/ds_client/ds_client_user.py

@@ -290,6 +290,25 @@ def add_archive_config(self, daac_config: dict):
         response = json.loads(response.text)
         return response
 
+    def get_archive_config(self):
+        if self.tenant is None or self.tenant_venue is None or self.collection is None:
+            raise ValueError(f'require to set tenant & tenant_venue & collection')
+        collection_id = ':'.join([self.urn, self.org, self.project, self.tenant, self.tenant_venue, self.get_complete_collection()])
+
+        request_url = f'{self._uds_url}collections/{collection_id}/archive/'
+        print(f'request_url: {request_url}')
+        print(f'token: {self._token_retriever.get_token()}')
+        s = requests.session()
+        s.trust_env = self._trust_env
+        response = s.get(url=request_url, headers={
+            'Authorization': f'Bearer {self._token_retriever.get_token()}',
+            'Content-Type': 'application/json',
+
+        }, verify=self._trust_env)
+        response.raise_for_status()
+        response = json.loads(response.text)
+        return response
+
     def archive_granule(self):
         if self.tenant is None or self.tenant_venue is None or self.collection is None or self.granule is None:
             raise ValueError(f'require to set tenant & tenant_venue & collection & granule')

mdps_ds_lib/lib/utils/factory_abstract.py

@@ -5,3 +5,7 @@ class FactoryAbstract(ABC):
     @abstractmethod
     def get_instance(self, class_type, **kwargs):
         return
+
+    @abstractmethod
+    def get_instance_from_env(self, **kwargs):
+        return

mdps_ds_lib/stac_fast_api_client/__init__.py

@@ -0,0 +1,73 @@
+import os
+from abc import ABC, abstractmethod
+
+import requests
+from dotenv import load_dotenv
+
+
+class SFAClientBase(ABC):
+    def __init__(self, ds_url: str, ds_stage: str):
+        load_dotenv()
+        self._trust_env = os.getenv('TRUST_ENV', 'FALSE').upper().strip() == 'TRUE'
+        self._base_url = f'{ds_url}/{ds_stage}'
+
+    @abstractmethod
+    def create_session(self):
+        s = requests.session()
+        s.trust_env = self._trust_env
+        # s.cookies.set('mod_auth_openidc_session', 'xxx-xxx-xxx')
+        # s.auth = HTTPBasicAuth('user', 'pass')
+        # s.headers.update({"Authorization": f"Bearer {token}"})
+        return s
+
+    def _handle_response(self, response):
+        """
+        Raises HTTPError with detailed response text if available.
+        """
+        try:
+            response.raise_for_status()
+        except requests.HTTPError as e:
+            # Attach response text to the exception for easier debugging
+            error_text = response.text.strip()
+            raise requests.HTTPError(
+                f"{e}\nResponse content: {error_text}"
+            ) from None
+        try:
+            return response.json()
+        except:
+            raise requests.HTTPError(f'invalid JSON response: {response.text}')
+
+    def get_collections(self, **params):
+        url = f"{self._base_url}/collections"
+        response = self.create_session().get(url, params=params)
+        return self._handle_response(response)
+
+    def get_collection(self, collection_id, **params) -> dict:  # Return a COllection STAC JSON
+        url = f"{self._base_url}/collections/{collection_id}"
+        response = self.create_session().get(url, params=params)
+        return self._handle_response(response)
+
+    def create_collection(self, collection):
+        url = f"{self._base_url}/collections"
+        response = self.create_session().post(url, json=collection)
+        # NOTE: result if not found: {"code":"NotFoundError","description":"Collection Invalid-Collection not found"}
+        return self._handle_response(response)
+
+    def get_items(self, collection_id, **params):
+        url = f"{self._base_url}/collections/{collection_id}/items"
+        response = self.create_session().get(url, params=params)
+        return self._handle_response(response)
+
+    def create_item(self, collection_id, item):
+        url = f"{self._base_url}/collections/{collection_id}/items"
+        my_session = self.create_session()
+        my_session.headers.update({'Content-Type': 'application/json'})
+        response = self.create_session().post(url, json=item)
+        return self._handle_response(response)
+
+    def update_item(self, collection_id, item_id, item, update_whole=True):
+        url = f"{self._base_url}/collections/{collection_id}/items/{item_id}"
+        my_session = self.create_session()
+        my_session.headers.update({'Content-Type': 'application/json'})
+        response = self.create_session().put(url, json=item) if update_whole else self.create_session().patch(url, json=item)
+        return self._handle_response(response)

mdps_ds_lib/stac_fast_api_client/sfa_client_base.py

@@ -0,0 +1,16 @@
+from requests.auth import HTTPBasicAuth
+
+from mdps_ds_lib.stac_fast_api_client.sfa_client_base import SFAClientBase
+
+
+class SFAClientBasicAuth(SFAClientBase):
+
+    def __init__(self, username: str, password: str, ds_url: str, ds_stage: str):
+        super().__init__(ds_url, ds_stage)
+        self.__user = username
+        self.__pass = password
+
+    def create_session(self):
+        s1 = super().create_session()
+        s1.auth = HTTPBasicAuth(self.__user, self.__pass)
+        return s1

mdps_ds_lib/stac_fast_api_client/sfa_client_basic_auth.py

@@ -0,0 +1,15 @@
+from requests.auth import HTTPBasicAuth
+
+from mdps_ds_lib.stac_fast_api_client.sfa_client_base import SFAClientBase
+
+
+class SFAClientBearer(SFAClientBase):
+
+    def __init__(self, bearer_token: str, ds_url: str, ds_stage: str):
+        super().__init__(ds_url, ds_stage)
+        self.__bearer_token = bearer_token
+
+    def create_session(self):
+        s1 = super().create_session()
+        s1.headers.update({"Authorization": f"Bearer {self.__bearer_token}"})
+        return s1

mdps_ds_lib/stac_fast_api_client/sfa_client_bearer.py

@@ -0,0 +1,16 @@
+from requests.auth import HTTPBasicAuth
+
+from mdps_ds_lib.stac_fast_api_client.sfa_client_base import SFAClientBase
+
+
+class SFAClientCookieToken(SFAClientBase):
+
+    def __init__(self, auth_key: str, auth_value: str, ds_url: str, ds_stage: str):
+        super().__init__(ds_url, ds_stage)
+        self.__auth_key = auth_key
+        self.__auth_value = auth_value
+
+    def create_session(self):
+        s1 = super().create_session()
+        s1.cookies.set(self.__auth_key, self.__auth_value)
+        return s1

mdps_ds_lib/stac_fast_api_client/sfa_client_cookie_token.py

@@ -0,0 +1,57 @@
+from mdps_ds_lib.lib.utils.factory_abstract import FactoryAbstract
+from mdps_ds_lib.stac_fast_api_client.sfa_client_base import SFAClientBase
+import os
+
+
+class SFAClientFactory(FactoryAbstract):
+    NO_AUTH = 'NO_AUTH'
+    BASIC_AUTH = 'BASIC_AUTH'
+    COOKIE_AUTH = 'COOKIE_AUTH'
+    BEARER_AUTH = 'BEARER_AUTH'
+
+    def get_instance_from_env(self, **kwargs) -> SFAClientBase:
+        if 'DS_URL' not in os.environ:
+            raise RuntimeError(f'missing mandatory env: DS_URL')
+        class_env = {'ds_url': os.getenv('DS_URL'),
+                     'ds_stage': '' if 'ds_stage'.upper() not in os.environ else os.getenv('ds_stage'.upper())}
+
+        class_type_env_map = {
+            SFAClientFactory.BASIC_AUTH: {
+                'SFA_USERNAME': 'username',
+                'SFA_PASSWORD': 'password'
+            },
+            SFAClientFactory.COOKIE_AUTH: {
+                'SFA_AUTH_KEY': 'auth_key',
+                'SFA_AUTH_VALUE': 'auth_value'
+            },
+            SFAClientFactory.BEARER_AUTH: {
+                'SFA_BEARER_TOKEN': 'bearer_token',
+            },
+            SFAClientFactory.NO_AUTH: {
+            },
+        }
+        chosen_class = None
+        for k, v in class_type_env_map.items():
+            if all([k1 in os.environ for k1 in list(v.keys())]):
+                for k1, v1 in v.items():
+                    class_env[v1] = os.getenv(k1)
+                chosen_class = k
+                break
+        if chosen_class is None:
+            raise NotImplementedError(f'unknown class type: missing ENVs. require one of {class_type_env_map}')
+        return self.get_instance(chosen_class, **class_env)
+
+    def get_instance(self, class_type, **kwargs) -> SFAClientBase:
+        if class_type == self.NO_AUTH:
+            from mdps_ds_lib.stac_fast_api_client.sfa_client_no_auth import SFAClientNoAuth
+            return SFAClientNoAuth(kwargs['ds_url'], kwargs['ds_stage'])
+        if class_type == self.BASIC_AUTH:
+            from mdps_ds_lib.stac_fast_api_client.sfa_client_basic_auth import SFAClientBasicAuth
+            return SFAClientBasicAuth(kwargs['username'], kwargs['password'], kwargs['ds_url'], kwargs['ds_stage'])
+        if class_type == self.COOKIE_AUTH:
+            from mdps_ds_lib.stac_fast_api_client.sfa_client_cookie_token import SFAClientCookieToken
+            return SFAClientCookieToken(kwargs['auth_key'], kwargs['auth_value'], kwargs['ds_url'], kwargs['ds_stage'])
+        if class_type == self.BEARER_AUTH:
+            from mdps_ds_lib.stac_fast_api_client.sfa_client_bearer import SFAClientBearer
+            return SFAClientBearer(kwargs['bearer_token'], kwargs['ds_url'], kwargs['ds_stage'])
+        raise NotImplementedError(f'unknown class type: {class_type}')