feat: Allow SNS to set external roles to publish messages (#110)

* feat: add AUDIT log level for upload

* chore: update outdated tests

* fix: allow empty str as RESULT_PATH_PREFIX & replace w/ default val

* fix: allowing optional original stac item

* feat: sns accept external role option

Author: wphyojpl

mdps_ds_lib/ds_client/ds_client_user.py

@@ -256,3 +256,21 @@ def delete_single_granule(self):
         response.raise_for_status()
         response = json.loads(response.text)
         return response
+
+    def archive_granule(self):
+        if self.tenant is None or self.tenant_venue is None or self.collection is None or self.granule is None:
+            raise ValueError(f'require to set tenant & tenant_venue & collection & granule')
+        collection_id_for_granules = ':'.join([self.urn, self.org, self.project, self.tenant, self.tenant_venue, self.get_complete_collection()])
+        granule_id_complete = ':'.join([collection_id_for_granules, self.granule])
+        request_url = f'{self._uds_url}collections/'
+        request_url = f'{request_url}{collection_id_for_granules}/archive/{granule_id_complete}/'
+        print(request_url)
+        s = requests.session()
+        s.trust_env = self._trust_env
+        response = s.put(url=request_url, headers={
+            'Authorization': f'Bearer {self._token_retriever.get_token()}',
+        }, verify=self._trust_env)
+        response.raise_for_status()
+        response = json.loads(response.text)
+        return response
+

mdps_ds_lib/lib/aws/aws_sns.py

@@ -5,16 +5,38 @@ class AwsSns(AwsCred):
     def __init__(self):
         super().__init__()
         self.__sns_client = self.get_client('sns')
+        self.__special_sns_client = None
         self.__topic_arn = ''
 
     def set_topic_arn(self, topic_arn):
         self.__topic_arn = topic_arn
         return self
 
-    def publish_message(self, msg_str: str):
+    def set_external_role(self, external_role_arn: str, external_role_session_name: str, external_role_duration: int =900):
+        sts_client = self.get_client('sts')
+        assumed_role = sts_client.assume_role(
+            RoleArn=external_role_arn,
+            RoleSessionName=external_role_session_name,
+            DurationSeconds=external_role_duration  # 12 hours max
+        )
+
+        credentials = assumed_role['Credentials']
+
+        self.__special_sns_client = self.get_session().client(
+            "sns",
+            aws_access_key_id=credentials['AccessKeyId'],
+            aws_secret_access_key=credentials['SecretAccessKey'],
+            aws_session_token=credentials['SessionToken'],
+        )
+        return self
+
+    def publish_message(self, msg_str: str, is_with_daac_role: bool=False):
         if self.__topic_arn == '':
             raise ValueError('missing topic arn to publish message')
-        response = self.__sns_client.publish(
+        if is_with_daac_role and self.__special_sns_client is None:
+            raise ValueError('sns client with external role NOT set')
+        my_sns = self.__special_sns_client if is_with_daac_role else self.__sns_client
+        response = my_sns.publish(
             TopicArn=self.__topic_arn,
             # TargetArn='string',  # not needed coz of we are using topic arn
             # PhoneNumber='string',  # not needed coz of we are using topic arn

tests/mdps_ds_lib/ds_client/test_ds_client_unity.py

@@ -2,13 +2,18 @@
 import os
 from unittest import TestCase
 
+from dotenv import load_dotenv
+
 from mdps_ds_lib.ds_client.auth_token.token_abstract import TokenAbstract
 from mdps_ds_lib.ds_client.auth_token.token_factory import TokenFactory
 from mdps_ds_lib.ds_client.ds_client_admin import DsClientAdmin
 from mdps_ds_lib.ds_client.ds_client_user import DsClientUser
 
 
 class TestDsClientAdmin(TestCase):
+    def setUp(self) -> None:
+        super().setUp()
+        load_dotenv()
     def test_01_admin(self):
         os.environ['TRUST_ENV'] = 'TRUE'
         os.environ['PASSWORD_TYPE'] = 'PARAM_STORE'
@@ -33,12 +38,6 @@ def test_01_admin(self):
 
     def test_query_granules_across_collections(self):
         os.environ['TRUST_ENV'] = 'TRUE'
-        os.environ['PASSWORD_TYPE'] = 'PARAM_STORE'
-        os.environ['USERNAME'] = '/unity/uds/user/wphyo/username'
-        os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap'
-        os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c'
-        os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com'
-
         os.environ['TOKEN_FACTORY'] = 'COGNITO'
         token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY'))
         client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx')
@@ -55,12 +54,6 @@ def test_query_granules_across_collections(self):
 
     def test_query_collections(self):
         os.environ['TRUST_ENV'] = 'TRUE'
-        os.environ['PASSWORD_TYPE'] = 'PARAM_STORE'
-        os.environ['USERNAME'] = '/unity/uds/user/wphyo/username'
-        os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap'
-        os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c'
-        os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com'
-
         os.environ['TOKEN_FACTORY'] = 'COGNITO'
         token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY'))
         client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx')
@@ -70,12 +63,6 @@ def test_query_collections(self):
 
     def test_query_single_collection(self):
         os.environ['TRUST_ENV'] = 'TRUE'
-        os.environ['PASSWORD_TYPE'] = 'PARAM_STORE'
-        os.environ['USERNAME'] = '/unity/uds/user/wphyo/username'
-        os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap'
-        os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c'
-        os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com'
-
         os.environ['TOKEN_FACTORY'] = 'COGNITO'
         token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY'))
         client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx')
@@ -92,12 +79,6 @@ def test_query_single_collection(self):
 
     def test_query_granules(self):
         os.environ['TRUST_ENV'] = 'TRUE'
-        os.environ['PASSWORD_TYPE'] = 'PARAM_STORE'
-        os.environ['USERNAME'] = '/unity/uds/user/wphyo/username'
-        os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap'
-        os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c'
-        os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com'
-
         os.environ['TOKEN_FACTORY'] = 'COGNITO'
         token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY'))
         client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx')
@@ -117,12 +98,6 @@ def test_query_granules(self):
 
     def test_query_granules02(self):
         os.environ['TRUST_ENV'] = 'TRUE'
-        os.environ['PASSWORD_TYPE'] = 'PARAM_STORE'
-        os.environ['USERNAME'] = '/unity/uds/user/wphyo/username'
-        os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap'
-        os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c'
-        os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com'
-
         os.environ['TOKEN_FACTORY'] = 'COGNITO'
         token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY'))
         client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'am-uds-dapa')
@@ -151,12 +126,6 @@ def test_query_granules02(self):
 
     def test_query_custom_properties(self):
         os.environ['TRUST_ENV'] = 'TRUE'
-        os.environ['PASSWORD_TYPE'] = 'PARAM_STORE'
-        os.environ['USERNAME'] = '/unity/uds/user/wphyo/username'
-        os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap'
-        os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c'
-        os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com'
-
         os.environ['TOKEN_FACTORY'] = 'COGNITO'
         token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY'))
         client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'am-uds-dapa')
@@ -178,12 +147,6 @@ def test_query_custom_properties(self):
 
     def test_query_single_granule(self):
         os.environ['TRUST_ENV'] = 'TRUE'
-        os.environ['PASSWORD_TYPE'] = 'PARAM_STORE'
-        os.environ['USERNAME'] = '/unity/uds/user/wphyo/username'
-        os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap'
-        os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c'
-        os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com'
-
         os.environ['TOKEN_FACTORY'] = 'COGNITO'
         token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY'))
         client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'am-uds-dapa')
@@ -202,12 +165,6 @@ def test_query_single_granule(self):
 
     def test_update_admin(self):
         os.environ['TRUST_ENV'] = 'TRUE'
-        os.environ['PASSWORD_TYPE'] = 'PARAM_STORE'
-        os.environ['USERNAME'] = '/unity/uds/user/wphyo/username'
-        os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap'
-        os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c'
-        os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com'
-
         os.environ['TOKEN_FACTORY'] = 'COGNITO'
         token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY'))
         client = DsClientAdmin(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx')
@@ -225,12 +182,6 @@ def test_update_admin(self):
 
     def test_delete_single_granule(self):
         os.environ['TRUST_ENV'] = 'TRUE'
-        os.environ['PASSWORD_TYPE'] = 'PARAM_STORE'
-        os.environ['USERNAME'] = '/unity/uds/user/wphyo/username'
-        os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap'
-        os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c'
-        os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com'
-
         os.environ['TOKEN_FACTORY'] = 'COGNITO'
         token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY'))
         client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx')
@@ -243,21 +194,53 @@ def test_delete_single_granule(self):
         client.tenant_venue = 'DEV'
         client.collection = 'DDD-01'
         client.collection_venue = '001'
-        client.granule = 'test_file10'
         # urn:nasa:unity:uds_local_test:DEV1:CHRP_16_DAY_REBIN___10:SNDR.SS1330.CHIRP.20230101T0000.m06.g001.L1_J1.std.v02_48.G.200101070318_REBIN
         print(client.delete_single_granule())
         return
 
     def test_query_catalog(self):
         os.environ['TRUST_ENV'] = 'TRUE'
-        os.environ['PASSWORD_TYPE'] = 'PARAM_STORE'
-        os.environ['USERNAME'] = '/unity/uds/user/wphyo/username'
-        os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap'
-        os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c'
-        os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com'
-
         os.environ['TOKEN_FACTORY'] = 'COGNITO'
         token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY'))
         client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx')  # data-sbx'
         print(json.dumps(client.query_catalog(), indent=4))
         return
+
+    def test_query_granules01(self):
+        os.environ['TRUST_ENV'] = 'TRUE'
+        # https://api.test.mdps.mcp.nasa.gov/am-uds-dapa/collections/URN:NASA:UNITY:unity:test:TRPSDL2ALLCRS1MGLOS___2/items
+        os.environ['TOKEN_FACTORY'] = 'COGNITO'
+        token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY'))
+        client = DsClientUser(token_retriever, 'https://api.test.mdps.mcp.nasa.gov', 'am-uds-dapa')
+        # client = DsClientAdmin(token_retriever, 'http://localhost:8005', 'data')
+
+        client.urn = 'URN'
+        client.org = 'NASA'
+        client.project = 'UNITY'
+        client.tenant = 'unity'
+        client.tenant_venue = 'test'
+        client.collection = 'TRPSDL2ALLCRS1MGLOS'
+        client.collection_venue = '2'
+        result = client.query_granules(sort_keys='+properties.datetime,-id')  # bbox='-114,32.5,-113,33.5'
+        print(json.dumps(result, indent=4))
+        return
+
+    def test_archive_one(self):
+        os.environ['TRUST_ENV'] = 'TRUE'
+        # https://api.test.mdps.mcp.nasa.gov/am-uds-dapa/collections/URN:NASA:UNITY:unity:test:TRPSDL2ALLCRS1MGLOS___2/items
+        os.environ['TOKEN_FACTORY'] = 'COGNITO'
+        token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY'))
+        client = DsClientUser(token_retriever, 'https://api.test.mdps.mcp.nasa.gov', 'am-uds-dapa')
+        # client = DsClientAdmin(token_retriever, 'http://localhost:8005', 'data')
+
+        client.urn = 'URN'
+        client.org = 'NASA'
+        client.project = 'UNITY'
+        client.tenant = 'unity'
+        client.tenant_venue = 'test'
+        client.collection = 'TRPSDL2ALLCRS1MGLOS'
+        client.collection_venue = '2'
+        client.granule = 'TROPESS_CrIS-JPSS1_L2_Standard_NH3_20250108_MUSES_R1p23_megacity_los_angeles_MGLOS_F2p5_J0'
+
+        print(client.archive_granule())
+        return