Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
b0801de
feat: add AUDIT log level for upload
wphyojpl Oct 14, 2024
00d6bde
chore: update outdated tests
wphyojpl Oct 14, 2024
327ef8d
Merge branch 'develop' of github.com:unity-sds/mdps-ds-lib into develop
wphyojpl Oct 21, 2024
6fd6910
chore: merge from develop
wphyojpl Oct 21, 2024
f77e47e
Merge branch 'develop' of github.com:unity-sds/mdps-ds-lib into develop
wphyojpl Oct 21, 2024
a51314b
fix: allow empty str as RESULT_PATH_PREFIX & replace w/ default val
wphyojpl Oct 21, 2024
2d72378
Merge branch 'develop' of github.com:unity-sds/mdps-ds-lib into develop
wphyojpl Oct 28, 2024
0c31b5f
Merge branch 'develop' of github.community-sds/mdps-ds-lib into develop
wphyojpl Oct 28, 2024
5701e60
Merge branch 'develop' of github.com:unity-sds/mdps-ds-lib into develop
wphyojpl Nov 7, 2024
b429a12
erge branch 'develop' of github.com:unity-sds/mdps-ds-lib into develop
wphyojpl Nov 7, 2024
87e82ae
Merge branch 'develop' of github.com:unity-sds/mdps-ds-lib into develop
wphyojpl Nov 26, 2024
b730339
Merge branch 'develop' of github.com:unity-sds/mdps-ds-lib into develop
wphyojpl Nov 26, 2024
a6d89f6
Merge branch 'develop' of github.com:unity-sds/mdps-ds-lib into develop
wphyojpl Dec 4, 2024
87a2858
Merge branch 'develop' of github.com:unity-sds/mdps-ds-lib into develop
wphyojpl Dec 9, 2024
efb0d8c
Merge branch 'develop' of github.com:unity-sds/mdps-ds-lib into develop
wphyojpl Dec 9, 2024
b908552
Merge branch 'develop' of github.com:unity-sds/mdps-ds-lib into develop
wphyojpl Dec 16, 2024
b66e867
Merge branch 'develop' of github.com:unity-sds/mdps-ds-lib into develop
wphyojpl Dec 16, 2024
51c9a9e
Merge branch 'develop' of github.com:unity-sds/mdps-ds-lib into develop
wphyojpl Dec 16, 2024
1ca94c8
Merge branch 'develop' of github.com:unity-sds/mdps-ds-lib into develop
wphyojpl Jan 8, 2025
9b6dc8d
Merge branch 'develop' of github.com:unity-sds/mdps-ds-lib into develop
wphyojpl Jan 9, 2025
919b679
Merge branch 'develop' of github.com:unity-sds/mdps-ds-lib into develop
wphyojpl Feb 10, 2025
6d95543
Merge branch 'develop' of github.com:unity-sds/mdps-ds-lib into develop
wphyojpl Feb 12, 2025
44697a5
Merge branch 'develop' of github.com:unity-sds/mdps-ds-lib into develop
wphyojpl Mar 18, 2025
75a0436
Merge branch 'develop' of github.com:unity-sds/mdps-ds-lib into develop
wphyojpl Mar 24, 2025
9e35099
Merge branch 'develop' of github.com:unity-sds/mdps-ds-lib into develop
wphyojpl Mar 24, 2025
494505d
fix: allowing optional original stac item
wphyojpl Apr 7, 2025
d8a0ff4
Merge branch 'develop' of github.com:unity-sds/mdps-ds-lib into develop
wphyojpl Apr 7, 2025
65be24b
Merge branch 'develop' of github.com:unity-sds/mdps-ds-lib into develop
wphyojpl Apr 7, 2025
8610202
Merge branch 'develop' of github.com:unity-sds/mdps-ds-lib into develop
wphyojpl Apr 22, 2025
a314344
Merge branch 'develop' of github.com:unity-sds/mdps-ds-lib into develop
wphyojpl Apr 29, 2025
9588b56
Merge branch 'develop' of github.com:unity-sds/mdps-ds-lib into develop
wphyojpl May 19, 2025
30d0272
feat: sns accept external role option
wphyojpl May 19, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions mdps_ds_lib/ds_client/ds_client_user.py
Original file line number Diff line number Diff line change
Expand Up @@ -256,3 +256,21 @@ def delete_single_granule(self):
response.raise_for_status()
response = json.loads(response.text)
return response

def archive_granule(self):
if self.tenant is None or self.tenant_venue is None or self.collection is None or self.granule is None:
raise ValueError(f'require to set tenant & tenant_venue & collection & granule')
collection_id_for_granules = ':'.join([self.urn, self.org, self.project, self.tenant, self.tenant_venue, self.get_complete_collection()])
granule_id_complete = ':'.join([collection_id_for_granules, self.granule])
request_url = f'{self._uds_url}collections/'
request_url = f'{request_url}{collection_id_for_granules}/archive/{granule_id_complete}/'
print(request_url)
s = requests.session()
s.trust_env = self._trust_env
response = s.put(url=request_url, headers={
'Authorization': f'Bearer {self._token_retriever.get_token()}',
}, verify=self._trust_env)
response.raise_for_status()
response = json.loads(response.text)
return response

26 changes: 24 additions & 2 deletions mdps_ds_lib/lib/aws/aws_sns.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,38 @@ class AwsSns(AwsCred):
def __init__(self):
super().__init__()
self.__sns_client = self.get_client('sns')
self.__special_sns_client = None
self.__topic_arn = ''

def set_topic_arn(self, topic_arn):
self.__topic_arn = topic_arn
return self

def publish_message(self, msg_str: str):
def set_external_role(self, external_role_arn: str, external_role_session_name: str, external_role_duration: int =900):
sts_client = self.get_client('sts')
assumed_role = sts_client.assume_role(
RoleArn=external_role_arn,
RoleSessionName=external_role_session_name,
DurationSeconds=external_role_duration # 12 hours max
)

credentials = assumed_role['Credentials']

self.__special_sns_client = self.get_session().client(
"sns",
aws_access_key_id=credentials['AccessKeyId'],
aws_secret_access_key=credentials['SecretAccessKey'],
aws_session_token=credentials['SessionToken'],
)
return self

def publish_message(self, msg_str: str, is_with_daac_role: bool=False):
if self.__topic_arn == '':
raise ValueError('missing topic arn to publish message')
response = self.__sns_client.publish(
if is_with_daac_role and self.__special_sns_client is None:
raise ValueError('sns client with external role NOT set')
my_sns = self.__special_sns_client if is_with_daac_role else self.__sns_client
response = my_sns.publish(
TopicArn=self.__topic_arn,
# TargetArn='string', # not needed coz of we are using topic arn
# PhoneNumber='string', # not needed coz of we are using topic arn
Expand Down
105 changes: 44 additions & 61 deletions tests/mdps_ds_lib/ds_client/test_ds_client_unity.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,18 @@
import os
from unittest import TestCase

from dotenv import load_dotenv

from mdps_ds_lib.ds_client.auth_token.token_abstract import TokenAbstract
from mdps_ds_lib.ds_client.auth_token.token_factory import TokenFactory
from mdps_ds_lib.ds_client.ds_client_admin import DsClientAdmin
from mdps_ds_lib.ds_client.ds_client_user import DsClientUser


class TestDsClientAdmin(TestCase):
def setUp(self) -> None:
super().setUp()
load_dotenv()
def test_01_admin(self):
os.environ['TRUST_ENV'] = 'TRUE'
os.environ['PASSWORD_TYPE'] = 'PARAM_STORE'
Expand All @@ -33,12 +38,6 @@ def test_01_admin(self):

def test_query_granules_across_collections(self):
os.environ['TRUST_ENV'] = 'TRUE'
os.environ['PASSWORD_TYPE'] = 'PARAM_STORE'
os.environ['USERNAME'] = '/unity/uds/user/wphyo/username'
os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap'
os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c'
os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com'

os.environ['TOKEN_FACTORY'] = 'COGNITO'
token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY'))
client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx')
Expand All @@ -55,12 +54,6 @@ def test_query_granules_across_collections(self):

def test_query_collections(self):
os.environ['TRUST_ENV'] = 'TRUE'
os.environ['PASSWORD_TYPE'] = 'PARAM_STORE'
os.environ['USERNAME'] = '/unity/uds/user/wphyo/username'
os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap'
os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c'
os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com'

os.environ['TOKEN_FACTORY'] = 'COGNITO'
token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY'))
client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx')
Expand All @@ -70,12 +63,6 @@ def test_query_collections(self):

def test_query_single_collection(self):
os.environ['TRUST_ENV'] = 'TRUE'
os.environ['PASSWORD_TYPE'] = 'PARAM_STORE'
os.environ['USERNAME'] = '/unity/uds/user/wphyo/username'
os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap'
os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c'
os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com'

os.environ['TOKEN_FACTORY'] = 'COGNITO'
token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY'))
client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx')
Expand All @@ -92,12 +79,6 @@ def test_query_single_collection(self):

def test_query_granules(self):
os.environ['TRUST_ENV'] = 'TRUE'
os.environ['PASSWORD_TYPE'] = 'PARAM_STORE'
os.environ['USERNAME'] = '/unity/uds/user/wphyo/username'
os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap'
os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c'
os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com'

os.environ['TOKEN_FACTORY'] = 'COGNITO'
token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY'))
client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx')
Expand All @@ -117,12 +98,6 @@ def test_query_granules(self):

def test_query_granules02(self):
os.environ['TRUST_ENV'] = 'TRUE'
os.environ['PASSWORD_TYPE'] = 'PARAM_STORE'
os.environ['USERNAME'] = '/unity/uds/user/wphyo/username'
os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap'
os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c'
os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com'

os.environ['TOKEN_FACTORY'] = 'COGNITO'
token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY'))
client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'am-uds-dapa')
Expand Down Expand Up @@ -151,12 +126,6 @@ def test_query_granules02(self):

def test_query_custom_properties(self):
os.environ['TRUST_ENV'] = 'TRUE'
os.environ['PASSWORD_TYPE'] = 'PARAM_STORE'
os.environ['USERNAME'] = '/unity/uds/user/wphyo/username'
os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap'
os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c'
os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com'

os.environ['TOKEN_FACTORY'] = 'COGNITO'
token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY'))
client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'am-uds-dapa')
Expand All @@ -178,12 +147,6 @@ def test_query_custom_properties(self):

def test_query_single_granule(self):
os.environ['TRUST_ENV'] = 'TRUE'
os.environ['PASSWORD_TYPE'] = 'PARAM_STORE'
os.environ['USERNAME'] = '/unity/uds/user/wphyo/username'
os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap'
os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c'
os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com'

os.environ['TOKEN_FACTORY'] = 'COGNITO'
token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY'))
client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'am-uds-dapa')
Expand All @@ -202,12 +165,6 @@ def test_query_single_granule(self):

def test_update_admin(self):
os.environ['TRUST_ENV'] = 'TRUE'
os.environ['PASSWORD_TYPE'] = 'PARAM_STORE'
os.environ['USERNAME'] = '/unity/uds/user/wphyo/username'
os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap'
os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c'
os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com'

os.environ['TOKEN_FACTORY'] = 'COGNITO'
token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY'))
client = DsClientAdmin(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx')
Expand All @@ -225,12 +182,6 @@ def test_update_admin(self):

def test_delete_single_granule(self):
os.environ['TRUST_ENV'] = 'TRUE'
os.environ['PASSWORD_TYPE'] = 'PARAM_STORE'
os.environ['USERNAME'] = '/unity/uds/user/wphyo/username'
os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap'
os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c'
os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com'

os.environ['TOKEN_FACTORY'] = 'COGNITO'
token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY'))
client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx')
Expand All @@ -243,21 +194,53 @@ def test_delete_single_granule(self):
client.tenant_venue = 'DEV'
client.collection = 'DDD-01'
client.collection_venue = '001'
client.granule = 'test_file10'
# urn:nasa:unity:uds_local_test:DEV1:CHRP_16_DAY_REBIN___10:SNDR.SS1330.CHIRP.20230101T0000.m06.g001.L1_J1.std.v02_48.G.200101070318_REBIN
print(client.delete_single_granule())
return

def test_query_catalog(self):
os.environ['TRUST_ENV'] = 'TRUE'
os.environ['PASSWORD_TYPE'] = 'PARAM_STORE'
os.environ['USERNAME'] = '/unity/uds/user/wphyo/username'
os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap'
os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c'
os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com'

os.environ['TOKEN_FACTORY'] = 'COGNITO'
token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY'))
client = DsClientUser(token_retriever, 'https://d3vc8w9zcq658.cloudfront.net', 'data-sbx') # data-sbx'
print(json.dumps(client.query_catalog(), indent=4))
return

def test_query_granules01(self):
os.environ['TRUST_ENV'] = 'TRUE'
# https://api.test.mdps.mcp.nasa.gov/am-uds-dapa/collections/URN:NASA:UNITY:unity:test:TRPSDL2ALLCRS1MGLOS___2/items
os.environ['TOKEN_FACTORY'] = 'COGNITO'
token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY'))
client = DsClientUser(token_retriever, 'https://api.test.mdps.mcp.nasa.gov', 'am-uds-dapa')
# client = DsClientAdmin(token_retriever, 'http://localhost:8005', 'data')

client.urn = 'URN'
client.org = 'NASA'
client.project = 'UNITY'
client.tenant = 'unity'
client.tenant_venue = 'test'
client.collection = 'TRPSDL2ALLCRS1MGLOS'
client.collection_venue = '2'
result = client.query_granules(sort_keys='+properties.datetime,-id') # bbox='-114,32.5,-113,33.5'
print(json.dumps(result, indent=4))
return

def test_archive_one(self):
os.environ['TRUST_ENV'] = 'TRUE'
# https://api.test.mdps.mcp.nasa.gov/am-uds-dapa/collections/URN:NASA:UNITY:unity:test:TRPSDL2ALLCRS1MGLOS___2/items
os.environ['TOKEN_FACTORY'] = 'COGNITO'
token_retriever: TokenAbstract = TokenFactory().get_instance(os.getenv('TOKEN_FACTORY'))
client = DsClientUser(token_retriever, 'https://api.test.mdps.mcp.nasa.gov', 'am-uds-dapa')
# client = DsClientAdmin(token_retriever, 'http://localhost:8005', 'data')

client.urn = 'URN'
client.org = 'NASA'
client.project = 'UNITY'
client.tenant = 'unity'
client.tenant_venue = 'test'
client.collection = 'TRPSDL2ALLCRS1MGLOS'
client.collection_venue = '2'
client.granule = 'TROPESS_CrIS-JPSS1_L2_Standard_NH3_20250108_MUSES_R1p23_megacity_los_angeles_MGLOS_F2p5_J0'

print(client.archive_granule())
return