fix: case insensitivity (#106)

* feat: add AUDIT log level for upload

* chore: update outdated tests

* fix: allow empty str as RESULT_PATH_PREFIX & replace w/ default val

* fix: allowing optional original stac item

* feat: add catalog endpoint + manual test for case insensitive

* fix: ancillary upload needs valid bbox

* fix: ancillary upload bugfix

* fix: <URN>:<ORG>:<Project>:<Tenant>:<Venue> are always upper case

* fix: update tests

Author: wphyojpl

mdps_ds_lib/ds_client/ds_client_user.py

@@ -93,6 +93,17 @@ def query_collections(self, limit=10):
 
         return response
 
+    def query_catalog(self):
+        request_url = f'{self._uds_url}catalog/'
+        s = requests.session()
+        s.trust_env = self._trust_env
+        response = s.get(url=request_url, headers={
+            'Authorization': f'Bearer {self._token_retriever.get_token()}',
+        }, verify=self._trust_env)
+        response.raise_for_status()
+        response = json.loads(response.text)
+        return response
+
     def query_collections_next(self):
         if self.__collection_query_next_page is None:
             return None

mdps_ds_lib/lib/cumulus_stac/granules_catalog.py

@@ -9,20 +9,31 @@
 
 
 class GranulesCatalog:
+
+    @staticmethod
+    def standardize_stage_out_collection_id_format(current_collection_id: str):
+        collection_identifier_parts = current_collection_id.split(':')
+        if len(collection_identifier_parts) < 6:
+            raise ValueError(
+                f'invalid collection ID. Need to be in <URN>:<ORG>:<Project>:<Tenant>:<Venue>:<Collection ID> but it is {current_collection_id}')
+        for i in range(5):
+            collection_identifier_parts[i] = collection_identifier_parts[i].upper()
+        current_collection_id = ':'.join(collection_identifier_parts)
+        current_collection_id = f'{current_collection_id}___001' if '___' not in current_collection_id else current_collection_id
+        return current_collection_id
+
     @staticmethod
     def get_unity_formatted_collection_id(current_collection_id: str, project_venue_set: tuple):
         if current_collection_id == '' or current_collection_id is None:
             raise ValueError(f'NULL or EMPTY collection_id: {current_collection_id}')
         collection_identifier_parts = current_collection_id.split(':')
         if len(collection_identifier_parts) >= 6:
             LOGGER.debug(f'current_collection_id is assumed to be in UNITY format: {current_collection_id}')
-            current_collection_id = f'{current_collection_id}___001' if '___' not in current_collection_id else current_collection_id
-            return current_collection_id
-
+            return GranulesCatalog.standardize_stage_out_collection_id_format(current_collection_id)
         LOGGER.info(f'current_collection_id is not UNITY formatted ID: {current_collection_id}')
         if project_venue_set[0] is None or project_venue_set[1] is None:
             raise ValueError(f'missing project or venue in ENV which is needed due to current_collection_id not UNITY format: {project_venue_set}')
-        new_collection = f'URN:NASA:UNITY:{project_venue_set[0]}:{project_venue_set[1]}:{current_collection_id}'
+        new_collection = f'URN:NASA:UNITY:{project_venue_set[0].upper()}:{project_venue_set[1].upper()}:{current_collection_id}'
         new_collection = f'{new_collection}___001' if '___' not in new_collection else new_collection
         LOGGER.info(f'UNITY formatted ID: {new_collection}')
         return new_collection

mdps_ds_lib/stage_in_out/upload_arbitrary_files_as_granules.py

@@ -51,7 +51,7 @@ def generate_sample_stac(self, filepath: str):
                              "type": "Point",
                              "coordinates": [0.0, 0.0]
                          },
-                         bbox=[0.0, 0.0, 0.0, 0.0],
+                         bbox=[-180, -90, 180, 90],
                          datetime=TimeUtils().parse_from_unix(0, True).get_datetime_obj(),
                          properties={
                              "start_datetime": TimeUtils.get_current_time(),
@@ -84,10 +84,11 @@ def execute_job(self, job_obj, lock) -> bool:
             s3_url = self.__s3.upload(job_obj, self.__staging_bucket, f'{self.__collection_id}/{sample_stac_item.id}', self.__delete_files)
             updating_assets[os.path.basename(s3_url)] = s3_url
             uploading_current_granule_stac = f'{s3_url}.stac.json'
-            self.__s3.set_s3_url(uploading_current_granule_stac)
-            self.__s3.upload_bytes(json.dumps(sample_stac_item.to_dict(False, False),indent=4).encode())
             updating_assets[os.path.basename(uploading_current_granule_stac)] = uploading_current_granule_stac
             self.__gc.update_assets_href(sample_stac_item, updating_assets)
+
+            self.__s3.set_s3_url(uploading_current_granule_stac)
+            self.__s3.upload_bytes(json.dumps(sample_stac_item.to_dict(False, False),indent=4).encode())
             self.__result_list.put(sample_stac_item.to_dict(False, False))
         except Exception as e:
             sample_stac_item.properties['upload_error'] = str(e)
@@ -116,6 +117,7 @@ def upload(self, **kwargs) -> str:
         self._set_props_from_env()
         if self._collection_id is None:
             raise ValueError(f'missing COLLECTION ID in ENV')
+        self._collection_id = GranulesCatalog.standardize_stage_out_collection_id_format(self._collection_id)
         output_dir = os.environ.get(self.OUTPUT_DIRECTORY)
         if not FileUtils.dir_exist(output_dir):
             raise ValueError(f'OUTPUT_DIRECTORY: {output_dir} does not exist')