patch for external file retriver changes

Author: mfshao

docs/howto/externalFileDownloading.md

@@ -29,9 +29,9 @@ This is specified in the `external_file_metadata` field. An example is shown bel
 }
 ```
 
-The `'external_oidc_idp'` field is required. It is used to determine how to get a token from the `workspace token service`.
+The `'file_retriever'` field is required. It is used to determine how to retrieve the file.
 
-The `'file_retriever'` field is also required. It is used to determine how to retrieve the file.
+The `'external_oidc_idp'` field is required if retrieving the file(s) requires to get a token from the `workspace token service`. if the file(s) can be retrieved without any tokens, this field could be omitted.
 
 The `'study_id'` and `'file_id'` fields are allowed but are not required.
 

gen3/tools/download/external_file_download.py

@@ -7,15 +7,14 @@
 
 """
 import json
-import os
 from pathlib import Path
 from typing import Any, Dict, List
 
 from cdislogging import get_logger
 
 from gen3.auth import Gen3Auth
 
-REQUIRED_EXTERNAL_FILE_METADATA_KEYS = ["external_oidc_idp", "file_retriever"]
+REQUIRED_EXTERNAL_FILE_METADATA_KEYS = ["file_retriever"]
 
 logger = get_logger("__name__", log_level="debug")
 
@@ -32,21 +31,25 @@ def download_files_from_metadata(
     external-file metadata and a retriever function.
 
     The external_file_metadata items should have the following
-    required keys: 'external_oidc_idp' and 'file_retriever'.
+    required key: 'file_retriever'.
     The items can have an additional optional key of
     'study_id' or 'file_id.
 
     An example is shown below:
     [
         {
-        "external_oidc_idp": "externaldata-keycloak",
-        "file_retriever": "QDR",
-        "study_id": "QDR_study_01"
+            "external_oidc_idp": "externaldata-keycloak",
+            "file_retriever": "QDR",
+            "study_id": "QDR_study_01"
         },
         {
-        "external_oidc_idp": "externaldata-keycloak",
-        "file_retriever": "QDR",
-        "file_id": "QDR_file_02"
+            "external_oidc_idp": "externaldata-keycloak",
+            "file_retriever": "QDR",
+            "file_id": "QDR_file_02"
+        },
+        {
+            "file_retriever": "Dataverse",
+            "file_id": "Dataverse_file_01"
         },
     ]
 
@@ -181,8 +184,7 @@ def extract_external_file_metadata(mds_object: dict) -> List:
 
 def is_valid_external_file_metadata(external_file_metadata: dict) -> bool:
     """
-    Check that the external_file dict has the required keys:
-    'external_oidc_idp' and 'file_retriever'.
+    Check that the external_file dict has the required key(s) as defined in 'REQUIRED_EXTERNAL_FILE_METADATA_KEYS'
 
     Args:
         external_file_metadata: dict

tests/download_tests/test_external_download.py

@@ -41,18 +41,15 @@ def test_load_all_metadata():
 
 
 def test_is_valid_external_file_metadata_invalid_input():
-    expected = False
-    # input is missing one of the required keys, 'external_oidc_idp' or 'file_retriever'
+    # input is missing one of the required key(s) 'file_retriever'
     external_file_metadata = {
         "external_oidc_idp": "test-external-idp",
         "file_id": "QDR_file_02",
     }
-    assert is_valid_external_file_metadata(external_file_metadata) == expected
-    external_file_metadata = {"file_retriever": "QDR", "file_id": "QDR_file_02"}
-    assert is_valid_external_file_metadata(external_file_metadata) == expected
+    assert is_valid_external_file_metadata(external_file_metadata) is False
     # input is not a dict
     external_file_metadata = ["some", "list"]
-    assert is_valid_external_file_metadata(external_file_metadata) == expected
+    assert is_valid_external_file_metadata(external_file_metadata) is False
 
 
 def test_is_valid_external_file_metadata(valid_external_file_metadata):