Merge branch 'develop' into feature/list-data-file

Author: rtapella

jupyter-notebooks/tutorials/3_working_with_jobs.ipynb

@@ -21,7 +21,7 @@
     "tags": []
    },
    "source": [
-    "## 0. Set up imports and predefined variables\n",
+    "## 0. Set up imports, predefined variables, and authentication\n",
     "\n",
     "For this Tutorial we will make use of a wrapper API that is called api.py (in the utils folder). It helps to manage things like authentication."
    ]
@@ -34,13 +34,108 @@
    "outputs": [],
    "source": [
     "from utils import api\n",
+    "import getpass\n",
     "import json\n",
     "from IPython.display import JSON\n",
     "import requests\n",
     "import time\n",
     "from datetime import datetime\n",
     "\n",
-    "app_name = \"job-l1b-cwl:develop\""
+    "app_name = \"job-l1b-cwl:develop\"\n",
+    "\n",
+    "# This portion of the code is env specific for Dev, Test, Ops, etc. \n",
+    "# define the environment as our test venue\n",
+    "env = {\n",
+    "    \"clientId\":\"71894molftjtie4dvvkbjeard0\",\n",
+    "}\n",
+    "\n",
+    "# The auth_json is template for authorizing with AWS Cognito for a token that can be used for calls to the data service.\n",
+    "# For now this is just an empty data structure. You will be prompted for your username and password in a few steps.\n",
+    "auth_json = '''{\n",
+    "     \"AuthParameters\" : {\n",
+    "        \"USERNAME\" : \"\",\n",
+    "        \"PASSWORD\" : \"\"\n",
+    "     },\n",
+    "     \"AuthFlow\" : \"USER_PASSWORD_AUTH\",\n",
+    "     \"ClientId\" : \"\"\n",
+    "  }'''"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "814a3678-dde6-44d9-ad94-5021b5813eb7",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "### Authentication Code\n",
+    "\n",
+    "The below method is a helper function for getting an access token for accessing Unity SDS services. You must pass the token along with any API requests in order to access the various Unity SDS services."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dcd5e87b-8f36-4985-977a-e6b4bc3173b5",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# This method is used for taking a username and password and client ID and fetching a cognito token\n",
+    "def get_token(username, password, clientID):\n",
+    "    aj = json.loads(auth_json)\n",
+    "    aj['AuthParameters']['USERNAME'] = username\n",
+    "    aj['AuthParameters']['PASSWORD'] = password\n",
+    "    aj['ClientId'] =clientID \n",
+    "    token = None\n",
+    "    try:\n",
+    "        response = requests.post('https://cognito-idp.us-west-2.amazonaws.com', headers={\"Content-Type\":\"application/x-amz-json-1.1\", \"X-Amz-Target\":\"AWSCognitoIdentityProviderService.InitiateAuth\"}, json=aj)\n",
+    "        token = response.json()['AuthenticationResult']['AccessToken']\n",
+    "    except:\n",
+    "        print(\"Error, check username and password and try again.\")\n",
+    "    return token"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "78549288-db6c-454c-a1dc-92d1a098b601",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "### Prompt for your Unity username and password\n",
+    "\n",
+    "These are required to get the token (described above) to connect to the data services."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1f69d59c-65a9-4014-924e-255fd8ff158d",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "print(\"Please enter your username...\")\n",
+    "user_name = input()\n",
+    "\n",
+    "print(\"Please enter your password...\")\n",
+    "password = getpass.getpass()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a142588e-5710-401a-bd62-b46dd109212b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "token = get_token(user_name, password, env['clientId'])\n",
+    "\n",
+    "if(token):\n",
+    "    print(\"Token received.\")"
    ]
   },
   {
@@ -62,7 +157,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "apps = api.get_apps()\n",
+    "apps = api.get_apps(token)\n",
     "for application in apps:\n",
     "    print(\"Process ID: {}\".format(application['id']))\n",
     "    print(\"Process Version: {}\".format(application['processVersion']))\n",
@@ -112,7 +207,7 @@
     "try:\n",
     "\n",
     "    # Store Job ID to use in future steps\n",
-    "    job_id = api.submit_job(app_name, data)\n",
+    "    job_id = api.submit_job(token, app_name, data)\n",
     "\n",
     "    # If the job submission is successful, print a success message along with the returned JOB-ID\n",
     "    print(\"\\nJob Submission Successful!\\nJOB ID: {}\\n\".format(job_id))\n",
@@ -145,12 +240,12 @@
     "\n",
     "try:\n",
     "    \n",
-    "    job_status = api.get_job_status(app_name, job_id)\n",
+    "    job_status = api.get_job_status(token, app_name, job_id)\n",
     "\n",
     "    while job_status == \"running\":\n",
     "        print(\"Status for job \\\"{}\\\" ({}): {}\".format(job_id, datetime.now().strftime(\"%H:%M:%S\"), job_status))\n",
     "        time.sleep(5)\n",
-    "        job_status = api.get_job_status(app_name, job_id)\n",
+    "        job_status = api.get_job_status(token, app_name, job_id)\n",
     "    \n",
     "    # Print the job status\n",
     "    print(\"\\nStatus for job \\\"{}\\\" ({}): {}\\n\".format(job_id, datetime.now().strftime(\"%H:%M:%S\"),job_status))\n",
@@ -183,7 +278,7 @@
     "\n",
     "try:\n",
     "\n",
-    "    job_data = api.get_job_by_id(app_name, job_id)\n",
+    "    job_data = api.get_job_by_id(token, app_name, job_id)\n",
     "\n",
     "    #for data in job_data:\n",
     "    #    print(\"ID: \" + data['id'])\n",
@@ -224,7 +319,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "jobs = api.get_jobs_for_app(app_name)\n",
+    "jobs = api.get_jobs_for_app(token, app_name)\n",
     "\n",
     "print(\"Jobs running for Process: \", app_name, \"\\n\")\n",
     "\n",

jupyter-notebooks/utils/api.py

@@ -2,11 +2,12 @@
 
 from . import settings
 
-def get_apps():
+def get_apps(token):
 
     try:
-        
-        response = requests.get(settings.WPST_API_DOMAIN + "/processes")
+        headers = _get_headers(token)
+        response = requests.get(settings.WPST_API_DOMAIN + "/processes", headers=headers)
+        response.raise_for_status()
         json_result = response.json()['processes']
 
     except requests.exceptions.HTTPError as e:
@@ -15,11 +16,12 @@ def get_apps():
 
     return json_result
 
-def get_application(app_name):
+def get_application(token, app_name):
 
     try:
-        
-        response = requests.get(settings.WPST_API_DOMAIN + "/processes/{}".format(app_name))
+        headers = _get_headers(token)
+        response = requests.get(settings.WPST_API_DOMAIN + "/processes/{}".format(app_name), headers=headers)
+        response.raise_for_status()
         json_result = response.json()['process']
 
     except requests.exceptions.HTTPError as e:
@@ -28,15 +30,14 @@ def get_application(app_name):
 
     return json_result
 
-def deploy_application(app_config):
-    
-    headers = {
-       'Content-type': 'application/json'
-    }
+def deploy_application(token, app_config):
 
     try:
-        
+        headers = _get_headers(token, {
+           'Content-type': 'application/json'
+        })
         response = requests.post(settings.WPST_API_DOMAIN + "/processes", headers=headers, json=app_config)
+        response.raise_for_status()
         json_result = response.json()
 
     except requests.exceptions.HTTPError as e:
@@ -45,11 +46,12 @@ def deploy_application(app_config):
 
     return json_result
 
-def dismiss_application(app_name):
+def dismiss_application(token, app_name):
 
     try:
-        
-        response = requests.delete(settings.WPST_API_DOMAIN + "/processes/{}".format(app_name))
+        headers = _get_headers(token)
+        response = requests.delete(settings.WPST_API_DOMAIN + "/processes/{}".format(app_name), headers=headers)
+        response.raise_for_status()
         json_result = response.json()['undeploymentResult']
 
     except requests.exceptions.HTTPError as e:
@@ -59,17 +61,18 @@ def dismiss_application(app_name):
     return json_result
 
 
-def submit_job(app_name, data):
+def submit_job(token, app_name, data):
 
-    headers = {
+    headers = _get_headers(token, {
        'Content-type': 'application/json'
-    }
+    })
 
     try:
         job_url = settings.WPST_API_DOMAIN + "/processes/{}/jobs".format(app_name)
-        r = requests.post(job_url, headers=headers, json=data)
+        response = requests.post(job_url, headers=headers, json=data)
+        response.raise_for_status()
 
-        job_location = r.headers['location']
+        job_location = response.headers['location']
         if "http://127.0.0.1:5000" in job_location:
             job_location = job_location.replace("http://127.0.0.1:5000",settings.WPST_API_DOMAIN)
 
@@ -81,26 +84,29 @@ def submit_job(app_name, data):
 
     return job_id
 
-def get_job_by_id(app_name, job_id):
+def get_job_by_id(token, app_name, job_id):
 
     try:
-        
+        headers = _get_headers(token)
         job_status_url = settings.WPST_API_DOMAIN + "/processes/{}/jobs/{}".format(app_name, job_id)
-        response = requests.get(job_status_url).json()
+        response = requests.get(job_status_url, headers=headers)
+        response.raise_for_status()
+        json_result = response.json()
 
     except requests.exceptions.HTTPError as e:
         # Add Logging Mechanism
         raise
 
-    return response
+    return json_result
 
 
-def get_job_status(app_name, job_id):
+def get_job_status(token, app_name, job_id):
 
     try:
-        
+        headers = _get_headers(token)
         job_status_url = settings.WPST_API_DOMAIN + "/processes/{}/jobs/{}".format(app_name, job_id)
-        response = requests.get(job_status_url)
+        response = requests.get(job_status_url, headers=headers)
+        response.raise_for_status()
         job_status = response.json()['status']
 
     except requests.exceptions.HTTPError as e:
@@ -109,12 +115,13 @@ def get_job_status(app_name, job_id):
 
     return job_status
 
-def get_job_result(app_name, job_id):
+def get_job_result(token, app_name, job_id):
 
     try:
-        
+        headers = _get_headers(token)
         job_result_url = settings.WPST_API_DOMAIN + "/processes/{}/jobs/{}/result".format(app_name, job_id)
-        response = requests.get(job_result_url)
+        response = requests.get(job_result_url, headers=headers)
+        response.raise_for_status()
         json_result = response.json()['outputs']
 
     except requests.exceptions.HTTPError as e:
@@ -123,28 +130,43 @@ def get_job_result(app_name, job_id):
 
     return json_result    
 
-def get_jobs_for_app(app_name):
+def get_jobs_for_app(token, app_name):
 
     try:
+        headers = _get_headers(token)
         job_url = settings.WPST_API_DOMAIN + "/processes/{}/jobs".format(app_name)
-        response = requests.get(job_url)
+        response = requests.get(job_url, headers=headers)
+        response.raise_for_status()
         json_result = response.json()['jobs']
     except:
         # Add Logging Mechanism
         raise
 
     return json_result
 
-def dismiss_job(app_name, job_id):
+def dismiss_job(token, app_name, job_id):
 
     try:
-        
+        headers = _get_headers(token)
         job_url = settings.WPST_API_DOMAIN + "/processes/{}/jobs/{}".format(app_name,job_id)
-        response = requests.delete(job_url)
+        response = requests.delete(job_url, headers=headers)
+        response.raise_for_status()
         json_result = response.json()['statusInfo']
 
     except:
         # Add Logging Mechanism
         raise
 
-    return json_result
+    return json_result
+
+
+def _get_headers(token, additional_headers = {}):
+    
+    if token == None:
+        raise Exception("No authorization token was provided.")
+    
+    base_header = {
+        "Authorization": "Bearer " + token,
+    }
+    
+    return {**base_header, **additional_headers}

jupyter-notebooks/utils/settings.py

@@ -1 +1 @@
-WPST_API_DOMAIN = "http://a706c2e790a7a46489443f4f64b521e5-1186363756.us-west-2.elb.amazonaws.com:5001"
+WPST_API_DOMAIN = "https://71t9bgvuyc.execute-api.us-west-2.amazonaws.com/test/ades-wpst"