Update notebook to change inline text to reflect what the

unity-app-generator is doing.

Add test scripts for running the now default modular app approach.
Rename the old test scripts director to reflect it for the monolithic
app approach.

Author: mcduffie

.gitignore

@@ -7,9 +7,10 @@ env/
 # Jupyter temporary directory
 .ipynb_checkpoints/
 
-# Output from process.ipynb
-summary_table.txt
-process_results.json
+# User created monolithic app job input file derived from template
+test/monolithic_app/cwl_job_input.yml
+
+# Output from process.ipynb and test scripts
 test/process_results/
 
 # Unity App Generator file

process.ipynb

@@ -32,8 +32,8 @@
     "\n",
     "The below cell is tagged as a 'paramter' cell. This enables us to overwrite the below values at runtime. There are some special values in the below cell.\n",
     "\n",
-    "* `input` is a special name, and it also has the `# type: stage-in` annotation. This should be a directory, and at run time it will be populated with a STAC catalog that contains files that have been staged for your algorithm to reference.\n",
-    "* `output` is a special name, and it also has the `# type: stage-out` annotation. This should be treated as a directory to which you write ALL of your output files along with a STAC catalog that references files you would like to persist outside of the algorithm run."
+    "* `input_stac_catalog_dir` has the `# type: stage-in` annotation. This variable tells your algorithm the name of the directory where to read inputs from. At run time it will be populated with a STAC catalog and the files it references. The application generator process will connect your variable name to the special `input` parameter in automatically generated CWL files.\n",
+    "* `output_stac_catalog_dir` has the `# type: stage-out` annotation. This is a directory where you write ALL of your output files along with a STAC catalog that references files you would like to persist outside of the algorithm run. The application generator process will connect your variable name to the special `output` parameter in automatically generated CWL files."
    ]
   },
   {
@@ -51,8 +51,8 @@
    },
    "outputs": [],
    "source": [
-    "input = 'test/stage_in/' # type: stage-in\n",
-    "output    = 'test/process_results/'                    # type: stage-out\n",
+    "input_stac_catalog_dir  = 'test/stage_in/'         # type: stage-in\n",
+    "output_stac_catalog_dir = 'test/process_results/'  # type: stage-out\n",
     "\n",
     "# Filename written to the working directory\n",
     "summary_table_filename = \"summary_table.txt\"\n",
@@ -70,7 +70,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 3,
    "id": "62471f5d-d898-46c1-89c1-b572851db551",
    "metadata": {},
    "outputs": [
@@ -83,7 +83,7 @@
     }
    ],
    "source": [
-    "input_catalog = os.path.join(input, \"catalog.json\")\n",
+    "input_catalog = os.path.join(input_stac_catalog_dir, \"catalog.json\")\n",
     "print(\"reading {}\".format(input_catalog))"
    ]
   },
@@ -99,7 +99,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 4,
    "id": "2eeaa5d4",
    "metadata": {},
    "outputs": [
@@ -123,7 +123,7 @@
        "'<table>\\n<thead>\\n<tr><th>argument_name          </th><th>type              </th><th>value  </th></tr>\\n</thead>\\n<tbody>\\n<tr><td>example_argument_int   </td><td>&lt;class &#x27;int&#x27;&gt;     </td><td>1      </td></tr>\\n<tr><td>example_argument_float </td><td>&lt;class &#x27;float&#x27;&gt;   </td><td>1.0    </td></tr>\\n<tr><td>example_argument_string</td><td>&lt;class &#x27;str&#x27;&gt;     </td><td>string </td></tr>\\n<tr><td>example_argument_bool  </td><td>&lt;class &#x27;bool&#x27;&gt;    </td><td>True   </td></tr>\\n<tr><td>example_argument_empty </td><td>&lt;class &#x27;NoneType&#x27;&gt;</td><td>       </td></tr>\\n</tbody>\\n</table>'"
       ]
      },
-     "execution_count": 10,
+     "execution_count": 4,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -158,18 +158,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 5,
    "id": "3a09d57c",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "['/Users/gangl/dev/unity/unity-OGC-example-application/test/stage_in/./SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc',\n",
-       " '/Users/gangl/dev/unity/unity-OGC-example-application/test/stage_in/./SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc']"
+       "['test/stage_in/SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc',\n",
+       " 'test/stage_in/SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc']"
       ]
      },
-     "execution_count": 11,
+     "execution_count": 5,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -178,7 +178,8 @@
     "inp_collection = Collection.from_stac(input_catalog)\n",
     "data_filenames = inp_collection.data_locations()\n",
     "\n",
-    "data_filenames"
+    "# Scrub base part of path so it is not displayed in Github with a home directory prefix\n",
+    "[ os.path.relpath(fn, os.curdir) for fn in data_filenames ]"
    ]
   },
   {
@@ -193,7 +194,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 6,
    "id": "9fbac209",
    "metadata": {},
    "outputs": [],
@@ -213,7 +214,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 7,
    "id": "d22c8670",
    "metadata": {},
    "outputs": [],
@@ -230,7 +231,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 8,
    "id": "3344bd15",
    "metadata": {},
    "outputs": [
@@ -251,7 +252,7 @@
        "'<table>\\n<thead>\\n<tr><th>product_name                                                               </th><th>product_name_type_id  </th><th>shortname  </th><th>product_version  </th><th>date_created        </th><th>time_coverage_start  </th><th>time_coverage_end   </th><th style=\"text-align: right;\">  geospatial_lat_mid</th><th style=\"text-align: right;\">  geospatial_lon_mid</th></tr>\\n</thead>\\n<tbody>\\n<tr><td>SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc</td><td>L1_AQ                 </td><td>SNDR13CHRP1</td><td>v02.48.00        </td><td>2021-04-25T05:59:08Z</td><td>2016-08-22T00:05:22Z </td><td>2016-08-22T00:11:22Z</td><td style=\"text-align: right;\">            -48.6062</td><td style=\"text-align: right;\">            12.4563 </td></tr>\\n<tr><td>SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc</td><td>L1_AQ                 </td><td>SNDR13CHRP1</td><td>v02.48.00        </td><td>2021-04-25T05:59:19Z</td><td>2016-08-22T00:11:22Z </td><td>2016-08-22T00:17:22Z</td><td style=\"text-align: right;\">            -69.3979</td><td style=\"text-align: right;\">            -1.98753</td></tr>\\n</tbody>\\n</table>'"
       ]
      },
-     "execution_count": 14,
+     "execution_count": 8,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -263,14 +264,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 9,
    "id": "014257f3",
    "metadata": {},
    "outputs": [],
    "source": [
     "# Write the table in text format\n",
-    "pathlib.Path(output).mkdir(parents=True, exist_ok=True)\n",
-    "output_filename = os.path.join(output, summary_table_filename)\n",
+    "pathlib.Path(output_stac_catalog_dir).mkdir(parents=True, exist_ok=True)\n",
+    "output_filename = os.path.join(output_stac_catalog_dir, summary_table_filename)\n",
     "with open(output_filename, \"w\") as summary_file:\n",
     "    summary_file.write(tabulate(table_data, headers=column_names))"
    ]
@@ -285,7 +286,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 10,
    "id": "b4aa5d3b",
    "metadata": {
     "editable": true,
@@ -303,38 +304,30 @@
     "dataset = Dataset(\n",
     "    name=summary_table_filename, \n",
     "    collection_id=out_collection.collection_id, \n",
-    "    start_time=datetime.utcnow().replace(tzinfo=timezone.utc).isoformat(), \n",
-    "    end_time=datetime.utcnow().replace(tzinfo=timezone.utc).isoformat(),\n",
-    "    creation_time=datetime.utcnow().replace(tzinfo=timezone.utc).isoformat(),\n",
+    "    start_time=datetime.now(timezone.utc).isoformat(), \n",
+    "    end_time=datetime.now(timezone.utc).isoformat(),\n",
+    "    creation_time=datetime.now(timezone.utc).isoformat(),\n",
     ")\n",
     "\n",
     "# Add output file(s) to the dataset\n",
     "dataset.add_data_file(DataFile(\"csv\", summary_table_filename, [\"data\"]))\n",
     "\n",
     "#when we run \"to_stac\" below, this file  will be generated. this needs to be added to the stac file itself for future reference.\n",
-    "dataset.add_data_file(DataFile(\"json\", output + \"/\" + summary_table_filename +'.json', [\"metadata\"] ))\n",
+    "dataset.add_data_file(DataFile(\"json\", os.path.join(output_stac_catalog_dir, summary_table_filename + '.json'), [\"metadata\"] ))\n",
     "\n",
     "\n",
     "# Add the dataset to the collection\n",
     "#out_collection.add_dataset(dataset)\n",
     "out_collection._datasets.append(dataset)\n",
     "\n",
-    "Collection.to_stac(out_collection, output)"
+    "Collection.to_stac(out_collection, output_stac_catalog_dir)"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "14f240b6-b5ea-4bf8-8764-ecf12eee6c16",
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {
   "celltoolbar": "Tags",
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "Python 3",
    "language": "python",
    "name": "python3"
   },
@@ -348,7 +341,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.8"
+   "version": "3.12.4"
   }
  },
  "nbformat": 4,

test/ogc_app/build_app.sh renamed to test/modular_app/build_app.sh

@@ -9,4 +9,4 @@ cd $BASE_DIR
 build_ogc_app init .
 
 build_ogc_app build_docker
-build_ogc_app build_cwl
+build_ogc_app build_cwl

test/modular_app/cwl_job_input.yml

@@ -0,0 +1,10 @@
+input:  # type 'Directory'
+    class: Directory
+    path: ../stage_in
+summary_table_filename: summary_table.txt  # default value of type 'string'.
+output_collection: urn:nasa:unity:unity:dev:unity-example-application___1  # default value of type 'string'.
+example_argument_string: string  # default value of type 'string'.
+example_argument_int: 1  # default value of type 'int'.
+example_argument_float: 1.0  # default value of type 'float'.
+example_argument_empty: a_string  # type 'string'
+example_argument_bool: true  # default value of type 'boolean'.

test/modular_app/test_cwl.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+SCRIPT_DIR=$(realpath $(dirname $0))
+BASE_DIR=$(realpath $(dirname $0)/../..)
+
+WORKFLOW_FILENAME=$BASE_DIR/.unity_app_gen/cwl/process.cwl
+JOB_INP_FILENAME=$SCRIPT_DIR/cwl_job_input.yml
+RUN_DIR=$BASE_DIR/test/process_results
+
+if [ ! -e $WORKFLOW_FILENAME ]; then
+    echo "ERROR: Run $SCRIPT_DIR/build_app.sh first to generate CWL output"
+    exit 1
+fi
+
+if [ ! -e $JOB_INP_FILENAME ]; then
+    echo "ERROR: Copy ${JOB_INP_FILENAME}.template to ${JOB_INP_FILENAME} and edit to include credentials"
+    exit 1
+fi
+
+mkdir -p $RUN_DIR
+cd $RUN_DIR
+
+# Detect if using Podman 
+if [ ! -z "$(which podman)" ]; then
+    use_podman_arg="--podman"
+fi
+
+cwltool \
+    --debug --leave-tmpdir --no-read-only \
+    $use_podman_arg \
+    "$WORKFLOW_FILENAME" "$JOB_INP_FILENAME" \
+    $* |& tee $RUN_DIR/test_modular_cwl.log

test/monolithic_app/build_app.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e 
+
+BASE_DIR=$(realpath $(dirname $0)/../..)
+
+cd $BASE_DIR
+
+build_ogc_app init .
+
+build_ogc_app build_docker
+build_ogc_app build_cwl --monolithic

test/ogc_app/cmr_results.json renamed to test/monolithic_app/cmr_results.json

@@ -21,4 +21,4 @@ stage_out:  # Anonymous record type.
     aws_region: us-west-2
     aws_session_token: null
     aws_secret_access_key: null
-    aws_access_key_id: null
+    aws_access_key_id: null

test/ogc_app/cwl_job_input.yml.template renamed to test/monolithic_app/cwl_job_input.yml.template

@@ -5,6 +5,7 @@ BASE_DIR=$(realpath $(dirname $0)/../..)
 
 WORKFLOW_FILENAME=$BASE_DIR/.unity_app_gen/cwl/workflow.cwl
 JOB_INP_FILENAME=$SCRIPT_DIR/cwl_job_input.yml
+RUN_DIR=$BASE_DIR/test/process_results
 
 if [ ! -e $WORKFLOW_FILENAME ]; then
     echo "ERROR: Run $SCRIPT_DIR/build_app.sh first to generate CWL output"
@@ -44,6 +45,9 @@ done
 echo "Using modified job input file $modified_job_inp_file:"
 cat $modified_job_inp_file
 
+mkdir -p $RUN_DIR
+cd $RUN_DIR
+
 # Detect if using Podman 
 if [ ! -z "$(which podman)" ]; then
     use_podman_arg="--podman"
@@ -53,6 +57,6 @@ cwltool \
     --debug --leave-tmpdir --no-read-only \
     $use_podman_arg \
     "$WORKFLOW_FILENAME" "$modified_job_inp_file" \
-    $*
+    $* |& tee $RUN_DIR/test_monolithic_cwl.log
 
 rm $modified_job_inp_file