Skip to content

Commit 504ed3b

Browse files
mcduffiemcduffie
committed
Rewrite example to use updated stage_in/stage_out mechanism of
unity-app-generator.
1 parent a4a148b commit 504ed3b

12 files changed

+665
-36
lines changed

‎.gitattributes

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
*.nc filter=lfs diff=lfs merge=lfs -text

‎.gitignore

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,15 @@
1+
# Editor temporary files
2+
.*.swp
3+
14
# Virtual environment
25
env/
36

47
# Jupyter temporary directory
58
.ipynb_checkpoints/
9+
10+
# Output from process.ipynb
11+
summary_table.txt
12+
process_results.json
13+
14+
# Unity App Generator file
15+
.unity_app_gen

‎example_job_input.json

Lines changed: 0 additions & 9 deletions
This file was deleted.

‎process.ipynb

Lines changed: 231 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,31 @@
22
"cells": [
33
{
44
"cell_type": "code",
5-
"execution_count": null,
5+
"execution_count": 1,
6+
"id": "075fe5a0",
7+
"metadata": {},
8+
"outputs": [],
9+
"source": [
10+
"import os\n",
11+
"from datetime import datetime \n",
12+
"from datetime import timezone"
13+
]
14+
},
15+
{
16+
"cell_type": "code",
17+
"execution_count": 2,
618
"id": "1251c9f4",
719
"metadata": {},
820
"outputs": [],
921
"source": [
10-
"import os"
22+
"from tabulate import tabulate\n",
23+
"import netCDF4\n",
24+
"from pystac import ItemCollection, Item, Asset"
1125
]
1226
},
1327
{
1428
"cell_type": "code",
15-
"execution_count": null,
29+
"execution_count": 3,
1630
"id": "04ac7f2d",
1731
"metadata": {
1832
"tags": [
@@ -21,59 +35,250 @@
2135
},
2236
"outputs": [],
2337
"source": [
24-
"input_filename = [{'path': '/tmp/input_file'}] # type: stage-in\n",
25-
"line_offset = 0\n",
26-
"output_directory = \"/tmp\""
38+
"input_stac_collection = 'test/stage_in/stage_in_results.json' # type: stage-in\n",
39+
"output_stac_collection = 'process_results.json' # type: stage-out\n",
40+
"\n",
41+
"# Filename written to the working directory\n",
42+
"summary_table_filename = \"summary_table.txt\"\n",
43+
"\n",
44+
"# Examples of arbitrary arguments of different data types\n",
45+
"example_argument_int = 1\n",
46+
"example_argument_float = 1.0\n",
47+
"example_argument_string = \"string\"\n",
48+
"example_argument_bool = True\n",
49+
"example_argument_empty = None"
50+
]
51+
},
52+
{
53+
"cell_type": "markdown",
54+
"id": "7926d21b",
55+
"metadata": {},
56+
"source": [
57+
"# Output Example Arguments\n",
58+
"\n",
59+
"Table useful for debugging connection of arguments from CWL into notebook. Used for unity-app-generator development and as an example, not an application requirement."
60+
]
61+
},
62+
{
63+
"cell_type": "code",
64+
"execution_count": 4,
65+
"id": "2eeaa5d4",
66+
"metadata": {},
67+
"outputs": [
68+
{
69+
"data": {
70+
"text/html": [
71+
"<table>\n",
72+
"<thead>\n",
73+
"<tr><th>argument_name </th><th>type </th><th>value </th></tr>\n",
74+
"</thead>\n",
75+
"<tbody>\n",
76+
"<tr><td>example_argument_int </td><td>&lt;class &#x27;int&#x27;&gt; </td><td>1 </td></tr>\n",
77+
"<tr><td>example_argument_float </td><td>&lt;class &#x27;float&#x27;&gt; </td><td>1.0 </td></tr>\n",
78+
"<tr><td>example_argument_string</td><td>&lt;class &#x27;str&#x27;&gt; </td><td>string </td></tr>\n",
79+
"<tr><td>example_argument_bool </td><td>&lt;class &#x27;bool&#x27;&gt; </td><td>True </td></tr>\n",
80+
"<tr><td>example_argument_empty </td><td>&lt;class &#x27;NoneType&#x27;&gt;</td><td> </td></tr>\n",
81+
"</tbody>\n",
82+
"</table>"
83+
],
84+
"text/plain": [
85+
"'<table>\\n<thead>\\n<tr><th>argument_name </th><th>type </th><th>value </th></tr>\\n</thead>\\n<tbody>\\n<tr><td>example_argument_int </td><td>&lt;class &#x27;int&#x27;&gt; </td><td>1 </td></tr>\\n<tr><td>example_argument_float </td><td>&lt;class &#x27;float&#x27;&gt; </td><td>1.0 </td></tr>\\n<tr><td>example_argument_string</td><td>&lt;class &#x27;str&#x27;&gt; </td><td>string </td></tr>\\n<tr><td>example_argument_bool </td><td>&lt;class &#x27;bool&#x27;&gt; </td><td>True </td></tr>\\n<tr><td>example_argument_empty </td><td>&lt;class &#x27;NoneType&#x27;&gt;</td><td> </td></tr>\\n</tbody>\\n</table>'"
86+
]
87+
},
88+
"execution_count": 4,
89+
"metadata": {},
90+
"output_type": "execute_result"
91+
}
92+
],
93+
"source": [
94+
"argument_names = [\n",
95+
" 'example_argument_int',\n",
96+
" 'example_argument_float',\n",
97+
" 'example_argument_string',\n",
98+
" 'example_argument_bool',\n",
99+
" 'example_argument_empty',\n",
100+
"]\n",
101+
"\n",
102+
"table_data = []\n",
103+
"column_names = ['argument_name', 'type', 'value']\n",
104+
"for arg_name in argument_names:\n",
105+
" arg_val = locals()[arg_name]\n",
106+
" table_data.append( (arg_name, type(arg_val), arg_val) )\n",
107+
"\n",
108+
"tabulate(table_data, headers=column_names, tablefmt='html')"
109+
]
110+
},
111+
{
112+
"cell_type": "markdown",
113+
"id": "60b7fa38",
114+
"metadata": {},
115+
"source": [
116+
"# Import Files from STAC Item Collection\n",
117+
"\n",
118+
"Load filenames from the stage_in STAC item collection file"
27119
]
28120
},
29121
{
30122
"cell_type": "code",
31-
"execution_count": null,
32-
"id": "e37390cf",
123+
"execution_count": 5,
124+
"id": "3a09d57c",
33125
"metadata": {},
34126
"outputs": [],
35127
"source": [
36-
"output_filename = os.path.join(output_directory, 'lines_numbered.txt')"
128+
"import json\n",
129+
"\n",
130+
"with open(input_stac_collection, \"r\") as collection_file:\n",
131+
" collection = ItemCollection.from_dict(json.load(collection_file))"
37132
]
38133
},
39134
{
40135
"cell_type": "code",
41-
"execution_count": null,
42-
"id": "198423ee",
136+
"execution_count": 6,
137+
"id": "72b03749",
43138
"metadata": {},
44139
"outputs": [],
45140
"source": [
46-
"print(f\"Reading {input_filename[0]['path']}\")\n",
141+
"data_filenames = []\n",
142+
"for item in collection.items:\n",
143+
" for asset_key in item.assets:\n",
144+
" asset = item.assets[asset_key]\n",
145+
" data_filenames.append(asset.href)"
146+
]
147+
},
148+
{
149+
"cell_type": "markdown",
150+
"id": "73e97444",
151+
"metadata": {},
152+
"source": [
153+
"# Summary Table\n",
47154
"\n",
48-
"with open(input_filename[0]['path']) as input_file:\n",
49-
" input_lines = input_file.readlines()"
155+
"AKA the result of this \"application\""
156+
]
157+
},
158+
{
159+
"cell_type": "code",
160+
"execution_count": 7,
161+
"id": "9fbac209",
162+
"metadata": {},
163+
"outputs": [],
164+
"source": [
165+
"column_names = [\n",
166+
" 'product_name',\n",
167+
" 'product_name_type_id',\n",
168+
" 'shortname',\n",
169+
" 'product_version',\n",
170+
" 'date_created',\n",
171+
" 'time_coverage_start',\n",
172+
" 'time_coverage_end',\n",
173+
" 'geospatial_lat_mid',\n",
174+
" 'geospatial_lon_mid',\n",
175+
"]"
176+
]
177+
},
178+
{
179+
"cell_type": "code",
180+
"execution_count": 8,
181+
"id": "d22c8670",
182+
"metadata": {},
183+
"outputs": [],
184+
"source": [
185+
"table_data = []\n",
186+
"for data_file in data_filenames:\n",
187+
" nc_file = netCDF4.Dataset(data_file, \"r\")\n",
188+
" \n",
189+
" table_row = []\n",
190+
" for col_name in column_names:\n",
191+
" table_row.append(getattr(nc_file, col_name))\n",
192+
" table_data.append(table_row)"
193+
]
194+
},
195+
{
196+
"cell_type": "code",
197+
"execution_count": 9,
198+
"id": "3344bd15",
199+
"metadata": {},
200+
"outputs": [
201+
{
202+
"data": {
203+
"text/html": [
204+
"<table>\n",
205+
"<thead>\n",
206+
"<tr><th>product_name </th><th>product_name_type_id </th><th>shortname </th><th>product_version </th><th>date_created </th><th>time_coverage_start </th><th>time_coverage_end </th><th style=\"text-align: right;\"> geospatial_lat_mid</th><th style=\"text-align: right;\"> geospatial_lon_mid</th></tr>\n",
207+
"</thead>\n",
208+
"<tbody>\n",
209+
"<tr><td>SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc</td><td>L1_AQ </td><td>SNDR13CHRP1</td><td>v02.48.00 </td><td>2021-04-25T05:59:08Z</td><td>2016-08-22T00:05:22Z </td><td>2016-08-22T00:11:22Z</td><td style=\"text-align: right;\"> -48.6062</td><td style=\"text-align: right;\"> 12.4563 </td></tr>\n",
210+
"<tr><td>SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc</td><td>L1_AQ </td><td>SNDR13CHRP1</td><td>v02.48.00 </td><td>2021-04-25T05:59:19Z</td><td>2016-08-22T00:11:22Z </td><td>2016-08-22T00:17:22Z</td><td style=\"text-align: right;\"> -69.3979</td><td style=\"text-align: right;\"> -1.98753</td></tr>\n",
211+
"</tbody>\n",
212+
"</table>"
213+
],
214+
"text/plain": [
215+
"'<table>\\n<thead>\\n<tr><th>product_name </th><th>product_name_type_id </th><th>shortname </th><th>product_version </th><th>date_created </th><th>time_coverage_start </th><th>time_coverage_end </th><th style=\"text-align: right;\"> geospatial_lat_mid</th><th style=\"text-align: right;\"> geospatial_lon_mid</th></tr>\\n</thead>\\n<tbody>\\n<tr><td>SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc</td><td>L1_AQ </td><td>SNDR13CHRP1</td><td>v02.48.00 </td><td>2021-04-25T05:59:08Z</td><td>2016-08-22T00:05:22Z </td><td>2016-08-22T00:11:22Z</td><td style=\"text-align: right;\"> -48.6062</td><td style=\"text-align: right;\"> 12.4563 </td></tr>\\n<tr><td>SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc</td><td>L1_AQ </td><td>SNDR13CHRP1</td><td>v02.48.00 </td><td>2021-04-25T05:59:19Z</td><td>2016-08-22T00:11:22Z </td><td>2016-08-22T00:17:22Z</td><td style=\"text-align: right;\"> -69.3979</td><td style=\"text-align: right;\"> -1.98753</td></tr>\\n</tbody>\\n</table>'"
216+
]
217+
},
218+
"execution_count": 9,
219+
"metadata": {},
220+
"output_type": "execute_result"
221+
}
222+
],
223+
"source": [
224+
"tabulate(table_data, headers=column_names, tablefmt='html')"
225+
]
226+
},
227+
{
228+
"cell_type": "code",
229+
"execution_count": 10,
230+
"id": "014257f3",
231+
"metadata": {},
232+
"outputs": [],
233+
"source": [
234+
"with open(summary_table_filename, \"w\") as summary_file:\n",
235+
" summary_file.write(tabulate(table_data, headers=column_names))"
236+
]
237+
},
238+
{
239+
"cell_type": "markdown",
240+
"id": "89224c4e",
241+
"metadata": {},
242+
"source": [
243+
"# Create stage-out item collection"
50244
]
51245
},
52246
{
53247
"cell_type": "code",
54-
"execution_count": null,
55-
"id": "07f23b67",
248+
"execution_count": 11,
249+
"id": "b48d4ffd",
56250
"metadata": {},
57251
"outputs": [],
58252
"source": [
59-
"print(f\"Creating output {output_filename}\")\n",
253+
"output_item = Item(\n",
254+
" id=os.path.splitext(summary_table_filename)[0],\n",
255+
" geometry=\"\",\n",
256+
" bbox=\"\",\n",
257+
" datetime=datetime.now(timezone.utc),\n",
258+
" properties={\n",
259+
" \"datetime\": datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z'),\n",
260+
" \"start_datetime\": datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z'),\n",
261+
" \"end_datetime\": datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z'),\n",
262+
" \"created\": datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z'),\n",
263+
" \"updated\": datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z'),\n",
264+
" },\n",
265+
" assets={\n",
266+
" \"data\": Asset(href=summary_table_filename,title=\"Main Data File\"),\n",
267+
" }\n",
268+
")\n",
60269
"\n",
61-
"line_number = line_offset\n",
62-
"with open(output_filename, \"w\") as output_file:\n",
63-
" for line in input_lines:\n",
64-
" line = line.rstrip()\n",
65-
" print(f\"{line_number} : {line}\", file=output_file)\n",
66-
" line_number += 1"
270+
"output_collection = ItemCollection([output_item])"
67271
]
68272
},
69273
{
70274
"cell_type": "code",
71-
"execution_count": null,
72-
"id": "a4219fd1",
275+
"execution_count": 12,
276+
"id": "f63f5499",
73277
"metadata": {},
74278
"outputs": [],
75279
"source": [
76-
"print(f\"Finished with {line_number} lines\")"
280+
"with open(output_stac_collection, \"w\") as collection_file:\n",
281+
" json.dump(output_collection.to_dict(), collection_file)"
77282
]
78283
}
79284
],

‎requirements.txt

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,4 @@
1-
papermill
1+
papermill
2+
pystac
3+
netCDF4
4+
tabulate

‎test/ogc_app/build_app.sh

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
BASE_DIR=$(realpath $(dirname $0)/../..)
6+
7+
cd $BASE_DIR
8+
9+
build_ogc_app.py init .
10+
11+
build_ogc_app.py build_docker
12+
build_ogc_app.py build_cwl

0 commit comments

Comments
 (0)