Skip to content

Commit 80e4d05

Browse files
mcduffiemcduffie
committed
Modify notebook to locate stage-in files from collection file path.
Write out a catalog file instead of a collection file using unity-py.
1 parent 1ce0725 commit 80e4d05

File tree

2 files changed

+82
-46
lines changed

2 files changed

+82
-46
lines changed

‎process.ipynb

Lines changed: 80 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
"metadata": {},
88
"outputs": [],
99
"source": [
10+
"import os\n",
11+
"\n",
1012
"# Application packages\n",
1113
"from tabulate import tabulate\n",
1214
"import netCDF4\n",
@@ -15,10 +17,9 @@
1517
"from unity_py.resources.collection import Collection\n",
1618
"\n",
1719
"# stage_out packages\n",
18-
"import os\n",
19-
"import json\n",
2020
"from datetime import datetime, timezone\n",
21-
"from pystac import ItemCollection, Item, Asset"
21+
"from unity_py.resources.dataset import Dataset\n",
22+
"from unity_py.resources.data_file import DataFile"
2223
]
2324
},
2425
{
@@ -32,8 +33,8 @@
3233
},
3334
"outputs": [],
3435
"source": [
35-
"input_stac_collection = 'test/stage_in/stage_in_results.json' # type: stage-in\n",
36-
"output_stac_collection = 'process_results.json' # type: stage-out\n",
36+
"input_stac_collection_file = 'test/stage_in/stage_in_results.json' # type: stage-in\n",
37+
"output_stac_catalog_dir = 'process_results/' # type: stage-out\n",
3738
"\n",
3839
"# Filename written to the working directory\n",
3940
"summary_table_filename = \"summary_table.txt\"\n",
@@ -120,10 +121,53 @@
120121
"execution_count": 4,
121122
"id": "3a09d57c",
122123
"metadata": {},
123-
"outputs": [],
124+
"outputs": [
125+
{
126+
"data": {
127+
"text/plain": [
128+
"['./SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc',\n",
129+
" './SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc']"
130+
]
131+
},
132+
"execution_count": 4,
133+
"metadata": {},
134+
"output_type": "execute_result"
135+
}
136+
],
137+
"source": [
138+
"inp_collection = Collection.from_stac(input_stac_collection_file)\n",
139+
"data_filenames = inp_collection.data_locations()\n",
140+
"\n",
141+
"data_filenames"
142+
]
143+
},
144+
{
145+
"cell_type": "code",
146+
"execution_count": 5,
147+
"id": "0671dfaf",
148+
"metadata": {},
149+
"outputs": [
150+
{
151+
"data": {
152+
"text/plain": [
153+
"['/home/mcduffie/Devel/unity/unity-example-application/test/stage_in/SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc',\n",
154+
" '/home/mcduffie/Devel/unity/unity-example-application/test/stage_in/SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc']"
155+
]
156+
},
157+
"execution_count": 5,
158+
"metadata": {},
159+
"output_type": "execute_result"
160+
}
161+
],
124162
"source": [
125-
"inp_collection = Collection.from_stac(input_stac_collection)\n",
126-
"data_filenames = inp_collection.data_locations()"
163+
"# Manually locate the full path of the files to match location of collection file\n",
164+
"import contextlib\n",
165+
"old_cwd = os.path.realpath(os.curdir)\n",
166+
"os.chdir(os.path.dirname(input_stac_collection_file))\n",
167+
"data_filenames = [ os.path.realpath(fn) for fn in data_filenames ]\n",
168+
"os.chdir(old_cwd)\n",
169+
" \n",
170+
"data_filenames"
127171
]
128172
},
129173
{
@@ -138,7 +182,7 @@
138182
},
139183
{
140184
"cell_type": "code",
141-
"execution_count": 5,
185+
"execution_count": 6,
142186
"id": "9fbac209",
143187
"metadata": {},
144188
"outputs": [],
@@ -158,7 +202,7 @@
158202
},
159203
{
160204
"cell_type": "code",
161-
"execution_count": 6,
205+
"execution_count": 7,
162206
"id": "d22c8670",
163207
"metadata": {},
164208
"outputs": [],
@@ -175,7 +219,7 @@
175219
},
176220
{
177221
"cell_type": "code",
178-
"execution_count": 7,
222+
"execution_count": 8,
179223
"id": "3344bd15",
180224
"metadata": {},
181225
"outputs": [
@@ -196,7 +240,7 @@
196240
"'<table>\\n<thead>\\n<tr><th>product_name </th><th>product_name_type_id </th><th>shortname </th><th>product_version </th><th>date_created </th><th>time_coverage_start </th><th>time_coverage_end </th><th style=\"text-align: right;\"> geospatial_lat_mid</th><th style=\"text-align: right;\"> geospatial_lon_mid</th></tr>\\n</thead>\\n<tbody>\\n<tr><td>SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc</td><td>L1_AQ </td><td>SNDR13CHRP1</td><td>v02.48.00 </td><td>2021-04-25T05:59:08Z</td><td>2016-08-22T00:05:22Z </td><td>2016-08-22T00:11:22Z</td><td style=\"text-align: right;\"> -48.6062</td><td style=\"text-align: right;\"> 12.4563 </td></tr>\\n<tr><td>SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc</td><td>L1_AQ </td><td>SNDR13CHRP1</td><td>v02.48.00 </td><td>2021-04-25T05:59:19Z</td><td>2016-08-22T00:11:22Z </td><td>2016-08-22T00:17:22Z</td><td style=\"text-align: right;\"> -69.3979</td><td style=\"text-align: right;\"> -1.98753</td></tr>\\n</tbody>\\n</table>'"
197241
]
198242
},
199-
"execution_count": 7,
243+
"execution_count": 8,
200244
"metadata": {},
201245
"output_type": "execute_result"
202246
}
@@ -208,13 +252,14 @@
208252
},
209253
{
210254
"cell_type": "code",
211-
"execution_count": 8,
255+
"execution_count": 9,
212256
"id": "014257f3",
213257
"metadata": {},
214258
"outputs": [],
215259
"source": [
216260
"# Write the table in text format\n",
217-
"with open(summary_table_filename, \"w\") as summary_file:\n",
261+
"output_filename = os.path.join(output_stac_catalog_dir, summary_table_filename)\n",
262+
"with open(output_filename, \"w\") as summary_file:\n",
218263
" summary_file.write(tabulate(table_data, headers=column_names))"
219264
]
220265
},
@@ -223,45 +268,36 @@
223268
"id": "89224c4e",
224269
"metadata": {},
225270
"source": [
226-
"# Create stage-out item collection"
271+
"# Create stage-out item catalog"
227272
]
228273
},
229274
{
230275
"cell_type": "code",
231-
"execution_count": 9,
232-
"id": "1029f88c",
276+
"execution_count": 10,
277+
"id": "b4aa5d3b",
233278
"metadata": {},
234279
"outputs": [],
235280
"source": [
236-
"output_item = Item(\n",
237-
" id=os.path.splitext(summary_table_filename)[0],\n",
238-
" geometry=\"\",\n",
239-
" bbox=\"\",\n",
240-
" datetime=datetime.now(timezone.utc),\n",
241-
" properties={\n",
242-
" \"datetime\": datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z'),\n",
243-
" \"start_datetime\": datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z'),\n",
244-
" \"end_datetime\": datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z'),\n",
245-
" \"created\": datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z'),\n",
246-
" \"updated\": datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z'),\n",
247-
" },\n",
248-
" assets={\n",
249-
" \"data\": Asset(href=summary_table_filename,title=\"Main Data File\"),\n",
250-
" }\n",
281+
"# Create a collection\n",
282+
"out_collection = Collection(\"SNDR13CHRP1AQCal_rebin\")\n",
283+
" \n",
284+
"# Create a Dataset for the collection\n",
285+
"dataset = Dataset(\n",
286+
" name=os.path.splitext(summary_table_filename)[0], \n",
287+
" collection_id=out_collection.collection_id, \n",
288+
" start_time=datetime.utcnow().replace(tzinfo=timezone.utc).isoformat(), \n",
289+
" end_time=datetime.utcnow().replace(tzinfo=timezone.utc).isoformat(),\n",
290+
" creation_time=datetime.utcnow().replace(tzinfo=timezone.utc).isoformat(),\n",
251291
")\n",
252292
"\n",
253-
"output_collection = ItemCollection([output_item])"
254-
]
255-
},
256-
{
257-
"cell_type": "code",
258-
"execution_count": 10,
259-
"id": "504c9271",
260-
"metadata": {},
261-
"outputs": [],
262-
"source": [
263-
"with open(output_stac_collection, \"w\") as collection_file:\n",
264-
" json.dump(output_collection.to_dict(), collection_file)"
293+
"# Add output file(s) to the dataset\n",
294+
"dataset.add_data_file(DataFile(\"data\", summary_table_filename))\n",
295+
"\n",
296+
"# Add the dataset to the collection\n",
297+
"#out_collection.add_dataset(dataset)\n",
298+
"out_collection._datasets.append(dataset)\n",
299+
"\n",
300+
"Collection.to_stac(out_collection, output_stac_catalog_dir)"
265301
]
266302
}
267303
],

‎test/stage_in/stage_in_results.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
"assets": {
2020
"data": {
2121
"title": "Download SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc",
22-
"href": "test/stage_in/SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc"
22+
"href": "./SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc"
2323
}
2424
},
2525
"type": "Feature",
@@ -96,7 +96,7 @@
9696
"assets": {
9797
"data": {
9898
"title": "Download SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc",
99-
"href": "test/stage_in/SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc"
99+
"href": "./SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc"
100100
}
101101
},
102102
"type": "Feature",

0 commit comments

Comments
 (0)