Skip to content

Commit 0348320

Browse files
anilnathaanilnatha
authored
Merge pull request #23 from unity-sds/develop
Merging notebook and API changes
2 parents 1be09c5 + f685edc commit 0348320

File tree

4 files changed

+340
-64
lines changed

4 files changed

+340
-64
lines changed

‎jupyter-notebooks/tutorials/2_working_with_data.ipynb

Lines changed: 177 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -191,22 +191,24 @@
191191
"url = env['url'] + \"am-uds-dapa/collections/\"+data_set+\"/items\"\n",
192192
"\n",
193193
"params = []\n",
194-
"params.append((\"limit\", 20))\n",
195-
"params.append((\"datetime\", \"1900-01-01T00:00:00Z/2022-11-01T02:31:12Z\")) # working through a bug that requires a time to be included.\n",
194+
"#params.append((\"limit\", 20))\n",
196195
"\n",
197196
"response = requests.get(url, headers={\"Authorization\": \"Bearer \" + token}, params=params)\n",
198197
"\n",
198+
"print(f\"Endpoint: \"+url)\n",
199+
"print(f\"Total number of files: {response.json()['numberMatched']}\")\n",
199200
"print(\"File IDs, titles, and hrefs in Collection \" + data_set + \"\\n\")\n",
200201
"\n",
201202
"features = response.json()['features']\n",
202203
"\n",
203204
"for data_file in features: {\n",
204-
" print(data_file['id']),\n",
205-
" print(data_file['assets']['metadata__data']['title']),\n",
206-
" print(data_file['assets']['metadata__data']['href']),\n",
205+
" print(\"For \"+ data_file['id']),\n",
206+
" print(\"File:\\t\\t\"+data_file['assets']['data']['href']),\n",
207+
" print(\"Metadata:\\t\"+data_file['assets']['metadata__data']['href']),\n",
207208
" print(\"\")\n",
208209
"}\n",
209210
"\n",
211+
"\n",
210212
"print(\"Full JSON response object:\")\n",
211213
"JSON(response.json())\n"
212214
]
@@ -240,24 +242,132 @@
240242
"params.append((\"datetime\", \"2000-11-01T00:00:00Z/2022-11-01T02:31:12Z\"))\n",
241243
"\n",
242244
"# limit - how many results to return in a single request\n",
243-
"params.append((\"limit\", 100))\n",
244-
"\n",
245-
"#offset - the number of results to offset, for paging\n",
246-
"#params.append((\"offset\", 100))\n",
245+
"#params.append((\"limit\", 10))\n",
247246
"\n",
248247
"response = requests.get(url, headers={\"Authorization\": \"Bearer \" + token}, params=params)\n",
249248
"\n",
249+
"print(f\"Total number of files: {response.json()['numberMatched']}\")\n",
250250
"print(\"File IDs, datetimes, and hrefs in Collection \" + data_set + \"\\n\")\n",
251251
"\n",
252252
"features = response.json()['features']\n",
253-
"\n",
254-
"for data_file in features: {\n",
255-
" print(data_file['id']),\n",
256-
" print(data_file['properties']['created']),\n",
257-
" print(data_file['assets']['metadata__data']['href']),\n",
258-
" print(\"\")\n",
253+
"while len(features) > 0:\n",
254+
" for data_file in features: {\n",
255+
" print(data_file['id']),\n",
256+
" print(data_file['properties']['created']),\n",
257+
" print(data_file['assets']['metadata__data']['href']),\n",
258+
" print(data_file['assets']['data']['href']),\n",
259+
" print(\"\")\n",
260+
" }\n",
261+
" # Get the next page of results\n",
262+
" response = requests.get(next(item for item in response.json()['links'] if item['rel'] == 'next')['href'], headers={\"Authorization\": \"Bearer \" + token}, params=params)\n",
263+
" features = response.json()['features']"
264+
]
265+
},
266+
{
267+
"cell_type": "markdown",
268+
"id": "d2aba320-4079-4fa0-943e-d97391c2ce8c",
269+
"metadata": {},
270+
"source": [
271+
"## Create a new Collection"
272+
]
273+
},
274+
{
275+
"cell_type": "code",
276+
"execution_count": null,
277+
"id": "acd23209-573e-4b72-9f5a-5f7b7f6be26d",
278+
"metadata": {},
279+
"outputs": [],
280+
"source": [
281+
"collection_id = \"NEW_COLLECTION_EXAMPLE_L1B___5\"\n",
282+
"collection = {\n",
283+
" \"type\": \"Collection\",\n",
284+
" \"id\": collection_id,\n",
285+
" \"stac_version\": \"1.0.0\",\n",
286+
" \"description\": \"TODO\",\n",
287+
" \"links\": [\n",
288+
" {\n",
289+
" \"rel\": \"root\",\n",
290+
" \"href\": \"./collection.json?bucket=unknown_bucket&regex=%7BcmrMetadata.Granule.Collection.ShortName%7D___%7BcmrMetadata.Granule.Collection.VersionId%7D\",\n",
291+
" \"type\": \"application/json\",\n",
292+
" \"title\": \"test_file01.nc\"\n",
293+
" },\n",
294+
" {\n",
295+
" \"rel\": \"item\",\n",
296+
" \"href\": \"./collection.json?bucket=protected&regex=%5Etest_file.%2A%5C.nc%24\",\n",
297+
" \"type\": \"data\",\n",
298+
" \"title\": \"test_file01.nc\"\n",
299+
" },\n",
300+
" {\n",
301+
" \"rel\": \"item\",\n",
302+
" \"href\": \"./collection.json?bucket=protected&regex=%5Etest_file.%2A%5C.nc%5C.cas%24\",\n",
303+
" \"type\": \"metadata\",\n",
304+
" \"title\": \"test_file01.nc.cas\"\n",
305+
" },\n",
306+
" {\n",
307+
" \"rel\": \"item\",\n",
308+
" \"href\": \"./collection.json?bucket=private&regex=%5Etest_file.%2A%5C.cmr%5C.xml%24\",\n",
309+
" \"type\": \"metadata\",\n",
310+
" \"title\": \"test_file01.cmr.xml\"\n",
311+
" }\n",
312+
" ],\n",
313+
" \"stac_extensions\": [],\n",
314+
" \"extent\": {\n",
315+
" \"spatial\": {\n",
316+
" \"bbox\": [\n",
317+
" [\n",
318+
" -180,\n",
319+
" -90,\n",
320+
" 180,\n",
321+
" 90\n",
322+
" ]\n",
323+
" ]\n",
324+
" },\n",
325+
" \"temporal\": {\n",
326+
" \"interval\": [\n",
327+
" [\n",
328+
" \"2022-10-04T00:00:00.000000Z\",\n",
329+
" \"2022-10-04T23:59:59.999999Z\"\n",
330+
" ]\n",
331+
" ]\n",
332+
" }\n",
333+
" },\n",
334+
" \"license\": \"proprietary\",\n",
335+
" \"summaries\": {\n",
336+
" \"granuleId\": [\n",
337+
" \"^test_file.*$\"\n",
338+
" ],\n",
339+
" \"granuleIdExtraction\": [\n",
340+
" \"(^test_file.*)(\\\\.nc|\\\\.nc\\\\.cas|\\\\.cmr\\\\.xml)\"\n",
341+
" ],\n",
342+
" \"process\": [\n",
343+
" \"snpp.level1\"\n",
344+
" ]\n",
345+
" }\n",
259346
"}\n",
260347
"\n",
348+
"url = env['url'] + \"am-uds-dapa/collections\"\n",
349+
"response = requests.post(url, headers={\"Authorization\": \"Bearer \" + token}, json=collection)\n",
350+
"print(response)"
351+
]
352+
},
353+
{
354+
"cell_type": "markdown",
355+
"id": "c48cc2e9-4d2a-4056-bf1a-59a4365960a4",
356+
"metadata": {},
357+
"source": [
358+
"## Get newly created Collection\n",
359+
"\n",
360+
"The collection creation may take a minute, so if the new collection isn't returned immediately, please retry."
361+
]
362+
},
363+
{
364+
"cell_type": "code",
365+
"execution_count": null,
366+
"metadata": {},
367+
"outputs": [],
368+
"source": [
369+
"url = env['url'] + \"am-uds-dapa/collections/\" + collection_id\n",
370+
"response = requests.get(url, headers={\"Authorization\": \"Bearer \" + token})\n",
261371
"print(\"Full JSON response object:\")\n",
262372
"JSON(response.json())"
263373
]
@@ -267,16 +377,34 @@
267377
"id": "7719daba",
268378
"metadata": {},
269379
"source": [
270-
"## Explore on your own (COMING SOON)\n",
380+
"## Explore on your own\n",
381+
"\n",
382+
"Given the endpoints above for finding collections and then finding data within those collections, try to craft a query by copying cells to find data from one of the other collections in the list above.\n",
271383
"\n",
272-
"Given the endpoints above for finding collections and then finding data within those collections, try to craft a query by copying cells to find data from one of the other collections in the list above. Some things to try\n",
384+
"Some things to try:\n",
273385
"\n",
274386
"* Find data in the Unity system for the L0_SNPP_EphAtt product type\n",
275387
"* Find data in the Unity system for the L1 SounderSIPS \n",
276388
"* Filter the collections above on a numer\n",
277389
"\n"
278390
]
279391
},
392+
{
393+
"cell_type": "markdown",
394+
"id": "d79e544a",
395+
"metadata": {},
396+
"source": [
397+
"## Credential-less data download\n",
398+
"\n",
399+
"When accessing data stores within the same venue, you'll be able to download data from S3 without credentials. \n",
400+
"\n",
401+
"**Note**, the following libraries are needed for this, and the below command can be run in a jupyter-terminal to install them:\n",
402+
"\n",
403+
"```\n",
404+
"conda install xarray netcdf4 hdf5 boto3 matplotlib\n",
405+
"```\n"
406+
]
407+
},
280408
{
281409
"cell_type": "code",
282410
"execution_count": null,
@@ -295,8 +423,39 @@
295423
"outputs": [],
296424
"source": [
297425
"s3 = boto3.client('s3')\n",
298-
"s3.download_file('uds-test-cumulus-protected', 'ATMS_SCIENCE_Group___1/P1570515ATMSSCIENCEAAT16001044919900.PDS', 'P1570515ATMSSCIENCEAAT16001044919900.PDS')"
426+
"s3.download_file('uds-test-cumulus-protected', 'SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.04.nc', 'test_file11.nc')"
427+
]
428+
},
429+
{
430+
"cell_type": "code",
431+
"execution_count": null,
432+
"id": "c09f8626",
433+
"metadata": {},
434+
"outputs": [],
435+
"source": [
436+
"import xarray as xr\n",
437+
"ds = xr.open_dataset('test_file11.nc')\n",
438+
"ds"
439+
299440
]
441+
},
442+
{
443+
"cell_type": "code",
444+
"execution_count": null,
445+
"id": "fba4dff8",
446+
"metadata": {},
447+
"outputs": [],
448+
"source": [
449+
"ds.band_surf_alt.plot()"
450+
]
451+
},
452+
{
453+
"cell_type": "code",
454+
"execution_count": null,
455+
"id": "9386a4f1",
456+
"metadata": {},
457+
"outputs": [],
458+
"source": []
300459
}
301460
],
302461
"metadata": {

0 commit comments

Comments
 (0)