From bcf782a2cca0939aa43a5f9ac39bff6ea3302dde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Fri, 25 Jul 2025 13:17:15 -0700 Subject: [PATCH 01/20] ENH: auto convert to JB2 --- myst.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 myst.yml diff --git a/myst.yml b/myst.yml new file mode 100644 index 00000000..a3461246 --- /dev/null +++ b/myst.yml @@ -0,0 +1,15 @@ +# See docs at: https://mystmd.org/guide/frontmatter +version: 1 +project: + id: 808b80d9-c29d-4dc7-8c64-8836d6ea4095 + # title: + # description: + # keywords: [] + # authors: [] + # github: + # To autogenerate a Table of Contents, run "jupyter book init --write-toc" +site: + template: book-theme + # options: + # favicon: favicon.ico + # logo: site_logo.png From bb4a7f922186e434d1e277c67ae9f25d5a96daca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Fri, 25 Jul 2025 15:20:19 -0700 Subject: [PATCH 02/20] CI: switching to mystmd builds, update artifact redirect --- .circleci/config.yml | 62 +++++++++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 21 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bdf639e8..62f6403a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,46 +1,66 @@ version: 2.1 -jobs: +jobs: build-docs: docker: - - image: cimg/python:3.12 - + - image: cimg/python:3.13 steps: - - attach_workspace: - at: ~/ - - checkout - - run: - name: Install CI dependencies - command: python -m pip install --upgrade tox + - restore_cache: + keys: + - node-cache-v1 - restore_cache: keys: - - jupyter_ch + - pip-cache-{{ checksum "site-requirements.txt" }} + + - run: + name: Install Node.js + command: | + curl -fsSL https://deb.nodesource.com/setup_24.x | sudo -E bash - + sudo apt-get install -y nodejs - run: - name: Build HTML rendering of notebooks - no_output_timeout: 30m + name: Install mystmd command: | - python -m tox -e py312-buildhtml + npm install mystmd + + - run: + name: Install Python dependencies + command: | + python -m pip install --upgrade pip + if [ -f site-requirements.txt ]; then pip install -r site-requirements.txt; fi + + - save_cache: + key: node-cache-v1 + paths: + - node_modules - save_cache: - key: jupyter_ch + key: pip-cache-{{ checksum "site-requirements.txt" }} paths: - - _build/.jupyter_cache + - ~/.cache/pip + + - run: + name: Build documentation + environment: + # Ensure this is same as store_artifacts path below + DOCS_PATH: _build/html + command: | + export BASE_URL="/output/job/$CIRCLE_WORKFLOW_JOB_ID/artifacts/0/$DOCS_PATH" + + (npx myst build --html) + + ## Temporary hack to remove unused thebe JS + rm _build/html/*thebe*.js - store_artifacts: path: _build/html - - persist_to_workspace: - root: _build - paths: - - html - workflows: version: 2 - default: + build-and-docs: jobs: - build-docs From 8d06dccefe62a8ef80e95fa5f8bcd3180e7d8f5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Fri, 25 Jul 2025 15:23:58 -0700 Subject: [PATCH 03/20] CI: update filename --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 62f6403a..5a6e0dbe 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ jobs: - restore_cache: keys: - - pip-cache-{{ checksum "site-requirements.txt" }} + - pip-cache-{{ checksum "site_requirements.txt" }} - run: name: Install Node.js @@ -31,7 +31,7 @@ jobs: name: Install Python dependencies command: | python -m pip install --upgrade pip - if [ -f site-requirements.txt ]; then pip install -r site-requirements.txt; fi + if [ -f site_requirements.txt ]; then pip install -r site_requirements.txt; fi - save_cache: key: node-cache-v1 @@ -39,7 +39,7 @@ jobs: - node_modules - save_cache: - key: pip-cache-{{ checksum "site-requirements.txt" }} + key: pip-cache-{{ checksum "site_requirements.txt" }} paths: - ~/.cache/pip From 0d01f152b011e48f876c313e79b2d5d11d2be387 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Fri, 25 Jul 2025 17:10:22 -0700 Subject: [PATCH 04/20] MAINT: Add TOC structure to myst.yml and remove toctrees from index --- index.md | 91 +------------------------------------------------------- myst.yml | 54 +++++++++++++++++++++++++++++++-- 2 files changed, 52 insertions(+), 93 deletions(-) diff --git a/index.md b/index.md index 807a00bb..81d44991 100644 --- a/index.md +++ b/index.md @@ -7,82 +7,19 @@ They cover topics like querying IRSA, working with catalogs in Parquet format, v ## Accessing IRSA's on-premises holdings using VO protocols -```{toctree} ---- -maxdepth: 1 -caption: VO on-prem data access ---- -tutorials/irsa-sia-examples/sia_2mass_allsky -tutorials/irsa-sia-examples/sia_allwise_atlas -tutorials/irsa-sia-examples/sia_cosmos -tutorials/irsa-sia-examples/siav2_seip -tutorials/cosmodc2/cosmoDC2_TAP_access.md - -``` + ## Accessing IRSA's cloud holdings These notebooks demonstrate how to access the IRSA-curated datasets that available in Amazon Web Services (AWS) S3 cloud storage buckets. -```{toctree} ---- -maxdepth: 1 -caption: Cloud data access ---- - -tutorials/cloud_access/cloud-access-intro -tutorials/parquet-catalog-demos/wise-allwise-catalog-demo -tutorials/parquet-catalog-demos/neowise-source-table-strategies -tutorials/parquet-catalog-demos/neowise-source-table-lightcurves -tutorials/openuniversesims/openuniverse2024_roman_simulated_timedomainsurvey -tutorials/openuniversesims/openuniverse2024_roman_simulated_wideareasurvey -tutorials/cloud_access/euclid-cloud-access - -``` ## Accessing Euclid data -### Euclid Early Release Observation - -```{toctree} ---- -maxdepth: 1 -caption: Euclid Early Release Observations ---- - -tutorials/euclid_access/Euclid_ERO - -``` - -### Euclid Quick Release 1 - -```{toctree} ---- -maxdepth: 1 -caption: Euclid Quick Release 1 ---- - -tutorials/euclid_access/1_Euclid_intro_MER_images -tutorials/euclid_access/2_Euclid_intro_MER_catalog -tutorials/euclid_access/3_Euclid_intro_1D_spectra -tutorials/euclid_access/4_Euclid_intro_PHZ_catalog -tutorials/euclid_access/5_Euclid_intro_SPE_catalog -tutorials/cloud_access/euclid-cloud-access - -``` ## Accessing SPHEREx data -```{toctree} ---- -maxdepth: 1 -caption: SPHEREx Data ---- - -tutorials/spherex/spherex_intro - -``` ## Interactive visualization in Python with Firefly @@ -91,43 +28,17 @@ These notebooks demonstrate how to use the Firefly visualization tools from Pyth It is used in archive user interfaces at [IRSA](https://irsa.ipac.caltech.edu), the [NASA Exoplanet Science Institute (NExScI)](https://nexsci.caltech.edu/), the [NASA/IPAC Extragalactic Database (NED)](https://ned.ipac.caltech.edu/), and the [Vera C. Rubin Observatory](https://www.lsst.org/). -```{toctree} ---- -maxdepth: 1 -caption: Visualizations with Firefly ---- -tutorials/firefly/SEDs_in_Firefly -tutorials/firefly/NEOWISE_light_curve_demo -tutorials/firefly/OpenUniverse2024Preview_Firefly - -``` ## Simulated Data -```{toctree} ---- -maxdepth: 1 -caption: Simulated Data ---- - -tutorials/roman_simulations/roman_hlss_number_density.md -``` ## Generally useful techniques These notebooks cover miscellaneous topics that users might find useful in their analysis of IRSA-curated data. -```{toctree} ---- -maxdepth: 1 -caption: Generic techniques ---- - -tutorials/parallelize/Parallelize_Convolution -``` *** diff --git a/myst.yml b/myst.yml index a3461246..39c96091 100644 --- a/myst.yml +++ b/myst.yml @@ -8,8 +8,56 @@ project: # authors: [] # github: # To autogenerate a Table of Contents, run "jupyter book init --write-toc" + toc: + # Auto-generated by `myst init --write-toc` + - title: IRSA Tutorials + file: index.md + - title: VO on-prem data access + children: + - file: tutorials/irsa-sia-examples/sia_2mass_allsky.md + - file: tutorials/irsa-sia-examples/sia_allwise_atlas.md + - file: tutorials/irsa-sia-examples/sia_cosmos.md + - file: tutorials/irsa-sia-examples/siav2_seip.md + - file: tutorials/cosmodc2/cosmoDC2_TAP_access.md + - title: Cloud data accesst + children: + - file: tutorials/cloud_access/cloud-access-intro.md + - file: tutorials/parquet-catalog-demos/wise-allwise-catalog-demo.md + - file: tutorials/parquet-catalog-demos/neowise-source-table-strategies.md + - file: tutorials/parquet-catalog-demos/neowise-source-table-lightcurves.md + - file: tutorials/openuniversesims/openuniverse2024_roman_simulated_timedomainsurvey.md + - file: tutorials/openuniversesims/openuniverse2024_roman_simulated_wideareasurvey.md + - file: tutorials/cloud_access/euclid-cloud-access.md + - title: Euclid + children: + - title: Euclid Early Release Observations + children: + - file: tutorials/euclid_access/Euclid_ERO.md + - title: Euclid Quick Release 1 + children: + - file: tutorials/euclid_access/1_Euclid_intro_MER_images.md + - file: tutorials/euclid_access/2_Euclid_intro_MER_catalog.md + - file: tutorials/euclid_access/3_Euclid_intro_1D_spectra.md + - file: tutorials/euclid_access/4_Euclid_intro_PHZ_catalog.md + - file: tutorials/euclid_access/5_Euclid_intro_SPE_catalog.md + - file: tutorials/cloud_access/euclid-cloud-access.md + - title: SPHEREx + children: + - file: tutorials/spherex/spherex_intro.md + - title: Visualizations with Firefly + children: + - file: tutorials/firefly/SEDs_in_Firefly.md + - file: tutorials/firefly/NEOWISE_light_curve_demo.md + - file: tutorials/firefly/OpenUniverse2024Preview_Firefly.md + - title: Simulated Data + children: + - file: tutorials/roman_simulations/roman_hlss_number_density.md + - title: Generic techniques + children: + - file: tutorials/parallelize/Parallelize_Convolution.md + site: template: book-theme - # options: - # favicon: favicon.ico - # logo: site_logo.png + options: + favicon: _static/irsa-favicon.ico + logo: _static/irsa_logo.png From 5fed83161b89ed9d53b1ab2c8eae60b21374c6c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Fri, 25 Jul 2025 17:28:28 -0700 Subject: [PATCH 05/20] Adding main title and url --- myst.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/myst.yml b/myst.yml index 39c96091..963c3335 100644 --- a/myst.yml +++ b/myst.yml @@ -1,15 +1,12 @@ # See docs at: https://mystmd.org/guide/frontmatter version: 1 project: - id: 808b80d9-c29d-4dc7-8c64-8836d6ea4095 - # title: + title: IRSA Tutorials # description: # keywords: [] # authors: [] - # github: - # To autogenerate a Table of Contents, run "jupyter book init --write-toc" + github: https://github.com/Caltech-IPAC/irsa-tutorials/ toc: - # Auto-generated by `myst init --write-toc` - title: IRSA Tutorials file: index.md - title: VO on-prem data access @@ -19,7 +16,7 @@ project: - file: tutorials/irsa-sia-examples/sia_cosmos.md - file: tutorials/irsa-sia-examples/siav2_seip.md - file: tutorials/cosmodc2/cosmoDC2_TAP_access.md - - title: Cloud data accesst + - title: Cloud data access children: - file: tutorials/cloud_access/cloud-access-intro.md - file: tutorials/parquet-catalog-demos/wise-allwise-catalog-demo.md From 22b5d6a7a68209cb07ccc4ada66d2ade8eb011a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Fri, 25 Jul 2025 17:52:32 -0700 Subject: [PATCH 06/20] ENH: gutting out landing page, adding some more metadata --- index.md | 37 +------------------------------------ myst.yml | 6 +++--- 2 files changed, 4 insertions(+), 39 deletions(-) diff --git a/index.md b/index.md index 81d44991..e4dea84d 100644 --- a/index.md +++ b/index.md @@ -1,47 +1,12 @@ -# Caltech/IPAC--IRSA Python Notebook Tutorials +# Caltech/IPAC—IRSA Python Notebook Tutorials These Python Jupyter Notebook tutorials demonstrate access methods and techniques for working with data served by the [NASA/IPAC Infrared Science Archive (IRSA)](https://irsa.ipac.caltech.edu). They cover topics like querying IRSA, working with catalogs in Parquet format, visualizing with Firefly, and general other techniques. -## Accessing IRSA's on-premises holdings using VO protocols - -## Accessing IRSA's cloud holdings - -These notebooks demonstrate how to access the IRSA-curated datasets that available in Amazon Web Services (AWS) S3 cloud storage buckets. - - - -## Accessing Euclid data - - -## Accessing SPHEREx data - - -## Interactive visualization in Python with Firefly - -These notebooks demonstrate how to use the Firefly visualization tools from Python. -[Firefly](https://github.com/Caltech-IPAC/firefly) is an open-source toolkit based on IVOA standards and designed to enable astronomical data archive access, exploratory data analysis, and visualization. - -It is used in archive user interfaces at [IRSA](https://irsa.ipac.caltech.edu), the [NASA Exoplanet Science Institute (NExScI)](https://nexsci.caltech.edu/), the [NASA/IPAC Extragalactic Database (NED)](https://ned.ipac.caltech.edu/), and the [Vera C. Rubin Observatory](https://www.lsst.org/). - - - -## Simulated Data - - - -## Generally useful techniques - -These notebooks cover miscellaneous topics that users might find useful in their analysis of IRSA-curated data. - - - -*** - ## About these notebooks **Authors:** IRSA Scientists and Developers wrote and maintain these notebooks. diff --git a/myst.yml b/myst.yml index 963c3335..3243f32e 100644 --- a/myst.yml +++ b/myst.yml @@ -2,9 +2,9 @@ version: 1 project: title: IRSA Tutorials - # description: - # keywords: [] - # authors: [] + description: IRSA Python Notebook tutorials to demostrate accessing data served by the NASA/IPAC Infrared Science Archive (IRSA) + keywords: [astronomy] + authors: [IRSA Scientists and Developers] github: https://github.com/Caltech-IPAC/irsa-tutorials/ toc: - title: IRSA Tutorials From 5b2e2fea9f0e99332ac09b5a31b1c8af96e452d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Fri, 25 Jul 2025 21:39:55 -0700 Subject: [PATCH 07/20] DOC: add subject --- myst.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/myst.yml b/myst.yml index 3243f32e..348231c8 100644 --- a/myst.yml +++ b/myst.yml @@ -2,6 +2,7 @@ version: 1 project: title: IRSA Tutorials + subject: IRSA Tutorials description: IRSA Python Notebook tutorials to demostrate accessing data served by the NASA/IPAC Infrared Science Archive (IRSA) keywords: [astronomy] authors: [IRSA Scientists and Developers] From 23959df9af1c2b26b39dd272c81d716449adf8d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Wed, 30 Jul 2025 10:17:13 -0700 Subject: [PATCH 08/20] MAINT: fix older of metadata keywords to make it easier to sed --- tutorials/spherex/spherex_intro.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutorials/spherex/spherex_intro.md b/tutorials/spherex/spherex_intro.md index ed3caeea..a16c99d4 100644 --- a/tutorials/spherex/spherex_intro.md +++ b/tutorials/spherex/spherex_intro.md @@ -7,9 +7,9 @@ jupytext: format_version: 0.13 jupytext_version: 1.17.2 kernelspec: - name: python3 display_name: Python 3 (ipykernel) language: python + name: python3 --- # Introduction to SPHEREx Spectral Images From 120988e8ff92d5576085e3d38a8e9f8cda4132e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Fri, 8 Aug 2025 11:52:19 -0700 Subject: [PATCH 09/20] ENH: separating out TOC --- myst.yml | 49 ++----------------------------------------------- toc.yml | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 47 deletions(-) create mode 100644 toc.yml diff --git a/myst.yml b/myst.yml index 348231c8..4df49601 100644 --- a/myst.yml +++ b/myst.yml @@ -7,53 +7,8 @@ project: keywords: [astronomy] authors: [IRSA Scientists and Developers] github: https://github.com/Caltech-IPAC/irsa-tutorials/ - toc: - - title: IRSA Tutorials - file: index.md - - title: VO on-prem data access - children: - - file: tutorials/irsa-sia-examples/sia_2mass_allsky.md - - file: tutorials/irsa-sia-examples/sia_allwise_atlas.md - - file: tutorials/irsa-sia-examples/sia_cosmos.md - - file: tutorials/irsa-sia-examples/siav2_seip.md - - file: tutorials/cosmodc2/cosmoDC2_TAP_access.md - - title: Cloud data access - children: - - file: tutorials/cloud_access/cloud-access-intro.md - - file: tutorials/parquet-catalog-demos/wise-allwise-catalog-demo.md - - file: tutorials/parquet-catalog-demos/neowise-source-table-strategies.md - - file: tutorials/parquet-catalog-demos/neowise-source-table-lightcurves.md - - file: tutorials/openuniversesims/openuniverse2024_roman_simulated_timedomainsurvey.md - - file: tutorials/openuniversesims/openuniverse2024_roman_simulated_wideareasurvey.md - - file: tutorials/cloud_access/euclid-cloud-access.md - - title: Euclid - children: - - title: Euclid Early Release Observations - children: - - file: tutorials/euclid_access/Euclid_ERO.md - - title: Euclid Quick Release 1 - children: - - file: tutorials/euclid_access/1_Euclid_intro_MER_images.md - - file: tutorials/euclid_access/2_Euclid_intro_MER_catalog.md - - file: tutorials/euclid_access/3_Euclid_intro_1D_spectra.md - - file: tutorials/euclid_access/4_Euclid_intro_PHZ_catalog.md - - file: tutorials/euclid_access/5_Euclid_intro_SPE_catalog.md - - file: tutorials/cloud_access/euclid-cloud-access.md - - title: SPHEREx - children: - - file: tutorials/spherex/spherex_intro.md - - title: Visualizations with Firefly - children: - - file: tutorials/firefly/SEDs_in_Firefly.md - - file: tutorials/firefly/NEOWISE_light_curve_demo.md - - file: tutorials/firefly/OpenUniverse2024Preview_Firefly.md - - title: Simulated Data - children: - - file: tutorials/roman_simulations/roman_hlss_number_density.md - - title: Generic techniques - children: - - file: tutorials/parallelize/Parallelize_Convolution.md - +extends: + - toc.yml site: template: book-theme options: diff --git a/toc.yml b/toc.yml new file mode 100644 index 00000000..b9bc4e01 --- /dev/null +++ b/toc.yml @@ -0,0 +1,48 @@ +version: 1 +project: + toc: + - title: IRSA Tutorials + file: index.md + - title: VO on-prem data access + children: + - file: tutorials/irsa-sia-examples/sia_2mass_allsky.md + - file: tutorials/irsa-sia-examples/sia_allwise_atlas.md + - file: tutorials/irsa-sia-examples/sia_cosmos.md + - file: tutorials/irsa-sia-examples/siav2_seip.md + - file: tutorials/cosmodc2/cosmoDC2_TAP_access.md + - title: Cloud data access + children: + - file: tutorials/cloud_access/cloud-access-intro.md + - file: tutorials/parquet-catalog-demos/wise-allwise-catalog-demo.md + - file: tutorials/parquet-catalog-demos/neowise-source-table-strategies.md + - file: tutorials/parquet-catalog-demos/neowise-source-table-lightcurves.md + - file: tutorials/openuniversesims/openuniverse2024_roman_simulated_timedomainsurvey.md + - file: tutorials/openuniversesims/openuniverse2024_roman_simulated_wideareasurvey.md + - file: tutorials/cloud_access/euclid-cloud-access.md + - title: Euclid + children: + - title: Euclid Early Release Observations + children: + - file: tutorials/euclid_access/Euclid_ERO.md + - title: Euclid Quick Release 1 + children: + - file: tutorials/euclid_access/1_Euclid_intro_MER_images.md + - file: tutorials/euclid_access/2_Euclid_intro_MER_catalog.md + - file: tutorials/euclid_access/3_Euclid_intro_1D_spectra.md + - file: tutorials/euclid_access/4_Euclid_intro_PHZ_catalog.md + - file: tutorials/euclid_access/5_Euclid_intro_SPE_catalog.md + - file: tutorials/cloud_access/euclid-cloud-access.md + - title: SPHEREx + children: + - file: tutorials/spherex/spherex_intro.md + - title: Visualizations with Firefly + children: + - file: tutorials/firefly/SEDs_in_Firefly.md + - file: tutorials/firefly/NEOWISE_light_curve_demo.md + - file: tutorials/firefly/OpenUniverse2024Preview_Firefly.md + - title: Simulated Data + children: + - file: tutorials/roman_simulations/roman_hlss_number_density.md + - title: Generic techniques + children: + - file: tutorials/parallelize/Parallelize_Convolution.md From c61cce83184326859413de3541f9d957c8e45e25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Fri, 8 Aug 2025 12:04:55 -0700 Subject: [PATCH 10/20] ENH: restructure TOC --- toc.yml | 59 +++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 23 deletions(-) diff --git a/toc.yml b/toc.yml index b9bc4e01..cac05583 100644 --- a/toc.yml +++ b/toc.yml @@ -3,22 +3,9 @@ project: toc: - title: IRSA Tutorials file: index.md - - title: VO on-prem data access - children: - - file: tutorials/irsa-sia-examples/sia_2mass_allsky.md - - file: tutorials/irsa-sia-examples/sia_allwise_atlas.md - - file: tutorials/irsa-sia-examples/sia_cosmos.md - - file: tutorials/irsa-sia-examples/siav2_seip.md - - file: tutorials/cosmodc2/cosmoDC2_TAP_access.md - - title: Cloud data access + - title: SPHEREx children: - - file: tutorials/cloud_access/cloud-access-intro.md - - file: tutorials/parquet-catalog-demos/wise-allwise-catalog-demo.md - - file: tutorials/parquet-catalog-demos/neowise-source-table-strategies.md - - file: tutorials/parquet-catalog-demos/neowise-source-table-lightcurves.md - - file: tutorials/openuniversesims/openuniverse2024_roman_simulated_timedomainsurvey.md - - file: tutorials/openuniversesims/openuniverse2024_roman_simulated_wideareasurvey.md - - file: tutorials/cloud_access/euclid-cloud-access.md + - file: tutorials/spherex/spherex_intro.md - title: Euclid children: - title: Euclid Early Release Observations @@ -32,17 +19,43 @@ project: - file: tutorials/euclid_access/4_Euclid_intro_PHZ_catalog.md - file: tutorials/euclid_access/5_Euclid_intro_SPE_catalog.md - file: tutorials/cloud_access/euclid-cloud-access.md - - title: SPHEREx + - title: WISE children: - - file: tutorials/spherex/spherex_intro.md - - title: Visualizations with Firefly + - title: AllWISE + children: + - file: tutorials/irsa-sia-examples/sia_allwise_atlas.md + - file: tutorials/parquet-catalog-demos/wise-allwise-catalog-demo.md + - title: NEOWISE + children: + - file: tutorials/firefly/NEOWISE_light_curve_demo.md + - file: tutorials/parquet-catalog-demos/neowise-source-table-strategies.md + - file: tutorials/parquet-catalog-demos/neowise-source-table-lightcurves.md + - title: Spitzer children: - - file: tutorials/firefly/SEDs_in_Firefly.md - - file: tutorials/firefly/NEOWISE_light_curve_demo.md - - file: tutorials/firefly/OpenUniverse2024Preview_Firefly.md + - title: Spitzer Enhanced Imaging Products (SEIP) + children: + - file: tutorials/irsa-sia-examples/siav2_seip.md + - title: 2MASS + children: + - file: tutorials/irsa-sia-examples/sia_2mass_allsky.md + - title: COSMOS + children: + - file: tutorials/irsa-sia-examples/sia_cosmos.md - title: Simulated Data children: - - file: tutorials/roman_simulations/roman_hlss_number_density.md - - title: Generic techniques + - title: Roman HLSS Galaxy Mock Catalog + children: + - file: tutorials/roman_simulations/roman_hlss_number_density.md + - title: OpenUniverse 2024 + children: + - file: tutorials/openuniversesims/openuniverse2024_roman_simulated_wideareasurvey.md + - file: tutorials/firefly/OpenUniverse2024Preview_Firefly.md + - file: tutorials/openuniversesims/openuniverse2024_roman_simulated_timedomainsurvey.md + - title: CosmoDC2 + children: + - file: tutorials/cosmodc2/cosmoDC2_TAP_access.md + - title: Special Topics children: + - file: tutorials/cloud_access/cloud-access-intro.md + - file: tutorials/firefly/SEDs_in_Firefly.md - file: tutorials/parallelize/Parallelize_Convolution.md From 5edebc37cbd24efc085670b238e65640d8035ebe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Mon, 25 Aug 2025 21:56:14 -0700 Subject: [PATCH 11/20] MAINT: adding myst build to tox --- .circleci/config.yml | 16 ++++++---------- tox.ini | 11 +++++------ 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5a6e0dbe..1401fe21 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,19 +30,13 @@ jobs: - run: name: Install Python dependencies command: | - python -m pip install --upgrade pip - if [ -f site_requirements.txt ]; then pip install -r site_requirements.txt; fi + python -m pip install --upgrade pip tox - save_cache: key: node-cache-v1 paths: - node_modules - - save_cache: - key: pip-cache-{{ checksum "site_requirements.txt" }} - paths: - - ~/.cache/pip - - run: name: Build documentation environment: @@ -51,10 +45,12 @@ jobs: command: | export BASE_URL="/output/job/$CIRCLE_WORKFLOW_JOB_ID/artifacts/0/$DOCS_PATH" - (npx myst build --html) + python -m tox -e py312-buildhtml - ## Temporary hack to remove unused thebe JS - rm _build/html/*thebe*.js + - save_cache: + key: pip-cache-{{ checksum "site_requirements.txt" }} + paths: + - ~/.cache/pip - store_artifacts: path: _build/html diff --git a/tox.ini b/tox.ini index f84e03ce..5eca0555 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,7 @@ setenv = FIREFLY_URL = https://irsa.ipac.caltech.edu/irsaviewer # fallback url for firefly notebooks if not defined in code devdeps: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/scientific-python-nightly-wheels/simple https://pypi.anaconda.org/astropy/simple -passenv = CI, CIRCLECI, GITHUB_EVENT_NAME, GITHUB_EVENT_PATH +passenv = CI, CIRCLECI, GITHUB_EVENT_NAME, GITHUB_EVENT_PATH, BASE_URL deps = # We use these files to specify all the dependencies, and below we override @@ -38,7 +38,7 @@ deps = devdeps: git+https://github.com/astropy/pyvo.git#egg=pyvo devdeps: git+https://github.com/astropy/astroquery.git#egg=astroquery -allowlist_externals = bash, sed +allowlist_externals = bash, sed, npx commands = pip freeze @@ -54,10 +54,9 @@ commands = !buildhtml: bash -c 'if [[ $GITHUB_EVENT_NAME == pull_request && -z "$(grep force_run:all_tests ${GITHUB_EVENT_PATH})" ]]; then git fetch origin main --depth=1; git diff origin/main --name-only tutorials | grep .md; else find tutorials -name "*.md"; fi | grep -vf ignore_testing | xargs jupytext --to notebook ' !buildhtml: pytest --nbval-lax -vv --suppress-no-test-exit-code --durations=10 tutorials - buildhtml: sphinx-build -b html . _build/html -nWT --keep-going - # SED magic to remove the toctree captions from the rendered index page while keeping them in the sidebar TOC - buildhtml: sed -E -i.bak '/caption-text/{N; s/.+caption-text.+\n
    /
      /; P;D;}' _build/html/index.html - buildhtml: bash -c 'rm _build/html/index.html.bak' + + buildhtml: npx myst build --execute + buildhtml: bash rm _build/html/*thebe*.js pip_pre = predeps: true From b87ed3c73cbbd363665563908cd188f14e35a87b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Mon, 25 Aug 2025 22:30:48 -0700 Subject: [PATCH 12/20] CI: creating ignore_circleci_testing file from existing conf.py option --- ignore_circleci_testing | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 ignore_circleci_testing diff --git a/ignore_circleci_testing b/ignore_circleci_testing new file mode 100644 index 00000000..09605002 --- /dev/null +++ b/ignore_circleci_testing @@ -0,0 +1,2 @@ +tutorials/parallelize/Parallelize_Convolution +tutorials/parquet-catalog-demos/wise-allwise-catalog-demo From 15fe884508e73f868634de7e918cb5e81539a497 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Mon, 25 Aug 2025 22:42:33 -0700 Subject: [PATCH 13/20] Adding ignore logic to all tox envs --- tox.ini | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index 5eca0555..2ba00513 100644 --- a/tox.ini +++ b/tox.ini @@ -45,16 +45,17 @@ commands = # Ignore testing the tutorials listed in ignore_testing file. We have some OS specific ignores, # too due to issues with e.g. multiprocessing and problems in upstream dependency - !buildhtml: bash -c 'if python -c "import platform; print(platform.platform())" | grep -i macos; then cat ignore_osx_testing >> ignore_testing; fi' - !buildhtml: bash -c 'if python -c "import platform; print(platform.platform())" | grep -i win; then cat ignore_windows_testing >> ignore_testing; fi' - - !buildhtml: bash -c 'if [[ $CI == true ]]; then cat ignore_gha_testing >> ignore_testing; fi' + bash -c 'if python -c "import platform; print(platform.platform())" | grep -i macos; then cat ignore_osx_testing >> ignore_testing; fi' + bash -c 'if python -c "import platform; print(platform.platform())" | grep -i win; then cat ignore_windows_testing >> ignore_testing; fi' + bash -c 'if [[ $CI == true ]]; then cat ignore_gha_testing >> ignore_testing; fi' + bash -c 'if [[ $CIRCLECI == true ]]; then cat ignore_circleci_testing >> ignore_testing; fi' # We only want to run CI in PRs for the notebooks we touched !buildhtml: bash -c 'if [[ $GITHUB_EVENT_NAME == pull_request && -z "$(grep force_run:all_tests ${GITHUB_EVENT_PATH})" ]]; then git fetch origin main --depth=1; git diff origin/main --name-only tutorials | grep .md; else find tutorials -name "*.md"; fi | grep -vf ignore_testing | xargs jupytext --to notebook ' !buildhtml: pytest --nbval-lax -vv --suppress-no-test-exit-code --durations=10 tutorials + buildhtml: bash -c "find tutorials -name '*md' | grep -f ignore_testing | sort | uniq | xargs -n 1 sed -i '' -e 's|name: python3|name: python3\nskip_execution: true|g'" buildhtml: npx myst build --execute buildhtml: bash rm _build/html/*thebe*.js From 96f9ac921cefe632723e017784f5c95c023f08c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Tue, 26 Aug 2025 22:43:33 -0700 Subject: [PATCH 14/20] Fixing env name for circleci --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1401fe21..4e44c9b0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,7 +45,7 @@ jobs: command: | export BASE_URL="/output/job/$CIRCLE_WORKFLOW_JOB_ID/artifacts/0/$DOCS_PATH" - python -m tox -e py312-buildhtml + python -m tox -e py313-buildhtml - save_cache: key: pip-cache-{{ checksum "site_requirements.txt" }} From 2af06aed763ed6ac4093411d39a7c62cc57d5c7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Tue, 26 Aug 2025 23:44:38 -0700 Subject: [PATCH 15/20] Fix the sed -i logic for ubuntu as it doesn't take an empty extension --- tox.ini | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 2ba00513..280e1a86 100644 --- a/tox.ini +++ b/tox.ini @@ -38,7 +38,7 @@ deps = devdeps: git+https://github.com/astropy/pyvo.git#egg=pyvo devdeps: git+https://github.com/astropy/astroquery.git#egg=astroquery -allowlist_externals = bash, sed, npx +allowlist_externals = bash, npx commands = pip freeze @@ -55,7 +55,9 @@ commands = !buildhtml: pytest --nbval-lax -vv --suppress-no-test-exit-code --durations=10 tutorials - buildhtml: bash -c "find tutorials -name '*md' | grep -f ignore_testing | sort | uniq | xargs -n 1 sed -i '' -e 's|name: python3|name: python3\nskip_execution: true|g'" + # sed -i needs a bit of hacky conditional on ubuntu to cover the case of an empty ignore + buildhtml: bash -c "find tutorials -name '*md' | grep -f ignore_testing | sort | uniq > ignore_execute; if [ -s ignore_execute ]; then cat ignore_execute | xargs -n 1 sed -i -e 's|name: python3|name: python3\nskip_execution: true|g';fi" + buildhtml: npx myst build --execute buildhtml: bash rm _build/html/*thebe*.js From 1ae96d69a542f546ef52db2af43d5140555f0558 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Wed, 27 Aug 2025 00:02:45 -0700 Subject: [PATCH 16/20] CI: adding back no output timeout --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4e44c9b0..ff6f3508 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,6 +39,7 @@ jobs: - run: name: Build documentation + no_output_timeout: 30m environment: # Ensure this is same as store_artifacts path below DOCS_PATH: _build/html From cc4c1dd698c44d356ef55d9052928accdda4a942 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Thu, 28 Aug 2025 06:44:38 -0700 Subject: [PATCH 17/20] CI: even larger timeout --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ff6f3508..843c0be0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,7 +39,7 @@ jobs: - run: name: Build documentation - no_output_timeout: 30m + no_output_timeout: 60m environment: # Ensure this is same as store_artifacts path below DOCS_PATH: _build/html From 9119cd024d11db8e0ff23e246d73d7bb8166be30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Thu, 28 Aug 2025 16:59:14 -0700 Subject: [PATCH 18/20] CI: don't need to cleanup files that are not generated --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index 280e1a86..570d239b 100644 --- a/tox.ini +++ b/tox.ini @@ -59,7 +59,6 @@ commands = buildhtml: bash -c "find tutorials -name '*md' | grep -f ignore_testing | sort | uniq > ignore_execute; if [ -s ignore_execute ]; then cat ignore_execute | xargs -n 1 sed -i -e 's|name: python3|name: python3\nskip_execution: true|g';fi" buildhtml: npx myst build --execute - buildhtml: bash rm _build/html/*thebe*.js pip_pre = predeps: true From 2f4a29b38c884326d697ce2d1345b9ad12bb766a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Thu, 28 Aug 2025 17:11:24 -0700 Subject: [PATCH 19/20] CI: downgrading node --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 843c0be0..603fdd12 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,7 +19,7 @@ jobs: - run: name: Install Node.js command: | - curl -fsSL https://deb.nodesource.com/setup_24.x | sudo -E bash - + curl -fsSL https://deb.nodesource.com/setup_22.x | sudo -E bash - sudo apt-get install -y nodejs - run: From 3c8007475ab90a68be5d4396dc8135abec8c36a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Thu, 28 Aug 2025 21:05:14 -0700 Subject: [PATCH 20/20] =?UTF-8?q?Adding=20--html=20=F0=9F=A4=A6=E2=80=8D?= =?UTF-8?q?=E2=99=80=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 570d239b..0fb91807 100644 --- a/tox.ini +++ b/tox.ini @@ -58,7 +58,7 @@ commands = # sed -i needs a bit of hacky conditional on ubuntu to cover the case of an empty ignore buildhtml: bash -c "find tutorials -name '*md' | grep -f ignore_testing | sort | uniq > ignore_execute; if [ -s ignore_execute ]; then cat ignore_execute | xargs -n 1 sed -i -e 's|name: python3|name: python3\nskip_execution: true|g';fi" - buildhtml: npx myst build --execute + buildhtml: npx myst build --execute --html pip_pre = predeps: true