Jl/redshiftrail (#1086)

* multiple test train splits

* code for running sklearn as training method and for plotting

* using parquet files for making metrics now

* fixed import

---------

Co-authored-by: Jackson Loper <jaloper@umich.edu>
Co-authored-by: Jeffrey Regier <jeff-regier@users.noreply.github.com>

Author: 3 people

bliss/surveys/dc2.py

@@ -3,6 +3,7 @@
 import logging
 import multiprocessing
 import pathlib
+from pathlib import Path
 from typing import List
 
 import numpy as np
@@ -320,7 +321,18 @@ def read_image_for_bands(self, image_index):
 class DC2FullCatalog(FullCatalog):
     @classmethod
     def from_file(cls, cat_path, wcs, height, width, **kwargs):
-        catalog = pd.read_pickle(cat_path)
+        # load catalog from either a string path or a Path
+
+        cat_path = Path(cat_path)
+        suffix = cat_path.suffix.lower()
+
+        if suffix == ".parquet":
+            catalog = pd.read_parquet(cat_path)
+        elif suffix in (".pkl", ".pickle"):
+            catalog = pd.read_pickle(cat_path)
+        else:
+            raise ValueError(f"Unsupported catalog file format: {suffix}")
+
         flux_r_band = catalog["flux_r"].values
         catalog = catalog.loc[flux_r_band > kwargs["catalog_min_r_flux"]]
 

case_studies/dc2_cataloging/generate_catalog.ipynb

@@ -29,7 +29,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -53,20 +53,20 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
     "truth_catalog_pickle_file = output_dir / \"truth_catalog.pkl\"\n",
     "GCRCatalogs.set_root_dir(\"/data/scratch/dc2_nfs/\")\n",
     "if truth_catalog_pickle_file.exists():\n",
-    "    with open(truth_catalog_pickle_file, \"rb\") as inputp: \n",
+    "    with open(truth_catalog_pickle_file, \"rb\") as inputp:\n",
     "        truth_cat_data = pickle.load(inputp)\n",
-    "else:   \n",
-    "    truth_cat = GCRCatalogs.load_catalog(\"desc_dc2_run2.2i_dr6_truth\") \n",
+    "else:\n",
+    "    truth_cat = GCRCatalogs.load_catalog(\"desc_dc2_run2.2i_dr6_truth\")\n",
     "    print(sorted(truth_cat.list_all_quantities()))\n",
     "    truth_cat_data = truth_cat.get_quantities([\n",
-    "        \"id\", \"match_objectId\", \"cosmodc2_id\", \"ra\", \"dec\", \"truth_type\", \n",
+    "        \"id\", \"match_objectId\", \"cosmodc2_id\", \"ra\", \"dec\", \"truth_type\",\n",
     "        \"flux_g\", \"flux_i\", \"flux_r\", \"flux_u\", \"flux_y\", \"flux_z\",\n",
     "        \"redshift\",\n",
     "    ])\n",
@@ -95,7 +95,7 @@
    "source": [
     "galaxy_params_pickle_file = output_dir / \"galaxy_params.pkl\"\n",
     "if galaxy_params_pickle_file.exists():\n",
-    "    with open(galaxy_params_pickle_file, \"rb\") as inputp: \n",
+    "    with open(galaxy_params_pickle_file, \"rb\") as inputp:\n",
     "        galaxy_params_data = pickle.load(inputp)\n",
     "else:\n",
     "    config_overwrite = dict(\n",
@@ -243,20 +243,20 @@
    "source": [
     "psf_params_pickle_file = output_dir / \"psf_params.pkl\"\n",
     "if psf_params_pickle_file.exists():\n",
-    "    with open(psf_params_pickle_file, \"rb\") as inputp: \n",
+    "    with open(psf_params_pickle_file, \"rb\") as inputp:\n",
     "        psf_params_data = pickle.load(inputp)\n",
     "else:\n",
     "    psf_params = GCRCatalogs.load_catalog(\"desc_dc2_run2.2i_dr6_object_with_truth_match\")\n",
     "    print(sorted(psf_params.list_all_quantities()))\n",
     "    psf_params_data = psf_params.get_quantities([\n",
     "        \"objectId\",\n",
     "        \"blendedness\",\n",
-    "        \"IxxPSF_pixel_g\", \"IxxPSF_pixel_z\", \n",
-    "        \"IxxPSF_pixel_r\", \"IxxPSF_pixel_i\", \"IxxPSF_pixel_u\", \n",
-    "        \"IxxPSF_pixel_y\", \"IyyPSF_pixel_g\", \"IyyPSF_pixel_z\", \n",
-    "        \"IyyPSF_pixel_r\", \"IyyPSF_pixel_i\", \"IyyPSF_pixel_u\", \n",
-    "        \"IyyPSF_pixel_y\", \"IxyPSF_pixel_g\", \"IxyPSF_pixel_z\", \n",
-    "        \"IxyPSF_pixel_r\", \"IxyPSF_pixel_i\", \"IxyPSF_pixel_u\", \n",
+    "        \"IxxPSF_pixel_g\", \"IxxPSF_pixel_z\",\n",
+    "        \"IxxPSF_pixel_r\", \"IxxPSF_pixel_i\", \"IxxPSF_pixel_u\",\n",
+    "        \"IxxPSF_pixel_y\", \"IyyPSF_pixel_g\", \"IyyPSF_pixel_z\",\n",
+    "        \"IyyPSF_pixel_r\", \"IyyPSF_pixel_i\", \"IyyPSF_pixel_u\",\n",
+    "        \"IyyPSF_pixel_y\", \"IxyPSF_pixel_g\", \"IxyPSF_pixel_z\",\n",
+    "        \"IxyPSF_pixel_r\", \"IxyPSF_pixel_i\", \"IxyPSF_pixel_u\",\n",
     "        \"IxyPSF_pixel_y\", \"psf_fwhm_g\", \"psf_fwhm_z\", \"psf_fwhm_r\",\n",
     "        \"psf_fwhm_i\", \"psf_fwhm_u\", \"psf_fwhm_y\"\n",
     "    ])\n",
@@ -280,18 +280,18 @@
    "outputs": [],
    "source": [
     "truth_galaxy_df = truth_cat_df.merge(\n",
-    "    galaxy_params_df, \n",
-    "    left_on=\"cosmodc2_id\", \n",
-    "    right_on=\"galaxy_id\", \n",
-    "    how=\"left\" \n",
+    "    galaxy_params_df,\n",
+    "    left_on=\"cosmodc2_id\",\n",
+    "    right_on=\"galaxy_id\",\n",
+    "    how=\"left\"\n",
     ")\n",
     "cosmodc2_mask = ~(np.isnan(truth_galaxy_df[\"ellipticity_1_true\"].values))\n",
     "truth_galaxy_df[\"cosmodc2_mask\"] = cosmodc2_mask\n",
     "truth_galaxy_psf_df = truth_galaxy_df.merge(\n",
-    "    psf_params_df, \n",
-    "    left_on=\"match_objectId\", \n",
-    "    right_on=\"objectId\", \n",
-    "    how=\"left\" \n",
+    "    psf_params_df,\n",
+    "    left_on=\"match_objectId\",\n",
+    "    right_on=\"objectId\",\n",
+    "    how=\"left\"\n",
     ")"
    ]
   },
@@ -495,7 +495,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# create a pickle file \n",
+    "# create a pickle file\n",
     "truth_galaxy_psf_df.to_pickle(output_dir / f\"merged_catalog_with_flux_over_{flux_min}.pkl\")"
    ]
   }

case_studies/redshift/artifact_creation.yaml

@@ -0,0 +1,29 @@
+defaults:
+    - ../../bliss/conf@_here_: base_config
+    - _self_
+
+# Inherits defaults from `base_config`. We add some user-defined
+# paths to store artifacts.
+paths:
+  root: /home/declan/bliss
+  data_dir: /data/scratch/declan/redshift/dc2
+  dc2_cached: ${paths.data_dir}/cached_dc2
+  processed_data_dir_rail: ${paths.data_dir}/processed_dc2_rail
+  plots: ${paths.data_dir}/plots
+  rail_checkpoints: ${paths.data_dir}/checkpoints/rail
+  dc2: /nfs/turbo/lsa-regier/lsstdesc-public/dc2
+  plot_dir: ${paths.data_dir}/plots
+  cosmodc2: /data/scratch/dc2_nfs/cosmoDC2_v1.1.4/
+
+surveys:
+  dc2:
+      _target_: case_studies.redshift.artifacts.redshift_dc2.RedshiftDC2DataModule
+      cached_data_path: ${paths.dc2_cached}
+      dc2_cat_path: ${paths.processed_data_dir_bliss}/merged_catalog.parquet
+      batch_size: 64
+      max_sources_per_tile: 5
+
+# used for visualization
+visualization:
+  mag_bin_cutoffs: [23.9, 24.1, 24.5, 24.9, 25.6]
+  redshift_bin_cutoffs: [0.5, 1, 1.5, 2, 2.5, 3]