From ec8e772b4f83b7695443eff02e3f00d24f838159 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Thu, 31 Jul 2025 15:31:52 -0400 Subject: [PATCH 001/132] add nexus --- sorc/build_all.sh | 8 +++++--- sorc/build_nexus.sh | 31 +++++++++++++++++++++++++++++++ sorc/link_workflow.sh | 22 ++++++++++++++++++++++ 3 files changed, 58 insertions(+), 3 deletions(-) create mode 100755 sorc/build_nexus.sh diff --git a/sorc/build_all.sh b/sorc/build_all.sh index f0bd7f425ef..0d1e38299d8 100755 --- a/sorc/build_all.sh +++ b/sorc/build_all.sh @@ -84,10 +84,10 @@ system_builds=( ["gfs"]="ufs_gfs gfs_utils ufs_utils upp ww3_gfs" ["gefs"]="ufs_gefs gfs_utils ufs_utils upp ww3_gefs" ["sfs"]="ufs_sfs gfs_utils ufs_utils upp ww3_gefs" - ["gcafs"]="ufs_gcafs gfs_utils ufs_utils upp" + ["gcafs"]="ufs_gcafs gfs_utils ufs_utils upp nexus gsi_utils" ["gsi"]="gsi_enkf gsi_monitor gsi_utils" ["gdas"]="gdas gsi_monitor gsi_utils" - ["all"]="ufs_gfs gfs_utils ufs_utils upp ww3_gfs ufs_gefs ufs_sfs ufs_gcafs ww3_gefs gdas gsi_enkf gsi_monitor gsi_utils" + ["all"]="ufs_gfs gfs_utils ufs_utils upp ww3_gfs ufs_gefs ufs_sfs ufs_gcafs ww3_gefs gdas gsi_enkf gsi_monitor gsi_utils nexus" ) logs_dir="${HOMEgfs}/sorc/logs" @@ -100,7 +100,7 @@ fi declare -A build_jobs build_opts build_scripts build_jobs=( ["ufs_gfs"]=8 ["ufs_gefs"]=8 ["ufs_sfs"]=8 ["ufs_gcafs"]=8 ["gdas"]=8 ["gsi_enkf"]=2 ["gfs_utils"]=1 ["ufs_utils"]=1 - ["ww3_gfs"]=1 ["ww3_gefs"]=1 ["gsi_utils"]=1 ["gsi_monitor"]=1 ["gfs_utils"]=1 ["upp"]=1 + ["ww3_gfs"]=1 ["ww3_gefs"]=1 ["gsi_utils"]=1 ["gsi_monitor"]=1 ["gfs_utils"]=1 ["upp"]=1 ["nexus"]=1 ) # Establish build options for each job @@ -122,6 +122,7 @@ build_opts=( ["gsi_utils"]="${_verbose_opt} ${_build_debug}" ["gsi_enkf"]="${_verbose_opt} ${_build_debug}" ["gsi_monitor"]="${_verbose_opt} ${_build_debug}" + ["nexus"]="${_verbose_opt} ${_build_debug}" ) # Set the build script name for each build @@ -140,6 +141,7 @@ build_scripts=( ["gsi_monitor"]="build_gsi_monitor.sh" ["gfs_utils"]="build_gfs_utils.sh" ["upp"]="build_upp.sh" + ["nexus"]="build_nexus.sh" ) # Check the requested systems to make sure we can build them diff --git a/sorc/build_nexus.sh b/sorc/build_nexus.sh new file mode 100755 index 00000000000..ea79fa14093 --- /dev/null +++ b/sorc/build_nexus.sh @@ -0,0 +1,31 @@ +#! /usr/bin/env bash +set -eux + +# shellcheck disable=SC2155 +readonly HOMEgfs_=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )/.." && pwd -P) + +OPTIND=1 +_opts="-f " # forces a clean build +while getopts ":j:dv" option; do + case "${option}" in + d) _opts+="-c -DCMAKE_BUILD_TYPE=Debug " ;; + j) BUILD_JOBS=${OPTARG};; + v) _opts+="-v ";; + :) + echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" + usage + ;; + *) + echo "[${BASH_SOURCE[0]}]: Unrecognized option: ${option}" + usage + ;; + esac +done +shift $((OPTIND-1)) + +# double quoting opts will not work since it is a string of options +# shellcheck disable=SC2086 +BUILD_JOBS="${BUILD_JOBS:-1}" \ +./nexus.fd/build.sh ${_opts} -f -w ${HOMEgfs_} + +exit diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 12054acfa7f..40e85523e91 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -287,6 +287,23 @@ if [[ -d "${HOMEgfs}/sorc/gsi_monitor.fd" ]]; then # ${LINK_OR_COPY} "${HOMEgfs}/sorc/gsi_monitor.fd/src/Radiance_Monitor/nwprod/gdas_radmon/parm/gdas_radmon.parm" . fi +#------------------------------ +#--add NEXUS files +#------------------------------ +if [[ -d "${HOMEgfs}/sorc/nexus.fd" ]]; then + cd "${HOMEgfs}/parm/chem" || exit 1 + if [[ -d nexus ]]; then + rm -rf nexus + fi + mkdir -p nexus/gocart + cd nexus/gocart || exit 1 + ${LINK_OR_COPY} "${HOMEgfs}/sorc/nexus.fd/config/gocart/NEXUS_Config.rc.j2" . + ${LINK_OR_COPY} "${HOMEgfs}/sorc/nexus.fd/config/gocart/HEMCO_sa_Grid.rc.j2" . + ${LINK_OR_COPY} "${HOMEgfs}/sorc/nexus.fd/config/gocart/HEMCO_sa_Time.rc.j2" . + ${LINK_OR_COPY} "${HOMEgfs}/sorc/nexus.fd/config/gocart/HEMCO_sa_Diag.rc.j2" . + ${LINK_OR_COPY} "${HOMEgfs}/sorc/nexus.fd/config/gocart/HEMCO_sa_Spec.rc.j2" . +fi + #------------------------------ #--link executables #------------------------------ @@ -396,6 +413,11 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd/install" ]]; then cp -af "${HOMEgfs}/sorc/gdas.cd/install/lib/." ./ fi +# NEXUS executable +if [[ -d "${HOMEgfs}/sorc/nexus.fd/build/bin" ]]; then + ${LINK_OR_COPY} "${HOMEgfs}/sorc/nexus.fd/build/bin/nexus" nexus.x +fi + #------------------------------ #--link source code directories #------------------------------ From d064d3a93604d97f22e3526872161ab37e2dabab Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Thu, 31 Jul 2025 15:33:36 -0400 Subject: [PATCH 002/132] add fire and nxs configs to config.aero --- .gitmodules | 4 ++ dev/jobs/prep_emissions.sh | 11 ++++ dev/parm/config/gcafs/config.aero.j2 | 82 ++++++++++++++++++++++++++-- 3 files changed, 93 insertions(+), 4 deletions(-) diff --git a/.gitmodules b/.gitmodules index c80d24c03aa..df696048db3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -25,3 +25,7 @@ [submodule "sorc/gsi_monitor.fd"] path = sorc/gsi_monitor.fd url = https://github.com/NOAA-EMC/GSI-Monitor.git +[submodule "sorc/nexus.fd"] + path = sorc/nexus.fd + url = https://github.com/NOAA-OAR-ARL/NEXUS.git + branch = feature/gcafs diff --git a/dev/jobs/prep_emissions.sh b/dev/jobs/prep_emissions.sh index 57e1c9d9c7c..32886764eb3 100755 --- a/dev/jobs/prep_emissions.sh +++ b/dev/jobs/prep_emissions.sh @@ -11,6 +11,17 @@ status=$? export job="prep_emissions" export jobid="${job}.$$" +############################################################### +# Source relevant configs +configs="base aero prep_emissions" +for config in ${configs}; do + source "${EXPDIR}/config.${config}" + status=$? + if [[ ${status} -ne 0 ]]; then + exit "${status}" + fi +done + ############################################################### # Execute the JJOB "${HOMEgfs}/jobs/JGLOBAL_PREP_EMISSIONS" diff --git a/dev/parm/config/gcafs/config.aero.j2 b/dev/parm/config/gcafs/config.aero.j2 index 719e100525f..d3a554f32db 100644 --- a/dev/parm/config/gcafs/config.aero.j2 +++ b/dev/parm/config/gcafs/config.aero.j2 @@ -2,14 +2,18 @@ # UFS-Aerosols settings -# Path to the input data tree +#================================================================================ +# 1. Aerosol settings +#================================================================================ export AERO_INPUTS_DIR="{{ AERO_INPUTS_DIR }}" +#----------------------------------------------- +# Diag Table and Field Table for GOCART aerosols +#----------------------------------------------- export AERO_DIAG_TABLE="${PARMgfs}/ufs/fv3/diag_table.aero" export AERO_FIELD_TABLE="${PARMgfs}/ufs/fv3/field_table.aero" -# Biomass burning emission dataset. Choose from: gbbepx, qfed, none -export AERO_EMIS_FIRE="qfed" -# Directory containing GOCART configuration files + +# Aerosol configuration export AERO_CONFIG_DIR="${PARMgfs}/ufs/gocart" # Aerosol convective scavenging factors (list of string array elements) @@ -20,4 +24,74 @@ export fscav_aero="'*:0.3','so2:0.0','msa:0.0','dms:0.0','nh3:0.4','nh4:0.6','bc # Number of diagnostic aerosol tracers (default: 0) export dnats_aero=2 +#================================================================================ +# 2. Aerosol emissions settings +#================================================================================ +# Biomass burning emission dataset. Choose from: gbbepx, qfed, none +export AERO_EMIS_FIRE="qfed" +export AERO_EMIS_FIRE_VERSION="061" +export AERO_EMIS_FIRE_HIST=1 # Use historical fire emissions +export FIRE_EMIS_NRT_DIR="${DCOMROOT}/YYYYMMDD/firewx" # Directory containing NRT fire emissions + +#=============================================================================== +# 3. NEXUS settings +#=============================================================================== +# NEXUS aerosol emissions dataset. Choose from: gocart, none + +# NEXUS configuration set +export NXS_CONFIG={{ NXS_CONFIG | default("gocart") }} # Options: gocart, none +export NXS_CONFIG_DIR="${PARMgfs}/chem/nexus/${NXS_CONFIG}" # Directory containing NEXUS configuration files + +#--------------------- +# NEXUS root directory +#--------------------- +export NXS_INPUT_DIR="{{ AERO_INPUTS_DIR }}" + +#-------------------------- +# NEXUS Time Step (seconds) +#-------------------------- +NXS_TSTEP={{ NXS_TSTEP | default(3600) }} # Default NEXUS time step in seconds + +#------------------ +# NEXUS Grid +#------------------ +export NXS_NX={{ NXS_NX | default(1440) }} +export NXS_NY={{ NXS_NY | default(720) }} +export NXS_XMIN={{ NXS_XMIN | default(-180.0) }} +export NXS_XMAX={{ NXS_XMAX | default(180.0) }} +export NXS_YMIN={{ NXS_YMIN | default(-90.0) }} +export NXS_YMAX={{ NXS_YMAX | default(90.0) }} +export NXS_NZ={{ NXS_NZ | default(1) }} + +#------------------- +# NEXUS Config Files +#------------------- +export NXS_GRID_NAME={{ NXS_GRID_NAME | default("HEMCO_sa_Grid.rc") }} +export NXS_TIME_NAME={{ NXS_TIME_NAME | default("HEMCO_sa_Time.rc") }} +export NXS_DIAG_NAME={{ NXS_DIAG_NAME | default("HEMCO_sa_Diag.rc") }} +export NXS_SPEC_NAME={{ NXS_SPEC_NAME | default("HEMCO_sa_Spec.rc") }} +export NXS_CONFIG_NAME={{ NXS_CONFIG_NAME | default("NEXUS_Config.rc") }} + +#------------------ +# NEXUS Diagnostics +#------------------ +export NXS_DIAG_PREFIX={{ NXS_DIAG_PREFIX | default("NXS_DIAG") }} +export NXS_DIAG_FREQ={{ NXS_DIAG_FREQ | default("Hourly") }} # Options: Hourly, Daily, Monthly + +#------------------ +# NEXUS Logging +#------------------ +export NXS_LOGFILE={{ NXS_LOGFILE | default("NEXUS.log") }} + +#------------------ +# NEXUS Emissions +#------------------ +export NXS_DO_MEGAN=.false # Use MEGAN biogenic emissions +export NXS_DO_CEDS2019=.true. # Use CEDS2019 emissions +export NXS_DO_CEDS2024=.false. # Use CEDS2024 emissions +export NXS_DO_HTAPv2=.true. # Use HTAPv2 emissions +export NXS_DO_HTAPv3=.false. # Use HTAPv3 emissions +export NXS_DO_CAMS=.false. # Use CAMS global emissions +export NXS_DO_CAMSTEMPO=.false # Use CAMS temporal emissions + echo "END: config.aero" From 8de888560c82607008ac6c5f2d1f99083f40680c Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Thu, 31 Jul 2025 15:34:24 -0400 Subject: [PATCH 003/132] seperate out into fire emision process and nexus emission process --- parm/chem/chem_emission.yaml.j2 | 23 ----------------------- parm/chem/fire_emission.yaml.j2 | 28 ++++++++++++++++++++++++++++ parm/chem/nxs_emission.yaml.j2 | 15 +++++++++++++++ 3 files changed, 43 insertions(+), 23 deletions(-) delete mode 100644 parm/chem/chem_emission.yaml.j2 create mode 100644 parm/chem/fire_emission.yaml.j2 create mode 100644 parm/chem/nxs_emission.yaml.j2 diff --git a/parm/chem/chem_emission.yaml.j2 b/parm/chem/chem_emission.yaml.j2 deleted file mode 100644 index 7583cbf82ca..00000000000 --- a/parm/chem/chem_emission.yaml.j2 +++ /dev/null @@ -1,23 +0,0 @@ -chem_emission: - config: - apply_quality_control: True - quality_control_threshold: 1.5 - GBBEPX_TEMPLATE: GBBEPx-all01GRID_YYYYMMDD.nc - QFED_VARS: {{ qfed_vars }} - GBBEPX_VARS: {{ gbbepx_vars }} - data_in: - mkdir: - - "{{ DATA }}" - copy: - {% for file in files_in %} - - ["{{ AERO_EMIS_FIRE_DIR }}/{{ file }}", "{{ DATA }}/{{ file }}"] - {% endfor %} - data_out: - mkdir: - - "{{ COMOUT_CHEM_HISTORY }}" - copy: - {% for file in processed_files %} - - ["{{ DATA }}/{{ file }}", "{{ COMOUT_CHEM_HISTORY }}/{{ file }}"] - {% endfor %} - - diff --git a/parm/chem/fire_emission.yaml.j2 b/parm/chem/fire_emission.yaml.j2 new file mode 100644 index 00000000000..0902e8cf48d --- /dev/null +++ b/parm/chem/fire_emission.yaml.j2 @@ -0,0 +1,28 @@ +fire_emission: + config: + apply_quality_control: True + quality_control_threshold: 1.5 + QFED_VARS: + {% for qvar in fire_vars %} + - "{{ qvar }}" + {% endfor %} + GBBEX_VARS: + {% for gvar in fire_vars %} + - "{{ gvar }}" + {% endfor %} + data_in: + mkdir: + - "{{ DATA }}" + copy: + {% for fin in rawfiles %} + - ["{{ fin }}", "{{ DATA }}/"] + {% endfor %} + data_out: + mkdir: + - "{{ COMOUT_CHEM_INPUT }}" + copy: + {% for fileout in processed_files %} + - ["{{ DATA }}/{{ fileout }}", "{{ COMOUT_CHEM_INPUT }}/"] + {% endfor %} + + diff --git a/parm/chem/nxs_emission.yaml.j2 b/parm/chem/nxs_emission.yaml.j2 new file mode 100644 index 00000000000..6c63c9529e2 --- /dev/null +++ b/parm/chem/nxs_emission.yaml.j2 @@ -0,0 +1,15 @@ +nxs_emission: + data_in: + link: + - ["{{ NXS_INPUT_DIR }}", "{{ LOCAL_INPUT_DIR }}"] + copy: + - ["{{ NXS_EXECUTABLE }}", "{{ WORK_DIR }}/"] + data_out: + mkdir: + - "{{ COMOUT_CHEM_INPUT }}" + - "{{ COMOUT_CHEM_RESTART }}" + copy: + {% for fileout in FINAL_OUTPUT %} + - ["{{ WORK_DIR }}/{{ fileout }}", "{{ COMOUT_CHEM_INPUT }}/"] + {% endfor %} + - ["{{ WORK_DIR }}/{{ RestartFile }}", "{{ COMOUT_CHEM_RESTART }}/"] \ No newline at end of file From b4c5982adbf0f3566ec2d030cb4bba26a3336e1b Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Thu, 31 Jul 2025 15:34:48 -0400 Subject: [PATCH 004/132] add input directories and source config.aero --- dev/parm/config/gcafs/config.prep_emissions | 2 ++ dev/parm/config/gfs/config.com | 2 ++ 2 files changed, 4 insertions(+) diff --git a/dev/parm/config/gcafs/config.prep_emissions b/dev/parm/config/gcafs/config.prep_emissions index fa411c27ad4..47bf5e26922 100644 --- a/dev/parm/config/gcafs/config.prep_emissions +++ b/dev/parm/config/gcafs/config.prep_emissions @@ -8,4 +8,6 @@ echo "BEGIN: config.prep_emissions" # Get task specific resources source "${EXPDIR}/config.resources" prep_emissions +source "${EXPDIR}/config.aero" + echo "END: config.prep_emissions" diff --git a/dev/parm/config/gfs/config.com b/dev/parm/config/gfs/config.com index bba449b0db3..f7e832ebae5 100644 --- a/dev/parm/config/gfs/config.com +++ b/dev/parm/config/gfs/config.com @@ -110,5 +110,7 @@ declare -rx COM_CHEM_HISTORY_TMPL=${COM_BASE}'/model/chem/history' declare -rx COM_CHEM_ANALYSIS_TMPL=${COM_BASE}'/analysis/chem' declare -rx COM_CHEM_BMAT_TMPL=${COM_CHEM_ANALYSIS_TMPL}'/bmatrix' declare -rx COM_CHEM_ANLMON_TMPL=${COM_BASE}'/products/chem/anlmon' +declare -rx COM_CHEM_INPUT_TMPL=${COM_BASE}'/model/chem/input' +declare -rx COM_CHEM_RESTART_TMPL=${COM_BASE}'/model/chem/restart' declare -rx COM_MED_RESTART_TMPL=${COM_BASE}'/model/med/restart' From 4f74323c8967f3e74f9c004de03a8a64238e0e13 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Thu, 31 Jul 2025 15:35:18 -0400 Subject: [PATCH 005/132] add COMIN_CHEM_INPUT --- jobs/JGLOBAL_FORECAST | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jobs/JGLOBAL_FORECAST b/jobs/JGLOBAL_FORECAST index 04366caf93f..c60b86294e1 100755 --- a/jobs/JGLOBAL_FORECAST +++ b/jobs/JGLOBAL_FORECAST @@ -92,6 +92,8 @@ if [[ "${DO_AERO_FCST}" == "YES" ]]; then COMOUT_CHEM_HISTORY:COM_CHEM_HISTORY_TMPL YMD="${PDY}" HH="${cyc}" RUN="${rCDUMP}" declare_from_tmpl -rx \ COMIN_TRACER_RESTART:COM_ATMOS_RESTART_TMPL + YMD="${PDY}" HH="${cyc}" RUN="${rCDUMP}" declare_from_tmpl -rx \ + COMIN_CHEM_INPUT:COM_CHEM_INPUT_TMPL fi From 90c5f80f9efcba6325bdebc1179e911c42d34089 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Thu, 31 Jul 2025 15:35:42 -0400 Subject: [PATCH 006/132] Add prep_emis for gcdas_half cycle --- dev/workflow/rocoto/gcafs_tasks.py | 180 ++++++++++++++++++++++++++++- 1 file changed, 179 insertions(+), 1 deletion(-) diff --git a/dev/workflow/rocoto/gcafs_tasks.py b/dev/workflow/rocoto/gcafs_tasks.py index 46baf5c2f67..e585581045a 100644 --- a/dev/workflow/rocoto/gcafs_tasks.py +++ b/dev/workflow/rocoto/gcafs_tasks.py @@ -121,13 +121,14 @@ def prep_emissions(self): str XML representation of the task """ + cycledef = f'{self.run}_half,{self.run}' if self.run in ['gcdas', 'enkfgcdas'] else self.run resources = self.get_resource('prep_emissions') task_name = f'{self.run}_prep_emissions' task_dict = {'task_name': task_name, 'resources': resources, 'envars': self.envars, - 'cycledef': self.run, + 'cycledef': cycledef, 'command': f'{self.HOMEgfs}/dev/jobs/prep_emissions.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -174,7 +175,16 @@ def offlineanl(self): return task def sfcanl(self): + """ + Create a task for surface analysis (sfcanl). + This task performs the surface analysis step in the workflow, depending on whether JEDI atmospheric variational analysis is enabled. + + Returns + ------- + str + XML representation of the task + """ deps = [] if self.options['do_jediatmvar']: dep_dict = {'type': 'task', 'name': f'gcdas_atmanlfinal'} @@ -201,7 +211,16 @@ def sfcanl(self): return task def prepatmiodaobs(self): + """ + Create a task for preparing atmospheric IODA observations. + + This task prepares the IODA observation files needed for data assimilation. + Returns + ------- + str + XML representation of the task + """ deps = [] dep_dict = {'type': 'task', 'name': f'gcdas_prep'} deps.append(rocoto.add_dependency(dep_dict)) @@ -225,7 +244,16 @@ def prepatmiodaobs(self): return task def atmanlinit(self): + """ + Create a task for atmospheric analysis initialization. + + This task initializes the atmospheric analysis, including hybrid variational analysis if enabled. + Returns + ------- + str + XML representation of the task + """ deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}_prepatmiodaobs'} deps.append(rocoto.add_dependency(dep_dict)) @@ -261,7 +289,16 @@ def atmanlinit(self): return task def atmanlvar(self): + """ + Create a task for atmospheric analysis variational step. + + This task performs the variational analysis step for the atmospheric component. + Returns + ------- + str + XML representation of the task + """ deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}_atmanlinit'} deps.append(rocoto.add_dependency(dep_dict)) @@ -285,7 +322,16 @@ def atmanlvar(self): return task def atmanlfv3inc(self): + """ + Create a task for applying FV3 increments to the atmospheric analysis. + This task applies the FV3 increment files to the atmospheric analysis fields. + + Returns + ------- + str + XML representation of the task + """ deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}_atmanlvar'} deps.append(rocoto.add_dependency(dep_dict)) @@ -309,7 +355,16 @@ def atmanlfv3inc(self): return task def atmanlfinal(self): + """ + Create a task for finalizing the atmospheric analysis. + This task finalizes the atmospheric analysis by applying all necessary increments and adjustments. + + Returns + ------- + str + XML representation of the task + """ deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}_atmanlfv3inc'} deps.append(rocoto.add_dependency(dep_dict)) @@ -333,7 +388,16 @@ def atmanlfinal(self): return task def prepobsaero(self): + """ + Create a task for preparing aerosol observation data. + + This task prepares the aerosol observation BUFR files needed for aerosol analysis. + Returns + ------- + str + XML representation of the task + """ dump_suffix = self._base["DUMP_SUFFIX"] dmpdir = self._base["DMPDIR"] dump_path = self._template_to_rocoto_cycstring(self._base["COM_OBSPROC_TMPL"], @@ -363,7 +427,16 @@ def prepobsaero(self): return task def aeroanlgenb(self): + """ + Create a task for generating aerosol background fields. + This task generates the background fields required for aerosol analysis. + + Returns + ------- + str + XML representation of the task + """ deps = [] dep_dict = {'type': 'metatask', 'name': f'{self.run}_fcst'} deps.append(rocoto.add_dependency(dep_dict)) @@ -387,7 +460,16 @@ def aeroanlgenb(self): return task def aeroanlinit(self): + """ + Create a task for initializing aerosol analysis. + This task initializes the aerosol analysis by preparing the necessary background and observation data. + + Returns + ------- + str + XML representation of the task + """ deps = [] dep_dict = {'type': 'task', 'name': 'gcdas_aeroanlgenb', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) @@ -413,7 +495,16 @@ def aeroanlinit(self): return task def aeroanlvar(self): + """ + Create a task for the aerosol analysis variational step. + + This task performs the variational analysis for the aerosol component. + Returns + ------- + str + XML representation of the task + """ deps = [] dep_dict = { 'type': 'task', 'name': f'{self.run}_aeroanlinit', @@ -439,7 +530,16 @@ def aeroanlvar(self): return task def aeroanlfinal(self): + """ + Create a task for finalizing the aerosol analysis. + + This task finalizes the aerosol analysis by applying all necessary increments and adjustments. + Returns + ------- + str + XML representation of the task + """ deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}_aeroanlvar'} deps.append(rocoto.add_dependency(dep_dict)) @@ -749,6 +849,16 @@ def efcs(self): # return task def atmanlupp(self): + """ + Create a task for UPP post-processing of the atmospheric analysis. + + This task runs the Unified Post Processor (UPP) on the atmospheric analysis output. + + Returns + ------- + str + XML representation of the task + """ postenvars = self.envars.copy() postenvar_dict = {'FHR3': '000', 'UPP_RUN': 'analysis'} @@ -785,6 +895,16 @@ def atmanlupp(self): return task def atmanlprod(self): + """ + Create a task for generating atmospheric analysis products. + + This task generates products from the atmospheric analysis output using UPP. + + Returns + ------- + str + XML representation of the task + """ postenvars = self.envars.copy() postenvar_dict = {'FHR_LIST': '-1'} for key, value in postenvar_dict.items(): @@ -814,12 +934,50 @@ def atmanlprod(self): return task def atmupp(self): + """ + Create a task for UPP post-processing of the atmospheric forecast. + + This task runs the Unified Post Processor (UPP) on the atmospheric forecast output. + + Returns + ------- + str + XML representation of the task + """ return self._upptask(upp_run='forecast', task_id='atmupp') def goesupp(self): + """ + Create a task for UPP post-processing of GOES satellite data. + + This task runs the Unified Post Processor (UPP) for GOES satellite output. + + Returns + ------- + str + XML representation of the task + """ return self._upptask(upp_run='goes', task_id='goesupp') def _upptask(self, upp_run="forecast", task_id="atmupp"): + """ + Helper method to create a UPP post-processing task. + + This method creates a Rocoto task for running the Unified Post Processor (UPP) + on either forecast or GOES satellite output, depending on the arguments. + + Parameters + ---------- + upp_run : str, optional + Type of UPP run ('forecast' or 'goes'). Default is 'forecast'. + task_id : str, optional + Identifier for the task. Default is 'atmupp'. + + Returns + ------- + str + XML representation of the task + """ VALID_UPP_RUN = ["forecast", "goes"] if upp_run not in VALID_UPP_RUN: @@ -1028,6 +1186,16 @@ def atmos_ensstat(self): return task def metp(self): + """ + Create a task for METplus verification. + + This task runs METplus to verify model output against observations for various cases. + + Returns + ------- + str + XML representation of the task + """ deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}_arch_vrfy'} deps.append(rocoto.add_dependency(dep_dict)) @@ -1085,6 +1253,16 @@ def metp(self): return task def anlstat(self): + """ + Create a task for analysis statistics. + + This task computes statistics for the analysis, including aerosol analysis if enabled. + + Returns + ------- + str + XML representation of the task + """ deps = [] if self.options['do_aero_anl']: dep_dict = {'type': 'task', 'name': f'{self.run}_aeroanlfinal'} From b9649972c83d01e758034b100045f801aa8fbd08 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Thu, 31 Jul 2025 15:35:57 -0400 Subject: [PATCH 007/132] Add com directories to prep emission --- jobs/JGLOBAL_PREP_EMISSIONS | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/jobs/JGLOBAL_PREP_EMISSIONS b/jobs/JGLOBAL_PREP_EMISSIONS index b545547fcb4..72012617349 100755 --- a/jobs/JGLOBAL_PREP_EMISSIONS +++ b/jobs/JGLOBAL_PREP_EMISSIONS @@ -10,8 +10,10 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "prep_emissions" -c "base prep_emissio ############################################## # Begin JOB SPECIFIC work ############################################## -# Generate COM variables from templates -# TODO: Add necessary COMIN, COMOUT variables for this job +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMOUT_CHEM_INPUT:COM_CHEM_INPUT_TMPL +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMOUT_CHEM_RESTART:COM_CHEM_RESTART_TMPL + +mkdir -p "${COMOUT_CHEM_INPUT}" ############################################################### # Run relevant script From ad249ef1fda880795acf5e8ae9feb3072f207f73 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Thu, 31 Jul 2025 15:37:59 -0400 Subject: [PATCH 008/132] Add newly generated data --- parm/ufs/gocart/ExtData.other | 16 ++++++++-------- parm/ufs/gocart/ExtData.qfed | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/parm/ufs/gocart/ExtData.other b/parm/ufs/gocart/ExtData.other index 5d2ddc51021..4facfaf5c12 100644 --- a/parm/ufs/gocart/ExtData.other +++ b/parm/ufs/gocart/ExtData.other @@ -7,7 +7,7 @@ TROPP 'Pa' Y N - 0.0 1.0 TROPP /dev/null:10000. #====== Dust Imports ================================================= -# FENGSHA input files. Note: regridding should be N or E - Use files with _FillValue != NaN +# FENGSHA input files. Note: regridding should be N or E - Use files with _FillValue != NaN DU_CLAY '1' Y E - none none clayfrac ExtData/nexus/FENGSHA/FENGSHA_2022_NESDIS_inputs_10km_v3.2.nc DU_SAND '1' Y E - none none sandfrac ExtData/nexus/FENGSHA/FENGSHA_2022_NESDIS_inputs_10km_v3.2.nc DU_SILT '1' Y E - none none siltfrac /dev/null @@ -17,12 +17,12 @@ DU_UTHRES '1' Y E - none none uthres ExtData/n #====== Sulfate Sources ================================================= # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -SU_ANTHROL1 NA Y Y %y4-%m2-%d2t12:00:00 none none SO2 ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -SU_ANTHROL2 NA Y Y %y4-%m2-%d2t12:00:00 none none SO2_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_ANTHROL1 NA N Y %y4-%m2-%d2t12:00:00 none none SO2 ChemInput/NXS_DIAG.%y4%m2%d2.nc +SU_ANTHROL2 NA N Y %y4-%m2-%d2t12:00:00 none none SO2_elev ChemInput/NXS_DIAG.%y4%m2%d2.nc # Ship emissions -SU_SHIPSO2 NA Y Y %y4-%m2-%d2t12:00:00 none none SO2_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -SU_SHIPSO4 NA Y Y %y4-%m2-%d2t12:00:00 none none SO4_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_SHIPSO2 NA N Y %y4-%m2-%d2t12:00:00 none none SO2_ship ChemInput/NXS_DIAG.%y4%m2%d2.nc +SU_SHIPSO4 NA N Y %y4-%m2-%d2t12:00:00 none none SO4_ship ChemInput/NXS_DIAG.%y4%m2%d2.nc # Aircraft fuel consumption SU_AIRCRAFT NA Y Y %y4-%m2-%d2t12:00:00 none none none /dev/null @@ -31,9 +31,9 @@ SU_AIRCRAFT NA Y Y %y4-%m2-%d2t12:00:00 none none none /dev/null SU_DMSO NA Y Y %y4-%m2-%d2t12:00:00 none none conc ExtData/MERRA2/sfc/DMSclim_sfcconcentration.x360_y181_t12.Lana2011.nc4 # Aviation emissions during the three phases of flight -SU_AVIATION_LTO NA Y Y %y4-%m2-%d2t12:00:00 none none so2_aviation ExtData/PIESA/sfc/HTAP/v2.2/htap-v2.2.emis_so2.aviation_lto.x3600_y1800_t12.2010.nc4 -SU_AVIATION_CDS NA Y Y %y4-%m2-%d2t12:00:00 none none so2_aviation ExtData/PIESA/sfc/HTAP/v2.2/htap-v2.2.emis_so2.aviation_cds.x3600_y1800_t12.2010.nc4 -SU_AVIATION_CRS NA Y Y %y4-%m2-%d2t12:00:00 none none so2_aviation ExtData/PIESA/sfc/HTAP/v2.2/htap-v2.2.emis_so2.aviation_crs.x3600_y1800_t12.2010.nc4 +SU_AVIATION_LTO NA N Y %y4-%m2-%d2t12:00:00 none none SO2_lto ChemInput/NXS_DIAG.%y4%m2%d2.nc +SU_AVIATION_CDS NA N Y %y4-%m2-%d2t12:00:00 none none SO2_cds ChemInput/NXS_DIAG.%y4%m2%d2.nc +SU_AVIATION_CRS NA N Y %y4-%m2-%d2t12:00:00 none none SO2_crs ChemInput/NXS_DIAG.%y4%m2%d2.nc # H2O2, OH and NO3 mixing ratios # -------------------------------------------------------------- diff --git a/parm/ufs/gocart/ExtData.qfed b/parm/ufs/gocart/ExtData.qfed index 805c1173e3f..60c2693ec0c 100644 --- a/parm/ufs/gocart/ExtData.qfed +++ b/parm/ufs/gocart/ExtData.qfed @@ -2,7 +2,7 @@ # QFED #-------------------------------------------------------------------------------------------------------------------------------- -SU_BIOMASS NA N Y %y4-%m2-%d2t12:00:00 none 0.7778 biomass ExtData/nexus/QFED/%y4/%m2/qfed2.emis_so2.061.%y4%m2%d2.nc4 -OC_BIOMASS NA N Y %y4-%m2-%d2t12:00:00 none 0.7778 biomass ExtData/nexus/QFED/%y4/%m2/qfed2.emis_oc.061.%y4%m2%d2.nc4 -BC_BIOMASS NA N Y %y4-%m2-%d2t12:00:00 none 0.7778 biomass ExtData/nexus/QFED/%y4/%m2/qfed2.emis_bc.061.%y4%m2%d2.nc4 -# EMI_NH3_BB NA N Y %y4-%m2-%d2t12:00:00 none 0.7778 biomass ExtData/nexus/QFED/%y4/%m2/qfed2.emis_nh3.061.%y4%m2%d2.nc4 +SU_BIOMASS NA N Y %y4-%m2-%d2t12:00:00 none 0.7778 SO2 ChemInput/FIRE_EMIS_%y4%m2%d2.nc # ExtData/nexus/QFED/%y4/%m2/qfed2.emis_so2.061.%y4%m2%d2.nc4 +OC_BIOMASS NA N Y %y4-%m2-%d2t12:00:00 none 0.7778 OC ChemInput/FIRE_EMIS_%y4%m2%d2.nc +BC_BIOMASS NA N Y %y4-%m2-%d2t12:00:00 none 0.7778 BC ChemInput/FIRE_EMIS_%y4%m2%d2.nc +# EMI_NH3_BB NA N Y %y4-%m2-%d2t12:00:00 none 0.7778 NH3 ChemInput/FIRE_EMIS_%y4%m2%d2.nc From 668e91ba600df8969c8b54912a0bf4732e44dd39 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Thu, 31 Jul 2025 15:41:34 -0400 Subject: [PATCH 009/132] add anthropogenic --- parm/ufs/gocart/ExtData.other | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/parm/ufs/gocart/ExtData.other b/parm/ufs/gocart/ExtData.other index 4facfaf5c12..bb50ec6a8da 100644 --- a/parm/ufs/gocart/ExtData.other +++ b/parm/ufs/gocart/ExtData.other @@ -63,19 +63,19 @@ OC_MTPO NA Y Y %y4-%m2-%d2t12:00:00 none none mtpo ExtData/nexus/MEGAN_ OC_BIOFUEL NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -OC_ANTEOC1 NA Y Y %y4-%m2-%d2t12:00:00 none none OC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -OC_ANTEOC2 NA Y Y %y4-%m2-%d2t12:00:00 none none OC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +OC_ANTEOC1 NA N Y %y4-%m2-%d2t12:00:00 none none OC ChemInput/NXS_DIAG.%y4%m2%d2.nc +OC_ANTEOC2 NA N Y %y4-%m2-%d2t12:00:00 none none OC_elev ChemInput/NXS_DIAG.%y4%m2%d2.nc # EDGAR based ship emissions -OC_SHIP NA Y Y %y4-%m2-%d2t12:00:00 none none OC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +OC_SHIP NA N Y %y4-%m2-%d2t12:00:00 none none OC_ship ChemInput/NXS_DIAG.%y4%m2%d2.nc # Aircraft fuel consumption OC_AIRCRAFT NA N Y %y4-%m2-%d2t12:00:00 none none oc_aviation /dev/null # Aviation emissions during the three phases of flight -OC_AVIATION_LTO NA Y Y %y4-%m2-%d2t12:00:00 none none oc_aviation ExtData/PIESA/sfc/HTAP/v2.2/htap-v2.2.emis_oc.aviation_lto.x3600_y1800_t12.2010.nc4 -OC_AVIATION_CDS NA Y Y %y4-%m2-%d2t12:00:00 none none oc_aviation ExtData/PIESA/sfc/HTAP/v2.2/htap-v2.2.emis_oc.aviation_cds.x3600_y1800_t12.2010.nc4 -OC_AVIATION_CRS NA Y Y %y4-%m2-%d2t12:00:00 none none oc_aviation ExtData/PIESA/sfc/HTAP/v2.2/htap-v2.2.emis_oc.aviation_crs.x3600_y1800_t12.2010.nc4 +OC_AVIATION_LTO NA N Y %y4-%m2-%d2t12:00:00 none none OC_lto ChemInput/NXS_DIAG.%y4%m2%d2.nc +OC_AVIATION_CDS NA N Y %y4-%m2-%d2t12:00:00 none none OC_cds ChemInput/NXS_DIAG.%y4%m2%d2.nc +OC_AVIATION_CRS NA N Y %y4-%m2-%d2t12:00:00 none none OC_crs ChemInput/NXS_DIAG.%y4%m2%d2.nc # SOA production pSOA_ANTHRO_VOC NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null From b534a8b9cb191bfc16648b1b0123a8aeec682589 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Thu, 31 Jul 2025 15:46:01 -0400 Subject: [PATCH 010/132] adding finishing touches --- parm/ufs/gocart/ExtData.other | 12 +- scripts/exglobal_prep_emissions.py | 20 +- ush/forecast_predet.sh | 51 +- ush/parsing_namelists_GOCART.sh | 30 +- ush/python/pygfs/__init__.py | 39 +- ush/python/pygfs/task/aero_emissions.py | 465 ------------ ush/python/pygfs/task/chem_fire_emission.py | 768 ++++++++++++++++++++ ush/python/pygfs/task/nxs_emission.py | 375 ++++++++++ 8 files changed, 1266 insertions(+), 494 deletions(-) delete mode 100644 ush/python/pygfs/task/aero_emissions.py create mode 100644 ush/python/pygfs/task/chem_fire_emission.py create mode 100644 ush/python/pygfs/task/nxs_emission.py diff --git a/parm/ufs/gocart/ExtData.other b/parm/ufs/gocart/ExtData.other index bb50ec6a8da..54e7a6df949 100644 --- a/parm/ufs/gocart/ExtData.other +++ b/parm/ufs/gocart/ExtData.other @@ -88,19 +88,19 @@ pSOA_ANTHRO_VOC NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null BC_BIOFUEL NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -BC_ANTEBC1 NA Y Y %y4-%m2-%d2t12:00:00 none none BC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -BC_ANTEBC2 NA Y Y %y4-%m2-%d2t12:00:00 none none BC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +BC_ANTEBC1 NA N Y %y4-%m2-%d2t12:00:00 none none BC ChemInput/NXS_DIAG.%y4%m2%d2.nc +BC_ANTEBC2 NA N Y %y4-%m2-%d2t12:00:00 none none BC_elev ChemInput/NXS_DIAG.%y4%m2%d2.nc # EDGAR based ship emissions -BC_SHIP NA Y Y %y4-%m2-%d2t12:00:00 none none BC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +BC_SHIP NA N Y %y4-%m2-%d2t12:00:00 none none BC_ship ChemInput/NXS_DIAG.%y4%m2%d2.nc # Aircraft fuel consumption BC_AIRCRAFT NA N Y %y4-%m2-%d2t12:00:00 none none bc_aviation /dev/null # Aviation emissions during the LTO, SDC and CRS phases of flight -BC_AVIATION_LTO NA Y Y %y4-%m2-%d2t12:00:00 none none bc_aviation ExtData/PIESA/sfc/HTAP/v2.2/htap-v2.2.emis_bc.aviation_lto.x3600_y1800_t12.2010.nc4 -BC_AVIATION_CDS NA Y Y %y4-%m2-%d2t12:00:00 none none bc_aviation ExtData/PIESA/sfc/HTAP/v2.2/htap-v2.2.emis_bc.aviation_cds.x3600_y1800_t12.2010.nc4 -BC_AVIATION_CRS NA Y Y %y4-%m2-%d2t12:00:00 none none bc_aviation ExtData/PIESA/sfc/HTAP/v2.2/htap-v2.2.emis_bc.aviation_crs.x3600_y1800_t12.2010.nc4 +BC_AVIATION_LTO NA N Y %y4-%m2-%d2t12:00:00 none none BC_lto ChemInput/NXS_DIAG.%y4%m2%d2.nc +BC_AVIATION_CDS NA N Y %y4-%m2-%d2t12:00:00 none none BC_cds ChemInput/NXS_DIAG.%y4%m2%d2.nc +BC_AVIATION_CRS NA N Y %y4-%m2-%d2t12:00:00 none none BC_crs ChemInput/NXS_DIAG.%y4%m2%d2.nc #============================================================================================================ # BROWN CARBON diff --git a/scripts/exglobal_prep_emissions.py b/scripts/exglobal_prep_emissions.py index 2cf718d7f6a..dadd1f39c3f 100755 --- a/scripts/exglobal_prep_emissions.py +++ b/scripts/exglobal_prep_emissions.py @@ -5,7 +5,7 @@ import os from wxflow import Logger, cast_strdict_as_dtypedict -from pygfs import AerosolEmissions +from pygfs import ChemFireEmissions, NXSEmissions # Initialize root logger @@ -19,8 +19,16 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the emissions pre-processing task - # emissions = AerosolEmissions(config) - # emissions.initialize() - # emissions.configure() - # emissions.execute(emissions.task_config.DATA, emissions.task_config.APRUN) - # emissions.finalize() + emissions = ChemFireEmissions(config) + emissions.initialize() + emissions.configure() + emissions.execute() + emissions.finalize() + + nxsemis = NXSEmissions(config) + nxsemis.initialize() + nxsemis.configure() + nxsemis.execute() + nxsemis.finalize() + + diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh index e859352b74d..508d2382c84 100755 --- a/ush/forecast_predet.sh +++ b/ush/forecast_predet.sh @@ -661,7 +661,7 @@ WW3_predet(){ # this file does not exist for structured, and the model can run without it (just slower init) if [[ -f "${FIXgfs}/wave/pnt_wght.${waveGRD}.nc" ]]; then cpreq "${FIXgfs}/wave/pnt_wght.${waveGRD}.nc" "${DATA}/pnt_wght.ww3.nc" - fi + fi if [[ "${WW3ICEINP}" == "YES" ]]; then local wavicefile="${COMIN_WAVE_PREP}/${RUN}wave.${WAVEICE_FID}.t${current_cycle:8:2}z.ice" @@ -786,5 +786,54 @@ GOCART_predet(){ # FHMAX gets modified when IAU is on, so keep origianl value for GOCART output GOCART_MAX=${FHMAX} + # Create the ChemInput directory in the local run directory + if [[ ! -d "${DATA}/ChemInput" ]]; then mkdir -p "${DATA}/ChemInput"; fi + + + # Copy Fire Emission Files ChemInput directory + local current + local YYYYMMDDHH + current="${current_cycle_begin}" + while [[ "${current}" -le "${current_cycle_end}" ]]; do + # Validate current is a valid date string + if ! YYYYMMDD=$(date -d "${current:0:8} ${current:8:2}:00:00" +%Y%m%d 2>/dev/null); then + echo "FATAL ERROR: Invalid date string '${current}' in GOCART_predet, ABORT!" + exit 1 + fi + local FireEmisFile="${COMIN_CHEM_INPUT}/FIRE_EMIS_${YYYYMMDD}.nc" + if [[ -f "${FireEmisFile}" ]]; then + cpreq "${FireEmisFile}" "${DATA}/ChemInput/" + else + echo "FATAL ERROR: GOCART input file '${FireEmisFile}' does not exist, ABORT!" + exit 1 + fi + + + # Increment by 1 day + current=$(date -d "${current:0:8} ${current:8:2} +1 day" +%Y%m%d%H) + done + + + # Copy NXS Emission Files ChemInput directory + # NXS files are hourly, so we need to loop through each hour in the cycle + current=$(date -d "${current_cycle_begin:0:8} ${current_cycle_begin:8:2}" +%Y%m%d%H) + cycleend=$(date -d "${current_cycle_end:0:8} ${current_cycle_end:8:2} +12 hour" +%Y%m%d%H) + while [[ "${current}" -le "${current_cycle_end}" ]]; do + if ! YYYYMMDD=$(date -d "${current:0:8} ${current:8:2}:00:00" +%Y%m%d 2>/dev/null); then + echo "FATAL ERROR: Invalid date string '${current}' in GOCART_predet, ABORT!" + exit 1 + fi + local NXSFile="${COMIN_CHEM_INPUT}/${NXS_DIAG_PREFIX}.${YYYYMMDD}.nc" + if [[ -f "${NXSFile}" ]]; then + cpreq "${NXSFile}" "${DATA}/ChemInput/" + else + echo "FATAL ERROR: GOCART input file '${NXSFile}' does not exist, ABORT!" + exit 1 + fi + # Increment by 1 hour + current=$(date -d "${current:0:8} ${current:8:2} +1 hour" +%Y%m%d%H) + done + + # GOCART output times can't be computed here because they may depend on FHROT } diff --git a/ush/parsing_namelists_GOCART.sh b/ush/parsing_namelists_GOCART.sh index 2a08a90b0bc..4388fd232cb 100755 --- a/ush/parsing_namelists_GOCART.sh +++ b/ush/parsing_namelists_GOCART.sh @@ -13,21 +13,21 @@ GOCART_namelists() { local inst_aod_freq="${fhout_aero_padded}0000" # Other gocart fields not currently used - local inst_du_ss_freq="120000" - local tavg_du_ss_freq="120000" - local inst_ca_freq="120000" - local inst_ni_freq="120000" - local inst_su_freq="120000" - local inst_du_bin_freq="010000" - local tavg_du_bin_freq="030000" - local inst_ss_bin_freq="060000" - local inst_ca_bin_freq="120000" - local inst_ni_bin_freq="120000" - local inst_su_bin_freq="120000" - local inst_2d_freq="030000" - local inst_3d_freq="060000" - local tavg_2d_rad_freq="120000" - local tavg_3d_rad_freq="120000" + local inst_du_ss_freq="${fhout_aero_padded}0000" + local tavg_du_ss_freq="${fhout_aero_padded}0000" + local inst_ca_freq="${fhout_aero_padded}0000" + local inst_ni_freq="${fhout_aero_padded}0000" + local inst_su_freq="${fhout_aero_padded}0000" + local inst_du_bin_freq="${fhout_aero_padded}0000" + local tavg_du_bin_freq="${fhout_aero_padded}0000" + local inst_ss_bin_freq="${fhout_aero_padded}0000" + local inst_ca_bin_freq="${fhout_aero_padded}0000" + local inst_ni_bin_freq="${fhout_aero_padded}0000" + local inst_su_bin_freq="${fhout_aero_padded}0000" + local inst_2d_freq="${fhout_aero_padded}0000" + local inst_3d_freq="${fhout_aero_padded}0000" + local tavg_2d_rad_freq="${fhout_aero_padded}0000" + local tavg_3d_rad_freq="${fhout_aero_padded}0000" for template_in in "${AERO_CONFIG_DIR}/"*.rc; do base_in="$(basename "${template_in}")" diff --git a/ush/python/pygfs/__init__.py b/ush/python/pygfs/__init__.py index d718c6c0c93..04493212d0c 100644 --- a/ush/python/pygfs/__init__.py +++ b/ush/python/pygfs/__init__.py @@ -1,8 +1,45 @@ +""" +pygfs +===== + +This package provides task classes and utilities for the GFS workflow, including analysis, chemistry, ensemble, marine, snow, and forecast processing. + +Modules +------- +- task.analysis: Analysis task +- task.chem_fire_emission: Chemistry fire emissions task +- task.nxs_emission: NEXUS emissions task +- task.aero_analysis: Aerosol analysis task +- task.aero_bmatrix: Aerosol background matrix task +- task.atm_analysis: Atmospheric analysis task +- task.atmens_analysis: Atmospheric ensemble analysis task +- task.ensemble_recenter: Ensemble recentering task +- task.fv3_analysis_calc: FV3 analysis calculation task +- task.marine_bmat: Marine background matrix task +- task.offline_analysis: Offline analysis task +- task.snow_analysis: Snow analysis task +- task.snowens_analysis: Snow ensemble analysis task +- task.upp: Unified Post Processor (UPP) task +- task.oceanice_products: Ocean/ice products task +- task.gfs_forecast: GFS forecast task +- utils.marine_da_utils: Marine data assimilation utilities +- task.fetch: Fetch task + +Attributes +---------- +__docformat__ : str + The documentation format for the module. +__version__ : str + The version of the pygfs package. +pygfs_directory : str + The absolute path to the pygfs package directory. +""" import os from .task.analysis import Analysis -from .task.aero_emissions import AerosolEmissions +from .task.chem_fire_emission import ChemFireEmissions +from .task.nxs_emission import NXSEmissions from .task.aero_analysis import AerosolAnalysis from .task.aero_bmatrix import AerosolBMatrix from .task.atm_analysis import AtmAnalysis diff --git a/ush/python/pygfs/task/aero_emissions.py b/ush/python/pygfs/task/aero_emissions.py deleted file mode 100644 index 6c17c135261..00000000000 --- a/ush/python/pygfs/task/aero_emissions.py +++ /dev/null @@ -1,465 +0,0 @@ -#!/usr/bin/env python3 - -import os -import re -import fnmatch -import xarray as xr -from logging import getLogger -from typing import Dict, Any, Union -from dateutil.rrule import DAILY, rrule -from pprint import pformat, pprint - -from wxflow import (AttrDict, - parse_j2yaml, - FileHandler, - logit, - Task, - to_timedelta, - WorkflowException, - Executable, which) - -logger = getLogger(__name__.split('.')[-1]) - - -class AerosolEmissions(Task): - """Chemistry Emissions pre-processing Task - """ - - @logit(logger, name="AeroEmission") - def __init__(self, config: Dict[str, Any]) -> None: - """Constructor for the Aerosol Emissions task - - Parameters - ---------- - config : Dict[str, Any] - Incoming configuration for the task from the environment - - Returns - ------- - None - """ - super().__init__(config) - - self.historical = bool(self.task_config.get('AERO_EMIS_FIRE_HIST', 0)) - nforecast_hours = self.task_config["FHMAX_GFS"] - self.start_date = self.task_config["PDY"] - self.end_date = self.start_date + to_timedelta(f'{nforecast_hours + 24}H') - self.forecast_dates = list(rrule(freq=DAILY, dtstart=self.start_date, until=self.end_date)) - - # # Extend task_config with localdict - # self.task_config = AttrDict(**self.task_config, **localdict) - - @logit(logger) - def initialize(self) -> None: - """Initialize the work directory and process chemical emissions configuration. - - This method performs the following steps: - 1. Loads and parses the chem_emission.yaml.j2 template - 2. Sets up template variables for emission configuration - 3. Creates necessary working directories - 4. Copies required input files to working directory - - Parameters - ---------- - None - - Returns - ------- - None - - Raises - ------ - WorkflowException - If the YAML template file is not found - If required directories cannot be created - If file copying operations fail - - Notes - ----- - The method expects the following configuration to be available: - - HOMEgfs : str - Base directory containing workflow configuration - - DATA : str - Working directory path - - COMOUT_CHEM_HISTORY : str - Output directory for chemical history files - - AERO_EMIS_FIRE_DIR : str - Directory containing fire emission data - - AERO_EMIS_FIRE_VERSION : str - Version of fire emission data (GBBEPx or QFED) - - The configuration is processed through a Jinja2 template system - and the resulting setup is stored in self.task_config. - """ - # # Parse the YAML template - # yaml_template = os.path.join(self.task_config.HOMEgfs, 'parm/chem/chem_emission.yaml.j2') - # if not os.path.exists(yaml_template): - # msg = f'YAML template not found: {yaml_template}' - # logger.error(msg) - # raise WorkflowException(msg) - # else: - # logger.debug(f'Found YAML template: {yaml_template}') - # yamlvars = parse_j2yaml(path=yaml_template, data=self.task_config) - # self.task_config.append(yamlvars) - # print(self.task_config) - - if self.historical: - logger.info(f'Processing historical emissions for {self.start_date} to {self.end_date}') - - # find the forecast dates that are in the historical period for the given emission dataset - for dates in self.forecast_dates: - if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': - files = self._find_gbbepx_files(dates, version=self.task_config.AERO_EMIS_FIRE_VERSION) - elif self.task_config.AERO_EMIS_FIRE.lower() == 'qfed': - files = self._find_qfed_files(dates, version=self.task_config.AERO_EMIS_FIRE_VERSION) - else: - logger.info(f'Processing forecast emissions for {self.start_date}') - - if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': - files = self._find_gbbepx_files( - self.start_date, - version=self.task_config.AERO_EMIS_FIRE_VERSION, - vars=self.task_config.gbbepx_vars - ) - elif self.task_config.AERO_EMIS_FIRE.lower() == 'qfed': - files = self._find_qfed_files( - self.start_date, - version=self.task_config.AERO_EMIS_FIRE_VERSION, - vars=self.task_config.qfed_vars.split() - ) - - # Set up template variables - pprint(self.task_config) - tmpl_dict = { - 'DATA': self.task_config.DATA, - 'COMOUT_CHEM_HISTORY': self.task_config.COMOUT_CHEM_HISTORY, - 'AERO_EMIS_FIRE_DIR': self.task_config.AERO_EMIS_FIRE_DIR, - 'AERO_EMIS_FIRE_VERSION': self.task_config.AERO_EMIS_FIRE_VERSION, - 'historical': self.historical, - 'forecast_dates': self.task_config.get('forecast_dates', []), - 'qfed_vars': self.task_config.get('QFED_VARS', - ["co", - "nox", - "so2", - "nh3", - "bc", - "oc"]), - 'gbbepx_vars': self.task_config.get('GBBEPX_VARS', - ["co", - "nox", - "so2", - "nh3", - "bc", - "oc"]), - "files_in": files - } - - # Parse template and update task configuration - logger.debug(f'Parsing YAML template: {yaml_template}') - yaml_config = parse_j2yaml(yaml_template, tmpl_dict) - self.task_config.update(yaml_config.get('chem_emission', {})) - - # Create directories - for dir_path in self.task_config.data_in.mkdir: - logger.info(f'Creating directory: {dir_path}') - os.makedirs(dir_path, exist_ok=True) - - # Copy input files - fh = FileHandler() - for file_pair in self.task_config.data_in.copy: - src = file_pair[0] - dst = os.path.join(self.task_config.DATA, os.path.basename(src)) - logger.info(f'Copying {src} to {dst}') - fh.copy(src, dst) - - @logit(logger) - def _get_unique_months(self): - """Extract unique months from forecast dates. - - This method finds all unique months present in the forecast dates - range. Useful for monthly-based emissions processing. - - Returns - ------- - set - Set of unique months as zero-padded strings (01-12) - - Notes - ----- - Uses self.forecast_dates which should be populated during initialization - Months are returned as strings with leading zeros (e.g., '01' for January) - """ - months = set(f"{date.month:02d}" for date in self.forecast_dates) - years = set(date.year for date in self.forecast_dates) - return months, years - - @logit(logger) - def execute(self, workdir: Union[str, os.PathLike]) -> None: - """Process emission files based on configuration. - - For GBBEPx files, converts them to COARDS compliant format and renames - according to template pattern. - - Parameters - ---------- - workdir : str | os.PathLike - work directory with the staged data - - Returns - ------- - None - - Notes - ----- - Uses GBBEPX_TEMPLATE from config to rename processed files - """ - logger.info(f"Processing emission files in {workdir}") - - if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': - # Process each GBBEPx file - for file_path in os.listdir(workdir): - if file_path.startswith('GBBEPx'): - full_path = os.path.join(workdir, file_path) - logger.info(f"Processing GBBEPx file: {file_path}") - - # Extract date from filename using regex - match = re.search(r"c(\d{8}).", file_path) - if not match: - logger.warning(f"Could not extract date from {file_path}, skipping") - continue - - current_date = match.group(1) - - # Convert to COARDS format - ds = self.GBBEPx_to_COARDS(full_path) - - # Generate new filename from template - template = self.task_config.config.GBBEPX_TEMPLATE - new_name = template.replace('YYYYMMDD', current_date) - output_path = os.path.join(workdir, f"processed_{new_name}") - - logger.info(f"Saving processed file to: {output_path}") - ds.to_netcdf(output_path) - - elif self.task_config.AERO_EMIS_FIRE.lower() == 'qfed': - logger.info("QFED files do not require processing, skipping execute step") - return - - logger.info("Emission processing complete") - - @logit(logger) - def finalize(self) -> None: - """Perform closing actions of the task. - Copy processed files from the DATA directory to COMOUT_CHEM_HISTORY. - - Returns - ------- - None - - Notes - ----- - Only copies processed GBBEPx files or QFED files based on configuration - Uses FileHandler for reliable file operations with logging - """ - logger.info("Finalizing chemical emissions processing") - - fh = FileHandler() - data_dir = self.task_config.DATA - comout_dir = self.task_config.COMOUT_CHEM_HISTORY - - if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': - pattern = "processed_GBBEPx*.nc" - else: - pattern = "qfed*.nc" - - processed_files = [] - for file_name in os.listdir(data_dir): - if fnmatch.fnmatch(file_name, pattern): - src = os.path.join(data_dir, file_name) - dst = os.path.join(comout_dir, file_name) - logger.info(f"Copying {src} to {dst}") - fh.copy(src, dst) - processed_files.append(file_name) - - self.task_config.update({'processed_files': processed_files}) - logger.info("Chemical emissions finalization complete") - - @logit(logger) - def _find_gbbepx_files(self, dates, version='v5r0'): - """Find GBBEPx files for the given date - - Parameters - ---------- - dates : str - Date for which to find GBBEPx files - version : str - Version of GBBEPx files to search for - - Returns - ------- - List[str] - List of GBBEPx files for the given date - """ - logger.info(f'Finding GBBEPx files for {dates}') - - # Find all possible months - months = self._get_unique_months() - - files_found = [] - # Find all possible files - for mon in months: - emis_file_dir = os.path.join(self.task_config.AERO_EMIS_FIRE_DIR, version, mon) - all_files = os.listdir(emis_file_dir) - - matching_files = [] - - pattern = r"s(\d{8})_e(\d{8})_c(\d{8})" - - for file_name in all_files: - match = re.match(pattern, file_name) - if match: - # start_date = match.group(1) - # end_date = match.group(2) - create_date = match.group(3) - - if dates[0] <= create_date and dates[-1] <= create_date: - matching_files.append(file_name) - files_found.extend(matching_files) - - return files_found - - @logit(logger) - def _find_qfed_files(self, dates, vars, version='061'): - """Find QFED files for the given date - - Parameters - ---------- - dates : str or datetime - Date for which to find QFED files - vars : list - List of variables to search for (e.g., bc, oc, co, etc.) - version : str - Version of QFED files to search for - - Returns - ------- - List[str] - List of QFED files for the given date - """ - logger.info(f'Finding QFED files for {dates}') - - # ensure version is a string - version = str(version).zfill(3) - - # Convert single date to list for consistent processing - if not isinstance(dates, list): - dates = [dates] - - # Format dates properly - date_strings = [d.strftime('%Y%m%d') if hasattr(d, 'strftime') else str(d) for d in dates] - - files_found = [] - - for date in dates: - # Extract year and month from the date - if hasattr(date, 'year') and hasattr(date, 'month'): - year = str(date.year) - month = f"{date.month:02d}" - else: - # If date is a string, try to parse it - date_str = str(date) - if len(date_str) >= 8: # YYYYMMDD format - year = date_str[:4] - month = date_str[4:6] - else: - logger.warning(f"Cannot parse date format: {date}") - continue - - emis_file_dir = os.path.join(self.task_config.AERO_EMIS_FIRE_DIR, year, month) - - if not os.path.exists(emis_file_dir): - logger.warning(f"Directory does not exist: {emis_file_dir}") - continue - - # Format date string for file matching - date_str = date.strftime('%Y%m%d') if hasattr(date, 'strftime') else str(date) - if len(date_str) > 8: # Format may be YYYY-MM-DD - date_str = date_str.replace('-', '') - - for v in vars: - # Match pattern like qfed2.emis_bc.{version}.20200118.nc4 - v_pattern = f"qfed2.emis_{v}.{version}.{date_str}.nc4" - full_path = os.path.join(emis_file_dir, v_pattern) - - # If exact match exists - if os.path.exists(full_path): - files_found.append(full_path) - logger.debug(f"Found exact QFED file: {full_path}") - - if not full_path: - logger.warning(f"File not found: {full_path}") - if not files_found: - logger.warning(f"No QFED files found for dates {date_strings} and variables {vars}") - - return files_found - - @logit(logger) - def GBBEPx_to_COARDS(fname: Union[str, os.PathLike]) -> xr.Dataset: - """Convert GBBEPx file to COARDS compliant format - - Parameters - ---------- - fname : str | os.PathLike - Input GBBEPx file path - - Returns - ------- - xr.Dataset - COARDS compliant dataset - """ - logger.info(f"Converting {fname} to COARDS format") - f = xr.open_dataset(fname, decode_cf=False) - - # Handle time dimension - if 'Time' in f.dims: - f = f.rename({"Time": 'time'}) - f.time.attrs['long_name'] = 'time' - - # Modify latitude and longitude attributes - f = f.rename({'Longitude': 'lon', 'Latitude': 'lat'}) - - # Validate and normalize coordinates - # Check longitude range and monotonicity - if not (f.lon.diff('lon') > 0).all(): - raise WorkflowException("Longitude values must be strictly increasing") - - # Ensure longitude is in [-180, 180] range - f['lon'] = xr.where(f.lon > 180, f.lon - 360, f.lon) - f = f.sortby('lon') # Sort after potential wrapping - - # Check latitude monotonicity - if not (f.lat.diff('lat') > 0).all(): - raise WorkflowException("Latitude values must be strictly increasing") - - f.lon.attrs.update({'long_name': 'Longitude', 'units': 'degrees_east'}) - f.lat.attrs.update({'long_name': 'Latitude', 'units': 'degrees_north'}) - - # Remove Element dimension if present - if 'Element' in f.dims: - f = f.drop_dims('Element') - - # Update variable attributes - for v in f.data_vars: - if v not in ['FirePerc', 'QCAll', 'NumSensor', 'CloudPerc']: - f[v].attrs['_FillValue'] = -9999.0 - elif v == 'FirePerc': - f[v].attrs.update({'units': '-', 'long_name': 'percent_of_fire_in_grid_cell'}) - elif v == 'CloudPerc': - f[v].attrs.update({'units': '-', 'long_name': 'percent_of_clouds_in_grid_cell'}) - elif v == 'NumSensor': - f[v].attrs['units'] = '-' - - # Set global attributes - f.attrs.update({'format': 'NetCDF', 'title': 'GBBEPx Fire Emissions'}) - - return f diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py new file mode 100644 index 00000000000..c4b969c57aa --- /dev/null +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -0,0 +1,768 @@ +#!/usr/bin/env python3 + +import os +import re +import fnmatch +import datetime +import xarray as xr +from logging import getLogger +from typing import Dict, Any, Union, List +from dateutil.rrule import DAILY, rrule +# from pprint import pprint + +from wxflow import (AttrDict, + parse_j2yaml, + FileHandler, + logit, + Task, + to_timedelta, + WorkflowException, + Executable, which) + +logger = getLogger(__name__.split('.')[-1]) + + +class ChemFireEmissions(Task): + """Chemistry Emissions pre-processing Task + """ + + @logit(logger, name="ChemFireEmissions") + def __init__(self, config: Dict[str, Any]) -> None: + """Constructor for the Chemistry Fire Emissions task + + Parameters + ---------- + config : Dict[str, Any] + Incoming configuration for the task from the environment + + Returns + ------- + None + """ + super().__init__(config) + + self.historical = bool(self.task_config.get('AERO_EMIS_FIRE_HIST', 1)) + self.AERO_INPUTS_DIR = self.task_config.get('AERO_INPUTS_DIR', None) + self.COMOUT_CHEM_INPUT = self.task_config.get('COMOUT_CHEM_INPUT', None) + nforecast_hours = self.task_config["FHMAX_GFS"] + self.start_date = self.task_config["PDY"] #- datetime.timedelta(days=1) + self.end_date = self.start_date + to_timedelta(f'{nforecast_hours + 24}H') + self.forecast_dates = list(rrule(freq=DAILY, dtstart=self.start_date, until=self.end_date)) + + @logit(logger) + def initialize(self) -> None: + """Initialize the work directory and process chemical emissions configuration. + + This method performs the following steps: + 1. Loads and parses the fire_emission.yaml.j2 template + 2. Sets up template variables for emission configuration + 3. Creates necessary working directories + 4. Copies required input files to working directory + 5. Sets up forecast dates and file paths for each date + + Parameters + ---------- + None + + Returns + ------- + None + + Raises + ------ + WorkflowException + If the YAML template file is not found + If required directories cannot be created + If file copying operations fail + + Notes + ----- + The method expects the following configuration to be available: + - HOMEgfs : str + Base directory containing workflow configuration + - DATA : str + Working directory path + - COMOUT_CHEM_INPUT : str + Output directory for chemical input files + - AERO_EMIS_FIRE_DIR : str + Directory containing fire emission data + - AERO_EMIS_FIRE_VERSION : str + Version of fire emission data (GBBEPx or QFED) + + The configuration is processed through a Jinja2 template system + and the resulting setup is stored in self.task_config. + """ + + if self.historical: + logger.info(f'Processing historical emissions for {self.start_date} to {self.end_date}') + + # print(self.task_config) + aero_inputs_dir = str(self.task_config.AERO_INPUTS_DIR) + aero_emis_fire = str(self.task_config.AERO_EMIS_FIRE) + aero_emis_fire_version = str(self.task_config.AERO_EMIS_FIRE_VERSION) + + logger.info(f'Using AERO_INPUTS_DIR: {aero_inputs_dir}') + logger.info(f'Using AERO_EMIS_FIRE: {aero_emis_fire}') + logger.info(f'Using AERO_EMIS_FIRE_VERSION: {aero_emis_fire_version}') + + AERO_EMIS_FIRE_DIR = os.path.join(aero_inputs_dir, + "nexus", + aero_emis_fire.upper()) + + logger.info(f'Final AERO_EMIS_FIRE_DIR: {AERO_EMIS_FIRE_DIR}') + + # find the forecast dates that are in the historical period for the given emission dataset + files_found = [] + for dates in self.forecast_dates: + if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': + gbbepx_vars = self.task_config.get('gbbepx_vars', ["co", "nox", "so2", "nh3", "bc", "oc"]) + files = self._find_gbbepx_files(dates, + gbbepx_vars, + version=self.task_config.AERO_EMIS_FIRE_VERSION, + aero_emis_fire_dir=AERO_EMIS_FIRE_DIR) + elif self.task_config.AERO_EMIS_FIRE.lower() == 'qfed': + + qfed_vars = self.task_config.get('qfed_vars', ["co", "nox", "so2", "nh3", "bc", "oc"]) + files = self._find_qfed_files(dates, + qfed_vars, + version=self.task_config.AERO_EMIS_FIRE_VERSION, + aero_emis_fire_dir=AERO_EMIS_FIRE_DIR) + files_found.extend(files) + logger.info(f'Found {len(files_found)} files for historical period') + else: + logger.info(f'Processing forecast emissions for {self.start_date}') + + if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': + files = self._find_gbbepx_files( + self.start_date, + self.task_config.gbbepx_vars, + version=self.task_config.AERO_EMIS_FIRE_VERSION, + vars=self.task_config.get('gbbepx_vars', ["co", "nox", "so2", "nh3", "bc", "oc"]) + ) + elif self.task_config.AERO_EMIS_FIRE.lower() == 'qfed': + # Get QFED variables with safe defaults + qfed_vars = self.task_config.get('qfed_vars', ["co", "nox", "so2", "nh3", "bc", "oc"]) + if isinstance(qfed_vars, str): + qfed_vars = qfed_vars.split() + # Ensure version is properly formatted + version = self.task_config.AERO_EMIS_FIRE_VERSION + if isinstance(version, int) or version.isdigit(): + version = str(version).zfill(3) # Pad with leading zeros if needed + + # Get fire emissions directory + aero_emis_fire_dir = getattr(self.task_config, 'AERO_EMIS_FIRE_DIR', None) + + files = self._find_qfed_files( + self.start_date, + vars=qfed_vars, + version=version, + aero_emis_fire_dir=aero_emis_fire_dir + ) + + # Fill the COMOUT_CHEM_INPUT with environment variables to create the full output path + processed_files = [] + for dt in self.forecast_dates: + processed_files.append( + dt.strftime("FIRE_EMIS_%Y%m%d.nc") + ) + + # pprint(self.task_config) + # Debug output for chemistry history directory + logger.info(f"Outputing files prescribed to {self.task_config.COMOUT_CHEM_INPUT}") + tmpl_dict = { + 'DATA': self.task_config.DATA, + 'COMOUT_CHEM_INPUT': self.task_config.COMOUT_CHEM_INPUT, + 'AERO_EMIS_FIRE_DIR': AERO_EMIS_FIRE_DIR, + 'AERO_EMIS_FIRE_VERSION': self.task_config.AERO_EMIS_FIRE_VERSION, + 'historical': self.historical, + 'forecast_dates': self.task_config.get('forecast_dates', []), + 'qfed_vars': self.task_config.get('qfed_vars', ["co", "nox", "so2", "nh3", "bc", "oc"]), + 'gbbepx_vars': self.task_config.get('gbbepx_vars', ["co", "nox", "so2", "nh3", "bc", "oc"]), + "rawfiles": files_found, + "startdate": self.start_date.strftime('%Y%m%d'), + "processed_files": processed_files, + } + + # Parse template and update task configuration + yaml_template = os.path.join(self.task_config.HOMEgfs, 'parm', 'chem', 'fire_emission.yaml.j2') + if not os.path.exists(yaml_template): + logger.warning(f"Template file not found: {yaml_template}, using default configuration") + yaml_config = {'fire_emission': {}} + else: + logger.debug(f'Parsing YAML template: {yaml_template}') + yaml_config = parse_j2yaml(yaml_template, tmpl_dict) + + self.task_config = AttrDict(**self.task_config, **yaml_config) + + # Create working directory and sync files using FileHandler + FileHandler(yaml_config.fire_emission.data_in).sync() + + input_files = {"rawfiles": [os.path.join(self.task_config.DATA, os.path.basename(file)) for file in files_found]} + self.task_config = AttrDict(**self.task_config, **input_files) + + @logit(logger) + def execute(self) -> None: + """Process emission files based on configuration. + + For GBBEPx files, converts them to COARDS compliant format and renames + according to template pattern. + For QFED files, combines all data into separate files for each forecast date. + + Parameters + ---------- + None + + Returns + ------- + None + + Notes + ----- + - Uses the task_config to determine the type of emissions to process + - For GBBEPx, it uses the GBBEPx_to_COARDS method to convert files + - For QFED, it combines files by date using the combine_qfed_files method + - Creates a separate output file for each date in self.forecast_dates + - Output files are named with pattern FIRE_EMIS_YYYYMMDD.nc for each date + - The processed files are added to the task_config for later use + - Uses the FileHandler for file operations + - Uses the logit decorator for logging + - Uses decode_cf=False when processing QFED files + """ + logger.info(f"Processing emission files in {self.task_config.DATA}") + + workdir = self.task_config.DATA + + processed_files = [] + + if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': + # Process GBBEPx files separately for each date + processed_files.extend(self._process_gbbepx_files(workdir)) + + elif self.task_config.AERO_EMIS_FIRE.lower() == 'qfed': + # Process QFED files for each forecast date + processed_files.extend(self._process_qfed_files(workdir)) + + # Add processed files to task_config + outdict = {'processed_files': processed_files} + self.task_config = AttrDict(**self.task_config, **outdict) + + logger.info("Emission processing execute phase complete") + + @logit(logger) + def finalize(self) -> None: + """Perform closing actions of the task. + Copy processed files from the DATA directory to COMOUT_CHEM_INPUT. + + Returns + ------- + None + + Notes + ----- + Only copies processed GBBEPx files or QFED files based on configuration + Uses FileHandler for reliable file operations with logging + """ + logger.info("Finalizing chemical emissions processing") + + FileHandler(self.task_config.fire_emission.data_out).sync() + + logger.info("Chemical emissions finalization complete") + + @logit(logger) + def _get_unique_months(self): + """Extract unique months and years from forecast dates. + + This method finds all unique months and years present in the forecast dates + range. Useful for monthly-based emissions processing. + + Returns + ------- + tuple + A tuple containing: + - set of unique months as zero-padded strings (01-12) + - set of unique years as integers + + Notes + ----- + Uses self.forecast_dates which should be populated during initialization + Months are returned as strings with leading zeros (e.g., '01' for January) + Years are returned as integers + """ + months = set(f"{date.month:02d}" for date in self.forecast_dates) + years = set(date.year for date in self.forecast_dates) + return months, years + + @logit(logger) + def _find_gbbepx_files(self, dates, version='v5r0'): + """Find GBBEPx files for the given date + + Parameters + ---------- + dates : str or list + Date or dates for which to find GBBEPx files + version : str + Version of GBBEPx files to search for + + Returns + ------- + List[str] + List of GBBEPx files for the given date(s) + """ + logger.info(f'Finding GBBEPx files for {dates}') + + # Find all possible months + months, years = self._get_unique_months() + + files_found = [] + # Find all possible files + for mon in months: + try: + emis_file_dir = os.path.join(self.task_config.AERO_EMIS_FIRE_DIR, version, mon) + if not os.path.exists(emis_file_dir): + logger.warning(f"Directory does not exist: {emis_file_dir}") + continue + + all_files = os.listdir(emis_file_dir) + + matching_files = [] + + # Look for both file patterns: + # Pattern 1: "GBBEPx-all01GRID_v4r0_blend_s202302240000000_e202302242359590_c202302250134090.nc" + # Pattern 2: "GBBEPx_all01GRID.emissions_v004_20150716.nc" + + for file_name in all_files: + match_found = False + + # Try pattern 1 with s/e/c date format + pattern1 = r".*s(\d{8}).*e(\d{8}).*c(\d{8}).*\.nc" + match = re.match(pattern1, file_name) + if match: + start_date = match.group(1) + # end_date = match.group(2) + create_date = match.group(3) + + # Check if the file's date matches any of our target dates + for date_str in date_strings: + # Match if the file start date is within our target dates + if date_str in start_date: + full_path = os.path.join(emis_file_dir, file_name) + matching_files.append(full_path) + logger.debug(f"Found GBBEPx file (pattern 1): {full_path}") + match_found = True + break + + # If no match yet, try pattern 2 with YYYYMMDD format at the end + if not match_found and "GBBEPx" in file_name: + pattern2 = r".*_(\d{8})\.nc" + match = re.match(pattern2, file_name) + if match: + file_date = match.group(1) + + # Check if the file's date matches any of our target dates + for date_str in date_strings: + if date_str in file_date: + full_path = os.path.join(emis_file_dir, file_name) + matching_files.append(full_path) + logger.debug(f"Found GBBEPx file (pattern 2): {full_path}") + break + + files_found.extend(matching_files) + except (FileNotFoundError, PermissionError) as e: + logger.warning(f"Error accessing directory {emis_file_dir}: {e}") + + return files_found + + @logit(logger) + def _find_qfed_files(self, dates, vars, version='061', aero_emis_fire_dir=None): + """Find QFED files for the given date(s) + + Parameters + ---------- + dates : str, datetime, or list + Date or dates for which to find QFED files + vars : list + List of variables to search for (e.g., bc, oc, co, etc.) + version : str + Version of QFED files to search for, will be zero-padded to 3 digits + aero_emis_fire_dir : str, optional + Directory containing fire emission data. If None, uses self.task_config.AERO_EMIS_FIRE_DIR + + Returns + ------- + List[str] + List of QFED files for the given date(s) and variables + """ + logger.info(f'Finding QFED files for {dates}') + + # Use provided directory or fall back to config value + logger.info(f'Using emissions directory: {aero_emis_fire_dir}') + + # ensure version is a string + version = str(version).zfill(3) + + # Convert single date to list for consistent processing + if not isinstance(dates, list): + dates = [dates] + + # Format dates properly + date_strings = [d.strftime('%Y%m%d') if hasattr(d, 'strftime') else str(d) for d in dates] + + files_found = [] + + for date in dates: + # Extract year and month from the date + if hasattr(date, 'year') and hasattr(date, 'month'): + year = str(date.year) + month = f"{date.month:02d}" + else: + # If date is a string, try to parse it + date_str = str(date) + if len(date_str) >= 8: # YYYYMMDD format + year = date_str[:4] + month = date_str[4:6] + else: + logger.warning(f"Cannot parse date format: {date}") + continue + + emis_file_dir = os.path.join(aero_emis_fire_dir, year, month) + + if not os.path.exists(emis_file_dir): + logger.warning(f"Directory does not exist: {emis_file_dir}") + continue + + # Format date string for file matching + date_str = date.strftime('%Y%m%d') if hasattr(date, 'strftime') else str(date) + if len(date_str) > 8: # Format may be YYYY-MM-DD + date_str = date_str.replace('-', '') + + for v in vars: + # Match pattern like qfed2.emis_bc.{version}.20200118.nc4 + v_pattern = f"qfed2.emis_{v}.{version}.{date_str}.nc4" + full_path = os.path.join(emis_file_dir, v_pattern) + + # If exact match exists + if os.path.exists(full_path): + files_found.append(full_path) + logger.debug(f"Found exact QFED file: {full_path}") + + if not os.path.exists(full_path): + logger.warning(f"File not found: {full_path}") + + if not files_found: + logger.warning(f"No QFED files found for dates {date_strings} and variables {vars}") + + return files_found + + @logit(logger) + def GBBEPx_to_COARDS(fname: Union[str, os.PathLike]) -> xr.Dataset: + """Convert GBBEPx file to COARDS compliant format + + Parameters + ---------- + fname : str | os.PathLike + Input GBBEPx file path + + Returns + ------- + xr.Dataset + COARDS compliant dataset + """ + logger.info(f"Converting {fname} to COARDS format") + f = xr.open_dataset(fname, decode_cf=False) + + # Handle time dimension + if 'Time' in f.dims: + f = f.rename({"Time": 'time'}) + f.time.attrs['long_name'] = 'time' + + # Modify latitude and longitude attributes + f = f.rename({'Longitude': 'lon', 'Latitude': 'lat'}) + + # Validate and normalize coordinates + # Check longitude range and monotonicity + if not (f.lon.diff('lon') > 0).all(): + raise WorkflowException("Longitude values must be strictly increasing") + + # Ensure longitude is in [-180, 180] range + f['lon'] = xr.where(f.lon > 180, f.lon - 360, f.lon) + f = f.sortby('lon') # Sort after potential wrapping + + # Check latitude monotonicity + if not (f.lat.diff('lat') > 0).all(): + raise WorkflowException("Latitude values must be strictly increasing") + + f.lon.attrs.update({'long_name': 'Longitude', 'units': 'degrees_east'}) + f.lat.attrs.update({'long_name': 'Latitude', 'units': 'degrees_north'}) + + # Remove Element dimension if present + if 'Element' in f.dims: + f = f.drop_dims('Element') + + # Update variable attributes + for v in f.data_vars: + if v not in ['FirePerc', 'QCAll', 'NumSensor', 'CloudPerc']: + f[v].attrs['_FillValue'] = -9999.0 + elif v == 'FirePerc': + f[v].attrs.update({'units': '-', 'long_name': 'percent_of_fire_in_grid_cell'}) + elif v == 'CloudPerc': + f[v].attrs.update({'units': '-', 'long_name': 'percent_of_clouds_in_grid_cell'}) + elif v == 'NumSensor': + f[v].attrs['units'] = '-' + + # Set global attributes + f.attrs.update({'format': 'NetCDF', 'title': 'GBBEPx Fire Emissions'}) + + return f + + @logit(logger) + def combine_qfed_files(self, qfed_files: List[str], output_path: str = None) -> xr.Dataset: + """Combine multiple QFED emission files into a single NetCDF file. + + Parameters + ---------- + qfed_files : List[str] + List of QFED file paths to combine + output_path : str, optional + Path where to save the combined file. If None, returns the dataset without saving. + + Returns + ------- + xr.Dataset + Combined dataset containing all QFED variables + + Notes + ----- + This function loads each file individually and combines them without using dask. + Uses decode_cf=False as required for QFED files. + Preprocessing renames biomass variables to uppercase emission type (e.g., biomass -> BC), + as well as related variables like biomass_tf -> BC_tf to avoid conflicts during merge. + Files are grouped by variable type, processed, and then combined using merge with compat='override'. + """ + if not qfed_files: + logger.warning("No QFED files provided to combine") + return None + + logger.info(f"Combining {len(qfed_files)} QFED files") + + try: + # Group files by variable type for easier processing + var_groups = {} + for file_path in qfed_files: + file_name = os.path.basename(file_path) + if "qfed2.emis_" in file_name: + parts = file_name.split('.') + if len(parts) >= 3: + var_type = parts[1].split('_')[1].lower() # Extract variable after emis_ + if var_type not in var_groups: + var_groups[var_type] = [] + var_groups[var_type].append(file_path) + + # Process each variable group + datasets_by_var = {} + for var_type, files in var_groups.items(): + logger.info(f"Processing {len(files)} files for variable: {var_type}") + var_datasets = [] + + for file_path in files: + logger.info(f"Opening file: {file_path}") + # Open dataset + ds = xr.open_dataset(file_path, decode_cf=False) + + # Get uppercase variable name for this type + var_name = var_type.upper() + + # Find all variables that need to be renamed for this emission type + rename_dict = {} + for dvar in ds.data_vars: + # Main biomass variable + if dvar == 'biomass': + rename_dict[dvar] = var_name + # Related variables like biomass_tf, biomass_xxx, etc. + elif dvar.startswith('biomass_'): + # Keep the suffix but prefix with the variable type + suffix = dvar[8:] # Get part after 'biomass_' + rename_dict[dvar] = f"{var_name}_{suffix}" + + # Rename all identified variables + if rename_dict: + logger.info(f"Renaming variables in {os.path.basename(file_path)}: {rename_dict}") + ds = ds.rename(rename_dict) + + var_datasets.append(ds) + + # Concatenate datasets for this variable along the time dimension if needed + if len(var_datasets) > 1: + try: + concat_ds = xr.concat(var_datasets, dim='time') + datasets_by_var[var_type] = concat_ds + except (ValueError, KeyError) as e: + logger.warning(f"Could not concatenate along time for {var_type}: {e}") + # If concatenation fails, just use the first dataset + datasets_by_var[var_type] = var_datasets[0] + for ds in var_datasets[1:]: + ds.close() + else: + datasets_by_var[var_type] = var_datasets[0] + + # Merge datasets across different variables + if datasets_by_var: + var_list = list(datasets_by_var.values()) + combined_ds = var_list[0] + + # Merge remaining datasets with compat='override' to handle conflicting values + for i in range(1, len(var_list)): + combined_ds = combined_ds.merge(var_list[i], compat='override') + + # Add global attributes + combined_ds.attrs.update({ + 'title': 'Combined QFED emissions', + 'source': 'QFED', + 'created_by': 'AerosolEmissions.combine_qfed_files', + 'creation_date': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + }) + + # Save to file if output path is provided + if output_path: + logger.info(f"Saving combined QFED dataset to {output_path}") + combined_ds.to_netcdf(output_path) + + # Close individual datasets to free memory + for ds_list in datasets_by_var.values(): + if hasattr(ds_list, 'close'): + ds_list.close() + + return combined_ds + else: + logger.warning("No valid datasets found to combine") + return None + + except Exception as e: + logger.error(f"Error combining QFED files: {e}") + import traceback + logger.error(f"Traceback: {traceback.format_exc()}") + return None + + + @logit(logger) + def _process_gbbepx_files(self, workdir: str) -> List[str]: + """Process GBBEPx files for each forecast date. + + Parameters + ---------- + workdir : str + Working directory path where processed files will be saved + + Returns + ------- + List[str] + List of processed file paths + + Notes + ----- + This method processes GBBEPx files for each forecast date by: + 1. Filtering raw files by date (if date filtering logic is implemented) + 2. Converting files to COARDS format using GBBEPx_to_COARDS + 3. Saving the processed dataset to a NetCDF file + 4. Returning the list of processed file paths + """ + logger.info(f"Processing GBBEPx files for {len(self.forecast_dates)} forecast dates") + processed_files = [] + + for forecast_date in self.forecast_dates: + date_str = forecast_date.strftime('%Y%m%d') + logger.info(f"Processing GBBEPx files for date {date_str}") + + # Filter files for this date - implement date filtering if needed + date_files = [] + for file in self.task_config.rawfiles: + # Add logic here to filter files by date if needed + date_files.append(file) + + if date_files: + # Process files for this date + ds = self.GBBEPx_to_COARDS(date_files[0]) # Use the first file for this date + + # Create output filename with date + outfile_name = f"FIRE_EMIS_{date_str}.nc" + outfile = os.path.join(workdir, outfile_name) + + # Save the processed dataset + comp = dict(zlib=True, complevel=2) + encoding = {var: comp for var in ds.data_vars} + ds.to_netcdf(outfile, encoding=encoding) + logger.info(f"Processed emission file saved to {outfile}") + + # Add to processed files list + processed_files.append(outfile) + + # Close dataset + ds.close() + else: + logger.warning(f"No GBBEPx files found for date {date_str}") + + return processed_files + + @logit(logger) + def _process_qfed_files(self, workdir: str) -> List[str]: + """Process QFED files for each forecast date. + + Parameters + ---------- + workdir : str + Working directory path where processed files will be saved + + Returns + ------- + List[str] + List of processed file paths + + Notes + ----- + This method processes QFED files for each forecast date by: + 1. Filtering raw files by date + 2. Combining files for each date using combine_qfed_files + 3. Saving the combined dataset to a NetCDF file + 4. Returning the list of processed file paths + """ + logger.info(f"Processing QFED files for {len(self.forecast_dates)} forecast dates") + processed_files = [] + + for forecast_date in self.forecast_dates: + date_str = forecast_date.strftime('%Y%m%d') + logger.info(f"Processing QFED files for date {date_str}") + + # Filter files for this date + date_files = [] + for file_path in self.task_config.rawfiles: + file_name = os.path.basename(file_path) + if date_str in file_name: + date_files.append(file_path) + + if date_files: + logger.info(f"Found {len(date_files)} QFED files for date {date_str}") + + # Combine QFED files for this date + ds = self.combine_qfed_files(date_files) + + if ds is not None: + # Create output filename with date + outfile_name = f"FIRE_EMIS_{date_str}.nc" + outfile = os.path.join(workdir, outfile_name) + + # Save the processed dataset + comp = dict(zlib=True, complevel=2) + encoding = {var: comp for var in ds.data_vars} + ds.to_netcdf(outfile, encoding=encoding) + logger.info(f"Processed emission file for {date_str} saved to {outfile}") + + # Add to processed files list + processed_files.append(outfile) + + # Close dataset + ds.close() + else: + logger.warning(f"Failed to combine QFED files for date {date_str}") + else: + logger.warning(f"No QFED files found for date {date_str}") + + return processed_files diff --git a/ush/python/pygfs/task/nxs_emission.py b/ush/python/pygfs/task/nxs_emission.py new file mode 100644 index 00000000000..a20503f94c7 --- /dev/null +++ b/ush/python/pygfs/task/nxs_emission.py @@ -0,0 +1,375 @@ +#!/usr/bin/env python3 + +import os +import re +import xarray as xr +import subprocess +import cftime +from logging import getLogger +from typing import Dict, Any, Union, List +from dateutil.rrule import DAILY, HOURLY, rrule +from pprint import pprint +from jinja2 import Environment, FileSystemLoader +from wxflow import (AttrDict, + FileHandler, + parse_j2yaml, + logit, + Task, + to_timedelta, + WorkflowException, + Executable, which) + +logger = getLogger(__name__.split('.')[-1]) + + +class NXSEmissions(Task): + """NEXUS Emissions pre-processing Task + """ + + @logit(logger, name="NXSEmissions") + def __init__(self, config: Dict[str, Any]) -> None: + """Constructor for the NEXUS Emissions task + + Parameters + ---------- + config : Dict[str, Any] + Incoming configuration for the task from the environment + + Returns + ------- + None + """ + super().__init__(config) + + self.task_config = AttrDict(config) + self.AERO_INPUTS_DIR = self.task_config.get('AERO_INPUTS_DIR', None) + self.COMOUT_CHEM_INPUT = self.task_config.get('COMOUT_CHEM_INPUT', None) + nforecast_hours = self.task_config["FHMAX_GFS"] + self.start_date = self.task_config["SDATE"] - to_timedelta('12H') + self.end_date = self.task_config["EDATE"] + to_timedelta('12H') + frequency = self.task_config.get("NXS_DIAG_FREQ", "Hourly") + if frequency == "Hourly": + self.forecast_dates = list(rrule(freq=HOURLY, dtstart=self.start_date, until=self.end_date)) + elif frequency == 'Daily': + self.forecast_dates = list(rrule(freq=DAILY, dtstart=self.start_date, until=self.end_date)) + else: + raise WorkflowException(f"Unsupported NXS_DIAG_FREQ: {frequency}") + + self.forecast_dates_daily = list(rrule(freq=DAILY, dtstart=self.start_date, until=self.end_date)) + + logger.info(f"NXSEmissions initialized with start date: {self.start_date}, end date: {self.end_date}") + + @logit(logger) + def initialize(self) -> None: + """Initialize the work directory and process chemical emissions configuration. + + This method performs the following steps: + 1. Render the NEXUS configuration files using Jinja2 templates + found in `parm/chem/nexus/$NXS_CONFIG` + 2. Sets up template variables for emission configuration + 3. Creates necessary working directories + 4. Copies required input files to working directory + 5. Sets up forecast dates and file paths for each date + + Parameters + ---------- + None + + Returns + ------- + None + + Raises + ------ + WorkflowException + If the YAML template file is not found + If required directories cannot be created + If file copying operations fail + + Notes + ----- + The method expects the following configuration to be available: + - HOMEgfs : str + Base directory containing workflow configuration + - DATA : str + Working directory path + - COMOUT_CHEM_INPUT : str + Output directory for chemical input files + - AERO_EMIS_FIRE_DIR : str + Directory containing fire emission data + + The configuration is processed through a Jinja2 template system + and the resulting setup is stored in self.task_config. + """ + logger.info("Initializing NEXUS emissions pre-processing task") + + # + logger.info("Rendering NEXUS configuration files") + # Check for required NEXUS configuration parameters + nxs_config_set = self.task_config.get('NXS_CONFIG', None) + if not nxs_config_set: + raise WorkflowException("NXS_CONFIG must be set in task configuration") + nxs_config_dir = self.task_config.get('NXS_CONFIG_DIR', None) + if not nxs_config_dir: + raise WorkflowException("NXS_CONFIG_DIR must be set in task configuration") + nxs_input_dir = self.task_config.get('NXS_INPUT_DIR', None) + if not nxs_input_dir: + raise WorkflowException("NXS_INPUT_DIR must be set in task configuration") + # Default NXS_TSTEP to 3600 seconds (1 hour) if not set + nxs_tstep = self.task_config.get('NXS_TSTEP', 3600) + if not nxs_tstep: + raise WorkflowException("NXS_TSTEP must be set in task configuration") + + logger.info(f"Using NXS_CONFIG: {nxs_config_set}") + logger.info(f"Using NXS_CONFIG_DIR: {nxs_config_dir}") + logger.info(f"Using NXS_INPUT_DIR: {nxs_input_dir}") + logger.info(f"Using NXS_TSTEP: {nxs_tstep}") + + # Check for grid parameters + if not self.task_config.get('NXS_NX', None): + raise WorkflowException("NXS_NX must be set in task configuration") + if not self.task_config.get('NXS_NY', None): + raise WorkflowException("NXS_NY must be set in task configuration") + if not self.task_config.get('NXS_NZ', None): + raise WorkflowException("NXS_NZ must be set in task configuration") + if not self.task_config.get('NXS_XMIN', None): + raise WorkflowException("NXS_XMIN must be set in task configuration") + if not self.task_config.get('NXS_XMAX', None): + raise WorkflowException("NXS_XMAX must be set in task configuration") + if not self.task_config.get('NXS_YMIN', None): + raise WorkflowException("NXS_YMIN must be set in task configuration") + + logger.info(f"Grid parameters: NXS_NX={self.task_config.NXS_NX}") + logger.info(f"Grid parameters: NXS_NY={self.task_config.NXS_NY}") + logger.info(f"Grid parameters: NXS_NZ={self.task_config.NXS_NZ}") + logger.info(f"Grid parameters: NXS_XMIN={self.task_config.NXS_XMIN}") + logger.info(f"Grid parameters: NXS_XMAX={self.task_config.NXS_XMAX}") + logger.info(f"Grid parameters: NXS_YMIN={self.task_config.NXS_YMIN}") + logger.info(f"Grid parameters: NXS_YMAX={self.task_config.NXS_YMAX}") + + processed_nxs_files = [] + final_output_files = [] + sorted_dates = sorted(self.forecast_dates) + for d in sorted_dates[:-1]: + fname = f"{self.task_config.NXS_DIAG_PREFIX}.{d.strftime('%Y%m%d%H')}00.nc" + fname_final = f"{self.task_config.NXS_DIAG_PREFIX}.{d.strftime('%Y%m%d')}.nc" + processed_nxs_files.append(fname) + final_output_files.append(fname_final) + self.processed_nxs_files = processed_nxs_files + # render the NEXUS configuration files + if not os.path.exists(nxs_config_dir): + raise WorkflowException(f"NEXUS configuration file not found: {nxs_config_dir}") + logger.info(f"Rendering NEXUS configuration from {nxs_config_dir}") + tmpl_dict = { + 'NXS_CONFIG': nxs_config_set, + 'NXS_CONFIG_DIR': nxs_config_dir, + 'NXS_INPUT_DIR': nxs_input_dir, + 'NXS_DIAG_PREFIX': self.task_config.NXS_DIAG_PREFIX, + 'NXS_TSTEP': nxs_tstep, + 'NXS_NX': self.task_config.NXS_NX, + 'NXS_NY': self.task_config.NXS_NY, + 'NXS_NZ': self.task_config.NXS_NZ, + 'NXS_XMIN': self.task_config.NXS_XMIN, + 'NXS_XMAX': self.task_config.NXS_XMAX, + 'NXS_YMIN': self.task_config.NXS_YMIN, + 'NXS_YMAX': self.task_config.NXS_YMAX, + 'LOCAL_INPUT_DIR': os.path.join(self.task_config.DATA, 'INPUT'), + 'NXS_EXECUTABLE': os.path.join(self.task_config.get('HOMEgfs', None), "exec/nexus.x"), + "WORK_DIR": self.task_config.DATA, + "NXS_DO_MEGAN": self.task_config.get('NXS_DO_MEGAN', False), + "NXS_DO_CEDS2019": self.task_config.get('NXS_DO_CEDS2019', True), + "NXS_DO_CEDS2024": self.task_config.get('NXS_DO_CEDS2024', False), + "NXS_DO_HTAPv2": self.task_config.get('NXS_DO_HTAPv2', True), + "NXS_DO_HTAPv3": self.task_config.get('NXS_DO_HTAPv3', False), + "NXS_DO_CAMS": self.task_config.get('NXS_DO_CAMS', False), + "NXS_DO_CAMSTEMPO": self.task_config.get('NXS_DO_CAMSTEMPO', False), + "start_date": self.start_date.strftime('%Y-%m-%d %H:%M:%S'), + "end_date": self.end_date.strftime('%Y-%m-%d %H:%M:%S'), + "FINAL_OUTPUT": final_output_files, + "COMOUT_CHEM_INPUT": self.task_config.COMOUT_CHEM_INPUT, + "COMOUT_CHEM_RESTART": self.task_config.COMOUT_CHEM_RESTART, + "RestartFile": f"HEMCO_restart.{self.end_date.strftime('%Y%m%d%H')}00.nc", + "processed_nxs_files": processed_nxs_files, + + } + + yaml_template = os.path.join(self.task_config.HOMEgfs, 'parm', 'chem', 'nxs_emission.yaml.j2') + if not os.path.exists(yaml_template): + logger.warning(f"Template file not found: {yaml_template}, using default configuration") + yaml_config = {'nxs_emission': {}} + else: + logger.debug(f'Parsing YAML template: {yaml_template}') + yaml_config = parse_j2yaml(yaml_template, tmpl_dict) + + # Add yaml configuration to task_config + self.task_config = AttrDict(**self.task_config, **yaml_config) + + # Link NEXUS input directory to the working directory + FileHandler(self.task_config.nxs_emission.data_in).sync() + logger.info(f"NEXUS input directory linked to {self.task_config.DATA}") + + # Render NXS Grid File + file_loader = FileSystemLoader(self.task_config.NXS_CONFIG_DIR) + env = Environment(loader=file_loader) + nxs_grid_template = env.get_template(f"{self.task_config.NXS_GRID_NAME}.j2") + self.task_config.NXS_GRID_TEMPLATE = nxs_grid_template.render(tmpl_dict) + outfile = os.path.join(self.task_config.DATA, self.task_config.NXS_GRID_NAME) + _write_txt_file(self.task_config.NXS_GRID_TEMPLATE, outfile) + logger.info(f"NEXUS grid file rendered successfully: written to {outfile}") + + # Render NXS Config File + nxs_config_template = env.get_template(f"{self.task_config.NXS_CONFIG_NAME}.j2") + self.task_config.NXS_CONFIG_TEMPLATE = nxs_config_template.render(tmpl_dict) + outfile = os.path.join(self.task_config.DATA, self.task_config.NXS_CONFIG_NAME) + _write_txt_file(self.task_config.NXS_CONFIG_TEMPLATE, outfile) + logger.info(f"NEXUS config file rendered successfully: written to {outfile}") + + # Render NXS Time File + nxs_time_template = env.get_template(f"{self.task_config.NXS_TIME_NAME}.j2") + self.task_config.NXS_TIME_TEMPLATE = nxs_time_template.render(tmpl_dict) + outfile = os.path.join(self.task_config.DATA, self.task_config.NXS_TIME_NAME) + _write_txt_file(self.task_config.NXS_TIME_TEMPLATE, outfile) + logger.info(f"NEXUS time file rendered successfully: written to {outfile}") + + + # Render NXS Diag File + nxs_diag_template = env.get_template(f"{self.task_config.NXS_DIAG_NAME}.j2") + self.task_config.NXS_DIAG_TEMPLATE = nxs_diag_template.render(tmpl_dict) + outfile = os.path.join(self.task_config.DATA, self.task_config.NXS_DIAG_NAME) + _write_txt_file(self.task_config.NXS_DIAG_TEMPLATE, outfile) + logger.info(f"NEXUS diag file rendered successfully: written to {outfile}") + + # Render NXS Spec File + nxs_spec_template = env.get_template(f"{self.task_config.NXS_SPEC_NAME}.j2") + self.task_config.NXS_SPEC_TEMPLATE = nxs_spec_template.render(tmpl_dict) + outfile = os.path.join(self.task_config.DATA, self.task_config.NXS_SPEC_NAME) + _write_txt_file(self.task_config.NXS_SPEC_TEMPLATE, outfile) + logger.info(f"NEXUS spec file rendered successfully: written to {outfile}") + + # pprint(self.task_config) + + @logit(logger) + def execute(self) -> None: + """Run NEXUS emission preprocessor based on configuration. + + This will run the NEXUS preprocessor executable with the provided configuration. + It will process the emission files based on the task configuration and forecast dates. + It will also handle different types of emissions based on the configuration. + + + Parameters + ---------- + None + + Returns + ------- + None + + Notes + ----- + - This method assumes that the NEXUS preprocessor executable is available in the PATH. + - It will log the processing steps and any issues encountered. + Raises + ------ + WorkflowException + If the NEXUS preprocessor executable is not found + If the working directory does not exist + If no emission files are found for processing + """ + logger.info(f"Running NEXUS emission preprocessor in {self.task_config.DATA}") + logger.info(f"NEXUS Logs: {self.task_config.DATA}/stdout") + logger.info(f"NEXUS Logs: {self.task_config.DATA}/stderr") + logger.info(f"NEXUS Logs: {self.task_config.DATA}/NEXUS.log") + + if not os.path.exists(self.task_config.DATA): + raise WorkflowException(f"Working directory does not exist: {self.task_config.DATA}") + + # pprint(self.task_config) + exe = Executable(self.task_config.launcher) + arg_list = ['--ntasks', + str(1), + 'nexus.x', + '-c', + self.task_config.NXS_CONFIG_NAME] + exe(*arg_list, output='stdout', error='stderr') + + logger.info("Concatenating processed NEXUS files...") + + files = sorted(self.processed_nxs_files) + dsets = [] + for f in files: + dsets.append(xr.open_dataset(f, decode_cf=False)) + + # Concatenate along time dimension + ds = xr.concat(dsets, dim="time") + + # Convert raw time values to datetime objects using cftime + if 'time' not in ds.dims: + raise WorkflowException("No 'time' dimension found in NEXUS output dataset.") + + time_var = ds['time'] + time_units = time_var.attrs.get('units', None) + time_calendar = time_var.attrs.get('calendar', 'standard') + if time_units is None: + raise WorkflowException("No 'units' attribute found for time variable.") + + # Convert time values to datetime objects + time_vals = time_var.values + time_dt = cftime.num2date(time_vals, units=time_units, calendar=time_calendar) + + # Group indices by day + from collections import defaultdict + day_to_indices = defaultdict(list) + for idx, dt in enumerate(time_dt): + day_to_indices[dt.strftime('%Y%m%d')].append(idx) + + encoding = {var: {"zlib": True, "complevel": 4} for var in ds.data_vars} + for day_str, indices in day_to_indices.items(): + daily_ds = ds.isel(time=indices) + outname = f"{self.task_config.NXS_DIAG_PREFIX}.{day_str}.nc" + daily_ds.to_netcdf(outname, format="NETCDF4", encoding=encoding) + logger.info(f"Wrote daily output: {outname}") + + logger.info("NEXUS emission processing execute phase complete") + + @logit(logger) + def finalize(self) -> None: + """Perform closing actions of the task. + Copy processed files from the DATA directory to COMOUT_CHEM_INPUT. + + Returns + ------- + None + + Notes + ----- + Only copies processed NEXUS files to the output directory. + Uses FileHandler for reliable file operations with logging + """ + logger.info("Finalizing NEXUS emissions processing") + + FileHandler(self.task_config.nxs_emission.data_out).sync() + + logger.info("Chemical emissions finalization complete") + +def _write_txt_file(content: str, file_path: Union[str, os.PathLike]) -> None: + """Write content to a text file. + + Parameters + ---------- + content : str + Content to write to the file. + file_path : Union[str, os.PathLike] + Path where the file will be created. + + Returns + ------- + None + + Notes + ----- + If the directory does not exist, it will be created. + """ + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, 'w') as f: + f.write(content) From c90dd91e616806f6c0cd1dbbbd4feb9c6361109f Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Wed, 6 Aug 2025 15:15:01 -0400 Subject: [PATCH 011/132] add nexus submodule --- sorc/nexus.fd | 1 + 1 file changed, 1 insertion(+) create mode 160000 sorc/nexus.fd diff --git a/sorc/nexus.fd b/sorc/nexus.fd new file mode 160000 index 00000000000..f6c76e6c5ac --- /dev/null +++ b/sorc/nexus.fd @@ -0,0 +1 @@ +Subproject commit f6c76e6c5ac4df86ce5e78f91fc076424934ed14 From dec6de52dc86699b8153db13a37823898256b1e1 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Thu, 7 Aug 2025 11:00:53 -0400 Subject: [PATCH 012/132] Update fire emissions directory path and adjust GOCART date handling logic --- dev/parm/config/gcafs/config.aero.j2 | 2 +- ush/forecast_predet.sh | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dev/parm/config/gcafs/config.aero.j2 b/dev/parm/config/gcafs/config.aero.j2 index d3a554f32db..1d9bf377ebd 100644 --- a/dev/parm/config/gcafs/config.aero.j2 +++ b/dev/parm/config/gcafs/config.aero.j2 @@ -31,7 +31,7 @@ export dnats_aero=2 export AERO_EMIS_FIRE="qfed" export AERO_EMIS_FIRE_VERSION="061" export AERO_EMIS_FIRE_HIST=1 # Use historical fire emissions -export FIRE_EMIS_NRT_DIR="${DCOMROOT}/YYYYMMDD/firewx" # Directory containing NRT fire emissions +export FIRE_EMIS_NRT_DIR="" #TODO set to DCOM for WCOSS2 "${DCOMROOT}/YYYYMMDD/firewx" # Directory containing NRT fire emissions #=============================================================================== # 3. NEXUS settings diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh index 508d2382c84..bd5b814ccfb 100755 --- a/ush/forecast_predet.sh +++ b/ush/forecast_predet.sh @@ -794,7 +794,8 @@ GOCART_predet(){ local current local YYYYMMDDHH current="${current_cycle_begin}" - while [[ "${current}" -le "${current_cycle_end}" ]]; do + cycleend=$(date -d "${current_cycle_end:0:8} ${current_cycle_end:8:2} +24 hour" +%Y%m%d%H) + while [[ "${current}" -le "${cycleend}" ]]; do # Validate current is a valid date string if ! YYYYMMDD=$(date -d "${current:0:8} ${current:8:2}:00:00" +%Y%m%d 2>/dev/null); then echo "FATAL ERROR: Invalid date string '${current}' in GOCART_predet, ABORT!" @@ -817,8 +818,8 @@ GOCART_predet(){ # Copy NXS Emission Files ChemInput directory # NXS files are hourly, so we need to loop through each hour in the cycle current=$(date -d "${current_cycle_begin:0:8} ${current_cycle_begin:8:2}" +%Y%m%d%H) - cycleend=$(date -d "${current_cycle_end:0:8} ${current_cycle_end:8:2} +12 hour" +%Y%m%d%H) - while [[ "${current}" -le "${current_cycle_end}" ]]; do + cycleend=$(date -d "${current_cycle_end:0:8} ${current_cycle_end:8:2} +24 hour" +%Y%m%d%H) + while [[ "${current}" -le "${cycleend}" ]]; do if ! YYYYMMDD=$(date -d "${current:0:8} ${current:8:2}:00:00" +%Y%m%d 2>/dev/null); then echo "FATAL ERROR: Invalid date string '${current}' in GOCART_predet, ABORT!" exit 1 From 0e512c93740536ff3786c7f03201e310ec3604ba Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Thu, 7 Aug 2025 11:08:46 -0400 Subject: [PATCH 013/132] pycodestyle fixes --- scripts/exglobal_prep_emissions.py | 2 -- ush/python/pygfs/task/chem_fire_emission.py | 3 +-- ush/python/pygfs/task/nxs_emission.py | 5 +---- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/scripts/exglobal_prep_emissions.py b/scripts/exglobal_prep_emissions.py index dadd1f39c3f..3876fd6c2f3 100755 --- a/scripts/exglobal_prep_emissions.py +++ b/scripts/exglobal_prep_emissions.py @@ -30,5 +30,3 @@ nxsemis.configure() nxsemis.execute() nxsemis.finalize() - - diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index c4b969c57aa..aeb0c4bc5f6 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -45,7 +45,7 @@ def __init__(self, config: Dict[str, Any]) -> None: self.AERO_INPUTS_DIR = self.task_config.get('AERO_INPUTS_DIR', None) self.COMOUT_CHEM_INPUT = self.task_config.get('COMOUT_CHEM_INPUT', None) nforecast_hours = self.task_config["FHMAX_GFS"] - self.start_date = self.task_config["PDY"] #- datetime.timedelta(days=1) + self.start_date = self.task_config["PDY"] self.end_date = self.start_date + to_timedelta(f'{nforecast_hours + 24}H') self.forecast_dates = list(rrule(freq=DAILY, dtstart=self.start_date, until=self.end_date)) @@ -642,7 +642,6 @@ def combine_qfed_files(self, qfed_files: List[str], output_path: str = None) -> logger.error(f"Traceback: {traceback.format_exc()}") return None - @logit(logger) def _process_gbbepx_files(self, workdir: str) -> List[str]: """Process GBBEPx files for each forecast date. diff --git a/ush/python/pygfs/task/nxs_emission.py b/ush/python/pygfs/task/nxs_emission.py index a20503f94c7..953f36982bf 100644 --- a/ush/python/pygfs/task/nxs_emission.py +++ b/ush/python/pygfs/task/nxs_emission.py @@ -231,7 +231,6 @@ def initialize(self) -> None: _write_txt_file(self.task_config.NXS_TIME_TEMPLATE, outfile) logger.info(f"NEXUS time file rendered successfully: written to {outfile}") - # Render NXS Diag File nxs_diag_template = env.get_template(f"{self.task_config.NXS_DIAG_NAME}.j2") self.task_config.NXS_DIAG_TEMPLATE = nxs_diag_template.render(tmpl_dict) @@ -246,8 +245,6 @@ def initialize(self) -> None: _write_txt_file(self.task_config.NXS_SPEC_TEMPLATE, outfile) logger.info(f"NEXUS spec file rendered successfully: written to {outfile}") - # pprint(self.task_config) - @logit(logger) def execute(self) -> None: """Run NEXUS emission preprocessor based on configuration. @@ -256,7 +253,6 @@ def execute(self) -> None: It will process the emission files based on the task configuration and forecast dates. It will also handle different types of emissions based on the configuration. - Parameters ---------- None @@ -352,6 +348,7 @@ def finalize(self) -> None: logger.info("Chemical emissions finalization complete") + def _write_txt_file(content: str, file_path: Union[str, os.PathLike]) -> None: """Write content to a text file. From 4bf0a40d23dee6892bbc626951f828e8c343471e Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Thu, 7 Aug 2025 11:15:12 -0400 Subject: [PATCH 014/132] shellcheck fixes --- dev/parm/config/gcafs/config.aero.j2 | 42 ++++++++++++++-------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/dev/parm/config/gcafs/config.aero.j2 b/dev/parm/config/gcafs/config.aero.j2 index 1d9bf377ebd..5b49b06ad13 100644 --- a/dev/parm/config/gcafs/config.aero.j2 +++ b/dev/parm/config/gcafs/config.aero.j2 @@ -30,8 +30,8 @@ export dnats_aero=2 # Biomass burning emission dataset. Choose from: gbbepx, qfed, none export AERO_EMIS_FIRE="qfed" export AERO_EMIS_FIRE_VERSION="061" -export AERO_EMIS_FIRE_HIST=1 # Use historical fire emissions -export FIRE_EMIS_NRT_DIR="" #TODO set to DCOM for WCOSS2 "${DCOMROOT}/YYYYMMDD/firewx" # Directory containing NRT fire emissions +export AERO_EMIS_FIRE_HIST=1 # Use historical fire emissions | 1 = true 0 = false +export FIRE_EMIS_NRT_DIR="" #TODO: set to DCOM for WCOSS2 "${DCOMROOT}/YYYYMMDD/firewx" # Directory containing NRT fire emissions #=============================================================================== # 3. NEXUS settings @@ -39,49 +39,49 @@ export FIRE_EMIS_NRT_DIR="" #TODO set to DCOM for WCOSS2 "${DCOMROOT}/YYYYMMDD/f # NEXUS aerosol emissions dataset. Choose from: gocart, none # NEXUS configuration set -export NXS_CONFIG={{ NXS_CONFIG | default("gocart") }} # Options: gocart, none +export NXS_CONFIG="{{ NXS_CONFIG | default('gocart') }}" # Options: gocart, none export NXS_CONFIG_DIR="${PARMgfs}/chem/nexus/${NXS_CONFIG}" # Directory containing NEXUS configuration files #--------------------- # NEXUS root directory #--------------------- -export NXS_INPUT_DIR="{{ AERO_INPUTS_DIR }}" - +export NXS_NY="{{ NXS_NY | default(720) }}" +export NXS_XMIN="{{ NXS_XMIN | default(-180.0) }}" #-------------------------- # NEXUS Time Step (seconds) #-------------------------- -NXS_TSTEP={{ NXS_TSTEP | default(3600) }} # Default NEXUS time step in seconds +export NXS_TSTEP="{{ NXS_TSTEP | default(3600) }}" # Default NEXUS time step in seconds #------------------ # NEXUS Grid #------------------ -export NXS_NX={{ NXS_NX | default(1440) }} -export NXS_NY={{ NXS_NY | default(720) }} -export NXS_XMIN={{ NXS_XMIN | default(-180.0) }} -export NXS_XMAX={{ NXS_XMAX | default(180.0) }} -export NXS_YMIN={{ NXS_YMIN | default(-90.0) }} -export NXS_YMAX={{ NXS_YMAX | default(90.0) }} -export NXS_NZ={{ NXS_NZ | default(1) }} +export NXS_NX="{{ NXS_NX | default(1440) }}" +export NXS_NY="{{ NXS_NY | default(720) }}" +export NXS_XMIN="{{ NXS_XMIN | default(-180.0) }}" +export NXS_XMAX="{{ NXS_XMAX | default(180.0) }}" +export NXS_YMIN="{{ NXS_YMIN | default(-90.0) }}" +export NXS_YMAX="{{ NXS_YMAX | default(90.0) }}" +export NXS_NZ="{{ NXS_NZ | default(1) }}" #------------------- # NEXUS Config Files #------------------- -export NXS_GRID_NAME={{ NXS_GRID_NAME | default("HEMCO_sa_Grid.rc") }} -export NXS_TIME_NAME={{ NXS_TIME_NAME | default("HEMCO_sa_Time.rc") }} -export NXS_DIAG_NAME={{ NXS_DIAG_NAME | default("HEMCO_sa_Diag.rc") }} -export NXS_SPEC_NAME={{ NXS_SPEC_NAME | default("HEMCO_sa_Spec.rc") }} -export NXS_CONFIG_NAME={{ NXS_CONFIG_NAME | default("NEXUS_Config.rc") }} +export NXS_GRID_NAME="{{ NXS_GRID_NAME | default("HEMCO_sa_Grid.rc") }}" +export NXS_TIME_NAME="{{ NXS_TIME_NAME | default("HEMCO_sa_Time.rc") }}" +export NXS_DIAG_NAME="{{ NXS_DIAG_NAME | default("HEMCO_sa_Diag.rc") }}" +export NXS_SPEC_NAME="{{ NXS_SPEC_NAME | default("HEMCO_sa_Spec.rc") }}" +export NXS_CONFIG_NAME="{{ NXS_CONFIG_NAME | default("NEXUS_Config.rc") }}" #------------------ # NEXUS Diagnostics #------------------ -export NXS_DIAG_PREFIX={{ NXS_DIAG_PREFIX | default("NXS_DIAG") }} -export NXS_DIAG_FREQ={{ NXS_DIAG_FREQ | default("Hourly") }} # Options: Hourly, Daily, Monthly +export NXS_DIAG_PREFIX="{{ NXS_DIAG_PREFIX | default("NXS_DIAG") }}" +export NXS_DIAG_FREQ="{{ NXS_DIAG_FREQ | default("Hourly") }}" # Options: Hourly, Daily, Monthly #------------------ # NEXUS Logging #------------------ -export NXS_LOGFILE={{ NXS_LOGFILE | default("NEXUS.log") }} +export NXS_LOGFILE="{{ NXS_LOGFILE | default("NEXUS.log") }}" #------------------ # NEXUS Emissions From 14d3bd789547af8e1e20ddadccb7d30152e9d42a Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Thu, 7 Aug 2025 11:18:58 -0400 Subject: [PATCH 015/132] more shellcheck fixes --- dev/parm/config/gcafs/config.aero.j2 | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dev/parm/config/gcafs/config.aero.j2 b/dev/parm/config/gcafs/config.aero.j2 index 5b49b06ad13..37af512db1e 100644 --- a/dev/parm/config/gcafs/config.aero.j2 +++ b/dev/parm/config/gcafs/config.aero.j2 @@ -66,22 +66,22 @@ export NXS_NZ="{{ NXS_NZ | default(1) }}" #------------------- # NEXUS Config Files #------------------- -export NXS_GRID_NAME="{{ NXS_GRID_NAME | default("HEMCO_sa_Grid.rc") }}" -export NXS_TIME_NAME="{{ NXS_TIME_NAME | default("HEMCO_sa_Time.rc") }}" -export NXS_DIAG_NAME="{{ NXS_DIAG_NAME | default("HEMCO_sa_Diag.rc") }}" -export NXS_SPEC_NAME="{{ NXS_SPEC_NAME | default("HEMCO_sa_Spec.rc") }}" -export NXS_CONFIG_NAME="{{ NXS_CONFIG_NAME | default("NEXUS_Config.rc") }}" +export NXS_GRID_NAME="{{ NXS_GRID_NAME | default('HEMCO_sa_Grid.rc') }}" +export NXS_TIME_NAME="{{ NXS_TIME_NAME | default('HEMCO_sa_Time.rc') }}" +export NXS_DIAG_NAME="{{ NXS_DIAG_NAME | default('HEMCO_sa_Diag.rc') }}" +export NXS_SPEC_NAME="{{ NXS_SPEC_NAME | default('HEMCO_sa_Spec.rc') }}" +export NXS_CONFIG_NAME="{{ NXS_CONFIG_NAME | default('NEXUS_Config.rc') }}" #------------------ # NEXUS Diagnostics #------------------ -export NXS_DIAG_PREFIX="{{ NXS_DIAG_PREFIX | default("NXS_DIAG") }}" -export NXS_DIAG_FREQ="{{ NXS_DIAG_FREQ | default("Hourly") }}" # Options: Hourly, Daily, Monthly +export NXS_DIAG_PREFIX="{{ NXS_DIAG_PREFIX | default('NXS_DIAG') }}" +export NXS_DIAG_FREQ="{{ NXS_DIAG_FREQ | default('Hourly') }}" # Options: Hourly, Daily, Monthly #------------------ # NEXUS Logging #------------------ -export NXS_LOGFILE="{{ NXS_LOGFILE | default("NEXUS.log") }}" +export NXS_LOGFILE="{{ NXS_LOGFILE | default('NEXUS.log') }}" #------------------ # NEXUS Emissions From 3ccdb91a0eebc1defe93671faf024d8ae847c9a8 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Thu, 7 Aug 2025 12:19:45 -0400 Subject: [PATCH 016/132] minor syntax fix that didn't cause error --- dev/parm/config/gcafs/config.aero.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/parm/config/gcafs/config.aero.j2 b/dev/parm/config/gcafs/config.aero.j2 index 37af512db1e..8bc8ebc619f 100644 --- a/dev/parm/config/gcafs/config.aero.j2 +++ b/dev/parm/config/gcafs/config.aero.j2 @@ -92,6 +92,6 @@ export NXS_DO_CEDS2024=.false. # Use CEDS2024 emissions export NXS_DO_HTAPv2=.true. # Use HTAPv2 emissions export NXS_DO_HTAPv3=.false. # Use HTAPv3 emissions export NXS_DO_CAMS=.false. # Use CAMS global emissions -export NXS_DO_CAMSTEMPO=.false # Use CAMS temporal emissions +export NXS_DO_CAMSTEMPO=.false. # Use CAMS temporal emissions echo "END: config.aero" From 5cbb1308a2191d7ef4ffcd26b82efc02ad56ac94 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Thu, 7 Aug 2025 12:19:56 -0400 Subject: [PATCH 017/132] update GBBEPX emmissions for new format --- parm/ufs/gocart/ExtData.gbbepx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/parm/ufs/gocart/ExtData.gbbepx b/parm/ufs/gocart/ExtData.gbbepx index 3bd516c772a..5d666757db4 100644 --- a/parm/ufs/gocart/ExtData.gbbepx +++ b/parm/ufs/gocart/ExtData.gbbepx @@ -1,8 +1,8 @@ #====== BIOMASS BURNING EMISSIONS ======================================= -# GBBEPx +# QFED #-------------------------------------------------------------------------------------------------------------------------------- -SU_BIOMASS NA N Y %y4-%m2-%d2t12:00:00 none 0.7778 SO2 ExtData/nexus/GBBEPx/GBBEPx_all01GRID.emissions_v003_%y4%m2%d2.nc -OC_BIOMASS NA N Y %y4-%m2-%d2t12:00:00 none 0.7778 OC ExtData/nexus/GBBEPx/GBBEPx_all01GRID.emissions_v003_%y4%m2%d2.nc -BC_BIOMASS NA N Y %y4-%m2-%d2t12:00:00 none 0.7778 BC ExtData/nexus/GBBEPx/GBBEPx_all01GRID.emissions_v003_%y4%m2%d2.nc -# EMI_NH3_BB NA N Y %y4-%m2-%d2t12:00:00 none 0.7778 NH3 ExtData/nexus/GBBEPx/GBBEPx_all01GRID.emissions_v003_%y4%m2%d2.nc +SU_BIOMASS NA N Y %y4-%m2-%d2t12:00:00 none 0.7778 SO2 ChemInput/FIRE_EMIS_%y4%m2%d2.nc +OC_BIOMASS NA N Y %y4-%m2-%d2t12:00:00 none 0.7778 OC ChemInput/FIRE_EMIS_%y4%m2%d2.nc +BC_BIOMASS NA N Y %y4-%m2-%d2t12:00:00 none 0.7778 BC ChemInput/FIRE_EMIS_%y4%m2%d2.nc +# EMI_NH3_BB NA N Y %y4-%m2-%d2t12:00:00 none 0.7778 NH3 ChemInput/FIRE_EMIS_%y4%m2%d2.nc From 5d44905648d9c97a9124abc0b2b86b891b7933af Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Fri, 8 Aug 2025 11:00:41 -0400 Subject: [PATCH 018/132] update nexus hash for CAMS data config --- sorc/nexus.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/nexus.fd b/sorc/nexus.fd index f6c76e6c5ac..c7b81cd4676 160000 --- a/sorc/nexus.fd +++ b/sorc/nexus.fd @@ -1 +1 @@ -Subproject commit f6c76e6c5ac4df86ce5e78f91fc076424934ed14 +Subproject commit c7b81cd46766a5bab2d39ec429527569232a6b29 From 8c39994a0f97aa3e53bd3a321369eb82a032aff3 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Fri, 8 Aug 2025 11:01:52 -0400 Subject: [PATCH 019/132] update nexus hash --- sorc/nexus.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/nexus.fd b/sorc/nexus.fd index c7b81cd4676..7a1feb35b41 160000 --- a/sorc/nexus.fd +++ b/sorc/nexus.fd @@ -1 +1 @@ -Subproject commit c7b81cd46766a5bab2d39ec429527569232a6b29 +Subproject commit 7a1feb35b413aea9fbdbe8094f0b9d475df2271b From 8647595be08579667fe64e6aca51defec2d0d0e0 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Mon, 11 Aug 2025 13:29:17 -0400 Subject: [PATCH 020/132] address comments --- dev/parm/config/gcafs/config.aero.j2 | 54 ++--- parm/chem/nexus_emission.yaml.j2 | 15 ++ parm/chem/nxs_emission.yaml.j2 | 15 -- parm/ufs/gocart/ExtData.other | 38 +-- scripts/exglobal_prep_emissions.py | 4 +- sorc/nexus.fd | 2 +- ush/python/pygfs/__init__.py | 2 +- ush/python/pygfs/task/chem_fire_emission.py | 7 +- .../{nxs_emission.py => nexus_emission.py} | 221 +++++++++--------- 9 files changed, 183 insertions(+), 175 deletions(-) create mode 100644 parm/chem/nexus_emission.yaml.j2 delete mode 100644 parm/chem/nxs_emission.yaml.j2 rename ush/python/pygfs/task/{nxs_emission.py => nexus_emission.py} (60%) diff --git a/dev/parm/config/gcafs/config.aero.j2 b/dev/parm/config/gcafs/config.aero.j2 index 8bc8ebc619f..41a5227afab 100644 --- a/dev/parm/config/gcafs/config.aero.j2 +++ b/dev/parm/config/gcafs/config.aero.j2 @@ -39,59 +39,59 @@ export FIRE_EMIS_NRT_DIR="" #TODO: set to DCOM for WCOSS2 "${DCOMROOT}/YYYYMMDD/ # NEXUS aerosol emissions dataset. Choose from: gocart, none # NEXUS configuration set -export NXS_CONFIG="{{ NXS_CONFIG | default('gocart') }}" # Options: gocart, none -export NXS_CONFIG_DIR="${PARMgfs}/chem/nexus/${NXS_CONFIG}" # Directory containing NEXUS configuration files +export NEXUS_CONFIG="{{ NEXUS_CONFIG | default('gocart') }}" # Options: gocart, none +export NEXUS_CONFIG_DIR="${PARMgfs}/chem/nexus/${NEXUS_CONFIG}" # Directory containing NEXUS configuration files #--------------------- # NEXUS root directory #--------------------- -export NXS_NY="{{ NXS_NY | default(720) }}" -export NXS_XMIN="{{ NXS_XMIN | default(-180.0) }}" +export NEXUS_NY="{{ NEXUS_NY | default(720) }}" +export NEXUS_XMIN="{{ NEXUS_XMIN | default(-180.0) }}" #-------------------------- # NEXUS Time Step (seconds) #-------------------------- -export NXS_TSTEP="{{ NXS_TSTEP | default(3600) }}" # Default NEXUS time step in seconds +export NEXUS_TSTEP="{{ NEXUS_TSTEP | default(3600) }}" # Default NEXUS time step in seconds #------------------ # NEXUS Grid #------------------ -export NXS_NX="{{ NXS_NX | default(1440) }}" -export NXS_NY="{{ NXS_NY | default(720) }}" -export NXS_XMIN="{{ NXS_XMIN | default(-180.0) }}" -export NXS_XMAX="{{ NXS_XMAX | default(180.0) }}" -export NXS_YMIN="{{ NXS_YMIN | default(-90.0) }}" -export NXS_YMAX="{{ NXS_YMAX | default(90.0) }}" -export NXS_NZ="{{ NXS_NZ | default(1) }}" +export NEXUS_NX="{{ NEXUS_NX | default(1440) }}" +export NEXUS_NY="{{ NEXUS_NY | default(720) }}" +export NEXUS_XMIN="{{ NEXUS_XMIN | default(-180.0) }}" +export NEXUS_XMAX="{{ NEXUS_XMAX | default(180.0) }}" +export NEXUS_YMIN="{{ NEXUS_YMIN | default(-90.0) }}" +export NEXUS_YMAX="{{ NEXUS_YMAX | default(90.0) }}" +export NEXUS_NZ="{{ NEXUS_NZ | default(1) }}" #------------------- # NEXUS Config Files #------------------- -export NXS_GRID_NAME="{{ NXS_GRID_NAME | default('HEMCO_sa_Grid.rc') }}" -export NXS_TIME_NAME="{{ NXS_TIME_NAME | default('HEMCO_sa_Time.rc') }}" -export NXS_DIAG_NAME="{{ NXS_DIAG_NAME | default('HEMCO_sa_Diag.rc') }}" -export NXS_SPEC_NAME="{{ NXS_SPEC_NAME | default('HEMCO_sa_Spec.rc') }}" -export NXS_CONFIG_NAME="{{ NXS_CONFIG_NAME | default('NEXUS_Config.rc') }}" +export NEXUS_GRID_NAME="{{ NEXUS_GRID_NAME | default('HEMCO_sa_Grid.rc') }}" +export NEXUS_TIME_NAME="{{ NEXUS_TIME_NAME | default('HEMCO_sa_Time.rc') }}" +export NEXUS_DIAG_NAME="{{ NEXUS_DIAG_NAME | default('HEMCO_sa_Diag.rc') }}" +export NEXUS_SPEC_NAME="{{ NEXUS_SPEC_NAME | default('HEMCO_sa_Spec.rc') }}" +export NEXUS_CONFIG_NAME="{{ NEXUS_CONFIG_NAME | default('NEXUS_Config.rc') }}" #------------------ # NEXUS Diagnostics #------------------ -export NXS_DIAG_PREFIX="{{ NXS_DIAG_PREFIX | default('NXS_DIAG') }}" -export NXS_DIAG_FREQ="{{ NXS_DIAG_FREQ | default('Hourly') }}" # Options: Hourly, Daily, Monthly +export NEXUS_DIAG_PREFIX="{{ NEXUS_DIAG_PREFIX | default('NEXUS_DIAG') }}" +export NEXUS_DIAG_FREQ="{{ NEXUS_DIAG_FREQ | default('Hourly') }}" # Options: Hourly, Daily, Monthly #------------------ # NEXUS Logging #------------------ -export NXS_LOGFILE="{{ NXS_LOGFILE | default('NEXUS.log') }}" +export NEXUS_LOGFILE="{{ NEXUS_LOGFILE | default('NEXUS.log') }}" #------------------ # NEXUS Emissions #------------------ -export NXS_DO_MEGAN=.false # Use MEGAN biogenic emissions -export NXS_DO_CEDS2019=.true. # Use CEDS2019 emissions -export NXS_DO_CEDS2024=.false. # Use CEDS2024 emissions -export NXS_DO_HTAPv2=.true. # Use HTAPv2 emissions -export NXS_DO_HTAPv3=.false. # Use HTAPv3 emissions -export NXS_DO_CAMS=.false. # Use CAMS global emissions -export NXS_DO_CAMSTEMPO=.false. # Use CAMS temporal emissions +export NEXUS_DO_MEGAN=.false # Use MEGAN biogenic emissions +export NEXUS_DO_CEDS2019=.true. # Use CEDS2019 emissions +export NEXUS_DO_CEDS2024=.false. # Use CEDS2024 emissions +export NEXUS_DO_HTAPv2=.true. # Use HTAPv2 emissions +export NEXUS_DO_HTAPv3=.false. # Use HTAPv3 emissions +export NEXUS_DO_CAMS=.false. # Use CAMS global emissions +export NEXUS_DO_CAMSTEMPO=.false. # Use CAMS temporal emissions echo "END: config.aero" diff --git a/parm/chem/nexus_emission.yaml.j2 b/parm/chem/nexus_emission.yaml.j2 new file mode 100644 index 00000000000..2be1d48cba6 --- /dev/null +++ b/parm/chem/nexus_emission.yaml.j2 @@ -0,0 +1,15 @@ +nexus_emission: + data_in: + link: + - ["{{ NEXUS_INPUT_DIR }}", "{{ LOCAL_INPUT_DIR }}"] + copy: + - ["{{ NEXUS_EXECUTABLE }}", "{{ DATA }}/"] + data_out: + mkdir: + - "{{ COMOUT_CHEM_INPUT }}" + - "{{ COMOUT_CHEM_RESTART }}" + copy: + {% for fileout in FINAL_OUTPUT %} + - ["{{ DATA }}/{{ fileout }}", "{{ COMOUT_CHEM_INPUT }}/"] + {% endfor %} + - ["{{ DATA }}/{{ RestartFile }}", "{{ COMOUT_CHEM_RESTART }}/"] \ No newline at end of file diff --git a/parm/chem/nxs_emission.yaml.j2 b/parm/chem/nxs_emission.yaml.j2 deleted file mode 100644 index 6c63c9529e2..00000000000 --- a/parm/chem/nxs_emission.yaml.j2 +++ /dev/null @@ -1,15 +0,0 @@ -nxs_emission: - data_in: - link: - - ["{{ NXS_INPUT_DIR }}", "{{ LOCAL_INPUT_DIR }}"] - copy: - - ["{{ NXS_EXECUTABLE }}", "{{ WORK_DIR }}/"] - data_out: - mkdir: - - "{{ COMOUT_CHEM_INPUT }}" - - "{{ COMOUT_CHEM_RESTART }}" - copy: - {% for fileout in FINAL_OUTPUT %} - - ["{{ WORK_DIR }}/{{ fileout }}", "{{ COMOUT_CHEM_INPUT }}/"] - {% endfor %} - - ["{{ WORK_DIR }}/{{ RestartFile }}", "{{ COMOUT_CHEM_RESTART }}/"] \ No newline at end of file diff --git a/parm/ufs/gocart/ExtData.other b/parm/ufs/gocart/ExtData.other index 54e7a6df949..18f574ed5cc 100644 --- a/parm/ufs/gocart/ExtData.other +++ b/parm/ufs/gocart/ExtData.other @@ -17,12 +17,12 @@ DU_UTHRES '1' Y E - none none uthres ExtData/n #====== Sulfate Sources ================================================= # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -SU_ANTHROL1 NA N Y %y4-%m2-%d2t12:00:00 none none SO2 ChemInput/NXS_DIAG.%y4%m2%d2.nc -SU_ANTHROL2 NA N Y %y4-%m2-%d2t12:00:00 none none SO2_elev ChemInput/NXS_DIAG.%y4%m2%d2.nc +SU_ANTHROL1 NA N Y %y4-%m2-%d2t12:00:00 none none SO2 ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +SU_ANTHROL2 NA N Y %y4-%m2-%d2t12:00:00 none none SO2_elev ChemInput/NEXUS_DIAG.%y4%m2%d2.nc # Ship emissions -SU_SHIPSO2 NA N Y %y4-%m2-%d2t12:00:00 none none SO2_ship ChemInput/NXS_DIAG.%y4%m2%d2.nc -SU_SHIPSO4 NA N Y %y4-%m2-%d2t12:00:00 none none SO4_ship ChemInput/NXS_DIAG.%y4%m2%d2.nc +SU_SHIPSO2 NA N Y %y4-%m2-%d2t12:00:00 none none SO2_ship ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +SU_SHIPSO4 NA N Y %y4-%m2-%d2t12:00:00 none none SO4_ship ChemInput/NEXUS_DIAG.%y4%m2%d2.nc # Aircraft fuel consumption SU_AIRCRAFT NA Y Y %y4-%m2-%d2t12:00:00 none none none /dev/null @@ -31,9 +31,9 @@ SU_AIRCRAFT NA Y Y %y4-%m2-%d2t12:00:00 none none none /dev/null SU_DMSO NA Y Y %y4-%m2-%d2t12:00:00 none none conc ExtData/MERRA2/sfc/DMSclim_sfcconcentration.x360_y181_t12.Lana2011.nc4 # Aviation emissions during the three phases of flight -SU_AVIATION_LTO NA N Y %y4-%m2-%d2t12:00:00 none none SO2_lto ChemInput/NXS_DIAG.%y4%m2%d2.nc -SU_AVIATION_CDS NA N Y %y4-%m2-%d2t12:00:00 none none SO2_cds ChemInput/NXS_DIAG.%y4%m2%d2.nc -SU_AVIATION_CRS NA N Y %y4-%m2-%d2t12:00:00 none none SO2_crs ChemInput/NXS_DIAG.%y4%m2%d2.nc +SU_AVIATION_LTO NA N Y %y4-%m2-%d2t12:00:00 none none SO2_lto ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +SU_AVIATION_CDS NA N Y %y4-%m2-%d2t12:00:00 none none SO2_cds ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +SU_AVIATION_CRS NA N Y %y4-%m2-%d2t12:00:00 none none SO2_crs ChemInput/NEXUS_DIAG.%y4%m2%d2.nc # H2O2, OH and NO3 mixing ratios # -------------------------------------------------------------- @@ -63,19 +63,19 @@ OC_MTPO NA Y Y %y4-%m2-%d2t12:00:00 none none mtpo ExtData/nexus/MEGAN_ OC_BIOFUEL NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -OC_ANTEOC1 NA N Y %y4-%m2-%d2t12:00:00 none none OC ChemInput/NXS_DIAG.%y4%m2%d2.nc -OC_ANTEOC2 NA N Y %y4-%m2-%d2t12:00:00 none none OC_elev ChemInput/NXS_DIAG.%y4%m2%d2.nc +OC_ANTEOC1 NA N Y %y4-%m2-%d2t12:00:00 none none OC ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +OC_ANTEOC2 NA N Y %y4-%m2-%d2t12:00:00 none none OC_elev ChemInput/NEXUS_DIAG.%y4%m2%d2.nc # EDGAR based ship emissions -OC_SHIP NA N Y %y4-%m2-%d2t12:00:00 none none OC_ship ChemInput/NXS_DIAG.%y4%m2%d2.nc +OC_SHIP NA N Y %y4-%m2-%d2t12:00:00 none none OC_ship ChemInput/NEXUS_DIAG.%y4%m2%d2.nc # Aircraft fuel consumption OC_AIRCRAFT NA N Y %y4-%m2-%d2t12:00:00 none none oc_aviation /dev/null # Aviation emissions during the three phases of flight -OC_AVIATION_LTO NA N Y %y4-%m2-%d2t12:00:00 none none OC_lto ChemInput/NXS_DIAG.%y4%m2%d2.nc -OC_AVIATION_CDS NA N Y %y4-%m2-%d2t12:00:00 none none OC_cds ChemInput/NXS_DIAG.%y4%m2%d2.nc -OC_AVIATION_CRS NA N Y %y4-%m2-%d2t12:00:00 none none OC_crs ChemInput/NXS_DIAG.%y4%m2%d2.nc +OC_AVIATION_LTO NA N Y %y4-%m2-%d2t12:00:00 none none OC_lto ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +OC_AVIATION_CDS NA N Y %y4-%m2-%d2t12:00:00 none none OC_cds ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +OC_AVIATION_CRS NA N Y %y4-%m2-%d2t12:00:00 none none OC_crs ChemInput/NEXUS_DIAG.%y4%m2%d2.nc # SOA production pSOA_ANTHRO_VOC NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null @@ -88,19 +88,19 @@ pSOA_ANTHRO_VOC NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null BC_BIOFUEL NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -BC_ANTEBC1 NA N Y %y4-%m2-%d2t12:00:00 none none BC ChemInput/NXS_DIAG.%y4%m2%d2.nc -BC_ANTEBC2 NA N Y %y4-%m2-%d2t12:00:00 none none BC_elev ChemInput/NXS_DIAG.%y4%m2%d2.nc +BC_ANTEBC1 NA N Y %y4-%m2-%d2t12:00:00 none none BC ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +BC_ANTEBC2 NA N Y %y4-%m2-%d2t12:00:00 none none BC_elev ChemInput/NEXUS_DIAG.%y4%m2%d2.nc # EDGAR based ship emissions -BC_SHIP NA N Y %y4-%m2-%d2t12:00:00 none none BC_ship ChemInput/NXS_DIAG.%y4%m2%d2.nc +BC_SHIP NA N Y %y4-%m2-%d2t12:00:00 none none BC_ship ChemInput/NEXUS_DIAG.%y4%m2%d2.nc # Aircraft fuel consumption BC_AIRCRAFT NA N Y %y4-%m2-%d2t12:00:00 none none bc_aviation /dev/null # Aviation emissions during the LTO, SDC and CRS phases of flight -BC_AVIATION_LTO NA N Y %y4-%m2-%d2t12:00:00 none none BC_lto ChemInput/NXS_DIAG.%y4%m2%d2.nc -BC_AVIATION_CDS NA N Y %y4-%m2-%d2t12:00:00 none none BC_cds ChemInput/NXS_DIAG.%y4%m2%d2.nc -BC_AVIATION_CRS NA N Y %y4-%m2-%d2t12:00:00 none none BC_crs ChemInput/NXS_DIAG.%y4%m2%d2.nc +BC_AVIATION_LTO NA N Y %y4-%m2-%d2t12:00:00 none none BC_lto ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +BC_AVIATION_CDS NA N Y %y4-%m2-%d2t12:00:00 none none BC_cds ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +BC_AVIATION_CRS NA N Y %y4-%m2-%d2t12:00:00 none none BC_crs ChemInput/NEXUS_DIAG.%y4%m2%d2.nc #============================================================================================================ # BROWN CARBON diff --git a/scripts/exglobal_prep_emissions.py b/scripts/exglobal_prep_emissions.py index 3876fd6c2f3..e517a52243a 100755 --- a/scripts/exglobal_prep_emissions.py +++ b/scripts/exglobal_prep_emissions.py @@ -5,7 +5,7 @@ import os from wxflow import Logger, cast_strdict_as_dtypedict -from pygfs import ChemFireEmissions, NXSEmissions +from pygfs import ChemFireEmissions, NEXUSEmissions # Initialize root logger @@ -25,7 +25,7 @@ emissions.execute() emissions.finalize() - nxsemis = NXSEmissions(config) + nxsemis = NEXUSEmissions(config) nxsemis.initialize() nxsemis.configure() nxsemis.execute() diff --git a/sorc/nexus.fd b/sorc/nexus.fd index 7a1feb35b41..df138fb26af 160000 --- a/sorc/nexus.fd +++ b/sorc/nexus.fd @@ -1 +1 @@ -Subproject commit 7a1feb35b413aea9fbdbe8094f0b9d475df2271b +Subproject commit df138fb26afdd1843175b2a830efde377701d9cc diff --git a/ush/python/pygfs/__init__.py b/ush/python/pygfs/__init__.py index 3e88e7f93a3..9a42e4f141f 100644 --- a/ush/python/pygfs/__init__.py +++ b/ush/python/pygfs/__init__.py @@ -39,7 +39,7 @@ from .task.analysis import Analysis from .task.chem_fire_emission import ChemFireEmissions -from .task.nxs_emission import NXSEmissions +from .task.nexus_emission import NEXUSEmissions from .task.aero_analysis import AerosolAnalysis from .task.aero_bmatrix import AerosolBMatrix from .task.atm_analysis import AtmAnalysis diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index aeb0c4bc5f6..4828eed008a 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -313,6 +313,11 @@ def _find_gbbepx_files(self, dates, version='v5r0'): # Find all possible months months, years = self._get_unique_months() + # Format dates properly for matching + if not isinstance(dates, list): + dates = [dates] + date_strings = [d.strftime('%Y%m%d') if hasattr(d, 'strftime') else str(d) for d in dates] + files_found = [] # Find all possible files for mon in months: @@ -454,7 +459,7 @@ def _find_qfed_files(self, dates, vars, version='061', aero_emis_fire_dir=None): return files_found @logit(logger) - def GBBEPx_to_COARDS(fname: Union[str, os.PathLike]) -> xr.Dataset: + def GBBEPx_to_COARDS(self, fname: Union[str, os.PathLike]) -> xr.Dataset: """Convert GBBEPx file to COARDS compliant format Parameters diff --git a/ush/python/pygfs/task/nxs_emission.py b/ush/python/pygfs/task/nexus_emission.py similarity index 60% rename from ush/python/pygfs/task/nxs_emission.py rename to ush/python/pygfs/task/nexus_emission.py index 953f36982bf..d3b7e4d2203 100644 --- a/ush/python/pygfs/task/nxs_emission.py +++ b/ush/python/pygfs/task/nexus_emission.py @@ -8,7 +8,7 @@ from logging import getLogger from typing import Dict, Any, Union, List from dateutil.rrule import DAILY, HOURLY, rrule -from pprint import pprint +# from pprint import pprint from jinja2 import Environment, FileSystemLoader from wxflow import (AttrDict, FileHandler, @@ -22,11 +22,11 @@ logger = getLogger(__name__.split('.')[-1]) -class NXSEmissions(Task): +class NEXUSEmissions(Task): """NEXUS Emissions pre-processing Task """ - @logit(logger, name="NXSEmissions") + @logit(logger, name="NEXUSEmissions") def __init__(self, config: Dict[str, Any]) -> None: """Constructor for the NEXUS Emissions task @@ -47,17 +47,17 @@ def __init__(self, config: Dict[str, Any]) -> None: nforecast_hours = self.task_config["FHMAX_GFS"] self.start_date = self.task_config["SDATE"] - to_timedelta('12H') self.end_date = self.task_config["EDATE"] + to_timedelta('12H') - frequency = self.task_config.get("NXS_DIAG_FREQ", "Hourly") + frequency = self.task_config.get("NEXUS_DIAG_FREQ", "Hourly") if frequency == "Hourly": self.forecast_dates = list(rrule(freq=HOURLY, dtstart=self.start_date, until=self.end_date)) elif frequency == 'Daily': self.forecast_dates = list(rrule(freq=DAILY, dtstart=self.start_date, until=self.end_date)) else: - raise WorkflowException(f"Unsupported NXS_DIAG_FREQ: {frequency}") + raise WorkflowException(f"Unsupported NEXUS_DIAG_FREQ: {frequency}") self.forecast_dates_daily = list(rrule(freq=DAILY, dtstart=self.start_date, until=self.end_date)) - logger.info(f"NXSEmissions initialized with start date: {self.start_date}, end date: {self.end_date}") + logger.info(f"NEXUSEmissions initialized with start date: {self.start_date}, end date: {self.end_date}") @logit(logger) def initialize(self) -> None: @@ -65,7 +65,7 @@ def initialize(self) -> None: This method performs the following steps: 1. Render the NEXUS configuration files using Jinja2 templates - found in `parm/chem/nexus/$NXS_CONFIG` + found in `parm/chem/nexus/$NEXUS_CONFIG` 2. Sets up template variables for emission configuration 3. Creates necessary working directories 4. Copies required input files to working directory @@ -106,97 +106,100 @@ def initialize(self) -> None: # logger.info("Rendering NEXUS configuration files") # Check for required NEXUS configuration parameters - nxs_config_set = self.task_config.get('NXS_CONFIG', None) - if not nxs_config_set: - raise WorkflowException("NXS_CONFIG must be set in task configuration") - nxs_config_dir = self.task_config.get('NXS_CONFIG_DIR', None) - if not nxs_config_dir: - raise WorkflowException("NXS_CONFIG_DIR must be set in task configuration") - nxs_input_dir = self.task_config.get('NXS_INPUT_DIR', None) - if not nxs_input_dir: - raise WorkflowException("NXS_INPUT_DIR must be set in task configuration") - # Default NXS_TSTEP to 3600 seconds (1 hour) if not set - nxs_tstep = self.task_config.get('NXS_TSTEP', 3600) - if not nxs_tstep: - raise WorkflowException("NXS_TSTEP must be set in task configuration") - - logger.info(f"Using NXS_CONFIG: {nxs_config_set}") - logger.info(f"Using NXS_CONFIG_DIR: {nxs_config_dir}") - logger.info(f"Using NXS_INPUT_DIR: {nxs_input_dir}") - logger.info(f"Using NXS_TSTEP: {nxs_tstep}") + required_nexus_params = [ + 'NEXUS_CONFIG', + 'NEXUS_CONFIG_DIR', + 'NEXUS_INPUT_DIR', + ] + for param in required_nexus_params: + if not self.task_config.get(param, None): + raise WorkflowException(f"{param} must be set in task configuration") + + nexus_config_set = self.task_config.get('NEXUS_CONFIG', None) + nexus_config_dir = self.task_config.get('NEXUS_CONFIG_DIR', None) + nexus_input_dir = self.task_config.get('NEXUS_INPUT_DIR', None + ) + # Default NEXUS_TSTEP to 3600 seconds (1 hour) if not set + nexus_tstep = self.task_config.get('NEXUS_TSTEP', 3600) + if not nexus_tstep: + raise WorkflowException("NEXUS_TSTEP must be set in task configuration") + + logger.info(f"Using NEXUS_CONFIG: {nexus_config_set}") + logger.info(f"Using NEXUS_CONFIG_DIR: {nexus_config_dir}") + logger.info(f"Using NEXUS_INPUT_DIR: {nexus_input_dir}") + logger.info(f"Using NEXUS_TSTEP: {nexus_tstep}") # Check for grid parameters - if not self.task_config.get('NXS_NX', None): - raise WorkflowException("NXS_NX must be set in task configuration") - if not self.task_config.get('NXS_NY', None): - raise WorkflowException("NXS_NY must be set in task configuration") - if not self.task_config.get('NXS_NZ', None): - raise WorkflowException("NXS_NZ must be set in task configuration") - if not self.task_config.get('NXS_XMIN', None): - raise WorkflowException("NXS_XMIN must be set in task configuration") - if not self.task_config.get('NXS_XMAX', None): - raise WorkflowException("NXS_XMAX must be set in task configuration") - if not self.task_config.get('NXS_YMIN', None): - raise WorkflowException("NXS_YMIN must be set in task configuration") - - logger.info(f"Grid parameters: NXS_NX={self.task_config.NXS_NX}") - logger.info(f"Grid parameters: NXS_NY={self.task_config.NXS_NY}") - logger.info(f"Grid parameters: NXS_NZ={self.task_config.NXS_NZ}") - logger.info(f"Grid parameters: NXS_XMIN={self.task_config.NXS_XMIN}") - logger.info(f"Grid parameters: NXS_XMAX={self.task_config.NXS_XMAX}") - logger.info(f"Grid parameters: NXS_YMIN={self.task_config.NXS_YMIN}") - logger.info(f"Grid parameters: NXS_YMAX={self.task_config.NXS_YMAX}") - - processed_nxs_files = [] + required_grid_params = [ + 'NEXUS_NX', + 'NEXUS_NY', + 'NEXUS_NZ', + 'NEXUS_XMIN', + 'NEXUS_XMAX', + 'NEXUS_YMIN' + ] + for param in required_grid_params: + if not self.task_config.get(param, None): + raise WorkflowException(f"{param} must be set in task configuration") + + logger.info(f"Grid parameters: NEXUS_NX={self.task_config.NEXUS_NX}") + logger.info(f"Grid parameters: NEXUS_NY={self.task_config.NEXUS_NY}") + logger.info(f"Grid parameters: NEXUS_NZ={self.task_config.NEXUS_NZ}") + logger.info(f"Grid parameters: NEXUS_XMIN={self.task_config.NEXUS_XMIN}") + logger.info(f"Grid parameters: NEXUS_XMAX={self.task_config.NEXUS_XMAX}") + logger.info(f"Grid parameters: NEXUS_YMIN={self.task_config.NEXUS_YMIN}") + logger.info(f"Grid parameters: NEXUS_YMAX={self.task_config.NEXUS_YMAX}") + + processed_nexus_files = [] final_output_files = [] sorted_dates = sorted(self.forecast_dates) for d in sorted_dates[:-1]: - fname = f"{self.task_config.NXS_DIAG_PREFIX}.{d.strftime('%Y%m%d%H')}00.nc" - fname_final = f"{self.task_config.NXS_DIAG_PREFIX}.{d.strftime('%Y%m%d')}.nc" - processed_nxs_files.append(fname) + fname = f"{self.task_config.NEXUS_DIAG_PREFIX}.{d.strftime('%Y%m%d%H')}00.nc" + fname_final = f"{self.task_config.NEXUS_DIAG_PREFIX}.{d.strftime('%Y%m%d')}.nc" + processed_nexus_files.append(fname) final_output_files.append(fname_final) - self.processed_nxs_files = processed_nxs_files + self.processed_nexus_files = processed_nexus_files # render the NEXUS configuration files - if not os.path.exists(nxs_config_dir): - raise WorkflowException(f"NEXUS configuration file not found: {nxs_config_dir}") - logger.info(f"Rendering NEXUS configuration from {nxs_config_dir}") + if not os.path.exists(nexus_config_dir): + raise WorkflowException(f"NEXUS configuration file not found: {nexus_config_dir}") + logger.info(f"Rendering NEXUS configuration from {nexus_config_dir}") tmpl_dict = { - 'NXS_CONFIG': nxs_config_set, - 'NXS_CONFIG_DIR': nxs_config_dir, - 'NXS_INPUT_DIR': nxs_input_dir, - 'NXS_DIAG_PREFIX': self.task_config.NXS_DIAG_PREFIX, - 'NXS_TSTEP': nxs_tstep, - 'NXS_NX': self.task_config.NXS_NX, - 'NXS_NY': self.task_config.NXS_NY, - 'NXS_NZ': self.task_config.NXS_NZ, - 'NXS_XMIN': self.task_config.NXS_XMIN, - 'NXS_XMAX': self.task_config.NXS_XMAX, - 'NXS_YMIN': self.task_config.NXS_YMIN, - 'NXS_YMAX': self.task_config.NXS_YMAX, + 'NEXUS_CONFIG': nexus_config_set, + 'NEXUS_CONFIG_DIR': nexus_config_dir, + 'NEXUS_INPUT_DIR': nexus_input_dir, + 'NEXUS_DIAG_PREFIX': self.task_config.NEXUS_DIAG_PREFIX, + 'NEXUS_TSTEP': nexus_tstep, + 'NEXUS_NX': self.task_config.NEXUS_NX, + 'NEXUS_NY': self.task_config.NEXUS_NY, + 'NEXUS_NZ': self.task_config.NEXUS_NZ, + 'NEXUS_XMIN': self.task_config.NEXUS_XMIN, + 'NEXUS_XMAX': self.task_config.NEXUS_XMAX, + 'NEXUS_YMIN': self.task_config.NEXUS_YMIN, + 'NEXUS_YMAX': self.task_config.NEXUS_YMAX, 'LOCAL_INPUT_DIR': os.path.join(self.task_config.DATA, 'INPUT'), - 'NXS_EXECUTABLE': os.path.join(self.task_config.get('HOMEgfs', None), "exec/nexus.x"), - "WORK_DIR": self.task_config.DATA, - "NXS_DO_MEGAN": self.task_config.get('NXS_DO_MEGAN', False), - "NXS_DO_CEDS2019": self.task_config.get('NXS_DO_CEDS2019', True), - "NXS_DO_CEDS2024": self.task_config.get('NXS_DO_CEDS2024', False), - "NXS_DO_HTAPv2": self.task_config.get('NXS_DO_HTAPv2', True), - "NXS_DO_HTAPv3": self.task_config.get('NXS_DO_HTAPv3', False), - "NXS_DO_CAMS": self.task_config.get('NXS_DO_CAMS', False), - "NXS_DO_CAMSTEMPO": self.task_config.get('NXS_DO_CAMSTEMPO', False), + 'NEXUS_EXECUTABLE': os.path.join(self.task_config.get('HOMEgfs', None), "exec/nexus.x"), + "DATA": self.task_config.DATA, + "NEXUS_DO_MEGAN": self.task_config.get('NEXUS_DO_MEGAN', False), + "NEXUS_DO_CEDS2019": self.task_config.get('NEXUS_DO_CEDS2019', True), + "NEXUS_DO_CEDS2024": self.task_config.get('NEXUS_DO_CEDS2024', False), + "NEXUS_DO_HTAPv2": self.task_config.get('NEXUS_DO_HTAPv2', True), + "NEXUS_DO_HTAPv3": self.task_config.get('NEXUS_DO_HTAPv3', False), + "NEXUS_DO_CAMS": self.task_config.get('NEXUS_DO_CAMS', False), + "NEXUS_DO_CAMSTEMPO": self.task_config.get('NEXUS_DO_CAMSTEMPO', False), "start_date": self.start_date.strftime('%Y-%m-%d %H:%M:%S'), "end_date": self.end_date.strftime('%Y-%m-%d %H:%M:%S'), "FINAL_OUTPUT": final_output_files, "COMOUT_CHEM_INPUT": self.task_config.COMOUT_CHEM_INPUT, "COMOUT_CHEM_RESTART": self.task_config.COMOUT_CHEM_RESTART, "RestartFile": f"HEMCO_restart.{self.end_date.strftime('%Y%m%d%H')}00.nc", - "processed_nxs_files": processed_nxs_files, + "processed_nexus_files": processed_nexus_files, } - yaml_template = os.path.join(self.task_config.HOMEgfs, 'parm', 'chem', 'nxs_emission.yaml.j2') + yaml_template = os.path.join(self.task_config.HOMEgfs, 'parm', 'chem', 'nexus_emission.yaml.j2') if not os.path.exists(yaml_template): logger.warning(f"Template file not found: {yaml_template}, using default configuration") - yaml_config = {'nxs_emission': {}} + yaml_config = {'nexus_emission': {}} else: logger.debug(f'Parsing YAML template: {yaml_template}') yaml_config = parse_j2yaml(yaml_template, tmpl_dict) @@ -205,44 +208,44 @@ def initialize(self) -> None: self.task_config = AttrDict(**self.task_config, **yaml_config) # Link NEXUS input directory to the working directory - FileHandler(self.task_config.nxs_emission.data_in).sync() + FileHandler(self.task_config.nexus_emission.data_in).sync() logger.info(f"NEXUS input directory linked to {self.task_config.DATA}") - # Render NXS Grid File - file_loader = FileSystemLoader(self.task_config.NXS_CONFIG_DIR) + # Render NEXUS Grid File + file_loader = FileSystemLoader(self.task_config.NEXUS_CONFIG_DIR) env = Environment(loader=file_loader) - nxs_grid_template = env.get_template(f"{self.task_config.NXS_GRID_NAME}.j2") - self.task_config.NXS_GRID_TEMPLATE = nxs_grid_template.render(tmpl_dict) - outfile = os.path.join(self.task_config.DATA, self.task_config.NXS_GRID_NAME) - _write_txt_file(self.task_config.NXS_GRID_TEMPLATE, outfile) + nexus_grid_template = env.get_template(f"{self.task_config.NEXUS_GRID_NAME}.j2") + self.task_config.NEXUS_GRID_TEMPLATE = nexus_grid_template.render(tmpl_dict) + outfile = os.path.join(self.task_config.DATA, self.task_config.NEXUS_GRID_NAME) + _write_txt_file(self.task_config.NEXUS_GRID_TEMPLATE, outfile) logger.info(f"NEXUS grid file rendered successfully: written to {outfile}") - # Render NXS Config File - nxs_config_template = env.get_template(f"{self.task_config.NXS_CONFIG_NAME}.j2") - self.task_config.NXS_CONFIG_TEMPLATE = nxs_config_template.render(tmpl_dict) - outfile = os.path.join(self.task_config.DATA, self.task_config.NXS_CONFIG_NAME) - _write_txt_file(self.task_config.NXS_CONFIG_TEMPLATE, outfile) + # Render NEXUS Config File + nexus_config_template = env.get_template(f"{self.task_config.NEXUS_CONFIG_NAME}.j2") + self.task_config.NEXUS_CONFIG_TEMPLATE = nexus_config_template.render(tmpl_dict) + outfile = os.path.join(self.task_config.DATA, self.task_config.NEXUS_CONFIG_NAME) + _write_txt_file(self.task_config.NEXUS_CONFIG_TEMPLATE, outfile) logger.info(f"NEXUS config file rendered successfully: written to {outfile}") - # Render NXS Time File - nxs_time_template = env.get_template(f"{self.task_config.NXS_TIME_NAME}.j2") - self.task_config.NXS_TIME_TEMPLATE = nxs_time_template.render(tmpl_dict) - outfile = os.path.join(self.task_config.DATA, self.task_config.NXS_TIME_NAME) - _write_txt_file(self.task_config.NXS_TIME_TEMPLATE, outfile) + # Render NEXUS Time File + nexus_time_template = env.get_template(f"{self.task_config.NEXUS_TIME_NAME}.j2") + self.task_config.NEXUS_TIME_TEMPLATE = nexus_time_template.render(tmpl_dict) + outfile = os.path.join(self.task_config.DATA, self.task_config.NEXUS_TIME_NAME) + _write_txt_file(self.task_config.NEXUS_TIME_TEMPLATE, outfile) logger.info(f"NEXUS time file rendered successfully: written to {outfile}") - # Render NXS Diag File - nxs_diag_template = env.get_template(f"{self.task_config.NXS_DIAG_NAME}.j2") - self.task_config.NXS_DIAG_TEMPLATE = nxs_diag_template.render(tmpl_dict) - outfile = os.path.join(self.task_config.DATA, self.task_config.NXS_DIAG_NAME) - _write_txt_file(self.task_config.NXS_DIAG_TEMPLATE, outfile) + # Render NEXUS Diag File + nexus_diag_template = env.get_template(f"{self.task_config.NEXUS_DIAG_NAME}.j2") + self.task_config.NEXUS_DIAG_TEMPLATE = nexus_diag_template.render(tmpl_dict) + outfile = os.path.join(self.task_config.DATA, self.task_config.NEXUS_DIAG_NAME) + _write_txt_file(self.task_config.NEXUS_DIAG_TEMPLATE, outfile) logger.info(f"NEXUS diag file rendered successfully: written to {outfile}") - # Render NXS Spec File - nxs_spec_template = env.get_template(f"{self.task_config.NXS_SPEC_NAME}.j2") - self.task_config.NXS_SPEC_TEMPLATE = nxs_spec_template.render(tmpl_dict) - outfile = os.path.join(self.task_config.DATA, self.task_config.NXS_SPEC_NAME) - _write_txt_file(self.task_config.NXS_SPEC_TEMPLATE, outfile) + # Render NEXUS Spec File + nexus_spec_template = env.get_template(f"{self.task_config.NEXUS_SPEC_NAME}.j2") + self.task_config.NEXUS_SPEC_TEMPLATE = nexus_spec_template.render(tmpl_dict) + outfile = os.path.join(self.task_config.DATA, self.task_config.NEXUS_SPEC_NAME) + _write_txt_file(self.task_config.NEXUS_SPEC_TEMPLATE, outfile) logger.info(f"NEXUS spec file rendered successfully: written to {outfile}") @logit(logger) @@ -286,12 +289,12 @@ def execute(self) -> None: str(1), 'nexus.x', '-c', - self.task_config.NXS_CONFIG_NAME] + self.task_config.NEXUS_CONFIG_NAME] exe(*arg_list, output='stdout', error='stderr') logger.info("Concatenating processed NEXUS files...") - files = sorted(self.processed_nxs_files) + files = sorted(self.processed_nexus_files) dsets = [] for f in files: dsets.append(xr.open_dataset(f, decode_cf=False)) @@ -322,7 +325,7 @@ def execute(self) -> None: encoding = {var: {"zlib": True, "complevel": 4} for var in ds.data_vars} for day_str, indices in day_to_indices.items(): daily_ds = ds.isel(time=indices) - outname = f"{self.task_config.NXS_DIAG_PREFIX}.{day_str}.nc" + outname = f"{self.task_config.NEXUS_DIAG_PREFIX}.{day_str}.nc" daily_ds.to_netcdf(outname, format="NETCDF4", encoding=encoding) logger.info(f"Wrote daily output: {outname}") @@ -344,7 +347,7 @@ def finalize(self) -> None: """ logger.info("Finalizing NEXUS emissions processing") - FileHandler(self.task_config.nxs_emission.data_out).sync() + FileHandler(self.task_config.nexus_emission.data_out).sync() logger.info("Chemical emissions finalization complete") From b586abd018296b14415a43437163eaba256c3c06 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Mon, 11 Aug 2025 13:33:08 -0400 Subject: [PATCH 021/132] fix: correct syntax error in NEXUS_INPUT_DIR retrieval --- ush/python/pygfs/task/nexus_emission.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/nexus_emission.py b/ush/python/pygfs/task/nexus_emission.py index d3b7e4d2203..703957c065c 100644 --- a/ush/python/pygfs/task/nexus_emission.py +++ b/ush/python/pygfs/task/nexus_emission.py @@ -117,8 +117,8 @@ def initialize(self) -> None: nexus_config_set = self.task_config.get('NEXUS_CONFIG', None) nexus_config_dir = self.task_config.get('NEXUS_CONFIG_DIR', None) - nexus_input_dir = self.task_config.get('NEXUS_INPUT_DIR', None - ) + nexus_input_dir = self.task_config.get('NEXUS_INPUT_DIR', None) + # Default NEXUS_TSTEP to 3600 seconds (1 hour) if not set nexus_tstep = self.task_config.get('NEXUS_TSTEP', 3600) if not nexus_tstep: From 70a300d515a30ade6be6f16a4321b73e6793ca6c Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Mon, 11 Aug 2025 14:00:02 -0400 Subject: [PATCH 022/132] remove duplicates --- dev/parm/config/gcafs/config.aero.j2 | 5 ----- 1 file changed, 5 deletions(-) diff --git a/dev/parm/config/gcafs/config.aero.j2 b/dev/parm/config/gcafs/config.aero.j2 index 41a5227afab..04003cbf4b4 100644 --- a/dev/parm/config/gcafs/config.aero.j2 +++ b/dev/parm/config/gcafs/config.aero.j2 @@ -42,11 +42,6 @@ export FIRE_EMIS_NRT_DIR="" #TODO: set to DCOM for WCOSS2 "${DCOMROOT}/YYYYMMDD/ export NEXUS_CONFIG="{{ NEXUS_CONFIG | default('gocart') }}" # Options: gocart, none export NEXUS_CONFIG_DIR="${PARMgfs}/chem/nexus/${NEXUS_CONFIG}" # Directory containing NEXUS configuration files -#--------------------- -# NEXUS root directory -#--------------------- -export NEXUS_NY="{{ NEXUS_NY | default(720) }}" -export NEXUS_XMIN="{{ NEXUS_XMIN | default(-180.0) }}" #-------------------------- # NEXUS Time Step (seconds) #-------------------------- From fa8d7ee97bd51b0a87a82b2f120b9666c10a10a4 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Mon, 11 Aug 2025 14:38:32 -0400 Subject: [PATCH 023/132] refactor: remove commented-out code and unnecessary check for NEXUS_TSTEP --- parm/chem/nexus_emission.yaml.j2 | 2 +- ush/python/pygfs/task/nexus_emission.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/parm/chem/nexus_emission.yaml.j2 b/parm/chem/nexus_emission.yaml.j2 index 2be1d48cba6..e5e2c42cf41 100644 --- a/parm/chem/nexus_emission.yaml.j2 +++ b/parm/chem/nexus_emission.yaml.j2 @@ -12,4 +12,4 @@ nexus_emission: {% for fileout in FINAL_OUTPUT %} - ["{{ DATA }}/{{ fileout }}", "{{ COMOUT_CHEM_INPUT }}/"] {% endfor %} - - ["{{ DATA }}/{{ RestartFile }}", "{{ COMOUT_CHEM_RESTART }}/"] \ No newline at end of file + - ["{{ DATA }}/{{ RestartFile }}", "{{ COMOUT_CHEM_RESTART }}/"] diff --git a/ush/python/pygfs/task/nexus_emission.py b/ush/python/pygfs/task/nexus_emission.py index 703957c065c..77c1142bf96 100644 --- a/ush/python/pygfs/task/nexus_emission.py +++ b/ush/python/pygfs/task/nexus_emission.py @@ -8,7 +8,6 @@ from logging import getLogger from typing import Dict, Any, Union, List from dateutil.rrule import DAILY, HOURLY, rrule -# from pprint import pprint from jinja2 import Environment, FileSystemLoader from wxflow import (AttrDict, FileHandler, @@ -121,8 +120,6 @@ def initialize(self) -> None: # Default NEXUS_TSTEP to 3600 seconds (1 hour) if not set nexus_tstep = self.task_config.get('NEXUS_TSTEP', 3600) - if not nexus_tstep: - raise WorkflowException("NEXUS_TSTEP must be set in task configuration") logger.info(f"Using NEXUS_CONFIG: {nexus_config_set}") logger.info(f"Using NEXUS_CONFIG_DIR: {nexus_config_dir}") From f2138072c876fcb14b617a2d998d87e6d258fad6 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Wed, 13 Aug 2025 12:53:41 -0400 Subject: [PATCH 024/132] update nexus hash --- sorc/nexus.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/nexus.fd b/sorc/nexus.fd index df138fb26af..6f731a49606 160000 --- a/sorc/nexus.fd +++ b/sorc/nexus.fd @@ -1 +1 @@ -Subproject commit df138fb26afdd1843175b2a830efde377701d9cc +Subproject commit 6f731a4960690167a73aef66f0e4372b0fb6f8ba From 7caea35d47ebf32b1780fdf40f7628244e147401 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Wed, 13 Aug 2025 14:10:30 -0400 Subject: [PATCH 025/132] update nexus --- sorc/nexus.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/nexus.fd b/sorc/nexus.fd index 6f731a49606..72bb61fe857 160000 --- a/sorc/nexus.fd +++ b/sorc/nexus.fd @@ -1 +1 @@ -Subproject commit 6f731a4960690167a73aef66f0e4372b0fb6f8ba +Subproject commit 72bb61fe857b46f94d888328d411b1a6d0a8cf4a From c340141e9fb411024e96e82225b0f88d18cc7c2b Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Wed, 13 Aug 2025 14:11:38 -0400 Subject: [PATCH 026/132] remove commented out imports --- ush/python/pygfs/task/chem_fire_emission.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 4828eed008a..8b0b8e8c408 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -8,7 +8,6 @@ from logging import getLogger from typing import Dict, Any, Union, List from dateutil.rrule import DAILY, rrule -# from pprint import pprint from wxflow import (AttrDict, parse_j2yaml, From 11549982748df90d960999b0e385b5dd6f35bdb2 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Fri, 15 Aug 2025 15:36:56 -0400 Subject: [PATCH 027/132] Refactor NEXUS emissions configuration and update forecast handling - Updated comments for MEGAN biogenic emissions in config.aero.j2 - Increased walltime for "prep_emissions" in config.resources - Adjusted NXS file handling in forecast_predet.sh to use correct prefix - Enhanced ChemFireEmissions and NEXUSEmissions classes to improve date handling and logging --- dev/parm/config/gcafs/config.aero.j2 | 4 +- dev/parm/config/gcafs/config.resources | 7 ++-- parm/ufs/gocart/ExtData.other | 42 ++++++++++----------- ush/forecast_predet.sh | 8 +--- ush/python/pygfs/task/chem_fire_emission.py | 16 ++++++-- ush/python/pygfs/task/nexus_emission.py | 25 ++++++++++-- 6 files changed, 63 insertions(+), 39 deletions(-) diff --git a/dev/parm/config/gcafs/config.aero.j2 b/dev/parm/config/gcafs/config.aero.j2 index 04003cbf4b4..14f63a44c6b 100644 --- a/dev/parm/config/gcafs/config.aero.j2 +++ b/dev/parm/config/gcafs/config.aero.j2 @@ -81,11 +81,11 @@ export NEXUS_LOGFILE="{{ NEXUS_LOGFILE | default('NEXUS.log') }}" #------------------ # NEXUS Emissions #------------------ -export NEXUS_DO_MEGAN=.false # Use MEGAN biogenic emissions +export NEXUS_DO_MEGAN=.false # TODO: Add MEGAN biogenic emissions in the furture export NEXUS_DO_CEDS2019=.true. # Use CEDS2019 emissions export NEXUS_DO_CEDS2024=.false. # Use CEDS2024 emissions export NEXUS_DO_HTAPv2=.true. # Use HTAPv2 emissions -export NEXUS_DO_HTAPv3=.false. # Use HTAPv3 emissions +export NEXUS_DO_HTAPv3=.false. # TODO: Currently only uses HTAPv2 for this. export NEXUS_DO_CAMS=.false. # Use CAMS global emissions export NEXUS_DO_CAMSTEMPO=.false. # Use CAMS temporal emissions diff --git a/dev/parm/config/gcafs/config.resources b/dev/parm/config/gcafs/config.resources index 1d32c979a4e..27be2f2ed23 100644 --- a/dev/parm/config/gcafs/config.resources +++ b/dev/parm/config/gcafs/config.resources @@ -812,10 +812,11 @@ case ${step} in ;; "prep_emissions") - export walltime="00:10:00" + export walltime="00:20:00" export ntasks=1 - export threads_per_task=1 - export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export threads_per_task=$(( max_tasks_per_node / 2 )) + export tasks_per_node=1 + export OMP_NUM_THREADS="${threads_per_task}" ;; "fcst" | "efcs") diff --git a/parm/ufs/gocart/ExtData.other b/parm/ufs/gocart/ExtData.other index 18f574ed5cc..b21b9b3d931 100644 --- a/parm/ufs/gocart/ExtData.other +++ b/parm/ufs/gocart/ExtData.other @@ -17,12 +17,12 @@ DU_UTHRES '1' Y E - none none uthres ExtData/n #====== Sulfate Sources ================================================= # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -SU_ANTHROL1 NA N Y %y4-%m2-%d2t12:00:00 none none SO2 ChemInput/NEXUS_DIAG.%y4%m2%d2.nc -SU_ANTHROL2 NA N Y %y4-%m2-%d2t12:00:00 none none SO2_elev ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +SU_ANTHROL1 NA N Y F0 none none SO2 ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +SU_ANTHROL2 NA N Y F0 none none SO2_elev ChemInput/NEXUS_DIAG.%y4%m2%d2.nc # Ship emissions -SU_SHIPSO2 NA N Y %y4-%m2-%d2t12:00:00 none none SO2_ship ChemInput/NEXUS_DIAG.%y4%m2%d2.nc -SU_SHIPSO4 NA N Y %y4-%m2-%d2t12:00:00 none none SO4_ship ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +SU_SHIPSO2 NA N Y F0 none none SO2_ship ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +SU_SHIPSO4 NA N Y F0 none none SO4_ship ChemInput/NEXUS_DIAG.%y4%m2%d2.nc # Aircraft fuel consumption SU_AIRCRAFT NA Y Y %y4-%m2-%d2t12:00:00 none none none /dev/null @@ -31,9 +31,9 @@ SU_AIRCRAFT NA Y Y %y4-%m2-%d2t12:00:00 none none none /dev/null SU_DMSO NA Y Y %y4-%m2-%d2t12:00:00 none none conc ExtData/MERRA2/sfc/DMSclim_sfcconcentration.x360_y181_t12.Lana2011.nc4 # Aviation emissions during the three phases of flight -SU_AVIATION_LTO NA N Y %y4-%m2-%d2t12:00:00 none none SO2_lto ChemInput/NEXUS_DIAG.%y4%m2%d2.nc -SU_AVIATION_CDS NA N Y %y4-%m2-%d2t12:00:00 none none SO2_cds ChemInput/NEXUS_DIAG.%y4%m2%d2.nc -SU_AVIATION_CRS NA N Y %y4-%m2-%d2t12:00:00 none none SO2_crs ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +SU_AVIATION_LTO NA N Y F0 none none SO2_lto ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +SU_AVIATION_CDS NA N Y F0 none none SO2_cds ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +SU_AVIATION_CRS NA N Y F0 none none SO2_crs ChemInput/NEXUS_DIAG.%y4%m2%d2.nc # H2O2, OH and NO3 mixing ratios # -------------------------------------------------------------- @@ -63,19 +63,19 @@ OC_MTPO NA Y Y %y4-%m2-%d2t12:00:00 none none mtpo ExtData/nexus/MEGAN_ OC_BIOFUEL NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -OC_ANTEOC1 NA N Y %y4-%m2-%d2t12:00:00 none none OC ChemInput/NEXUS_DIAG.%y4%m2%d2.nc -OC_ANTEOC2 NA N Y %y4-%m2-%d2t12:00:00 none none OC_elev ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +OC_ANTEOC1 NA N Y F0 none none OC ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +OC_ANTEOC2 NA N Y F0 none none OC_elev ChemInput/NEXUS_DIAG.%y4%m2%d2.nc # EDGAR based ship emissions -OC_SHIP NA N Y %y4-%m2-%d2t12:00:00 none none OC_ship ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +OC_SHIP NA N Y F0 none none OC_ship ChemInput/NEXUS_DIAG.%y4%m2%d2.nc # Aircraft fuel consumption OC_AIRCRAFT NA N Y %y4-%m2-%d2t12:00:00 none none oc_aviation /dev/null # Aviation emissions during the three phases of flight -OC_AVIATION_LTO NA N Y %y4-%m2-%d2t12:00:00 none none OC_lto ChemInput/NEXUS_DIAG.%y4%m2%d2.nc -OC_AVIATION_CDS NA N Y %y4-%m2-%d2t12:00:00 none none OC_cds ChemInput/NEXUS_DIAG.%y4%m2%d2.nc -OC_AVIATION_CRS NA N Y %y4-%m2-%d2t12:00:00 none none OC_crs ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +OC_AVIATION_LTO NA N Y F0 none none OC_lto ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +OC_AVIATION_CDS NA N Y F0 none none OC_cds ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +OC_AVIATION_CRS NA N Y F0 none none OC_crs ChemInput/NEXUS_DIAG.%y4%m2%d2.nc # SOA production pSOA_ANTHRO_VOC NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null @@ -88,19 +88,19 @@ pSOA_ANTHRO_VOC NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null BC_BIOFUEL NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -BC_ANTEBC1 NA N Y %y4-%m2-%d2t12:00:00 none none BC ChemInput/NEXUS_DIAG.%y4%m2%d2.nc -BC_ANTEBC2 NA N Y %y4-%m2-%d2t12:00:00 none none BC_elev ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +BC_ANTEBC1 NA N Y F0 none none BC ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +BC_ANTEBC2 NA N Y F0 none none BC_elev ChemInput/NEXUS_DIAG.%y4%m2%d2.nc # EDGAR based ship emissions -BC_SHIP NA N Y %y4-%m2-%d2t12:00:00 none none BC_ship ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +BC_SHIP NA N Y F0 none none BC_ship ChemInput/NEXUS_DIAG.%y4%m2%d2.nc # Aircraft fuel consumption -BC_AIRCRAFT NA N Y %y4-%m2-%d2t12:00:00 none none bc_aviation /dev/null +BC_AIRCRAFT NA N Y F0 none none bc_aviation /dev/null # Aviation emissions during the LTO, SDC and CRS phases of flight -BC_AVIATION_LTO NA N Y %y4-%m2-%d2t12:00:00 none none BC_lto ChemInput/NEXUS_DIAG.%y4%m2%d2.nc -BC_AVIATION_CDS NA N Y %y4-%m2-%d2t12:00:00 none none BC_cds ChemInput/NEXUS_DIAG.%y4%m2%d2.nc -BC_AVIATION_CRS NA N Y %y4-%m2-%d2t12:00:00 none none BC_crs ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +BC_AVIATION_LTO NA N Y F0 none none BC_lto ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +BC_AVIATION_CDS NA N Y F0 none none BC_cds ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +BC_AVIATION_CRS NA N Y F0 none none BC_crs ChemInput/NEXUS_DIAG.%y4%m2%d2.nc #============================================================================================================ # BROWN CARBON @@ -144,4 +144,4 @@ pSOA_BIOB_VOC NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null # # If using 64 levels please replace this section with the correct values (ie replace 127 with 64) # NITRATE_HNO3 'mol mol-1' Y N %y4-%m2-%d2T12:00:00 none 0.20 hno3 ExtData/PIESA/L127/GMI.vmr_HNO3.x144_y91.t12.2006.nc4 # # -------------------------------------------------------------- -# NI_regionMask NA Y V - none none REGION_MASK ExtData/PIESA/sfc/ARCTAS.region_mask.x540_y361.2008.nc +# NI_regionMask NA Y V - none none REGION_MASK ExtData/PIESA/sfc/ARCTAS.region_mask.x540_y361.2008.nc \ No newline at end of file diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh index bd5b814ccfb..149255e0bcf 100755 --- a/ush/forecast_predet.sh +++ b/ush/forecast_predet.sh @@ -789,7 +789,6 @@ GOCART_predet(){ # Create the ChemInput directory in the local run directory if [[ ! -d "${DATA}/ChemInput" ]]; then mkdir -p "${DATA}/ChemInput"; fi - # Copy Fire Emission Files ChemInput directory local current local YYYYMMDDHH @@ -809,22 +808,20 @@ GOCART_predet(){ exit 1 fi - # Increment by 1 day current=$(date -d "${current:0:8} ${current:8:2} +1 day" +%Y%m%d%H) done - # Copy NXS Emission Files ChemInput directory # NXS files are hourly, so we need to loop through each hour in the cycle current=$(date -d "${current_cycle_begin:0:8} ${current_cycle_begin:8:2}" +%Y%m%d%H) - cycleend=$(date -d "${current_cycle_end:0:8} ${current_cycle_end:8:2} +24 hour" +%Y%m%d%H) + cycleend=$(date -d "${current_cycle_end:0:8} ${current_cycle_end:8:2} +3 hour" +%Y%m%d%H) while [[ "${current}" -le "${cycleend}" ]]; do if ! YYYYMMDD=$(date -d "${current:0:8} ${current:8:2}:00:00" +%Y%m%d 2>/dev/null); then echo "FATAL ERROR: Invalid date string '${current}' in GOCART_predet, ABORT!" exit 1 fi - local NXSFile="${COMIN_CHEM_INPUT}/${NXS_DIAG_PREFIX}.${YYYYMMDD}.nc" + local NXSFile="${COMIN_CHEM_INPUT}/${NEXUS_DIAG_PREFIX}.${YYYYMMDD}.nc" if [[ -f "${NXSFile}" ]]; then cpreq "${NXSFile}" "${DATA}/ChemInput/" else @@ -835,6 +832,5 @@ GOCART_predet(){ current=$(date -d "${current:0:8} ${current:8:2} +1 hour" +%Y%m%d%H) done - # GOCART output times can't be computed here because they may depend on FHROT } diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 8b0b8e8c408..4cd3d235b50 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -43,10 +43,20 @@ def __init__(self, config: Dict[str, Any]) -> None: self.historical = bool(self.task_config.get('AERO_EMIS_FIRE_HIST', 1)) self.AERO_INPUTS_DIR = self.task_config.get('AERO_INPUTS_DIR', None) self.COMOUT_CHEM_INPUT = self.task_config.get('COMOUT_CHEM_INPUT', None) - nforecast_hours = self.task_config["FHMAX_GFS"] - self.start_date = self.task_config["PDY"] - self.end_date = self.start_date + to_timedelta(f'{nforecast_hours + 24}H') + + # get the nforecast hours - gcdas will use FHMAX and gcafs will use FHMAX_GFS + if 'das' in self.task_config['RUN']: + nforecast_hours = self.task_config["FHMAX"] + else: + nforecast_hours = self.task_config["FHMAX_GFS"] + + # Create start date based on SDATE - 12 hours + self.start_date = self.task_config["SDATE"] + + # end date = SDATE + nforecast hours + 36 + self.end_date = self.task_config["SDATE"] + to_timedelta(f'{nforecast_hours + 24}H') self.forecast_dates = list(rrule(freq=DAILY, dtstart=self.start_date, until=self.end_date)) + logger.info(f"Forecast dates: {self.forecast_dates}") @logit(logger) def initialize(self) -> None: diff --git a/ush/python/pygfs/task/nexus_emission.py b/ush/python/pygfs/task/nexus_emission.py index 77c1142bf96..4556aefda35 100644 --- a/ush/python/pygfs/task/nexus_emission.py +++ b/ush/python/pygfs/task/nexus_emission.py @@ -4,6 +4,7 @@ import re import xarray as xr import subprocess +import numpy as np import cftime from logging import getLogger from typing import Dict, Any, Union, List @@ -41,11 +42,22 @@ def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) self.task_config = AttrDict(config) + self.AERO_INPUTS_DIR = self.task_config.get('AERO_INPUTS_DIR', None) self.COMOUT_CHEM_INPUT = self.task_config.get('COMOUT_CHEM_INPUT', None) - nforecast_hours = self.task_config["FHMAX_GFS"] - self.start_date = self.task_config["SDATE"] - to_timedelta('12H') - self.end_date = self.task_config["EDATE"] + to_timedelta('12H') + + # get the nforecast hours - gcdas will use FHMAX and gcafs will use FHMAX_GFS + if 'das' in self.task_config['RUN']: + nforecast_hours = self.task_config["FHMAX"] + else: + nforecast_hours = self.task_config["FHMAX_GFS"] + + # Create start date based on SDATE + self.start_date = self.task_config["SDATE"] + self.total_hrs = nforecast_hours + 2 + self.end_date = self.task_config["SDATE"] + to_timedelta(f'{self.total_hrs}H') + + # Create the forecast dates based on start_date and end_date frequency = self.task_config.get("NEXUS_DIAG_FREQ", "Hourly") if frequency == "Hourly": self.forecast_dates = list(rrule(freq=HOURLY, dtstart=self.start_date, until=self.end_date)) @@ -155,6 +167,8 @@ def initialize(self) -> None: fname_final = f"{self.task_config.NEXUS_DIAG_PREFIX}.{d.strftime('%Y%m%d')}.nc" processed_nexus_files.append(fname) final_output_files.append(fname_final) + final_output_files = list(set(final_output_files)) + logger.info(f"Final output files: {final_output_files}") self.processed_nexus_files = processed_nexus_files # render the NEXUS configuration files if not os.path.exists(nexus_config_dir): @@ -310,9 +324,11 @@ def execute(self) -> None: raise WorkflowException("No 'units' attribute found for time variable.") # Convert time values to datetime objects - time_vals = time_var.values + time_vals = np.arange(len(files)) + ds = ds.assign_coords(time=time_vals) time_dt = cftime.num2date(time_vals, units=time_units, calendar=time_calendar) + units_string = f"hours since {self.start_date.strftime('%Y-%m-%d %H:%M:%S')}" # Group indices by day from collections import defaultdict day_to_indices = defaultdict(list) @@ -322,6 +338,7 @@ def execute(self) -> None: encoding = {var: {"zlib": True, "complevel": 4} for var in ds.data_vars} for day_str, indices in day_to_indices.items(): daily_ds = ds.isel(time=indices) + daily_ds.time.attrs['units'] = units_string outname = f"{self.task_config.NEXUS_DIAG_PREFIX}.{day_str}.nc" daily_ds.to_netcdf(outname, format="NETCDF4", encoding=encoding) logger.info(f"Wrote daily output: {outname}") From b077961c1aea4d4b285e883bb927ed54825c7ae9 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Sun, 17 Aug 2025 01:18:07 -0400 Subject: [PATCH 028/132] Update aviation emissions data and adjust forecast timing in GOCART and NEXUS processing --- parm/ufs/gocart/ExtData.other | 20 +-- ush/forecast_predet.sh | 2 +- ush/python/pygfs/task/chem_fire_emission.py | 2 +- ush/python/pygfs/task/nexus_emission.py | 140 +++++++++++++++----- 4 files changed, 117 insertions(+), 47 deletions(-) diff --git a/parm/ufs/gocart/ExtData.other b/parm/ufs/gocart/ExtData.other index d6dffc68faa..f2b60d5eb39 100644 --- a/parm/ufs/gocart/ExtData.other +++ b/parm/ufs/gocart/ExtData.other @@ -33,9 +33,9 @@ SU_AIRCRAFT NA Y Y %y4-%m2-%d2t12:00:00 none none none /dev/null SU_DMSO NA Y Y %y4-%m2-%d2t12:00:00 none none conc ExtData/MERRA2/sfc/DMSclim_sfcconcentration.x360_y181_t12.Lana2011.nc4 # Aviation emissions during the three phases of flight -SU_AVIATION_LTO NA N Y F0 none none SO2_lto ChemInput/NEXUS_DIAG.%y4%m2%d2.nc -SU_AVIATION_CDS NA N Y F0 none none SO2_cds ChemInput/NEXUS_DIAG.%y4%m2%d2.nc -SU_AVIATION_CRS NA N Y F0 none none SO2_crs ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +SU_AVIATION_LTO NA Y Y %y4-%m2-%d2t12:00:00 none none so2_aviation ExtData/PIESA/sfc/HTAP/v2.2/htap-v2.2.emis_so2.aviation_lto.x3600_y1800_t12.2010.nc4 +SU_AVIATION_CDS NA Y Y %y4-%m2-%d2t12:00:00 none none so2_aviation ExtData/PIESA/sfc/HTAP/v2.2/htap-v2.2.emis_so2.aviation_cds.x3600_y1800_t12.2010.nc4 +SU_AVIATION_CRS NA Y Y %y4-%m2-%d2t12:00:00 none none so2_aviation ExtData/PIESA/sfc/HTAP/v2.2/htap-v2.2.emis_so2.aviation_crs.x3600_y1800_t12.2010.nc4 # H2O2, OH and NO3 mixing ratios # -------------------------------------------------------------- @@ -75,9 +75,9 @@ OC_SHIP NA N Y F0 none none OC_ship ChemInput/NEXUS_DIAG.%y4%m2%d2.nc OC_AIRCRAFT NA N Y %y4-%m2-%d2t12:00:00 none none oc_aviation /dev/null # Aviation emissions during the three phases of flight -OC_AVIATION_LTO NA N Y F0 none none OC_lto ChemInput/NEXUS_DIAG.%y4%m2%d2.nc -OC_AVIATION_CDS NA N Y F0 none none OC_cds ChemInput/NEXUS_DIAG.%y4%m2%d2.nc -OC_AVIATION_CRS NA N Y F0 none none OC_crs ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +OC_AVIATION_LTO NA Y Y %y4-%m2-%d2t12:00:00 none none oc_aviation ExtData/PIESA/sfc/HTAP/v2.2/htap-v2.2.emis_oc.aviation_lto.x3600_y1800_t12.2010.nc4 +OC_AVIATION_CDS NA Y Y %y4-%m2-%d2t12:00:00 none none oc_aviation ExtData/PIESA/sfc/HTAP/v2.2/htap-v2.2.emis_oc.aviation_cds.x3600_y1800_t12.2010.nc4 +OC_AVIATION_CRS NA Y Y %y4-%m2-%d2t12:00:00 none none oc_aviation ExtData/PIESA/sfc/HTAP/v2.2/htap-v2.2.emis_oc.aviation_crs.x3600_y1800_t12.2010.nc4 # SOA production pSOA_ANTHRO_VOC NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null @@ -100,9 +100,9 @@ BC_SHIP NA N Y F0 none none BC_ship ChemInput/NEXUS_DIAG.%y4%m2%d2.nc BC_AIRCRAFT NA N Y F0 none none bc_aviation /dev/null # Aviation emissions during the LTO, SDC and CRS phases of flight -BC_AVIATION_LTO NA N Y F0 none none BC_lto ChemInput/NEXUS_DIAG.%y4%m2%d2.nc -BC_AVIATION_CDS NA N Y F0 none none BC_cds ChemInput/NEXUS_DIAG.%y4%m2%d2.nc -BC_AVIATION_CRS NA N Y F0 none none BC_crs ChemInput/NEXUS_DIAG.%y4%m2%d2.nc +BC_AVIATION_LTO NA Y Y %y4-%m2-%d2t12:00:00 none none bc_aviation ExtData/PIESA/sfc/HTAP/v2.2/htap-v2.2.emis_bc.aviation_lto.x3600_y1800_t12.2010.nc4 +BC_AVIATION_CDS NA Y Y %y4-%m2-%d2t12:00:00 none none bc_aviation ExtData/PIESA/sfc/HTAP/v2.2/htap-v2.2.emis_bc.aviation_cds.x3600_y1800_t12.2010.nc4 +BC_AVIATION_CRS NA Y Y %y4-%m2-%d2t12:00:00 none none bc_aviation ExtData/PIESA/sfc/HTAP/v2.2/htap-v2.2.emis_bc.aviation_crs.x3600_y1800_t12.2010.nc4 #============================================================================================================ # BROWN CARBON @@ -146,4 +146,4 @@ pSOA_BIOB_VOC NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null # # If using 64 levels please replace this section with the correct values (ie replace 127 with 64) # NITRATE_HNO3 'mol mol-1' Y N %y4-%m2-%d2T12:00:00 none 0.20 hno3 ExtData/PIESA/L127/GMI.vmr_HNO3.x144_y91.t12.2006.nc4 # # -------------------------------------------------------------- -# NI_regionMask NA Y V - none none REGION_MASK ExtData/PIESA/sfc/ARCTAS.region_mask.x540_y361.2008.nc \ No newline at end of file +# NI_regionMask NA Y V - none none REGION_MASK ExtData/PIESA/sfc/ARCTAS.region_mask.x540_y361.2008.nc diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh index 149255e0bcf..4c8a47d7287 100755 --- a/ush/forecast_predet.sh +++ b/ush/forecast_predet.sh @@ -815,7 +815,7 @@ GOCART_predet(){ # Copy NXS Emission Files ChemInput directory # NXS files are hourly, so we need to loop through each hour in the cycle current=$(date -d "${current_cycle_begin:0:8} ${current_cycle_begin:8:2}" +%Y%m%d%H) - cycleend=$(date -d "${current_cycle_end:0:8} ${current_cycle_end:8:2} +3 hour" +%Y%m%d%H) + cycleend=$(date -d "${current_cycle_end:0:8} ${current_cycle_end:8:2} +1 hour" +%Y%m%d%H) while [[ "${current}" -le "${cycleend}" ]]; do if ! YYYYMMDD=$(date -d "${current:0:8} ${current:8:2}:00:00" +%Y%m%d 2>/dev/null); then echo "FATAL ERROR: Invalid date string '${current}' in GOCART_predet, ABORT!" diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 4cd3d235b50..7fb2b2e0204 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -53,7 +53,7 @@ def __init__(self, config: Dict[str, Any]) -> None: # Create start date based on SDATE - 12 hours self.start_date = self.task_config["SDATE"] - # end date = SDATE + nforecast hours + 36 + # end date = SDATE + nforecast hours + 24 self.end_date = self.task_config["SDATE"] + to_timedelta(f'{nforecast_hours + 24}H') self.forecast_dates = list(rrule(freq=DAILY, dtstart=self.start_date, until=self.end_date)) logger.info(f"Forecast dates: {self.forecast_dates}") diff --git a/ush/python/pygfs/task/nexus_emission.py b/ush/python/pygfs/task/nexus_emission.py index 4556aefda35..1ef85c0ea43 100644 --- a/ush/python/pygfs/task/nexus_emission.py +++ b/ush/python/pygfs/task/nexus_emission.py @@ -2,6 +2,7 @@ import os import re +from collections import defaultdict import xarray as xr import subprocess import numpy as np @@ -54,7 +55,7 @@ def __init__(self, config: Dict[str, Any]) -> None: # Create start date based on SDATE self.start_date = self.task_config["SDATE"] - self.total_hrs = nforecast_hours + 2 + self.total_hrs = nforecast_hours + 3 self.end_date = self.task_config["SDATE"] + to_timedelta(f'{self.total_hrs}H') # Create the forecast dates based on start_date and end_date @@ -305,44 +306,103 @@ def execute(self) -> None: logger.info("Concatenating processed NEXUS files...") + # sort the files even though they should be sorted already | safety check files = sorted(self.processed_nexus_files) - dsets = [] - for f in files: - dsets.append(xr.open_dataset(f, decode_cf=False)) - - # Concatenate along time dimension - ds = xr.concat(dsets, dim="time") - - # Convert raw time values to datetime objects using cftime - if 'time' not in ds.dims: - raise WorkflowException("No 'time' dimension found in NEXUS output dataset.") - - time_var = ds['time'] - time_units = time_var.attrs.get('units', None) - time_calendar = time_var.attrs.get('calendar', 'standard') - if time_units is None: - raise WorkflowException("No 'units' attribute found for time variable.") - - # Convert time values to datetime objects - time_vals = np.arange(len(files)) - ds = ds.assign_coords(time=time_vals) - time_dt = cftime.num2date(time_vals, units=time_units, calendar=time_calendar) - - units_string = f"hours since {self.start_date.strftime('%Y-%m-%d %H:%M:%S')}" - # Group indices by day - from collections import defaultdict - day_to_indices = defaultdict(list) - for idx, dt in enumerate(time_dt): - day_to_indices[dt.strftime('%Y%m%d')].append(idx) - - encoding = {var: {"zlib": True, "complevel": 4} for var in ds.data_vars} - for day_str, indices in day_to_indices.items(): - daily_ds = ds.isel(time=indices) - daily_ds.time.attrs['units'] = units_string + + for i in files: + + if not os.path.exists(i): + logger.warning(f"NEXUS file not found: {i}") + continue + else: + logger.info(f"NEXUS file found: {i}") + + for f, dates in zip(files, self.forecast_dates): + logger.info(f" - {f}, {dates}") + + # find the day indexes for each unique day + # this returns a dictionary + # example: + # { + # datetime.date(2024, 1, 5): [0, 1, 3], + # datetime.date(2024, 1, 6): [2] + # } + day_indexes = _get_day_indices(self.forecast_dates[:-1]) # hemco doesn't write out the last timestep + + # now loop over each days + for date, indexes in day_indexes.items(): + day_str = date.strftime('%Y%m%d') + logger.info(f"Processing NEXUS files for date: {date}") + + dsets = [] + print(indexes, len(files)) + for index in indexes: + # list files for log + logger.info(f" - {files[index]}, {index}") + + # now concatenate the files per day + if os.path.exists(files[index]) is False: + break + ds = xr.open_dataset(files[index], decode_cf=False) + + # update time coordinate + ds = ds.assign_coords(time=('time',[index])) + + # set time units to reference start-date + ds.time.attrs['units'] = self.start_date.strftime('hours since %Y-%m-%d %H:00:00') + + # append + dsets.append(ds) + + # concatenate all the files for this day + if dsets is None: + break + else: + ds = xr.concat(dsets, dim='time') + + encoding = {var: {"zlib": True, "complevel": 2} for var in ds.data_vars} outname = f"{self.task_config.NEXUS_DIAG_PREFIX}.{day_str}.nc" - daily_ds.to_netcdf(outname, format="NETCDF4", encoding=encoding) + ds.to_netcdf(outname, format="NETCDF4", encoding=encoding) logger.info(f"Wrote daily output: {outname}") + # dsets = [] + # for index, fname in enumerate(files): + # dset = xr.open_dataset(fname,decode_cf=False) + # dsets.append(dset.assign_coords(time=[index])) + + # # Concatenate along time dimension + # ds = xr.concat(dsets, dim="time") + # ds.time.attrs['units'] = self.start_date.strftime('hours since %y-%m-%d %H:00:00') + + # # Convert raw time values to datetime objects using cftime + # if 'time' not in ds.dims: + # raise WorkflowException("No 'time' dimension found in NEXUS output dataset.") + + # time_var = ds['time'] + # time_units = time_var.attrs.get('units', None) + # time_calendar = time_var.attrs.get('calendar', 'standard') + # if time_units is None: + # raise WorkflowException("No 'units' attribute found for time variable.") + + # # Convert time values to datetime objects + # time_vals = np.arange(len(files)) + # ds = ds.assign_coords(time=time_vals) + # time_dt = cftime.num2date(time_vals, units=time_units, calendar=time_calendar) + + # # Group indices by day + + # day_to_indices = defaultdict(list) + # for idx, dt in enumerate(time_dt): + # day_to_indices[dt.strftime('%y%m%d')].append(idx) + + # encoding = {var: {"zlib": True, "complevel": 4} for var in ds.data_vars} + # for day_str, indices in day_to_indices.items(): + # daily_ds = ds.isel(time=indices) + # daily_ds.time.attrs['units'] = units_string + # outname = f"{self.task_config.NEXUS_DIAG_PREFIX}.{day_str}.nc" + # daily_ds.to_netcdf(outname, format="NETCDF4", encoding=encoding) + # logger.info(f"Wrote daily output: {outname}") + logger.info("NEXUS emission processing execute phase complete") @logit(logger) @@ -387,3 +447,13 @@ def _write_txt_file(content: str, file_path: Union[str, os.PathLike]) -> None: os.makedirs(os.path.dirname(file_path), exist_ok=True) with open(file_path, 'w') as f: f.write(content) + +def _get_day_indices(date_list): + day_indices = {} + for idx, dt_obj in enumerate(date_list): + # Extract the date part (ignores time) + day = dt_obj.date() + if day not in day_indices: + day_indices[day] = [] + day_indices[day].append(idx) + return day_indices From d282736c51d8951d819391f41279395c5c609dc5 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Sun, 17 Aug 2025 01:30:51 -0400 Subject: [PATCH 029/132] Update NEXUS emissions task to use SDATE_GFS for date calculations and add debug output for day indices --- ush/python/pygfs/task/nexus_emission.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ush/python/pygfs/task/nexus_emission.py b/ush/python/pygfs/task/nexus_emission.py index 1ef85c0ea43..30881589c0a 100644 --- a/ush/python/pygfs/task/nexus_emission.py +++ b/ush/python/pygfs/task/nexus_emission.py @@ -54,9 +54,9 @@ def __init__(self, config: Dict[str, Any]) -> None: nforecast_hours = self.task_config["FHMAX_GFS"] # Create start date based on SDATE - self.start_date = self.task_config["SDATE"] + self.start_date = self.task_config["SDATE_GFS"] self.total_hrs = nforecast_hours + 3 - self.end_date = self.task_config["SDATE"] + to_timedelta(f'{self.total_hrs}H') + self.end_date = self.task_config["SDATE_GFS"] + to_timedelta(f'{self.total_hrs}H') # Create the forecast dates based on start_date and end_date frequency = self.task_config.get("NEXUS_DIAG_FREQ", "Hourly") @@ -328,7 +328,8 @@ def execute(self) -> None: # datetime.date(2024, 1, 6): [2] # } day_indexes = _get_day_indices(self.forecast_dates[:-1]) # hemco doesn't write out the last timestep - + from pprint import pprint + pprint(day_indexes) # now loop over each days for date, indexes in day_indexes.items(): day_str = date.strftime('%Y%m%d') From e61516e7176cddfce83a7037b2e30aa2a4f68c3f Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Sun, 17 Aug 2025 01:31:43 -0400 Subject: [PATCH 030/132] Update ChemFireEmissions to use SDATE_GFS for start and end date calculations --- ush/python/pygfs/task/chem_fire_emission.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 7fb2b2e0204..80ec1688b2b 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -51,10 +51,10 @@ def __init__(self, config: Dict[str, Any]) -> None: nforecast_hours = self.task_config["FHMAX_GFS"] # Create start date based on SDATE - 12 hours - self.start_date = self.task_config["SDATE"] + self.start_date = self.task_config["SDATE_GFS"] # end date = SDATE + nforecast hours + 24 - self.end_date = self.task_config["SDATE"] + to_timedelta(f'{nforecast_hours + 24}H') + self.end_date = self.task_config["SDATE_GFS"] + to_timedelta(f'{nforecast_hours + 24}H') self.forecast_dates = list(rrule(freq=DAILY, dtstart=self.start_date, until=self.end_date)) logger.info(f"Forecast dates: {self.forecast_dates}") From 39e454b91fd2ea0d1ca6c1851cf5638c236ce321 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Sun, 17 Aug 2025 01:37:43 -0400 Subject: [PATCH 031/132] Update subproject commit reference in nexus.fd --- sorc/nexus.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/nexus.fd b/sorc/nexus.fd index 72bb61fe857..6c660b10a58 160000 --- a/sorc/nexus.fd +++ b/sorc/nexus.fd @@ -1 +1 @@ -Subproject commit 72bb61fe857b46f94d888328d411b1a6d0a8cf4a +Subproject commit 6c660b10a58e3596db527753106daea35ef152a6 From 6ad4adf46461f91cf9d2faac65c8d0e0fb9ca4c7 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Sun, 17 Aug 2025 15:19:38 -0400 Subject: [PATCH 032/132] Refactor _get_day_indices function to group datetime indices by day, including midnight in both days --- ush/python/pygfs/task/nexus_emission.py | 42 ++++++++++++++++++------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/ush/python/pygfs/task/nexus_emission.py b/ush/python/pygfs/task/nexus_emission.py index 30881589c0a..0bce5a579b9 100644 --- a/ush/python/pygfs/task/nexus_emission.py +++ b/ush/python/pygfs/task/nexus_emission.py @@ -328,8 +328,6 @@ def execute(self) -> None: # datetime.date(2024, 1, 6): [2] # } day_indexes = _get_day_indices(self.forecast_dates[:-1]) # hemco doesn't write out the last timestep - from pprint import pprint - pprint(day_indexes) # now loop over each days for date, indexes in day_indexes.items(): day_str = date.strftime('%Y%m%d') @@ -449,12 +447,34 @@ def _write_txt_file(content: str, file_path: Union[str, os.PathLike]) -> None: with open(file_path, 'w') as f: f.write(content) -def _get_day_indices(date_list): - day_indices = {} - for idx, dt_obj in enumerate(date_list): - # Extract the date part (ignores time) - day = dt_obj.date() - if day not in day_indices: - day_indices[day] = [] - day_indices[day].append(idx) - return day_indices + +def _get_day_indices(datetimes): + """ + Group indices of datetimes by day, including midnight in both days. + + Parameters + ---------- + datetimes : list of datetime.datetime + List of datetime objects. + + Returns + ------- + dict + Dictionary mapping datetime.datetime (at midnight) to list of indices. + Each day includes all hours from 00:00 of that day through 00:00 of the next day, + and the midnight index is included in both days. + """ + from collections import defaultdict + from datetime import timedelta + + grouped = defaultdict(list) + + for idx, dt in enumerate(datetimes): + day_dt = dt.replace(hour=0, minute=0, second=0, microsecond=0) + grouped[day_dt].append(idx) + # If this is exactly midnight, also add to previous day + if dt.hour == 0 and dt.minute == 0 and dt.second == 0 and dt.microsecond == 0: + prev_day = day_dt - timedelta(days=1) + grouped[prev_day].append(idx) + + return dict(grouped) From a3f1c3622068ec3d7a0d36ea7ee6b279b025820e Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Sun, 17 Aug 2025 15:20:07 -0400 Subject: [PATCH 033/132] Comment out data copying logic in GOCART_predet function for future implementation - link for now using ${NLN} --- ush/forecast_predet.sh | 89 ++++++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 43 deletions(-) diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh index 4c8a47d7287..5afefe1133d 100755 --- a/ush/forecast_predet.sh +++ b/ush/forecast_predet.sh @@ -786,51 +786,54 @@ GOCART_predet(){ # FHMAX gets modified when IAU is on, so keep origianl value for GOCART output GOCART_MAX=${FHMAX} + #TODO: fix to copying data so that its required + ${NLN} "${COMIN_CHEM_INPUT}" "${DATA}/ChemInput" # Create the ChemInput directory in the local run directory - if [[ ! -d "${DATA}/ChemInput" ]]; then mkdir -p "${DATA}/ChemInput"; fi - - # Copy Fire Emission Files ChemInput directory - local current - local YYYYMMDDHH - current="${current_cycle_begin}" - cycleend=$(date -d "${current_cycle_end:0:8} ${current_cycle_end:8:2} +24 hour" +%Y%m%d%H) - while [[ "${current}" -le "${cycleend}" ]]; do - # Validate current is a valid date string - if ! YYYYMMDD=$(date -d "${current:0:8} ${current:8:2}:00:00" +%Y%m%d 2>/dev/null); then - echo "FATAL ERROR: Invalid date string '${current}' in GOCART_predet, ABORT!" - exit 1 - fi - local FireEmisFile="${COMIN_CHEM_INPUT}/FIRE_EMIS_${YYYYMMDD}.nc" - if [[ -f "${FireEmisFile}" ]]; then - cpreq "${FireEmisFile}" "${DATA}/ChemInput/" - else - echo "FATAL ERROR: GOCART input file '${FireEmisFile}' does not exist, ABORT!" - exit 1 - fi - - # Increment by 1 day - current=$(date -d "${current:0:8} ${current:8:2} +1 day" +%Y%m%d%H) - done - # Copy NXS Emission Files ChemInput directory - # NXS files are hourly, so we need to loop through each hour in the cycle - current=$(date -d "${current_cycle_begin:0:8} ${current_cycle_begin:8:2}" +%Y%m%d%H) - cycleend=$(date -d "${current_cycle_end:0:8} ${current_cycle_end:8:2} +1 hour" +%Y%m%d%H) - while [[ "${current}" -le "${cycleend}" ]]; do - if ! YYYYMMDD=$(date -d "${current:0:8} ${current:8:2}:00:00" +%Y%m%d 2>/dev/null); then - echo "FATAL ERROR: Invalid date string '${current}' in GOCART_predet, ABORT!" - exit 1 - fi - local NXSFile="${COMIN_CHEM_INPUT}/${NEXUS_DIAG_PREFIX}.${YYYYMMDD}.nc" - if [[ -f "${NXSFile}" ]]; then - cpreq "${NXSFile}" "${DATA}/ChemInput/" - else - echo "FATAL ERROR: GOCART input file '${NXSFile}' does not exist, ABORT!" - exit 1 - fi - # Increment by 1 hour - current=$(date -d "${current:0:8} ${current:8:2} +1 hour" +%Y%m%d%H) - done + # if [[ ! -d "${DATA}/ChemInput" ]]; then mkdir -p "${DATA}/ChemInput"; fi + + # # Copy Fire Emission Files ChemInput directory + # local current + # local YYYYMMDDHH + # current="${current_cycle_begin}" + # cycleend=$(date -d "${current_cycle_end:0:8} ${current_cycle_end:8:2} +24 hour" +%Y%m%d%H) + # while [[ "${current}" -le "${cycleend}" ]]; do + # # Validate current is a valid date string + # if ! YYYYMMDD=$(date -d "${current:0:8} ${current:8:2}:00:00" +%Y%m%d 2>/dev/null); then + # echo "FATAL ERROR: Invalid date string '${current}' in GOCART_predet, ABORT!" + # exit 1 + # fi + # local FireEmisFile="${COMIN_CHEM_INPUT}/FIRE_EMIS_${YYYYMMDD}.nc" + # if [[ -f "${FireEmisFile}" ]]; then + # cpreq "${FireEmisFile}" "${DATA}/ChemInput/" + # else + # echo "FATAL ERROR: GOCART input file '${FireEmisFile}' does not exist, ABORT!" + # exit 1 + # fi + + # # Increment by 1 day + # current=$(date -d "${current:0:8} ${current:8:2} +1 day" +%Y%m%d%H) + # done + + # # Copy NXS Emission Files ChemInput directory + # # NXS files are hourly, so we need to loop through each hour in the cycle + # current=$(date -d "${current_cycle_begin:0:8} ${current_cycle_begin:8:2}" +%Y%m%d%H) + # cycleend=$(date -d "${current_cycle_end:0:8} ${current_cycle_end:8:2} +1 hour" +%Y%m%d%H) + # while [[ "${current}" -le "${cycleend}" ]]; do + # if ! YYYYMMDD=$(date -d "${current:0:8} ${current:8:2}:00:00" +%Y%m%d 2>/dev/null); then + # echo "FATAL ERROR: Invalid date string '${current}' in GOCART_predet, ABORT!" + # exit 1 + # fi + # local NXSFile="${COMIN_CHEM_INPUT}/${NEXUS_DIAG_PREFIX}.${YYYYMMDD}.nc" + # if [[ -f "${NXSFile}" ]]; then + # cpreq "${NXSFile}" "${DATA}/ChemInput/" + # else + # echo "FATAL ERROR: GOCART input file '${NXSFile}' does not exist, ABORT!" + # exit 1 + # fi + # # Increment by 1 hour + # current=$(date -d "${current:0:8} ${current:8:2} +1 hour" +%Y%m%d%H) + # done # GOCART output times can't be computed here because they may depend on FHROT } From c406f0565fe7f3022fa5f226686342312ddb63c2 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Sun, 17 Aug 2025 23:15:02 -0400 Subject: [PATCH 034/132] Update soil_drylimit_factor to improve emissions calculations --- parm/ufs/gocart/DU2G_instance_DU.rc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/ufs/gocart/DU2G_instance_DU.rc b/parm/ufs/gocart/DU2G_instance_DU.rc index 8001798189d..39fb0f83050 100644 --- a/parm/ufs/gocart/DU2G_instance_DU.rc +++ b/parm/ufs/gocart/DU2G_instance_DU.rc @@ -55,6 +55,6 @@ emission_scheme: fengsha # choose among: fengsha, ginoux, k14 alpha: 0.16 gamma: 1.0 soil_moisture_factor: 1 -soil_drylimit_factor: 1 +soil_drylimit_factor: 1.75 vertical_to_horizontal_flux_ratio_limit: 2.e-04 drag_partition_option: 2 \ No newline at end of file From ecf691ec1b1fc9f5c79919b61527e18b0a4c7186 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Mon, 18 Aug 2025 00:11:11 -0400 Subject: [PATCH 035/132] change RUN variable in declare_from_tmpl for chemical input --- jobs/JGLOBAL_FORECAST | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/JGLOBAL_FORECAST b/jobs/JGLOBAL_FORECAST index c60b86294e1..8cd44dfacec 100755 --- a/jobs/JGLOBAL_FORECAST +++ b/jobs/JGLOBAL_FORECAST @@ -92,7 +92,7 @@ if [[ "${DO_AERO_FCST}" == "YES" ]]; then COMOUT_CHEM_HISTORY:COM_CHEM_HISTORY_TMPL YMD="${PDY}" HH="${cyc}" RUN="${rCDUMP}" declare_from_tmpl -rx \ COMIN_TRACER_RESTART:COM_ATMOS_RESTART_TMPL - YMD="${PDY}" HH="${cyc}" RUN="${rCDUMP}" declare_from_tmpl -rx \ + YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx \ COMIN_CHEM_INPUT:COM_CHEM_INPUT_TMPL fi From b53778cc0398d8f09dfae525e7230812e78ce241 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Mon, 18 Aug 2025 00:11:32 -0400 Subject: [PATCH 036/132] Enhance ChemFireEmissions class to log forecast period details and adjust end date calculation --- ush/python/pygfs/task/chem_fire_emission.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 80ec1688b2b..d6779f28344 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -5,10 +5,10 @@ import fnmatch import datetime import xarray as xr +import numpy as np from logging import getLogger from typing import Dict, Any, Union, List from dateutil.rrule import DAILY, rrule - from wxflow import (AttrDict, parse_j2yaml, FileHandler, @@ -49,13 +49,21 @@ def __init__(self, config: Dict[str, Any]) -> None: nforecast_hours = self.task_config["FHMAX"] else: nforecast_hours = self.task_config["FHMAX_GFS"] + logger.info(f"Number of forecast hours: {nforecast_hours}") # Create start date based on SDATE - 12 hours self.start_date = self.task_config["SDATE_GFS"] + logger.info(f"Start date: {self.start_date}") # end date = SDATE + nforecast hours + 24 - self.end_date = self.task_config["SDATE_GFS"] + to_timedelta(f'{nforecast_hours + 24}H') - self.forecast_dates = list(rrule(freq=DAILY, dtstart=self.start_date, until=self.end_date)) + self.end_date = self.task_config["SDATE_GFS"] + to_timedelta(f'{nforecast_hours + 36}H') + logger.info(f"End date: {self.end_date}") + + # Calculate number of days spanned by start and end date (inclusive) + numdays = (self.end_date.date() - self.start_date.date()).days + 1 + logger.info(f"Number of days in forecast period: {numdays}") + + self.forecast_dates = list(rrule(freq=DAILY, dtstart=self.start_date, count=numdays)) logger.info(f"Forecast dates: {self.forecast_dates}") @logit(logger) From fdff9078ad86c2e519ee8e9229d8a5fbbba32d78 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Mon, 18 Aug 2025 00:35:14 -0400 Subject: [PATCH 037/132] Add NEXUS_INPUT_DIR for NEXUS emissions data source - temporarily hardcoded for gaeac6 --- dev/parm/config/gcafs/config.aero.j2 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dev/parm/config/gcafs/config.aero.j2 b/dev/parm/config/gcafs/config.aero.j2 index 14f63a44c6b..2e7580fb037 100644 --- a/dev/parm/config/gcafs/config.aero.j2 +++ b/dev/parm/config/gcafs/config.aero.j2 @@ -42,6 +42,10 @@ export FIRE_EMIS_NRT_DIR="" #TODO: set to DCOM for WCOSS2 "${DCOMROOT}/YYYYMMDD/ export NEXUS_CONFIG="{{ NEXUS_CONFIG | default('gocart') }}" # Options: gocart, none export NEXUS_CONFIG_DIR="${PARMgfs}/chem/nexus/${NEXUS_CONFIG}" # Directory containing NEXUS configuration files +# NEXUS Inputs +# TODO: when this is merged this will point to AERO_INPUTS_DIR for operations +export NEXUS_INPUT_DIR="/gpfs/f6/bil-fire3/world-shared/Emissions/GEFS_ExtData/nexus" + #-------------------------- # NEXUS Time Step (seconds) #-------------------------- From e2271cbe1485c5638ca2da003a96498c6c6e6d1f Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Mon, 18 Aug 2025 00:36:54 -0400 Subject: [PATCH 038/132] Add commented placeholder for NEXUS_INPUT_DIR --- dev/parm/config/gcafs/config.aero.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/parm/config/gcafs/config.aero.j2 b/dev/parm/config/gcafs/config.aero.j2 index 2e7580fb037..b3df15618d8 100644 --- a/dev/parm/config/gcafs/config.aero.j2 +++ b/dev/parm/config/gcafs/config.aero.j2 @@ -44,6 +44,7 @@ export NEXUS_CONFIG_DIR="${PARMgfs}/chem/nexus/${NEXUS_CONFIG}" # Directory cont # NEXUS Inputs # TODO: when this is merged this will point to AERO_INPUTS_DIR for operations +# export NEXUS_INPUT_DIR="${AERO_INPUTS_DIR}/nexus" export NEXUS_INPUT_DIR="/gpfs/f6/bil-fire3/world-shared/Emissions/GEFS_ExtData/nexus" #-------------------------- From b374701a109048e5e04bbeb206658d46b7bde42a Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Mon, 18 Aug 2025 00:41:47 -0400 Subject: [PATCH 039/132] Fix formatting in NEXUSEmissions class for better readability --- ush/python/pygfs/task/nexus_emission.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/nexus_emission.py b/ush/python/pygfs/task/nexus_emission.py index 0bce5a579b9..1cea6fab3f3 100644 --- a/ush/python/pygfs/task/nexus_emission.py +++ b/ush/python/pygfs/task/nexus_emission.py @@ -327,7 +327,7 @@ def execute(self) -> None: # datetime.date(2024, 1, 5): [0, 1, 3], # datetime.date(2024, 1, 6): [2] # } - day_indexes = _get_day_indices(self.forecast_dates[:-1]) # hemco doesn't write out the last timestep + day_indexes = _get_day_indices(self.forecast_dates[:-1]) # hemco doesn't write out the last timestep # now loop over each days for date, indexes in day_indexes.items(): day_str = date.strftime('%Y%m%d') @@ -345,7 +345,7 @@ def execute(self) -> None: ds = xr.open_dataset(files[index], decode_cf=False) # update time coordinate - ds = ds.assign_coords(time=('time',[index])) + ds = ds.assign_coords(time=('time', [index])) # set time units to reference start-date ds.time.attrs['units'] = self.start_date.strftime('hours since %Y-%m-%d %H:00:00') From eada24d88e2b2d1a99baef4b9a8af7604bf499b7 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 27 Aug 2025 11:31:04 -0400 Subject: [PATCH 040/132] Fix indentation in nexus_emission.py --- ush/python/pygfs/task/nexus_emission.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/nexus_emission.py b/ush/python/pygfs/task/nexus_emission.py index 1cea6fab3f3..22978da981a 100644 --- a/ush/python/pygfs/task/nexus_emission.py +++ b/ush/python/pygfs/task/nexus_emission.py @@ -327,7 +327,7 @@ def execute(self) -> None: # datetime.date(2024, 1, 5): [0, 1, 3], # datetime.date(2024, 1, 6): [2] # } - day_indexes = _get_day_indices(self.forecast_dates[:-1]) # hemco doesn't write out the last timestep + day_indexes = _get_day_indices(self.forecast_dates[:-1]) # hemco doesn't write out the last timestep # now loop over each days for date, indexes in day_indexes.items(): day_str = date.strftime('%Y%m%d') From ca694318e7737a8a6f82fe247350cda7bc26dffe Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 3 Sep 2025 09:52:07 -0400 Subject: [PATCH 041/132] Update dev/workflow/rocoto/gcafs_tasks.py Co-authored-by: Cory Martin --- dev/workflow/rocoto/gcafs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/workflow/rocoto/gcafs_tasks.py b/dev/workflow/rocoto/gcafs_tasks.py index e585581045a..9bde4a2eb45 100644 --- a/dev/workflow/rocoto/gcafs_tasks.py +++ b/dev/workflow/rocoto/gcafs_tasks.py @@ -428,7 +428,7 @@ def prepobsaero(self): def aeroanlgenb(self): """ - Create a task for generating aerosol background fields. + Create a task for generating aerosol background error files. This task generates the background fields required for aerosol analysis. From d9b62fc186455f57ceee74012ded8eb1594aab66 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 3 Sep 2025 10:06:39 -0400 Subject: [PATCH 042/132] Add SettlingSolver options to resource files --- parm/ufs/gocart/CA2G_instance_CA.bc.rc | 4 ++++ parm/ufs/gocart/CA2G_instance_CA.br.rc | 4 ++++ parm/ufs/gocart/CA2G_instance_CA.oc.rc | 4 ++++ parm/ufs/gocart/DU2G_instance_DU.rc | 6 +++++- parm/ufs/gocart/NI2G_instance_NI.rc | 4 ++++ parm/ufs/gocart/SS2G_instance_SS.rc | 4 +++- parm/ufs/gocart/SU2G_instance_SU.rc | 4 ++++ 7 files changed, 28 insertions(+), 2 deletions(-) diff --git a/parm/ufs/gocart/CA2G_instance_CA.bc.rc b/parm/ufs/gocart/CA2G_instance_CA.bc.rc index 9ac462fa3e6..19b20a6e65d 100644 --- a/parm/ufs/gocart/CA2G_instance_CA.bc.rc +++ b/parm/ufs/gocart/CA2G_instance_CA.bc.rc @@ -30,6 +30,10 @@ fwet_ice: 0.0 1.0 fwet_snow: 0.0 1.0 fwet_rain: 0.0 1.0 +# SettlingSolver options +# Options: 'gocart' or 'ufs' +settling_scheme: 'ufs' + # Scavenging efficiency per bin [km-1] (NOT USED UNLESS RAS IS CALLED) fscav: 0.0 0.4 diff --git a/parm/ufs/gocart/CA2G_instance_CA.br.rc b/parm/ufs/gocart/CA2G_instance_CA.br.rc index e983240212b..679d33a42c7 100644 --- a/parm/ufs/gocart/CA2G_instance_CA.br.rc +++ b/parm/ufs/gocart/CA2G_instance_CA.br.rc @@ -33,6 +33,10 @@ fwet_ice: 0.0 0.4 fwet_snow: 0.0 0.4 fwet_rain: 0.0 0.4 +# SettlingSolver options +# Options: 'gocart' or 'ufs' +settling_scheme: 'ufs' + # Scavenging efficiency per bin [km-1] (NOT USED UNLESS RAS IS CALLED) fscav: 0.0 0.4 diff --git a/parm/ufs/gocart/CA2G_instance_CA.oc.rc b/parm/ufs/gocart/CA2G_instance_CA.oc.rc index f24d65e4c47..e510fec2eb2 100644 --- a/parm/ufs/gocart/CA2G_instance_CA.oc.rc +++ b/parm/ufs/gocart/CA2G_instance_CA.oc.rc @@ -40,6 +40,10 @@ fwet_ice: 0.0 1.0 fwet_snow: 0.0 1.0 fwet_rain: 0.0 1.0 +# SettlingSolver options +# Options: 'gocart' or 'ufs' +settling_scheme: 'ufs' + # Scavenging efficiency per bin [km-1] (NOT USED UNLESS RAS IS CALLED) fscav: 0.0 0.4 diff --git a/parm/ufs/gocart/DU2G_instance_DU.rc b/parm/ufs/gocart/DU2G_instance_DU.rc index 39fb0f83050..d1a0d98edff 100644 --- a/parm/ufs/gocart/DU2G_instance_DU.rc +++ b/parm/ufs/gocart/DU2G_instance_DU.rc @@ -57,4 +57,8 @@ gamma: 1.0 soil_moisture_factor: 1 soil_drylimit_factor: 1.75 vertical_to_horizontal_flux_ratio_limit: 2.e-04 -drag_partition_option: 2 \ No newline at end of file +drag_partition_option: 2 + +# SettlingSolver options +# Options: 'gocart' or 'ufs' +settling_scheme: 'ufs' \ No newline at end of file diff --git a/parm/ufs/gocart/NI2G_instance_NI.rc b/parm/ufs/gocart/NI2G_instance_NI.rc index 73db6010732..0c5d22218c9 100644 --- a/parm/ufs/gocart/NI2G_instance_NI.rc +++ b/parm/ufs/gocart/NI2G_instance_NI.rc @@ -31,3 +31,7 @@ sigma: 2.0 2.0 2.0 2.0 2.0 pressure_lid_in_hPa: 0.01 rhFlag: 0 + +# SettlingSolver options +# Options: 'gocart' or 'ufs' +settling_scheme: 'ufs' diff --git a/parm/ufs/gocart/SS2G_instance_SS.rc b/parm/ufs/gocart/SS2G_instance_SS.rc index 5616616ea6c..6bc90ae3b7c 100644 --- a/parm/ufs/gocart/SS2G_instance_SS.rc +++ b/parm/ufs/gocart/SS2G_instance_SS.rc @@ -48,4 +48,6 @@ fwet_ice: 1.0 1.0 1.0 1.0 1.0 fwet_snow: 1.0 1.0 1.0 1.0 1.0 fwet_rain: 1.0 1.0 1.0 1.0 1.0 - +# SettlingSolver options +# Options: 'gocart' or 'ufs' +settling_scheme: 'ufs' diff --git a/parm/ufs/gocart/SU2G_instance_SU.rc b/parm/ufs/gocart/SU2G_instance_SU.rc index 2efa5ec49f8..1613283d23b 100644 --- a/parm/ufs/gocart/SU2G_instance_SU.rc +++ b/parm/ufs/gocart/SU2G_instance_SU.rc @@ -23,6 +23,10 @@ aircraft_fuel_emission_factor: 0.0008 # Scavenging efficiency per bin [km-1] (NOT USED UNLESS RAS IS CALLED) fscav: 0.0 0.0 0.4 0.4 +# SettlingSolver options +# Options: 'gocart' or 'ufs' +settling_scheme: 'ufs' + # Dry particle radius [um], used for settling particle_radius_microns: 0.0 0.0 0.35 0.0 From 9179b1e0f7b86dd4a3c9c8e7aa668f9a9c9090d0 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Tue, 9 Sep 2025 09:36:19 -0400 Subject: [PATCH 043/132] Update NEXUSEmissions to use CDATE for start and end dates; add logging for computed values --- ush/python/pygfs/task/nexus_emission.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/nexus_emission.py b/ush/python/pygfs/task/nexus_emission.py index 22978da981a..831d3248376 100644 --- a/ush/python/pygfs/task/nexus_emission.py +++ b/ush/python/pygfs/task/nexus_emission.py @@ -11,6 +11,7 @@ from typing import Dict, Any, Union, List from dateutil.rrule import DAILY, HOURLY, rrule from jinja2 import Environment, FileSystemLoader +from pprint import pprint from wxflow import (AttrDict, FileHandler, parse_j2yaml, @@ -43,6 +44,7 @@ def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) self.task_config = AttrDict(config) + pprint(self.task_config) self.AERO_INPUTS_DIR = self.task_config.get('AERO_INPUTS_DIR', None) self.COMOUT_CHEM_INPUT = self.task_config.get('COMOUT_CHEM_INPUT', None) @@ -54,9 +56,13 @@ def __init__(self, config: Dict[str, Any]) -> None: nforecast_hours = self.task_config["FHMAX_GFS"] # Create start date based on SDATE - self.start_date = self.task_config["SDATE_GFS"] + self.start_date = self.task_config["CDATE"] self.total_hrs = nforecast_hours + 3 - self.end_date = self.task_config["SDATE_GFS"] + to_timedelta(f'{self.total_hrs}H') + self.end_date = self.task_config["CDATE"] + to_timedelta(f'{self.total_hrs}H') + + logger.info(f'SDATE_GFS: {self.start_date}') + logger.info(f'nforecast_hours: {nforecast_hours}') + logger.info(f'Computed end_date: {self.end_date} (total_hrs={self.total_hrs})') # Create the forecast dates based on start_date and end_date frequency = self.task_config.get("NEXUS_DIAG_FREQ", "Hourly") From 599f7b002ba9b9c786d206615e9d1040165dee80 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Thu, 18 Sep 2025 13:07:15 -0400 Subject: [PATCH 044/132] Update ush/python/pygfs/task/nexus_emission.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- ush/python/pygfs/task/nexus_emission.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/python/pygfs/task/nexus_emission.py b/ush/python/pygfs/task/nexus_emission.py index 831d3248376..b2bcc480717 100644 --- a/ush/python/pygfs/task/nexus_emission.py +++ b/ush/python/pygfs/task/nexus_emission.py @@ -11,7 +11,6 @@ from typing import Dict, Any, Union, List from dateutil.rrule import DAILY, HOURLY, rrule from jinja2 import Environment, FileSystemLoader -from pprint import pprint from wxflow import (AttrDict, FileHandler, parse_j2yaml, From 5d65aacf215922ec0fda15bb839fef01c926bdfe Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Thu, 18 Sep 2025 13:07:43 -0400 Subject: [PATCH 045/132] Update ush/python/pygfs/task/chem_fire_emission.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- ush/python/pygfs/task/chem_fire_emission.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index d6779f28344..ef3642b7f7e 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -467,9 +467,6 @@ def _find_qfed_files(self, dates, vars, version='061', aero_emis_fire_dir=None): files_found.append(full_path) logger.debug(f"Found exact QFED file: {full_path}") - if not os.path.exists(full_path): - logger.warning(f"File not found: {full_path}") - if not files_found: logger.warning(f"No QFED files found for dates {date_strings} and variables {vars}") From b890b78f168f240e336f2200293a48d9033bdaee Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Thu, 18 Sep 2025 15:16:43 -0400 Subject: [PATCH 046/132] Refactor emissions task instantiation and enhance build script usage information --- scripts/exglobal_prep_emissions.py | 10 ++--- sorc/build_nexus.sh | 8 ++++ ush/python/pygfs/task/chem_fire_emission.py | 8 +++- ush/python/pygfs/task/nexus_emission.py | 42 +-------------------- 4 files changed, 21 insertions(+), 47 deletions(-) diff --git a/scripts/exglobal_prep_emissions.py b/scripts/exglobal_prep_emissions.py index e517a52243a..46f4be24989 100755 --- a/scripts/exglobal_prep_emissions.py +++ b/scripts/exglobal_prep_emissions.py @@ -19,11 +19,11 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the emissions pre-processing task - emissions = ChemFireEmissions(config) - emissions.initialize() - emissions.configure() - emissions.execute() - emissions.finalize() + fireemis = ChemFireEmissions(config) + fireemis.initialize() + fireemis.configure() + fireemis.execute() + fireemis.finalize() nxsemis = NEXUSEmissions(config) nxsemis.initialize() diff --git a/sorc/build_nexus.sh b/sorc/build_nexus.sh index ea79fa14093..c202fc67565 100755 --- a/sorc/build_nexus.sh +++ b/sorc/build_nexus.sh @@ -1,6 +1,14 @@ #! /usr/bin/env bash set -eux +usage() { + echo "Usage: $0 [-d] [-j ] [-v]" + echo " -d Build in debug mode" + echo " -j Number of parallel build jobs" + echo " -v Verbose build output" + exit 1 +} + # shellcheck disable=SC2155 readonly HOMEgfs_=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )/.." && pwd -P) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index ef3642b7f7e..77ad731113a 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -51,11 +51,11 @@ def __init__(self, config: Dict[str, Any]) -> None: nforecast_hours = self.task_config["FHMAX_GFS"] logger.info(f"Number of forecast hours: {nforecast_hours}") - # Create start date based on SDATE - 12 hours + # Create start date based on SDATE self.start_date = self.task_config["SDATE_GFS"] logger.info(f"Start date: {self.start_date}") - # end date = SDATE + nforecast hours + 24 + # end date = SDATE + nforecast hours + 36 self.end_date = self.task_config["SDATE_GFS"] + to_timedelta(f'{nforecast_hours + 36}H') logger.info(f"End date: {self.end_date}") @@ -258,6 +258,10 @@ def execute(self) -> None: elif self.task_config.AERO_EMIS_FIRE.lower() == 'qfed': # Process QFED files for each forecast date processed_files.extend(self._process_qfed_files(workdir)) + + else: + logger.warning(f"Unknown AERO_EMIS_FIRE type: {self.task_config.AERO_EMIS_FIRE}") + raise WorkflowException(f"Unsupported AERO_EMIS_FIRE type: {self.task_config.AERO_EMIS_FIRE}") # Add processed files to task_config outdict = {'processed_files': processed_files} diff --git a/ush/python/pygfs/task/nexus_emission.py b/ush/python/pygfs/task/nexus_emission.py index b2bcc480717..73f855ac55c 100644 --- a/ush/python/pygfs/task/nexus_emission.py +++ b/ush/python/pygfs/task/nexus_emission.py @@ -151,7 +151,8 @@ def initialize(self) -> None: 'NEXUS_NZ', 'NEXUS_XMIN', 'NEXUS_XMAX', - 'NEXUS_YMIN' + 'NEXUS_YMIN', + 'NEXUS_YMAX' ] for param in required_grid_params: if not self.task_config.get(param, None): @@ -339,7 +340,6 @@ def execute(self) -> None: logger.info(f"Processing NEXUS files for date: {date}") dsets = [] - print(indexes, len(files)) for index in indexes: # list files for log logger.info(f" - {files[index]}, {index}") @@ -369,44 +369,6 @@ def execute(self) -> None: ds.to_netcdf(outname, format="NETCDF4", encoding=encoding) logger.info(f"Wrote daily output: {outname}") - # dsets = [] - # for index, fname in enumerate(files): - # dset = xr.open_dataset(fname,decode_cf=False) - # dsets.append(dset.assign_coords(time=[index])) - - # # Concatenate along time dimension - # ds = xr.concat(dsets, dim="time") - # ds.time.attrs['units'] = self.start_date.strftime('hours since %y-%m-%d %H:00:00') - - # # Convert raw time values to datetime objects using cftime - # if 'time' not in ds.dims: - # raise WorkflowException("No 'time' dimension found in NEXUS output dataset.") - - # time_var = ds['time'] - # time_units = time_var.attrs.get('units', None) - # time_calendar = time_var.attrs.get('calendar', 'standard') - # if time_units is None: - # raise WorkflowException("No 'units' attribute found for time variable.") - - # # Convert time values to datetime objects - # time_vals = np.arange(len(files)) - # ds = ds.assign_coords(time=time_vals) - # time_dt = cftime.num2date(time_vals, units=time_units, calendar=time_calendar) - - # # Group indices by day - - # day_to_indices = defaultdict(list) - # for idx, dt in enumerate(time_dt): - # day_to_indices[dt.strftime('%y%m%d')].append(idx) - - # encoding = {var: {"zlib": True, "complevel": 4} for var in ds.data_vars} - # for day_str, indices in day_to_indices.items(): - # daily_ds = ds.isel(time=indices) - # daily_ds.time.attrs['units'] = units_string - # outname = f"{self.task_config.NEXUS_DIAG_PREFIX}.{day_str}.nc" - # daily_ds.to_netcdf(outname, format="NETCDF4", encoding=encoding) - # logger.info(f"Wrote daily output: {outname}") - logger.info("NEXUS emission processing execute phase complete") @logit(logger) From 31aebbde166023eac04baa13677c7a37c0905eb3 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Thu, 18 Sep 2025 15:30:05 -0400 Subject: [PATCH 047/132] adding additional comments in the file to give users better descriptions of variables, where they are used and how to use them --- dev/parm/config/gcafs/config.aero.j2 | 101 +++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/dev/parm/config/gcafs/config.aero.j2 b/dev/parm/config/gcafs/config.aero.j2 index b3df15618d8..51bfd6264e3 100644 --- a/dev/parm/config/gcafs/config.aero.j2 +++ b/dev/parm/config/gcafs/config.aero.j2 @@ -1,97 +1,198 @@ #! /usr/bin/env bash # UFS-Aerosols settings +# This configuration file sets up environment variables for aerosol modeling in the UFS (Unified Forecast System) Aerosols component. +# It configures aerosol inputs, diagnostics, emissions, and the NEXUS emissions preprocessor for GCAFS (Global Coupled Aerosol Forecast System). +# Used in GFS (Global Forecast System) workflows for initializing and running aerosol simulations in FV3 (Finite-Volume Cubed-Sphere) dynamical core. #================================================================================ # 1. Aerosol settings #================================================================================ +# General settings for aerosol tracers, diagnostics, and scavenging in the GOCART (Goddard Chemistry Aerosol Radiation and Transport) model. +# These are used in the atmospheric model to handle aerosol transport, chemistry, and interaction with radiation/cloud processes. export AERO_INPUTS_DIR="{{ AERO_INPUTS_DIR }}" +# Base directory for aerosol input data files (e.g., initial conditions, climatologies). +# This path is mounted or staged in the workflow and referenced by the model for reading aerosol fields. #----------------------------------------------- # Diag Table and Field Table for GOCART aerosols #----------------------------------------------- +# Configuration files defining diagnostic outputs and field registrations for aerosol variables in GOCART. +# diag_table.aero: Specifies which aerosol fields to output and at what frequency (used by FMS diagnostics). +# field_table.aero: Registers prognostic/diagnostic tracers with the FV3 dynamical core (e.g., for advection, diffusion). export AERO_DIAG_TABLE="${PARMgfs}/ufs/fv3/diag_table.aero" export AERO_FIELD_TABLE="${PARMgfs}/ufs/fv3/field_table.aero" # Aerosol configuration export AERO_CONFIG_DIR="${PARMgfs}/ufs/gocart" +# Directory containing GOCART-specific namelists, parameters, and runtime configs (e.g., namelist.aero). +# Loaded during model initialization to set aerosol scheme parameters like time steps, vertical levels. # Aerosol convective scavenging factors (list of string array elements) # Element syntax: ':'. Use = * to set default factor for all aerosol tracers +# Scavenging factors represent the fraction of aerosol removed by convective precipitation (wet deposition). +# Used in the convection scheme (e.g., SAS or NF_CONV) to compute in-cloud scavenging rates for each tracer. +# * = default for unspecified tracers; specific gases like SO2 have lower factors due to solubility. # Scavenging factors are set to 0 (no scavenging) if unset export fscav_aero="'*:0.3','so2:0.0','msa:0.0','dms:0.0','nh3:0.4','nh4:0.6','bc1:0.6','bc2:0.6','oc1:0.4','oc2:0.4','dust1:0.6','dust2:0.6', 'dust3:0.6','dust4:0.6','dust5:0.6','seas1:0.5','seas2:0.5','seas3:0.5','seas4:0.5','seas5:0.5'" # # Number of diagnostic aerosol tracers (default: 0) +# Specifies how many additional diagnostic (non-prognostic) aerosol tracers to include in the model output. +# Used in GOCART to control verbosity of diagnostics; higher values add more fields for analysis/post-processing. export dnats_aero=2 #================================================================================ # 2. Aerosol emissions settings #================================================================================ +# Configuration for surface emissions of aerosols and precursors (e.g., from fires, anthropogenic sources). +# These drive the source terms in the GOCART continuity equation for each tracer. # Biomass burning emission dataset. Choose from: gbbepx, qfed, none +# Dataset for wildfire and biomass burning emissions (e.g., black/organic carbon, CO). +# qfed: Quick Fire Emission Dataset (near-real-time, version specified below). +# gbbepx: Global Biomass Burning Emissions Product (alternative). +# none: Disable fire emissions. +# Used in prep_emissions scripts to fetch/interpolate data to model grid. export AERO_EMIS_FIRE="qfed" export AERO_EMIS_FIRE_VERSION="061" + +# Version of the selected fire emissions dataset (e.g., for QFEDv2.5, version 061). +# Determines which historical or NRT files to load from input directories. export AERO_EMIS_FIRE_HIST=1 # Use historical fire emissions | 1 = true 0 = false + +# Flag to enable historical (climatological) fire emissions instead of NRT for testing/spin-up. +# When true, uses fixed-year data; false uses real-time from FIRE_EMIS_NRT_DIR. +# Path to near-real-time (NRT) fire emissions data, updated daily (e.g., from satellites like MODIS). +# On WCOSS2, points to DCOM (Data Communication) root for operational runs; empty for testing. +# Processed by scripts like exglobal_prep_emissions.py to generate input files for GOCART. export FIRE_EMIS_NRT_DIR="" #TODO: set to DCOM for WCOSS2 "${DCOMROOT}/YYYYMMDD/firewx" # Directory containing NRT fire emissions + + #=============================================================================== # 3. NEXUS settings #=============================================================================== +# NEXUS (Next-generation Emissions eXchange Utility System) is a preprocessor for anthropogenic/biogenic emissions. +# Generates time-varying, gridded emission inputs for GOCART from inventories like CEDS, HTAP, CAMS. +# Runs offline before the forecast, outputting netCDF files read by the model via AERO_INPUTS_DIR. # NEXUS aerosol emissions dataset. Choose from: gocart, none +# Specifies the emission species set for NEXUS processing (gocart for GOCART-compatible tracers like SO2, BC, OC, dust). +# none: Skip NEXUS entirely, use other emission sources or zero emissions. # NEXUS configuration set +#------------------------- export NEXUS_CONFIG="{{ NEXUS_CONFIG | default('gocart') }}" # Options: gocart, none + +# Runtime choice of NEXUS config variant; defaults to gocart for standard aerosol tracers. +# Overrides via Jinja2 templating in workflow (e.g., for different chemistry schemes). export NEXUS_CONFIG_DIR="${PARMgfs}/chem/nexus/${NEXUS_CONFIG}" # Directory containing NEXUS configuration files # NEXUS Inputs +#--------------- # TODO: when this is merged this will point to AERO_INPUTS_DIR for operations # export NEXUS_INPUT_DIR="${AERO_INPUTS_DIR}/nexus" +# Directory for static/dynamic input data used by NEXUS (e.g., emission inventories, masks, meteo fields). +# Currently hardcoded for development; will use shared AERO_INPUTS_DIR in production for consistency. export NEXUS_INPUT_DIR="/gpfs/f6/bil-fire3/world-shared/Emissions/GEFS_ExtData/nexus" +# Specific path for GCAFS external data on this filesystem. +# Contains emission datasets (e.g., CEDS2019/2024, HTAPv2, CAMS) processed by NEXUS. #-------------------------- # NEXUS Time Step (seconds) #-------------------------- +# Temporal resolution for emission interpolation in NEXUS (e.g., hourly outputs). +# Must align with model coupling time; used in HEMCO time management for diurnal/seasonal scaling. +# 3600s = 1 hour; adjustable for finer/coarser emission updates (e.g., 1800s for sub-hourly). export NEXUS_TSTEP="{{ NEXUS_TSTEP | default(3600) }}" # Default NEXUS time step in seconds #------------------ # NEXUS Grid #------------------ +# Defines the emission grid for NEXUS processing (0.5x0.5 degree global lat-lon). +# Emissions are interpolated from this grid to the FV3 cubed-sphere grid during prep. +# Number of longitude points (1440 for 0.25-degree resolution; here 1440 ~0.25deg). export NEXUS_NX="{{ NEXUS_NX | default(1440) }}" +# Number of latitude points (720 for 0.25-degree). export NEXUS_NY="{{ NEXUS_NY | default(720) }}" +# Western boundary longitude (global coverage). export NEXUS_XMIN="{{ NEXUS_XMIN | default(-180.0) }}" +# Eastern boundary longitude. export NEXUS_XMAX="{{ NEXUS_XMAX | default(180.0) }}" +# Southern boundary latitude. export NEXUS_YMIN="{{ NEXUS_YMIN | default(-90.0) }}" +# Northern boundary latitude. export NEXUS_YMAX="{{ NEXUS_YMAX | default(90.0) }}" +# Number of vertical levels (1 for surface emissions; higher for vertical profiles if needed). export NEXUS_NZ="{{ NEXUS_NZ | default(1) }}" #------------------- # NEXUS Config Files #------------------- +# HEMCO (Harmonized Emissions Component) runtime configuration files used by NEXUS. +# These define species mappings, time scales, grid alignments, and diagnostic flags. + +# Grid definition file: Specifies emission grid (lat-lon bounds, resolution) and interpolation options to model grid. export NEXUS_GRID_NAME="{{ NEXUS_GRID_NAME | default('HEMCO_sa_Grid.rc') }}" +# Time management file: Defines temporal patterns (diurnal, weekly, monthly) for scaling emissions. export NEXUS_TIME_NAME="{{ NEXUS_TIME_NAME | default('HEMCO_sa_Time.rc') }}" +# Diagnostics file: Controls which emission fields to output for verification (e.g., total SO2, BC emissions). export NEXUS_DIAG_NAME="{{ NEXUS_DIAG_NAME | default('HEMCO_sa_Diag.rc') }}" +# Species mapping file: Links emission inventories to GOCART tracers (e.g., CEDS SO2 to model SO2). export NEXUS_SPEC_NAME="{{ NEXUS_SPEC_NAME | default('HEMCO_sa_Spec.rc') }}" +# Master config file: Orchestrates all HEMCO components, emission sources, and runtime flags for NEXUS. export NEXUS_CONFIG_NAME="{{ NEXUS_CONFIG_NAME | default('NEXUS_Config.rc') }}" #------------------ # NEXUS Diagnostics #------------------ +# Settings for outputting NEXUS-processed emissions for model input and verification. +# Outputs are netCDF files with gridded, time-varying sources read by GOCART at each time step. + +# Base filename prefix for diagnostic output files (e.g., NEXUS_DIAG_YYYYMMDD.nc). export NEXUS_DIAG_PREFIX="{{ NEXUS_DIAG_PREFIX | default('NEXUS_DIAG') }}" +# Frequency of diagnostic emission outputs; Hourly for detailed analysis, coarser for storage efficiency. export NEXUS_DIAG_FREQ="{{ NEXUS_DIAG_FREQ | default('Hourly') }}" # Options: Hourly, Daily, Monthly #------------------ # NEXUS Logging #------------------ +# Controls NEXUS execution logs for debugging and monitoring in the workflow. + +# Output log file for NEXUS run; captures errors, warnings, and processing summaries. +# Reviewed in post-processing or if emissions fail to generate. export NEXUS_LOGFILE="{{ NEXUS_LOGFILE | default('NEXUS.log') }}" #------------------ # NEXUS Emissions #------------------ +# Flags to enable/disable specific emission inventories processed by NEXUS. +# Multiple can be true for blended emissions; used in NEXUS_Config.rc to select sources. +# Emissions are scaled by region, sector (e.g., industry, transport), and time. + +# Flag for MEGAN (Model of Emissions of Gases and Aerosols from Nature) biogenic VOC/PM emissions. +# Currently disabled; future integration for isoprene, terpenes affecting secondary organic aerosols. export NEXUS_DO_MEGAN=.false # TODO: Add MEGAN biogenic emissions in the furture + +# Enable Community Emissions Data System 2019 inventory for anthropogenic aerosols/gases (e.g., SO2, NOx, PM2.5). +# Global, gridded data for 1750-2019; used for historical and recent baseline emissions. export NEXUS_DO_CEDS2019=.true. # Use CEDS2019 emissions + +# Enable newer CEDS 2024 update (if available); mutually exclusive with 2019 for consistency. export NEXUS_DO_CEDS2024=.false. # Use CEDS2024 emissions + +# Hemispheric Transport of Air Pollution version 2: Regional anthropogenic emissions for Europe/Asia/N. America. +# Focuses on transboundary pollution; supplements CEDS for finer regional detail. export NEXUS_DO_HTAPv2=.true. # Use HTAPv2 emissions + +# HTAP version 3 flag; disabled pending updates to datasets and NEXUS compatibility. export NEXUS_DO_HTAPv3=.false. # TODO: Currently only uses HTAPv2 for this. + +# Copernicus Atmosphere Monitoring Service global reanalysis emissions. +# Alternative to CEDS/HTAP for consistent meteo-coupled emissions; disabled here. export NEXUS_DO_CAMS=.false. # Use CAMS global emissions + +# CAMS temporal disaggregation (e.g., hourly profiles for CAMS data). +# Enables time-varying scaling when CAMS is active. export NEXUS_DO_CAMSTEMPO=.false. # Use CAMS temporal emissions +#================================================================================ echo "END: config.aero" From b258a5f7163cfab988f931ba0dee2f141f277969 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Thu, 18 Sep 2025 15:32:58 -0400 Subject: [PATCH 048/132] pycodestyle fix W293 --- ush/python/pygfs/task/chem_fire_emission.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 77ad731113a..5e19c609b29 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -258,7 +258,6 @@ def execute(self) -> None: elif self.task_config.AERO_EMIS_FIRE.lower() == 'qfed': # Process QFED files for each forecast date processed_files.extend(self._process_qfed_files(workdir)) - else: logger.warning(f"Unknown AERO_EMIS_FIRE type: {self.task_config.AERO_EMIS_FIRE}") raise WorkflowException(f"Unsupported AERO_EMIS_FIRE type: {self.task_config.AERO_EMIS_FIRE}") From e5701612924740f635266b5868e0660974769b7f Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Tue, 23 Sep 2025 11:40:17 -0400 Subject: [PATCH 049/132] Remove debug print statements from NEXUSEmissions class initialization and execution methods --- ush/python/pygfs/task/nexus_emission.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ush/python/pygfs/task/nexus_emission.py b/ush/python/pygfs/task/nexus_emission.py index 73f855ac55c..a9da11213a1 100644 --- a/ush/python/pygfs/task/nexus_emission.py +++ b/ush/python/pygfs/task/nexus_emission.py @@ -43,7 +43,6 @@ def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) self.task_config = AttrDict(config) - pprint(self.task_config) self.AERO_INPUTS_DIR = self.task_config.get('AERO_INPUTS_DIR', None) self.COMOUT_CHEM_INPUT = self.task_config.get('COMOUT_CHEM_INPUT', None) @@ -301,7 +300,6 @@ def execute(self) -> None: if not os.path.exists(self.task_config.DATA): raise WorkflowException(f"Working directory does not exist: {self.task_config.DATA}") - # pprint(self.task_config) exe = Executable(self.task_config.launcher) arg_list = ['--ntasks', str(1), From dc18f43161659ae59fe29621d6da36b7ace3b533 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Tue, 23 Sep 2025 13:06:41 -0400 Subject: [PATCH 050/132] put a check on which launcher is used and adjust arglist --- ush/python/pygfs/task/nexus_emission.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/ush/python/pygfs/task/nexus_emission.py b/ush/python/pygfs/task/nexus_emission.py index a9da11213a1..4307e9a680b 100644 --- a/ush/python/pygfs/task/nexus_emission.py +++ b/ush/python/pygfs/task/nexus_emission.py @@ -301,11 +301,14 @@ def execute(self) -> None: raise WorkflowException(f"Working directory does not exist: {self.task_config.DATA}") exe = Executable(self.task_config.launcher) - arg_list = ['--ntasks', - str(1), - 'nexus.x', - '-c', - self.task_config.NEXUS_CONFIG_NAME] + if 'mpiexec' in self.task_config.launcher: + arg_list = ['-n', str(1), 'nexus.x', '-c', self.task_config.NEXUS_CONFIG_NAME] + else: + arg_list = ['--ntasks', + str(1), + 'nexus.x', + '-c', + self.task_config.NEXUS_CONFIG_NAME] exe(*arg_list, output='stdout', error='stderr') logger.info("Concatenating processed NEXUS files...") From af03b78882654f479717d95a5854c6a24aa0b1e1 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Tue, 23 Sep 2025 13:22:20 -0400 Subject: [PATCH 051/132] update chem_fire_emission to use CDATE for start_date --- ush/python/pygfs/task/chem_fire_emission.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 5e19c609b29..add2825b8b9 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -52,11 +52,11 @@ def __init__(self, config: Dict[str, Any]) -> None: logger.info(f"Number of forecast hours: {nforecast_hours}") # Create start date based on SDATE - self.start_date = self.task_config["SDATE_GFS"] + self.start_date = self.task_config["CDATE"] logger.info(f"Start date: {self.start_date}") # end date = SDATE + nforecast hours + 36 - self.end_date = self.task_config["SDATE_GFS"] + to_timedelta(f'{nforecast_hours + 36}H') + self.end_date = self.task_config["CDATE"] + to_timedelta(f'{nforecast_hours + 36}H') logger.info(f"End date: {self.end_date}") # Calculate number of days spanned by start and end date (inclusive) From 31670ea35e7dac2c022f45bf9fa5ec2693020ded Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Tue, 23 Sep 2025 14:42:15 -0400 Subject: [PATCH 052/132] remove the lookup for gbbepx_vars --- ush/python/pygfs/task/chem_fire_emission.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index add2825b8b9..22db156ff2f 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -132,7 +132,7 @@ def initialize(self) -> None: files_found = [] for dates in self.forecast_dates: if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': - gbbepx_vars = self.task_config.get('gbbepx_vars', ["co", "nox", "so2", "nh3", "bc", "oc"]) + gbbepx_vars = ["co", "nox", "so2", "nh3", "bc", "oc"] files = self._find_gbbepx_files(dates, gbbepx_vars, version=self.task_config.AERO_EMIS_FIRE_VERSION, @@ -154,7 +154,7 @@ def initialize(self) -> None: self.start_date, self.task_config.gbbepx_vars, version=self.task_config.AERO_EMIS_FIRE_VERSION, - vars=self.task_config.get('gbbepx_vars', ["co", "nox", "so2", "nh3", "bc", "oc"]) + vars=["co", "nox", "so2", "nh3", "bc", "oc"] ) elif self.task_config.AERO_EMIS_FIRE.lower() == 'qfed': # Get QFED variables with safe defaults @@ -194,7 +194,7 @@ def initialize(self) -> None: 'historical': self.historical, 'forecast_dates': self.task_config.get('forecast_dates', []), 'qfed_vars': self.task_config.get('qfed_vars', ["co", "nox", "so2", "nh3", "bc", "oc"]), - 'gbbepx_vars': self.task_config.get('gbbepx_vars', ["co", "nox", "so2", "nh3", "bc", "oc"]), + 'gbbepx_vars': ["co", "nox", "so2", "nh3", "bc", "oc"], "rawfiles": files_found, "startdate": self.start_date.strftime('%Y%m%d'), "processed_files": processed_files, From 09e0854c24b54979ebdd1082b4b8c0816e7fd7b9 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Tue, 23 Sep 2025 14:55:57 -0400 Subject: [PATCH 053/132] fix error on line 155 --- ush/python/pygfs/task/chem_fire_emission.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 22db156ff2f..9a8362b6d76 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -152,7 +152,7 @@ def initialize(self) -> None: if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': files = self._find_gbbepx_files( self.start_date, - self.task_config.gbbepx_vars, + gbbepx_vars, version=self.task_config.AERO_EMIS_FIRE_VERSION, vars=["co", "nox", "so2", "nh3", "bc", "oc"] ) From be3d6734671b2f25749c33426f69437280050d33 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Tue, 23 Sep 2025 14:57:01 -0400 Subject: [PATCH 054/132] Add gbbepx_vars definition for emissions processing --- ush/python/pygfs/task/chem_fire_emission.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 9a8362b6d76..fca0722878c 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -150,6 +150,7 @@ def initialize(self) -> None: logger.info(f'Processing forecast emissions for {self.start_date}') if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': + gbbepx_vars = ["co", "nox", "so2", "nh3", "bc", "oc"] files = self._find_gbbepx_files( self.start_date, gbbepx_vars, From 8fe0af044fbfa251ef23762d233fd0d8c85303ea Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Tue, 23 Sep 2025 15:15:45 -0400 Subject: [PATCH 055/132] Remove redundant gbbepx_vars definition in ChemFireEmissions class --- ush/python/pygfs/task/chem_fire_emission.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index fca0722878c..0ebc1143a86 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -132,9 +132,7 @@ def initialize(self) -> None: files_found = [] for dates in self.forecast_dates: if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': - gbbepx_vars = ["co", "nox", "so2", "nh3", "bc", "oc"] files = self._find_gbbepx_files(dates, - gbbepx_vars, version=self.task_config.AERO_EMIS_FIRE_VERSION, aero_emis_fire_dir=AERO_EMIS_FIRE_DIR) elif self.task_config.AERO_EMIS_FIRE.lower() == 'qfed': @@ -150,12 +148,9 @@ def initialize(self) -> None: logger.info(f'Processing forecast emissions for {self.start_date}') if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': - gbbepx_vars = ["co", "nox", "so2", "nh3", "bc", "oc"] files = self._find_gbbepx_files( self.start_date, - gbbepx_vars, - version=self.task_config.AERO_EMIS_FIRE_VERSION, - vars=["co", "nox", "so2", "nh3", "bc", "oc"] + version=self.task_config.AERO_EMIS_FIRE_VERSION ) elif self.task_config.AERO_EMIS_FIRE.lower() == 'qfed': # Get QFED variables with safe defaults From 0d9bdd1945f3a4fa9d285d4effb5563bfd5e795f Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Tue, 23 Sep 2025 15:35:23 -0400 Subject: [PATCH 056/132] Update AERO_EMIS_FIRE_DIR for NRT forecast emissions processing --- ush/python/pygfs/task/chem_fire_emission.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 0ebc1143a86..4cf18e07233 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -144,10 +144,11 @@ def initialize(self) -> None: aero_emis_fire_dir=AERO_EMIS_FIRE_DIR) files_found.extend(files) logger.info(f'Found {len(files_found)} files for historical period') - else: + else: # NRT Forecast emisssions logger.info(f'Processing forecast emissions for {self.start_date}') if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': + self.task_config['AERO_EMIS_FIRE_DIR'] = self.task_config.get('AERO_EMIS_FIRE_NRT_DIR', None) files = self._find_gbbepx_files( self.start_date, version=self.task_config.AERO_EMIS_FIRE_VERSION From e9165e3ed7017c6993a27d66e01e289c8f1959d3 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Tue, 23 Sep 2025 15:54:04 -0400 Subject: [PATCH 057/132] update gbbepx file pattern as creation date could be different than start date --- ush/python/pygfs/task/chem_fire_emission.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 4cf18e07233..0b90169dda0 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -356,7 +356,7 @@ def _find_gbbepx_files(self, dates, version='v5r0'): match_found = False # Try pattern 1 with s/e/c date format - pattern1 = r".*s(\d{8}).*e(\d{8}).*c(\d{8}).*\.nc" + pattern1 = r"GBBEPx-all01GRID.*_s(\d{8}).*_e(\d{8}).*\.nc" match = re.match(pattern1, file_name) if match: start_date = match.group(1) From 06a29ebafcc7c18d8bcc2bd16228e24897837649 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Tue, 23 Sep 2025 16:10:39 -0400 Subject: [PATCH 058/132] Implement GBBEPx NRT fire file search functionality for wcoss dcom and update method references --- ush/python/pygfs/task/chem_fire_emission.py | 52 ++++++++++++++++++++- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 0b90169dda0..07d5ccec966 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -149,7 +149,7 @@ def initialize(self) -> None: if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': self.task_config['AERO_EMIS_FIRE_DIR'] = self.task_config.get('AERO_EMIS_FIRE_NRT_DIR', None) - files = self._find_gbbepx_files( + files = self._find_gbbepx_nrt_fires( self.start_date, version=self.task_config.AERO_EMIS_FIRE_VERSION ) @@ -309,6 +309,54 @@ def _get_unique_months(self): years = set(date.year for date in self.forecast_dates) return months, years + + @logit(logger) + def _find_gbbepx_nrt_fires(self, emis_file_dir): + """Find GBBEPx NRT fire files in the specified directory. + + Parameters + ---------- + emis_file_dir : str + Directory to search for GBBEPx NRT fire files + + Returns + ------- + List[str] + List of found GBBEPx NRT fire files + + Notes + ----- + Searches for files matching the pattern "GBBEPx-all01GRID_v4r0_blend_sYYYYMMDD000000_eYYYYMMDD235959_cYYYYMMDDHHMMSS.nc" + where YYYYMMDD represents the date components. + """ + logger.info(f'Finding GBBEPx NRT fire files in {emis_file_dir}') + + if not os.path.exists(emis_file_dir): + logger.warning(f"Directory does not exist: {emis_file_dir}") + return [] + + all_files = os.listdir(emis_file_dir) + matching_files = [] + + # Look for pattern: "GBBEPx-all01GRID_v4r0_blend_s202302240000000_e202302242359590_c202302250134090.nc" + pattern = r"GBBEPx-all01GRID.*_s(\d{8}).*_e(\d{8}).*\.nc" + + if all_files is None: + logger.warning(f"No files found in directory: {emis_file_dir}") + logger.warning(f'Checking the previous date') + emis_file_dir = emis_file_dir.replace(self.start_date.strftime('%Y/%m'), (self.start_date - datetime.timedelta(days=1)).strftime('%Y/%m')) + if not os.path.exists(emis_file_dir): + logger.warning(f"Directory does not exist: {emis_file_dir}") + return [] + for file_name in all_files: + match = re.match(pattern, file_name) + if match: + full_path = os.path.join(emis_file_dir, file_name) + matching_files.append(full_path) + logger.debug(f"Found GBBEPx NRT fire file: {full_path}") + + return matching_files + @logit(logger) def _find_gbbepx_files(self, dates, version='v5r0'): """Find GBBEPx files for the given date @@ -339,7 +387,7 @@ def _find_gbbepx_files(self, dates, version='v5r0'): # Find all possible files for mon in months: try: - emis_file_dir = os.path.join(self.task_config.AERO_EMIS_FIRE_DIR, version, mon) + emis_file_dir = self.task_config.AERO_EMIS_FIRE_DIR if not os.path.exists(emis_file_dir): logger.warning(f"Directory does not exist: {emis_file_dir}") continue From 2959b0394cf9d93ad681751e8de4c509b16c91e7 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 09:34:58 -0400 Subject: [PATCH 059/132] Add NRT_DIRECTORY configuration and render_template method for GBBEPx NRT emissions processing --- parm/chem/fire_emission.yaml.j2 | 2 + ush/python/pygfs/task/chem_fire_emission.py | 44 +++++++++++++++++++-- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/parm/chem/fire_emission.yaml.j2 b/parm/chem/fire_emission.yaml.j2 index 0902e8cf48d..201cb7e4974 100644 --- a/parm/chem/fire_emission.yaml.j2 +++ b/parm/chem/fire_emission.yaml.j2 @@ -1,3 +1,4 @@ +{% set cycle_YMD = current_cycle | to_YMD %} fire_emission: config: apply_quality_control: True @@ -10,6 +11,7 @@ fire_emission: {% for gvar in fire_vars %} - "{{ gvar }}" {% endfor %} + NRT_DIRECTORY: "{{ AERO_EMIS_FIRE_NRT_DIR }}/{{cycle_YMD}}/firewx" data_in: mkdir: - "{{ DATA }}" diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 07d5ccec966..72ad37ffcc4 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -144,13 +144,22 @@ def initialize(self) -> None: aero_emis_fire_dir=AERO_EMIS_FIRE_DIR) files_found.extend(files) logger.info(f'Found {len(files_found)} files for historical period') - else: # NRT Forecast emisssions + else: + #=============================================== + # # NRT Forecast emisssions + #=============================================== logger.info(f'Processing forecast emissions for {self.start_date}') + # GBBEPx NRT files are in a different directory structure + # Render the template with the current cycle to get the correct path + tmp_dict = {'current_cycle': self.start_date, + 'AERO_EMIS_FIRE_NRT_DIR': self.task_config.get('AERO_EMIS_FIRE_NRT_DIR', None) + } + self.render_template(tmp_dict) + if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': - self.task_config['AERO_EMIS_FIRE_DIR'] = self.task_config.get('AERO_EMIS_FIRE_NRT_DIR', None) + self.task_config['AERO_EMIS_FIRE_DIR'] = self.task_config.get('NRT_DIRECTORY', None) files = self._find_gbbepx_nrt_fires( - self.start_date, version=self.task_config.AERO_EMIS_FIRE_VERSION ) elif self.task_config.AERO_EMIS_FIRE.lower() == 'qfed': @@ -832,3 +841,32 @@ def _process_qfed_files(self, workdir: str) -> List[str]: logger.warning(f"No QFED files found for date {date_str}") return processed_files + + + @logit(logger) + def render_template(self, tmp_dict: Dict()) -> None: + """Render the YAML template and set up task configuration. + + This method performs the following steps: + 1. Loads and parses the YAML template file using Jinja2 + 2. Fills in configuration parameters using environment variables and task attributes + 3. Updates the task configuration with the rendered YAML content + + Parameters + ---------- + tmp_dict : Dict + Dictionary containing template variables and their values + + """ + logger.info("Rendering YAML template") + # Parse template and update task configuration + yaml_template = os.path.join(self.task_config.HOMEgfs, 'parm', 'chem', 'fire_emission.yaml.j2') + if not os.path.exists(yaml_template): + logger.warning(f"Template file not found: {yaml_template}, using default configuration") + yaml_config = {'fire_emission': {}} + else: + logger.debug(f'Parsing YAML template: {yaml_template}') + yaml_config = parse_j2yaml(yaml_template, tmpl_dict) + + self.task_config = AttrDict(**self.task_config, **yaml_config) + From 116591469fb77706153ff16aa69f61a79aa23094 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 09:37:13 -0400 Subject: [PATCH 060/132] Fix type hint for tmp_dict parameter in render_template method --- ush/python/pygfs/task/chem_fire_emission.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 72ad37ffcc4..66ed6620615 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -844,7 +844,7 @@ def _process_qfed_files(self, workdir: str) -> List[str]: @logit(logger) - def render_template(self, tmp_dict: Dict()) -> None: + def render_template(self, tmp_dict: Dict[str, Any]) -> None: """Render the YAML template and set up task configuration. This method performs the following steps: From 031f03caf36f3db8170a9a1a4b4af8011cdc56b3 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 09:38:14 -0400 Subject: [PATCH 061/132] Fix parameter name in render_template method for clarity --- ush/python/pygfs/task/chem_fire_emission.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 66ed6620615..07472a76cd1 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -844,7 +844,7 @@ def _process_qfed_files(self, workdir: str) -> List[str]: @logit(logger) - def render_template(self, tmp_dict: Dict[str, Any]) -> None: + def render_template(self, tmpl_dict: Dict[str, Any]) -> None: """Render the YAML template and set up task configuration. This method performs the following steps: From 3d841b92d5ff2f5608d36bc3096b76438230826f Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 09:39:52 -0400 Subject: [PATCH 062/132] Refactor _find_gbbepx_nrt_fires call to use single argument for version --- ush/python/pygfs/task/chem_fire_emission.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 07472a76cd1..4975613e12e 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -159,9 +159,7 @@ def initialize(self) -> None: if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': self.task_config['AERO_EMIS_FIRE_DIR'] = self.task_config.get('NRT_DIRECTORY', None) - files = self._find_gbbepx_nrt_fires( - version=self.task_config.AERO_EMIS_FIRE_VERSION - ) + files = self._find_gbbepx_nrt_fires(self.task_config.AERO_EMIS_FIRE_VERSION) elif self.task_config.AERO_EMIS_FIRE.lower() == 'qfed': # Get QFED variables with safe defaults qfed_vars = self.task_config.get('qfed_vars', ["co", "nox", "so2", "nh3", "bc", "oc"]) From 0293bb58a427dc1c31f6f7a67b4696686b156558 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 09:42:39 -0400 Subject: [PATCH 063/132] Fix reference to AERO_EMIS_FIRE_DIR in render_template method to use task_config --- ush/python/pygfs/task/chem_fire_emission.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 4975613e12e..8ed44cfc896 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -193,7 +193,7 @@ def initialize(self) -> None: tmpl_dict = { 'DATA': self.task_config.DATA, 'COMOUT_CHEM_INPUT': self.task_config.COMOUT_CHEM_INPUT, - 'AERO_EMIS_FIRE_DIR': AERO_EMIS_FIRE_DIR, + 'AERO_EMIS_FIRE_DIR': self.task_config.AERO_EMIS_FIRE_DIR, 'AERO_EMIS_FIRE_VERSION': self.task_config.AERO_EMIS_FIRE_VERSION, 'historical': self.historical, 'forecast_dates': self.task_config.get('forecast_dates', []), From 3f6a0fc845b8b45e6c4e0f2496bc30307c0b2e2d Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 09:43:58 -0400 Subject: [PATCH 064/132] Update AERO_EMIS_FIRE_DIR in task_config for historical emissions logging --- ush/python/pygfs/task/chem_fire_emission.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 8ed44cfc896..ed477c18c10 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -144,6 +144,7 @@ def initialize(self) -> None: aero_emis_fire_dir=AERO_EMIS_FIRE_DIR) files_found.extend(files) logger.info(f'Found {len(files_found)} files for historical period') + self.task_config["AERO_EMIS_FIRE_DIR"] = AERO_EMIS_FIRE_DIR else: #=============================================== # # NRT Forecast emisssions From 7b115cdcd6bb8d81ea0f224c3b2b39d7bb183268 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 09:46:02 -0400 Subject: [PATCH 065/132] Rename variables for clarity in ChemFireEmissions class --- ush/python/pygfs/task/chem_fire_emission.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index ed477c18c10..e255558cd43 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -160,7 +160,7 @@ def initialize(self) -> None: if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': self.task_config['AERO_EMIS_FIRE_DIR'] = self.task_config.get('NRT_DIRECTORY', None) - files = self._find_gbbepx_nrt_fires(self.task_config.AERO_EMIS_FIRE_VERSION) + files_found = self._find_gbbepx_nrt_fires(self.task_config.AERO_EMIS_FIRE_VERSION) elif self.task_config.AERO_EMIS_FIRE.lower() == 'qfed': # Get QFED variables with safe defaults qfed_vars = self.task_config.get('qfed_vars', ["co", "nox", "so2", "nh3", "bc", "oc"]) @@ -174,7 +174,7 @@ def initialize(self) -> None: # Get fire emissions directory aero_emis_fire_dir = getattr(self.task_config, 'AERO_EMIS_FIRE_DIR', None) - files = self._find_qfed_files( + files_found = self._find_qfed_files( self.start_date, vars=qfed_vars, version=version, From ec5aeab18ce3222038aa78877cf0c4f903252101 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 09:48:36 -0400 Subject: [PATCH 066/132] Refactor render_template method to return YAML configuration for improved data handling --- ush/python/pygfs/task/chem_fire_emission.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index e255558cd43..a7eedaf40a5 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -156,10 +156,10 @@ def initialize(self) -> None: tmp_dict = {'current_cycle': self.start_date, 'AERO_EMIS_FIRE_NRT_DIR': self.task_config.get('AERO_EMIS_FIRE_NRT_DIR', None) } - self.render_template(tmp_dict) + yaml_config = self.render_template(tmp_dict) if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': - self.task_config['AERO_EMIS_FIRE_DIR'] = self.task_config.get('NRT_DIRECTORY', None) + self.task_config['AERO_EMIS_FIRE_DIR'] = yaml_config.fire_emission.config['NRT_DIRECTORY'] files_found = self._find_gbbepx_nrt_fires(self.task_config.AERO_EMIS_FIRE_VERSION) elif self.task_config.AERO_EMIS_FIRE.lower() == 'qfed': # Get QFED variables with safe defaults @@ -867,5 +867,5 @@ def render_template(self, tmpl_dict: Dict[str, Any]) -> None: logger.debug(f'Parsing YAML template: {yaml_template}') yaml_config = parse_j2yaml(yaml_template, tmpl_dict) - self.task_config = AttrDict(**self.task_config, **yaml_config) + return yaml_config From 3beb1d0972f8d81677da34b1c0f4eebb54413d9a Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 09:52:00 -0400 Subject: [PATCH 067/132] Add debug logging for file search in _find_gbbepx_files method --- ush/python/pygfs/task/chem_fire_emission.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index a7eedaf40a5..eda46984449 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -345,7 +345,9 @@ def _find_gbbepx_nrt_fires(self, emis_file_dir): all_files = os.listdir(emis_file_dir) matching_files = [] - + logger.debug(f"Searching in directory: {emis_file_dir}") + logger.debug(f"Total files in directory: {len(all_files)} files") + logger.debug(f"Files found in directory: {all_files}") # Look for pattern: "GBBEPx-all01GRID_v4r0_blend_s202302240000000_e202302242359590_c202302250134090.nc" pattern = r"GBBEPx-all01GRID.*_s(\d{8}).*_e(\d{8}).*\.nc" From fd4d432a3bb2c85d628e639a5ffbd42a67761e8a Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 09:54:50 -0400 Subject: [PATCH 068/132] Refactor render_template call to directly access AERO_EMIS_FIRE_NRT_DIR from task_config --- ush/python/pygfs/task/chem_fire_emission.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index eda46984449..627c461984e 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -154,7 +154,7 @@ def initialize(self) -> None: # GBBEPx NRT files are in a different directory structure # Render the template with the current cycle to get the correct path tmp_dict = {'current_cycle': self.start_date, - 'AERO_EMIS_FIRE_NRT_DIR': self.task_config.get('AERO_EMIS_FIRE_NRT_DIR', None) + 'AERO_EMIS_FIRE_NRT_DIR': self.task_config.AERO_EMIS_FIRE_NRT_DIR } yaml_config = self.render_template(tmp_dict) From f5e62d1026b81e7c853afc4c6f2ab35858e2e80a Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 09:57:22 -0400 Subject: [PATCH 069/132] Update NRT_DIRECTORY reference to use FIRE_EMIS_NRT_DIR in YAML template and ChemFireEmissions class --- parm/chem/fire_emission.yaml.j2 | 2 +- ush/python/pygfs/task/chem_fire_emission.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/chem/fire_emission.yaml.j2 b/parm/chem/fire_emission.yaml.j2 index 201cb7e4974..29f6008e164 100644 --- a/parm/chem/fire_emission.yaml.j2 +++ b/parm/chem/fire_emission.yaml.j2 @@ -11,7 +11,7 @@ fire_emission: {% for gvar in fire_vars %} - "{{ gvar }}" {% endfor %} - NRT_DIRECTORY: "{{ AERO_EMIS_FIRE_NRT_DIR }}/{{cycle_YMD}}/firewx" + NRT_DIRECTORY: "{{ FIRE_EMIS_NRT_DIR }}/{{cycle_YMD}}/firewx" data_in: mkdir: - "{{ DATA }}" diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 627c461984e..e4c31c2ad05 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -154,7 +154,7 @@ def initialize(self) -> None: # GBBEPx NRT files are in a different directory structure # Render the template with the current cycle to get the correct path tmp_dict = {'current_cycle': self.start_date, - 'AERO_EMIS_FIRE_NRT_DIR': self.task_config.AERO_EMIS_FIRE_NRT_DIR + 'FIRE_EMIS_NRT_DIR': self.task_config.FIRE_EMIS_NRT_DIR } yaml_config = self.render_template(tmp_dict) From e57f5c332730e5cf30a5d0283e03a484e4c8bdb3 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 09:59:53 -0400 Subject: [PATCH 070/132] Add logging for GBBEPx NRT file discovery in ChemFireEmissions class --- ush/python/pygfs/task/chem_fire_emission.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index e4c31c2ad05..ed0ae76da75 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -161,6 +161,8 @@ def initialize(self) -> None: if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': self.task_config['AERO_EMIS_FIRE_DIR'] = yaml_config.fire_emission.config['NRT_DIRECTORY'] files_found = self._find_gbbepx_nrt_fires(self.task_config.AERO_EMIS_FIRE_VERSION) + logger.info(f'Found {len(files_found)} GBBEPx NRT files for {self.start_date}' ) + logger.info(f"files found: {files_found}") elif self.task_config.AERO_EMIS_FIRE.lower() == 'qfed': # Get QFED variables with safe defaults qfed_vars = self.task_config.get('qfed_vars', ["co", "nox", "so2", "nh3", "bc", "oc"]) From 3d3a3b3323d80981a579f83c244b8cff21c7938c Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 10:02:52 -0400 Subject: [PATCH 071/132] Add logging for historical emissions flag in ChemFireEmissions constructor --- ush/python/pygfs/task/chem_fire_emission.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index ed0ae76da75..7d63c34df22 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -41,6 +41,7 @@ def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) self.historical = bool(self.task_config.get('AERO_EMIS_FIRE_HIST', 1)) + logger.info(f"Historical emissions flag: {self.historical}") self.AERO_INPUTS_DIR = self.task_config.get('AERO_INPUTS_DIR', None) self.COMOUT_CHEM_INPUT = self.task_config.get('COMOUT_CHEM_INPUT', None) From e1bc13b7649b89cf22695b477a80e89263d20fe8 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 10:08:52 -0400 Subject: [PATCH 072/132] Fix GBBEPx NRT fires directory handling in ChemFireEmissions class --- ush/python/pygfs/task/chem_fire_emission.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 7d63c34df22..caec8b2752c 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -161,7 +161,7 @@ def initialize(self) -> None: if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': self.task_config['AERO_EMIS_FIRE_DIR'] = yaml_config.fire_emission.config['NRT_DIRECTORY'] - files_found = self._find_gbbepx_nrt_fires(self.task_config.AERO_EMIS_FIRE_VERSION) + files_found = self._find_gbbepx_nrt_fires(yaml_config.fire_emission.config['NRT_DIRECTORY']) logger.info(f'Found {len(files_found)} GBBEPx NRT files for {self.start_date}' ) logger.info(f"files found: {files_found}") elif self.task_config.AERO_EMIS_FIRE.lower() == 'qfed': @@ -322,7 +322,7 @@ def _get_unique_months(self): @logit(logger) - def _find_gbbepx_nrt_fires(self, emis_file_dir): + def _find_gbbepx_nrt_fires(self, NRT_DIRECTORY: str) -> List[str]: """Find GBBEPx NRT fire files in the specified directory. Parameters @@ -340,15 +340,15 @@ def _find_gbbepx_nrt_fires(self, emis_file_dir): Searches for files matching the pattern "GBBEPx-all01GRID_v4r0_blend_sYYYYMMDD000000_eYYYYMMDD235959_cYYYYMMDDHHMMSS.nc" where YYYYMMDD represents the date components. """ - logger.info(f'Finding GBBEPx NRT fire files in {emis_file_dir}') + logger.info(f'Finding GBBEPx NRT fire files in {NRT_DIIRECTORY}') if not os.path.exists(emis_file_dir): - logger.warning(f"Directory does not exist: {emis_file_dir}") + logger.warning(f"Directory does not exist: {NRT_DIRECTORY}") return [] - all_files = os.listdir(emis_file_dir) + all_files = os.listdir(NRT_DIRECTORY) matching_files = [] - logger.debug(f"Searching in directory: {emis_file_dir}") + logger.debug(f"Searching in directory: {NRT_DIRECTORY}") logger.debug(f"Total files in directory: {len(all_files)} files") logger.debug(f"Files found in directory: {all_files}") # Look for pattern: "GBBEPx-all01GRID_v4r0_blend_s202302240000000_e202302242359590_c202302250134090.nc" From 843ae71c2c3c30ba23edec05ca2c037846f557bd Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 10:11:32 -0400 Subject: [PATCH 073/132] Fix NRT_DIRECTORY access in ChemFireEmissions class to use dot notation --- ush/python/pygfs/task/chem_fire_emission.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index caec8b2752c..94884e220a2 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -160,8 +160,8 @@ def initialize(self) -> None: yaml_config = self.render_template(tmp_dict) if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': - self.task_config['AERO_EMIS_FIRE_DIR'] = yaml_config.fire_emission.config['NRT_DIRECTORY'] - files_found = self._find_gbbepx_nrt_fires(yaml_config.fire_emission.config['NRT_DIRECTORY']) + self.task_config['AERO_EMIS_FIRE_DIR'] = yaml_config.fire_emission.config.NRT_DIRECTORY + files_found = self._find_gbbepx_nrt_fires(yaml_config.fire_emission.config.NRT_DIRECTORY) logger.info(f'Found {len(files_found)} GBBEPx NRT files for {self.start_date}' ) logger.info(f"files found: {files_found}") elif self.task_config.AERO_EMIS_FIRE.lower() == 'qfed': From 4d825380e4770b5073ecc393ec39e13c03f2aeb9 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 10:11:59 -0400 Subject: [PATCH 074/132] Add pprint for debugging YAML configuration in ChemFireEmissions class --- ush/python/pygfs/task/chem_fire_emission.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 94884e220a2..25dadba8e36 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -17,7 +17,7 @@ to_timedelta, WorkflowException, Executable, which) - +from pprint import pprint logger = getLogger(__name__.split('.')[-1]) @@ -162,6 +162,7 @@ def initialize(self) -> None: if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': self.task_config['AERO_EMIS_FIRE_DIR'] = yaml_config.fire_emission.config.NRT_DIRECTORY files_found = self._find_gbbepx_nrt_fires(yaml_config.fire_emission.config.NRT_DIRECTORY) + pprint(yaml_config) logger.info(f'Found {len(files_found)} GBBEPx NRT files for {self.start_date}' ) logger.info(f"files found: {files_found}") elif self.task_config.AERO_EMIS_FIRE.lower() == 'qfed': From de6b35f5862527eba36d95f69c4c000a9e84ad23 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 10:14:32 -0400 Subject: [PATCH 075/132] Add pprint for debugging YAML configuration in ChemFireEmissions class --- ush/python/pygfs/task/chem_fire_emission.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 25dadba8e36..027f59467ba 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -158,11 +158,10 @@ def initialize(self) -> None: 'FIRE_EMIS_NRT_DIR': self.task_config.FIRE_EMIS_NRT_DIR } yaml_config = self.render_template(tmp_dict) - + pprint(yaml_config) if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': self.task_config['AERO_EMIS_FIRE_DIR'] = yaml_config.fire_emission.config.NRT_DIRECTORY files_found = self._find_gbbepx_nrt_fires(yaml_config.fire_emission.config.NRT_DIRECTORY) - pprint(yaml_config) logger.info(f'Found {len(files_found)} GBBEPx NRT files for {self.start_date}' ) logger.info(f"files found: {files_found}") elif self.task_config.AERO_EMIS_FIRE.lower() == 'qfed': From 21f9043c92dcafee39cdca17eab116ea9ae127ef Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 10:15:42 -0400 Subject: [PATCH 076/132] Add pprint for NRT_DIRECTORY in ChemFireEmissions class --- ush/python/pygfs/task/chem_fire_emission.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 027f59467ba..f5b1e6f2658 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -159,6 +159,7 @@ def initialize(self) -> None: } yaml_config = self.render_template(tmp_dict) pprint(yaml_config) + pprint(yaml_config.fire_emission.config.NRT_DIRECTORY) if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': self.task_config['AERO_EMIS_FIRE_DIR'] = yaml_config.fire_emission.config.NRT_DIRECTORY files_found = self._find_gbbepx_nrt_fires(yaml_config.fire_emission.config.NRT_DIRECTORY) From 33779e0175db71e1e5d74318a4c3d021d890dea1 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 10:17:01 -0400 Subject: [PATCH 077/132] Fix typo in NRT_DIRECTORY variable name in GBBEPx file search logging --- ush/python/pygfs/task/chem_fire_emission.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index f5b1e6f2658..7d2a45d1863 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -341,9 +341,9 @@ def _find_gbbepx_nrt_fires(self, NRT_DIRECTORY: str) -> List[str]: Searches for files matching the pattern "GBBEPx-all01GRID_v4r0_blend_sYYYYMMDD000000_eYYYYMMDD235959_cYYYYMMDDHHMMSS.nc" where YYYYMMDD represents the date components. """ - logger.info(f'Finding GBBEPx NRT fire files in {NRT_DIIRECTORY}') + logger.info(f'Finding GBBEPx NRT fire files in {NRT_DIRECTORY}') - if not os.path.exists(emis_file_dir): + if not os.path.exists(NRT_DIRECTORY): logger.warning(f"Directory does not exist: {NRT_DIRECTORY}") return [] From e3b04f1c41d1fea5c9e8061c913ec9ec247403a0 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 10:18:15 -0400 Subject: [PATCH 078/132] Fix NRT_DIRECTORY variable usage in _find_gbbepx_nrt_fires method --- ush/python/pygfs/task/chem_fire_emission.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 7d2a45d1863..07f1bac67ac 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -356,16 +356,16 @@ def _find_gbbepx_nrt_fires(self, NRT_DIRECTORY: str) -> List[str]: pattern = r"GBBEPx-all01GRID.*_s(\d{8}).*_e(\d{8}).*\.nc" if all_files is None: - logger.warning(f"No files found in directory: {emis_file_dir}") + logger.warning(f"No files found in directory: {NRT_DIRECTORY}") logger.warning(f'Checking the previous date') - emis_file_dir = emis_file_dir.replace(self.start_date.strftime('%Y/%m'), (self.start_date - datetime.timedelta(days=1)).strftime('%Y/%m')) - if not os.path.exists(emis_file_dir): - logger.warning(f"Directory does not exist: {emis_file_dir}") + NRT_DIRECTORY = NRT_DIRECTORY.replace(self.start_date.strftime('%Y/%m'), (self.start_date - datetime.timedelta(days=1)).strftime('%Y/%m')) + if not os.path.exists(NRT_DIRECTORY): + logger.warning(f"Directory does not exist: {NRT_DIRECTORY}") return [] for file_name in all_files: match = re.match(pattern, file_name) if match: - full_path = os.path.join(emis_file_dir, file_name) + full_path = os.path.join(NRT_DIRECTORY, file_name) matching_files.append(full_path) logger.debug(f"Found GBBEPx NRT fire file: {full_path}") From 000de2f7c8b370dccf872e4da8f4c4eac8d9cd7d Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 10:33:09 -0400 Subject: [PATCH 079/132] Refactor GBBEPx processing logic for historical and non-historical cases; remove redundant pprint statements --- ush/python/pygfs/task/chem_fire_emission.py | 29 +++++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 07f1bac67ac..d93e1434a95 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -158,8 +158,6 @@ def initialize(self) -> None: 'FIRE_EMIS_NRT_DIR': self.task_config.FIRE_EMIS_NRT_DIR } yaml_config = self.render_template(tmp_dict) - pprint(yaml_config) - pprint(yaml_config.fire_emission.config.NRT_DIRECTORY) if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': self.task_config['AERO_EMIS_FIRE_DIR'] = yaml_config.fire_emission.config.NRT_DIRECTORY files_found = self._find_gbbepx_nrt_fires(yaml_config.fire_emission.config.NRT_DIRECTORY) @@ -748,7 +746,28 @@ def _process_gbbepx_files(self, workdir: str) -> List[str]: logger.info(f"Processing GBBEPx files for {len(self.forecast_dates)} forecast dates") processed_files = [] - for forecast_date in self.forecast_dates: + historical = self.task_config.get('historical', True) + if not historical: # only one file to process for multiple dates (need to change time in each file) + logger.info("Non-historical GBBEPx processing - only one file expected") + if self.task_config.rawfiles: + ds = self.GBBEPx_to_COARDS(self.task_config.rawfiles[0]) + + for index, forecast_date in enumerate(self.forecast_dates): + ds['time'] = index # set time dimension to index (zero for the first date, 1 for the second, etc) + # Save the processed dataset + outfile_name = f"FIRE_EMIS_{self.start_date.strftime('%Y%m%d')}.nc" + outfile = os.path.join(workdir, outfile_name) + comp = dict(zlib=True, complevel=2) + encoding = {var: comp for var in ds.data_vars} + ds.to_netcdf(outfile, encoding=encoding) + logger.info(f"Processed emission file saved to {outfile}") + processed_files.append(outfile) + ds.close() + else: + logger.warning("No raw GBBEPx files found for non-historical processing") + return processed_files + + for forecast_date,date_file in zip(self.forecast_dates, self.task_config.rawfiles): date_str = forecast_date.strftime('%Y%m%d') logger.info(f"Processing GBBEPx files for date {date_str}") @@ -758,9 +777,9 @@ def _process_gbbepx_files(self, workdir: str) -> List[str]: # Add logic here to filter files by date if needed date_files.append(file) - if date_files: + if date_file: # Process files for this date - ds = self.GBBEPx_to_COARDS(date_files[0]) # Use the first file for this date + ds = self.GBBEPx_to_COARDS(date_file) # Use the first file for this date # Create output filename with date outfile_name = f"FIRE_EMIS_{date_str}.nc" From cd77e7b703d4ec9ff58468d9aa98f8572e19b09c Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 10:40:57 -0400 Subject: [PATCH 080/132] Update end date calculation and enhance logging for GBBEPx processing --- ush/python/pygfs/task/chem_fire_emission.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index d93e1434a95..fd77c1f87d3 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -56,8 +56,8 @@ def __init__(self, config: Dict[str, Any]) -> None: self.start_date = self.task_config["CDATE"] logger.info(f"Start date: {self.start_date}") - # end date = SDATE + nforecast hours + 36 - self.end_date = self.task_config["CDATE"] + to_timedelta(f'{nforecast_hours + 36}H') + # end date = SDATE + nforecast hours + 24 + self.end_date = self.task_config["CDATE"] + to_timedelta(f'{nforecast_hours + 24}H') logger.info(f"End date: {self.end_date}") # Calculate number of days spanned by start and end date (inclusive) @@ -750,9 +750,12 @@ def _process_gbbepx_files(self, workdir: str) -> List[str]: if not historical: # only one file to process for multiple dates (need to change time in each file) logger.info("Non-historical GBBEPx processing - only one file expected") if self.task_config.rawfiles: + logger.info(f"Processing single GBBEPx file: {self.task_config.rawfiles[0]}") ds = self.GBBEPx_to_COARDS(self.task_config.rawfiles[0]) for index, forecast_date in enumerate(self.forecast_dates): + logger.info(f"Setting time for forecast date: {forecast_date}") + # Set time coordinate to the forecast date ds['time'] = index # set time dimension to index (zero for the first date, 1 for the second, etc) # Save the processed dataset outfile_name = f"FIRE_EMIS_{self.start_date.strftime('%Y%m%d')}.nc" From 00d6c6f41c1afedbf9419870882de3738734dd96 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 10:48:05 -0400 Subject: [PATCH 081/132] Refactor GBBEPx processing logic to use instance variable for historical flag --- ush/python/pygfs/task/chem_fire_emission.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index fd77c1f87d3..b8d9d2b922b 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -746,8 +746,7 @@ def _process_gbbepx_files(self, workdir: str) -> List[str]: logger.info(f"Processing GBBEPx files for {len(self.forecast_dates)} forecast dates") processed_files = [] - historical = self.task_config.get('historical', True) - if not historical: # only one file to process for multiple dates (need to change time in each file) + if not self.historical: # only one file to process for multiple dates (need to change time in each file) logger.info("Non-historical GBBEPx processing - only one file expected") if self.task_config.rawfiles: logger.info(f"Processing single GBBEPx file: {self.task_config.rawfiles[0]}") From b2d3fae456067b0fed2a6bdd36e3e1dee95d1153 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 11:02:10 -0400 Subject: [PATCH 082/132] Update time coordinate assignment in ChemFireEmissions to use assign_coords method for clarity --- ush/python/pygfs/task/chem_fire_emission.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index b8d9d2b922b..326a8d2b265 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -754,8 +754,9 @@ def _process_gbbepx_files(self, workdir: str) -> List[str]: for index, forecast_date in enumerate(self.forecast_dates): logger.info(f"Setting time for forecast date: {forecast_date}") - # Set time coordinate to the forecast date - ds['time'] = index # set time dimension to index (zero for the first date, 1 for the second, etc) + # Set time coordinate to the forecast date index (0, 1, 2, ...) + # this assumes a daily frequency + ds = ds.assign_coords(time=index) # set time dimension to index (zero for the first date, 1 for the second, etc) # Save the processed dataset outfile_name = f"FIRE_EMIS_{self.start_date.strftime('%Y%m%d')}.nc" outfile = os.path.join(workdir, outfile_name) From 68d64bb86789aa918fc56c365db299032b0282fb Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 11:15:46 -0400 Subject: [PATCH 083/132] Fix time coordinate assignment in ChemFireEmissions to use a list for index --- ush/python/pygfs/task/chem_fire_emission.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 326a8d2b265..6974e2c2249 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -756,7 +756,7 @@ def _process_gbbepx_files(self, workdir: str) -> List[str]: logger.info(f"Setting time for forecast date: {forecast_date}") # Set time coordinate to the forecast date index (0, 1, 2, ...) # this assumes a daily frequency - ds = ds.assign_coords(time=index) # set time dimension to index (zero for the first date, 1 for the second, etc) + ds = ds.assign_coords(time=[index]) # set time dimension to index (zero for the first date, 1 for the second, etc) # Save the processed dataset outfile_name = f"FIRE_EMIS_{self.start_date.strftime('%Y%m%d')}.nc" outfile = os.path.join(workdir, outfile_name) From 1cb4c2a7916a0ab0944ed7bfc45d88a645f45b7c Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 11:18:06 -0400 Subject: [PATCH 084/132] Fix output filename generation in ChemFireEmissions to use forecast date instead of start date --- ush/python/pygfs/task/chem_fire_emission.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 6974e2c2249..eaed23a8804 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -758,7 +758,7 @@ def _process_gbbepx_files(self, workdir: str) -> List[str]: # this assumes a daily frequency ds = ds.assign_coords(time=[index]) # set time dimension to index (zero for the first date, 1 for the second, etc) # Save the processed dataset - outfile_name = f"FIRE_EMIS_{self.start_date.strftime('%Y%m%d')}.nc" + outfile_name = f"FIRE_EMIS_{forecast_date.strftime('%Y%m%d')}.nc" outfile = os.path.join(workdir, outfile_name) comp = dict(zlib=True, complevel=2) encoding = {var: comp for var in ds.data_vars} From 96f6fe19eeb7fbbf80fed1d027abb3da4b44f876 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 11:21:13 -0400 Subject: [PATCH 085/132] Comment out time coordinate assignment in ChemFireEmissions to prevent incorrect indexing --- ush/python/pygfs/task/chem_fire_emission.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index eaed23a8804..ade68664575 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -756,7 +756,7 @@ def _process_gbbepx_files(self, workdir: str) -> List[str]: logger.info(f"Setting time for forecast date: {forecast_date}") # Set time coordinate to the forecast date index (0, 1, 2, ...) # this assumes a daily frequency - ds = ds.assign_coords(time=[index]) # set time dimension to index (zero for the first date, 1 for the second, etc) + # ds = ds.assign_coords(time=[index]) # set time dimension to index (zero for the first date, 1 for the second, etc) # Save the processed dataset outfile_name = f"FIRE_EMIS_{forecast_date.strftime('%Y%m%d')}.nc" outfile = os.path.join(workdir, outfile_name) From 45702638f28d123cd04ef1700aff6c5228722a4c Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 11:23:34 -0400 Subject: [PATCH 086/132] Update time coordinate assignment in ChemFireEmissions to set units based on forecast date --- ush/python/pygfs/task/chem_fire_emission.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index ade68664575..10f3a36ea12 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -754,9 +754,7 @@ def _process_gbbepx_files(self, workdir: str) -> List[str]: for index, forecast_date in enumerate(self.forecast_dates): logger.info(f"Setting time for forecast date: {forecast_date}") - # Set time coordinate to the forecast date index (0, 1, 2, ...) - # this assumes a daily frequency - # ds = ds.assign_coords(time=[index]) # set time dimension to index (zero for the first date, 1 for the second, etc) + ds.time.attrs['units'] = f'days since {forecast_date.strftime("%Y-%m-%d 12:00:00")}' # Save the processed dataset outfile_name = f"FIRE_EMIS_{forecast_date.strftime('%Y%m%d')}.nc" outfile = os.path.join(workdir, outfile_name) From dcf80a644ee00c7dcf4f20c9aa1ac8089dc90e8b Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 11:26:36 -0400 Subject: [PATCH 087/132] Update longitude attributes in ChemFireEmissions to include standard name and axis, and remove coordinates attribute from relevant variables --- ush/python/pygfs/task/chem_fire_emission.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 10f3a36ea12..22f2143e684 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -561,6 +561,8 @@ def GBBEPx_to_COARDS(self, fname: Union[str, os.PathLike]) -> xr.Dataset: # Check longitude range and monotonicity if not (f.lon.diff('lon') > 0).all(): raise WorkflowException("Longitude values must be strictly increasing") + f.lon.attrs['standard_name'] = 'longitude' + f.lon.attrs['axis'] = 'X' # Ensure longitude is in [-180, 180] range f['lon'] = xr.where(f.lon > 180, f.lon - 360, f.lon) @@ -587,6 +589,8 @@ def GBBEPx_to_COARDS(self, fname: Union[str, os.PathLike]) -> xr.Dataset: f[v].attrs.update({'units': '-', 'long_name': 'percent_of_clouds_in_grid_cell'}) elif v == 'NumSensor': f[v].attrs['units'] = '-' + if 'coordinates' in f[v].attrs: + del f[v].attrs['coordinates'] # Set global attributes f.attrs.update({'format': 'NetCDF', 'title': 'GBBEPx Fire Emissions'}) From cd2abc28e88be6abc67945a431d41f9f2552b232 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 14:31:58 -0400 Subject: [PATCH 088/132] Refactor execution command in NEXUSEmissions to use APRUN for launching nexus.x --- .../cases/yamls/gcafs_cycled_noDA_defaults_dev.yaml | 2 +- ush/python/pygfs/task/nexus_emission.py | 11 ++--------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/dev/ci/cases/yamls/gcafs_cycled_noDA_defaults_dev.yaml b/dev/ci/cases/yamls/gcafs_cycled_noDA_defaults_dev.yaml index 34bbdbe11b4..abf945e794d 100644 --- a/dev/ci/cases/yamls/gcafs_cycled_noDA_defaults_dev.yaml +++ b/dev/ci/cases/yamls/gcafs_cycled_noDA_defaults_dev.yaml @@ -1,5 +1,5 @@ defaults: - !INC {{ HOMEgfs }}/dev/parm/config/sfs/yaml/defaults.yaml + !INC {{ HOMEgfs }}/dev/parm/config/gcafs/yaml/defaults.yaml base: DO_TEST_MODE: "NO" USE_AERO_ANL: "NO" diff --git a/ush/python/pygfs/task/nexus_emission.py b/ush/python/pygfs/task/nexus_emission.py index 4307e9a680b..58d4e6b0a6e 100644 --- a/ush/python/pygfs/task/nexus_emission.py +++ b/ush/python/pygfs/task/nexus_emission.py @@ -300,15 +300,8 @@ def execute(self) -> None: if not os.path.exists(self.task_config.DATA): raise WorkflowException(f"Working directory does not exist: {self.task_config.DATA}") - exe = Executable(self.task_config.launcher) - if 'mpiexec' in self.task_config.launcher: - arg_list = ['-n', str(1), 'nexus.x', '-c', self.task_config.NEXUS_CONFIG_NAME] - else: - arg_list = ['--ntasks', - str(1), - 'nexus.x', - '-c', - self.task_config.NEXUS_CONFIG_NAME] + exe = Executable(self.task_config.APRUN) + arg_list = [ 'nexus.x', '-c', self.task_config.NEXUS_CONFIG_NAME] exe(*arg_list, output='stdout', error='stderr') logger.info("Concatenating processed NEXUS files...") From d2df1b1a5d3c2fafcaf32d1001e2e14200fc5c68 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 24 Sep 2025 14:35:36 -0400 Subject: [PATCH 089/132] Fix execution command in NEXUSEmissions to use relative path for nexus.x --- ush/python/pygfs/task/nexus_emission.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/nexus_emission.py b/ush/python/pygfs/task/nexus_emission.py index 58d4e6b0a6e..bd92609fa06 100644 --- a/ush/python/pygfs/task/nexus_emission.py +++ b/ush/python/pygfs/task/nexus_emission.py @@ -301,7 +301,7 @@ def execute(self) -> None: raise WorkflowException(f"Working directory does not exist: {self.task_config.DATA}") exe = Executable(self.task_config.APRUN) - arg_list = [ 'nexus.x', '-c', self.task_config.NEXUS_CONFIG_NAME] + arg_list = [ './nexus.x', '-c', self.task_config.NEXUS_CONFIG_NAME] exe(*arg_list, output='stdout', error='stderr') logger.info("Concatenating processed NEXUS files...") From 0691f99a4d9ffd8f4eeee6d887560722cab54bea Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Thu, 25 Sep 2025 13:25:48 -0400 Subject: [PATCH 090/132] Update aerosol optics file versions and enhance NEXUS emission file rendering --- dev/parm/config/gcafs/config.aero.j2 | 49 ++++++++++-- parm/ufs/gocart/CA2G_instance_CA.bc.rc | 2 +- parm/ufs/gocart/CA2G_instance_CA.oc.rc | 2 +- parm/ufs/gocart/DU2G_instance_DU.rc | 4 +- parm/ufs/gocart/SS2G_instance_SS.rc | 2 +- parm/ufs/gocart/SU2G_instance_SU.rc | 2 +- ush/python/pygfs/task/chem_fire_emission.py | 84 ++++++++++----------- ush/python/pygfs/task/nexus_emission.py | 55 ++++++++------ 8 files changed, 122 insertions(+), 78 deletions(-) diff --git a/dev/parm/config/gcafs/config.aero.j2 b/dev/parm/config/gcafs/config.aero.j2 index 51bfd6264e3..375c2b89ef7 100644 --- a/dev/parm/config/gcafs/config.aero.j2 +++ b/dev/parm/config/gcafs/config.aero.j2 @@ -1,32 +1,46 @@ #! /usr/bin/env bash +#================================================================================ # UFS-Aerosols settings # This configuration file sets up environment variables for aerosol modeling in the UFS (Unified Forecast System) Aerosols component. # It configures aerosol inputs, diagnostics, emissions, and the NEXUS emissions preprocessor for GCAFS (Global Coupled Aerosol Forecast System). # Used in GFS (Global Forecast System) workflows for initializing and running aerosol simulations in FV3 (Finite-Volume Cubed-Sphere) dynamical core. +#================================================================================ +echo "BEGIN: config.aero" #================================================================================ # 1. Aerosol settings #================================================================================ # General settings for aerosol tracers, diagnostics, and scavenging in the GOCART (Goddard Chemistry Aerosol Radiation and Transport) model. # These are used in the atmospheric model to handle aerosol transport, chemistry, and interaction with radiation/cloud processes. -export AERO_INPUTS_DIR="{{ AERO_INPUTS_DIR }}" + # Base directory for aerosol input data files (e.g., initial conditions, climatologies). # This path is mounted or staged in the workflow and referenced by the model for reading aerosol fields. +#--------------------------------------------------------------------------------------------------- +export AERO_INPUTS_DIR="{{ AERO_INPUTS_DIR }}" +# Temporary comment until we sync between machines +#export AERO_INPUTS_DIR="/lfs/h2/emc/lam/noscrub/barry.baker/gcafs/GCAFS" # WCOSS2 path for GCAFS external data +#export AERO_INPUTS_DIR="/gpfs/f6/bil-fire3/world-shared/Emissions/GEFS_ExtData # GAEA C6 path for GCAFS external data -#----------------------------------------------- +#------------------------------------------------- # Diag Table and Field Table for GOCART aerosols -#----------------------------------------------- +#------------------------------------------------- + # Configuration files defining diagnostic outputs and field registrations for aerosol variables in GOCART. # diag_table.aero: Specifies which aerosol fields to output and at what frequency (used by FMS diagnostics). # field_table.aero: Registers prognostic/diagnostic tracers with the FV3 dynamical core (e.g., for advection, diffusion). +#--------------------------------------------------------------------------------------------------- export AERO_DIAG_TABLE="${PARMgfs}/ufs/fv3/diag_table.aero" export AERO_FIELD_TABLE="${PARMgfs}/ufs/fv3/field_table.aero" +#================================================================================ # Aerosol configuration -export AERO_CONFIG_DIR="${PARMgfs}/ufs/gocart" +#================================================================================ + # Directory containing GOCART-specific namelists, parameters, and runtime configs (e.g., namelist.aero). # Loaded during model initialization to set aerosol scheme parameters like time steps, vertical levels. +#--------------------------------------------------------------------------------------------------- +export AERO_CONFIG_DIR="${PARMgfs}/ufs/gocart" # Aerosol convective scavenging factors (list of string array elements) # Element syntax: ':'. Use = * to set default factor for all aerosol tracers @@ -34,11 +48,13 @@ export AERO_CONFIG_DIR="${PARMgfs}/ufs/gocart" # Used in the convection scheme (e.g., SAS or NF_CONV) to compute in-cloud scavenging rates for each tracer. # * = default for unspecified tracers; specific gases like SO2 have lower factors due to solubility. # Scavenging factors are set to 0 (no scavenging) if unset +#--------------------------------------------------------------------------------------------------- export fscav_aero="'*:0.3','so2:0.0','msa:0.0','dms:0.0','nh3:0.4','nh4:0.6','bc1:0.6','bc2:0.6','oc1:0.4','oc2:0.4','dust1:0.6','dust2:0.6', 'dust3:0.6','dust4:0.6','dust5:0.6','seas1:0.5','seas2:0.5','seas3:0.5','seas4:0.5','seas5:0.5'" -# + # Number of diagnostic aerosol tracers (default: 0) # Specifies how many additional diagnostic (non-prognostic) aerosol tracers to include in the model output. # Used in GOCART to control verbosity of diagnostics; higher values add more fields for analysis/post-processing. +#--------------------------------------------------------------------------------------------------- export dnats_aero=2 #================================================================================ @@ -52,11 +68,13 @@ export dnats_aero=2 # gbbepx: Global Biomass Burning Emissions Product (alternative). # none: Disable fire emissions. # Used in prep_emissions scripts to fetch/interpolate data to model grid. +#--------------------------------------------------------------------------------------------------- export AERO_EMIS_FIRE="qfed" export AERO_EMIS_FIRE_VERSION="061" # Version of the selected fire emissions dataset (e.g., for QFEDv2.5, version 061). # Determines which historical or NRT files to load from input directories. +#--------------------------------------------------------------------------------------------------- export AERO_EMIS_FIRE_HIST=1 # Use historical fire emissions | 1 = true 0 = false # Flag to enable historical (climatological) fire emissions instead of NRT for testing/spin-up. @@ -64,6 +82,7 @@ export AERO_EMIS_FIRE_HIST=1 # Use historical fire emissions | 1 = true 0 = fals # Path to near-real-time (NRT) fire emissions data, updated daily (e.g., from satellites like MODIS). # On WCOSS2, points to DCOM (Data Communication) root for operational runs; empty for testing. # Processed by scripts like exglobal_prep_emissions.py to generate input files for GOCART. +#--------------------------------------------------------------------------------------------------- export FIRE_EMIS_NRT_DIR="" #TODO: set to DCOM for WCOSS2 "${DCOMROOT}/YYYYMMDD/firewx" # Directory containing NRT fire emissions @@ -84,6 +103,7 @@ export NEXUS_CONFIG="{{ NEXUS_CONFIG | default('gocart') }}" # Options: gocart, # Runtime choice of NEXUS config variant; defaults to gocart for standard aerosol tracers. # Overrides via Jinja2 templating in workflow (e.g., for different chemistry schemes). +#--------------------------------------------------------------------------------------------------- export NEXUS_CONFIG_DIR="${PARMgfs}/chem/nexus/${NEXUS_CONFIG}" # Directory containing NEXUS configuration files # NEXUS Inputs @@ -92,9 +112,12 @@ export NEXUS_CONFIG_DIR="${PARMgfs}/chem/nexus/${NEXUS_CONFIG}" # Directory cont # export NEXUS_INPUT_DIR="${AERO_INPUTS_DIR}/nexus" # Directory for static/dynamic input data used by NEXUS (e.g., emission inventories, masks, meteo fields). # Currently hardcoded for development; will use shared AERO_INPUTS_DIR in production for consistency. -export NEXUS_INPUT_DIR="/gpfs/f6/bil-fire3/world-shared/Emissions/GEFS_ExtData/nexus" # Specific path for GCAFS external data on this filesystem. # Contains emission datasets (e.g., CEDS2019/2024, HTAPv2, CAMS) processed by NEXUS. +#--------------------------------------------------------------------------------------------------- +export NEXUS_INPUT_DIR=${AERO_INPUTS_DIR}/nexus + + #-------------------------- # NEXUS Time Step (seconds) @@ -102,6 +125,7 @@ export NEXUS_INPUT_DIR="/gpfs/f6/bil-fire3/world-shared/Emissions/GEFS_ExtData/n # Temporal resolution for emission interpolation in NEXUS (e.g., hourly outputs). # Must align with model coupling time; used in HEMCO time management for diurnal/seasonal scaling. # 3600s = 1 hour; adjustable for finer/coarser emission updates (e.g., 1800s for sub-hourly). +#--------------------------------------------------------------------------------------------------- export NEXUS_TSTEP="{{ NEXUS_TSTEP | default(3600) }}" # Default NEXUS time step in seconds #------------------ @@ -110,18 +134,31 @@ export NEXUS_TSTEP="{{ NEXUS_TSTEP | default(3600) }}" # Default NEXUS time step # Defines the emission grid for NEXUS processing (0.5x0.5 degree global lat-lon). # Emissions are interpolated from this grid to the FV3 cubed-sphere grid during prep. # Number of longitude points (1440 for 0.25-degree resolution; here 1440 ~0.25deg). +#----------------------------------------------------- export NEXUS_NX="{{ NEXUS_NX | default(1440) }}" + # Number of latitude points (720 for 0.25-degree). +#-------------------------------------------------- export NEXUS_NY="{{ NEXUS_NY | default(720) }}" + # Western boundary longitude (global coverage). +#------------------------------------------------- export NEXUS_XMIN="{{ NEXUS_XMIN | default(-180.0) }}" + # Eastern boundary longitude. +#-------------------------------------------------- export NEXUS_XMAX="{{ NEXUS_XMAX | default(180.0) }}" + # Southern boundary latitude. +#-------------------------------------------------- export NEXUS_YMIN="{{ NEXUS_YMIN | default(-90.0) }}" + # Northern boundary latitude. +#--------------------------------------------------- export NEXUS_YMAX="{{ NEXUS_YMAX | default(90.0) }}" + # Number of vertical levels (1 for surface emissions; higher for vertical profiles if needed). +#-------------------------------------------------- export NEXUS_NZ="{{ NEXUS_NZ | default(1) }}" #------------------- diff --git a/parm/ufs/gocart/CA2G_instance_CA.bc.rc b/parm/ufs/gocart/CA2G_instance_CA.bc.rc index 19b20a6e65d..165e94d700c 100644 --- a/parm/ufs/gocart/CA2G_instance_CA.bc.rc +++ b/parm/ufs/gocart/CA2G_instance_CA.bc.rc @@ -5,7 +5,7 @@ nbins: 2 aerosol_radBands_optics_file: ExtData/optics/opticsBands_BC.v1_3.RRTMG.nc -aerosol_monochromatic_optics_file: ExtData/monochromatic/optics_BC.v1_3.nc +aerosol_monochromatic_optics_file: ExtData/monochromatic/optics_BC.v1_5.nc # Aircraft emission factor: convert input unit to kg C aircraft_fuel_emission_factor: 1.0000 diff --git a/parm/ufs/gocart/CA2G_instance_CA.oc.rc b/parm/ufs/gocart/CA2G_instance_CA.oc.rc index e510fec2eb2..c90e4ffeb4e 100644 --- a/parm/ufs/gocart/CA2G_instance_CA.oc.rc +++ b/parm/ufs/gocart/CA2G_instance_CA.oc.rc @@ -3,7 +3,7 @@ # aerosol_radBands_optics_file: ExtData/optics/opticsBands_OC.v1_3.RRTMG.nc -aerosol_monochromatic_optics_file: ExtData/monochromatic/optics_OC.v1_3.nc +aerosol_monochromatic_optics_file: ExtData/monochromatic/optics_OC.v1_5.nc # Aircraft emission factor: convert input unit to kg C aircraft_fuel_emission_factor: 1.0000 diff --git a/parm/ufs/gocart/DU2G_instance_DU.rc b/parm/ufs/gocart/DU2G_instance_DU.rc index d1a0d98edff..fef964a4834 100644 --- a/parm/ufs/gocart/DU2G_instance_DU.rc +++ b/parm/ufs/gocart/DU2G_instance_DU.rc @@ -3,7 +3,7 @@ # aerosol_radBands_optics_file: ExtData/optics/opticsBands_DU.v15_3.RRTMG.nc -aerosol_monochromatic_optics_file: ExtData/monochromatic/optics_DU.v15_3.nc +aerosol_monochromatic_optics_file: ExtData/monochromatic/optics_DU.v15_5.nc particle_radius_microns: 0.73 1.4 2.4 4.5 8.0 @@ -55,7 +55,7 @@ emission_scheme: fengsha # choose among: fengsha, ginoux, k14 alpha: 0.16 gamma: 1.0 soil_moisture_factor: 1 -soil_drylimit_factor: 1.75 +soil_drylimit_factor: 1.0 vertical_to_horizontal_flux_ratio_limit: 2.e-04 drag_partition_option: 2 diff --git a/parm/ufs/gocart/SS2G_instance_SS.rc b/parm/ufs/gocart/SS2G_instance_SS.rc index 6bc90ae3b7c..5d093650c20 100644 --- a/parm/ufs/gocart/SS2G_instance_SS.rc +++ b/parm/ufs/gocart/SS2G_instance_SS.rc @@ -3,7 +3,7 @@ # aerosol_radBands_optics_file: ExtData/optics/opticsBands_SS.v3_3.RRTMG.nc -aerosol_monochromatic_optics_file: ExtData/monochromatic/optics_SS.v3_3.nc +aerosol_monochromatic_optics_file: ExtData/monochromatic/optics_SS.v3_5.nc particle_radius_microns: 0.079 0.316 1.119 2.818 7.772 diff --git a/parm/ufs/gocart/SU2G_instance_SU.rc b/parm/ufs/gocart/SU2G_instance_SU.rc index 1613283d23b..2e5928ed238 100644 --- a/parm/ufs/gocart/SU2G_instance_SU.rc +++ b/parm/ufs/gocart/SU2G_instance_SU.rc @@ -3,7 +3,7 @@ # aerosol_radBands_optics_file: ExtData/optics/opticsBands_SU.v1_3.RRTMG.nc -aerosol_monochromatic_optics_file: ExtData/monochromatic/optics_SU.v1_3.nc +aerosol_monochromatic_optics_file: ExtData/monochromatic/optics_SU.v1_5.nc nbins: 4 diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 22f2143e684..140d6331e1b 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -122,6 +122,10 @@ def initialize(self) -> None: logger.info(f'Using AERO_INPUTS_DIR: {aero_inputs_dir}') logger.info(f'Using AERO_EMIS_FIRE: {aero_emis_fire}') logger.info(f'Using AERO_EMIS_FIRE_VERSION: {aero_emis_fire_version}') + + fire_emission_template = os.path.join(self.task_config.HOMEgfs, 'parm', 'chem', 'fire_emission.yaml.j2') + if not os.path.exists(fire_emission_template): + raise WorkflowException(f"Fire emission template file not found: {fire_emission_template}") AERO_EMIS_FIRE_DIR = os.path.join(aero_inputs_dir, "nexus", @@ -147,21 +151,20 @@ def initialize(self) -> None: logger.info(f'Found {len(files_found)} files for historical period') self.task_config["AERO_EMIS_FIRE_DIR"] = AERO_EMIS_FIRE_DIR else: - #=============================================== - # # NRT Forecast emisssions - #=============================================== + # =============================================== + # NRT Forecast emissions + # =============================================== logger.info(f'Processing forecast emissions for {self.start_date}') # GBBEPx NRT files are in a different directory structure - # Render the template with the current cycle to get the correct path + # Render the template with the current cycle to get the correct path tmp_dict = {'current_cycle': self.start_date, - 'FIRE_EMIS_NRT_DIR': self.task_config.FIRE_EMIS_NRT_DIR - } + 'FIRE_EMIS_NRT_DIR': self.task_config.FIRE_EMIS_NRT_DIR} yaml_config = self.render_template(tmp_dict) if self.task_config.AERO_EMIS_FIRE.lower() == 'gbbepx': self.task_config['AERO_EMIS_FIRE_DIR'] = yaml_config.fire_emission.config.NRT_DIRECTORY files_found = self._find_gbbepx_nrt_fires(yaml_config.fire_emission.config.NRT_DIRECTORY) - logger.info(f'Found {len(files_found)} GBBEPx NRT files for {self.start_date}' ) + logger.info(f'Found {len(files_found)} GBBEPx NRT files for {self.start_date}') logger.info(f"files found: {files_found}") elif self.task_config.AERO_EMIS_FIRE.lower() == 'qfed': # Get QFED variables with safe defaults @@ -319,7 +322,6 @@ def _get_unique_months(self): years = set(date.year for date in self.forecast_dates) return months, years - @logit(logger) def _find_gbbepx_nrt_fires(self, NRT_DIRECTORY: str) -> List[str]: """Find GBBEPx NRT fire files in the specified directory. @@ -368,7 +370,7 @@ def _find_gbbepx_nrt_fires(self, NRT_DIRECTORY: str) -> List[str]: logger.debug(f"Found GBBEPx NRT fire file: {full_path}") return matching_files - + @logit(logger) def _find_gbbepx_files(self, dates, version='v5r0'): """Find GBBEPx files for the given date @@ -411,7 +413,6 @@ def _find_gbbepx_files(self, dates, version='v5r0'): # Look for both file patterns: # Pattern 1: "GBBEPx-all01GRID_v4r0_blend_s202302240000000_e202302242359590_c202302250134090.nc" # Pattern 2: "GBBEPx_all01GRID.emissions_v004_20150716.nc" - for file_name in all_files: match_found = False @@ -750,7 +751,7 @@ def _process_gbbepx_files(self, workdir: str) -> List[str]: logger.info(f"Processing GBBEPx files for {len(self.forecast_dates)} forecast dates") processed_files = [] - if not self.historical: # only one file to process for multiple dates (need to change time in each file) + if not self.historical: # only one file to process for multiple dates (need to change time in each file) logger.info("Non-historical GBBEPx processing - only one file expected") if self.task_config.rawfiles: logger.info(f"Processing single GBBEPx file: {self.task_config.rawfiles[0]}") @@ -768,41 +769,40 @@ def _process_gbbepx_files(self, workdir: str) -> List[str]: logger.info(f"Processed emission file saved to {outfile}") processed_files.append(outfile) ds.close() - else: - logger.warning("No raw GBBEPx files found for non-historical processing") - return processed_files - - for forecast_date,date_file in zip(self.forecast_dates, self.task_config.rawfiles): - date_str = forecast_date.strftime('%Y%m%d') - logger.info(f"Processing GBBEPx files for date {date_str}") + else: + logger.warning("No raw GBBEPx files found for non-historical processing") + else: + for forecast_date, date_file in zip(self.forecast_dates, self.task_config.rawfiles): + date_str = forecast_date.strftime('%Y%m%d') + logger.info(f"Processing GBBEPx files for date {date_str}") - # Filter files for this date - implement date filtering if needed - date_files = [] - for file in self.task_config.rawfiles: - # Add logic here to filter files by date if needed - date_files.append(file) + # Filter files for this date - implement date filtering if needed + date_files = [] + for file in self.task_config.rawfiles: + # Add logic here to filter files by date if needed + date_files.append(file) - if date_file: - # Process files for this date - ds = self.GBBEPx_to_COARDS(date_file) # Use the first file for this date + if date_file: + # Process files for this date + ds = self.GBBEPx_to_COARDS(date_file) # Use the first file for this date - # Create output filename with date - outfile_name = f"FIRE_EMIS_{date_str}.nc" - outfile = os.path.join(workdir, outfile_name) + # Create output filename with date + outfile_name = f"FIRE_EMIS_{date_str}.nc" + outfile = os.path.join(workdir, outfile_name) - # Save the processed dataset - comp = dict(zlib=True, complevel=2) - encoding = {var: comp for var in ds.data_vars} - ds.to_netcdf(outfile, encoding=encoding) - logger.info(f"Processed emission file saved to {outfile}") + # Save the processed dataset + comp = dict(zlib=True, complevel=2) + encoding = {var: comp for var in ds.data_vars} + ds.to_netcdf(outfile, encoding=encoding) + logger.info(f"Processed emission file saved to {outfile}") - # Add to processed files list - processed_files.append(outfile) + # Add to processed files list + processed_files.append(outfile) - # Close dataset - ds.close() - else: - logger.warning(f"No GBBEPx files found for date {date_str}") + # Close dataset + ds.close() + else: + logger.warning(f"No GBBEPx files found for date {date_str}") return processed_files @@ -872,7 +872,7 @@ def _process_qfed_files(self, workdir: str) -> List[str]: return processed_files - @logit(logger) + @logit(logger) def render_template(self, tmpl_dict: Dict[str, Any]) -> None: """Render the YAML template and set up task configuration. @@ -896,6 +896,4 @@ def render_template(self, tmpl_dict: Dict[str, Any]) -> None: else: logger.debug(f'Parsing YAML template: {yaml_template}') yaml_config = parse_j2yaml(yaml_template, tmpl_dict) - return yaml_config - diff --git a/ush/python/pygfs/task/nexus_emission.py b/ush/python/pygfs/task/nexus_emission.py index bd92609fa06..9211a5a120c 100644 --- a/ush/python/pygfs/task/nexus_emission.py +++ b/ush/python/pygfs/task/nexus_emission.py @@ -1,24 +1,20 @@ #!/usr/bin/env python3 import os -import re from collections import defaultdict import xarray as xr -import subprocess -import numpy as np -import cftime from logging import getLogger from typing import Dict, Any, Union, List from dateutil.rrule import DAILY, HOURLY, rrule -from jinja2 import Environment, FileSystemLoader from wxflow import (AttrDict, FileHandler, parse_j2yaml, logit, Task, + JInja, to_timedelta, WorkflowException, - Executable, which) + Executable) logger = getLogger(__name__.split('.')[-1]) @@ -229,40 +225,53 @@ def initialize(self) -> None: logger.info(f"NEXUS input directory linked to {self.task_config.DATA}") # Render NEXUS Grid File - file_loader = FileSystemLoader(self.task_config.NEXUS_CONFIG_DIR) - env = Environment(loader=file_loader) - nexus_grid_template = env.get_template(f"{self.task_config.NEXUS_GRID_NAME}.j2") - self.task_config.NEXUS_GRID_TEMPLATE = nexus_grid_template.render(tmpl_dict) + nexus_grid_template = os.path.join(self.task_config.NEXUS_CONFIG_DIR, f"{self.task_config.NEXUS_GRID_NAME}.j2") + logger.info(f"Rendering NEXUS grid file using template: {nexus_grid_template}") + if not os.path.exists(nexus_grid_template): + raise WorkflowException(f"NEXUS grid template file not found: {nexus_grid_template}") + j2_renderer = Jinja(nexus_grid_template, tmpl_dict) outfile = os.path.join(self.task_config.DATA, self.task_config.NEXUS_GRID_NAME) - _write_txt_file(self.task_config.NEXUS_GRID_TEMPLATE, outfile) + j2_renderer.save(outfile) logger.info(f"NEXUS grid file rendered successfully: written to {outfile}") # Render NEXUS Config File - nexus_config_template = env.get_template(f"{self.task_config.NEXUS_CONFIG_NAME}.j2") - self.task_config.NEXUS_CONFIG_TEMPLATE = nexus_config_template.render(tmpl_dict) + nexus_config_template = os.path.join(self.task_config.NEXUS_CONFIG_DIR, f"{self.task_config.NEXUS_CONFIG_NAME}.j2") + logger.info(f"Rendering NEXUS config file using template: {nexus_config_template}") + if not os.path.exists(nexus_config_template): + raise WorkflowException(f"NEXUS config template file not found: {nexus_config_template}") + j2_renderer = Jinja(nexus_config_template, tmpl_dict) outfile = os.path.join(self.task_config.DATA, self.task_config.NEXUS_CONFIG_NAME) - _write_txt_file(self.task_config.NEXUS_CONFIG_TEMPLATE, outfile) + j2_renderer.save(outfile) logger.info(f"NEXUS config file rendered successfully: written to {outfile}") # Render NEXUS Time File - nexus_time_template = env.get_template(f"{self.task_config.NEXUS_TIME_NAME}.j2") - self.task_config.NEXUS_TIME_TEMPLATE = nexus_time_template.render(tmpl_dict) + nexus_time_template = os.path.join(self.task_config.NEXUS_CONFIG_DIR, f"{self.task_config.NEXUS_TIME_NAME}.j2") + logger.info(f"Rendering NEXUS time file using template: {nexus_time_template}") + if not os.path.exists(nexus_time_template): + raise WorkflowException(f"NEXUS time template file not found: {nexus_time_template}") + j2_renderer = Jinja(nexus_time_template, tmpl_dict) outfile = os.path.join(self.task_config.DATA, self.task_config.NEXUS_TIME_NAME) - _write_txt_file(self.task_config.NEXUS_TIME_TEMPLATE, outfile) + j2_renderer.save(outfile) logger.info(f"NEXUS time file rendered successfully: written to {outfile}") # Render NEXUS Diag File - nexus_diag_template = env.get_template(f"{self.task_config.NEXUS_DIAG_NAME}.j2") - self.task_config.NEXUS_DIAG_TEMPLATE = nexus_diag_template.render(tmpl_dict) + nexus_diag_template = os.path.join(self.task_config.NEXUS_CONFIG_DIR, f"{self.task_config.NEXUS_DIAG_NAME}.j2") + logger.info(f"Rendering NEXUS diag file using template: {nexus_diag_template}") + if not os.path.exists(nexus_diag_template): + raise WorkflowException(f"NEXUS diag template file not found: {nexus_diag_template}") + j2_renderer = Jinja(nexus_diag_template, tmpl_dict) outfile = os.path.join(self.task_config.DATA, self.task_config.NEXUS_DIAG_NAME) - _write_txt_file(self.task_config.NEXUS_DIAG_TEMPLATE, outfile) + j2_renderer.save(outfile) logger.info(f"NEXUS diag file rendered successfully: written to {outfile}") # Render NEXUS Spec File - nexus_spec_template = env.get_template(f"{self.task_config.NEXUS_SPEC_NAME}.j2") - self.task_config.NEXUS_SPEC_TEMPLATE = nexus_spec_template.render(tmpl_dict) + nexus_spec_template = os.path.join(self.task_config.NEXUS_CONFIG_DIR, f"{self.task_config.NEXUS_SPEC_NAME}.j2") + logger.info(f"Rendering NEXUS spec file using template: {nexus_spec_template}") + if not os.path.exists(nexus_spec_template): + raise WorkflowException(f"NEXUS spec template file not found: {nexus_spec_template}") + j2_renderer = Jinja(nexus_spec_template, tmpl_dict) outfile = os.path.join(self.task_config.DATA, self.task_config.NEXUS_SPEC_NAME) - _write_txt_file(self.task_config.NEXUS_SPEC_TEMPLATE, outfile) + j2_renderer.save(outfile) logger.info(f"NEXUS spec file rendered successfully: written to {outfile}") @logit(logger) From 483f5d77d63f68f56401866e2300322050fc140c Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Thu, 25 Sep 2025 13:31:28 -0400 Subject: [PATCH 091/132] Fix typo in import statement for Jinja in nexus_emission.py --- ush/python/pygfs/task/nexus_emission.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/nexus_emission.py b/ush/python/pygfs/task/nexus_emission.py index 9211a5a120c..386c31800b8 100644 --- a/ush/python/pygfs/task/nexus_emission.py +++ b/ush/python/pygfs/task/nexus_emission.py @@ -11,7 +11,7 @@ parse_j2yaml, logit, Task, - JInja, + Jinja, to_timedelta, WorkflowException, Executable) From 464fa44d79bede837ab818b91e57f195535ce19e Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Thu, 25 Sep 2025 14:24:53 -0400 Subject: [PATCH 092/132] pycodestyle fixes --- dev/ush/compare_f90nml.py | 3 +- scripts/exglobal_stage_ic.py | 2 +- ush/python/pygfs/task/analysis.py | 2 +- ush/python/pygfs/task/atm_analysis.py | 2 +- ush/python/pygfs/task/chem_fire_emission.py | 5 ++- ush/python/pygfs/task/ensemble_recenter.py | 4 +-- ush/python/pygfs/task/marine_letkf.py | 2 +- ush/python/pygfs/task/nexus_emission.py | 2 +- ush/python/pygfs/task/oceanice_products.py | 36 ++++++++++----------- ush/python/pygfs/task/snowens_analysis.py | 2 +- 10 files changed, 29 insertions(+), 31 deletions(-) diff --git a/dev/ush/compare_f90nml.py b/dev/ush/compare_f90nml.py index f3c5573a927..a3d47e76992 100755 --- a/dev/ush/compare_f90nml.py +++ b/dev/ush/compare_f90nml.py @@ -77,8 +77,7 @@ def _print_diffs(diff_dict: Dict) -> None: max_len = len(max(diff_dict[path], key=len)) for kk in diff_dict[path].keys(): items = diff_dict[path][kk] - print( - f"{kk:>{max_len+2}} : {' | '.join(map(str, diff_dict[path][kk]))}") + print(f"{kk:>{max_len + 2}} : {' | '.join(map(str, diff_dict[path][kk]))}") # noqa: E226 _print_diffs(result) diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index 3aaafaa96fd..1de3507b8bf 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -30,7 +30,7 @@ def main(): for key in keys: # Make sure OCNRES is three digits if key == "OCNRES": - stage.task_config.OCNRES = f"{stage.task_config.OCNRES :03d}" + stage.task_config.OCNRES = f"{stage.task_config.OCNRES:03d}" # noqa: E203 stage_dict[key] = stage.task_config[key] # Also import all COM* directory and template variables diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index 1d8b38483b0..48cd8eaac59 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -29,7 +29,7 @@ def __init__(self, config: Dict[str, Any]) -> None: # Store location of GDASApp jinja2 templates self.gdasapp_j2tmpl_dir = os.path.join(self.task_config.PARMgfs, 'gdas') # fix ocnres - self.task_config.OCNRES = f"{self.task_config.OCNRES :03d}" + self.task_config.OCNRES = f"{self.task_config.OCNRES:03d}" def initialize(self) -> None: super().initialize() diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index dae5469a1c3..2da7c0d0160 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -272,7 +272,7 @@ def finalize(self) -> None: inc_copy = {'copy': []} for itile in range(6): src = os.path.join(self.task_config.DATA, "anl", - f"{self.task_config.APREFIX}cubed_sphere_grid_atminc.tile{itile+1}.nc") + f"{self.task_config.APREFIX}cubed_sphere_grid_atminc.tile{itile + 1}.nc") dest = self.task_config.COMOUT_ATMOS_ANALYSIS inc_copy['copy'].append([src, dest]) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 140d6331e1b..85cefa80650 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -122,7 +122,7 @@ def initialize(self) -> None: logger.info(f'Using AERO_INPUTS_DIR: {aero_inputs_dir}') logger.info(f'Using AERO_EMIS_FIRE: {aero_emis_fire}') logger.info(f'Using AERO_EMIS_FIRE_VERSION: {aero_emis_fire_version}') - + fire_emission_template = os.path.join(self.task_config.HOMEgfs, 'parm', 'chem', 'fire_emission.yaml.j2') if not os.path.exists(fire_emission_template): raise WorkflowException(f"Fire emission template file not found: {fire_emission_template}") @@ -871,7 +871,6 @@ def _process_qfed_files(self, workdir: str) -> List[str]: return processed_files - @logit(logger) def render_template(self, tmpl_dict: Dict[str, Any]) -> None: """Render the YAML template and set up task configuration. @@ -880,7 +879,7 @@ def render_template(self, tmpl_dict: Dict[str, Any]) -> None: 1. Loads and parses the YAML template file using Jinja2 2. Fills in configuration parameters using environment variables and task attributes 3. Updates the task configuration with the rendered YAML content - + Parameters ---------- tmp_dict : Dict diff --git a/ush/python/pygfs/task/ensemble_recenter.py b/ush/python/pygfs/task/ensemble_recenter.py index 2f50f45501e..5501f7ead3a 100644 --- a/ush/python/pygfs/task/ensemble_recenter.py +++ b/ush/python/pygfs/task/ensemble_recenter.py @@ -166,10 +166,10 @@ def finalize(self) -> None: hr = format(fh, '03') for itile in range(6): src = os.path.join(self.task_config.DATA, memchar, - f"{self.task_config.APREFIX_ENS}cubed_sphere_grid_ratmi{hr}.tile{itile+1}.nc") + f"{self.task_config.APREFIX_ENS}cubed_sphere_grid_ratmi{hr}.tile{itile + 1}.nc") if fh == 6: dest = os.path.join(incdir, - f"{self.task_config.APREFIX_ENS}cubed_sphere_grid_ratminc.tile{itile+1}.nc") + f"{self.task_config.APREFIX_ENS}cubed_sphere_grid_ratminc.tile{itile + 1}.nc") else: dest = incdir fh_dict['copy'].append([src, dest]) diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index b8ec5cdddbb..dfffefacf00 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -57,7 +57,7 @@ def __init__(self, config: Dict) -> None: self.task_config.PARMmarine = os.path.join(self.task_config.PARMgfs, 'gdas', 'marine') self.task_config.app_path_observations = self.task_config.MARINE_JCB_GDAS_OBS self.task_config.letkf_app = "true" - self.task_config.OPREFIX = f"{self.task_config.RUN.replace('enkf','')}.t{self.task_config.cyc:02d}z." + self.task_config.OPREFIX = f"{self.task_config.RUN.replace('enkf', '')}.t{self.task_config.cyc:02d}z." @logit(logger) def initialize(self): diff --git a/ush/python/pygfs/task/nexus_emission.py b/ush/python/pygfs/task/nexus_emission.py index 386c31800b8..ac0f7d29180 100644 --- a/ush/python/pygfs/task/nexus_emission.py +++ b/ush/python/pygfs/task/nexus_emission.py @@ -310,7 +310,7 @@ def execute(self) -> None: raise WorkflowException(f"Working directory does not exist: {self.task_config.DATA}") exe = Executable(self.task_config.APRUN) - arg_list = [ './nexus.x', '-c', self.task_config.NEXUS_CONFIG_NAME] + arg_list = ['./nexus.x', '-c', self.task_config.NEXUS_CONFIG_NAME] exe(*arg_list, output='stdout', error='stderr') logger.info("Concatenating processed NEXUS files...") diff --git a/ush/python/pygfs/task/oceanice_products.py b/ush/python/pygfs/task/oceanice_products.py index d319608ad14..2ad8ff84bab 100644 --- a/ush/python/pygfs/task/oceanice_products.py +++ b/ush/python/pygfs/task/oceanice_products.py @@ -66,24 +66,24 @@ def __init__(self, config: Dict[str, Any]) -> None: # TODO: This is a bit of a hack, but it works for now # FIXME: find a better way to provide the averaging period - avg_period = f"{forecast_hour-interval:03d}-{forecast_hour:03d}" - - # Extend task_config with localdict - localdict = AttrDict( - {'component': self.task_config.COMPONENT, - 'forecast_hour': forecast_hour, - 'valid_datetime': valid_datetime, - 'avg_period': avg_period, - 'model_grid': model_grid, - 'interval': interval, - 'product_grids': self.VALID_PRODUCT_GRIDS[model_grid]} - ) - self.task_config = AttrDict(**self.task_config, **localdict) - - # Read the oceanice_products.yaml file for common configuration - logger.info(f"Read the ocean ice products configuration yaml file {self.task_config.OCEANICEPRODUCTS_CONFIG}") - self.task_config.oceanice_yaml = parse_j2yaml(self.task_config.OCEANICEPRODUCTS_CONFIG, self.task_config) - logger.debug(f"oceanice_yaml:\n{pformat(self.task_config.oceanice_yaml)}") + avg_period = f"{forecast_hour - interval:03d}-{forecast_hour:03d}" + + # Extend task_config with localdict + localdict = AttrDict( + {'component': self.task_config.COMPONENT, + 'forecast_hour': forecast_hour, + 'valid_datetime': valid_datetime, + 'avg_period': avg_period, + 'model_grid': model_grid, + 'interval': interval, + 'product_grids': self.VALID_PRODUCT_GRIDS[model_grid]} + ) + self.task_config = AttrDict(**self.task_config, **localdict) + + # Read the oceanice_products.yaml file for common configuration + logger.info(f"Read the ocean ice products configuration yaml file {self.task_config.OCEANICEPRODUCTS_CONFIG}") + self.task_config.oceanice_yaml = parse_j2yaml(self.task_config.OCEANICEPRODUCTS_CONFIG, self.task_config) + logger.debug(f"oceanice_yaml:\n{pformat(self.task_config.oceanice_yaml)}") @staticmethod @logit(logger) diff --git a/ush/python/pygfs/task/snowens_analysis.py b/ush/python/pygfs/task/snowens_analysis.py index 3d903bd118a..dbec8b49164 100644 --- a/ush/python/pygfs/task/snowens_analysis.py +++ b/ush/python/pygfs/task/snowens_analysis.py @@ -56,7 +56,7 @@ def __init__(self, config: Dict[str, Any]): _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config['assim_freq']}H") / 2) # fix ocnres - self.task_config.OCNRES = f"{self.task_config.OCNRES :03d}" + self.task_config.OCNRES = f"{self.task_config.OCNRES: 03d}" # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( From 78aa5e100fbe42e261cb1d9ab804d59b40d17a39 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Thu, 25 Sep 2025 15:50:09 -0400 Subject: [PATCH 093/132] Update ush/python/pygfs/task/snowens_analysis.py Co-authored-by: Jiarui Dong --- ush/python/pygfs/task/snowens_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/snowens_analysis.py b/ush/python/pygfs/task/snowens_analysis.py index dbec8b49164..dc6154a2190 100644 --- a/ush/python/pygfs/task/snowens_analysis.py +++ b/ush/python/pygfs/task/snowens_analysis.py @@ -56,7 +56,7 @@ def __init__(self, config: Dict[str, Any]): _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config['assim_freq']}H") / 2) # fix ocnres - self.task_config.OCNRES = f"{self.task_config.OCNRES: 03d}" + self.task_config.OCNRES = f"{self.task_config.OCNRES:03d}" # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( From 87d756fb4d96e3b3352907ad33048a18b6ebddf3 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Thu, 25 Sep 2025 15:54:20 -0400 Subject: [PATCH 094/132] update dust to use drag_partition_option: 1 with the new file --- parm/ufs/gocart/DU2G_instance_DU.rc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/ufs/gocart/DU2G_instance_DU.rc b/parm/ufs/gocart/DU2G_instance_DU.rc index fef964a4834..02306202bc9 100644 --- a/parm/ufs/gocart/DU2G_instance_DU.rc +++ b/parm/ufs/gocart/DU2G_instance_DU.rc @@ -57,7 +57,7 @@ gamma: 1.0 soil_moisture_factor: 1 soil_drylimit_factor: 1.0 vertical_to_horizontal_flux_ratio_limit: 2.e-04 -drag_partition_option: 2 +drag_partition_option: 1 # SettlingSolver options # Options: 'gocart' or 'ufs' From 1378357219cf2623457777f2afeaa686edcce6d5 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Thu, 25 Sep 2025 15:55:41 -0400 Subject: [PATCH 095/132] update dust inputs in ExtData.other --- parm/ufs/gocart/ExtData.other | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/parm/ufs/gocart/ExtData.other b/parm/ufs/gocart/ExtData.other index f2b60d5eb39..def6a3e8078 100644 --- a/parm/ufs/gocart/ExtData.other +++ b/parm/ufs/gocart/ExtData.other @@ -13,9 +13,9 @@ DU_SAND '1' Y E - none none sandfrac ExtData/n DU_SILT '1' Y E - none none siltfrac /dev/null DU_SSM '1' Y E - none none sep /dev/null:1.0 DU_UTHRES '1' Y E - none none uthres ExtData/nexus/FENGSHA/FENGSHA_2022_NESDIS_inputs_10km_v3.2.nc -DU_RDRAG '1' Y E %y4-%m2-%d2t12:00:00 none none PC ExtData/nexus/FENGSHA/FENGSHA_New_Method_NESDISv1.1_9km.nc -DU_GVF '1' Y E %y4-%m2-%d2T12:00:00 none none GVF ExtData/nexus/FENGSHA/FENGSHA_GVF_LAI2.nc -DU_LAI '1' Y E %y4-%m2-%d2T12:00:00 none none LAI ExtData/nexus/FENGSHA/FENGSHA_GVF_LAI2.nc +DU_RDRAG '1' Y E %y4-%m2-%d2t12:00:00 none none PC ExtData/nexus/FENGSHA/FENGSHA_2022_NESDIS_inputs_10km_v4.nc +DU_GVF '1' Y E %y4-%m2-%d2T12:00:00 none none GVF /dev/null:0.0 +DU_LAI '1' Y E %y4-%m2-%d2T12:00:00 none none LAI /dev/null/:0.0 #====== Sulfate Sources ================================================= # Anthropogenic (BF & FF) emissions -- allowed to input as two layers From 488b075dc7cf27b83a857d093ba857bbdd52c367 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Fri, 26 Sep 2025 09:05:39 -0400 Subject: [PATCH 096/132] rollback and add pycodestyle fix --- ush/python/pygfs/task/oceanice_products.py | 36 +++++++++++----------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/ush/python/pygfs/task/oceanice_products.py b/ush/python/pygfs/task/oceanice_products.py index 2ad8ff84bab..278775ffeda 100644 --- a/ush/python/pygfs/task/oceanice_products.py +++ b/ush/python/pygfs/task/oceanice_products.py @@ -66,24 +66,24 @@ def __init__(self, config: Dict[str, Any]) -> None: # TODO: This is a bit of a hack, but it works for now # FIXME: find a better way to provide the averaging period - avg_period = f"{forecast_hour - interval:03d}-{forecast_hour:03d}" - - # Extend task_config with localdict - localdict = AttrDict( - {'component': self.task_config.COMPONENT, - 'forecast_hour': forecast_hour, - 'valid_datetime': valid_datetime, - 'avg_period': avg_period, - 'model_grid': model_grid, - 'interval': interval, - 'product_grids': self.VALID_PRODUCT_GRIDS[model_grid]} - ) - self.task_config = AttrDict(**self.task_config, **localdict) - - # Read the oceanice_products.yaml file for common configuration - logger.info(f"Read the ocean ice products configuration yaml file {self.task_config.OCEANICEPRODUCTS_CONFIG}") - self.task_config.oceanice_yaml = parse_j2yaml(self.task_config.OCEANICEPRODUCTS_CONFIG, self.task_config) - logger.debug(f"oceanice_yaml:\n{pformat(self.task_config.oceanice_yaml)}") + avg_period = f"{forecast_hour-interval:03d} - {forecast_hour:03d}" + + # Extend task_config with localdict + localdict = AttrDict( + {'component': self.task_config.COMPONENT, + 'forecast_hour': forecast_hour, + 'valid_datetime': valid_datetime, + 'avg_period': avg_period, + 'model_grid': model_grid, + 'interval': interval, + 'product_grids': self.VALID_PRODUCT_GRIDS[model_grid]} + ) + self.task_config = AttrDict(**self.task_config, **localdict) + + # Read the oceanice_products.yaml file for common configuration + logger.info(f"Read the ocean ice products configuration yaml file {self.task_config.OCEANICEPRODUCTS_CONFIG}") + self.task_config.oceanice_yaml = parse_j2yaml(self.task_config.OCEANICEPRODUCTS_CONFIG, self.task_config) + logger.debug(f"oceanice_yaml:\n{pformat(self.task_config.oceanice_yaml)}") @staticmethod @logit(logger) From cbcb9cdaa827aaac92b8628d081959897956d10c Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Fri, 26 Sep 2025 09:45:13 -0400 Subject: [PATCH 097/132] Enhance GBBEPx file search logic to check for previous dates and improve error logging --- ush/python/pygfs/task/chem_fire_emission.py | 23 +++++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 85cefa80650..03e5542667c 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -342,26 +342,31 @@ def _find_gbbepx_nrt_fires(self, NRT_DIRECTORY: str) -> List[str]: where YYYYMMDD represents the date components. """ logger.info(f'Finding GBBEPx NRT fire files in {NRT_DIRECTORY}') + dates_to_look_for = range(0, 3) # today and two previous days + + for find_date_index in dates_to_look_for: + find_date = self.start_date - datetime.timedelta(days=find_date_index) + logger.info(f'Looking for files for date: {find_date.strftime("%Y%m%d")}') + NRT_DIRECTORY = NRT_DIRECTORY.replace(self.start_date.strftime('%Y%m%d'), + find_date.strftime('%Y%m%d')) + if not os.path.exists(NRT_DIRECTORY): + logger.warning(f"Directory does not exist: {NRT_DIRECTORY}") + continue + else: + break if not os.path.exists(NRT_DIRECTORY): - logger.warning(f"Directory does not exist: {NRT_DIRECTORY}") + logger.error(f"Could not find a valid NRT_DIRECTORY for GBBEPx files") return [] all_files = os.listdir(NRT_DIRECTORY) matching_files = [] - logger.debug(f"Searching in directory: {NRT_DIRECTORY}") + logger.info(f"Searching in directory: {NRT_DIRECTORY}") logger.debug(f"Total files in directory: {len(all_files)} files") logger.debug(f"Files found in directory: {all_files}") # Look for pattern: "GBBEPx-all01GRID_v4r0_blend_s202302240000000_e202302242359590_c202302250134090.nc" pattern = r"GBBEPx-all01GRID.*_s(\d{8}).*_e(\d{8}).*\.nc" - if all_files is None: - logger.warning(f"No files found in directory: {NRT_DIRECTORY}") - logger.warning(f'Checking the previous date') - NRT_DIRECTORY = NRT_DIRECTORY.replace(self.start_date.strftime('%Y/%m'), (self.start_date - datetime.timedelta(days=1)).strftime('%Y/%m')) - if not os.path.exists(NRT_DIRECTORY): - logger.warning(f"Directory does not exist: {NRT_DIRECTORY}") - return [] for file_name in all_files: match = re.match(pattern, file_name) if match: From 40ce24a624ee4e093c94d8710611a7212c2bda97 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Fri, 26 Sep 2025 10:09:48 -0400 Subject: [PATCH 098/132] update netcdf header for NRT fire emission --- ush/python/pygfs/task/chem_fire_emission.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 03e5542667c..fc824d6fc21 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -764,7 +764,8 @@ def _process_gbbepx_files(self, workdir: str) -> List[str]: for index, forecast_date in enumerate(self.forecast_dates): logger.info(f"Setting time for forecast date: {forecast_date}") - ds.time.attrs['units'] = f'days since {forecast_date.strftime("%Y-%m-%d 12:00:00")}' + # Set time dimension to index for days since (0, 1, 2, ..., nforecast_dates -1) + ds = ds.assign(time=[float(index)]) # Save the processed dataset outfile_name = f"FIRE_EMIS_{forecast_date.strftime('%Y%m%d')}.nc" outfile = os.path.join(workdir, outfile_name) From 89b3439f549fdbb472aaf09125321a481b3cfc5a Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Fri, 26 Sep 2025 10:09:58 -0400 Subject: [PATCH 099/132] fix linking for nexus.x --- sorc/link_workflow.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 373e1f6542e..91719df1245 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -422,6 +422,7 @@ fi # NEXUS executable if [[ -d "${HOMEgfs}/sorc/nexus.fd/build/bin" ]]; then + cd "${HOMEgfs}/exec" || exit 1 ${LINK_OR_COPY} "${HOMEgfs}/sorc/nexus.fd/build/bin/nexus" nexus.x fi From d74878271a90e47df2782a7da9ccf1d829b5953a Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Fri, 26 Sep 2025 10:15:08 -0400 Subject: [PATCH 100/132] Add attributes for time dimension in processed dataset --- ush/python/pygfs/task/chem_fire_emission.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index fc824d6fc21..479fb3c4cd3 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -766,6 +766,10 @@ def _process_gbbepx_files(self, workdir: str) -> List[str]: logger.info(f"Setting time for forecast date: {forecast_date}") # Set time dimension to index for days since (0, 1, 2, ..., nforecast_dates -1) ds = ds.assign(time=[float(index)]) + ds.time.attrs['long_name'] = 'time' + ds.time.attrs['units'] = f'days since {self.start_date.strftime("%Y-%m-%d 00:00:00")}' + ds.time.attrs['calendar'] = 'gregorian' + # Save the processed dataset outfile_name = f"FIRE_EMIS_{forecast_date.strftime('%Y%m%d')}.nc" outfile = os.path.join(workdir, outfile_name) From e4f80fbe57dfc9b9a4e031b388a9a2b705d9b726 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Fri, 26 Sep 2025 10:17:43 -0400 Subject: [PATCH 101/132] Add check for NEXUS preprocessor executable availability --- ush/python/pygfs/task/nexus_emission.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ush/python/pygfs/task/nexus_emission.py b/ush/python/pygfs/task/nexus_emission.py index ac0f7d29180..2f6e098ecff 100644 --- a/ush/python/pygfs/task/nexus_emission.py +++ b/ush/python/pygfs/task/nexus_emission.py @@ -310,6 +310,10 @@ def execute(self) -> None: raise WorkflowException(f"Working directory does not exist: {self.task_config.DATA}") exe = Executable(self.task_config.APRUN) + + if not exe.is_exe_available('nexus.x'): + raise WorkflowException("NEXUS preprocessor executable 'nexus.x' not found in PATH") + arg_list = ['./nexus.x', '-c', self.task_config.NEXUS_CONFIG_NAME] exe(*arg_list, output='stdout', error='stderr') From 9903eb8b094a022420387161bfeb9e3a5b28c03a Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Fri, 26 Sep 2025 10:22:37 -0400 Subject: [PATCH 102/132] Refactor NEXUS executable check to use os.path.exists for improved clarity --- ush/python/pygfs/task/nexus_emission.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/nexus_emission.py b/ush/python/pygfs/task/nexus_emission.py index 2f6e098ecff..b056adc0032 100644 --- a/ush/python/pygfs/task/nexus_emission.py +++ b/ush/python/pygfs/task/nexus_emission.py @@ -311,10 +311,10 @@ def execute(self) -> None: exe = Executable(self.task_config.APRUN) - if not exe.is_exe_available('nexus.x'): + if os.path.exists("nexus.x") is False: raise WorkflowException("NEXUS preprocessor executable 'nexus.x' not found in PATH") - arg_list = ['./nexus.x', '-c', self.task_config.NEXUS_CONFIG_NAME] + arg_list = ['nexus.x', '-c', self.task_config.NEXUS_CONFIG_NAME] exe(*arg_list, output='stdout', error='stderr') logger.info("Concatenating processed NEXUS files...") From e0fda09c0817a1abde4198f854b094c872ad8c4e Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Fri, 26 Sep 2025 10:47:38 -0400 Subject: [PATCH 103/132] Update NEXUS executable path to use relative path for improved portability --- ush/python/pygfs/task/nexus_emission.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/nexus_emission.py b/ush/python/pygfs/task/nexus_emission.py index b056adc0032..4e0a274cf8d 100644 --- a/ush/python/pygfs/task/nexus_emission.py +++ b/ush/python/pygfs/task/nexus_emission.py @@ -314,7 +314,7 @@ def execute(self) -> None: if os.path.exists("nexus.x") is False: raise WorkflowException("NEXUS preprocessor executable 'nexus.x' not found in PATH") - arg_list = ['nexus.x', '-c', self.task_config.NEXUS_CONFIG_NAME] + arg_list = ['./nexus.x', '-c', self.task_config.NEXUS_CONFIG_NAME] exe(*arg_list, output='stdout', error='stderr') logger.info("Concatenating processed NEXUS files...") From ba7c5ab4e403fc9b67a6f2e4de34e2495402669b Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Fri, 26 Sep 2025 10:48:14 -0400 Subject: [PATCH 104/132] Update commented AERO_INPUTS_DIR path for consistency across machines --- dev/parm/config/gcafs/config.aero.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/parm/config/gcafs/config.aero.j2 b/dev/parm/config/gcafs/config.aero.j2 index 375c2b89ef7..0ad6f2ee8fc 100644 --- a/dev/parm/config/gcafs/config.aero.j2 +++ b/dev/parm/config/gcafs/config.aero.j2 @@ -19,7 +19,7 @@ echo "BEGIN: config.aero" #--------------------------------------------------------------------------------------------------- export AERO_INPUTS_DIR="{{ AERO_INPUTS_DIR }}" # Temporary comment until we sync between machines -#export AERO_INPUTS_DIR="/lfs/h2/emc/lam/noscrub/barry.baker/gcafs/GCAFS" # WCOSS2 path for GCAFS external data +#export AERO_INPUTS_DIR="/lfs/h2/emc/lam/noscrub/barry.baker/Emissions/gcafs/GCAFS" # WCOSS2 path for GCAFS external data #export AERO_INPUTS_DIR="/gpfs/f6/bil-fire3/world-shared/Emissions/GEFS_ExtData # GAEA C6 path for GCAFS external data #------------------------------------------------- From 086b91fdc03afc2fb323f58787d7590e7daffbc2 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Fri, 26 Sep 2025 15:14:32 -0400 Subject: [PATCH 105/132] Adjust start and end date calculations for fire emissions processing; update time attributes in dataset output --- ush/python/pygfs/task/chem_fire_emission.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 479fb3c4cd3..46010dcb7d6 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -53,11 +53,11 @@ def __init__(self, config: Dict[str, Any]) -> None: logger.info(f"Number of forecast hours: {nforecast_hours}") # Create start date based on SDATE - self.start_date = self.task_config["CDATE"] + self.start_date = self.task_config["CDATE"] - to_timedelta('24H') # include previous day logger.info(f"Start date: {self.start_date}") - # end date = SDATE + nforecast hours + 24 - self.end_date = self.task_config["CDATE"] + to_timedelta(f'{nforecast_hours + 24}H') + # end date = SDATE + nforecast hours + 36 + self.end_date = self.task_config["CDATE"] + to_timedelta(f'{nforecast_hours + 36}H') logger.info(f"End date: {self.end_date}") # Calculate number of days spanned by start and end date (inclusive) @@ -567,8 +567,6 @@ def GBBEPx_to_COARDS(self, fname: Union[str, os.PathLike]) -> xr.Dataset: # Check longitude range and monotonicity if not (f.lon.diff('lon') > 0).all(): raise WorkflowException("Longitude values must be strictly increasing") - f.lon.attrs['standard_name'] = 'longitude' - f.lon.attrs['axis'] = 'X' # Ensure longitude is in [-180, 180] range f['lon'] = xr.where(f.lon > 180, f.lon - 360, f.lon) @@ -767,15 +765,15 @@ def _process_gbbepx_files(self, workdir: str) -> List[str]: # Set time dimension to index for days since (0, 1, 2, ..., nforecast_dates -1) ds = ds.assign(time=[float(index)]) ds.time.attrs['long_name'] = 'time' - ds.time.attrs['units'] = f'days since {self.start_date.strftime("%Y-%m-%d 00:00:00")}' - ds.time.attrs['calendar'] = 'gregorian' + ds.time.attrs['units'] = f'days since {self.start_date.strftime("%Y-%m-%d 12:00:00")}' + ds.time.attrs['time_increment'] = 240000 # 24 hours in HHMMSS format # Save the processed dataset outfile_name = f"FIRE_EMIS_{forecast_date.strftime('%Y%m%d')}.nc" outfile = os.path.join(workdir, outfile_name) comp = dict(zlib=True, complevel=2) encoding = {var: comp for var in ds.data_vars} - ds.to_netcdf(outfile, encoding=encoding) + ds.to_netcdf(outfile, encoding=encoding, unlimited_dims=['time']) logger.info(f"Processed emission file saved to {outfile}") processed_files.append(outfile) ds.close() @@ -803,7 +801,7 @@ def _process_gbbepx_files(self, workdir: str) -> List[str]: # Save the processed dataset comp = dict(zlib=True, complevel=2) encoding = {var: comp for var in ds.data_vars} - ds.to_netcdf(outfile, encoding=encoding) + ds.to_netcdf(outfile, encoding=encoding, unlimited_dims=['time']) logger.info(f"Processed emission file saved to {outfile}") # Add to processed files list From 4074b9989842a058823e641dc397115d228fec83 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Mon, 29 Sep 2025 14:39:24 -0400 Subject: [PATCH 106/132] move the prep_emis to the same dependency as stage_ic in gcdas_fcst --- dev/workflow/rocoto/gcafs_tasks.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dev/workflow/rocoto/gcafs_tasks.py b/dev/workflow/rocoto/gcafs_tasks.py index 56ae830adb3..5dcdec3d3bb 100644 --- a/dev/workflow/rocoto/gcafs_tasks.py +++ b/dev/workflow/rocoto/gcafs_tasks.py @@ -715,10 +715,6 @@ def _fcst_cycled(self): dep = rocoto.add_dependency(dep_dict) dependencies = rocoto.create_dependency(dep=dep) - if self.options['do_aero_fcst']: - dep_dict = {'type': 'task', 'name': f'{self.run}_prep_emissions'} - dependencies.append(rocoto.add_dependency(dep_dict)) - if self.options['use_aero_anl']: dep_dict = {'type': 'task', 'name': f'{anldep}_aeroanlfinal'} dependencies.append(rocoto.add_dependency(dep_dict)) @@ -728,6 +724,11 @@ def _fcst_cycled(self): if self.run in ['gcdas']: dep_dict = {'type': 'task', 'name': f'{self.run}_stage_ic'} dependencies.append(rocoto.add_dependency(dep_dict)) + + if self.options['do_aero_fcst']: + dep_dict = {'type': 'task', 'name': f'{self.run}_prep_emissions'} + dependencies.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies) cycledef = 'gcdas_half,gcdas' if self.run in ['gcdas'] else self.run From 35d0d8bf3538ae32473ca184a4afdd052c53ac7f Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Mon, 29 Sep 2025 14:49:11 -0400 Subject: [PATCH 107/132] correct variable --- parm/ufs/gocart/ExtData.gbbepx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/ufs/gocart/ExtData.gbbepx b/parm/ufs/gocart/ExtData.gbbepx index 5d666757db4..686a8f20815 100644 --- a/parm/ufs/gocart/ExtData.gbbepx +++ b/parm/ufs/gocart/ExtData.gbbepx @@ -1,6 +1,6 @@ #====== BIOMASS BURNING EMISSIONS ======================================= -# QFED +# GBBEPx #-------------------------------------------------------------------------------------------------------------------------------- SU_BIOMASS NA N Y %y4-%m2-%d2t12:00:00 none 0.7778 SO2 ChemInput/FIRE_EMIS_%y4%m2%d2.nc OC_BIOMASS NA N Y %y4-%m2-%d2t12:00:00 none 0.7778 OC ChemInput/FIRE_EMIS_%y4%m2%d2.nc From d25339cb0356e95c27b5e696e6a1489784b6a5c0 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Mon, 29 Sep 2025 14:49:29 -0400 Subject: [PATCH 108/132] fix gcdas forecast dependencies --- dev/workflow/rocoto/gcafs_tasks.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dev/workflow/rocoto/gcafs_tasks.py b/dev/workflow/rocoto/gcafs_tasks.py index 5dcdec3d3bb..6296c84bcc3 100644 --- a/dev/workflow/rocoto/gcafs_tasks.py +++ b/dev/workflow/rocoto/gcafs_tasks.py @@ -719,17 +719,17 @@ def _fcst_cycled(self): dep_dict = {'type': 'task', 'name': f'{anldep}_aeroanlfinal'} dependencies.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) - if self.run in ['gcdas']: dep_dict = {'type': 'task', 'name': f'{self.run}_stage_ic'} dependencies.append(rocoto.add_dependency(dep_dict)) - if self.options['do_aero_fcst']: - dep_dict = {'type': 'task', 'name': f'{self.run}_prep_emissions'} - dependencies.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies) + + if self.options['do_aero_fcst']: + dep_dict = {'type': 'task', 'name': f'{self.run}_prep_emissions'} + dependencies.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies) + dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) cycledef = 'gcdas_half,gcdas' if self.run in ['gcdas'] else self.run From 8e3d0d3ea0530579b34a059bf550b495cd2174fb Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Mon, 29 Sep 2025 14:49:40 -0400 Subject: [PATCH 109/132] udpate gbbepx processing --- ush/python/pygfs/task/chem_fire_emission.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 46010dcb7d6..fb90878f408 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -554,7 +554,7 @@ def GBBEPx_to_COARDS(self, fname: Union[str, os.PathLike]) -> xr.Dataset: """ logger.info(f"Converting {fname} to COARDS format") f = xr.open_dataset(fname, decode_cf=False) - + f = f[['OC', 'BC', 'SO2', 'NOx', 'CO', 'NH3']] # Handle time dimension if 'Time' in f.dims: f = f.rename({"Time": 'time'}) @@ -579,6 +579,16 @@ def GBBEPx_to_COARDS(self, fname: Union[str, os.PathLike]) -> xr.Dataset: f.lon.attrs.update({'long_name': 'Longitude', 'units': 'degrees_east'}) f.lat.attrs.update({'long_name': 'Latitude', 'units': 'degrees_north'}) + # remove unnessicary attributes + del f['lat'].attrs['valid_range'] + del f['lat'].attrs['scale_factor'] + del f['lat'].attrs['add_offset'] + del f['lat'].attrs['_FillValue'] + del f['time'].attrs['begin_date'] + del f['time'].attrs['begin_time'] + del f['time'].attrs['time_increment'] + del f['time'].attrs['calendar'] + # Remove Element dimension if present if 'Element' in f.dims: f = f.drop_dims('Element') @@ -763,15 +773,14 @@ def _process_gbbepx_files(self, workdir: str) -> List[str]: for index, forecast_date in enumerate(self.forecast_dates): logger.info(f"Setting time for forecast date: {forecast_date}") # Set time dimension to index for days since (0, 1, 2, ..., nforecast_dates -1) - ds = ds.assign(time=[float(index)]) + # ds = ds.assign(time=[float(index)]) ds.time.attrs['long_name'] = 'time' - ds.time.attrs['units'] = f'days since {self.start_date.strftime("%Y-%m-%d 12:00:00")}' - ds.time.attrs['time_increment'] = 240000 # 24 hours in HHMMSS format + ds.time.attrs['units'] = f'minutes since {forecast_date.strftime("%Y-%m-%d 12:00:00")}' # Save the processed dataset outfile_name = f"FIRE_EMIS_{forecast_date.strftime('%Y%m%d')}.nc" outfile = os.path.join(workdir, outfile_name) - comp = dict(zlib=True, complevel=2) + comp = dict(zlib=True, complevel=2, _FillValue=0.0) encoding = {var: comp for var in ds.data_vars} ds.to_netcdf(outfile, encoding=encoding, unlimited_dims=['time']) logger.info(f"Processed emission file saved to {outfile}") From 5da1e6ec7414f84ee68bae6ac7ed604664392296 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Mon, 29 Sep 2025 15:05:54 -0400 Subject: [PATCH 110/132] change fill value --- ush/python/pygfs/task/chem_fire_emission.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index fb90878f408..b585ded3902 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -780,7 +780,7 @@ def _process_gbbepx_files(self, workdir: str) -> List[str]: # Save the processed dataset outfile_name = f"FIRE_EMIS_{forecast_date.strftime('%Y%m%d')}.nc" outfile = os.path.join(workdir, outfile_name) - comp = dict(zlib=True, complevel=2, _FillValue=0.0) + comp = dict(zlib=True, complevel=2, _FillValue=None) encoding = {var: comp for var in ds.data_vars} ds.to_netcdf(outfile, encoding=encoding, unlimited_dims=['time']) logger.info(f"Processed emission file saved to {outfile}") From 431f15dfd6765926f4301eec26051e29efc9026f Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Mon, 29 Sep 2025 15:21:08 -0400 Subject: [PATCH 111/132] change rdrag parameter --- parm/ufs/gocart/ExtData.other | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/ufs/gocart/ExtData.other b/parm/ufs/gocart/ExtData.other index def6a3e8078..0f3ec37329c 100644 --- a/parm/ufs/gocart/ExtData.other +++ b/parm/ufs/gocart/ExtData.other @@ -13,7 +13,7 @@ DU_SAND '1' Y E - none none sandfrac ExtData/n DU_SILT '1' Y E - none none siltfrac /dev/null DU_SSM '1' Y E - none none sep /dev/null:1.0 DU_UTHRES '1' Y E - none none uthres ExtData/nexus/FENGSHA/FENGSHA_2022_NESDIS_inputs_10km_v3.2.nc -DU_RDRAG '1' Y E %y4-%m2-%d2t12:00:00 none none PC ExtData/nexus/FENGSHA/FENGSHA_2022_NESDIS_inputs_10km_v4.nc +DU_RDRAG '1' Y E %y4-%m2-%d2t12:00:00 none none albedo_drag ExtData/nexus/FENGSHA/FENGSHA_2022_NESDIS_inputs_10km_v4.nc DU_GVF '1' Y E %y4-%m2-%d2T12:00:00 none none GVF /dev/null:0.0 DU_LAI '1' Y E %y4-%m2-%d2T12:00:00 none none LAI /dev/null/:0.0 From 5d8fa73241683090ab0d8174a6624ba2702e166e Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Thu, 2 Oct 2025 15:08:07 +0000 Subject: [PATCH 112/132] update nexus hash --- sorc/nexus.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/nexus.fd b/sorc/nexus.fd index 6c660b10a58..e900334dbf6 160000 --- a/sorc/nexus.fd +++ b/sorc/nexus.fd @@ -1 +1 @@ -Subproject commit 6c660b10a58e3596db527753106daea35ef152a6 +Subproject commit e900334dbf62dcdf6f81e687933134e2630468fa From 063da78243f049869ac552a528fae04c5287da18 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Thu, 2 Oct 2025 17:49:30 +0000 Subject: [PATCH 113/132] updating nexus hash again --- sorc/nexus.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/nexus.fd b/sorc/nexus.fd index e900334dbf6..a06deeb4d23 160000 --- a/sorc/nexus.fd +++ b/sorc/nexus.fd @@ -1 +1 @@ -Subproject commit e900334dbf62dcdf6f81e687933134e2630468fa +Subproject commit a06deeb4d231094895b5b955fb10f4e799ddf8c8 From eb0eb719650714a132bb5bf566e85b590d64ae60 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Fri, 3 Oct 2025 14:59:01 +0000 Subject: [PATCH 114/132] Create 'Restarts' directory in task_config.DATA during initialization --- ush/python/pygfs/task/nexus_emission.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ush/python/pygfs/task/nexus_emission.py b/ush/python/pygfs/task/nexus_emission.py index 4e0a274cf8d..5ef2d27b969 100644 --- a/ush/python/pygfs/task/nexus_emission.py +++ b/ush/python/pygfs/task/nexus_emission.py @@ -274,6 +274,9 @@ def initialize(self) -> None: j2_renderer.save(outfile) logger.info(f"NEXUS spec file rendered successfully: written to {outfile}") + #create a directory in the self.task_config.DATA/Restarts + os.makedirs(os.path.join(self.task_config.DATA, 'Restarts'), exist_ok=True) + @logit(logger) def execute(self) -> None: """Run NEXUS emission preprocessor based on configuration. From 392234465265137ff98024e020e77cb29a431376 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Fri, 3 Oct 2025 15:02:29 +0000 Subject: [PATCH 115/132] add logging for restarts directory creation --- ush/python/pygfs/task/nexus_emission.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ush/python/pygfs/task/nexus_emission.py b/ush/python/pygfs/task/nexus_emission.py index 5ef2d27b969..632cad4f264 100644 --- a/ush/python/pygfs/task/nexus_emission.py +++ b/ush/python/pygfs/task/nexus_emission.py @@ -276,6 +276,8 @@ def initialize(self) -> None: #create a directory in the self.task_config.DATA/Restarts os.makedirs(os.path.join(self.task_config.DATA, 'Restarts'), exist_ok=True) + logger.info(f"Created Restarts directory: {os.path.join(self.task_config.DATA, 'Restarts')}") + @logit(logger) def execute(self) -> None: From 47e0e42fd52338a4bea62204da9a8abcff190679 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Mon, 6 Oct 2025 14:45:48 +0000 Subject: [PATCH 116/132] add nexus and gsi_utils for gcafs build_compute --- dev/workflow/build_opts.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dev/workflow/build_opts.yaml b/dev/workflow/build_opts.yaml index 38b6d4cde59..4815ab25590 100644 --- a/dev/workflow/build_opts.yaml +++ b/dev/workflow/build_opts.yaml @@ -32,6 +32,8 @@ systems: - "gefs_ww3_prepost" gcafs: - "gcafs_model" + - "nexus" + - "gsi_utils" build: gfs_model: command: "./build_ufs.sh -e gfs_model.x" @@ -97,3 +99,8 @@ build: command: "./build_gdas.sh" cores: 40 walltime: "01:45:00" + + nexus: + command: "./build_nexus.sh" + cores: 8 + walltime: "00:20:00" From 1132ef8e794fd46e17cbfe0a0999e733b4f6a9c6 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Mon, 6 Oct 2025 14:48:36 +0000 Subject: [PATCH 117/132] update nexus restart data directory with updated nexus --- parm/chem/nexus_emission.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/chem/nexus_emission.yaml.j2 b/parm/chem/nexus_emission.yaml.j2 index e5e2c42cf41..f97133a45cc 100644 --- a/parm/chem/nexus_emission.yaml.j2 +++ b/parm/chem/nexus_emission.yaml.j2 @@ -12,4 +12,4 @@ nexus_emission: {% for fileout in FINAL_OUTPUT %} - ["{{ DATA }}/{{ fileout }}", "{{ COMOUT_CHEM_INPUT }}/"] {% endfor %} - - ["{{ DATA }}/{{ RestartFile }}", "{{ COMOUT_CHEM_RESTART }}/"] + - ["{{ DATA }}/Restarts/{{ RestartFile }}", "{{ COMOUT_CHEM_RESTART }}/"] From c3678d2cb9de6398e31b0fbc95ec32107a24fe9b Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Mon, 6 Oct 2025 18:41:06 +0000 Subject: [PATCH 118/132] update nexus hash --- sorc/nexus.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/nexus.fd b/sorc/nexus.fd index a06deeb4d23..aa90ddf17be 160000 --- a/sorc/nexus.fd +++ b/sorc/nexus.fd @@ -1 +1 @@ -Subproject commit a06deeb4d231094895b5b955fb10f4e799ddf8c8 +Subproject commit aa90ddf17be9e0fd4e7795553926d3ade40205e6 From 7c78a09cb0522450d87e903afa2bc33889c2ad80 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Tue, 7 Oct 2025 13:42:01 +0000 Subject: [PATCH 119/132] upating nexus hash --- sorc/nexus.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/nexus.fd b/sorc/nexus.fd index aa90ddf17be..a06deeb4d23 160000 --- a/sorc/nexus.fd +++ b/sorc/nexus.fd @@ -1 +1 @@ -Subproject commit aa90ddf17be9e0fd4e7795553926d3ade40205e6 +Subproject commit a06deeb4d231094895b5b955fb10f4e799ddf8c8 From 5301ccab39ace0299e2bcb57a39c6c31277dc73e Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Tue, 7 Oct 2025 14:05:00 +0000 Subject: [PATCH 120/132] update nexus hash --- sorc/nexus | 1 + 1 file changed, 1 insertion(+) create mode 160000 sorc/nexus diff --git a/sorc/nexus b/sorc/nexus new file mode 160000 index 00000000000..a06deeb4d23 --- /dev/null +++ b/sorc/nexus @@ -0,0 +1 @@ +Subproject commit a06deeb4d231094895b5b955fb10f4e799ddf8c8 From bc960363d2302e8c8d7f889d453e8c79370ff382 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Tue, 14 Oct 2025 19:41:08 +0000 Subject: [PATCH 121/132] Update GCAFS documentation and resource configuration --- dev/parm/config/gcafs/config.resources.URSA | 4 + docs/source/gcafs.rst | 216 ++++++++++++++++---- 2 files changed, 177 insertions(+), 43 deletions(-) diff --git a/dev/parm/config/gcafs/config.resources.URSA b/dev/parm/config/gcafs/config.resources.URSA index ae555e97489..34bf80003a6 100644 --- a/dev/parm/config/gcafs/config.resources.URSA +++ b/dev/parm/config/gcafs/config.resources.URSA @@ -43,6 +43,10 @@ case ${step} in export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) fi ;; + + "offlineanl") + export memory="240GB" + ;; "eupd") case "${CASE}" in diff --git a/docs/source/gcafs.rst b/docs/source/gcafs.rst index 2b0c1971c44..d2bceec3b1c 100644 --- a/docs/source/gcafs.rst +++ b/docs/source/gcafs.rst @@ -1,6 +1,6 @@ -===================================== -Global Chemistry and Aerosol Forecast -===================================== +==================================================== +Global Chemistry and Aerosol Forecast System (GCAFS) +==================================================== Overview -------- @@ -14,7 +14,9 @@ Key Features * Interactive GOCART aerosol module for forecasting dust, sea salt, sulfate, black carbon, and organic carbon * Optional full atmospheric chemistry with gas-phase and heterogeneous reactions -* Integration with biomass burning emissions sources (QFED, GBBEPX) +* Integration with biomass burning emissions sources (QFED, GBBEPx) +* NEXUS emissions preprocessing system for anthropogenic and biogenic sources +* Support for multiple emission inventories (CEDS, HTAP, CAMS) * Aerosol-radiation-cloud interactions * Optional aerosol data assimilation @@ -62,6 +64,45 @@ The GCAFS workflow includes these main tasks: The workflow is managed by the Rocoto workflow manager, with tasks defined in the ``workflow/rocoto/gcafs_tasks.py`` file. +Configuration Files +------------------ + +GCAFS configuration is managed through several key files in the ``parm/config/gcafs/`` directory: + +### config.aero.j2 + +The primary configuration file for aerosol settings, containing: + +**Aerosol Model Settings:** + +.. code-block:: bash + + export AERO_INPUTS_DIR="/path/to/aerosol/data" # Base directory for aerosol input data + export AERO_CONFIG_DIR="${PARMgfs}/ufs/gocart" # GOCART configuration files + export fscav_aero="'*:0.3','so2:0.0',..." # Convective scavenging factors + export dnats_aero=2 # Number of diagnostic tracers + +**Fire Emissions Settings:** + +.. code-block:: bash + + export AERO_EMIS_FIRE="gbbepx" # Fire dataset: gbbepx, qfed, none + export AERO_EMIS_FIRE_VERSION="061" # Dataset version + export AERO_EMIS_FIRE_HIST=1 # Historical (1) vs NRT (0) + +**NEXUS Emissions Settings:** + +.. code-block:: bash + + export NEXUS_CONFIG="gocart" # NEXUS configuration set + export NEXUS_TSTEP=3600 # Time step (seconds) + export NEXUS_DO_CEDS2019=.true. # Enable CEDS 2019 + export NEXUS_DO_HTAPv2=.true. # Enable HTAP v2 + export NEXUS_DO_CAMS=.false. # Enable CAMS + +These settings are processed as Jinja2 templates, allowing for experiment-specific customization +through template variables like ``{{ NEXUS_CONFIG | default('gocart') }}``. + Emissions Preprocessing ----------------------- @@ -72,48 +113,97 @@ The ``prep_emissions`` task is a critical component of the GCAFS workflow that p This task performs several important functions: 1. **Configuration Generation**: Creates customized GOCART configuration files from templates -2. **Emissions File Preparation**: Processes and prepares emissions data files -3. **Historical Data Handling**: Retrieves historical fire emissions when needed -4. **Fire Emissions Selection**: Configures the selected biomass burning emissions source (QFED/GBBEPx) -5. **Template Variable Processing**: Processes all template variables in the configuration files +2. **Fire Emissions Processing**: Handles biomass burning emissions from QFED or GBBEPx datasets +3. **NEXUS Preprocessing**: Processes anthropogenic and biogenic emissions through the NEXUS system +4. **Emissions File Preparation**: Generates model-ready emissions data files +5. **Historical Data Handling**: Retrieves historical emissions when needed for testing or spin-up +6. **Template Variable Processing**: Processes all template variables in the configuration files The task is implemented in ``ush/python/pygfs/task/aero_emissions.py`` as the ``AerosolEmissions`` class. -### Detailed Workflow +### Fire Emissions Configuration -When the ``prep_emissions`` task runs, it follows these steps: +GCAFS supports multiple biomass burning emission datasets that can be configured through the ``config.aero`` file: -1. **Initialization**: - ```python - def initialize(self): - # Parse the YAML template for chemistry emissions - yaml_template = os.path.join(self.task_config.HOMEgfs, 'parm/chem/chem_emission.yaml.j2') - yamlvars = parse_j2yaml(path=yaml_template) - self.task_config.append(yamlvars) - ``` +**Available Fire Emission Datasets:** - This loads the base configuration template and merges it with the task configuration. +* **GBBEPx** (Global Biomass Burning Emissions Product): NOAA/NWS operational fire emissions +* **QFED** (Quick Fire Emission Dataset): NASA fire emissions with near-real-time updates +* **None**: Disable fire emissions entirely +**Configuration Options:** -2. **Historical Fire Emission Handling**: - ```python - if self.task_config.fire_emissions == 'historical': - # Handle historical fire emissions - self.task_config.fire_emissions = 'historical' - self.task_config.fire_emissions_file = os.path.join(self.task_config.HOMEgfs, 'parm/chem/historical_fire_emissions.txt') - ``` +.. code-block:: bash - This sets up the task to use historical fire emissions data if specified. + # Select fire emissions dataset + export AERO_EMIS_FIRE="gbbepx" # Options: gbbepx, qfed, none + export AERO_EMIS_FIRE_VERSION="061" # Dataset version + export AERO_EMIS_FIRE_HIST=1 # Use historical (1) or near-real-time (0) + + # Directories for emissions data + export FIRE_EMIS_NRT_DIR="" # Near-real-time data location + export FIRE_EMIS_DIR="" # Historical data location -3. **Fire Emission Configuration**: - ```python - if self.task_config.fire_emissions == 'qfed': - # Configure QFED emissions - self.task_config.fire_emissions = 'qfed' - self.task_config.fire_emissions_file = os.path.join(self.task_config.HOMEgfs, 'parm/chem/qfed_fire_emissions.txt') - ``` +### NEXUS Emissions Preprocessing + +NEXUS (Next-generation Emissions eXchange Utility System) preprocesses anthropogenic and biogenic emissions from multiple global inventories: + +**Supported Emission Inventories:** + +* **CEDS** (Community Emissions Data System): Global anthropogenic emissions (2019/2024 versions) +* **HTAP** (Hemispheric Transport of Air Pollution): Regional high-resolution emissions (v2/v3) +* **CAMS** (Copernicus Atmosphere Monitoring Service): European reanalysis emissions +* **MEGAN** (Model of Emissions of Gases and Aerosols from Nature): Biogenic emissions (future) + +**NEXUS Configuration:** + +.. code-block:: bash + + # NEXUS system configuration + export NEXUS_CONFIG="gocart" # Configuration set (gocart, none) + export NEXUS_TSTEP=3600 # Time step in seconds + + # Grid specification (0.25-degree global) + export NEXUS_NX=1440 # Longitude points + export NEXUS_NY=720 # Latitude points + + # Enable/disable emission inventories + export NEXUS_DO_CEDS2019=.true. # CEDS 2019 emissions + export NEXUS_DO_CEDS2024=.false. # CEDS 2024 emissions + export NEXUS_DO_HTAPv2=.true. # HTAP v2 emissions + export NEXUS_DO_CAMS=.false. # CAMS emissions + +### Emission Dataset Details + +**Fire Emissions:** + +* **GBBEPx (Global Biomass Burning Emissions Product)**: + - Operational NOAA/NWS fire emissions based on VIIRS satellite data + - Near-real-time updates with ~6-hour latency + - Includes wildfire, agricultural burning, and prescribed burns + +* **QFED (Quick Fire Emission Dataset)**: + - NASA fire emissions using MODIS satellite observations + - Available in near-real-time and historical versions + - High spatial resolution with detailed speciation + +**Anthropogenic/Biogenic Emissions:** + +* **CEDS (Community Emissions Data System)**: + - Global gridded emissions inventory (1750-2019/2024) + - Anthropogenic sources: energy, industry, transport, residential, agriculture + - Species: SO2, NOx, CO, NH3, black carbon, organic carbon, PM2.5 + +* **HTAP (Hemispheric Transport of Air Pollution)**: + - Regional high-resolution emissions for Europe, Asia, North America + - Focuses on transboundary air pollution + - Complements CEDS with finer spatial detail + +* **CAMS (Copernicus Atmosphere Monitoring Service)**: + - European Centre reanalysis emissions + - Consistent with meteorological fields + - Includes temporal disaggregation capabilities - This sets up the task to use QFED emissions data if specified. GOCART Configuration Files -------------------------- @@ -156,13 +246,22 @@ are replaced at runtime with values from the workflow configuration. Emissions Configuration ~~~~~~~~~~~~~~~~~~~~~~~ -External data sources for emissions are configured in: +External data sources for emissions are configured through ExtData resource files: - **ExtData.gbbepx**: GBBEPx biomass burning emissions configuration -- **ExtData.qfed**: QFED fire emissions configuration -- **ExtData.other**: Anthropogenic, biogenic, and other emission sources +- **ExtData.qfed**: QFED fire emissions configuration +- **ExtData.nexus**: NEXUS-processed anthropogenic/biogenic emissions +- **ExtData.other**: Additional emission sources (volcanic, lightning, etc.) - **ExtData.none**: Placeholder configuration when emissions are disabled +The NEXUS system processes emissions through HEMCO (Harmonized Emissions Component) configuration files: + +- **NEXUS_Config.rc**: Master configuration orchestrating all emission sources +- **HEMCO_sa_Grid.rc**: Grid definition and interpolation settings +- **HEMCO_sa_Time.rc**: Temporal scaling patterns (diurnal, weekly, seasonal) +- **HEMCO_sa_Spec.rc**: Species mapping between inventories and GOCART tracers +- **HEMCO_sa_Diag.rc**: Diagnostic output configuration + To modify the aerosol configuration, edit these files or create custom versions in your experiment directory. The file ``gocart_tracer.list`` defines the complete set of aerosol tracers used in the model. @@ -174,7 +273,7 @@ The ExtData configuration files specify how external data sources are imported i .. code-block:: none # Import Name | Units | Clim | Regrid | Time Template | Offset | Scale | Var on File | File Template - OC_BIOMASS NA N Y %y4-%m2-%d2t12:00:00 none 0.7778 biomass ExtData/nexus/QFED/%y4/%m2/qfed2.emis_oc.006.%y4%m2%d2.nc4 + OC_BIOMASS NA N Y %y4-%m2-%d2t12:00:00 none 0.7778 OC ChemInput/FIRE_EMIS.%y4%m2%d2.nc4 Field descriptions: @@ -203,6 +302,24 @@ For example, in the QFED configuration: This imports SO2 emissions from QFED into the SU_BIOMASS variable, using a scale factor of 0.7778, from files with a date-based naming pattern. +### NEXUS ExtData Configuration + +The NEXUS-processed emissions are configured through **ExtData.nexus**, which handles anthropogenic and biogenic emissions from multiple inventories. Example entries: + +.. code-block:: none + + # Anthropogenic SO2 from CEDS + SU_ANTHRO NA N Y %y4-%m2-%d2t12:00:00 none none so2_anthro ExtData/nexus/CEDS/%y4/CEDS.emis_so2.%y4%m2%d2.nc4 + + # Black carbon from HTAP + BC_ANTHRO NA N Y %y4-%m2-%d2t12:00:00 none none bc_anthro ExtData/nexus/HTAP/%y4/HTAP.emis_bc.%y4%m2%d2.nc4 + +The NEXUS preprocessing system generates these files by: + +1. Reading emission inventories (CEDS, HTAP, CAMS) from ``NEXUS_INPUT_DIR`` +2. Applying temporal scaling patterns (diurnal, weekly, seasonal) +3. Regridding to the model resolution +4. Outputting model-ready netCDF files with standardized variable names AERO_HISTORY.rc File Details ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -275,12 +392,25 @@ corresponding frequency parameters are properly set in your workflow. Output Products --------------- -GCAFS produces standard meteorological outputs plus aerosol fields including: +GCAFS produces standard meteorological outputs plus comprehensive aerosol fields including: +**Core Aerosol Fields:** * Aerosol mass concentrations (dust, sea salt, sulfate, black carbon, organic carbon) -* Aerosol optical depth fields +* Aerosol optical depth fields at multiple wavelengths * PM2.5 and PM10 concentrations +* Aerosol extinction coefficients + +**Process-Specific Diagnostics:** +* Emission fields from fires and anthropogenic sources (when NEXUS diagnostics enabled) +* Dry and wet deposition fluxes +* Optical properties (single scattering albedo, asymmetry parameter) +* Column-integrated aerosol mass + +**Advanced Outputs:** +* 3D aerosol concentrations on model levels +* Aerosol number concentrations * Full chemical species concentrations when running with chemistry enabled +* NEXUS diagnostic emissions for verification -Output frequency is controlled by the standard global-workflow configuration options -in the same manner as GFS. +Output frequency and collections are controlled through the ``AERO_HISTORY.rc`` configuration file, +with standard global-workflow configuration options determining the base output settings. From 709e4357cb1485af1aedcef9e099b1043d5554ea Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Tue, 14 Oct 2025 19:42:04 +0000 Subject: [PATCH 122/132] fix issue with historical gbbepx data --- ush/python/pygfs/task/chem_fire_emission.py | 88 ++++++++++++++------- 1 file changed, 61 insertions(+), 27 deletions(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index b585ded3902..38ddbd27c4d 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -6,6 +6,7 @@ import datetime import xarray as xr import numpy as np +import shutil from logging import getLogger from typing import Dict, Any, Union, List from dateutil.rrule import DAILY, rrule @@ -17,7 +18,6 @@ to_timedelta, WorkflowException, Executable, which) -from pprint import pprint logger = getLogger(__name__.split('.')[-1]) @@ -127,9 +127,14 @@ def initialize(self) -> None: if not os.path.exists(fire_emission_template): raise WorkflowException(f"Fire emission template file not found: {fire_emission_template}") - AERO_EMIS_FIRE_DIR = os.path.join(aero_inputs_dir, - "nexus", - aero_emis_fire.upper()) + if os.path.exists(self.task_config.FIRE_EMIS_DIR): + logger.info(f"AERO_EMIS_FIRE_DIR already set: {self.task_config.FIRE_EMIS_DIR}") + AERO_EMIS_FIRE_DIR = self.task_config.FIRE_EMIS_DIR + else: + logger.info("AERO_EMIS_FIRE_DIR not set, constructing from AERO_INPUTS_DIR and AERO_EMIS_FIRE") + AERO_EMIS_FIRE_DIR = os.path.join(aero_inputs_dir, + "nexus", + aero_emis_fire.upper()) logger.info(f'Final AERO_EMIS_FIRE_DIR: {AERO_EMIS_FIRE_DIR}') @@ -193,7 +198,6 @@ def initialize(self) -> None: dt.strftime("FIRE_EMIS_%Y%m%d.nc") ) - # pprint(self.task_config) # Debug output for chemistry history directory logger.info(f"Outputing files prescribed to {self.task_config.COMOUT_CHEM_INPUT}") tmpl_dict = { @@ -374,10 +378,21 @@ def _find_gbbepx_nrt_fires(self, NRT_DIRECTORY: str) -> List[str]: matching_files.append(full_path) logger.debug(f"Found GBBEPx NRT fire file: {full_path}") - return matching_files + # Remove duplicates while preserving order (safety check) + unique_files = [] + seen = set() + for file_path in matching_files: + if file_path not in seen: + unique_files.append(file_path) + seen.add(file_path) + + if len(unique_files) < len(matching_files): + logger.info(f"Found {len(unique_files)} unique GBBEPx NRT files (removed {len(matching_files) - len(unique_files)} duplicates)") + + return unique_files @logit(logger) - def _find_gbbepx_files(self, dates, version='v5r0'): + def _find_gbbepx_files(self, dates, aero_emis_fire_dir=None, version='v5r0'): """Find GBBEPx files for the given date Parameters @@ -406,7 +421,7 @@ def _find_gbbepx_files(self, dates, version='v5r0'): # Find all possible files for mon in months: try: - emis_file_dir = self.task_config.AERO_EMIS_FIRE_DIR + emis_file_dir = aero_emis_fire_dir if not os.path.exists(emis_file_dir): logger.warning(f"Directory does not exist: {emis_file_dir}") continue @@ -458,7 +473,16 @@ def _find_gbbepx_files(self, dates, version='v5r0'): except (FileNotFoundError, PermissionError) as e: logger.warning(f"Error accessing directory {emis_file_dir}: {e}") - return files_found + # Remove duplicates while preserving order + unique_files = [] + seen = set() + for file_path in files_found: + if file_path not in seen: + unique_files.append(file_path) + seen.add(file_path) + + logger.info(f"Found {len(unique_files)} unique GBBEPx files (removed {len(files_found) - len(unique_files)} duplicates)") + return unique_files @logit(logger) def _find_qfed_files(self, dates, vars, version='061', aero_emis_fire_dir=None): @@ -535,6 +559,18 @@ def _find_qfed_files(self, dates, vars, version='061', aero_emis_fire_dir=None): if not files_found: logger.warning(f"No QFED files found for dates {date_strings} and variables {vars}") + else: + # Remove duplicates while preserving order + unique_files = [] + seen = set() + for file_path in files_found: + if file_path not in seen: + unique_files.append(file_path) + seen.add(file_path) + + if len(unique_files) < len(files_found): + logger.info(f"Found {len(unique_files)} unique QFED files (removed {len(files_found) - len(unique_files)} duplicates)") + files_found = unique_files return files_found @@ -555,6 +591,10 @@ def GBBEPx_to_COARDS(self, fname: Union[str, os.PathLike]) -> xr.Dataset: logger.info(f"Converting {fname} to COARDS format") f = xr.open_dataset(fname, decode_cf=False) f = f[['OC', 'BC', 'SO2', 'NOx', 'CO', 'NH3']] + if 'time' in f.dims and 'lon' in f.dims and 'lat' in f.dims: + logger.info("File already in COARDS format") + return None # Already in COARDS format + # Handle time dimension if 'Time' in f.dims: f = f.rename({"Time": 'time'}) @@ -789,37 +829,31 @@ def _process_gbbepx_files(self, workdir: str) -> List[str]: else: logger.warning("No raw GBBEPx files found for non-historical processing") else: + logger.info(f"RAWFILES for historical GBBEPx processing: {self.task_config.rawfiles}") for forecast_date, date_file in zip(self.forecast_dates, self.task_config.rawfiles): date_str = forecast_date.strftime('%Y%m%d') - logger.info(f"Processing GBBEPx files for date {date_str}") + logger.info(f"Processing GBBEPx files for date {date_str} from file {date_file}") - # Filter files for this date - implement date filtering if needed - date_files = [] - for file in self.task_config.rawfiles: - # Add logic here to filter files by date if needed - date_files.append(file) + # Create output filename with date + outfile_name = f"FIRE_EMIS_{date_str}.nc" + outfile = os.path.join(workdir, outfile_name) - if date_file: - # Process files for this date - ds = self.GBBEPx_to_COARDS(date_file) # Use the first file for this date - - # Create output filename with date - outfile_name = f"FIRE_EMIS_{date_str}.nc" - outfile = os.path.join(workdir, outfile_name) + ds = self.GBBEPx_to_COARDS(date_file) + if ds is None: # file was already in COARDS format + logger.info(f"File {date_file} already in COARDS format, copying to {outfile}") + shutil.copy(date_file, outfile) + else: # Save the processed dataset comp = dict(zlib=True, complevel=2) encoding = {var: comp for var in ds.data_vars} ds.to_netcdf(outfile, encoding=encoding, unlimited_dims=['time']) logger.info(f"Processed emission file saved to {outfile}") - # Add to processed files list - processed_files.append(outfile) - # Close dataset ds.close() - else: - logger.warning(f"No GBBEPx files found for date {date_str}") + + processed_files.append(outfile) return processed_files From 1fab3b08aed90a4e43fb2dc905fb8072f12133b3 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Tue, 14 Oct 2025 19:42:45 +0000 Subject: [PATCH 123/132] make gbbepx default and add historical FIRE_EMIS_DIR --- dev/parm/config/gcafs/config.aero.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/parm/config/gcafs/config.aero.j2 b/dev/parm/config/gcafs/config.aero.j2 index 0ad6f2ee8fc..a4f44db94ea 100644 --- a/dev/parm/config/gcafs/config.aero.j2 +++ b/dev/parm/config/gcafs/config.aero.j2 @@ -69,7 +69,7 @@ export dnats_aero=2 # none: Disable fire emissions. # Used in prep_emissions scripts to fetch/interpolate data to model grid. #--------------------------------------------------------------------------------------------------- -export AERO_EMIS_FIRE="qfed" +export AERO_EMIS_FIRE="gbbepx" export AERO_EMIS_FIRE_VERSION="061" # Version of the selected fire emissions dataset (e.g., for QFEDv2.5, version 061). @@ -84,7 +84,7 @@ export AERO_EMIS_FIRE_HIST=1 # Use historical fire emissions | 1 = true 0 = fals # Processed by scripts like exglobal_prep_emissions.py to generate input files for GOCART. #--------------------------------------------------------------------------------------------------- export FIRE_EMIS_NRT_DIR="" #TODO: set to DCOM for WCOSS2 "${DCOMROOT}/YYYYMMDD/firewx" # Directory containing NRT fire emissions - +export FIRE_EMIS_DIR="" # Directory containing historical fire emissions #=============================================================================== From 0d2ce2f9647a68d4080f3399529c55c24d0abfcd Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Tue, 14 Oct 2025 19:51:27 +0000 Subject: [PATCH 124/132] pycodestyle changes --- ush/python/pygfs/task/chem_fire_emission.py | 14 +++++++------- ush/python/pygfs/task/nexus_emission.py | 3 +-- ush/python/pygfs/task/oceanice_products.py | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 38ddbd27c4d..e876f22572d 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -133,8 +133,8 @@ def initialize(self) -> None: else: logger.info("AERO_EMIS_FIRE_DIR not set, constructing from AERO_INPUTS_DIR and AERO_EMIS_FIRE") AERO_EMIS_FIRE_DIR = os.path.join(aero_inputs_dir, - "nexus", - aero_emis_fire.upper()) + "nexus", + aero_emis_fire.upper()) logger.info(f'Final AERO_EMIS_FIRE_DIR: {AERO_EMIS_FIRE_DIR}') @@ -385,10 +385,10 @@ def _find_gbbepx_nrt_fires(self, NRT_DIRECTORY: str) -> List[str]: if file_path not in seen: unique_files.append(file_path) seen.add(file_path) - + if len(unique_files) < len(matching_files): logger.info(f"Found {len(unique_files)} unique GBBEPx NRT files (removed {len(matching_files) - len(unique_files)} duplicates)") - + return unique_files @logit(logger) @@ -480,7 +480,7 @@ def _find_gbbepx_files(self, dates, aero_emis_fire_dir=None, version='v5r0'): if file_path not in seen: unique_files.append(file_path) seen.add(file_path) - + logger.info(f"Found {len(unique_files)} unique GBBEPx files (removed {len(files_found) - len(unique_files)} duplicates)") return unique_files @@ -567,7 +567,7 @@ def _find_qfed_files(self, dates, vars, version='061', aero_emis_fire_dir=None): if file_path not in seen: unique_files.append(file_path) seen.add(file_path) - + if len(unique_files) < len(files_found): logger.info(f"Found {len(unique_files)} unique QFED files (removed {len(files_found) - len(unique_files)} duplicates)") files_found = unique_files @@ -840,7 +840,7 @@ def _process_gbbepx_files(self, workdir: str) -> List[str]: ds = self.GBBEPx_to_COARDS(date_file) - if ds is None: # file was already in COARDS format + if ds is None: # file was already in COARDS format logger.info(f"File {date_file} already in COARDS format, copying to {outfile}") shutil.copy(date_file, outfile) else: diff --git a/ush/python/pygfs/task/nexus_emission.py b/ush/python/pygfs/task/nexus_emission.py index 632cad4f264..ba5d12b525b 100644 --- a/ush/python/pygfs/task/nexus_emission.py +++ b/ush/python/pygfs/task/nexus_emission.py @@ -274,11 +274,10 @@ def initialize(self) -> None: j2_renderer.save(outfile) logger.info(f"NEXUS spec file rendered successfully: written to {outfile}") - #create a directory in the self.task_config.DATA/Restarts + # create a directory in the self.task_config.DATA/Restarts os.makedirs(os.path.join(self.task_config.DATA, 'Restarts'), exist_ok=True) logger.info(f"Created Restarts directory: {os.path.join(self.task_config.DATA, 'Restarts')}") - @logit(logger) def execute(self) -> None: """Run NEXUS emission preprocessor based on configuration. diff --git a/ush/python/pygfs/task/oceanice_products.py b/ush/python/pygfs/task/oceanice_products.py index 278775ffeda..fdfd19a4243 100644 --- a/ush/python/pygfs/task/oceanice_products.py +++ b/ush/python/pygfs/task/oceanice_products.py @@ -66,7 +66,7 @@ def __init__(self, config: Dict[str, Any]) -> None: # TODO: This is a bit of a hack, but it works for now # FIXME: find a better way to provide the averaging period - avg_period = f"{forecast_hour-interval:03d} - {forecast_hour:03d}" + avg_period = f"{forecast_hour - interval:03d} - {forecast_hour:03d}" # Extend task_config with localdict localdict = AttrDict( From 73ac75234d1c43f5a946ccff0298d03b82935018 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Wed, 15 Oct 2025 13:02:38 +0000 Subject: [PATCH 125/132] updating nexus commit --- sorc/nexus | 1 - sorc/nexus.fd | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 160000 sorc/nexus diff --git a/sorc/nexus b/sorc/nexus deleted file mode 160000 index a06deeb4d23..00000000000 --- a/sorc/nexus +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a06deeb4d231094895b5b955fb10f4e799ddf8c8 diff --git a/sorc/nexus.fd b/sorc/nexus.fd index a06deeb4d23..bf36c04b75d 160000 --- a/sorc/nexus.fd +++ b/sorc/nexus.fd @@ -1 +1 @@ -Subproject commit a06deeb4d231094895b5b955fb10f4e799ddf8c8 +Subproject commit bf36c04b75d4de9929a9e78b13befd50cbd871ae From 382dcb535fb45e804774d48936f88d474935c3b2 Mon Sep 17 00:00:00 2001 From: bbakernoaa Date: Wed, 15 Oct 2025 13:07:07 +0000 Subject: [PATCH 126/132] updating nexus hash --- sorc/nexus.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/nexus.fd b/sorc/nexus.fd index bf36c04b75d..7b0772b7a64 160000 --- a/sorc/nexus.fd +++ b/sorc/nexus.fd @@ -1 +1 @@ -Subproject commit bf36c04b75d4de9929a9e78b13befd50cbd871ae +Subproject commit 7b0772b7a649e073902622a3a4250eff428b6663 From 57edce1113b868bedcbee2e57c48253accb167ab Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 15 Oct 2025 11:46:02 -0400 Subject: [PATCH 127/132] Update dev/parm/config/gcafs/config.aero.j2 Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- dev/parm/config/gcafs/config.aero.j2 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dev/parm/config/gcafs/config.aero.j2 b/dev/parm/config/gcafs/config.aero.j2 index a4f44db94ea..fb55f38f2ef 100644 --- a/dev/parm/config/gcafs/config.aero.j2 +++ b/dev/parm/config/gcafs/config.aero.j2 @@ -70,10 +70,9 @@ export dnats_aero=2 # Used in prep_emissions scripts to fetch/interpolate data to model grid. #--------------------------------------------------------------------------------------------------- export AERO_EMIS_FIRE="gbbepx" -export AERO_EMIS_FIRE_VERSION="061" - # Version of the selected fire emissions dataset (e.g., for QFEDv2.5, version 061). # Determines which historical or NRT files to load from input directories. +export AERO_EMIS_FIRE_VERSION="061" #--------------------------------------------------------------------------------------------------- export AERO_EMIS_FIRE_HIST=1 # Use historical fire emissions | 1 = true 0 = false From 14f5674fa84fb0557599fe724cdc2ddb506b1f2e Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 15 Oct 2025 11:46:22 -0400 Subject: [PATCH 128/132] Update docs/source/gcafs.rst Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- docs/source/gcafs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/gcafs.rst b/docs/source/gcafs.rst index d2bceec3b1c..ba4be17c0b3 100644 --- a/docs/source/gcafs.rst +++ b/docs/source/gcafs.rst @@ -79,7 +79,7 @@ The primary configuration file for aerosol settings, containing: export AERO_INPUTS_DIR="/path/to/aerosol/data" # Base directory for aerosol input data export AERO_CONFIG_DIR="${PARMgfs}/ufs/gocart" # GOCART configuration files - export fscav_aero="'*:0.3','so2:0.0',..." # Convective scavenging factors + export fscav_aero="'*:0.3','so2:0.0',..." # Convective scavenging factors export dnats_aero=2 # Number of diagnostic tracers **Fire Emissions Settings:** From a0a436bd2eab377fb5e0a50f51314e257bc856e3 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 15 Oct 2025 16:22:44 -0400 Subject: [PATCH 129/132] Update scripts/exglobal_stage_ic.py Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- scripts/exglobal_stage_ic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index 1de3507b8bf..6c65edce9b7 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -30,7 +30,7 @@ def main(): for key in keys: # Make sure OCNRES is three digits if key == "OCNRES": - stage.task_config.OCNRES = f"{stage.task_config.OCNRES:03d}" # noqa: E203 + stage.task_config.OCNRES = f"{stage.task_config.OCNRES:03d}" stage_dict[key] = stage.task_config[key] # Also import all COM* directory and template variables From 31316c9f0bbbb9839290b142ab4bff76844c79fc Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 15 Oct 2025 16:22:54 -0400 Subject: [PATCH 130/132] Update ush/python/pygfs/task/chem_fire_emission.py Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- ush/python/pygfs/task/chem_fire_emission.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index e876f22572d..46d313ab36a 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -5,7 +5,6 @@ import fnmatch import datetime import xarray as xr -import numpy as np import shutil from logging import getLogger from typing import Dict, Any, Union, List From 069be650d8e1c2eca1da6f710e986b68d73088da Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 15 Oct 2025 16:24:18 -0400 Subject: [PATCH 131/132] Update ush/python/pygfs/task/chem_fire_emission.py Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- ush/python/pygfs/task/chem_fire_emission.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/python/pygfs/task/chem_fire_emission.py b/ush/python/pygfs/task/chem_fire_emission.py index 46d313ab36a..a3059724632 100644 --- a/ush/python/pygfs/task/chem_fire_emission.py +++ b/ush/python/pygfs/task/chem_fire_emission.py @@ -2,7 +2,6 @@ import os import re -import fnmatch import datetime import xarray as xr import shutil From 87f320b3f2fec9cc68b978f7f13d88879c2e01e0 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Wed, 15 Oct 2025 16:24:45 -0400 Subject: [PATCH 132/132] Update dev/ush/compare_f90nml.py Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- dev/ush/compare_f90nml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/ush/compare_f90nml.py b/dev/ush/compare_f90nml.py index a3d47e76992..2aa0e745db2 100755 --- a/dev/ush/compare_f90nml.py +++ b/dev/ush/compare_f90nml.py @@ -77,7 +77,7 @@ def _print_diffs(diff_dict: Dict) -> None: max_len = len(max(diff_dict[path], key=len)) for kk in diff_dict[path].keys(): items = diff_dict[path][kk] - print(f"{kk:>{max_len + 2}} : {' | '.join(map(str, diff_dict[path][kk]))}") # noqa: E226 + print(f"{kk:>{max_len + 2}} : {' | '.join(map(str, diff_dict[path][kk]))}") _print_diffs(result)