diff --git a/dev/ci/cases/pr/C96_gcafs_cycled.yaml b/dev/ci/cases/pr/C96_gcafs_cycled.yaml index 58d9c8ab5f3..a51d4277648 100644 --- a/dev/ci/cases/pr/C96_gcafs_cycled.yaml +++ b/dev/ci/cases/pr/C96_gcafs_cycled.yaml @@ -15,9 +15,7 @@ experiment: yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/gcafs_cycled_defaults_ci.yaml skip_ci_on_hosts: - - orion - gaeac5 - - hercules - awsepicglobalworkflow - ursa diff --git a/dev/parm/config/gcafs/config.fetch b/dev/parm/config/gcafs/config.fetch index 7cce1d1be27..bcd65154126 100644 --- a/dev/parm/config/gcafs/config.fetch +++ b/dev/parm/config/gcafs/config.fetch @@ -17,7 +17,12 @@ if [[ "${PDY}${cyc}" -gt "${SDATE}" ]]; then gdas_version="prod" fi export gdas_version - export FETCH_YAML_TMPL_LIST="${PARMgfs}/fetch/${NET}_${APP}_gdas-anl.yaml.j2,${PARMgfs}/fetch/${NET}_${APP}_gdas-dtfanl.yaml.j2" + if [[ "${machine}" == "ORION" || "${machine}" == "HERCULES" ]]; then + FETCH_YAML_TMPL_LIST="${PARMgfs}/fetch/${NET}_${APP}_gdas-anl_msu.yaml.j2,${PARMgfs}/fetch/${NET}_${APP}_gdas-dtfanl_msu.yaml.j2" + else + FETCH_YAML_TMPL_LIST="${PARMgfs}/fetch/${NET}_${APP}_gdas-anl.yaml.j2,${PARMgfs}/fetch/${NET}_${APP}_gdas-dtfanl.yaml.j2" + fi + export FETCH_YAML_TMPL_LIST else # fetch based on first cycle # Determine start type diff --git a/dev/ush/fetch_gdas_for_gcafs.sh b/dev/ush/fetch_gdas_for_gcafs.sh new file mode 100755 index 00000000000..b8e1513493f --- /dev/null +++ b/dev/ush/fetch_gdas_for_gcafs.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# This script fetches GDAS data for GCAFS from HPSS +# and places it in a new tar file. +set -e +set -u + +# Get arguments from command line +if [[ "${#}" -ne 2 ]]; then + echo "Usage: ${0} " + exit 1 +fi +YYYYMMDDHH=${1} +OUTPUT_DIR=${2} +# Create output directory if it doesn't exist +mkdir -p "${OUTPUT_DIR}" +mkdir -p "${OUTPUT_DIR}/tmp" + +# determine GDAS version based on date +if [[ "${YYYYMMDDHH}" -ge "2022112900" ]]; then + gdas_version="v16.3" +elif [[ "${YYYYMMDDHH}" -ge "2022062700" ]]; then + gdas_version="v16.2" +else + gdas_version="prod" +fi + +# break date and time into components +cycle_Y=${YYYYMMDDHH:0:4} +cycle_M=${YYYYMMDDHH:4:2} +cycle_D=${YYYYMMDDHH:6:2} +cyc=${YYYYMMDDHH:8:2} +# cycle_YM is YYYYMM +cycle_YM="${cycle_Y}${cycle_M}" +# PDY is YYYYMMDD +PDY="${cycle_Y}${cycle_M}${cycle_D}" + +# HPSS path for the two tar files +hpss_path_root="/NCEPPROD/hpssprod/runhistory/rh${cycle_Y}/${cycle_YM}/${PDY}" +hpss_file_nc="com_gfs_${gdas_version}_gdas.${PDY}_${cyc}.gdas_nc.tar" +hpss_file_restart="com_gfs_${gdas_version}_gdas.${PDY}_${cyc}.gdas_restart.tar" + +# get the names of the files to extract +atmanl="./gdas.${PDY}/${cyc}/atmos/gdas.t${cyc}z.atmanl.nc" +dtfanl="./gdas.${PDY}/${cyc}/atmos/gdas.t${cyc}z.dtfanl.nc" + +# Fetch the tar files from HPSS +cd "${OUTPUT_DIR}/tmp" + +htar -xvf "${hpss_path_root}/${hpss_file_nc}" "${atmanl}" +htar -xvf "${hpss_path_root}/${hpss_file_restart}" "${dtfanl}" + +# create the output tar files +echo "creating output tar files" +tar cvf "${hpss_file_nc}" "${atmanl}" +tar cvf "${hpss_file_restart}" "${dtfanl}" + +# Move the tar files to the output directory +echo "moving tar files to ${OUTPUT_DIR}" +mv "${hpss_file_nc}" "${OUTPUT_DIR}/" +mv "${hpss_file_restart}" "${OUTPUT_DIR}/" + +# Clean up temporary directory +rm -rf "${OUTPUT_DIR}/tmp" +echo "GDAS data for ${YYYYMMDDHH} has been successfully fetched and stored in ${OUTPUT_DIR}." +# End of script +exit 0 diff --git a/parm/fetch/gcafs_ATMA_gdas-anl_msu.yaml.j2 b/parm/fetch/gcafs_ATMA_gdas-anl_msu.yaml.j2 new file mode 100644 index 00000000000..0c9f87bee86 --- /dev/null +++ b/parm/fetch/gcafs_ATMA_gdas-anl_msu.yaml.j2 @@ -0,0 +1,12 @@ +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set cycle_Y = current_cycle | strftime("%Y") %} +{% set cycle_YM = current_cycle | strftime("%Y%m") %} +{% set cycle_YMD = current_cycle | to_YMD %} +{% set cycle_HH = current_cycle | strftime("%H") %} +target: + tarball: "/work/noaa/da/cmartin/june2025/gcafs/gcafs-tars/com_gfs_{{ gdas_version }}_gdas.{{ cycle_YMD }}_{{ cycle_HH }}.gdas_nc.tar" + on_hpss: False + contents: + # Atmospheric analysis + - ./gdas.{{ cycle_YMD }}/{{ cycle_HH }}/atmos/gdas.t{{ cycle_HH }}z.atmanl.nc + destination: "{{ ROTDIR }}" diff --git a/parm/fetch/gcafs_ATMA_gdas-dtfanl_msu.yaml.j2 b/parm/fetch/gcafs_ATMA_gdas-dtfanl_msu.yaml.j2 new file mode 100644 index 00000000000..ecba78fc90f --- /dev/null +++ b/parm/fetch/gcafs_ATMA_gdas-dtfanl_msu.yaml.j2 @@ -0,0 +1,12 @@ +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set cycle_Y = current_cycle | strftime("%Y") %} +{% set cycle_YM = current_cycle | strftime("%Y%m") %} +{% set cycle_YMD = current_cycle | to_YMD %} +{% set cycle_HH = current_cycle | strftime("%H") %} +target: + tarball: "/work/noaa/da/cmartin/june2025/gcafs/gcafs-tars/com_gfs_{{ gdas_version }}_gdas.{{ cycle_YMD }}_{{ cycle_HH }}.gdas_restart.tar" + on_hpss: False + contents: + # NSST analysis + - ./gdas.{{ cycle_YMD }}/{{ cycle_HH }}/atmos/gdas.t{{ cycle_HH }}z.dtfanl.nc + destination: "{{ ROTDIR }}" diff --git a/ush/python/pygfs/task/fetch.py b/ush/python/pygfs/task/fetch.py index 5a3b0d09460..cedebc9abaa 100755 --- a/ush/python/pygfs/task/fetch.py +++ b/ush/python/pygfs/task/fetch.py @@ -3,6 +3,7 @@ import os from logging import getLogger from typing import Any, Dict +import tarfile from wxflow import (Task, htar, logit, parse_j2yaml, chdir) @@ -85,12 +86,10 @@ def execute_pull_data(self, fetchdir_set: Dict[str, Any]) -> None: if on_hpss is True: # htar all files in fnames htar_obj = htar.Htar() htar_obj.xvf(tarball, f_names) - else: # tar all files in fnames - raise NotImplementedError("The fetch job does not yet support pulling from local archives") - -# with tarfile.open(dest, "w") as tar: -# for filename in f_names: -# tar.add(filename) + else: # extract from a specified tarball + with tarfile.open(tarball, "r") as tar: + members = [m for m in tar.getmembers() if m.name in f_names] + tar.extractall(members=members) # Verify all data files were extracted missing_files = [] for f in f_names: