From 318baa6c8763c21ace38052703a181c5da56826e Mon Sep 17 00:00:00 2001 From: anton-seaice Date: Mon, 29 Jun 2026 14:19:31 +1000 Subject: [PATCH 1/9] Use sys.argv for metadata --- chlorophyll/chl_climatology_and_fill.py | 10 +++----- .../generate_xml_datm_era5.py | 2 +- .../generate_xml_datm_jra55.py | 2 +- .../generate_xml_drof_jra55.py | 2 +- .../generate_bottom_roughness_polyfit.py | 10 +------- .../generate_bottom_roughness_regrid.py | 13 ++--------- .../generate_tide_amplitude.py | 12 ++-------- grid_generation/generate_vertical_grid.py | 12 +++------- mesh_generation/generate_mesh.py | 23 +++---------------- regrid_common.py | 13 ----------- .../apply_bathy_mom_restarts.py | 5 ++-- restart_modifications/remask_cpl_restart.py | 11 +++------ .../generate_rofi_pattern.py | 10 +++----- scripts_common.py | 2 +- .../make_sfc_restore_from_regridded_woa.py | 6 +++-- wombat_ic_generation/SFe_climatology.py | 10 +++----- wombat_ic_generation/co2_iaf.py | 11 +++------ wombat_ic_generation/generate_wombat_ic.py | 10 +++----- wombat_ic_generation/regrid_forcing.py | 12 +++------- 19 files changed, 43 insertions(+), 133 deletions(-) diff --git a/chlorophyll/chl_climatology_and_fill.py b/chlorophyll/chl_climatology_and_fill.py index ecd645ee..741ae1d1 100644 --- a/chlorophyll/chl_climatology_and_fill.py +++ b/chlorophyll/chl_climatology_and_fill.py @@ -70,13 +70,9 @@ def main(): input_directory = os.path.abspath(args.input_directory) output_filename = os.path.abspath(args.output_filename) - this_file = os.path.normpath(__file__) - - # Add some info about how the file was generated - runcmd = ( - f"python3 {os.path.basename(this_file)} --input-directory={input_directory} " - f"--output-filename={output_filename}" - ) + # Obtain metadata + this_file = sys.argv[0] + runcmd = f"{sys.executable} {' '.join(sys.argv)}" history_attrs = { "history": get_provenance_metadata(this_file, runcmd), diff --git a/data_stream_xml_generation/generate_xml_datm_era5.py b/data_stream_xml_generation/generate_xml_datm_era5.py index ea279f20..23eb1ec0 100644 --- a/data_stream_xml_generation/generate_xml_datm_era5.py +++ b/data_stream_xml_generation/generate_xml_datm_era5.py @@ -86,7 +86,7 @@ # Obtain metadata this_file = sys.argv[0] -runcmd = " ".join(sys.argv) +runcmd = f"{sys.executable} {' '.join(sys.argv)}" metadata_info = get_provenance_metadata(this_file, runcmd) # Add metadata diff --git a/data_stream_xml_generation/generate_xml_datm_jra55.py b/data_stream_xml_generation/generate_xml_datm_jra55.py index 91339653..b487fb07 100644 --- a/data_stream_xml_generation/generate_xml_datm_jra55.py +++ b/data_stream_xml_generation/generate_xml_datm_jra55.py @@ -62,7 +62,7 @@ # Obtain metadata this_file = sys.argv[0] -runcmd = " ".join(sys.argv) +runcmd = f"{sys.executable} {' '.join(sys.argv)}" metadata_info = get_provenance_metadata(this_file, runcmd) # Add metadata diff --git a/data_stream_xml_generation/generate_xml_drof_jra55.py b/data_stream_xml_generation/generate_xml_drof_jra55.py index 455a6253..633ece2e 100644 --- a/data_stream_xml_generation/generate_xml_drof_jra55.py +++ b/data_stream_xml_generation/generate_xml_drof_jra55.py @@ -43,7 +43,7 @@ # Obtain metadata this_file = sys.argv[0] -runcmd = " ".join(sys.argv) +runcmd = f"{sys.executable} {' '.join(sys.argv)}" metadata_info = get_provenance_metadata(this_file, runcmd) # Add metadata diff --git a/external_tidal_generation/generate_bottom_roughness_polyfit.py b/external_tidal_generation/generate_bottom_roughness_polyfit.py index 8dfc0e4a..0f298bfe 100644 --- a/external_tidal_generation/generate_bottom_roughness_polyfit.py +++ b/external_tidal_generation/generate_bottom_roughness_polyfit.py @@ -384,15 +384,7 @@ def main(): # Add provenance metadata and MD5 hashes for input files. this_file = os.path.normpath(__file__) - runcmd = ( - f"mpirun -n $PBS_NCPUS python3 {os.path.basename(this_file)} " - f"--high-res-topo-file={args.high_res_topo_file} " - f"--hgrid-file={args.hgrid_file} " - f"--topog-file={args.topog_file} " - f"--chunk-lat={args.chunk_lat} " - f"--chunk-lon={args.chunk_lon} " - f"--output={args.output}" - ) + runcmd = f"mpirun -n $PBS_NCPUS python3 {' '.join(sys.argv)} " history = get_provenance_metadata(this_file, runcmd) global_attrs = {"history": history} diff --git a/external_tidal_generation/generate_bottom_roughness_regrid.py b/external_tidal_generation/generate_bottom_roughness_regrid.py index 567ee1e0..2cad0e6f 100644 --- a/external_tidal_generation/generate_bottom_roughness_regrid.py +++ b/external_tidal_generation/generate_bottom_roughness_regrid.py @@ -396,17 +396,8 @@ def main(): print("Regridding done!") # Add provenance metadata and MD5 hashes for input files. - this_file = os.path.normpath(__file__) - runcmd = ( - f"python3 {os.path.basename(this_file)} " - f"--topog_file={args.topog_file} " - f"--hgrid_file={args.hgrid_file} " - f"--woa_intermediate_file={args.woa_intermediate_file} " - f"--output_file={args.output_file} " - f"--method={args.method} " - f"--periodic_regrid={args.periodic_regrid} " - f"--periodic_lon_laplace={args.periodic_lon_laplace}" - ) + this_file = sys.argv[0] + runcmd = f"{sys.executable} {' '.join(sys.argv)}" history = get_provenance_metadata(this_file, runcmd) global_attrs = {"history": history} diff --git a/external_tidal_generation/generate_tide_amplitude.py b/external_tidal_generation/generate_tide_amplitude.py index a707f4d9..b186ec7f 100644 --- a/external_tidal_generation/generate_tide_amplitude.py +++ b/external_tidal_generation/generate_tide_amplitude.py @@ -277,16 +277,8 @@ def main(): tideamp = tideamp.rename({"x": "xh", "y": "yh"}) # Add provenance metadata and MD5 hashes for input files. - this_file = os.path.normpath(__file__) - runcmd = ( - f"python3 {os.path.basename(this_file)} " - f"--hgrid-file={args.hgrid_file} " - f"--topog-file={args.topog_file} " - f"--method={args.method} " - f"--data-path={args.data_path} " - f"--output={args.output} " - ) - + this_file = sys.argv[0] + runcmd = f"{sys.executable} {' '.join(sys.argv)}" history = get_provenance_metadata(this_file, runcmd) global_attrs = {"history": history} diff --git a/grid_generation/generate_vertical_grid.py b/grid_generation/generate_vertical_grid.py index a310728f..0e8a702f 100644 --- a/grid_generation/generate_vertical_grid.py +++ b/grid_generation/generate_vertical_grid.py @@ -91,15 +91,9 @@ def generate_vertical_grid(H, dzd, min_dz, depfac, output_filename): : len(real_prop_z) ] # Trim the spacing values to match the adjusted depth levels - this_file = os.path.normpath(__file__) - - # Add some info about how the file was generated - runcmd = ( - f"python3 {os.path.basename(this_file)} --H={H} --depfac={depfac} " - f"--dzd={dzd} " - f"--min_dz={min_dz} " - f"--output={output_filename} " - ) + # Obtain metadata + this_file = sys.argv[0] + runcmd = f"{sys.executable} {' '.join(sys.argv)}" # Write to NetCDF file write_netcdf_file(output_filename, real_prop_z, this_file, runcmd) diff --git a/mesh_generation/generate_mesh.py b/mesh_generation/generate_mesh.py index d312f445..eb8f6893 100755 --- a/mesh_generation/generate_mesh.py +++ b/mesh_generation/generate_mesh.py @@ -589,26 +589,9 @@ def main(): if topog_filename: topog_filename = os.path.abspath(topog_filename) - this_file = os.path.normpath(__file__) - - # Add some info about how the file was generated - runcmd = ( - f"python3 {os.path.basename(this_file)} --grid-type={grid_type} --grid-filename={grid_filename} " - f"--mesh-filename={mesh_filename}" - ) - if topog_filename: - runcmd += f" --topog-filename={topog_filename}" - if minimum_depth: - runcmd += f" --minimum-depth={minimum_depth}" - if masking_depth: - runcmd += f" --masking-depth={masking_depth}" - runcmd += f" --wrap-lons={wrap_lons}" - if lon_name: - runcmd += f" --lon-name={lon_name}" - if lat_name: - runcmd += f" --lat-name={lat_name}" - if area_name: - runcmd += f" --area-name={area_name}" + # Obtain metadata + this_file = sys.argv[0] + runcmd = f"{sys.executable} {' '.join(sys.argv)}" global_attrs = {"history": get_provenance_metadata(this_file, runcmd)} diff --git a/regrid_common.py b/regrid_common.py index dbb01ab7..414ab1f4 100644 --- a/regrid_common.py +++ b/regrid_common.py @@ -119,19 +119,6 @@ def parse_cli(self): if self.mask_filename: self.mask_filename = os.path.abspath(self.mask_filename) - # some info about how the file was generated - - runcmd_args = f"--hgrid-filename={self.hgrid_filename} --output-filename={self.output_filename}" - - if self.mask_filename: - runcmd_args += f" --mask-filename={self.mask_filename}" - if self.lon_name: - runcmd_args += f" --lon-name={self.lon_name}" - if self.lat_name: - runcmd_args += f" --lat-name={self.lat_name}" - - self.runcmd_args = runcmd_args - ## NOTE: it's implied that forcing_filename is set outside this class def open_datasets(self): diff --git a/restart_modifications/apply_bathy_mom_restarts.py b/restart_modifications/apply_bathy_mom_restarts.py index f8fa41f4..64a48596 100644 --- a/restart_modifications/apply_bathy_mom_restarts.py +++ b/restart_modifications/apply_bathy_mom_restarts.py @@ -145,8 +145,9 @@ def main(): print(f"[COPY] {tf} -> {out_f}") shutil.copy(tf, out_f) - this_file = os.path.normpath(__file__) - runcmd = f"python3 {os.path.basename(this_file)} --template_dir {args.template_dir} --old_dir {args.old_dir} --output_dir {args.output_dir} --template_prefix {args.template_prefix} --old_prefix {args.old_prefix} --nprocs {args.nprocs}" + # Obtain metadata + this_file = sys.argv[0] + runcmd = f"{sys.executable} {' '.join(sys.argv)}" print("[INFO] Starting update of variables...") # update each restart file on a different proc diff --git a/restart_modifications/remask_cpl_restart.py b/restart_modifications/remask_cpl_restart.py index 0419bd1c..5fa5e238 100644 --- a/restart_modifications/remask_cpl_restart.py +++ b/restart_modifications/remask_cpl_restart.py @@ -157,14 +157,9 @@ def main(): unmask_file(f, mask, missing_value, skip_vars=skip_vars) apply_mask_file(f, mask, skip_vars=skip_vars) - this_file = os.path.normpath(__file__) - runcmd = ( - f"python3 {os.path.basename(this_file)} " - f"--input_file {args.input_file} " - f"--output_file {args.output_file} " - f"--mask_file {args.mask_file} " - f"--mask_var {args.mask_var}" - ) + # Obtain metadata + this_file = sys.argv[0] + runcmd = f"{sys.executable} {' '.join(sys.argv)}" # Add metadata f.setncattr( diff --git a/rof_pattern_generation/generate_rofi_pattern.py b/rof_pattern_generation/generate_rofi_pattern.py index 04f0dbba..8c5625c2 100755 --- a/rof_pattern_generation/generate_rofi_pattern.py +++ b/rof_pattern_generation/generate_rofi_pattern.py @@ -179,13 +179,9 @@ def main(): "units": "days since 0001-01-01 00:00:00", } - # Add some info about how the file was generated - this_file = os.path.normpath(__file__) - - runcmd = ( - f"python3 {os.path.basename(this_file)} --topog-file={regrid.args.topog_file} " - f"{regrid.runcmd_args}" - ) + # Obtain metadata + this_file = sys.argv[0] + runcmd = f"{sys.executable} {' '.join(sys.argv)}" # Info about input data used file_hashes = [ diff --git a/scripts_common.py b/scripts_common.py index 10e7e639..d6b4ecfe 100644 --- a/scripts_common.py +++ b/scripts_common.py @@ -127,7 +127,7 @@ def get_provenance_metadata(file, runcmd): warn( f"{file} not under git version control! Add your file to a repository before generating any production output." ) - prepend += f"{file}: " + prepend += f"{os.path.abspath(file)}: " return prepend + runcmd diff --git a/surface_restoring_generation/make_sfc_restore_from_regridded_woa.py b/surface_restoring_generation/make_sfc_restore_from_regridded_woa.py index c17e4478..b5f8abf1 100755 --- a/surface_restoring_generation/make_sfc_restore_from_regridded_woa.py +++ b/surface_restoring_generation/make_sfc_restore_from_regridded_woa.py @@ -93,8 +93,10 @@ def main(input_path, variable_to_smooth, output_file): out_ds["time"] = out_ds.time.assign_attrs({"modulo": " "}) - this_file = os.path.normpath(__file__) - runcmd = f"python3 {os.path.basename(this_file)} --input_path={input_path} --var={variable_to_smooth} --output_file={output_file}" + # Obtain metadata + this_file = sys.argv[0] + runcmd = f"{sys.executable} {' '.join(sys.argv)}" + out_ds = out_ds.assign_attrs( { "history": get_provenance_metadata(this_file, runcmd), diff --git a/wombat_ic_generation/SFe_climatology.py b/wombat_ic_generation/SFe_climatology.py index 5a8b6489..c157f990 100644 --- a/wombat_ic_generation/SFe_climatology.py +++ b/wombat_ic_generation/SFe_climatology.py @@ -65,13 +65,9 @@ def main(): input_filename = os.path.abspath(args.input_filename) output_filename = os.path.abspath(args.output_filename) - this_file = os.path.normpath(__file__) - - # Add some info about how the file was generated - runcmd = ( - f"python3 {os.path.basename(this_file)} --input-filename={input_filename} " - f"--output-filename={output_filename}" - ) + # Obtain metadata + this_file = sys.argv[0] + runcmd = f"{sys.executable} {' '.join(sys.argv)}" history_attrs = { "history": get_provenance_metadata(this_file, runcmd), diff --git a/wombat_ic_generation/co2_iaf.py b/wombat_ic_generation/co2_iaf.py index 4e2f2fbf..db751b67 100644 --- a/wombat_ic_generation/co2_iaf.py +++ b/wombat_ic_generation/co2_iaf.py @@ -91,14 +91,9 @@ def main(): hgrid_filename = os.path.abspath(args.hgrid_filename) output_filename = os.path.abspath(args.output_filename) - this_file = os.path.normpath(__file__) - - # provenance metadata - runcmd = ( - f"python3 {os.path.basename(this_file)} --co2-cmip-filename={co2_cmip_filename} " - f"--co2-noaa-filename={co2_noaa_filename} --hgrid-filename={hgrid_filename} " - f"--output-filename={output_filename}" - ) + # Obtain metadata + this_file = sys.argv[0] + runcmd = f"{sys.executable} {' '.join(sys.argv)}" file_hashes = [ f"{co2_cmip_filename} (md5 hash: {md5sum(co2_cmip_filename)})", diff --git a/wombat_ic_generation/generate_wombat_ic.py b/wombat_ic_generation/generate_wombat_ic.py index a156d7be..b8e13298 100755 --- a/wombat_ic_generation/generate_wombat_ic.py +++ b/wombat_ic_generation/generate_wombat_ic.py @@ -457,13 +457,9 @@ def main(): wombat_version = args.wombat_version output_file = args.output_file - this_file = os.path.normpath(__file__) - - # Add some info about how the file was generated - runcmd = ( - f"python3 {this_file} --wombat-version={wombat_version} " - f"--output-file={os.path.abspath(output_file)}" - ) + # Obtain metadata + this_file = sys.argv[0] + runcmd = f"{sys.executable} {' '.join(sys.argv)}" global_attrs = { "history": get_provenance_metadata(this_file, runcmd), diff --git a/wombat_ic_generation/regrid_forcing.py b/wombat_ic_generation/regrid_forcing.py index 3ec0a5ef..c4963fe2 100644 --- a/wombat_ic_generation/regrid_forcing.py +++ b/wombat_ic_generation/regrid_forcing.py @@ -93,15 +93,9 @@ def main(): forcing_regrid["nx"].encoding |= {"dtype": "float32"} forcing_regrid["ny"].encoding |= {"dtype": "float32"} - # Add some info about how the file was generated - this_file = os.path.normpath(__file__) - - runcmd = ( - f"python3 {os.path.basename(this_file)} --forcing-filename={regrid.forcing_filename} " - f"{regrid.runcmd_args}" - ) - if regrid.args.homogenize: - runcmd += " --homogenize" + # Obtain metadata + this_file = sys.argv[0] + runcmd = f"{sys.executable} {' '.join(sys.argv)}" global_attrs = { "history": get_provenance_metadata(this_file, runcmd), From 07bb1ec4762068a6d522f9b44a393452e7aafa57 Mon Sep 17 00:00:00 2001 From: anton-seaice Date: Mon, 29 Jun 2026 15:59:20 +1000 Subject: [PATCH 2/9] Standardise input files metadata --- ...nerate_bottom_roughness_intermediate_woa.py | 12 +++++------- .../generate_bottom_roughness_polyfit.py | 14 +++++++------- .../generate_bottom_roughness_regrid.py | 12 +++++------- .../generate_tide_amplitude.py | 10 ++++------ grid_generation/generate_vertical_grid.py | 2 +- make_ryf/make_ryf.py | 8 ++++---- mesh_generation/generate_mesh.py | 8 ++------ mesh_generation/generate_rof_weights.py | 4 ++-- payu_config/postscript/concat_ice_daily.py | 2 +- regrid_common.py | 11 +++-------- .../generate_rofi_pattern.py | 18 ++++++++---------- scripts_common.py | 10 ++++++++++ .../make_sfc_restore_from_regridded_woa.py | 4 ++-- wombat_ic_generation/co2_iaf.py | 13 +++---------- wombat_ic_generation/generate_wombat_ic.py | 6 +++--- 15 files changed, 60 insertions(+), 74 deletions(-) diff --git a/external_tidal_generation/generate_bottom_roughness_intermediate_woa.py b/external_tidal_generation/generate_bottom_roughness_intermediate_woa.py index 9523d1e7..cc3f91e9 100644 --- a/external_tidal_generation/generate_bottom_roughness_intermediate_woa.py +++ b/external_tidal_generation/generate_bottom_roughness_intermediate_woa.py @@ -135,7 +135,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, md5sum +from scripts_common import get_provenance_metadata, get_provenance_input_files def coriolis_f(lat: xr.DataArray) -> xr.DataArray: @@ -856,12 +856,10 @@ def main(): history = get_provenance_metadata(this_file, runcmd) global_attrs = {"history": history} - file_hashes = [ - f"{args.woa_temp_file} (md5 hash: {md5sum(args.woa_temp_file)})", - f"{args.woa_salt_file} (md5 hash: {md5sum(args.woa_salt_file)})", - f"{args.synbath_file} (md5 hash: {md5sum(args.synbath_file)})", - ] - global_attrs["inputFile"] = ", ".join(file_hashes) + + input_files = [args.woa_temp_file, args.woa_salt_file, args.synbath_file] + global_attrs["inputFile"] = get_provenance_input_files(input_files) + ds_woa_output.attrs.update(global_attrs) ds_woa_output.to_netcdf(args.woa_intermediate_file) diff --git a/external_tidal_generation/generate_bottom_roughness_polyfit.py b/external_tidal_generation/generate_bottom_roughness_polyfit.py index 0f298bfe..6c4398e0 100644 --- a/external_tidal_generation/generate_bottom_roughness_polyfit.py +++ b/external_tidal_generation/generate_bottom_roughness_polyfit.py @@ -39,7 +39,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, md5sum +from scripts_common import get_provenance_metadata, get_provenance_input_files from mesh_generation.generate_mesh import mom6_mask_detection @@ -388,12 +388,12 @@ def main(): history = get_provenance_metadata(this_file, runcmd) global_attrs = {"history": history} - file_hashes = [ - f"{args.high_res_topo_file} (md5 hash: {md5sum(args.high_res_topo_file)})", - f"{args.hgrid_file} (md5 hash: {md5sum(args.hgrid_file)})", - f"{args.topog_file} (md5 hash: {md5sum(args.topog_file)})", - ] - global_attrs["inputFile"] = ", ".join(file_hashes) + + input_files = [args.high_res_topo_file, args.hgrid_file, args.topog_file] + global_attrs["inputFile"] = get_provenance_input_files( + get_provenance_input_files(input_files) + ) + h2_out.attrs.update(global_attrs) h2_out.to_netcdf(args.output) diff --git a/external_tidal_generation/generate_bottom_roughness_regrid.py b/external_tidal_generation/generate_bottom_roughness_regrid.py index 2cad0e6f..f050d6d5 100644 --- a/external_tidal_generation/generate_bottom_roughness_regrid.py +++ b/external_tidal_generation/generate_bottom_roughness_regrid.py @@ -53,7 +53,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, md5sum +from scripts_common import get_provenance_metadata, get_provenance_input_files from mesh_generation.generate_mesh import mom6_mask_detection @@ -401,12 +401,10 @@ def main(): history = get_provenance_metadata(this_file, runcmd) global_attrs = {"history": history} - file_hashes = [ - f"{args.hgrid_file} (md5 hash: {md5sum(args.hgrid_file)})", - f"{args.topog_file} (md5 hash: {md5sum(args.topog_file)})", - f"{args.woa_intermediate_file} (md5 hash: {md5sum(args.woa_intermediate_file)})", - ] - global_attrs["inputFile"] = ", ".join(file_hashes) + + input_files = [args.hgrid_file, args.topog_file, args.woa_intermediate_file] + global_attrs["inputFile"] = get_provenance_input_files(input_files) + regrid_depth_var.attrs.update(global_attrs) output_path = Path(args.output_file) diff --git a/external_tidal_generation/generate_tide_amplitude.py b/external_tidal_generation/generate_tide_amplitude.py index b186ec7f..1838e6c3 100644 --- a/external_tidal_generation/generate_tide_amplitude.py +++ b/external_tidal_generation/generate_tide_amplitude.py @@ -38,7 +38,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, md5sum +from scripts_common import get_provenance_metadata, get_provenance_input_files from mesh_generation.generate_mesh import mom6_mask_detection PRIMARY_CONSTITUENTS = ["m2", "s2", "n2", "k2", "k1", "o1", "p1", "q1"] @@ -283,11 +283,9 @@ def main(): global_attrs = {"history": history} # add md5 hashes for input files - file_hashes = [ - f"{args.hgrid_file} (md5 hash: {md5sum(args.hgrid_file)})", - f"{args.topog_file} (md5 hash: {md5sum(args.topog_file)})", - ] - global_attrs["inputFile"] = ", ".join(file_hashes) + input_files = [args.hgrid_file, args.topog_file] + global_attrs["inputFile"] = get_provenance_input_files(input_files) + tideamp.attrs.update(global_attrs) tideamp.to_netcdf(args.output, unlimited_dims=["time"]) diff --git a/grid_generation/generate_vertical_grid.py b/grid_generation/generate_vertical_grid.py index 0e8a702f..bbe7314a 100644 --- a/grid_generation/generate_vertical_grid.py +++ b/grid_generation/generate_vertical_grid.py @@ -38,7 +38,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, md5sum +from scripts_common import get_provenance_metadata # Define a small constant to initialize the iteration and prevent numerical issues epsilon = 0.001 diff --git a/make_ryf/make_ryf.py b/make_ryf/make_ryf.py index b86d1a44..1630a644 100644 --- a/make_ryf/make_ryf.py +++ b/make_ryf/make_ryf.py @@ -38,7 +38,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, md5sum +from scripts_common import get_provenance_metadata, get_provenance_input_files FILLVALUE = 1e20 # compression settings to use @@ -113,7 +113,7 @@ ds = {} for var in variables: - ryf_files = str() + input_files = [] print(var) for y in (year1, year2): if source_data == "jra55v1p4" or source_data == "jra55v1p6": @@ -147,7 +147,7 @@ print("Loading {} for {}".format(files[0], y)) ds[y] = xarray.open_dataset(files[0], decode_coords=False) # save info for metadata - ryf_files += f"{files[0]} (md5 hash: {md5sum(files[0])}, )" + input_files.append(files[0]) # Make a copy of the second year without time_bnds ryf = ds[baseyear].drop_vars("time_bnds") ryf.encoding = ds[baseyear].encoding @@ -208,7 +208,7 @@ this_file = os.path.normpath(__file__) runcmd = f"python3 {os.path.basename(this_file)}" ryf.attrs |= {"RYF_creation": get_provenance_metadata(this_file, runcmd)} - ryf.attrs |= {"RYF_inputFiles": ryf_files} + ryf.attrs |= {"RYF_input_files": get_provenance_input_files(input_files)} outfile = "RYF.{}.{}_{}.nc".format(var, year1, year2) print("Writing ", outfile) diff --git a/mesh_generation/generate_mesh.py b/mesh_generation/generate_mesh.py index eb8f6893..f58cb02c 100755 --- a/mesh_generation/generate_mesh.py +++ b/mesh_generation/generate_mesh.py @@ -43,7 +43,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, md5sum +from scripts_common import get_provenance_metadata, get_provenance_input_files EARTH_R = 6.37122e6 @@ -205,11 +205,7 @@ def create_mesh(self, wrap_lons=True, global_attrs=None): "timeGenerated": f"{datetime.now()}", "created_by": f"{os.environ.get('USER')}", } - if self.inputs: - file_hashes = [] - for input in self.inputs: - file_hashes.append(f"{input} (md5 hash: {md5sum(input)})") - ds.attrs["inputFile"] = ", ".join(file_hashes) + ds.attrs["inputFile"] = get_provenance_input_files(self.inputs) # add git info to history if global_attrs: diff --git a/mesh_generation/generate_rof_weights.py b/mesh_generation/generate_rof_weights.py index 88c63401..40e298d0 100755 --- a/mesh_generation/generate_rof_weights.py +++ b/mesh_generation/generate_rof_weights.py @@ -42,7 +42,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, md5sum +from scripts_common import get_provenance_metadata, get_provenance_input_files TEMP_WEIGHTS_F = "temp_weights.nc" COMP_ENCODING = {"complevel": 1, "compression": "zlib"} # compression settings to use @@ -139,7 +139,7 @@ def drof_remapping_weights(mesh_filename, weights_filename, nx, ny, global_attrs # add global attributes weights_ds.attrs = { "gridType": "unstructured mesh", - "inputFile": f"{mesh_filename} (md5 hash: {md5sum(mesh_filename)})", + "inputFile": get_provenance_input_files([mesh_filename]), } # add git info to history diff --git a/payu_config/postscript/concat_ice_daily.py b/payu_config/postscript/concat_ice_daily.py index 28ee6bd3..9f133766 100644 --- a/payu_config/postscript/concat_ice_daily.py +++ b/payu_config/postscript/concat_ice_daily.py @@ -29,7 +29,7 @@ path_root = Path(__file__).parents[2] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, md5sum +from scripts_common import get_provenance_metadata CICE_DAILY_FN = "access-om3.cice.1day.mean.????-??-??.nc" MONTHLY_STUB_FN = "access-om3.cice.1day.mean." diff --git a/regrid_common.py b/regrid_common.py index 414ab1f4..1c84afb9 100644 --- a/regrid_common.py +++ b/regrid_common.py @@ -216,17 +216,12 @@ def save_output(self): forcing_regrid = self.forcing_regrid # Info about input data used - file_hashes = [ - f"{self.forcing_filename} (md5 hash: {md5sum(self.forcing_filename)})", - f"{self.hgrid_filename} (md5 hash: {md5sum(self.hgrid_filename)})", - ] + input_files = [self.forcing_filename, self.hgrid_filename] if self.mask_filename: - file_hashes.append( - f"{self.mask_filename} (md5 hash: {md5sum(self.mask_filename)})" - ) + input_files.append(self.mask_filename) global_attrs = { - "inputFile": ", ".join(file_hashes), + "inputFile": get_provenance_input_files(input_files), } forcing_regrid.attrs = forcing_regrid.attrs | global_attrs diff --git a/rof_pattern_generation/generate_rofi_pattern.py b/rof_pattern_generation/generate_rofi_pattern.py index 8c5625c2..fb8ea8cc 100755 --- a/rof_pattern_generation/generate_rofi_pattern.py +++ b/rof_pattern_generation/generate_rofi_pattern.py @@ -40,7 +40,7 @@ sys.path.append(str(path_root)) from regrid_common import Regrid_Common -from scripts_common import get_provenance_metadata, md5sum +from scripts_common import get_provenance_metadata, get_provenance_input_files from mesh_generation.generate_mesh import mom6_mask_detection # in a climatology, with 365 day calendar, whats the day of the middle of each month @@ -184,21 +184,19 @@ def main(): runcmd = f"{sys.executable} {' '.join(sys.argv)}" # Info about input data used - file_hashes = [ - f"{AQ_MELT_PATTERN} (md5 hash: {md5sum(AQ_MELT_PATTERN)})", - f"{GL_MELT_PATTERN} (md5 hash: {md5sum(GL_MELT_PATTERN)})", - f"{regrid.hgrid_filename} (md5 hash: {md5sum(regrid.hgrid_filename)})", - f"{regrid.args.topog_file} (md5 hash: {md5sum(regrid.args.topog_file)})", + input_files = [ + AQ_MELT_PATTERN, + GL_MELT_PATTERN, + regrid.hgrid_filename, + regrid.args.topog_file, ] if regrid.mask_filename: - file_hashes.append( - f"{regrid.mask_filename} (md5 hash: {md5sum(regrid.mask_filename)})" - ) + input_files.append(regrid.mask_filename) global_attrs |= { "description": "Mankoff 2025 iceberg spreading climatology remapped onto an ACCESS-OM3 grid", "history": get_provenance_metadata(this_file, runcmd), - "inputFile": ", ".join(file_hashes), + "inputFile": get_provenance_input_files(input_files), } weights_ds.attrs = weights_ds.attrs | global_attrs diff --git a/scripts_common.py b/scripts_common.py index d6b4ecfe..b4235a56 100644 --- a/scripts_common.py +++ b/scripts_common.py @@ -144,3 +144,13 @@ def md5sum(path): for chunk in iter(lambda: fd.read(length), b""): md5.update(chunk) return md5.hexdigest() + + +def get_provenance_input_files(input_files): + """ + Return a formatted string of provided input files and their md5 hashes + """ + file_hashes = [] + for input in input_files: + file_hashes.append(f"{os.path.abspath(input)} (md5 hash: {md5sum(input)})") + return ", ".join(file_hashes) diff --git a/surface_restoring_generation/make_sfc_restore_from_regridded_woa.py b/surface_restoring_generation/make_sfc_restore_from_regridded_woa.py index b5f8abf1..27c932d0 100755 --- a/surface_restoring_generation/make_sfc_restore_from_regridded_woa.py +++ b/surface_restoring_generation/make_sfc_restore_from_regridded_woa.py @@ -38,7 +38,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, md5sum +from scripts_common import get_provenance_metadata, get_provenance_input_files def smooth2d(src): @@ -100,7 +100,7 @@ def main(input_path, variable_to_smooth, output_file): out_ds = out_ds.assign_attrs( { "history": get_provenance_metadata(this_file, runcmd), - "input_files": [f"{f}(md5sum:{md5sum(f)})" for f in file_paths], + "input_files": get_provenance_input_files(file_paths), } ) diff --git a/wombat_ic_generation/co2_iaf.py b/wombat_ic_generation/co2_iaf.py index db751b67..ff9dc8b3 100644 --- a/wombat_ic_generation/co2_iaf.py +++ b/wombat_ic_generation/co2_iaf.py @@ -38,7 +38,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, md5sum +from scripts_common import get_provenance_metadata, get_provenance_input_files xr.set_options(keep_attrs=True) @@ -86,24 +86,17 @@ def main(): args = parser.parse_args() - co2_cmip_filename = os.path.abspath(args.co2_cmip_filename) - co2_noaa_filename = os.path.abspath(args.co2_noaa_filename) - hgrid_filename = os.path.abspath(args.hgrid_filename) output_filename = os.path.abspath(args.output_filename) # Obtain metadata this_file = sys.argv[0] runcmd = f"{sys.executable} {' '.join(sys.argv)}" - file_hashes = [ - f"{co2_cmip_filename} (md5 hash: {md5sum(co2_cmip_filename)})", - f"{co2_noaa_filename} (md5 hash: {md5sum(co2_noaa_filename)})", - f"{hgrid_filename} (md5 hash: {md5sum(hgrid_filename)})", - ] + input_files = [co2_cmip_filename, co2_noaa_filename, hgrid_filename] global_attrs = { "history": get_provenance_metadata(this_file, runcmd), - "inputFile": ", ".join(file_hashes), + "inputFile": get_provenance_input_files(input_files), } # Load the input data diff --git a/wombat_ic_generation/generate_wombat_ic.py b/wombat_ic_generation/generate_wombat_ic.py index b8e13298..ef3b4a31 100755 --- a/wombat_ic_generation/generate_wombat_ic.py +++ b/wombat_ic_generation/generate_wombat_ic.py @@ -75,7 +75,7 @@ def _open_woa_data(file, var, global_attrs, hcoord_int=None, vcoord_int=None): "WOA23 (https://www.ncei.noaa.gov/access/world-ocean-atlas-2023/)" ) - global_attrs["inputFile"].add(f"{file} (md5 hash: {md5sum(file)})") + global_attrs["inputFile"].add(f"{os.path.abspath(file)} (md5 hash: {md5sum(file)})") return _standardise_coords(da, hcoord_int, vcoord_int) @@ -96,7 +96,7 @@ def _open_glodap_data(file, var, global_attrs, hcoord_int=None, vcoord_int=None) "GLODAPv2 mapped data (https://glodap.info/index.php/mapped-data-product/)" ) - global_attrs["inputFile"].add(f"{file} (md5 hash: {md5sum(file)})") + global_attrs["inputFile"].add(f"{os.path.abspath(file)} (md5 hash: {md5sum(file)})") return _standardise_coords(da, hcoord_int, vcoord_int) @@ -113,7 +113,7 @@ def _open_huang_data(file, global_attrs, hcoord_int=None, vcoord_int=None): "Huang et al 2022 dFe data (https://zenodo.org/records/6994318)" ) - global_attrs["inputFile"].add(f"{file} (md5 hash: {md5sum(file)})") + global_attrs["inputFile"].add(f"{os.path.abspath(file)} (md5 hash: {md5sum(file)})") # Extend Latitudes from -90-90 new_lats = np.concatenate( From 875f2e9ea343b6fda6af52d8d4b8f1439ee320a2 Mon Sep 17 00:00:00 2001 From: anton-seaice Date: Tue, 30 Jun 2026 09:32:31 +1000 Subject: [PATCH 3/9] refactor get provenance metadata to return a dictionary --- chlorophyll/chl_climatology_and_fill.py | 10 +-------- .../generate_xml_datm_era5.py | 6 ++--- .../generate_xml_datm_jra55.py | 6 ++--- .../generate_xml_drof_jra55.py | 6 ++--- ...erate_bottom_roughness_intermediate_woa.py | 7 ++---- .../generate_bottom_roughness_polyfit.py | 11 ++-------- .../generate_bottom_roughness_regrid.py | 11 ++-------- .../generate_tide_amplitude.py | 11 ++-------- grid_generation/generate_vertical_grid.py | 10 +++------ make_ryf/make_ryf.py | 7 ++---- mesh_generation/generate_mesh.py | 22 ++++--------------- mesh_generation/generate_rof_weights.py | 21 ++++-------------- payu_config/postscript/build_intake_ds.py | 12 +++------- payu_config/postscript/concat_ice_daily.py | 4 +--- .../apply_bathy_mom_restarts.py | 10 ++------- restart_modifications/remask_cpl_restart.py | 6 +---- .../generate_rofi_pattern.py | 5 ++--- scripts_common.py | 20 ++++++++++++----- .../make_sfc_restore_from_regridded_woa.py | 9 ++------ wombat_ic_generation/SFe_climatology.py | 12 +++------- wombat_ic_generation/co2_iaf.py | 7 ++---- wombat_ic_generation/generate_wombat_ic.py | 7 ++---- wombat_ic_generation/regrid_forcing.py | 7 +----- 23 files changed, 62 insertions(+), 165 deletions(-) diff --git a/chlorophyll/chl_climatology_and_fill.py b/chlorophyll/chl_climatology_and_fill.py index 741ae1d1..1930de9d 100644 --- a/chlorophyll/chl_climatology_and_fill.py +++ b/chlorophyll/chl_climatology_and_fill.py @@ -70,14 +70,6 @@ def main(): input_directory = os.path.abspath(args.input_directory) output_filename = os.path.abspath(args.output_filename) - # Obtain metadata - this_file = sys.argv[0] - runcmd = f"{sys.executable} {' '.join(sys.argv)}" - - history_attrs = { - "history": get_provenance_metadata(this_file, runcmd), - } - # Load the input data and compute the monthly climatology input_files = sorted(glob.glob(f"{input_directory}/*.nc")) @@ -156,7 +148,7 @@ def main(): chl.time.attrs["long_name"] = "Time" chl.time.attrs["standard_name"] = "time" chl.time.attrs["axis"] = "T" - chl.attrs |= history_attrs + chl.attrs |= get_provenance_metadata(input_files) comp = dict(zlib=True, complevel=4) encoding = {var: comp for var in chl.data_vars} # Time coords should be double type according for CF conventions diff --git a/data_stream_xml_generation/generate_xml_datm_era5.py b/data_stream_xml_generation/generate_xml_datm_era5.py index 23eb1ec0..4f6b0f9b 100644 --- a/data_stream_xml_generation/generate_xml_datm_era5.py +++ b/data_stream_xml_generation/generate_xml_datm_era5.py @@ -85,9 +85,7 @@ root = Element("file", id="stream", version="2.0") # Obtain metadata -this_file = sys.argv[0] -runcmd = f"{sys.executable} {' '.join(sys.argv)}" -metadata_info = get_provenance_metadata(this_file, runcmd) +metadata_info = get_provenance_metadata() # Add metadata metadata = SubElement(root, "metadata") @@ -95,7 +93,7 @@ SubElement(metadata, "date_generated").text = datetime.now().strftime( "%Y-%m-%d %H:%M:%S" ) -SubElement(metadata, "history").text = metadata_info +SubElement(metadata, "history").text = metadata_info["history"] # Generate stream info elements with changing years for stream_name, era5_prefix, datavar_pairs, mapalgo, offset_seconds in STREAM_SPECS: diff --git a/data_stream_xml_generation/generate_xml_datm_jra55.py b/data_stream_xml_generation/generate_xml_datm_jra55.py index b487fb07..19c1f4d0 100644 --- a/data_stream_xml_generation/generate_xml_datm_jra55.py +++ b/data_stream_xml_generation/generate_xml_datm_jra55.py @@ -61,9 +61,7 @@ root = Element("file", id="stream", version="2.0") # Obtain metadata -this_file = sys.argv[0] -runcmd = f"{sys.executable} {' '.join(sys.argv)}" -metadata_info = get_provenance_metadata(this_file, runcmd) +metadata_info = get_provenance_metadata() # Add metadata metadata = SubElement(root, "metadata") @@ -71,7 +69,7 @@ SubElement(metadata, "date_generated").text = datetime.now().strftime( "%Y-%m-%d %H:%M:%S" ) -SubElement(metadata, "history").text = metadata_info +SubElement(metadata, "history").text = metadata_info["history"] # Define the stream info names and corresponding var names stream_info_names = [*STREAMS_AVE, *STREAMS_PT] diff --git a/data_stream_xml_generation/generate_xml_drof_jra55.py b/data_stream_xml_generation/generate_xml_drof_jra55.py index 633ece2e..5b331a74 100644 --- a/data_stream_xml_generation/generate_xml_drof_jra55.py +++ b/data_stream_xml_generation/generate_xml_drof_jra55.py @@ -42,9 +42,7 @@ root = Element("file", id="stream", version="2.0") # Obtain metadata -this_file = sys.argv[0] -runcmd = f"{sys.executable} {' '.join(sys.argv)}" -metadata_info = get_provenance_metadata(this_file, runcmd) +metadata_info = get_provenance_metadata() # Add metadata metadata = SubElement(root, "metadata") @@ -52,7 +50,7 @@ SubElement(metadata, "date_generated").text = datetime.now().strftime( "%Y-%m-%d %H:%M:%S" ) -SubElement(metadata, "history").text = metadata_info +SubElement(metadata, "history").text = metadata_info["history"] # Define the stream info names and corresponding var names stream_info_data = [ diff --git a/external_tidal_generation/generate_bottom_roughness_intermediate_woa.py b/external_tidal_generation/generate_bottom_roughness_intermediate_woa.py index cc3f91e9..4ee2b524 100644 --- a/external_tidal_generation/generate_bottom_roughness_intermediate_woa.py +++ b/external_tidal_generation/generate_bottom_roughness_intermediate_woa.py @@ -135,7 +135,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, get_provenance_input_files +from scripts_common import get_provenance_metadata def coriolis_f(lat: xr.DataArray) -> xr.DataArray: @@ -854,11 +854,8 @@ def main(): f"--woa-intermediate-file={args.woa_intermediate_file} " ) - history = get_provenance_metadata(this_file, runcmd) - global_attrs = {"history": history} - input_files = [args.woa_temp_file, args.woa_salt_file, args.synbath_file] - global_attrs["inputFile"] = get_provenance_input_files(input_files) + global_attrs = get_provenance_metadata(input_files) ds_woa_output.attrs.update(global_attrs) diff --git a/external_tidal_generation/generate_bottom_roughness_polyfit.py b/external_tidal_generation/generate_bottom_roughness_polyfit.py index 6c4398e0..e75c5574 100644 --- a/external_tidal_generation/generate_bottom_roughness_polyfit.py +++ b/external_tidal_generation/generate_bottom_roughness_polyfit.py @@ -39,7 +39,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, get_provenance_input_files +from scripts_common import get_provenance_metadata from mesh_generation.generate_mesh import mom6_mask_detection @@ -383,16 +383,9 @@ def main(): ) # Add provenance metadata and MD5 hashes for input files. - this_file = os.path.normpath(__file__) runcmd = f"mpirun -n $PBS_NCPUS python3 {' '.join(sys.argv)} " - - history = get_provenance_metadata(this_file, runcmd) - global_attrs = {"history": history} - input_files = [args.high_res_topo_file, args.hgrid_file, args.topog_file] - global_attrs["inputFile"] = get_provenance_input_files( - get_provenance_input_files(input_files) - ) + global_attrs = get_provenance_metadata(input_files, runcmd) h2_out.attrs.update(global_attrs) diff --git a/external_tidal_generation/generate_bottom_roughness_regrid.py b/external_tidal_generation/generate_bottom_roughness_regrid.py index f050d6d5..0bb985b7 100644 --- a/external_tidal_generation/generate_bottom_roughness_regrid.py +++ b/external_tidal_generation/generate_bottom_roughness_regrid.py @@ -53,7 +53,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, get_provenance_input_files +from scripts_common import get_provenance_metadata from mesh_generation.generate_mesh import mom6_mask_detection @@ -396,15 +396,8 @@ def main(): print("Regridding done!") # Add provenance metadata and MD5 hashes for input files. - this_file = sys.argv[0] - runcmd = f"{sys.executable} {' '.join(sys.argv)}" - - history = get_provenance_metadata(this_file, runcmd) - global_attrs = {"history": history} - input_files = [args.hgrid_file, args.topog_file, args.woa_intermediate_file] - global_attrs["inputFile"] = get_provenance_input_files(input_files) - + global_attrs = get_provenance_metadata(input_files) regrid_depth_var.attrs.update(global_attrs) output_path = Path(args.output_file) diff --git a/external_tidal_generation/generate_tide_amplitude.py b/external_tidal_generation/generate_tide_amplitude.py index 1838e6c3..f3b021f4 100644 --- a/external_tidal_generation/generate_tide_amplitude.py +++ b/external_tidal_generation/generate_tide_amplitude.py @@ -38,7 +38,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, get_provenance_input_files +from scripts_common import get_provenance_metadata from mesh_generation.generate_mesh import mom6_mask_detection PRIMARY_CONSTITUENTS = ["m2", "s2", "n2", "k2", "k1", "o1", "p1", "q1"] @@ -277,15 +277,8 @@ def main(): tideamp = tideamp.rename({"x": "xh", "y": "yh"}) # Add provenance metadata and MD5 hashes for input files. - this_file = sys.argv[0] - runcmd = f"{sys.executable} {' '.join(sys.argv)}" - history = get_provenance_metadata(this_file, runcmd) - global_attrs = {"history": history} - - # add md5 hashes for input files input_files = [args.hgrid_file, args.topog_file] - global_attrs["inputFile"] = get_provenance_input_files(input_files) - + global_attrs = get_provenance_metadata(input_files) tideamp.attrs.update(global_attrs) tideamp.to_netcdf(args.output, unlimited_dims=["time"]) diff --git a/grid_generation/generate_vertical_grid.py b/grid_generation/generate_vertical_grid.py index bbe7314a..ccc85636 100644 --- a/grid_generation/generate_vertical_grid.py +++ b/grid_generation/generate_vertical_grid.py @@ -91,12 +91,8 @@ def generate_vertical_grid(H, dzd, min_dz, depfac, output_filename): : len(real_prop_z) ] # Trim the spacing values to match the adjusted depth levels - # Obtain metadata - this_file = sys.argv[0] - runcmd = f"{sys.executable} {' '.join(sys.argv)}" - # Write to NetCDF file - write_netcdf_file(output_filename, real_prop_z, this_file, runcmd) + write_netcdf_file(output_filename, real_prop_z, this_file) print( f"SUCCESS! A vertical grid with {len(real_prop_z) - 1} levels has been generated. " @@ -105,7 +101,7 @@ def generate_vertical_grid(H, dzd, min_dz, depfac, output_filename): ) -def write_netcdf_file(output_filename, real_prop_z, this_file, runcmd): +def write_netcdf_file(output_filename, real_prop_z, this_file): """Function to write vertical grid data to a NetCDF file.""" # Convert to float32 (single precision) to ensure values are exactly representable in single precision, # then convert back to float64 (double precision) for storage in NetCDF. @@ -119,7 +115,7 @@ def write_netcdf_file(output_filename, real_prop_z, this_file, runcmd): zeta.standard_name = "depth" zeta.long_name = "vertical grid depth at top and bottom of each cell" eddyfile.variables["zeta"][:] = real_prop_z_float64 - eddyfile.setncatts({"history": get_provenance_metadata(this_file, runcmd)}) + eddyfile.setncatts(get_provenance_metadata()) # Obtain metadata eddyfile.close() diff --git a/make_ryf/make_ryf.py b/make_ryf/make_ryf.py index 1630a644..8ffaada3 100644 --- a/make_ryf/make_ryf.py +++ b/make_ryf/make_ryf.py @@ -38,7 +38,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, get_provenance_input_files +from scripts_common import get_provenance_metadata FILLVALUE = 1e20 # compression settings to use @@ -205,10 +205,7 @@ ) # Add some info about how the file was generated - this_file = os.path.normpath(__file__) - runcmd = f"python3 {os.path.basename(this_file)}" - ryf.attrs |= {"RYF_creation": get_provenance_metadata(this_file, runcmd)} - ryf.attrs |= {"RYF_input_files": get_provenance_input_files(input_files)} + ryf.attrs |= get_provenance_metadata(input_files) outfile = "RYF.{}.{}_{}.nc".format(var, year1, year2) print("Writing ", outfile) diff --git a/mesh_generation/generate_mesh.py b/mesh_generation/generate_mesh.py index f58cb02c..511d3b78 100755 --- a/mesh_generation/generate_mesh.py +++ b/mesh_generation/generate_mesh.py @@ -43,7 +43,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, get_provenance_input_files +from scripts_common import get_provenance_metadata EARTH_R = 6.37122e6 @@ -126,7 +126,7 @@ def __init__( self.mesh = None - def create_mesh(self, wrap_lons=True, global_attrs=None): + def create_mesh(self, wrap_lons=True): """ Create the mesh as an xarray Dataset @@ -134,8 +134,6 @@ def create_mesh(self, wrap_lons=True, global_attrs=None): ---------- wrap_lons: boolean, optional If True, wrap longitude values into the range between 0 and 360 - global_attrs: dict - Global attributes to the mesh object """ if wrap_lons: @@ -205,11 +203,7 @@ def create_mesh(self, wrap_lons=True, global_attrs=None): "timeGenerated": f"{datetime.now()}", "created_by": f"{os.environ.get('USER')}", } - ds.attrs["inputFile"] = get_provenance_input_files(self.inputs) - - # add git info to history - if global_attrs: - ds.attrs |= global_attrs + ds.attrs |= get_provenance_metadata(input_files=self.inputs) self.mesh = ds @@ -585,12 +579,6 @@ def main(): if topog_filename: topog_filename = os.path.abspath(topog_filename) - # Obtain metadata - this_file = sys.argv[0] - runcmd = f"{sys.executable} {' '.join(sys.argv)}" - - global_attrs = {"history": get_provenance_metadata(this_file, runcmd)} - mesh = gridtype_dispatch[grid_type]( grid_filename, topog_filename, @@ -601,9 +589,7 @@ def main(): masking_depth, ) - mesh.create_mesh(wrap_lons=wrap_lons, global_attrs=global_attrs).write( - mesh_filename - ) + mesh.create_mesh(wrap_lons=wrap_lons).write(mesh_filename) if __name__ == "__main__": diff --git a/mesh_generation/generate_rof_weights.py b/mesh_generation/generate_rof_weights.py index 40e298d0..935859ea 100755 --- a/mesh_generation/generate_rof_weights.py +++ b/mesh_generation/generate_rof_weights.py @@ -42,13 +42,13 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, get_provenance_input_files +from scripts_common import get_provenance_metadata TEMP_WEIGHTS_F = "temp_weights.nc" COMP_ENCODING = {"complevel": 1, "compression": "zlib"} # compression settings to use -def drof_remapping_weights(mesh_filename, weights_filename, nx, ny, global_attrs=None): +def drof_remapping_weights(mesh_filename, weights_filename, nx, ny): # We need to generate remapping weights for use in the mediator, such that the overall volume of runoff is conserved # and no runoff is mapped onto land cells. Inside the mediator, the grid doesn't change as we run the mediator with # the ocean grid (the DROF component does the remapping from JRA grid to mediator grid). Therefore we use the @@ -139,12 +139,10 @@ def drof_remapping_weights(mesh_filename, weights_filename, nx, ny, global_attrs # add global attributes weights_ds.attrs = { "gridType": "unstructured mesh", - "inputFile": get_provenance_input_files([mesh_filename]), } # add git info to history - if global_attrs: - weights_ds.attrs |= global_attrs + weights_ds.attrs |= get_provenance_metadata([mesh_filename]) # save (compressed) encoding = {} @@ -188,19 +186,8 @@ def main(): ) args = parser.parse_args() - mesh_filename = os.path.abspath(args.mesh_filename) - weights_filename = os.path.abspath(args.weights_filename) - this_file = os.path.normpath(__file__) - - # Add some info about how the file was generated - runcmd = f"python3 {os.path.basename(this_file)} --mesh_filename={mesh_filename} --nx={args.nx} --ny={args.ny} --weights_filename={weights_filename} " - - global_attrs = {"history": get_provenance_metadata(this_file, runcmd)} - - drof_remapping_weights( - mesh_filename, weights_filename, args.nx, args.ny, global_attrs - ) + drof_remapping_weights(args.mesh_filename, args.weights_filename, args.nx, args.ny) return True diff --git a/payu_config/postscript/build_intake_ds.py b/payu_config/postscript/build_intake_ds.py index ac4c7595..3709a9ad 100644 --- a/payu_config/postscript/build_intake_ds.py +++ b/payu_config/postscript/build_intake_ds.py @@ -39,18 +39,12 @@ def description(): warn(f"{METADATA_FILENAME} not found in config folder") uuid = False - # Check git status of this .py file - this_file = os.path.normpath(__file__) - - runcmd = f"python3 {os.path.basename(this_file)}" - - # Get string "Created using $file: $command" - provenance = get_provenance_metadata(this_file, runcmd) + provenance = get_provenance_metadata() if uuid: - description = f"intake-esm datastore for experiment {uuid}, in folder {os.getcwd()}. {provenance}. (md5 hash: {md5sum(this_file)})" + description = f"intake-esm datastore for experiment {uuid}, in folder {os.getcwd()}. {provenance['history']}." else: - description = f"intake-esm datastore for experiment in folder {os.getcwd()}. {provenance}. (md5 hash: {md5sum(this_file)})" + description = f"intake-esm datastore for experiment in folder {os.getcwd()}. {provenance['history']}." return description diff --git a/payu_config/postscript/concat_ice_daily.py b/payu_config/postscript/concat_ice_daily.py index 9f133766..50e9bbd9 100644 --- a/payu_config/postscript/concat_ice_daily.py +++ b/payu_config/postscript/concat_ice_daily.py @@ -134,9 +134,7 @@ def __init__(self, directory=None, assume_gadi=True): del daily_ds.attrs["comment3"] # Add some info about how the file was generated - this_file = os.path.normpath(__file__) - runcmd = f"python3 {os.path.basename(this_file)} --directory={os.path.abspath(directory)}" - daily_ds.attrs["postprocessing"] = get_provenance_metadata(this_file, runcmd) + daily_ds.attrs |= get_provenance_metadata() self.directory = directory self.daily_ds = daily_ds diff --git a/restart_modifications/apply_bathy_mom_restarts.py b/restart_modifications/apply_bathy_mom_restarts.py index 64a48596..3e56faac 100644 --- a/restart_modifications/apply_bathy_mom_restarts.py +++ b/restart_modifications/apply_bathy_mom_restarts.py @@ -45,14 +45,14 @@ def copy_data(args): - in_f, out_f, runcmd, this_f, template_f = args + in_f, out_f, template_f = args print(f"\n[INFO] Processing:\n OLD: {in_f}\n OUT: {out_f}") try: with nc.Dataset(in_f) as in_fp, nc.Dataset(out_f, "r+") as out_fp: # Add metadata out_fp.setncattr("title", "Updated restart file with new bathymetry mask") out_fp.setncattr("source", "Generated by fix_mom6_restarts.py") - out_fp.setncattr("history", get_provenance_metadata(this_f, runcmd)) + out_fp.setncatts(get_provenance_metadata()) out_fp.setncattr( "template_file", f"{os.path.abspath(template_f)} (md5 hash: {md5sum(template_f)})", @@ -145,10 +145,6 @@ def main(): print(f"[COPY] {tf} -> {out_f}") shutil.copy(tf, out_f) - # Obtain metadata - this_file = sys.argv[0] - runcmd = f"{sys.executable} {' '.join(sys.argv)}" - print("[INFO] Starting update of variables...") # update each restart file on a different proc pool = mp.Pool(args.nprocs) @@ -157,8 +153,6 @@ def main(): zip( old_files, output_files, - [runcmd] * len(old_files), - [this_file] * len(old_files), template_files, ), ) diff --git a/restart_modifications/remask_cpl_restart.py b/restart_modifications/remask_cpl_restart.py index 5fa5e238..80acfd58 100644 --- a/restart_modifications/remask_cpl_restart.py +++ b/restart_modifications/remask_cpl_restart.py @@ -157,16 +157,12 @@ def main(): unmask_file(f, mask, missing_value, skip_vars=skip_vars) apply_mask_file(f, mask, skip_vars=skip_vars) - # Obtain metadata - this_file = sys.argv[0] - runcmd = f"{sys.executable} {' '.join(sys.argv)}" - # Add metadata f.setncattr( "title", "Coupler restart fields updated with land mask of modified bathymetry", ) - f.setncattr("history", get_provenance_metadata(this_file, runcmd)) + f.setncatts(get_provenance_metadata()) f.setncattr( "mask_file", f"{os.path.abspath(args.mask_file)} (md5 hash: {md5sum(args.mask_file)})", diff --git a/rof_pattern_generation/generate_rofi_pattern.py b/rof_pattern_generation/generate_rofi_pattern.py index fb8ea8cc..72bf93fe 100755 --- a/rof_pattern_generation/generate_rofi_pattern.py +++ b/rof_pattern_generation/generate_rofi_pattern.py @@ -40,7 +40,7 @@ sys.path.append(str(path_root)) from regrid_common import Regrid_Common -from scripts_common import get_provenance_metadata, get_provenance_input_files +from scripts_common import get_provenance_metadata from mesh_generation.generate_mesh import mom6_mask_detection # in a climatology, with 365 day calendar, whats the day of the middle of each month @@ -195,9 +195,8 @@ def main(): global_attrs |= { "description": "Mankoff 2025 iceberg spreading climatology remapped onto an ACCESS-OM3 grid", - "history": get_provenance_metadata(this_file, runcmd), - "inputFile": get_provenance_input_files(input_files), } + global_attrs |= get_provenance_metadata(input_files) weights_ds.attrs = weights_ds.attrs | global_attrs diff --git a/scripts_common.py b/scripts_common.py index b4235a56..a105a572 100644 --- a/scripts_common.py +++ b/scripts_common.py @@ -7,7 +7,7 @@ import subprocess -import os +import os, sys from warnings import warn import io import hashlib @@ -97,15 +97,20 @@ def username(file): return name -def get_provenance_metadata(file, runcmd): +def get_provenance_metadata(input_files=None, runcmd=None): """ - Return a string with the provenance of the file being run. Warn if the file is not pushed to the git upstream repository. + Return a dictionary with the provenance of the file being run. Warn if the + file is not pushed to the git upstream repository. arguments: file: the path to the file being run runcmd: the command used to run the file (with any arguments) """ + file = sys.argv[0] # script being run + if runcmd is None: + runcmd = f"{sys.executable} {' '.join(sys.argv)}" + prepend = ( f"Created by {username(file)} on {datetime.now().strftime('%Y-%m-%d')}, using " ) @@ -127,9 +132,14 @@ def get_provenance_metadata(file, runcmd): warn( f"{file} not under git version control! Add your file to a repository before generating any production output." ) - prepend += f"{os.path.abspath(file)}: " + prepend += f"{os.path.abspath(file)} (md5 hash: {md5sum(file)}): " + + attrs = {"history": prepend + runcmd} + + if input_files is not None: + attrs["inputFile"] = get_provenance_input_files(input_files) - return prepend + runcmd + return attrs def md5sum(path): diff --git a/surface_restoring_generation/make_sfc_restore_from_regridded_woa.py b/surface_restoring_generation/make_sfc_restore_from_regridded_woa.py index 27c932d0..a2df37d7 100755 --- a/surface_restoring_generation/make_sfc_restore_from_regridded_woa.py +++ b/surface_restoring_generation/make_sfc_restore_from_regridded_woa.py @@ -38,7 +38,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, get_provenance_input_files +from scripts_common import get_provenance_metadata def smooth2d(src): @@ -97,12 +97,7 @@ def main(input_path, variable_to_smooth, output_file): this_file = sys.argv[0] runcmd = f"{sys.executable} {' '.join(sys.argv)}" - out_ds = out_ds.assign_attrs( - { - "history": get_provenance_metadata(this_file, runcmd), - "input_files": get_provenance_input_files(file_paths), - } - ) + out_ds = out_ds.assign_attrs(get_provenance_metadata(file_paths)) # Save out_ds[variable_to_smooth].encoding.setdefault( diff --git a/wombat_ic_generation/SFe_climatology.py b/wombat_ic_generation/SFe_climatology.py index c157f990..f632cacc 100644 --- a/wombat_ic_generation/SFe_climatology.py +++ b/wombat_ic_generation/SFe_climatology.py @@ -62,17 +62,11 @@ def main(): ) args = parser.parse_args() - input_filename = os.path.abspath(args.input_filename) - output_filename = os.path.abspath(args.output_filename) + input_filename = args.input_filename + output_filename = args.output_filename # Obtain metadata - this_file = sys.argv[0] - runcmd = f"{sys.executable} {' '.join(sys.argv)}" - - history_attrs = { - "history": get_provenance_metadata(this_file, runcmd), - "inputFile": f"{input_filename} (md5 hash: {md5sum(input_filename)})", - } + history_attrs = get_provenance_metadata(input_files=[input_filename]) # Load the input data ds = xr.open_dataset(input_filename).compute() diff --git a/wombat_ic_generation/co2_iaf.py b/wombat_ic_generation/co2_iaf.py index ff9dc8b3..b9fc2913 100644 --- a/wombat_ic_generation/co2_iaf.py +++ b/wombat_ic_generation/co2_iaf.py @@ -38,7 +38,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, get_provenance_input_files +from scripts_common import get_provenance_metadata xr.set_options(keep_attrs=True) @@ -94,10 +94,7 @@ def main(): input_files = [co2_cmip_filename, co2_noaa_filename, hgrid_filename] - global_attrs = { - "history": get_provenance_metadata(this_file, runcmd), - "inputFile": get_provenance_input_files(input_files), - } + global_attrs = get_provenance_metadata(input_files) # Load the input data co2_cmip = xr.open_dataset(co2_cmip_filename, decode_cf=False).compute() diff --git a/wombat_ic_generation/generate_wombat_ic.py b/wombat_ic_generation/generate_wombat_ic.py index ef3b4a31..28ce8e12 100755 --- a/wombat_ic_generation/generate_wombat_ic.py +++ b/wombat_ic_generation/generate_wombat_ic.py @@ -457,14 +457,11 @@ def main(): wombat_version = args.wombat_version output_file = args.output_file - # Obtain metadata - this_file = sys.argv[0] - runcmd = f"{sys.executable} {' '.join(sys.argv)}" - global_attrs = { - "history": get_provenance_metadata(this_file, runcmd), "inputFile": set(), } + # Obtain metadata + global_attrs |= get_provenance_metadata() xr.set_options(keep_attrs=True) template = _open_woa_data( diff --git a/wombat_ic_generation/regrid_forcing.py b/wombat_ic_generation/regrid_forcing.py index c4963fe2..efb4b72b 100644 --- a/wombat_ic_generation/regrid_forcing.py +++ b/wombat_ic_generation/regrid_forcing.py @@ -94,12 +94,7 @@ def main(): forcing_regrid["ny"].encoding |= {"dtype": "float32"} # Obtain metadata - this_file = sys.argv[0] - runcmd = f"{sys.executable} {' '.join(sys.argv)}" - - global_attrs = { - "history": get_provenance_metadata(this_file, runcmd), - } + global_attrs = get_provenance_metadata() forcing_regrid.attrs = forcing_regrid.attrs | global_attrs From e4f8e893703c811045d888d6de2e8d00eb2fa94c Mon Sep 17 00:00:00 2001 From: anton-seaice Date: Tue, 30 Jun 2026 12:58:45 +1000 Subject: [PATCH 4/9] review comments --- ...erate_bottom_roughness_intermediate_woa.py | 26 ++++++------------- regrid_common.py | 2 +- .../generate_rofi_pattern.py | 4 --- scripts_common.py | 5 ++-- .../make_sfc_restore_from_regridded_woa.py | 3 --- wombat_ic_generation/co2_iaf.py | 6 +---- 6 files changed, 13 insertions(+), 33 deletions(-) diff --git a/external_tidal_generation/generate_bottom_roughness_intermediate_woa.py b/external_tidal_generation/generate_bottom_roughness_intermediate_woa.py index 4ee2b524..75768660 100644 --- a/external_tidal_generation/generate_bottom_roughness_intermediate_woa.py +++ b/external_tidal_generation/generate_bottom_roughness_intermediate_woa.py @@ -838,24 +838,14 @@ def main(): ) # Add provenance metadata and MD5 hashes for input files. - this_file = os.path.normpath(__file__) - runcmd = ( - f"mpirun -n $PBS_NCPUS python3 {os.path.basename(this_file)} " - f"--woa_temp_file={args.woa_temp_file} " - f"--woa_salt_file={args.woa_salt_file} " - f"--synbath_file={args.synbath_file} " - f"--chunk-lat={args.chunk_lat} " - f"--chunk-lon={args.chunk_lon} " - f"--nradial={args.nradial} " - f"--ntheta={args.ntheta} " - f"--earth-radius={args.earth_radius} " - f"--omega={args.omega} " - f"--print-every={args.print_every} " - f"--woa-intermediate-file={args.woa_intermediate_file} " - ) - - input_files = [args.woa_temp_file, args.woa_salt_file, args.synbath_file] - global_attrs = get_provenance_metadata(input_files) + runcmd = f"mpirun -n $PBS_NCPUS python3 {' '.join(sys.argv)} " + input_files = [ + args.woa_temp_file, + args.woa_salt_file, + args.synbath_file, + args.woa_intermediate_file, + ] + global_attrs = get_provenance_metadata(input_files, runcmd) ds_woa_output.attrs.update(global_attrs) diff --git a/regrid_common.py b/regrid_common.py index 1c84afb9..ee297532 100644 --- a/regrid_common.py +++ b/regrid_common.py @@ -15,7 +15,7 @@ import scipy.sparse as sp import scipy.sparse.linalg as spla -from scripts_common import md5sum +from scripts_common import get_provenance_metadata def _guess_longitude_name(ds): diff --git a/rof_pattern_generation/generate_rofi_pattern.py b/rof_pattern_generation/generate_rofi_pattern.py index 72bf93fe..41a00976 100755 --- a/rof_pattern_generation/generate_rofi_pattern.py +++ b/rof_pattern_generation/generate_rofi_pattern.py @@ -179,10 +179,6 @@ def main(): "units": "days since 0001-01-01 00:00:00", } - # Obtain metadata - this_file = sys.argv[0] - runcmd = f"{sys.executable} {' '.join(sys.argv)}" - # Info about input data used input_files = [ AQ_MELT_PATTERN, diff --git a/scripts_common.py b/scripts_common.py index a105a572..bc1c6368 100644 --- a/scripts_common.py +++ b/scripts_common.py @@ -103,8 +103,9 @@ def get_provenance_metadata(input_files=None, runcmd=None): file is not pushed to the git upstream repository. arguments: - file: the path to the file being run - runcmd: the command used to run the file (with any arguments) + input_files: list of input files being used in the script being run (optional) + runcmd: the command used to run the file, with any arguments. Optional - + defaults to the python executable + input arguments """ file = sys.argv[0] # script being run diff --git a/surface_restoring_generation/make_sfc_restore_from_regridded_woa.py b/surface_restoring_generation/make_sfc_restore_from_regridded_woa.py index a2df37d7..696091fe 100755 --- a/surface_restoring_generation/make_sfc_restore_from_regridded_woa.py +++ b/surface_restoring_generation/make_sfc_restore_from_regridded_woa.py @@ -94,9 +94,6 @@ def main(input_path, variable_to_smooth, output_file): out_ds["time"] = out_ds.time.assign_attrs({"modulo": " "}) # Obtain metadata - this_file = sys.argv[0] - runcmd = f"{sys.executable} {' '.join(sys.argv)}" - out_ds = out_ds.assign_attrs(get_provenance_metadata(file_paths)) # Save diff --git a/wombat_ic_generation/co2_iaf.py b/wombat_ic_generation/co2_iaf.py index b9fc2913..33b8bd82 100644 --- a/wombat_ic_generation/co2_iaf.py +++ b/wombat_ic_generation/co2_iaf.py @@ -89,11 +89,7 @@ def main(): output_filename = os.path.abspath(args.output_filename) # Obtain metadata - this_file = sys.argv[0] - runcmd = f"{sys.executable} {' '.join(sys.argv)}" - - input_files = [co2_cmip_filename, co2_noaa_filename, hgrid_filename] - + input_files = [args.co2_cmip_filename, args.co2_noaa_filename, args.hgrid_filename] global_attrs = get_provenance_metadata(input_files) # Load the input data From 5e82d99cc126eef19da613cce6f87105e8da3bcc Mon Sep 17 00:00:00 2001 From: anton-seaice Date: Tue, 30 Jun 2026 14:03:05 +1000 Subject: [PATCH 5/9] fixes --- grid_generation/generate_vertical_grid.py | 4 ++-- payu_config/postscript/build_intake_ds.py | 2 +- regrid_common.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/grid_generation/generate_vertical_grid.py b/grid_generation/generate_vertical_grid.py index ccc85636..76cff9ea 100644 --- a/grid_generation/generate_vertical_grid.py +++ b/grid_generation/generate_vertical_grid.py @@ -92,7 +92,7 @@ def generate_vertical_grid(H, dzd, min_dz, depfac, output_filename): ] # Trim the spacing values to match the adjusted depth levels # Write to NetCDF file - write_netcdf_file(output_filename, real_prop_z, this_file) + write_netcdf_file(output_filename, real_prop_z) print( f"SUCCESS! A vertical grid with {len(real_prop_z) - 1} levels has been generated. " @@ -101,7 +101,7 @@ def generate_vertical_grid(H, dzd, min_dz, depfac, output_filename): ) -def write_netcdf_file(output_filename, real_prop_z, this_file): +def write_netcdf_file(output_filename, real_prop_z): """Function to write vertical grid data to a NetCDF file.""" # Convert to float32 (single precision) to ensure values are exactly representable in single precision, # then convert back to float64 (double precision) for storage in NetCDF. diff --git a/payu_config/postscript/build_intake_ds.py b/payu_config/postscript/build_intake_ds.py index 3709a9ad..268de232 100644 --- a/payu_config/postscript/build_intake_ds.py +++ b/payu_config/postscript/build_intake_ds.py @@ -19,7 +19,7 @@ path_root = Path(__file__).parents[2] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, md5sum +from scripts_common import get_provenance_metadata METADATA_FILENAME = "metadata.yaml" UUID_FIELD = "experiment_uuid" diff --git a/regrid_common.py b/regrid_common.py index ee297532..9343ee09 100644 --- a/regrid_common.py +++ b/regrid_common.py @@ -15,7 +15,7 @@ import scipy.sparse as sp import scipy.sparse.linalg as spla -from scripts_common import get_provenance_metadata +from scripts_common import get_provenance_input_files def _guess_longitude_name(ds): From 79cc9e1f0d50d166fab737ef602a20b74bab309a Mon Sep 17 00:00:00 2001 From: anton-seaice Date: Tue, 30 Jun 2026 14:15:06 +1000 Subject: [PATCH 6/9] corrections --- scripts_common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts_common.py b/scripts_common.py index bc1c6368..cfe4e2d8 100644 --- a/scripts_common.py +++ b/scripts_common.py @@ -108,7 +108,7 @@ def get_provenance_metadata(input_files=None, runcmd=None): defaults to the python executable + input arguments """ - file = sys.argv[0] # script being run + file = os.path.abspath(sys.argv[0]) # script being run if runcmd is None: runcmd = f"{sys.executable} {' '.join(sys.argv)}" @@ -133,7 +133,7 @@ def get_provenance_metadata(input_files=None, runcmd=None): warn( f"{file} not under git version control! Add your file to a repository before generating any production output." ) - prepend += f"{os.path.abspath(file)} (md5 hash: {md5sum(file)}): " + prepend += f"{file} (md5 hash: {md5sum(file)}): " attrs = {"history": prepend + runcmd} From 1349da106c84b5bed9d33309b19a6e9fba8c9389 Mon Sep 17 00:00:00 2001 From: anton-seaice Date: Wed, 1 Jul 2026 13:52:55 +1000 Subject: [PATCH 7/9] corrections --- wombat_ic_generation/co2_iaf.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/wombat_ic_generation/co2_iaf.py b/wombat_ic_generation/co2_iaf.py index 33b8bd82..76ab5255 100644 --- a/wombat_ic_generation/co2_iaf.py +++ b/wombat_ic_generation/co2_iaf.py @@ -86,10 +86,13 @@ def main(): args = parser.parse_args() + co2_cmip_filename = os.path.abspath(args.co2_cmip_filename) + co2_noaa_filename = os.path.abspath(args.co2_noaa_filename) + hgrid_filename = os.path.abspath(args.hgrid_filename) output_filename = os.path.abspath(args.output_filename) # Obtain metadata - input_files = [args.co2_cmip_filename, args.co2_noaa_filename, args.hgrid_filename] + input_files = [co2_cmip_filename, co2_noaa_filename, hgrid_filename] global_attrs = get_provenance_metadata(input_files) # Load the input data From 182486ccd25d7c7e674d7bc470c409fad7ad74c0 Mon Sep 17 00:00:00 2001 From: anton-seaice Date: Thu, 2 Jul 2026 16:36:05 +1000 Subject: [PATCH 8/9] Review comments --- .../generate_bottom_roughness_intermediate_woa.py | 1 - external_tidal_generation/pbs_bottom_roughness.pbs | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/external_tidal_generation/generate_bottom_roughness_intermediate_woa.py b/external_tidal_generation/generate_bottom_roughness_intermediate_woa.py index 75768660..0de8ef8d 100644 --- a/external_tidal_generation/generate_bottom_roughness_intermediate_woa.py +++ b/external_tidal_generation/generate_bottom_roughness_intermediate_woa.py @@ -843,7 +843,6 @@ def main(): args.woa_temp_file, args.woa_salt_file, args.synbath_file, - args.woa_intermediate_file, ] global_attrs = get_provenance_metadata(input_files, runcmd) diff --git a/external_tidal_generation/pbs_bottom_roughness.pbs b/external_tidal_generation/pbs_bottom_roughness.pbs index 163441b3..ccbbf4d7 100644 --- a/external_tidal_generation/pbs_bottom_roughness.pbs +++ b/external_tidal_generation/pbs_bottom_roughness.pbs @@ -6,6 +6,7 @@ #PBS -l walltime=10:00:00 #PBS -l mem=500GB #PBS -l ncpus=72 +#PBS -l wd ##PBS -j oe # Bottom roughness generation (h^2) on Gadi @@ -67,8 +68,8 @@ LOG_INTERMEDIATE="${PATH_TO_SAVE}/log_bottom_roughness_intermediate_${RESOLUTION LOG_REGRID="${PATH_TO_SAVE}/log_bottom_roughness_regrid_${RESOLUTION}.log" # script paths -INTERMEDIATE_SCRIPT="/g/data/vk83/apps/om3-scripts/external_tidal_generation/generate_bottom_roughness_intermediate_woa.py" -REGRID_SCRIPT="/g/data/vk83/apps/om3-scripts/external_tidal_generation/generate_bottom_roughness_regrid.py" +INTERMEDIATE_SCRIPT="/g/data/tm70/as2285/making_inputs/make_om3_topo/om3-scripts/external_tidal_generation/generate_bottom_roughness_intermediate_woa.py" +REGRID_SCRIPT="/g/data/tm70/as2285/making_inputs/make_om3_topo/om3-scripts/external_tidal_generation/generate_bottom_roughness_regrid.py" # Input paths - shared across resolutions # Loads woa23 0.25deg annual mean climatology From 15d2cb66b059a113529e213fcb1920fb0a551eb4 Mon Sep 17 00:00:00 2001 From: anton-seaice Date: Fri, 3 Jul 2026 12:25:56 +1000 Subject: [PATCH 9/9] corrections --- external_tidal_generation/pbs_bottom_roughness.pbs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/external_tidal_generation/pbs_bottom_roughness.pbs b/external_tidal_generation/pbs_bottom_roughness.pbs index ccbbf4d7..163441b3 100644 --- a/external_tidal_generation/pbs_bottom_roughness.pbs +++ b/external_tidal_generation/pbs_bottom_roughness.pbs @@ -6,7 +6,6 @@ #PBS -l walltime=10:00:00 #PBS -l mem=500GB #PBS -l ncpus=72 -#PBS -l wd ##PBS -j oe # Bottom roughness generation (h^2) on Gadi @@ -68,8 +67,8 @@ LOG_INTERMEDIATE="${PATH_TO_SAVE}/log_bottom_roughness_intermediate_${RESOLUTION LOG_REGRID="${PATH_TO_SAVE}/log_bottom_roughness_regrid_${RESOLUTION}.log" # script paths -INTERMEDIATE_SCRIPT="/g/data/tm70/as2285/making_inputs/make_om3_topo/om3-scripts/external_tidal_generation/generate_bottom_roughness_intermediate_woa.py" -REGRID_SCRIPT="/g/data/tm70/as2285/making_inputs/make_om3_topo/om3-scripts/external_tidal_generation/generate_bottom_roughness_regrid.py" +INTERMEDIATE_SCRIPT="/g/data/vk83/apps/om3-scripts/external_tidal_generation/generate_bottom_roughness_intermediate_woa.py" +REGRID_SCRIPT="/g/data/vk83/apps/om3-scripts/external_tidal_generation/generate_bottom_roughness_regrid.py" # Input paths - shared across resolutions # Loads woa23 0.25deg annual mean climatology