diff --git a/chlorophyll/chl_climatology_and_fill.py b/chlorophyll/chl_climatology_and_fill.py index ecd645ee..1930de9d 100644 --- a/chlorophyll/chl_climatology_and_fill.py +++ b/chlorophyll/chl_climatology_and_fill.py @@ -70,18 +70,6 @@ def main(): input_directory = os.path.abspath(args.input_directory) output_filename = os.path.abspath(args.output_filename) - this_file = os.path.normpath(__file__) - - # Add some info about how the file was generated - runcmd = ( - f"python3 {os.path.basename(this_file)} --input-directory={input_directory} " - f"--output-filename={output_filename}" - ) - - history_attrs = { - "history": get_provenance_metadata(this_file, runcmd), - } - # Load the input data and compute the monthly climatology input_files = sorted(glob.glob(f"{input_directory}/*.nc")) @@ -160,7 +148,7 @@ def main(): chl.time.attrs["long_name"] = "Time" chl.time.attrs["standard_name"] = "time" chl.time.attrs["axis"] = "T" - chl.attrs |= history_attrs + chl.attrs |= get_provenance_metadata(input_files) comp = dict(zlib=True, complevel=4) encoding = {var: comp for var in chl.data_vars} # Time coords should be double type according for CF conventions diff --git a/data_stream_xml_generation/generate_xml_datm_era5.py b/data_stream_xml_generation/generate_xml_datm_era5.py index ea279f20..4f6b0f9b 100644 --- a/data_stream_xml_generation/generate_xml_datm_era5.py +++ b/data_stream_xml_generation/generate_xml_datm_era5.py @@ -85,9 +85,7 @@ root = Element("file", id="stream", version="2.0") # Obtain metadata -this_file = sys.argv[0] -runcmd = " ".join(sys.argv) -metadata_info = get_provenance_metadata(this_file, runcmd) +metadata_info = get_provenance_metadata() # Add metadata metadata = SubElement(root, "metadata") @@ -95,7 +93,7 @@ SubElement(metadata, "date_generated").text = datetime.now().strftime( "%Y-%m-%d %H:%M:%S" ) -SubElement(metadata, "history").text = metadata_info +SubElement(metadata, "history").text = metadata_info["history"] # Generate stream info elements with changing years for stream_name, era5_prefix, datavar_pairs, mapalgo, offset_seconds in STREAM_SPECS: diff --git a/data_stream_xml_generation/generate_xml_datm_jra55.py b/data_stream_xml_generation/generate_xml_datm_jra55.py index 91339653..19c1f4d0 100644 --- a/data_stream_xml_generation/generate_xml_datm_jra55.py +++ b/data_stream_xml_generation/generate_xml_datm_jra55.py @@ -61,9 +61,7 @@ root = Element("file", id="stream", version="2.0") # Obtain metadata -this_file = sys.argv[0] -runcmd = " ".join(sys.argv) -metadata_info = get_provenance_metadata(this_file, runcmd) +metadata_info = get_provenance_metadata() # Add metadata metadata = SubElement(root, "metadata") @@ -71,7 +69,7 @@ SubElement(metadata, "date_generated").text = datetime.now().strftime( "%Y-%m-%d %H:%M:%S" ) -SubElement(metadata, "history").text = metadata_info +SubElement(metadata, "history").text = metadata_info["history"] # Define the stream info names and corresponding var names stream_info_names = [*STREAMS_AVE, *STREAMS_PT] diff --git a/data_stream_xml_generation/generate_xml_drof_jra55.py b/data_stream_xml_generation/generate_xml_drof_jra55.py index 455a6253..5b331a74 100644 --- a/data_stream_xml_generation/generate_xml_drof_jra55.py +++ b/data_stream_xml_generation/generate_xml_drof_jra55.py @@ -42,9 +42,7 @@ root = Element("file", id="stream", version="2.0") # Obtain metadata -this_file = sys.argv[0] -runcmd = " ".join(sys.argv) -metadata_info = get_provenance_metadata(this_file, runcmd) +metadata_info = get_provenance_metadata() # Add metadata metadata = SubElement(root, "metadata") @@ -52,7 +50,7 @@ SubElement(metadata, "date_generated").text = datetime.now().strftime( "%Y-%m-%d %H:%M:%S" ) -SubElement(metadata, "history").text = metadata_info +SubElement(metadata, "history").text = metadata_info["history"] # Define the stream info names and corresponding var names stream_info_data = [ diff --git a/external_tidal_generation/generate_bottom_roughness_intermediate_woa.py b/external_tidal_generation/generate_bottom_roughness_intermediate_woa.py index 9523d1e7..0de8ef8d 100644 --- a/external_tidal_generation/generate_bottom_roughness_intermediate_woa.py +++ b/external_tidal_generation/generate_bottom_roughness_intermediate_woa.py @@ -135,7 +135,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, md5sum +from scripts_common import get_provenance_metadata def coriolis_f(lat: xr.DataArray) -> xr.DataArray: @@ -838,30 +838,14 @@ def main(): ) # Add provenance metadata and MD5 hashes for input files. - this_file = os.path.normpath(__file__) - runcmd = ( - f"mpirun -n $PBS_NCPUS python3 {os.path.basename(this_file)} " - f"--woa_temp_file={args.woa_temp_file} " - f"--woa_salt_file={args.woa_salt_file} " - f"--synbath_file={args.synbath_file} " - f"--chunk-lat={args.chunk_lat} " - f"--chunk-lon={args.chunk_lon} " - f"--nradial={args.nradial} " - f"--ntheta={args.ntheta} " - f"--earth-radius={args.earth_radius} " - f"--omega={args.omega} " - f"--print-every={args.print_every} " - f"--woa-intermediate-file={args.woa_intermediate_file} " - ) - - history = get_provenance_metadata(this_file, runcmd) - global_attrs = {"history": history} - file_hashes = [ - f"{args.woa_temp_file} (md5 hash: {md5sum(args.woa_temp_file)})", - f"{args.woa_salt_file} (md5 hash: {md5sum(args.woa_salt_file)})", - f"{args.synbath_file} (md5 hash: {md5sum(args.synbath_file)})", + runcmd = f"mpirun -n $PBS_NCPUS python3 {' '.join(sys.argv)} " + input_files = [ + args.woa_temp_file, + args.woa_salt_file, + args.synbath_file, ] - global_attrs["inputFile"] = ", ".join(file_hashes) + global_attrs = get_provenance_metadata(input_files, runcmd) + ds_woa_output.attrs.update(global_attrs) ds_woa_output.to_netcdf(args.woa_intermediate_file) diff --git a/external_tidal_generation/generate_bottom_roughness_polyfit.py b/external_tidal_generation/generate_bottom_roughness_polyfit.py index 8dfc0e4a..e75c5574 100644 --- a/external_tidal_generation/generate_bottom_roughness_polyfit.py +++ b/external_tidal_generation/generate_bottom_roughness_polyfit.py @@ -39,7 +39,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, md5sum +from scripts_common import get_provenance_metadata from mesh_generation.generate_mesh import mom6_mask_detection @@ -383,25 +383,10 @@ def main(): ) # Add provenance metadata and MD5 hashes for input files. - this_file = os.path.normpath(__file__) - runcmd = ( - f"mpirun -n $PBS_NCPUS python3 {os.path.basename(this_file)} " - f"--high-res-topo-file={args.high_res_topo_file} " - f"--hgrid-file={args.hgrid_file} " - f"--topog-file={args.topog_file} " - f"--chunk-lat={args.chunk_lat} " - f"--chunk-lon={args.chunk_lon} " - f"--output={args.output}" - ) + runcmd = f"mpirun -n $PBS_NCPUS python3 {' '.join(sys.argv)} " + input_files = [args.high_res_topo_file, args.hgrid_file, args.topog_file] + global_attrs = get_provenance_metadata(input_files, runcmd) - history = get_provenance_metadata(this_file, runcmd) - global_attrs = {"history": history} - file_hashes = [ - f"{args.high_res_topo_file} (md5 hash: {md5sum(args.high_res_topo_file)})", - f"{args.hgrid_file} (md5 hash: {md5sum(args.hgrid_file)})", - f"{args.topog_file} (md5 hash: {md5sum(args.topog_file)})", - ] - global_attrs["inputFile"] = ", ".join(file_hashes) h2_out.attrs.update(global_attrs) h2_out.to_netcdf(args.output) diff --git a/external_tidal_generation/generate_bottom_roughness_regrid.py b/external_tidal_generation/generate_bottom_roughness_regrid.py index 567ee1e0..0bb985b7 100644 --- a/external_tidal_generation/generate_bottom_roughness_regrid.py +++ b/external_tidal_generation/generate_bottom_roughness_regrid.py @@ -53,7 +53,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, md5sum +from scripts_common import get_provenance_metadata from mesh_generation.generate_mesh import mom6_mask_detection @@ -396,26 +396,8 @@ def main(): print("Regridding done!") # Add provenance metadata and MD5 hashes for input files. - this_file = os.path.normpath(__file__) - runcmd = ( - f"python3 {os.path.basename(this_file)} " - f"--topog_file={args.topog_file} " - f"--hgrid_file={args.hgrid_file} " - f"--woa_intermediate_file={args.woa_intermediate_file} " - f"--output_file={args.output_file} " - f"--method={args.method} " - f"--periodic_regrid={args.periodic_regrid} " - f"--periodic_lon_laplace={args.periodic_lon_laplace}" - ) - - history = get_provenance_metadata(this_file, runcmd) - global_attrs = {"history": history} - file_hashes = [ - f"{args.hgrid_file} (md5 hash: {md5sum(args.hgrid_file)})", - f"{args.topog_file} (md5 hash: {md5sum(args.topog_file)})", - f"{args.woa_intermediate_file} (md5 hash: {md5sum(args.woa_intermediate_file)})", - ] - global_attrs["inputFile"] = ", ".join(file_hashes) + input_files = [args.hgrid_file, args.topog_file, args.woa_intermediate_file] + global_attrs = get_provenance_metadata(input_files) regrid_depth_var.attrs.update(global_attrs) output_path = Path(args.output_file) diff --git a/external_tidal_generation/generate_tide_amplitude.py b/external_tidal_generation/generate_tide_amplitude.py index a707f4d9..f3b021f4 100644 --- a/external_tidal_generation/generate_tide_amplitude.py +++ b/external_tidal_generation/generate_tide_amplitude.py @@ -38,7 +38,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, md5sum +from scripts_common import get_provenance_metadata from mesh_generation.generate_mesh import mom6_mask_detection PRIMARY_CONSTITUENTS = ["m2", "s2", "n2", "k2", "k1", "o1", "p1", "q1"] @@ -277,25 +277,8 @@ def main(): tideamp = tideamp.rename({"x": "xh", "y": "yh"}) # Add provenance metadata and MD5 hashes for input files. - this_file = os.path.normpath(__file__) - runcmd = ( - f"python3 {os.path.basename(this_file)} " - f"--hgrid-file={args.hgrid_file} " - f"--topog-file={args.topog_file} " - f"--method={args.method} " - f"--data-path={args.data_path} " - f"--output={args.output} " - ) - - history = get_provenance_metadata(this_file, runcmd) - global_attrs = {"history": history} - - # add md5 hashes for input files - file_hashes = [ - f"{args.hgrid_file} (md5 hash: {md5sum(args.hgrid_file)})", - f"{args.topog_file} (md5 hash: {md5sum(args.topog_file)})", - ] - global_attrs["inputFile"] = ", ".join(file_hashes) + input_files = [args.hgrid_file, args.topog_file] + global_attrs = get_provenance_metadata(input_files) tideamp.attrs.update(global_attrs) tideamp.to_netcdf(args.output, unlimited_dims=["time"]) diff --git a/grid_generation/generate_vertical_grid.py b/grid_generation/generate_vertical_grid.py index a310728f..76cff9ea 100644 --- a/grid_generation/generate_vertical_grid.py +++ b/grid_generation/generate_vertical_grid.py @@ -38,7 +38,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, md5sum +from scripts_common import get_provenance_metadata # Define a small constant to initialize the iteration and prevent numerical issues epsilon = 0.001 @@ -91,18 +91,8 @@ def generate_vertical_grid(H, dzd, min_dz, depfac, output_filename): : len(real_prop_z) ] # Trim the spacing values to match the adjusted depth levels - this_file = os.path.normpath(__file__) - - # Add some info about how the file was generated - runcmd = ( - f"python3 {os.path.basename(this_file)} --H={H} --depfac={depfac} " - f"--dzd={dzd} " - f"--min_dz={min_dz} " - f"--output={output_filename} " - ) - # Write to NetCDF file - write_netcdf_file(output_filename, real_prop_z, this_file, runcmd) + write_netcdf_file(output_filename, real_prop_z) print( f"SUCCESS! A vertical grid with {len(real_prop_z) - 1} levels has been generated. " @@ -111,7 +101,7 @@ def generate_vertical_grid(H, dzd, min_dz, depfac, output_filename): ) -def write_netcdf_file(output_filename, real_prop_z, this_file, runcmd): +def write_netcdf_file(output_filename, real_prop_z): """Function to write vertical grid data to a NetCDF file.""" # Convert to float32 (single precision) to ensure values are exactly representable in single precision, # then convert back to float64 (double precision) for storage in NetCDF. @@ -125,7 +115,7 @@ def write_netcdf_file(output_filename, real_prop_z, this_file, runcmd): zeta.standard_name = "depth" zeta.long_name = "vertical grid depth at top and bottom of each cell" eddyfile.variables["zeta"][:] = real_prop_z_float64 - eddyfile.setncatts({"history": get_provenance_metadata(this_file, runcmd)}) + eddyfile.setncatts(get_provenance_metadata()) # Obtain metadata eddyfile.close() diff --git a/make_ryf/make_ryf.py b/make_ryf/make_ryf.py index b86d1a44..8ffaada3 100644 --- a/make_ryf/make_ryf.py +++ b/make_ryf/make_ryf.py @@ -38,7 +38,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, md5sum +from scripts_common import get_provenance_metadata FILLVALUE = 1e20 # compression settings to use @@ -113,7 +113,7 @@ ds = {} for var in variables: - ryf_files = str() + input_files = [] print(var) for y in (year1, year2): if source_data == "jra55v1p4" or source_data == "jra55v1p6": @@ -147,7 +147,7 @@ print("Loading {} for {}".format(files[0], y)) ds[y] = xarray.open_dataset(files[0], decode_coords=False) # save info for metadata - ryf_files += f"{files[0]} (md5 hash: {md5sum(files[0])}, )" + input_files.append(files[0]) # Make a copy of the second year without time_bnds ryf = ds[baseyear].drop_vars("time_bnds") ryf.encoding = ds[baseyear].encoding @@ -205,10 +205,7 @@ ) # Add some info about how the file was generated - this_file = os.path.normpath(__file__) - runcmd = f"python3 {os.path.basename(this_file)}" - ryf.attrs |= {"RYF_creation": get_provenance_metadata(this_file, runcmd)} - ryf.attrs |= {"RYF_inputFiles": ryf_files} + ryf.attrs |= get_provenance_metadata(input_files) outfile = "RYF.{}.{}_{}.nc".format(var, year1, year2) print("Writing ", outfile) diff --git a/mesh_generation/generate_mesh.py b/mesh_generation/generate_mesh.py index d312f445..511d3b78 100755 --- a/mesh_generation/generate_mesh.py +++ b/mesh_generation/generate_mesh.py @@ -43,7 +43,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, md5sum +from scripts_common import get_provenance_metadata EARTH_R = 6.37122e6 @@ -126,7 +126,7 @@ def __init__( self.mesh = None - def create_mesh(self, wrap_lons=True, global_attrs=None): + def create_mesh(self, wrap_lons=True): """ Create the mesh as an xarray Dataset @@ -134,8 +134,6 @@ def create_mesh(self, wrap_lons=True, global_attrs=None): ---------- wrap_lons: boolean, optional If True, wrap longitude values into the range between 0 and 360 - global_attrs: dict - Global attributes to the mesh object """ if wrap_lons: @@ -205,15 +203,7 @@ def create_mesh(self, wrap_lons=True, global_attrs=None): "timeGenerated": f"{datetime.now()}", "created_by": f"{os.environ.get('USER')}", } - if self.inputs: - file_hashes = [] - for input in self.inputs: - file_hashes.append(f"{input} (md5 hash: {md5sum(input)})") - ds.attrs["inputFile"] = ", ".join(file_hashes) - - # add git info to history - if global_attrs: - ds.attrs |= global_attrs + ds.attrs |= get_provenance_metadata(input_files=self.inputs) self.mesh = ds @@ -589,29 +579,6 @@ def main(): if topog_filename: topog_filename = os.path.abspath(topog_filename) - this_file = os.path.normpath(__file__) - - # Add some info about how the file was generated - runcmd = ( - f"python3 {os.path.basename(this_file)} --grid-type={grid_type} --grid-filename={grid_filename} " - f"--mesh-filename={mesh_filename}" - ) - if topog_filename: - runcmd += f" --topog-filename={topog_filename}" - if minimum_depth: - runcmd += f" --minimum-depth={minimum_depth}" - if masking_depth: - runcmd += f" --masking-depth={masking_depth}" - runcmd += f" --wrap-lons={wrap_lons}" - if lon_name: - runcmd += f" --lon-name={lon_name}" - if lat_name: - runcmd += f" --lat-name={lat_name}" - if area_name: - runcmd += f" --area-name={area_name}" - - global_attrs = {"history": get_provenance_metadata(this_file, runcmd)} - mesh = gridtype_dispatch[grid_type]( grid_filename, topog_filename, @@ -622,9 +589,7 @@ def main(): masking_depth, ) - mesh.create_mesh(wrap_lons=wrap_lons, global_attrs=global_attrs).write( - mesh_filename - ) + mesh.create_mesh(wrap_lons=wrap_lons).write(mesh_filename) if __name__ == "__main__": diff --git a/mesh_generation/generate_rof_weights.py b/mesh_generation/generate_rof_weights.py index 88c63401..935859ea 100755 --- a/mesh_generation/generate_rof_weights.py +++ b/mesh_generation/generate_rof_weights.py @@ -42,13 +42,13 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, md5sum +from scripts_common import get_provenance_metadata TEMP_WEIGHTS_F = "temp_weights.nc" COMP_ENCODING = {"complevel": 1, "compression": "zlib"} # compression settings to use -def drof_remapping_weights(mesh_filename, weights_filename, nx, ny, global_attrs=None): +def drof_remapping_weights(mesh_filename, weights_filename, nx, ny): # We need to generate remapping weights for use in the mediator, such that the overall volume of runoff is conserved # and no runoff is mapped onto land cells. Inside the mediator, the grid doesn't change as we run the mediator with # the ocean grid (the DROF component does the remapping from JRA grid to mediator grid). Therefore we use the @@ -139,12 +139,10 @@ def drof_remapping_weights(mesh_filename, weights_filename, nx, ny, global_attrs # add global attributes weights_ds.attrs = { "gridType": "unstructured mesh", - "inputFile": f"{mesh_filename} (md5 hash: {md5sum(mesh_filename)})", } # add git info to history - if global_attrs: - weights_ds.attrs |= global_attrs + weights_ds.attrs |= get_provenance_metadata([mesh_filename]) # save (compressed) encoding = {} @@ -188,19 +186,8 @@ def main(): ) args = parser.parse_args() - mesh_filename = os.path.abspath(args.mesh_filename) - weights_filename = os.path.abspath(args.weights_filename) - this_file = os.path.normpath(__file__) - - # Add some info about how the file was generated - runcmd = f"python3 {os.path.basename(this_file)} --mesh_filename={mesh_filename} --nx={args.nx} --ny={args.ny} --weights_filename={weights_filename} " - - global_attrs = {"history": get_provenance_metadata(this_file, runcmd)} - - drof_remapping_weights( - mesh_filename, weights_filename, args.nx, args.ny, global_attrs - ) + drof_remapping_weights(args.mesh_filename, args.weights_filename, args.nx, args.ny) return True diff --git a/payu_config/postscript/build_intake_ds.py b/payu_config/postscript/build_intake_ds.py index ac4c7595..268de232 100644 --- a/payu_config/postscript/build_intake_ds.py +++ b/payu_config/postscript/build_intake_ds.py @@ -19,7 +19,7 @@ path_root = Path(__file__).parents[2] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, md5sum +from scripts_common import get_provenance_metadata METADATA_FILENAME = "metadata.yaml" UUID_FIELD = "experiment_uuid" @@ -39,18 +39,12 @@ def description(): warn(f"{METADATA_FILENAME} not found in config folder") uuid = False - # Check git status of this .py file - this_file = os.path.normpath(__file__) - - runcmd = f"python3 {os.path.basename(this_file)}" - - # Get string "Created using $file: $command" - provenance = get_provenance_metadata(this_file, runcmd) + provenance = get_provenance_metadata() if uuid: - description = f"intake-esm datastore for experiment {uuid}, in folder {os.getcwd()}. {provenance}. (md5 hash: {md5sum(this_file)})" + description = f"intake-esm datastore for experiment {uuid}, in folder {os.getcwd()}. {provenance['history']}." else: - description = f"intake-esm datastore for experiment in folder {os.getcwd()}. {provenance}. (md5 hash: {md5sum(this_file)})" + description = f"intake-esm datastore for experiment in folder {os.getcwd()}. {provenance['history']}." return description diff --git a/payu_config/postscript/concat_ice_daily.py b/payu_config/postscript/concat_ice_daily.py index 28ee6bd3..50e9bbd9 100644 --- a/payu_config/postscript/concat_ice_daily.py +++ b/payu_config/postscript/concat_ice_daily.py @@ -29,7 +29,7 @@ path_root = Path(__file__).parents[2] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, md5sum +from scripts_common import get_provenance_metadata CICE_DAILY_FN = "access-om3.cice.1day.mean.????-??-??.nc" MONTHLY_STUB_FN = "access-om3.cice.1day.mean." @@ -134,9 +134,7 @@ def __init__(self, directory=None, assume_gadi=True): del daily_ds.attrs["comment3"] # Add some info about how the file was generated - this_file = os.path.normpath(__file__) - runcmd = f"python3 {os.path.basename(this_file)} --directory={os.path.abspath(directory)}" - daily_ds.attrs["postprocessing"] = get_provenance_metadata(this_file, runcmd) + daily_ds.attrs |= get_provenance_metadata() self.directory = directory self.daily_ds = daily_ds diff --git a/regrid_common.py b/regrid_common.py index dbb01ab7..9343ee09 100644 --- a/regrid_common.py +++ b/regrid_common.py @@ -15,7 +15,7 @@ import scipy.sparse as sp import scipy.sparse.linalg as spla -from scripts_common import md5sum +from scripts_common import get_provenance_input_files def _guess_longitude_name(ds): @@ -119,19 +119,6 @@ def parse_cli(self): if self.mask_filename: self.mask_filename = os.path.abspath(self.mask_filename) - # some info about how the file was generated - - runcmd_args = f"--hgrid-filename={self.hgrid_filename} --output-filename={self.output_filename}" - - if self.mask_filename: - runcmd_args += f" --mask-filename={self.mask_filename}" - if self.lon_name: - runcmd_args += f" --lon-name={self.lon_name}" - if self.lat_name: - runcmd_args += f" --lat-name={self.lat_name}" - - self.runcmd_args = runcmd_args - ## NOTE: it's implied that forcing_filename is set outside this class def open_datasets(self): @@ -229,17 +216,12 @@ def save_output(self): forcing_regrid = self.forcing_regrid # Info about input data used - file_hashes = [ - f"{self.forcing_filename} (md5 hash: {md5sum(self.forcing_filename)})", - f"{self.hgrid_filename} (md5 hash: {md5sum(self.hgrid_filename)})", - ] + input_files = [self.forcing_filename, self.hgrid_filename] if self.mask_filename: - file_hashes.append( - f"{self.mask_filename} (md5 hash: {md5sum(self.mask_filename)})" - ) + input_files.append(self.mask_filename) global_attrs = { - "inputFile": ", ".join(file_hashes), + "inputFile": get_provenance_input_files(input_files), } forcing_regrid.attrs = forcing_regrid.attrs | global_attrs diff --git a/restart_modifications/apply_bathy_mom_restarts.py b/restart_modifications/apply_bathy_mom_restarts.py index f8fa41f4..3e56faac 100644 --- a/restart_modifications/apply_bathy_mom_restarts.py +++ b/restart_modifications/apply_bathy_mom_restarts.py @@ -45,14 +45,14 @@ def copy_data(args): - in_f, out_f, runcmd, this_f, template_f = args + in_f, out_f, template_f = args print(f"\n[INFO] Processing:\n OLD: {in_f}\n OUT: {out_f}") try: with nc.Dataset(in_f) as in_fp, nc.Dataset(out_f, "r+") as out_fp: # Add metadata out_fp.setncattr("title", "Updated restart file with new bathymetry mask") out_fp.setncattr("source", "Generated by fix_mom6_restarts.py") - out_fp.setncattr("history", get_provenance_metadata(this_f, runcmd)) + out_fp.setncatts(get_provenance_metadata()) out_fp.setncattr( "template_file", f"{os.path.abspath(template_f)} (md5 hash: {md5sum(template_f)})", @@ -145,9 +145,6 @@ def main(): print(f"[COPY] {tf} -> {out_f}") shutil.copy(tf, out_f) - this_file = os.path.normpath(__file__) - runcmd = f"python3 {os.path.basename(this_file)} --template_dir {args.template_dir} --old_dir {args.old_dir} --output_dir {args.output_dir} --template_prefix {args.template_prefix} --old_prefix {args.old_prefix} --nprocs {args.nprocs}" - print("[INFO] Starting update of variables...") # update each restart file on a different proc pool = mp.Pool(args.nprocs) @@ -156,8 +153,6 @@ def main(): zip( old_files, output_files, - [runcmd] * len(old_files), - [this_file] * len(old_files), template_files, ), ) diff --git a/restart_modifications/remask_cpl_restart.py b/restart_modifications/remask_cpl_restart.py index 0419bd1c..80acfd58 100644 --- a/restart_modifications/remask_cpl_restart.py +++ b/restart_modifications/remask_cpl_restart.py @@ -157,21 +157,12 @@ def main(): unmask_file(f, mask, missing_value, skip_vars=skip_vars) apply_mask_file(f, mask, skip_vars=skip_vars) - this_file = os.path.normpath(__file__) - runcmd = ( - f"python3 {os.path.basename(this_file)} " - f"--input_file {args.input_file} " - f"--output_file {args.output_file} " - f"--mask_file {args.mask_file} " - f"--mask_var {args.mask_var}" - ) - # Add metadata f.setncattr( "title", "Coupler restart fields updated with land mask of modified bathymetry", ) - f.setncattr("history", get_provenance_metadata(this_file, runcmd)) + f.setncatts(get_provenance_metadata()) f.setncattr( "mask_file", f"{os.path.abspath(args.mask_file)} (md5 hash: {md5sum(args.mask_file)})", diff --git a/rof_pattern_generation/generate_rofi_pattern.py b/rof_pattern_generation/generate_rofi_pattern.py index 04f0dbba..41a00976 100755 --- a/rof_pattern_generation/generate_rofi_pattern.py +++ b/rof_pattern_generation/generate_rofi_pattern.py @@ -40,7 +40,7 @@ sys.path.append(str(path_root)) from regrid_common import Regrid_Common -from scripts_common import get_provenance_metadata, md5sum +from scripts_common import get_provenance_metadata from mesh_generation.generate_mesh import mom6_mask_detection # in a climatology, with 365 day calendar, whats the day of the middle of each month @@ -179,31 +179,20 @@ def main(): "units": "days since 0001-01-01 00:00:00", } - # Add some info about how the file was generated - this_file = os.path.normpath(__file__) - - runcmd = ( - f"python3 {os.path.basename(this_file)} --topog-file={regrid.args.topog_file} " - f"{regrid.runcmd_args}" - ) - # Info about input data used - file_hashes = [ - f"{AQ_MELT_PATTERN} (md5 hash: {md5sum(AQ_MELT_PATTERN)})", - f"{GL_MELT_PATTERN} (md5 hash: {md5sum(GL_MELT_PATTERN)})", - f"{regrid.hgrid_filename} (md5 hash: {md5sum(regrid.hgrid_filename)})", - f"{regrid.args.topog_file} (md5 hash: {md5sum(regrid.args.topog_file)})", + input_files = [ + AQ_MELT_PATTERN, + GL_MELT_PATTERN, + regrid.hgrid_filename, + regrid.args.topog_file, ] if regrid.mask_filename: - file_hashes.append( - f"{regrid.mask_filename} (md5 hash: {md5sum(regrid.mask_filename)})" - ) + input_files.append(regrid.mask_filename) global_attrs |= { "description": "Mankoff 2025 iceberg spreading climatology remapped onto an ACCESS-OM3 grid", - "history": get_provenance_metadata(this_file, runcmd), - "inputFile": ", ".join(file_hashes), } + global_attrs |= get_provenance_metadata(input_files) weights_ds.attrs = weights_ds.attrs | global_attrs diff --git a/scripts_common.py b/scripts_common.py index 10e7e639..cfe4e2d8 100644 --- a/scripts_common.py +++ b/scripts_common.py @@ -7,7 +7,7 @@ import subprocess -import os +import os, sys from warnings import warn import io import hashlib @@ -97,15 +97,21 @@ def username(file): return name -def get_provenance_metadata(file, runcmd): +def get_provenance_metadata(input_files=None, runcmd=None): """ - Return a string with the provenance of the file being run. Warn if the file is not pushed to the git upstream repository. + Return a dictionary with the provenance of the file being run. Warn if the + file is not pushed to the git upstream repository. arguments: - file: the path to the file being run - runcmd: the command used to run the file (with any arguments) + input_files: list of input files being used in the script being run (optional) + runcmd: the command used to run the file, with any arguments. Optional - + defaults to the python executable + input arguments """ + file = os.path.abspath(sys.argv[0]) # script being run + if runcmd is None: + runcmd = f"{sys.executable} {' '.join(sys.argv)}" + prepend = ( f"Created by {username(file)} on {datetime.now().strftime('%Y-%m-%d')}, using " ) @@ -127,9 +133,14 @@ def get_provenance_metadata(file, runcmd): warn( f"{file} not under git version control! Add your file to a repository before generating any production output." ) - prepend += f"{file}: " + prepend += f"{file} (md5 hash: {md5sum(file)}): " + + attrs = {"history": prepend + runcmd} - return prepend + runcmd + if input_files is not None: + attrs["inputFile"] = get_provenance_input_files(input_files) + + return attrs def md5sum(path): @@ -144,3 +155,13 @@ def md5sum(path): for chunk in iter(lambda: fd.read(length), b""): md5.update(chunk) return md5.hexdigest() + + +def get_provenance_input_files(input_files): + """ + Return a formatted string of provided input files and their md5 hashes + """ + file_hashes = [] + for input in input_files: + file_hashes.append(f"{os.path.abspath(input)} (md5 hash: {md5sum(input)})") + return ", ".join(file_hashes) diff --git a/surface_restoring_generation/make_sfc_restore_from_regridded_woa.py b/surface_restoring_generation/make_sfc_restore_from_regridded_woa.py index c17e4478..696091fe 100755 --- a/surface_restoring_generation/make_sfc_restore_from_regridded_woa.py +++ b/surface_restoring_generation/make_sfc_restore_from_regridded_woa.py @@ -38,7 +38,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, md5sum +from scripts_common import get_provenance_metadata def smooth2d(src): @@ -93,14 +93,8 @@ def main(input_path, variable_to_smooth, output_file): out_ds["time"] = out_ds.time.assign_attrs({"modulo": " "}) - this_file = os.path.normpath(__file__) - runcmd = f"python3 {os.path.basename(this_file)} --input_path={input_path} --var={variable_to_smooth} --output_file={output_file}" - out_ds = out_ds.assign_attrs( - { - "history": get_provenance_metadata(this_file, runcmd), - "input_files": [f"{f}(md5sum:{md5sum(f)})" for f in file_paths], - } - ) + # Obtain metadata + out_ds = out_ds.assign_attrs(get_provenance_metadata(file_paths)) # Save out_ds[variable_to_smooth].encoding.setdefault( diff --git a/wombat_ic_generation/SFe_climatology.py b/wombat_ic_generation/SFe_climatology.py index 5a8b6489..f632cacc 100644 --- a/wombat_ic_generation/SFe_climatology.py +++ b/wombat_ic_generation/SFe_climatology.py @@ -62,21 +62,11 @@ def main(): ) args = parser.parse_args() - input_filename = os.path.abspath(args.input_filename) - output_filename = os.path.abspath(args.output_filename) + input_filename = args.input_filename + output_filename = args.output_filename - this_file = os.path.normpath(__file__) - - # Add some info about how the file was generated - runcmd = ( - f"python3 {os.path.basename(this_file)} --input-filename={input_filename} " - f"--output-filename={output_filename}" - ) - - history_attrs = { - "history": get_provenance_metadata(this_file, runcmd), - "inputFile": f"{input_filename} (md5 hash: {md5sum(input_filename)})", - } + # Obtain metadata + history_attrs = get_provenance_metadata(input_files=[input_filename]) # Load the input data ds = xr.open_dataset(input_filename).compute() diff --git a/wombat_ic_generation/co2_iaf.py b/wombat_ic_generation/co2_iaf.py index 4e2f2fbf..76ab5255 100644 --- a/wombat_ic_generation/co2_iaf.py +++ b/wombat_ic_generation/co2_iaf.py @@ -38,7 +38,7 @@ path_root = Path(__file__).parents[1] sys.path.append(str(path_root)) -from scripts_common import get_provenance_metadata, md5sum +from scripts_common import get_provenance_metadata xr.set_options(keep_attrs=True) @@ -91,25 +91,9 @@ def main(): hgrid_filename = os.path.abspath(args.hgrid_filename) output_filename = os.path.abspath(args.output_filename) - this_file = os.path.normpath(__file__) - - # provenance metadata - runcmd = ( - f"python3 {os.path.basename(this_file)} --co2-cmip-filename={co2_cmip_filename} " - f"--co2-noaa-filename={co2_noaa_filename} --hgrid-filename={hgrid_filename} " - f"--output-filename={output_filename}" - ) - - file_hashes = [ - f"{co2_cmip_filename} (md5 hash: {md5sum(co2_cmip_filename)})", - f"{co2_noaa_filename} (md5 hash: {md5sum(co2_noaa_filename)})", - f"{hgrid_filename} (md5 hash: {md5sum(hgrid_filename)})", - ] - - global_attrs = { - "history": get_provenance_metadata(this_file, runcmd), - "inputFile": ", ".join(file_hashes), - } + # Obtain metadata + input_files = [co2_cmip_filename, co2_noaa_filename, hgrid_filename] + global_attrs = get_provenance_metadata(input_files) # Load the input data co2_cmip = xr.open_dataset(co2_cmip_filename, decode_cf=False).compute() diff --git a/wombat_ic_generation/generate_wombat_ic.py b/wombat_ic_generation/generate_wombat_ic.py index a156d7be..28ce8e12 100755 --- a/wombat_ic_generation/generate_wombat_ic.py +++ b/wombat_ic_generation/generate_wombat_ic.py @@ -75,7 +75,7 @@ def _open_woa_data(file, var, global_attrs, hcoord_int=None, vcoord_int=None): "WOA23 (https://www.ncei.noaa.gov/access/world-ocean-atlas-2023/)" ) - global_attrs["inputFile"].add(f"{file} (md5 hash: {md5sum(file)})") + global_attrs["inputFile"].add(f"{os.path.abspath(file)} (md5 hash: {md5sum(file)})") return _standardise_coords(da, hcoord_int, vcoord_int) @@ -96,7 +96,7 @@ def _open_glodap_data(file, var, global_attrs, hcoord_int=None, vcoord_int=None) "GLODAPv2 mapped data (https://glodap.info/index.php/mapped-data-product/)" ) - global_attrs["inputFile"].add(f"{file} (md5 hash: {md5sum(file)})") + global_attrs["inputFile"].add(f"{os.path.abspath(file)} (md5 hash: {md5sum(file)})") return _standardise_coords(da, hcoord_int, vcoord_int) @@ -113,7 +113,7 @@ def _open_huang_data(file, global_attrs, hcoord_int=None, vcoord_int=None): "Huang et al 2022 dFe data (https://zenodo.org/records/6994318)" ) - global_attrs["inputFile"].add(f"{file} (md5 hash: {md5sum(file)})") + global_attrs["inputFile"].add(f"{os.path.abspath(file)} (md5 hash: {md5sum(file)})") # Extend Latitudes from -90-90 new_lats = np.concatenate( @@ -457,18 +457,11 @@ def main(): wombat_version = args.wombat_version output_file = args.output_file - this_file = os.path.normpath(__file__) - - # Add some info about how the file was generated - runcmd = ( - f"python3 {this_file} --wombat-version={wombat_version} " - f"--output-file={os.path.abspath(output_file)}" - ) - global_attrs = { - "history": get_provenance_metadata(this_file, runcmd), "inputFile": set(), } + # Obtain metadata + global_attrs |= get_provenance_metadata() xr.set_options(keep_attrs=True) template = _open_woa_data( diff --git a/wombat_ic_generation/regrid_forcing.py b/wombat_ic_generation/regrid_forcing.py index 3ec0a5ef..efb4b72b 100644 --- a/wombat_ic_generation/regrid_forcing.py +++ b/wombat_ic_generation/regrid_forcing.py @@ -93,19 +93,8 @@ def main(): forcing_regrid["nx"].encoding |= {"dtype": "float32"} forcing_regrid["ny"].encoding |= {"dtype": "float32"} - # Add some info about how the file was generated - this_file = os.path.normpath(__file__) - - runcmd = ( - f"python3 {os.path.basename(this_file)} --forcing-filename={regrid.forcing_filename} " - f"{regrid.runcmd_args}" - ) - if regrid.args.homogenize: - runcmd += " --homogenize" - - global_attrs = { - "history": get_provenance_metadata(this_file, runcmd), - } + # Obtain metadata + global_attrs = get_provenance_metadata() forcing_regrid.attrs = forcing_regrid.attrs | global_attrs