diff --git a/examples/demo8.py b/examples/demo8.py index 83b6e0c4b5..6bb17d271f 100644 --- a/examples/demo8.py +++ b/examples/demo8.py @@ -35,10 +35,10 @@ New features introduced in this demo: -- galsim.config.Process(config, logger) -- galsim.config.ProcessInput(config, logger) -- galsim.config.BuildFile(config, file_num, logger) -- image = galsim.config.BuildImage(config, image_num, logger) +- galsim.config.Process(config) +- galsim.config.ProcessInput(config) +- galsim.config.BuildFile(config, file_num) +- image = galsim.config.BuildImage(config, image_num) - galsim.fits.read(file_name) """ @@ -153,19 +153,13 @@ def main(argv): # # image = galsim.config.BuildImage(config, image_num) # - # All of the above functions also have an optional kwarg, logger, which can take a - # logger object to output diagnostic information if desired. We'll use that option here - # to output the progress of the build as we go. Our logger is set with level=logging.INFO - # which means it will output a modest amount of text along the way. Using level=logging.DEBUG - # will output a lot of text, useful when diagnosing a mysterious crash. And using - # level=logging.WARNING or higher will be pretty silent unless there is a problem. t1 = time.time() # Build the image. # In this case, there is only a single image, so image_num=0. This is the default, so we # can actually omit this parameter for brevity. - image = galsim.config.BuildImage(config, logger=logger) + image = galsim.config.BuildImage(config) # At this point you could do something interesting with the image in memory. # After all, that was kind of the point of using BuildImage rather than the other higher @@ -202,7 +196,7 @@ def main(argv): 'file_name' : multi_file_name } # Again, we are just building one file, so use the default value of file_num=0. - galsim.config.BuildFile(config, logger=logger) + galsim.config.BuildFile(config) t3 = time.time() diff --git a/examples/des/blend.py b/examples/des/blend.py index c3fae1dcb0..b6d7800986 100644 --- a/examples/des/blend.py +++ b/examples/des/blend.py @@ -25,7 +25,7 @@ # Define the Blend stamp type # -def BuildBlendProfiles(self, config, base, psf, gsparams, logger): +def BuildBlendProfiles(self, config, base, psf, gsparams): """ Build a list of galaxy profiles, each convolved with the psf, to use for the blend image. @@ -38,7 +38,7 @@ def BuildBlendProfiles(self, config, base, psf, gsparams, logger): self.neighbor_gals = [] for i in range(n_neighbors): - gal = galsim.config.BuildGSObject(base, 'gal', gsparams=gsparams, logger=logger)[0] + gal = galsim.config.BuildGSObject(base, 'gal', gsparams=gsparams)[0] self.neighbor_gals.append(gal) # Remove the current stuff from base['gal'] so we don't get the same galaxy # each time. @@ -55,8 +55,7 @@ def BuildBlendProfiles(self, config, base, psf, gsparams, logger): pos = galsim.config.ParseValue(config_pos, 'pos', base, galsim.PositionD)[0] self.neighbor_pos.append(pos) - self.main_gal = galsim.config.BuildGSObject(base, 'gal', gsparams=gsparams, - logger=logger)[0] + self.main_gal = galsim.config.BuildGSObject(base, 'gal', gsparams=gsparams)[0] profiles = [ self.main_gal ] profiles += [ gal.shift(pos) for gal, pos in zip(self.neighbor_gals, self.neighbor_pos) ] @@ -68,7 +67,7 @@ def BuildBlendProfiles(self, config, base, psf, gsparams, logger): class BlendBuilder(galsim.config.StampBuilder): - def setup(self, config, base, xsize, ysize, ignore, logger): + def setup(self, config, base, xsize, ysize, ignore): """Do the appropriate setup for a Blend stamp. """ self.first = None # Mark that we don't have anything stored yet. @@ -80,19 +79,19 @@ def setup(self, config, base, xsize, ysize, ignore, logger): # Now farm off to the regular stamp setup function the rest of the work of parsing # the size and position of the stamp. ignore = ignore + ['n_neighbors', 'min_sep', 'max_sep'] - return super(BlendBuilder,self).setup(config, base, xsize, ysize, ignore, logger) + return super(BlendBuilder,self).setup(config, base, xsize, ysize, ignore) - def buildProfile(self, config, base, psf, gsparams, logger): - return BuildBlendProfiles(self, config, base, psf, gsparams, logger) + def buildProfile(self, config, base, psf, gsparams): + return BuildBlendProfiles(self, config, base, psf, gsparams) - def draw(self, profiles, image, method, offset, config, base, logger): + def draw(self, profiles, image, method, offset, config, base): """ Draw the profiles onto the stamp. """ n_neighbors = len(profiles)-1 # Draw the central galaxy using the basic draw function. - image = galsim.config.DrawBasic(profiles[0], image, method, offset, config, base, logger) + image = galsim.config.DrawBasic(profiles[0], image, method, offset, config, base) # We'll want a copy of just the neighbors for the deblend image. # Otherwise we could have just drawn these on the main image with add_to_image = True @@ -101,7 +100,7 @@ def draw(self, profiles, image, method, offset, config, base, logger): # Draw all the neighbor stamps for p in profiles[1:]: - galsim.config.DrawBasic(p, self.neighbor_image, method, offset, config, base, logger, + galsim.config.DrawBasic(p, self.neighbor_image, method, offset, config, base, add_to_image=True) # Save this in base for the deblend output @@ -111,12 +110,12 @@ def draw(self, profiles, image, method, offset, config, base, logger): return image - def whiten(self, profiles, image, config, base, logger): + def whiten(self, profiles, image, config, base): """ Whiten the noise on the stamp according to the existing noise in all the profiles. """ total = galsim.Add(profiles) - return super(BlendBuilder,self).whiten(total, image, config, base, logger) + return super(BlendBuilder,self).whiten(total, image, config, base) galsim.config.RegisterStampType('Blend', BlendBuilder()) @@ -132,7 +131,7 @@ class BlendSetBuilder(galsim.config.StampBuilder): # This is the same as the setup function for Blend, so there is a bit of duplicated code # here, but it was simpler to have BlendSetBuilder derive directly from StampBuilder # so the super calls go directly to that. - def setup(self, config, base, xsize, ysize, ignore, logger): + def setup(self, config, base, xsize, ysize, ignore): """Do the appropriate setup for a Blend stamp. """ # Make sure that we start over on the start of a new file. @@ -146,9 +145,9 @@ def setup(self, config, base, xsize, ysize, ignore, logger): # Now farm off to the regular stamp setup function the rest of the work of parsing # the size and position of the stamp. ignore = ignore + ['n_neighbors', 'min_sep', 'max_sep'] - return super(BlendSetBuilder, self).setup(config, base, xsize, ysize, ignore, logger) + return super(BlendSetBuilder, self).setup(config, base, xsize, ysize, ignore) - def buildProfile(self, config, base, psf, gsparams, logger): + def buildProfile(self, config, base, psf, gsparams): """ Build a list of galaxy profiles, each convolved with the psf, to use for the blend image. """ @@ -158,12 +157,12 @@ def buildProfile(self, config, base, psf, gsparams, logger): return None else: # Run the above BuildBlendProfiles function to build the profiles. - self.profiles = BuildBlendProfiles(self, config, base, psf, gsparams, logger) + self.profiles = BuildBlendProfiles(self, config, base, psf, gsparams) # And mark this as the first object in the set self.first = base['obj_num'] return self.profiles - def draw(self, profiles, image, method, offset, config, base, logger): + def draw(self, profiles, image, method, offset, config, base): """ Draw the profiles onto the stamp. """ @@ -188,8 +187,7 @@ def draw(self, profiles, image, method, offset, config, base, logger): self.full_images = [] for prof in profiles: im = galsim.ImageF(bounds=bounds, wcs=wcs) - galsim.config.DrawBasic(prof, im, method, offset-im.true_center, config, base, - logger) + galsim.config.DrawBasic(prof, im, method, offset-im.true_center, config, base) self.full_images.append(im) # Figure out what bounds to use for the cutouts. @@ -219,7 +217,7 @@ def draw(self, profiles, image, method, offset, config, base, logger): return image - def whiten(self, profiles, image, config, base, logger): + def whiten(self, profiles, image, config, base): """ Whiten the noise on the stamp according to the existing noise in all the profiles. """ @@ -229,7 +227,7 @@ def whiten(self, profiles, image, config, base, logger): self.current_var = 0 for prof, full_im in zip(self.profiles, self.full_images): self.current_var += super(BlendSetBuilder,self).whiten( - prof, full_im, config, base, logger) + prof, full_im, config, base) if self.current_var != 0: # Then we whitened the noise somewhere. Rebuild the stamp image.setZero() @@ -239,7 +237,7 @@ def whiten(self, profiles, image, config, base, logger): return self.current_var - def addNoise(self, config, base, image, current_var, logger): + def addNoise(self, config, base, image, current_var): """Add the sky and noise""" # We want the noise realization to be the same for all galaxies in the set, # so we only generate the noise the first time and save it, pulling out the right cutout @@ -251,7 +249,7 @@ def addNoise(self, config, base, image, current_var, logger): self.full_noise_image = self.full_images[0].copy() self.full_noise_image.setZero() self.full_noise_image, self.current_var = super(BlendSetBuilder,self).addNoise( - config, base, self.full_noise_image, current_var, logger) + config, base, self.full_noise_image, current_var) image += self.full_noise_image[self.bounds] return image, self.current_var @@ -273,7 +271,7 @@ def addNoise(self, config, base, image, current_var, logger): class DeblendBuilder(galsim.config.ExtraOutputBuilder): - def processStamp(self, obj_num, config, base, logger): + def processStamp(self, obj_num, config, base): """Save the stamps of just the neighbor fluxes. We'll subtract them from the full image at the end. """ @@ -289,7 +287,7 @@ def processStamp(self, obj_num, config, base, logger): # Save it in the scratch dict using this obj_num as the key. self.scratch[obj_num] = im - def processImage(self, index, obj_nums, config, base, logger): + def processImage(self, index, obj_nums, config, base): """Copy the full final image over and then subtract off the neighbor-only fluxes. """ # Start with a copy of the regular final image. @@ -313,7 +311,7 @@ def processImage(self, index, obj_nums, config, base, logger): class DeblendMedsBuilder(DeblendBuilder): - def finalize(self, config, base, main_data, logger): + def finalize(self, config, base, main_data): """Convert from the list of images we've been making into a list of MultiExposureObjects we can use to write the MEDS file. """ @@ -331,7 +329,7 @@ def finalize(self, config, base, main_data, logger): k1 = k2 return obj_list - def writeFile(self, file_name, config, base, logger): + def writeFile(self, file_name, config, base): galsim.des.WriteMEDS(self.final_data, file_name) diff --git a/examples/des/des_wcs.py b/examples/des/des_wcs.py index 0db4d63c3c..9f45c42b51 100644 --- a/examples/des/des_wcs.py +++ b/examples/des/des_wcs.py @@ -32,7 +32,7 @@ def get_random_chipnum(ud, bad_ccds): class DES_SlowLocalWCSBuilder(galsim.config.WCSBuilder): - def buildWCS(self, config, base, logger): + def buildWCS(self, config, base): """Build a local WCS from the given location in a DES focal plane, given a directory with the image files. By default, it will pick a random chipnum and image position, but these can be optionally specified. @@ -144,7 +144,7 @@ def get_chip_wcs(self, chipnum): class DES_LocalWCSBuilder(galsim.config.WCSBuilder): - def buildWCS(self, config, base, logger): + def buildWCS(self, config, base): """Build a local WCS from the given location in a DES focal plane. This function is used in conjunction with the des_wcs input field, which loads all the diff --git a/examples/great3/noise_free.py b/examples/great3/noise_free.py index 9f7558d202..c70c1a94ab 100644 --- a/examples/great3/noise_free.py +++ b/examples/great3/noise_free.py @@ -27,13 +27,13 @@ class NoiseFreeBuilder(galsim.config.ExtraOutputBuilder): """ # The function to call at the end of building each stamp - def processStamp(self, obj_num, config, base, logger): + def processStamp(self, obj_num, config, base): if base['do_noise_in_stamps']: noise_free_im = base['current_stamp'].copy() self.scratch[obj_num] = noise_free_im # The function to call at the end of building each image - def processImage(self, index, obj_nums, config, base, logger): + def processImage(self, index, obj_nums, config, base): if len(self.scratch) > 0.: # If we have been accumulating the stamp images, build the total from them. image = galsim.ImageF(base['image_bounds'], wcs=base['wcs'], init_value=0.) diff --git a/examples/mixed_scene.py b/examples/mixed_scene.py index babe0ecd05..30080ffc8d 100644 --- a/examples/mixed_scene.py +++ b/examples/mixed_scene.py @@ -16,11 +16,14 @@ # and/or other materials provided with the distribution. # +import logging import galsim +logger = logging.getLogger(__name__) + class MixedSceneBuilder(galsim.config.StampBuilder): - def setup(self, config, base, xsize, ysize, ignore, logger): + def setup(self, config, base, xsize, ysize, ignore): if 'objects' not in config: raise AttributeError('objets field is required for MixedScene stamp type') objects = config['objects'] @@ -76,20 +79,20 @@ def setup(self, config, base, xsize, ysize, ignore, logger): ignore = ignore + ['objects', 'magnify', 'shear', 'obj_type'] # Now go on and do the rest of the normal setup. - return super(MixedSceneBuilder, self).setup(config,base,xsize,ysize,ignore,logger) + return super(MixedSceneBuilder, self).setup(config,base,xsize,ysize,ignore) - def buildProfile(self, config, base, psf, gsparams, logger): + def buildProfile(self, config, base, psf, gsparams): obj_type = base['current_obj_type'] logger.info('obj %d: Drawing %s', base['obj_num'], obj_type) # Make the appropriate object using the obj_type field - obj = galsim.config.BuildGSObject(base, obj_type, gsparams=gsparams, logger=logger)[0] + obj = galsim.config.BuildGSObject(base, obj_type, gsparams=gsparams)[0] # Also save this in case useful for some calculation. base['current_obj'] = obj # Only shear and magnify are allowed, but this general TransformObject function will # work to implement those. - obj, safe = galsim.config.TransformObject(obj, config, base, logger) + obj, safe = galsim.config.TransformObject(obj, config, base) if psf: if obj: diff --git a/galsim/config/bandpass.py b/galsim/config/bandpass.py index fd768b4168..6a03c8a1f8 100644 --- a/galsim/config/bandpass.py +++ b/galsim/config/bandpass.py @@ -19,33 +19,32 @@ import logging from astropy.units import Quantity, Unit -from .util import LoggerWrapper from .value import ParseValue, GetAllParams, GetIndex from .input import RegisterInputConnectedType from ..errors import GalSimConfigError, GalSimConfigValueError from ..bandpass import Bandpass from ..utilities import basestring +logger = logging.getLogger(__name__) + # This module-level dict will store all the registered Bandpass types. # See the RegisterBandpassType function at the end of this file. # The keys are the (string) names of the Bandpass types, and the values will be builders that know # how to build the Bandpass object. valid_bandpass_types = {} -def BuildBandpass(config, key, base, logger=None): +def BuildBandpass(config, key, base): """Read the Bandpass parameters from config[key] and return a constructed Bandpass object. Parameters: config: A dict with the configuration information. key: The key name in config indicating which object to build. base: The base dict of the configuration. - logger: Optionally, provide a logger for logging debug statements. [default: None] Returns: (bandpass, safe) where bandpass is a Bandpass instance, and safe is whether it is safe to reuse. """ - logger = LoggerWrapper(logger) logger.debug('obj %d: Start BuildBandpass key = %s',base.get('obj_num',0),key) param = config[key] @@ -79,7 +78,7 @@ def BuildBandpass(config, key, base, logger=None): list(valid_bandpass_types.keys())) logger.debug('obj %d: Building bandpass type %s', base.get('obj_num',0), bandpass_type) builder = valid_bandpass_types[bandpass_type] - bandpass, safe = builder.buildBandpass(param, base, logger) + bandpass, safe = builder.buildBandpass(param, base) logger.debug('obj %d: bandpass = %s', base.get('obj_num',0), bandpass) param['current'] = bandpass, safe, Bandpass, index, index_key @@ -92,7 +91,7 @@ class BandpassBuilder: The base class defines the call signatures of the methods that any derived class should follow. """ - def buildBandpass(self, config, base, logger): + def buildBandpass(self, config, base): """Build the Bandpass based on the specifications in the config dict. Note: Sub-classes must override this function with a real implementation. @@ -100,7 +99,6 @@ def buildBandpass(self, config, base, logger): Parameters: config: The configuration dict for the bandpass type. base: The base configuration dict. - logger: If provided, a logger for logging debug statements. Returns: the constructed Bandpass object. @@ -120,18 +118,16 @@ class FileBandpassBuilder(BandpassBuilder): red_limit (float or Quantity) A cutoff wavelength on the red side (default: None) zeropoint (float or str) A zeropoint to use (default: None) """ - def buildBandpass(self, config, base, logger): + def buildBandpass(self, config, base): """Build the Bandpass based on the specifications in the config dict. Parameters: config: The configuration dict for the bandpass type. base: The base configuration dict. - logger: If provided, a logger for logging debug statements. Returns: the constructed Bandpass object. """ - logger = LoggerWrapper(logger) req = { 'file_name': str, diff --git a/galsim/config/extra.py b/galsim/config/extra.py index 634dfa6751..a275815a8a 100644 --- a/galsim/config/extra.py +++ b/galsim/config/extra.py @@ -16,16 +16,19 @@ # and/or other materials provided with the distribution. # +import logging import os from multiprocessing.managers import ListProxy, DictProxy -from .util import LoggerWrapper, SetDefaultExt, RetryIO, SafeManager, single_threaded +from .util import SetDefaultExt, RetryIO, SafeManager, single_threaded from .value import ParseValue from .image import GetNObjForImage from ..utilities import ensure_dir from ..errors import GalSimConfigValueError, GalSimConfigError from ..fits import writeMulti +logger = logging.getLogger(__name__) + # This file handles the processing of extra output items in addition to the primary output file # in config['output']. The ones that are defined natively in GalSim are psf, weight, badpix, # and truth. See extra_*.py for the specific functions for each of these. @@ -36,7 +39,7 @@ # builder classes that will perform the different processing functions. valid_extra_outputs = {} -def SetupExtraOutput(config, logger=None): +def SetupExtraOutput(config): """ Set up the extra output items as necessary, including building Managers for the work space so they can work safely in multi-processing mode. Each builder will be placed in @@ -44,9 +47,7 @@ def SetupExtraOutput(config, logger=None): Parameters: config: The configuration dict. - logger: If given, a logger object to log progress. [default: None] """ - logger = LoggerWrapper(logger) output = config['output'] file_num = config.get('file_num',0) @@ -105,28 +106,27 @@ class OutputManager(SafeManager): pass # Create the builder, giving it the data and scratch objects as work space. field = config['output'][key] builder = valid_extra_outputs[key] - builder.initialize(data, scratch, field, config, logger) + builder.initialize(data, scratch, field, config) # And store it in the config dict config['extra_builder'][key] = builder logger.debug('file %d: Setup output %s object',file_num,key) -def SetupExtraOutputsForImage(config, logger=None): +def SetupExtraOutputsForImage(config): """Perform any necessary setup for the extra output items at the start of a new image. Parameters: config: The configuration dict. - logger: If given, a logger object to log progress. [default: None] """ if 'output' in config: if 'extra_builder' not in config: - SetupExtraOutput(config, logger) + SetupExtraOutput(config) for key, builder in config['extra_builder'].items(): field = config['output'][key] - builder.setupImage(field, config, logger) + builder.setupImage(field, config) -def ProcessExtraOutputsForStamp(config, skip, logger=None): +def ProcessExtraOutputsForStamp(config, skip): """Run the appropriate processing code for any extra output items that need to do something at the end of building each object. @@ -136,7 +136,6 @@ def ProcessExtraOutputsForStamp(config, skip, logger=None): Parameters: config: The configuration dict. skip: Was the drawing of this object skipped? - logger: If given, a logger object to log progress. [default: None] """ if 'output' in config: obj_num = config['obj_num'] @@ -144,18 +143,17 @@ def ProcessExtraOutputsForStamp(config, skip, logger=None): field = config['output'][key] if skip: config['_skipped_obj_nums'][obj_num] = None - builder.processSkippedStamp(obj_num, field, config, logger) + builder.processSkippedStamp(obj_num, field, config) else: - builder.processStamp(obj_num, field, config, logger) + builder.processStamp(obj_num, field, config) -def ProcessExtraOutputsForImage(config, logger=None): +def ProcessExtraOutputsForImage(config): """Run the appropriate processing code for any extra output items that need to do something at the end of building each image Parameters: config: The configuration dict. - logger: If given, a logger object to log progress. [default: None] """ if 'output' in config: obj_nums = None @@ -176,10 +174,10 @@ def ProcessExtraOutputsForImage(config, logger=None): obj_nums = [ n for n in obj_nums if n not in skipped ] field = config['output'][key] index = image_num - start_image_num - builder.processImage(index, obj_nums, field, config, logger) + builder.processImage(index, obj_nums, field, config) -def WriteExtraOutputs(config, main_data, logger=None): +def WriteExtraOutputs(config, main_data): """Write the extra output objects to files. This gets run at the end of the functions for building the regular output files. @@ -187,9 +185,7 @@ def WriteExtraOutputs(config, main_data, logger=None): Parameters: config: The configuration dict. main_data: The main file data in case it is needed. - logger: If given, a logger object to log progress. [default: None] """ - logger = LoggerWrapper(logger) output = config['output'] if 'retry_io' in output: ntries = ParseValue(config['output'],'retry_io',config,int)[0] @@ -242,17 +238,17 @@ def WriteExtraOutputs(config, main_data, logger=None): continue # Do any final processing that needs to happen. - builder.ensureFinalized(field, config, main_data, logger) + builder.ensureFinalized(field, config, main_data) # Call the write function, possibly multiple times to account for IO failures. write_func = builder.writeFile - args = (file_name,field,config,logger) - RetryIO(write_func, args, ntries, file_name, logger) + args = (file_name,field,config) + RetryIO(write_func, args, ntries, file_name) config['extra_last_file'][key] = file_name logger.debug('file %d: Wrote %s to %r',config['file_num'],key,file_name) -def AddExtraOutputHDUs(config, main_data, logger=None): +def AddExtraOutputHDUs(config, main_data): """Write the extra output objects to either HDUS or images as appropriate and add them to the existing data. @@ -265,7 +261,6 @@ def AddExtraOutputHDUs(config, main_data, logger=None): config: The configuration dict. main_data: The main file data as a list of images. Usually just [image] where image is the primary image to be written to the output file. - logger: If given, a logger object to log progress. [default: None] Returns: data with additional hdus added @@ -283,10 +278,10 @@ def AddExtraOutputHDUs(config, main_data, logger=None): raise GalSimConfigValueError("hdu is invalid or a duplicate.",hdu) # Do any final processing that needs to happen. - builder.ensureFinalized(field, config, main_data, logger) + builder.ensureFinalized(field, config, main_data) # Build the HDU for this output object. - hdus[hdu] = builder.writeHdu(field,config,logger) + hdus[hdu] = builder.writeHdu(field,config) first = len(main_data) for h in range(first,len(hdus)+first): @@ -296,7 +291,7 @@ def AddExtraOutputHDUs(config, main_data, logger=None): hdulist = [ hdus[k] for k in range(first,len(hdus)+first) ] return main_data + hdulist -def CheckNoExtraOutputHDUs(config, output_type, logger=None): +def CheckNoExtraOutputHDUs(config, output_type): """Check that none of the extra output objects want to add to the HDU list. Raises an exception if one of them has an hdu field. @@ -305,9 +300,7 @@ def CheckNoExtraOutputHDUs(config, output_type, logger=None): config: The configuration dict. output_type: A string to use in the error message to indicate which output type had a problem. - logger: If given, a logger object to log progress. [default: None] """ - logger = LoggerWrapper(logger) output = config['output'] for key in config['extra_builder'].keys(): field = output[key] @@ -318,20 +311,19 @@ def CheckNoExtraOutputHDUs(config, output_type, logger=None): "Output type %s cannot add extra images as HDUs"%output_type) -def GetFinalExtraOutput(key, config, main_data=[], logger=None): +def GetFinalExtraOutput(key, config, main_data=[]): """Get the finalized output object for the given extra output key Parameters: key: The name of the output field in config['output'] config: The configuration dict. main_data: The main file data in case it is needed. [default: []] - logger: If given, a logger object to log progress. [default: None] Returns: the final data to be output. """ field = config['output'][key] - return config['extra_builder'][key].ensureFinalized(field, config, main_data, logger) + return config['extra_builder'][key].ensureFinalized(field, config, main_data) class ExtraOutputBuilder: """A base class for building some kind of extra output object along with the main output. @@ -354,7 +346,7 @@ class can override to perform specific processing at any of several steps in the any information you want to persist into the scratch or data objects, which are set up to handle the multiprocessing communication properly. """ - def initialize(self, data, scratch, config, base, logger): + def initialize(self, data, scratch, config, base): """Do any initial setup for this builder at the start of a new output file. The base class implementation saves two work space items into self.data and self.scratch @@ -365,13 +357,12 @@ def initialize(self, data, scratch, config, base, logger): scratch: An empty dict that can be used as work space. config: The configuration field for this output object. base: The base configuration dict. - logger: If given, a logger object to log progress. [default: None] """ self.data = data self.scratch = scratch self.final_data = None - def setupImage(self, config, base, logger): + def setupImage(self, config, base): """Perform any necessary setup at the start of an image. This function will be called at the start of each image to allow for any setup that @@ -380,11 +371,10 @@ def setupImage(self, config, base, logger): Parameters: config: The configuration field for this output object. base: The base configuration dict. - logger: If given, a logger object to log progress. [default: None] """ pass - def processStamp(self, obj_num, config, base, logger): + def processStamp(self, obj_num, config, base): """Perform any necessary processing at the end of each stamp construction. This function will be called after each stamp is built, but before the noise is added, @@ -398,11 +388,10 @@ def processStamp(self, obj_num, config, base, logger): obj_num: The object number config: The configuration field for this output object. base: The base configuration dict. - logger: If given, a logger object to log progress. [default: None] """ pass # pragma: no cover (all our ExtraBuilders override this function.) - def processSkippedStamp(self, obj_num, config, base, logger): + def processSkippedStamp(self, obj_num, config, base): """Perform any necessary processing for stamps that were skipped in the normal processing. This function will be called for stamps that are not built because they were skipped @@ -414,11 +403,10 @@ def processSkippedStamp(self, obj_num, config, base, logger): obj_num: The object number config: The configuration field for this output object. base: The base configuration dict. - logger: If given, a logger object to log progress. [default: None] """ pass - def processImage(self, index, obj_nums, config, base, logger): + def processImage(self, index, obj_nums, config, base): """Perform any necessary processing at the end of each image construction. This function will be called after each full image is built. @@ -436,11 +424,10 @@ def processImage(self, index, obj_nums, config, base, logger): obj_nums: The object numbers that were used for this image. config: The configuration field for this output object. base: The base configuration dict. - logger: If given, a logger object to log progress. [default: None] """ pass - def ensureFinalized(self, config, base, main_data, logger): + def ensureFinalized(self, config, base, main_data): """A helper function in the base class to make sure finalize only gets called once by the different possible locations that might need it to have been called. @@ -448,16 +435,15 @@ def ensureFinalized(self, config, base, main_data, logger): config: The configuration field for this output object. base: The base configuration dict. main_data: The main file data in case it is needed. - logger: If given, a logger object to log progress. [default: None] Returns: the final version of the object. """ if self.final_data is None: - self.final_data = self.finalize(config, base, main_data, logger) + self.final_data = self.finalize(config, base, main_data) return self.final_data - def finalize(self, config, base, main_data, logger): + def finalize(self, config, base, main_data): """Perform any final processing at the end of all the image processing. This function will be called after all images have been built. @@ -470,14 +456,13 @@ def finalize(self, config, base, main_data, logger): config: The configuration field for this output object. base: The base configuration dict. main_data: The main file data in case it is needed. - logger: If given, a logger object to log progress. [default: None] Returns: The final version of the object. """ return self.data - def writeFile(self, file_name, config, base, logger): + def writeFile(self, file_name, config, base): """Write this output object to a file. The base class implementation is appropriate for the cas that the result of finalize @@ -487,11 +472,10 @@ def writeFile(self, file_name, config, base, logger): file_name: The file to write to. config: The configuration field for this output object. base: The base configuration dict. - logger: If given, a logger object to log progress. [default: None] """ writeMulti(self.final_data, file_name) - def writeHdu(self, config, base, logger): + def writeHdu(self, config, base): """Write the data to a FITS HDU with the data for this output object. The base class implementation is appropriate for the cas that the result of finalize @@ -500,7 +484,6 @@ def writeHdu(self, config, base, logger): Parameters: config: The configuration field for this output object. base: The base configuration dict. - logger: If given, a logger object to log progress. [default: None] Returns: an HDU with the output data. diff --git a/galsim/config/extra_badpix.py b/galsim/config/extra_badpix.py index 2372f3a529..178b65ddc0 100644 --- a/galsim/config/extra_badpix.py +++ b/galsim/config/extra_badpix.py @@ -31,7 +31,7 @@ class BadPixBuilder(ExtraOutputBuilder): """ # The function to call at the end of building each stamp - def processStamp(self, obj_num, config, base, logger): + def processStamp(self, obj_num, config, base): # Note: This is just a placeholder for now. Once we implement defects, saturation, etc., # these features should be marked in the badpix mask. For now though, all pixels = 0. if base['do_noise_in_stamps']: @@ -39,7 +39,7 @@ def processStamp(self, obj_num, config, base, logger): self.scratch[obj_num] = badpix_im # The function to call at the end of building each image - def processImage(self, index, obj_nums, config, base, logger): + def processImage(self, index, obj_nums, config, base): image = ImageS(base['image_bounds'], wcs=base['wcs'], init_value=0) if len(self.scratch) > 0.: # If we have been accumulating the variance on the stamps, build the total from them. diff --git a/galsim/config/extra_psf.py b/galsim/config/extra_psf.py index 1c1122ac86..36e55e8966 100644 --- a/galsim/config/extra_psf.py +++ b/galsim/config/extra_psf.py @@ -29,10 +29,12 @@ from ..position import PositionD from ..errors import GalSimConfigValueError, GalSimConfigError +logger = logging.getLogger(__name__) + # The psf extra output type builds an Image of the PSF at the same locations as the galaxies. # The code the actually draws the PSF on a postage stamp. -def DrawPSFStamp(psf, config, base, bounds, offset, method, logger): +def DrawPSFStamp(psf, config, base, bounds, offset, method): """ Draw an image using the given psf profile. @@ -92,7 +94,7 @@ class ExtraPSFBuilder(ExtraOutputBuilder): a TiledImage, since you wouldn't typically want the PSF images to overlap. But it just follows whatever pattern of stamp locations the main image has. """ - def processStamp(self, obj_num, config, base, logger): + def processStamp(self, obj_num, config, base): # If this doesn't exist, an appropriate exception will be raised. psf = base['psf']['current'][0] draw_method = GetCurrentValue('draw_method', base['stamp'], str, base) @@ -123,14 +125,14 @@ def processStamp(self, obj_num, config, base, logger): offset += ParseValue(config, 'offset', base, PositionD)[0] logger.debug('obj %d: psf offset: %s',base.get('obj_num',0),str(offset)) - psf_im = DrawPSFStamp(psf,config,base,bounds,offset,draw_method,logger) + psf_im = DrawPSFStamp(psf,config,base,bounds,offset,draw_method) if 'signal_to_noise' in config: base['current_noise_image'] = base['current_stamp'] - AddNoise(base,psf_im,current_var=0,logger=logger) + AddNoise(base,psf_im,current_var=0) self.scratch[obj_num] = psf_im # The function to call at the end of building each image - def processImage(self, index, obj_nums, config, base, logger): + def processImage(self, index, obj_nums, config, base): image = Image(base['image_bounds'], wcs=base['wcs'], init_value=0., dtype=base['current_image'].dtype) # Make sure to only use the stamps for objects in this image. diff --git a/galsim/config/extra_truth.py b/galsim/config/extra_truth.py index f400037b19..57c0efcd63 100644 --- a/galsim/config/extra_truth.py +++ b/galsim/config/extra_truth.py @@ -16,6 +16,7 @@ # and/or other materials provided with the distribution. # +import logging import sys import numpy as np from .extra import ExtraOutputBuilder, RegisterExtraOutput @@ -24,6 +25,8 @@ from ..catalog import OutputCatalog from ..utilities import basestring +logger = logging.getLogger(__name__) + # The truth extra output type builds an OutputCatalog with truth information about each of the # objects being built by the configuration processing. It stores the appropriate row information # in scratch space for each stamp and then adds them in order at the end of the file processing. @@ -41,7 +44,7 @@ class TruthBuilder(ExtraOutputBuilder): current values of various quantities for each constructed object. """ # The function to call at the end of building each stamp - def processStamp(self, obj_num, config, base, logger): + def processStamp(self, obj_num, config, base): cols = config['columns'] row = [] types = [] @@ -89,7 +92,7 @@ def _type(self, v): return type(v) # The function to call at the end of building each file to finalize the truth catalog - def finalize(self, config, base, main_data, logger): + def finalize(self, config, base, main_data): # Make the OutputCatalog cols = config['columns'] # Note: Provide a default here, because if all items were skipped it would otherwise @@ -106,11 +109,11 @@ def finalize(self, config, base, main_data, logger): return cat # This becomes self.final_data # Write the catalog to a file - def writeFile(self, file_name, config, base, logger): + def writeFile(self, file_name, config, base): self.final_data.write(file_name) # Create an HDU of the FITS binary table. - def writeHdu(self, config, base, logger): + def writeHdu(self, config, base): return self.final_data.writeFitsHdu() # Register this as a valid extra output diff --git a/galsim/config/extra_weight.py b/galsim/config/extra_weight.py index 6df834c187..0b60147407 100644 --- a/galsim/config/extra_weight.py +++ b/galsim/config/extra_weight.py @@ -38,7 +38,7 @@ class WeightBuilder(ExtraOutputBuilder): """ # The function to call at the end of building each stamp - def processStamp(self, obj_num, config, base, logger): + def processStamp(self, obj_num, config, base): if base['do_noise_in_stamps']: weight_im = ImageF(base['current_stamp'].bounds, wcs=base['wcs'], init_value=0.) if 'include_obj_var' in config: @@ -46,11 +46,11 @@ def processStamp(self, obj_num, config, base, logger): else: include_obj_var = False base['current_noise_image'] = base['current_stamp'] - AddNoiseVariance(base,weight_im,include_obj_var,logger) + AddNoiseVariance(base,weight_im,include_obj_var) self.scratch[obj_num] = weight_im # The function to call at the end of building each image - def processImage(self, index, obj_nums, config, base, logger): + def processImage(self, index, obj_nums, config, base): image = ImageF(base['image_bounds'], wcs=base['wcs'], init_value=0.) if len(self.scratch) > 0.: # If we have been accumulating the variance on the stamps, build the total from them. @@ -70,7 +70,7 @@ def processImage(self, index, obj_nums, config, base, logger): else: include_obj_var = False base['current_noise_image'] = base['current_image'] - AddNoiseVariance(base,image,include_obj_var,logger) + AddNoiseVariance(base,image,include_obj_var) # Now invert the variance image to get weight map. # Note that any zeros present in the image are maintained as zeros after inversion. diff --git a/galsim/config/gsobject.py b/galsim/config/gsobject.py index 5754330c6b..84505afb60 100644 --- a/galsim/config/gsobject.py +++ b/galsim/config/gsobject.py @@ -16,8 +16,9 @@ # and/or other materials provided with the distribution. # import inspect +import logging -from .util import LoggerWrapper, GetIndex, GetRNG, get_cls_params, CleanConfig +from .util import GetIndex, GetRNG, get_cls_params, CleanConfig from .value import ParseValue, GetCurrentValue, GetAllParams, CheckAllParams, SetDefaultIndex from .input import RegisterInputConnectedType from .sed import BuildSED @@ -35,6 +36,8 @@ from ..chromatic import ChromaticAtmosphere from ..celestial import CelestialCoord +logger = logging.getLogger(__name__) + # This file handles the building of GSObjects in the config['psf'] and config['gal'] fields. # This file includes many of the simple object types. Additional types are defined in # gsobject_ring.py, input_real.py, and input_cosmos.py. @@ -57,7 +60,7 @@ def __init__(self, message=None): self.msg = message -def BuildGSObject(config, key, base=None, gsparams={}, logger=None): +def BuildGSObject(config, key, base=None, gsparams={}): """Build a GSObject from the parameters in config[key]. Parameters: @@ -68,8 +71,6 @@ def BuildGSObject(config, key, base=None, gsparams={}, logger=None): at this level will be added to the list. This should be a dict with whatever kwargs should be used in constructing the GSParams object. [default: {}] - logger: Optionally, provide a logger for logging debug statements. - [default: None] Returns: the tuple (gsobject, safe), where ``gsobject`` is the built object, and ``safe`` is @@ -77,7 +78,6 @@ def BuildGSObject(config, key, base=None, gsparams={}, logger=None): """ from .. import __dict__ as galsim_dict - logger = LoggerWrapper(logger) if base is None: base = config @@ -117,14 +117,13 @@ def BuildGSObject(config, key, base=None, gsparams={}, logger=None): cobj, csafe, cvalue_type, cindex, cindex_type = param['current'] if csafe or cindex//repeat == index//repeat: # If logging, explain why we are using the current object. - if logger: - if csafe: - logger.debug('obj %d: current is safe',base.get('obj_num',0)) - elif repeat > 1: - logger.debug('obj %d: repeat = %d, index = %d, use current object', - base.get('obj_num',0),repeat,index) - else: - logger.debug('obj %d: This object is already current', base.get('obj_num',0)) + if csafe: + logger.debug('obj %d: current is safe',base.get('obj_num',0)) + elif repeat > 1: + logger.debug('obj %d: repeat = %d, index = %d, use current object', + base.get('obj_num',0),repeat,index) + else: + logger.debug('obj %d: This object is already current', base.get('obj_num',0)) return cobj, csafe @@ -188,9 +187,9 @@ def BuildGSObject(config, key, base=None, gsparams={}, logger=None): raise GalSimConfigValueError("Unrecognised gsobject type", type_name) if inspect.isclass(build_func) and issubclass(build_func, (GSObject, ChromaticObject)): - gsobject, safe = _BuildSimple(build_func, param, base, ignore, gsparams, logger) + gsobject, safe = _BuildSimple(build_func, param, base, ignore, gsparams) else: - gsobject, safe = build_func(param, base, ignore, gsparams, logger) + gsobject, safe = build_func(param, base, ignore, gsparams) # Apply any SED and redshift that might be present. if 'redshift' in param: @@ -201,7 +200,7 @@ def BuildGSObject(config, key, base=None, gsparams={}, logger=None): 'sed field.') param['sed']['redshift'] = param.pop('redshift') - gsobject, safe1 = ApplySED(gsobject, param, base, logger) + gsobject, safe1 = ApplySED(gsobject, param, base) safe = safe and safe1 if 'redshift' in param: @@ -234,7 +233,7 @@ def BuildGSObject(config, key, base=None, gsparams={}, logger=None): pass # Apply any dilation, ellip, shear, etc. modifications. - gsobject, safe1 = TransformObject(gsobject, param, base, logger) + gsobject, safe1 = TransformObject(gsobject, param, base) safe = safe and safe1 # Re-get index and index_key in case something changed when building the object. @@ -265,17 +264,16 @@ def UpdateGSParams(gsparams, config, base): ret.update(kwargs) return ret -def ApplySED(gsobject, config, base, logger): +def ApplySED(gsobject, config, base): """Read and apply an SED to the base gsobject Parameters: gsobject: The base GSObject config: A dict with the configuration information. base: The base dict of the configuration. - logger: A logger for logging debug statements. """ if 'sed' in config: - sed, safe = BuildSED(config, 'sed', base, logger) + sed, safe = BuildSED(config, 'sed', base) return gsobject * sed, safe else: return gsobject, True @@ -285,7 +283,7 @@ def ApplySED(gsobject, config, base, logger): # These are not imported into galsim.config namespace. # -def _BuildSimple(build_func, config, base, ignore, gsparams, logger): +def _BuildSimple(build_func, config, base, ignore, gsparams): """Build a simple GSObject (i.e. one without a specialized _Build function) or any other GalSim object that defines _req_params, _opt_params and _single_params. """ @@ -298,7 +296,7 @@ def _BuildSimple(build_func, config, base, ignore, gsparams, logger): if gsparams: kwargs['gsparams'] = GSParams(**gsparams) if takes_rng: - kwargs['rng'] = GetRNG(config, base, logger, type_name) + kwargs['rng'] = GetRNG(config, base, type_name) safe = False logger.debug('obj %d: kwargs = %s',base.get('obj_num',0),kwargs) @@ -307,13 +305,13 @@ def _BuildSimple(build_func, config, base, ignore, gsparams, logger): return build_func(**kwargs), safe -def _BuildNone(config, base, ignore, gsparams, logger): +def _BuildNone(config, base, ignore, gsparams): """Special type=None returns None. """ return None, True -def _BuildAdd(config, base, ignore, gsparams, logger): +def _BuildAdd(config, base, ignore, gsparams): """Build a Sum object. """ req = { 'items' : list } @@ -328,7 +326,7 @@ def _BuildAdd(config, base, ignore, gsparams, logger): safe = True for i in range(len(items)): - gsobject, safe1 = BuildGSObject(items, i, base, gsparams, logger) + gsobject, safe1 = BuildGSObject(items, i, base, gsparams) # Skip items with flux=0 if 'flux' in items[i] and GetCurrentValue('flux',items[i],float,base) == 0.: logger.debug('obj %d: Not including component with flux == 0',base.get('obj_num',0)) @@ -361,7 +359,7 @@ def _BuildAdd(config, base, ignore, gsparams, logger): return gsobject, safe -def _BuildConvolve(config, base, ignore, gsparams, logger): +def _BuildConvolve(config, base, ignore, gsparams): """Build a Convolution object. """ req = { 'items' : list } @@ -375,7 +373,7 @@ def _BuildConvolve(config, base, ignore, gsparams, logger): raise GalSimConfigError("items entry for type=Convolve is not a list.") safe = True for i in range(len(items)): - gsobject, safe1 = BuildGSObject(items, i, base, gsparams, logger) + gsobject, safe1 = BuildGSObject(items, i, base, gsparams) safe = safe and safe1 gsobjects.append(gsobject) @@ -390,7 +388,7 @@ def _BuildConvolve(config, base, ignore, gsparams, logger): return gsobject, safe -def _BuildList(config, base, ignore, gsparams, logger): +def _BuildList(config, base, ignore, gsparams): """Build a GSObject selected from a List. """ req = { 'items' : list } @@ -408,12 +406,12 @@ def _BuildList(config, base, ignore, gsparams, logger): if index < 0 or index >= len(items): raise GalSimConfigError("index %d out of bounds for List"%index) - gsobject, safe1 = BuildGSObject(items, index, base, gsparams, logger) + gsobject, safe1 = BuildGSObject(items, index, base, gsparams) safe = safe and safe1 return gsobject, safe -def _BuildEval(config, base, ignore, gsparams, logger): +def _BuildEval(config, base, ignore, gsparams): """Build a GSObject from an Eval string """ from .value_eval import _GenerateFromEval @@ -453,7 +451,7 @@ def ParseAberrations(key, config, base, name): else: return None -def _BuildJointOpticalPSF(cls, config, base, ignore, gsparams, logger): +def _BuildJointOpticalPSF(cls, config, base, ignore, gsparams): req, opt, single, _ = get_cls_params(cls) kwargs, safe = GetAllParams(config, base, req, opt, single, ignore = ['aberrations'] + ignore) @@ -462,18 +460,18 @@ def _BuildJointOpticalPSF(cls, config, base, ignore, gsparams, logger): return cls(**kwargs), safe -def _BuildOpticalPSF(config, base, ignore, gsparams, logger): +def _BuildOpticalPSF(config, base, ignore, gsparams): """Build an OpticalPSF. """ - return _BuildJointOpticalPSF(OpticalPSF, config, base, ignore, gsparams, logger) + return _BuildJointOpticalPSF(OpticalPSF, config, base, ignore, gsparams) -def _BuildChromaticOpticalPSF(config, base, ignore, gsparams, logger): +def _BuildChromaticOpticalPSF(config, base, ignore, gsparams): """Build a ChromaticOpticalPSF. """ # All the code for this is the same as for OpticalPSF, so use a shared implementation above. - return _BuildJointOpticalPSF(ChromaticOpticalPSF, config, base, ignore, gsparams, logger) + return _BuildJointOpticalPSF(ChromaticOpticalPSF, config, base, ignore, gsparams) -def _BuildChromaticAtmosphere(config, base, ignore, gsparams, logger): +def _BuildChromaticAtmosphere(config, base, ignore, gsparams): """Build a ChromaticAtmosphere. """ req = {'base_wavelength' : float} @@ -493,7 +491,7 @@ def _BuildChromaticAtmosphere(config, base, ignore, gsparams, logger): if 'base_profile' not in config: raise GalSimConfigError("Attribute base_profile is required for type=ChromaticAtmosphere") - base_profile, safe1 = BuildGSObject(config, 'base_profile', base, gsparams, logger) + base_profile, safe1 = BuildGSObject(config, 'base_profile', base, gsparams) safe = safe and safe1 if 'zenith_angle' not in kwargs: @@ -512,7 +510,7 @@ def _BuildChromaticAtmosphere(config, base, ignore, gsparams, logger): # Now the functions for performing transformations # -def TransformObject(gsobject, config, base, logger): +def TransformObject(gsobject, config, base): """Applies ellipticity, rotation, gravitational shearing and centroid shifting to a supplied GSObject, in that order. @@ -520,12 +518,10 @@ def TransformObject(gsobject, config, base, logger): gsobject: The GSObject to be transformed. config: A dict with the tranformation information for this object. base: The base dict of the configuration. - logger: A logger for logging debug statements. Returns: transformed GSObject. """ - logger = LoggerWrapper(logger) # The transformations are applied in the following order: _transformation_list = [ ('dilate', _Dilate), @@ -544,35 +540,35 @@ def TransformObject(gsobject, config, base, logger): safe = True for key, func in _transformation_list: if key in config: - gsobject, safe1 = func(gsobject, config, key, base, logger) + gsobject, safe1 = func(gsobject, config, key, base) safe = safe and safe1 return gsobject, safe -def _Shear(gsobject, config, key, base, logger): +def _Shear(gsobject, config, key, base): shear, safe = ParseValue(config, key, base, Shear) logger.debug('obj %d: shear = %f,%f',base.get('obj_num',0),shear.g1,shear.g2) gsobject = gsobject._shear(shear) return gsobject, safe -def _Rotate(gsobject, config, key, base, logger): +def _Rotate(gsobject, config, key, base): theta, safe = ParseValue(config, key, base, Angle) logger.debug('obj %d: theta = %f rad',base.get('obj_num',0),theta.rad) gsobject = gsobject.rotate(theta) return gsobject, safe -def _ScaleFlux(gsobject, config, key, base, logger): +def _ScaleFlux(gsobject, config, key, base): flux_ratio, safe = ParseValue(config, key, base, float) logger.debug('obj %d: flux_ratio = %f',base.get('obj_num',0),flux_ratio) gsobject = gsobject * flux_ratio return gsobject, safe -def _Dilate(gsobject, config, key, base, logger): +def _Dilate(gsobject, config, key, base): scale, safe = ParseValue(config, key, base, float) logger.debug('obj %d: scale = %f',base.get('obj_num',0),scale) gsobject = gsobject.dilate(scale) return gsobject, safe -def _Lens(gsobject, config, key, base, logger): +def _Lens(gsobject, config, key, base): shear, safe = ParseValue(config[key], 'shear', base, Shear) mu, safe1 = ParseValue(config[key], 'mu', base, float) safe = safe and safe1 @@ -581,13 +577,13 @@ def _Lens(gsobject, config, key, base, logger): gsobject = gsobject._lens(shear.g1, shear.g2, mu) return gsobject, safe -def _Magnify(gsobject, config, key, base, logger): +def _Magnify(gsobject, config, key, base): mu, safe = ParseValue(config, key, base, float) logger.debug('obj %d: mu = %f',base.get('obj_num',0),mu) gsobject = gsobject.magnify(mu) return gsobject, safe -def _Shift(gsobject, config, key, base, logger): +def _Shift(gsobject, config, key, base): shift, safe = ParseValue(config, key, base, PositionD) logger.debug('obj %d: shift = %f,%f',base.get('obj_num',0),shift.x,shift.y) gsobject = gsobject._shift(shift.x, shift.y) @@ -605,7 +601,6 @@ def RegisterObjectType(type_name, build_func, input_type=None): are present and not valid for the object being built. 4. The gsparams parameter is a dict of kwargs that should be used to build a GSParams object to use when building this object. - 5. The logger parameter is a logging.Logger object to use for logging progress if desired. 6. The return value of build_func should be a tuple consisting of the object and a boolean, safe, which indicates whether the generated object is safe to use again rather than regenerate for subsequent postage stamps. e.g. if a PSF has all constant values, then it @@ -618,7 +613,7 @@ def RegisterObjectType(type_name, build_func, input_type=None): build_func: A function to build a GSObject from the config information. The call signature is:: - obj, safe = Build(config, base, ignore, gsparams, logger) + obj, safe = Build(config, base, ignore, gsparams) input_type: If the type requires an input object, give the key name of the input type here. (If it uses more than one, this may be a list.) diff --git a/galsim/config/image.py b/galsim/config/image.py index 3e9c660a6a..51c8bc4b5f 100644 --- a/galsim/config/image.py +++ b/galsim/config/image.py @@ -16,9 +16,11 @@ # and/or other materials provided with the distribution. # +import logging + import numpy as np -from .util import LoggerWrapper, UpdateNProc, MultiProcess, SetupConfigRNG +from .util import UpdateNProc, MultiProcess, SetupConfigRNG from .input import SetupInput, SetupInputsForImage from .value import ParseValue, GetAllParams from .wcs import BuildWCS @@ -33,6 +35,8 @@ from ..image import Image from ..noise import VariableGaussianNoise +logger = logging.getLogger(__name__) + # This file handles the building of an image by parsing config['image']. # This file includes the basic functionality, but it calls out to helper functions # for parts of the process that are different for different image types. It includes @@ -46,7 +50,7 @@ valid_image_types = {} -def BuildImages(nimages, config, image_num=0, obj_num=0, logger=None): +def BuildImages(nimages, config, image_num=0, obj_num=0): """ Build a number of postage stamp images as specified by the config dict. @@ -55,12 +59,10 @@ def BuildImages(nimages, config, image_num=0, obj_num=0, logger=None): config: The configuration dict. image_num: If given, the current image number. [default: 0] obj_num: If given, the first object number in the image. [default: 0] - logger: If given, a logger object to log progress. [default: None] Returns: a list of images """ - logger = LoggerWrapper(logger) logger.debug('file %d: BuildImages nimages = %d: image, obj = %d,%d', config.get('file_num',0),nimages,image_num,obj_num) @@ -74,7 +76,7 @@ def BuildImages(nimages, config, image_num=0, obj_num=0, logger=None): if nimages > 1 and 'nproc' in image: nproc = ParseValue(image, 'nproc', config, int)[0] # Update this in case the config value is -1 - nproc = UpdateNProc(nproc, nimages, config, logger) + nproc = UpdateNProc(nproc, nimages, config) else: nproc = 1 @@ -87,10 +89,10 @@ def BuildImages(nimages, config, image_num=0, obj_num=0, logger=None): for k in range(nimages): kwargs = { 'image_num' : image_num, 'obj_num' : obj_num } jobs.append(kwargs) - obj_num += GetNObjForImage(config, image_num, logger=logger) + obj_num += GetNObjForImage(config, image_num) image_num += 1 - def done_func(logger, proc, k, image, t): + def done_func(proc, k, image, t): if image is not None: # Note: numpy shape is y,x ys, xs = image.array.shape @@ -99,7 +101,7 @@ def done_func(logger, proc, k, image, t): image_num = jobs[k]['image_num'] logger.info(s0 + 'Image %d: size = %d x %d, time = %f sec', image_num, xs, ys, t) - def except_func(logger, proc, k, e, tr): + def except_func(proc, k, e, tr): if proc is None: s0 = '' else: s0 = '%s: '%proc image_num = jobs[k]['image_num'] @@ -108,10 +110,10 @@ def except_func(logger, proc, k, e, tr): logger.error('Aborting the rest of this file') # Convert to the tasks structure we need for MultiProcess - tasks = MakeImageTasks(config, jobs, logger) + tasks = MakeImageTasks(config, jobs) images = MultiProcess(nproc, config, BuildImage, tasks, 'image', - logger=logger, timeout=timeout, + timeout=timeout, done_func=done_func, except_func=except_func) logger.debug('file %d: Done making images',config.get('file_num',0)) @@ -120,7 +122,7 @@ def except_func(logger, proc, k, e, tr): return images -def SetupConfigImageNum(config, image_num, obj_num, logger=None): +def SetupConfigImageNum(config, image_num, obj_num): """Do the basic setup of the config dict at the image processing level. Includes: @@ -135,9 +137,7 @@ def SetupConfigImageNum(config, image_num, obj_num, logger=None): config: The configuration dict. image_num: The current image number. obj_num: The first object number in the image. - logger: If given, a logger object to log progress. [default: None] """ - logger = LoggerWrapper(logger) config['image_num'] = image_num config['obj_num'] = obj_num config['index_key'] = 'image_num' @@ -160,15 +160,15 @@ def SetupConfigImageNum(config, image_num, obj_num, logger=None): list(valid_image_types.keys())) # In case this hasn't been done yet. - SetupInput(config, logger) + SetupInput(config) # Build the rng to use at the image level. - seed = SetupConfigRNG(config, logger=logger) + seed = SetupConfigRNG(config) logger.debug('image %d: seed = %d',image_num,seed) -def SetupConfigImageSize(config, xsize, ysize, logger=None): +def SetupConfigImageSize(config, xsize, ysize): """Do some further setup of the config dict at the image processing level based on the provided image size. @@ -186,9 +186,7 @@ def SetupConfigImageSize(config, xsize, ysize, logger=None): config: The configuration dict. xsize: The size of the image in the x-dimension. ysize: The size of the image in the y-dimension. - logger: If given, a logger object to log progress. [default: None] """ - logger = LoggerWrapper(logger) config['image_xsize'] = xsize config['image_ysize'] = ysize image = config['image'] @@ -210,7 +208,7 @@ def SetupConfigImageSize(config, xsize, ysize, logger=None): config['image_bounds'] = bounds # Build the wcs - wcs = BuildWCS(image, 'wcs', config, logger) + wcs = BuildWCS(image, 'wcs', config) config['wcs'] = wcs # If the WCS is a PixelScale or OffsetWCS, then store the pixel_scale in base. The @@ -239,7 +237,7 @@ def SetupConfigImageSize(config, xsize, ysize, logger=None): 'use_flux_sky_areas' ] + stamp_image_keys -def BuildImage(config, image_num=0, obj_num=0, logger=None): +def BuildImage(config, image_num=0, obj_num=0): """ Build an Image according to the information in config. @@ -247,18 +245,16 @@ def BuildImage(config, image_num=0, obj_num=0, logger=None): config: The configuration dict. image_num: If given, the current image number. [default: 0] obj_num: If given, the first object number in the image. [default: 0] - logger: If given, a logger object to log progress. [default: None] Returns: the final image """ from .extra import SetupExtraOutputsForImage, ProcessExtraOutputsForImage - logger = LoggerWrapper(logger) logger.debug('image %d: BuildImage: image, obj = %d,%d',image_num,image_num,obj_num) # Setup basic things in the top-level config dict that we will need. - SetupConfigImageNum(config, image_num, obj_num, logger) + SetupConfigImageNum(config, image_num, obj_num) cfg_image = config['image'] # Use cfg_image to avoid name confusion with the actual image # we will build later. @@ -266,34 +262,34 @@ def BuildImage(config, image_num=0, obj_num=0, logger=None): # Do the necessary initial setup for this image type. builder = valid_image_types[image_type] - xsize, ysize = builder.setup(cfg_image, config, image_num, obj_num, image_ignore, logger) + xsize, ysize = builder.setup(cfg_image, config, image_num, obj_num, image_ignore) # Given this image size (which may be 0,0, in which case it will be set automatically later), # do some basic calculations - SetupConfigImageSize(config, xsize, ysize, logger) + SetupConfigImageSize(config, xsize, ysize) logger.debug('image %d: image_size = %d, %d',image_num,xsize,ysize) logger.debug('image %d: image_origin = %s',image_num,config['image_origin']) logger.debug('image %d: image_center = %s',image_num,config['image_center']) # Sometimes an input field needs to do something special at the start of an image. - SetupInputsForImage(config, logger) + SetupInputsForImage(config) # Likewise for the extra output items. - SetupExtraOutputsForImage(config, logger) + SetupExtraOutputsForImage(config) # If there is a bandpass field, load it into config['bandpass'] - bp = builder.buildBandpass(cfg_image, config, image_num, obj_num, logger) + bp = builder.buildBandpass(cfg_image, config, image_num, obj_num) if bp is not None: config['bandpass'] = bp # If there is a sensor, build it now. - sensor = builder.buildSensor(cfg_image, config, image_num, obj_num, logger) + sensor = builder.buildSensor(cfg_image, config, image_num, obj_num) if sensor is not None: config['sensor'] = sensor # Actually build the image now. This is the main working part of this function. # It calls out to the appropriate build function for this image type. - image, current_var = builder.buildImage(cfg_image, config, image_num, obj_num, logger) + image, current_var = builder.buildImage(cfg_image, config, image_num, obj_num) # Store the current image in the base-level config for reference config['current_image'] = image @@ -315,14 +311,14 @@ def BuildImage(config, image_num=0, obj_num=0, logger=None): config['index_key'] = 'image_num' # Do whatever processing is required for the extra output items. - ProcessExtraOutputsForImage(config,logger) + ProcessExtraOutputsForImage(config) - builder.addNoise(image, cfg_image, config, image_num, obj_num, current_var, logger) + builder.addNoise(image, cfg_image, config, image_num, obj_num, current_var) return image -def GetNObjForImage(config, image_num, logger=None, approx=False): +def GetNObjForImage(config, image_num, approx=False): """ Get the number of objects that will be made for the image number image_num based on the information in the config dict. @@ -330,7 +326,6 @@ def GetNObjForImage(config, image_num, logger=None, approx=False): Parameters: config: The configuration dict. image_num: The current image number. - logger: If given, a logger object to log progress. approx: Whether an approximate/overestimate is ok [default: False] Returns: @@ -342,10 +337,10 @@ def GetNObjForImage(config, image_num, logger=None, approx=False): raise GalSimConfigValueError("Invalid image.type.", image_type, list(valid_image_types.keys())) return valid_image_types[image_type].getNObj(image, config, image_num, - logger=logger, approx=approx) + approx=approx) -def FlattenNoiseVariance(config, full_image, stamps, current_vars, logger): +def FlattenNoiseVariance(config, full_image, stamps, current_vars): """This is a helper function to bring the noise level up to a constant value across the image. If some of the galaxies are RealGalaxy objects and noise whitening (or symmetrizing) is turned on, then there will already be some noise in the @@ -358,12 +353,10 @@ def FlattenNoiseVariance(config, full_image, stamps, current_vars, logger): full_image: The full image onto which the noise should be added. stamps: A list of the individual postage stamps. current_vars: A list of the current variance in each postage stamps. - logger: If given, a logger object to log progress. Returns: the final variance in the image """ - logger = LoggerWrapper(logger) rng = config['image_num_rng'] nobjects = len(stamps) max_current_var = max(tuple(current_vars) + (0,)) # Include 0 in case current_vars is empty. @@ -391,7 +384,7 @@ def FlattenNoiseVariance(config, full_image, stamps, current_vars, logger): return max_current_var -def MakeImageTasks(config, jobs, logger): +def MakeImageTasks(config, jobs): """Turn a list of jobs into a list of tasks. See the doc string for galsim.config.MultiProcess for the meaning of this distinction. @@ -407,14 +400,13 @@ def MakeImageTasks(config, jobs, logger): config: The configuration dict jobs: A list of jobs to split up into tasks. Each job in the list is a dict of parameters that includes 'image_num' and 'obj_num'. - logger: If given, a logger object to log progress. Returns: a list of tasks """ image = config.get('image', {}) image_type = image.get('type', 'Single') - return valid_image_types[image_type].makeTasks(image, config, jobs, logger) + return valid_image_types[image_type].makeTasks(image, config, jobs) class ImageBuilder: @@ -424,7 +416,7 @@ class ImageBuilder: It also includes the implementation of the default image type: Single. """ - def setup(self, config, base, image_num, obj_num, ignore, logger): + def setup(self, config, base, image_num, obj_num, ignore): """Do the initialization and setup for building the image. This figures out the size that the image will be, but doesn't actually build it yet. @@ -436,7 +428,6 @@ def setup(self, config, base, image_num, obj_num, ignore, logger): obj_num: The first object number in the image. ignore: A list of parameters that are allowed to be in config that we can ignore here. i.e. it won't be an error if these parameters are present. - logger: If given, a logger object to log progress. Returns: xsize, ysize @@ -463,7 +454,7 @@ def setup(self, config, base, image_num, obj_num, ignore, logger): return xsize, ysize - def buildBandpass(self, config, base, image_num, obj_num, logger): + def buildBandpass(self, config, base, image_num, obj_num): """If thre is a 'bandpass' field in config['image'], load it. Parameters: @@ -471,17 +462,16 @@ def buildBandpass(self, config, base, image_num, obj_num, logger): base: The base configuration dict. image_num: The current image number. obj_num: The first object number in the image. - logger: If given, a logger object to log progress. Returns: a gasim.Bandpass or None """ if 'bandpass' in config: - return BuildBandpass(config, 'bandpass', base, logger)[0] + return BuildBandpass(config, 'bandpass', base)[0] else: return None - def buildSensor(self, config, base, image_num, obj_num, logger): + def buildSensor(self, config, base, image_num, obj_num): """Build the sensor if given in the config dict. Parameters: @@ -489,17 +479,16 @@ def buildSensor(self, config, base, image_num, obj_num, logger): base: The base configuration dict. image_num: The current image number. obj_num: The first object number in the image. - logger: If given, a logger object to log progress. Returns: a galsim.Sensor or None """ if 'sensor' in config: - return BuildSensor(config, 'sensor', base, logger) + return BuildSensor(config, 'sensor', base) else: return None - def buildImage(self, config, base, image_num, obj_num, logger): + def buildImage(self, config, base, image_num, obj_num): """Build an Image based on the parameters in the config dict. For Single, this is just an image consisting of a single postage stamp. @@ -509,7 +498,6 @@ def buildImage(self, config, base, image_num, obj_num, logger): base: The base configuration dict. image_num: The current image number. obj_num: The first object number in the image. - logger: If given, a logger object to log progress. Returns: the final image and the current noise variance in the image as a tuple @@ -519,7 +507,7 @@ def buildImage(self, config, base, image_num, obj_num, logger): logger.debug('image %d: Single Image: size = %s, %s',image_num,xsize,ysize) image, current_var = BuildStamp( - base, obj_num=obj_num, xsize=xsize, ysize=ysize, do_noise=True, logger=logger) + base, obj_num=obj_num, xsize=xsize, ysize=ysize, do_noise=True) if image is not None: image.wcs = base['wcs'] # in case stamp has a local jacobian. @@ -532,7 +520,7 @@ def buildImage(self, config, base, image_num, obj_num, logger): return image, current_var - def makeTasks(self, config, base, jobs, logger): + def makeTasks(self, config, base, jobs): """Turn a list of jobs into a list of tasks. Each task is performed separately in multi-processing runs, so this provides a mechanism @@ -557,14 +545,13 @@ def makeTasks(self, config, base, jobs, logger): base: The base configuration dict. jobs: A list of jobs to split up into tasks. Each job in the list is a dict of parameters that includes 'image_num' and 'obj_num'. - logger: If given, a logger object to log progress. Returns: a list of tasks """ - return MakeStampTasks(base, jobs, logger) + return MakeStampTasks(base, jobs) - def addNoise(self, image, config, base, image_num, obj_num, current_var, logger): + def addNoise(self, image, config, base, image_num, obj_num, current_var): """Add the final noise to the image. In the base class, this is a no op, since it directs the BuildStamp function to build @@ -578,11 +565,10 @@ def addNoise(self, image, config, base, image_num, obj_num, current_var, logger) image_num: The current image number. obj_num: The first object number in the image. current_var: The current noise variance in each postage stamps. - logger: If given, a logger object to log progress. """ pass - def getNObj(self, config, base, image_num, logger=None, approx=False): + def getNObj(self, config, base, image_num, approx=False): """Get the number of objects that will be built for this image. For Single, this is just 1, but other image types would figure this out from the @@ -592,7 +578,6 @@ def getNObj(self, config, base, image_num, logger=None, approx=False): config: The configuration dict for the image field. base: The base configuration dict. image_num: The current image number. - logger: If given, a logger object to log progress. approx: Whether an approximate/overestimate is ok [default: False] Returns: diff --git a/galsim/config/image_scattered.py b/galsim/config/image_scattered.py index f067a3b6df..7518daf9e1 100644 --- a/galsim/config/image_scattered.py +++ b/galsim/config/image_scattered.py @@ -27,12 +27,14 @@ from ..errors import GalSimConfigError, GalSimConfigValueError from ..image import Image +logger = logging.getLogger(__name__) + # This file adds image type Scattered, which places individual stamps at arbitrary # locations on a larger image. class ScatteredImageBuilder(ImageBuilder): - def setup(self, config, base, image_num, obj_num, ignore, logger): + def setup(self, config, base, image_num, obj_num, ignore): """Do the initialization and setup for building the image. This figures out the size that the image will be, but doesn't actually build it yet. @@ -44,7 +46,6 @@ def setup(self, config, base, image_num, obj_num, ignore, logger): obj_num: The first object number in the image. ignore: A list of parameters that are allowed to be in config that we can ignore here. i.e. it won't be an error if these parameters are present. - logger: If given, a logger object to log progress. Returns: xsize, ysize @@ -52,7 +53,7 @@ def setup(self, config, base, image_num, obj_num, ignore, logger): logger.debug('image %d: Building Scattered: image, obj = %d,%d', image_num,image_num,obj_num) - self.nobjects = self.getNObj(config, base, image_num, logger=logger) + self.nobjects = self.getNObj(config, base, image_num) logger.debug('image %d: nobj = %d',image_num,self.nobjects) # These are allowed for Scattered, but we don't use them here. @@ -79,7 +80,7 @@ def setup(self, config, base, image_num, obj_num, ignore, logger): return full_xsize, full_ysize - def buildImage(self, config, base, image_num, obj_num, logger): + def buildImage(self, config, base, image_num, obj_num): """Build an Image containing multiple objects placed at arbitrary locations. Parameters: @@ -87,7 +88,6 @@ def buildImage(self, config, base, image_num, obj_num, logger): base: The base configuration dict. image_num: The current image number. obj_num: The first object number in the image. - logger: If given, a logger object to log progress. Returns: the final image and the current noise variance in the image as a tuple @@ -115,7 +115,7 @@ def buildImage(self, config, base, image_num, obj_num, logger): } stamps, current_vars = BuildStamps( - self.nobjects, base, logger=logger, obj_num=obj_num, do_noise=False) + self.nobjects, base, obj_num=obj_num, do_noise=False) base['index_key'] = 'image_num' @@ -138,11 +138,11 @@ def buildImage(self, config, base, image_num, obj_num, logger): # Bring the image so far up to a flat noise variance current_var = FlattenNoiseVariance( - base, full_image, stamps, current_vars, logger) + base, full_image, stamps, current_vars) return full_image, current_var - def makeTasks(self, config, base, jobs, logger): + def makeTasks(self, config, base, jobs): """Turn a list of jobs into a list of tasks. Here we just have one job per task. @@ -152,14 +152,13 @@ def makeTasks(self, config, base, jobs, logger): base: The base configuration dict. jobs: A list of jobs to split up into tasks. Each job in the list is a dict of parameters that includes 'image_num' and 'obj_num'. - logger: If given, a logger object to log progress. Returns: a list of tasks """ return [ [ (job, k) ] for k, job in enumerate(jobs) ] - def addNoise(self, image, config, base, image_num, obj_num, current_var, logger): + def addNoise(self, image, config, base, image_num, obj_num, current_var): """Add the final noise to a Scattered image Parameters: @@ -169,21 +168,19 @@ def addNoise(self, image, config, base, image_num, obj_num, current_var, logger) image_num: The current image number. obj_num: The first object number in the image. current_var: The current noise variance in each postage stamps. - logger: If given, a logger object to log progress. """ base['current_noise_image'] = base['current_image'] AddSky(base,image) - AddNoise(base,image,current_var,logger) + AddNoise(base,image,current_var) - def getNObj(self, config, base, image_num, logger=None, approx=False): + def getNObj(self, config, base, image_num, approx=False): """Get the number of objects that will be built for this image. Parameters: config: The configuration dict for the image field. base: The base configuration dict. image_num: The current image number. - logger: If given, a logger object to log progress. approx: Whether an approximate/overestimate is ok [default: False] Returns: @@ -195,7 +192,7 @@ def getNObj(self, config, base, image_num, logger=None, approx=False): # Allow nobjects to be automatic based on input catalog if 'nobjects' not in config: - nobj = ProcessInputNObjects(base, logger=logger, approx=approx) + nobj = ProcessInputNObjects(base, approx=approx) if nobj is None: raise GalSimConfigError( "Attribute nobjects is required for image.type = Scattered") diff --git a/galsim/config/image_tiled.py b/galsim/config/image_tiled.py index 8bd149b6ef..b86cd23001 100644 --- a/galsim/config/image_tiled.py +++ b/galsim/config/image_tiled.py @@ -28,12 +28,14 @@ from ..image import Image from .. import random +logger = logging.getLogger(__name__) + # This file adds image type Tiled, which builds a larger image by tiling nx x ny individual # postage stamps. class TiledImageBuilder(ImageBuilder): - def setup(self, config, base, image_num, obj_num, ignore, logger): + def setup(self, config, base, image_num, obj_num, ignore): """Do the initialization and setup for building the image. This figures out the size that the image will be, but doesn't actually build it yet. @@ -45,7 +47,6 @@ def setup(self, config, base, image_num, obj_num, ignore, logger): obj_num: The first object number in the image. ignore: A list of parameters that are allowed to be in config that we can ignore here. i.e. it won't be an error if these parameters are present. - logger: If given, a logger object to log progress. Returns: xsize, ysize @@ -101,7 +102,7 @@ def setup(self, config, base, image_num, obj_num, ignore, logger): return full_xsize, full_ysize - def buildImage(self, config, base, image_num, obj_num, logger): + def buildImage(self, config, base, image_num, obj_num): """ Build an Image consisting of a tiled array of postage stamps. @@ -110,7 +111,6 @@ def buildImage(self, config, base, image_num, obj_num, logger): base: The base configuration dict. image_num: The current image number. obj_num: The first object number in the image. - logger: If given, a logger object to log progress. Returns: the final image and the current noise variance in the image as a tuple @@ -133,7 +133,7 @@ def buildImage(self, config, base, image_num, obj_num, logger): elif order.startswith('rand'): ix_list = [ ix for ix in range(self.nx_tiles) for iy in range(self.ny_tiles) ] iy_list = [ iy for ix in range(self.nx_tiles) for iy in range(self.ny_tiles) ] - rng = GetRNG(config, base, logger, 'TiledImage, order = '+order) + rng = GetRNG(config, base, 'TiledImage, order = '+order) random.permute(rng, ix_list, iy_list) else: raise GalSimConfigValueError("Invalid order.", order, ('row', 'col', 'random')) @@ -155,7 +155,7 @@ def buildImage(self, config, base, image_num, obj_num, logger): } stamps, current_vars = BuildStamps( - nobjects, base, logger=logger, obj_num=obj_num, + nobjects, base, obj_num=obj_num, xsize=self.stamp_xsize, ysize=self.stamp_ysize, do_noise=self.do_noise_in_stamps) base['index_key'] = 'image_num' @@ -172,10 +172,10 @@ def buildImage(self, config, base, image_num, obj_num, logger): current_var = 0 if not self.do_noise_in_stamps: current_var = FlattenNoiseVariance( - base, full_image, stamps, current_vars, logger) + base, full_image, stamps, current_vars) return full_image, current_var - def makeTasks(self, config, base, jobs, logger): + def makeTasks(self, config, base, jobs): """Turn a list of jobs into a list of tasks. Here we just have one job per task. @@ -185,14 +185,13 @@ def makeTasks(self, config, base, jobs, logger): base: The base configuration dict. jobs: A list of jobs to split up into tasks. Each job in the list is a dict of parameters that includes 'image_num' and 'obj_num'. - logger: If given, a logger object to log progress. Returns: a list of tasks """ return [ [ (job, k) ] for k, job in enumerate(jobs) ] - def addNoise(self, image, config, base, image_num, obj_num, current_var, logger): + def addNoise(self, image, config, base, image_num, obj_num, current_var): """Add the final noise to a Tiled image Parameters: @@ -202,23 +201,21 @@ def addNoise(self, image, config, base, image_num, obj_num, current_var, logger) image_num: The current image number. obj_num: The first object number in the image. current_var: The current noise variance in each postage stamps. - logger: If given, a logger object to log progress. """ # If didn't do noise above in the stamps, then need to do it here. if not self.do_noise_in_stamps: # Apply the sky and noise to the full image base['current_noise_image'] = base['current_image'] AddSky(base,image) - AddNoise(base,image,current_var,logger) + AddNoise(base,image,current_var) - def getNObj(self, config, base, image_num, logger=None, approx=False): + def getNObj(self, config, base, image_num, approx=False): """Get the number of objects that will be built for this image. Parameters: config: The configuration dict for the image field. base: The base configuration dict. image_num: The current image number. - logger: If given, a logger object to log progress. approx: Whether an approximate/overestimate is ok [default: False] Returns: diff --git a/galsim/config/input.py b/galsim/config/input.py index b6afb0cd0c..a9a53b4444 100644 --- a/galsim/config/input.py +++ b/galsim/config/input.py @@ -16,15 +16,19 @@ # and/or other materials provided with the distribution. # +import logging + from multiprocessing.managers import NamespaceProxy -from .util import LoggerWrapper, RemoveCurrent, GetRNG, GetLoggerProxy, get_cls_params +from .util import RemoveCurrent, GetRNG, get_cls_params from .util import SafeManager, GetIndex, PropagateIndexKeyRNGNum, single_threaded from .value import ParseValue, CheckAllParams, GetAllParams, SetDefaultIndex, _GetBoolValue from .value import RegisterValueType from ..errors import GalSimConfigError, GalSimConfigValueError, GalSimError from ..catalog import Catalog, Dict +logger = logging.getLogger(__name__) + # This file handles processing the input items according to the specifications in config['input']. # This file includes the basic functionality, which is often sufficient for simple input types, # but it has hooks to allow more customized behavior where necessary. See input_*.py for examples. @@ -77,7 +81,7 @@ def InputProxy(target): return ProxyType -def ProcessInput(config, logger=None, file_scope_only=False, safe_only=False): +def ProcessInput(config, file_scope_only=False, safe_only=False): """ Process the input field, reading in any specified input files or setting up any objects that need to be initialized. @@ -93,7 +97,6 @@ def ProcessInput(config, logger=None, file_scope_only=False, safe_only=False): Parameters: config: The configuration dict to process - logger: If given, a logger object to log progress. [default: None] file_scope_only: If True, only process the input items that are marked as being possibly relevant for file- and image-level items. [default: False] safe_only: If True, only process the input items whose construction parameters @@ -101,7 +104,6 @@ def ProcessInput(config, logger=None, file_scope_only=False, safe_only=False): used by multiple processes if appropriate. [default: False] """ if 'input' in config: - logger = LoggerWrapper(logger) file_num = config.get('file_num',0) logger.debug('file %d: Start ProcessInput',file_num) @@ -166,14 +168,14 @@ class InputManager(SafeManager): pass if file_scope_only and not loader.file_scope: continue logger.debug('file %d: Process input key %s',file_num,key) - LoadAllInputObj(config, key, safe_only, logger) + LoadAllInputObj(config, key, safe_only) # Check that there are no other attributes specified. valid_keys = valid_input_types.keys() CheckAllParams(config['input'], ignore=valid_keys) -def SetupInput(config, logger=None): +def SetupInput(config): """Process the input field if it hasn't been processed yet. This is mostly useful if the user isn't running through the full processing and just starting @@ -182,16 +184,15 @@ def SetupInput(config, logger=None): Parameters: config: The configuration dict in which to setup the input items. - logger: If given, a logger object to log progress. [default: None] """ if '_input_objs' not in config: # Make sure any user-set index keys are propagated properly. if 'input' in config: PropagateIndexKeyRNGNum(config['input']) - ProcessInput(config, logger=logger) + ProcessInput(config) -def LoadAllInputObj(config, key, safe_only=False, logger=None): +def LoadAllInputObj(config, key, safe_only=False): """Load all items of a single input type, named key, with definition given by the dict field. This function just detects if the dict item for this key is a list and calls LoadInputObj @@ -209,16 +210,15 @@ def LoadAllInputObj(config, key, safe_only=False, logger=None): config: The configuration dict to process key: The key name of this input type safe_only: Only load "safe" input objects. - logger: If given, a logger object to log progress. [default: None] """ fields = config['input'][key] nfields = len(fields) if isinstance(fields, list) else 1 for num in range(nfields): - input_obj = LoadInputObj(config, key, num, safe_only, logger) + input_obj = LoadInputObj(config, key, num, safe_only) return input_obj -def LoadInputObj(config, key, num=0, safe_only=False, logger=None): +def LoadInputObj(config, key, num=0, safe_only=False): """Load a single input object, named key, with definition given by the dict field. .. note:: @@ -234,12 +234,10 @@ def LoadInputObj(config, key, num=0, safe_only=False, logger=None): key: The key name of this input type num: Which number in the list of this key, if needed. [default: 0] safe_only: Only load "safe" input objects. - logger: If given, a logger object to log progress. [default: None] Returns: The constructed input object, which is also saved in config['_input_objs'][key] """ - logger = LoggerWrapper(logger) if '_input_objs' not in config: config['_input_objs'] = {} all_input_objs = config['_input_objs'] @@ -268,7 +266,7 @@ def LoadInputObj(config, key, num=0, safe_only=False, logger=None): # Not loaded or not current. logger.debug('file %d: Build input type %s',file_num,key) try: - kwargs, safe = loader.getKwargs(field, config, logger) + kwargs, safe = loader.getKwargs(field, config) except Exception as e: # If an exception was raised here, and we are doing the safe_only run, # then it probably needed an rng that we don't have yet. So really, that @@ -280,9 +278,9 @@ def LoadInputObj(config, key, num=0, safe_only=False, logger=None): if str(e).startswith("No input"): dep_input = str(e).split()[2] logger.info("%s input seems to depend on %s. Try loading that.", key, dep_input) - input_obj = LoadAllInputObj(config, dep_input, safe_only=safe_only, logger=logger) + input_obj = LoadAllInputObj(config, dep_input, safe_only=safe_only) # Now recurse to try this key again. - return LoadInputObj(config, key, num=num, safe_only=safe_only, logger=logger) + return LoadInputObj(config, key, num=num, safe_only=safe_only) if safe_only: logger.debug('file %d: caught exception: %s', file_num,e) safe = False @@ -297,13 +295,9 @@ def LoadInputObj(config, key, num=0, safe_only=False, logger=None): logger.debug('file %d: %s kwargs = %s',file_num,key,kwargs) use_proxy = ('_input_manager' in config and 'current_nproc' not in config - and valid_input_types[key].useProxy(config, logger)) + and valid_input_types[key].useProxy(config)) if use_proxy: tag = key + str(num) - if 'logger' in kwargs: - # Loggers can't be pickled. (At least prior to py3.7. Maybe they fixed this?) - # So if we have a logger, switch it for a proxy instead. - kwargs['logger'] = GetLoggerProxy(kwargs['logger']) input_obj = getattr(config['_input_manager'],tag)(**kwargs) else: input_obj = loader.init_func(**kwargs) @@ -318,7 +312,7 @@ def LoadInputObj(config, key, num=0, safe_only=False, logger=None): logger.info('Input %s has %d objects', key, input_obj.getNObjects()) input_objs[num] = input_obj - loader.initialize(input_objs, num, config, logger) + loader.initialize(input_objs, num, config) # Invalidate any currently cached values that use this kind of input object: # TODO: This isn't quite correct if there are multiple versions of this input @@ -336,7 +330,7 @@ def LoadInputObj(config, key, num=0, safe_only=False, logger=None): return input_obj -def ProcessInputNObjects(config, logger=None, approx=False): +def ProcessInputNObjects(config, approx=False): """Process the input field, just enough to determine the number of objects. Some input items are relevant for determining the number of objects in a file or image. @@ -353,20 +347,18 @@ def ProcessInputNObjects(config, logger=None, approx=False): Parameters: config: The configuration dict to process - logger: If given, a logger object to log progress. [default: None] approx: Whether an approximate count is ok. [default: False] Returns: the number of objects to use. """ - logger = LoggerWrapper(logger) if 'input' in config: - SetupInput(config, logger=logger) + SetupInput(config) for key in valid_input_types: loader = valid_input_types[key] if key in config['input'] and loader.has_nobj: # If it's a list, just use the first one. - input_obj = LoadInputObj(config, key, num=0, logger=logger) + input_obj = LoadInputObj(config, key, num=0) if approx and hasattr(input_obj, 'getApproxNObjects'): nobj = input_obj.getApproxNObjects() else: @@ -378,15 +370,14 @@ def ProcessInputNObjects(config, logger=None, approx=False): return None -def SetupInputsForImage(config, logger=None): +def SetupInputsForImage(config): """Do any necessary setup of the input items at the start of an image. Parameters: config: The configuration dict to process - logger: If given, a logger object to log progress. [default: None] """ if 'input' in config: - SetupInput(config, logger=logger) + SetupInput(config) for key in valid_input_types: loader = valid_input_types[key] if key in config['input']: @@ -398,7 +389,7 @@ def SetupInputsForImage(config, logger=None): for num in range(len(fields)): field = fields[num] input_obj = input_objs[num] - loader.setupImage(input_obj, field, config, logger) + loader.setupImage(input_obj, field, config) def GetNumInputObj(input_type, base): """Get the number of input objects of the given type @@ -477,10 +468,6 @@ class InputLoader: dict input object. Thus, dict is our canonical example of an input type for which this parameter should be True. - takes_logger - Whether the input object has a logger attribute. If so, and a proxy is being - used, then the logger will be replaced with a logger proxy. [default: False] - use_proxy Whether to use a proxy for commicating between processes. This is normally necessary whenever multiprocessing is being used, but there are cases where it @@ -497,12 +484,11 @@ class InputLoader: A function to provide the necessary arguments to worker_init. [default: None] This is required whenever worker_init is not None. """ - def __init__(self, init_func, has_nobj=False, file_scope=False, takes_logger=False, + def __init__(self, init_func, has_nobj=False, file_scope=False, use_proxy=True, worker_init=None, worker_initargs=None): self.init_func = init_func self.has_nobj = has_nobj self.file_scope = file_scope - self.takes_logger = takes_logger self.use_proxy = use_proxy if (worker_init is None) != (worker_initargs is None): raise GalSimError("Must provide both worker_init and worker_initargs") @@ -510,7 +496,7 @@ def __init__(self, init_func, has_nobj=False, file_scope=False, takes_logger=Fal worker_init_fns.append(worker_init) worker_initargs_fns.append(worker_initargs) - def getKwargs(self, config, base, logger): + def getKwargs(self, config, base): """Parse the config dict and return the kwargs needed to build the input object. The default implementation looks for special class attributes called: @@ -534,7 +520,6 @@ def getKwargs(self, config, base, logger): Parameters: config: The config dict for this input item base: The base config dict - logger: If given, a logger object to log progress. [default: None] Returns: kwargs, safe @@ -542,14 +527,12 @@ def getKwargs(self, config, base, logger): req, opt, single, takes_rng = get_cls_params(self.init_func) kwargs, safe = GetAllParams(config, base, req=req, opt=opt, single=single) if takes_rng: - rng = GetRNG(config, base, logger, 'input '+self.init_func.__name__) + rng = GetRNG(config, base, 'input '+self.init_func.__name__) kwargs['rng'] = rng safe = False - if self.takes_logger: - kwargs['logger'] = logger return kwargs, safe - def useProxy(self, config, logger=None): + def useProxy(self, config): """Return whether to use a proxy for the input object. The default behavior is to return self.use_proxy, which is set by the constructor @@ -558,11 +541,10 @@ def useProxy(self, config, logger=None): Parameters: config: The base configuration dict. - logger: If given, a logger object to log progress. [default: None] """ return self.use_proxy - def setupImage(self, input_obj, config, base, logger): + def setupImage(self, input_obj, config, base): """Do any necessary setup at the start of each image. In the base class, this function does not do anything. But see PowerSpectrumLoader @@ -572,11 +554,10 @@ def setupImage(self, input_obj, config, base, logger): input_obj: The input object to use config: The configuration dict for the input type base: The base configuration dict. - logger: If given, a logger object to log progress. [default: None] """ pass - def initialize(self, input_objs, num, base, logger): + def initialize(self, input_objs, num, base): """Do any global setup for input objects right after they are loaded. In the base class, this function does not do anything. It can be used to do @@ -592,7 +573,6 @@ def initialize(self, input_objs, num, base, logger): input_objs: The (current) list of input objects. num: The entry in the list that was loaded. base: The base configuration dict. - logger: If given, a logger object to log progress. [default: None] """ pass diff --git a/galsim/config/input_cosmos.py b/galsim/config/input_cosmos.py index 7eed1ab9d4..234e11e0f4 100644 --- a/galsim/config/input_cosmos.py +++ b/galsim/config/input_cosmos.py @@ -26,67 +26,66 @@ from ..gsparams import GSParams from ..galaxy_sample import GalaxySample, COSMOSCatalog +logger = logging.getLogger(__name__) + # This file adds input types cosmos_catalog and sample_galaxy and gsobject types # COSMOSGalaxy and SampleGalaxy. # The SampleGalaxy doesn't need anything special other than registration as a valid input type. -# However, we do make a custom Loader so that we can add a logger line with some information about -# the number of objects in the catalog that passed the initial cuts and other basic catalog info. class SampleLoader(InputLoader): def __init__(self, cls, input_field): self.cls_name = cls.__name__ self.input_field = input_field super().__init__(cls) - def setupImage(self, cosmos_cat, config, base, logger): - if logger: - # Only report as a warning the first time. After that, use info. - first = not base.get('_SampleLoader_reported_as_warning',False) - base['_SampleLoader_reported_as_warning'] = True - if first: - log_level = logging.WARNING - else: - log_level = logging.INFO - # It should be required that base['input']['cosmos_catalog'] exists, but - # just in case someone calls this in a weird way, use get() with defaults. - cc = base.get('input',{}).get(self.input_field,{}) - if isinstance(cc,list): cc = cc[0] - out_str = '' - if 'sample' in cc: - out_str += '\n sample = %s'%cc['sample'] - if 'dir' in cc: - out_str += '\n dir = %s'%cc['dir'] - if 'file_name' in cc: - out_str += '\n file_name = %s'%cc['file_name'] - if out_str != '': - logger.log(log_level, 'Using user-specified %s: %s',self.cls_name, out_str) - logger.info("file %d: Sample catalog has %d total objects; %d passed initial cuts.", - base['file_num'], cosmos_cat.getNTot(), cosmos_cat.nobjects) - if base.get('gal',{}).get('gal_type',None) == 'parametric': - logger.log(log_level,"Using parametric galaxies.") - else: - logger.log(log_level,"Using real galaxies.") + def setupImage(self, cosmos_cat, config, base): + # Only report as a warning the first time. After that, use info. + first = not base.get('_SampleLoader_reported_as_warning',False) + base['_SampleLoader_reported_as_warning'] = True + if first: + log_level = logging.WARNING + else: + log_level = logging.INFO + # It should be required that base['input']['cosmos_catalog'] exists, but + # just in case someone calls this in a weird way, use get() with defaults. + cc = base.get('input',{}).get(self.input_field,{}) + if isinstance(cc,list): cc = cc[0] + out_str = '' + if 'sample' in cc: + out_str += '\n sample = %s'%cc['sample'] + if 'dir' in cc: + out_str += '\n dir = %s'%cc['dir'] + if 'file_name' in cc: + out_str += '\n file_name = %s'%cc['file_name'] + if out_str != '': + logger.log(log_level, 'Using user-specified %s: %s',self.cls_name, out_str) + logger.info("file %d: Sample catalog has %d total objects; %d passed initial cuts.", + base['file_num'], cosmos_cat.getNTot(), cosmos_cat.nobjects) + if base.get('gal',{}).get('gal_type',None) == 'parametric': + logger.log(log_level,"Using parametric galaxies.") + else: + logger.log(log_level,"Using real galaxies.") RegisterInputType('cosmos_catalog', SampleLoader(COSMOSCatalog, 'cosmos_catalog')) RegisterInputType('galaxy_sample', SampleLoader(GalaxySample, 'galaxy_sample')) # The gsobject types coupled to these are COSMOSGalaxy and SampleGalaxy respectively. -def _BuildCOSMOSGalaxy(config, base, ignore, gsparams, logger): +def _BuildCOSMOSGalaxy(config, base, ignore, gsparams): """Build a COSMOS galaxy using the cosmos_catalog input item. """ sample_cat = GetInputObj('cosmos_catalog', config, base, 'COSMOSGalaxy') - return _FinishBuildSampleGalaxy(config, base, ignore, gsparams, logger, + return _FinishBuildSampleGalaxy(config, base, ignore, gsparams, sample_cat, 'COSMOSGalaxy') -def _BuildSampleGalaxy(config, base, ignore, gsparams, logger): +def _BuildSampleGalaxy(config, base, ignore, gsparams): """Build a sample galaxy using the galaxy_sample input item. """ sample_cat = GetInputObj('galaxy_sample', config, base, 'SampleGalaxy') - return _FinishBuildSampleGalaxy(config, base, ignore, gsparams, logger, + return _FinishBuildSampleGalaxy(config, base, ignore, gsparams, sample_cat, 'SampleGalaxy') -def _FinishBuildSampleGalaxy(config, base, ignore, gsparams, logger, sample_cat, cls_name): +def _FinishBuildSampleGalaxy(config, base, ignore, gsparams, sample_cat, cls_name): ignore = ignore + ['num'] # Special: if galaxies are selected based on index, and index is Sequence or Random, and max @@ -107,7 +106,7 @@ def _FinishBuildSampleGalaxy(config, base, ignore, gsparams, logger, sample_cat, kwargs, safe = GetAllParams(config, base, opt=opt, ignore=ignore) if gsparams: kwargs['gsparams'] = GSParams(**gsparams) - rng = GetRNG(config, base, logger, cls_name) + rng = GetRNG(config, base, cls_name) if 'index' not in kwargs: kwargs['index'], n_rng_calls = sample_cat.selectRandomIndex(1, rng=rng, _n_rng_calls=True) diff --git a/galsim/config/input_image.py b/galsim/config/input_image.py index 459e5d7438..a52d7f1178 100644 --- a/galsim/config/input_image.py +++ b/galsim/config/input_image.py @@ -16,11 +16,14 @@ # and/or other materials provided with the distribution. # +import logging from .input import InputLoader, RegisterInputType from .value import GetAllParams from ..fits import read +logger = logging.getLogger(__name__) + # This file adds input type initial_image. class InitialImageLoader(InputLoader): @@ -29,16 +32,14 @@ def __init__(self): self.init_func = read self.has_nobj = False self.file_scope = False - self.takes_logger = False self.use_proxy = False - def getKwargs(self, config, base, logger): + def getKwargs(self, config, base): """Parse the config dict and return the kwargs needed to build the PowerSpectrum object. Parameters: config: The configuration dict for 'power_spectrum' base: The base configuration dict - logger: If given, a logger object to log progress. Returns: kwargs, safe @@ -47,7 +48,7 @@ def getKwargs(self, config, base, logger): opt = { 'dir': str, 'read_header': bool } return GetAllParams(config, base, req=req, opt=opt) - def setupImage(self, input_obj, config, base, logger=None): + def setupImage(self, input_obj, config, base): """Set up the PowerSpectrum input object's gridded values based on the size of the image and the grid spacing. @@ -55,7 +56,6 @@ def setupImage(self, input_obj, config, base, logger=None): input_obj: The PowerSpectrum object to use config: The configuration dict for 'power_spectrum' base: The base configuration dict. - logger: If given, a logger object to log progress. """ base['current_image'] = input_obj.copy() diff --git a/galsim/config/input_nfw.py b/galsim/config/input_nfw.py index c47b7253b7..84a86517b1 100644 --- a/galsim/config/input_nfw.py +++ b/galsim/config/input_nfw.py @@ -16,19 +16,19 @@ # and/or other materials provided with the distribution. # +import logging + from .input import InputLoader, GetInputObj, RegisterInputType from .value import GetCurrentValue, CheckAllParams, GetAllParams, RegisterValueType -from .util import LoggerWrapper from ..errors import GalSimConfigError, GalSimConfigValueError from ..shear import Shear from ..nfw_halo import NFWHalo +logger = logging.getLogger(__name__) + # This file adds input type nfw_halo and value types NFWHaloShear and NFWHaloMagnification. -class NFWLoader(InputLoader): - def setupImage(self, input_obj, config, base, logger=None): - # Just attach the logger to the input_obj so we can use it when evaluating values. - input_obj.logger = LoggerWrapper(logger) +NFWLoader = InputLoader # Register this as a valid input type RegisterInputType('nfw_halo', NFWLoader(NFWHalo)) @@ -37,7 +37,6 @@ def _GenerateFromNFWHaloShear(config, base, value_type): """Return a shear calculated from an NFWHalo object. """ nfw_halo = GetInputObj('nfw_halo', config, base, 'NFWHaloShear') - logger = nfw_halo.logger if 'uv_pos' not in base: raise GalSimConfigError("NFWHaloShear requested, but no position defined.") @@ -68,7 +67,6 @@ def _GenerateFromNFWHaloMagnification(config, base, value_type): """Return a magnification calculated from an NFWHalo object. """ nfw_halo = GetInputObj('nfw_halo', config, base, 'NFWHaloMagnification') - logger = nfw_halo.logger if 'uv_pos' not in base: raise GalSimConfigError("NFWHaloMagnification requested, but no position defined.") diff --git a/galsim/config/input_powerspectrum.py b/galsim/config/input_powerspectrum.py index ce08208ba2..8c5d319a80 100644 --- a/galsim/config/input_powerspectrum.py +++ b/galsim/config/input_powerspectrum.py @@ -21,7 +21,7 @@ import logging from .input import InputLoader, GetInputObj, RegisterInputType -from .util import LoggerWrapper, SetupConfigRNG, GetRNG +from .util import SetupConfigRNG, GetRNG from .value import ParseValue, GetAllParams, CheckAllParams, RegisterValueType from .stamp import ParseWorldPos from ..errors import GalSimConfigError, GalSimConfigValueError @@ -30,6 +30,8 @@ from ..random import BaseDeviate from ..lensing_ps import PowerSpectrum +logger = logging.getLogger(__name__) + # This file adds input type nfw_halo and value types PowerSpectrumShear and # PowerSpectrumMagnification. @@ -40,18 +42,16 @@ class PowerSpectrumLoader(InputLoader): - def getKwargs(self, config, base, logger): + def getKwargs(self, config, base): """Parse the config dict and return the kwargs needed to build the PowerSpectrum object. Parameters: config: The configuration dict for 'power_spectrum' base: The base configuration dict - logger: If given, a logger object to log progress. Returns: kwargs, safe """ - logger = LoggerWrapper(logger) # If we are going to use a different rebuilding cadence than the normal once per image, # then in order for this feature to work properly in a multiprocessing context, @@ -80,10 +80,10 @@ def getKwargs(self, config, base, logger): base['image']['random_seed'] = rs orig_index_key = base.get('index_key', 'file_num') base['index_key'] = 'file_num' - SetupConfigRNG(base, logger=logger) + SetupConfigRNG(base) if image_num is not None: base['index_key'] = 'image_num' - SetupConfigRNG(base, logger=logger) + SetupConfigRNG(base) base['index_key'] = orig_index_key base['obj_num'] = obj_num base['image_num'] = image_num @@ -95,7 +95,7 @@ def getKwargs(self, config, base, logger): opt = PowerSpectrum._opt_params return GetAllParams(config, base, opt=opt, ignore=ignore) - def setupImage(self, input_obj, config, base, logger=None): + def setupImage(self, input_obj, config, base): """Set up the PowerSpectrum input object's gridded values based on the size of the image and the grid spacing. @@ -103,11 +103,7 @@ def setupImage(self, input_obj, config, base, logger=None): input_obj: The PowerSpectrum object to use config: The configuration dict for 'power_spectrum' base: The base configuration dict. - logger: If given, a logger object to log progress. """ - logger = LoggerWrapper(logger) - # Attach the logger to the input_obj so we can use it when evaluating values. - input_obj.logger = logger if 'grid_spacing' in config: grid_spacing = ParseValue(config, 'grid_spacing', base, float)[0] @@ -148,7 +144,7 @@ def setupImage(self, input_obj, config, base, logger=None): variance = None if 'center' in config: - center = ParseWorldPos(config, 'center', base, logger) + center = ParseWorldPos(config, 'center', base) elif base['wcs']._isCelestial: center = PositionD(0,0) else: @@ -163,7 +159,7 @@ def setupImage(self, input_obj, config, base, logger=None): return config['current_setup_index'] = index - rng = GetRNG(config, base, logger, 'PowerSpectrum') + rng = GetRNG(config, base, 'PowerSpectrum') # We don't care about the output here. This just builds the grid, which we'll # access for each object using its position. @@ -191,7 +187,6 @@ def _GenerateFromPowerSpectrumShear(config, base, value_type): """Return a shear calculated from a PowerSpectrum object. """ power_spectrum = GetInputObj('power_spectrum', config, base, 'PowerSpectrumShear') - logger = power_spectrum.logger if 'uv_pos' not in base: raise GalSimConfigError("PowerSpectrumShear requested, but no position defined.") @@ -228,7 +223,6 @@ def _GenerateFromPowerSpectrumMagnification(config, base, value_type): """Return a magnification calculated from a PowerSpectrum object. """ power_spectrum = GetInputObj('power_spectrum', config, base, 'PowerSpectrumMagnification') - logger = power_spectrum.logger if 'uv_pos' not in base: raise GalSimConfigError( diff --git a/galsim/config/input_real.py b/galsim/config/input_real.py index 9aa807b073..fceb70c4d5 100644 --- a/galsim/config/input_real.py +++ b/galsim/config/input_real.py @@ -16,6 +16,8 @@ # and/or other materials provided with the distribution. # +import logging + from .input import InputLoader, GetInputObj, RegisterInputType, GetNumInputObj from .util import GetRNG, get_cls_params from .value import GetAllParams, SetDefaultIndex @@ -24,15 +26,17 @@ from ..errors import GalSimConfigError from ..real import RealGalaxyCatalog, RealGalaxy, ChromaticRealGalaxy +logger = logging.getLogger(__name__) + # This file adds input type real_catalog and gsobject types RealGalaxy and RealGalaxyOriginal. # The RealGalaxyCatalog doesn't need anything special other than registration as a valid # input type. -RegisterInputType('real_catalog', InputLoader(RealGalaxyCatalog, takes_logger=True)) +RegisterInputType('real_catalog', InputLoader(RealGalaxyCatalog)) # There are two gsobject types that are coupled to this: RealGalaxy and RealGalaxyOriginal. -def _BuildRealGalaxy(config, base, ignore, gsparams, logger, param_name='RealGalaxy'): +def _BuildRealGalaxy(config, base, ignore, gsparams, param_name='RealGalaxy'): """Build a RealGalaxy from the real_catalog input item. """ real_cat = GetInputObj('real_catalog', config, base, param_name) @@ -53,7 +57,7 @@ def _BuildRealGalaxy(config, base, ignore, gsparams, logger, param_name='RealGal kwargs, safe = GetAllParams(config, base, req, opt, single, ignore = ignore + ['num']) if gsparams: kwargs['gsparams'] = GSParams(**gsparams) - kwargs['rng'] = GetRNG(config, base, logger, param_name) + kwargs['rng'] = GetRNG(config, base, param_name) if 'index' in kwargs: index = kwargs['index'] @@ -69,14 +73,14 @@ def _BuildRealGalaxy(config, base, ignore, gsparams, logger, param_name='RealGal return gal, safe -def _BuildRealGalaxyOriginal(config, base, ignore, gsparams, logger): +def _BuildRealGalaxyOriginal(config, base, ignore, gsparams): """Return the original image from a RealGalaxy using the real_catalog input item. """ - gal, safe = _BuildRealGalaxy(config, base, ignore, gsparams, logger, + gal, safe = _BuildRealGalaxy(config, base, ignore, gsparams, param_name='RealGalaxyOriginal') return gal.original_gal, safe -def _BuildChromaticRealGalaxy(config, base, ignore, gsparams, logger): +def _BuildChromaticRealGalaxy(config, base, ignore, gsparams): """Build a ChromaticRealGalaxy from several real_catalog input items. """ param_name = 'ChromaticRealGalaxy' @@ -101,7 +105,7 @@ def _BuildChromaticRealGalaxy(config, base, ignore, gsparams, logger): kwargs, safe = GetAllParams(config, base, req, opt, single, ignore) if gsparams: kwargs['gsparams'] = GSParams(**gsparams) - kwargs['rng'] = GetRNG(config, base, logger, param_name) + kwargs['rng'] = GetRNG(config, base, param_name) if 'index' in kwargs: index = kwargs['index'] diff --git a/galsim/config/noise.py b/galsim/config/noise.py index 685275e627..d107a4e1bf 100644 --- a/galsim/config/noise.py +++ b/galsim/config/noise.py @@ -20,7 +20,7 @@ import numpy as np import math -from .util import LoggerWrapper, GetIndex, GetRNG +from .util import GetIndex, GetRNG from .value import ParseValue, GetCurrentValue, GetAllParams from .input import RegisterInputConnectedType from ..errors import GalSimConfigError, GalSimConfigValueError @@ -30,6 +30,8 @@ from ..noise import GaussianNoise, PoissonNoise, DeviateNoise, CCDNoise from ..correlatednoise import getCOSMOSNoise, BaseCorrelatedNoise, UncorrelatedNoise +logger = logging.getLogger(__name__) + # This file handles the functionality for adding noise and the sky to an image after # drawing the objects. @@ -58,7 +60,7 @@ def AddSky(config, im): config['index_key'] = orig_index_key -def AddNoise(config, im, current_var=0., logger=None): +def AddNoise(config, im, current_var=0.): """ Add noise to an image according to the noise specifications in the noise dict. @@ -66,14 +68,12 @@ def AddNoise(config, im, current_var=0., logger=None): config: The (base) configuration dict im: The image onto which to add the noise current_var: The current noise variance present in the image already [default: 0] - logger: If given, a logger object to log progress. [default: None] Returns: Variance added to the image (units are ADU if gain != 1) """ from .stamp import SetupConfigObjNum - logger = LoggerWrapper(logger) if 'noise' in config['image']: noise = config['image']['noise'] else: # No noise. @@ -94,12 +94,12 @@ def AddNoise(config, im, current_var=0., logger=None): rng = GetRNG(noise, config) # This makes sure draw_method is properly copied over and given a default value. - SetupConfigObjNum(config, config.get('obj_num',0), logger) + SetupConfigObjNum(config, config.get('obj_num',0)) draw_method = GetCurrentValue('draw_method', config['stamp'], str, config) builder = valid_noise_types[noise_type] config['index_key'] = 'image_num' - var = builder.addNoise(noise, config, im, rng, current_var, draw_method, logger) + var = builder.addNoise(noise, config, im, rng, current_var, draw_method) config['index_key'] = orig_index_key return var @@ -137,7 +137,7 @@ def CalculateNoiseVariance(config, full=False): return var -def AddNoiseVariance(config, im, include_obj_var=False, logger=None): +def AddNoiseVariance(config, im, include_obj_var=False): """ Add the noise variance to an image according to the noise specifications in the noise dict. Typically, this is used for building a weight map, which is typically the inverse variance. @@ -149,12 +149,10 @@ def AddNoiseVariance(config, im, include_obj_var=False, logger=None): models that have a component based on the number of photons. Note: if this is True, the returned variance will not include this contribution to the noise variance. [default: False] - logger: If given, a logger object to log progress. [default: None] Returns: the variance in the image (units are ADU if gain != 1) """ - logger = LoggerWrapper(logger) if 'noise' in config['image']: noise = config['image']['noise'] else: # No noise. @@ -171,10 +169,10 @@ def AddNoiseVariance(config, im, include_obj_var=False, logger=None): config['index_key'] = 'image_num' builder = valid_noise_types[noise_type] - builder.addNoiseVariance(noise, config, im, include_obj_var, logger) + builder.addNoiseVariance(noise, config, im, include_obj_var) config['index_key'] = orig_index_key -def GetSky(config, base, logger=None, full=False): +def GetSky(config, base, full=False): """Parse the sky information and return either a float value for the sky level per pixel or an image, as needed. @@ -186,7 +184,6 @@ def GetSky(config, base, logger=None, full=False): config: The configuration field with the sky specification, which can be either base['image'] or base['image']['noise'] base: The base configuration dict - logger: If given, a logger object to log progress. [default: None] full: If the sky level is variable across the image, return the full image with the sky at every pixel. Otherwise, just return the sky at the image center. @@ -194,7 +191,6 @@ def GetSky(config, base, logger=None, full=False): Returns: sky, either a float value or an Image. (The latter only if full=True) """ - logger = LoggerWrapper(logger) if 'sky_level' in config: if 'sky_level_pixel' in config: raise GalSimConfigValueError( @@ -255,7 +251,7 @@ class NoiseBuilder: The base class doesn't do anything, but it defines the call signatures of the methods that derived classes should use for the different specific noise types. """ - def addNoise(self, config, base, im, rng, current_var, draw_method, logger): + def addNoise(self, config, base, im, rng, current_var, draw_method): """Read the noise parameters from the config dict and add the appropriate noise to the given image. @@ -266,7 +262,6 @@ def addNoise(self, config, base, im, rng, current_var, draw_method, logger): rng: The random number generator to use for adding the noise. current_var: The current noise variance present in the image already. draw_method: The method that was used to draw the objects on the image. - logger: If given, a logger object to log progress. Returns: the variance of the noise model (units are ADU if gain != 1) @@ -288,7 +283,7 @@ def getNoiseVariance(self, config, base, full=False): """ raise NotImplementedError("The %s class has not overridden addNoise"%self.__class__) - def addNoiseVariance(self, config, base, im, include_obj_var, logger): + def addNoiseVariance(self, config, base, im, include_obj_var): """Read the noise parameters from the config dict and add the appropriate noise variance to the given image. @@ -308,7 +303,6 @@ def addNoiseVariance(self, config, base, im, include_obj_var, logger): object flux in addition to the sky flux. Only relevant for noise models that are based on the image flux values such as Poisson and CCDNoise. - logger: If given, a logger object to log progress. """ im += self.getNoiseVariance(config, base, full=True) @@ -318,7 +312,7 @@ def addNoiseVariance(self, config, base, im, include_obj_var, logger): class GaussianNoiseBuilder(NoiseBuilder): - def addNoise(self, config, base, im, rng, current_var, draw_method, logger): + def addNoise(self, config, base, im, rng, current_var, draw_method): # Read the noise variance var = self.getNoiseVariance(config, base) @@ -362,11 +356,11 @@ def getNoiseVariance(self, config, base, full=False): class PoissonNoiseBuilder(NoiseBuilder): - def addNoise(self, config, base, im, rng, current_var, draw_method, logger): + def addNoise(self, config, base, im, rng, current_var, draw_method): # Get how much extra sky to assume from the image.noise attribute. - sky = GetSky(base['image'], base, logger, full=True) - extra_sky = GetSky(config, base, logger, full=True) + sky = GetSky(base['image'], base, full=True) + extra_sky = GetSky(config, base, full=True) total_sky = sky + extra_sky # for the return value if isinstance(total_sky, Image): var = np.mean(total_sky.array) @@ -424,7 +418,7 @@ def getNoiseVariance(self, config, base, full=False): sky += GetSky(config, base, full=full) return sky - def addNoiseVariance(self, config, base, im, include_obj_var, logger): + def addNoiseVariance(self, config, base, im, include_obj_var): if include_obj_var: # The current image at this point should be the noise-free, sky-free image, # which is the object variance in each pixel. @@ -455,7 +449,7 @@ def getCCDNoiseParams(self, config, base): return gain, read_noise, read_noise_var - def addNoise(self, config, base, im, rng, current_var, draw_method, logger): + def addNoise(self, config, base, im, rng, current_var, draw_method): # This process goes a lot like the Poisson routine. There are just two differences. # First, the Poisson noise is in electrons, not ADU, and now we allow for a gain = e-/ADU, @@ -464,8 +458,8 @@ def addNoise(self, config, base, im, rng, current_var, draw_method, logger): gain, read_noise, read_noise_var = self.getCCDNoiseParams(config, base) # Get how much extra sky to assume from the image.noise attribute. - sky = GetSky(base['image'], base, logger, full=True) - extra_sky = GetSky(config, base, logger, full=True) + sky = GetSky(base['image'], base, full=True) + extra_sky = GetSky(config, base, full=True) total_sky = sky + extra_sky if isinstance(total_sky, Image): var_adu = np.mean(total_sky.array) / gain + read_noise_var / gain**2 @@ -551,7 +545,7 @@ def getNoiseVariance(self, config, base, full=False): read_noise_var_adu = read_noise_var / gain**2 return sky / gain + read_noise_var_adu - def addNoiseVariance(self, config, base, im, include_obj_var, logger): + def addNoiseVariance(self, config, base, im, include_obj_var): gain, read_noise, read_noise_var = self.getCCDNoiseParams(config, base) if include_obj_var: # The current image at this point should be the noise-free, sky-free image, @@ -590,7 +584,7 @@ def _readNoiseFile(self, config, base, rng=None): rng = GetRNG(config, base) return BaseCorrelatedNoise.from_file(rng=rng, **kwargs) - def addNoise(self, config, base, im, rng, current_var, draw_method, logger): + def addNoise(self, config, base, im, rng, current_var, draw_method): # Build the correlated noise cn = self.getNoise(config,base,rng) diff --git a/galsim/config/output.py b/galsim/config/output.py index 7d91fcaf06..1049672608 100644 --- a/galsim/config/output.py +++ b/galsim/config/output.py @@ -20,7 +20,7 @@ import logging import time -from .util import LoggerWrapper, UpdateNProc, CopyConfig, MultiProcess, SetupConfigRNG +from .util import UpdateNProc, CopyConfig, MultiProcess, SetupConfigRNG from .util import RetryIO, SetDefaultExt from .value import ParseValue, CheckAllParams from .input import ProcessInput @@ -31,6 +31,8 @@ from ..utilities import ensure_dir from ..fits import writeMulti +logger = logging.getLogger(__name__) + # This file handles building the output files according to the specifications in config['output']. # This file includes the basic functionality, but it calls out to helper functions for the # different types of output files. It includes the implementation of the default output type, @@ -42,7 +44,7 @@ # that will perform the different stages of processing to construct and write the output file(s). valid_output_types = {} -def BuildFiles(nfiles, config, file_num=0, logger=None, except_abort=False): +def BuildFiles(nfiles, config, file_num=0, except_abort=False): """ Build a number of output files as specified in config. @@ -50,14 +52,12 @@ def BuildFiles(nfiles, config, file_num=0, logger=None, except_abort=False): nfiles: The number of files to build. config: A configuration dict. file_num: If given, the first file_num. [default: 0] - logger: If given, a logger object to log progress. [default: None] except_abort: Whether to abort processing when a file raises an exception (True) or just report errors and continue on (False). [default: False] Returns: the final config dict that was used. """ - logger = LoggerWrapper(logger) t1 = time.time() # The next line relies on getting errors when the rng is undefined. However, the default @@ -68,7 +68,7 @@ def BuildFiles(nfiles, config, file_num=0, logger=None, except_abort=False): # Process the input field for the first file. Often there are "safe" input items # that won't need to be reprocessed each time. So do them here once and keep them # in the config for all file_nums. This is more important if nproc != 1. - ProcessInput(config, logger=logger, safe_only=True) + ProcessInput(config, safe_only=True) jobs = [] # Will be a list of the kwargs to use for each job info = [] # Will be a list of (file_num, file_name) correspongind to each jobs. @@ -86,7 +86,7 @@ def BuildFiles(nfiles, config, file_num=0, logger=None, except_abort=False): if nfiles > 1 and 'nproc' in output: nproc = ParseValue(output, 'nproc', config, int)[0] # Update this in case the config value is -1 - nproc = UpdateNProc(nproc, nfiles, config, logger) + nproc = UpdateNProc(nproc, nfiles, config) # We'll want a pristine version later to give to the workers. else: nproc = 1 @@ -102,16 +102,16 @@ def BuildFiles(nfiles, config, file_num=0, logger=None, except_abort=False): return orig_config for k in range(nfiles + first_file_num): - SetupConfigFileNum(config, file_num, image_num, obj_num, logger) + SetupConfigFileNum(config, file_num, image_num, obj_num) builder = valid_output_types[output['type']] - builder.setup(output, config, file_num, logger) + builder.setup(output, config, file_num) # Process the input fields that might be relevant at file scope: - ProcessInput(config, logger=logger, file_scope_only=True) + ProcessInput(config, file_scope_only=True) # Get the number of objects in each image for this file. - nobj = GetNObjForFile(config, file_num, image_num, logger=logger, approx=True) + nobj = GetNObjForFile(config, file_num, image_num, approx=True) # The kwargs to pass to BuildFile kwargs = { @@ -124,7 +124,7 @@ def BuildFiles(nfiles, config, file_num=0, logger=None, except_abort=False): # Get the file_name here, in case it needs to create directories, which is not # safe to do with multiple processes. (At least not without extra code in the # getFilename function...) - file_name = builder.getFilename(output, config, logger) + file_name = builder.getFilename(output, config) jobs.append(kwargs) info.append( (file_num, file_name) ) @@ -135,17 +135,17 @@ def BuildFiles(nfiles, config, file_num=0, logger=None, except_abort=False): image_num += len(nobj) obj_num += sum(nobj) - def done_func(logger, proc, k, result, t2): + def done_func(proc, k, result, t2): file_num, file_name = info[k] file_name2, t = result # This is the t for which 0 means the file was skipped. if file_name2 != file_name: # pragma: no cover (I think this should never happen.) raise GalSimConfigError("Files seem to be out of sync. %s != %s", file_name, file_name2) - if t != 0 and logger: + if t != 0: if proc is None: s0 = '' else: s0 = '%s: '%proc logger.warning(s0 + 'File %d = %s: time = %f sec', file_num, file_name, t) - def except_func(logger, proc, k, e, tr): + def except_func(proc, k, e, tr): file_num, file_name = info[k] if proc is None: s0 = '' else: s0 = '%s: '%proc @@ -162,7 +162,7 @@ def except_func(logger, proc, k, e, tr): tasks = [ [ (job, k) ] for (k, job) in enumerate(jobs) ] results = MultiProcess(nproc, orig_config, BuildFile, tasks, 'file', - logger=logger, timeout=timeout, + timeout=timeout, done_func=done_func, except_func=except_func, except_abort=except_abort) t2 = time.time() @@ -187,7 +187,7 @@ def except_func(logger, proc, k, e, tr): output_ignore = [ 'nproc', 'timeout', 'skip', 'noclobber', 'retry_io' ] -def BuildFile(config, file_num=0, image_num=0, obj_num=0, logger=None): +def BuildFile(config, file_num=0, image_num=0, obj_num=0): """ Build an output file as specified in config. @@ -196,24 +196,22 @@ def BuildFile(config, file_num=0, image_num=0, obj_num=0, logger=None): file_num: If given, the current file_num. [default: 0] image_num: If given, the current image_num. [default: 0] obj_num: If given, the current obj_num. [default: 0] - logger: If given, a logger object to log progress. [default: None] Returns: (file_name, t), a tuple of the file name and the time taken to build file Note: t==0 indicates that this file was skipped. """ - logger = LoggerWrapper(logger) t1 = time.time() - SetupConfigFileNum(config, file_num, image_num, obj_num, logger) + SetupConfigFileNum(config, file_num, image_num, obj_num) output = config['output'] output_type = output['type'] builder = valid_output_types[output_type] - builder.setup(output, config, file_num, logger) + builder.setup(output, config, file_num) # Put these values in the config dict so we won't have to run them again later if # we need them. e.g. ExtraOuput processing uses these. - nobj = GetNObjForFile(config, file_num, image_num, logger=logger) + nobj = GetNObjForFile(config, file_num, image_num) nimages = len(nobj) config['nimages'] = nimages config['nobj'] = nobj @@ -221,11 +219,11 @@ def BuildFile(config, file_num=0, image_num=0, obj_num=0, logger=None): file_num,output_type,nimages,image_num) # Make sure the inputs and extra outputs are set up properly. - ProcessInput(config, logger=logger) - SetupExtraOutput(config, logger=logger) + ProcessInput(config) + SetupExtraOutput(config) # Get the file name - file_name = builder.getFilename(output, config, logger) + file_name = builder.getFilename(output, config) # Check if we ought to skip this file if 'skip' in output and ParseValue(output, 'skip', config, bool)[0]: @@ -244,7 +242,7 @@ def BuildFile(config, file_num=0, image_num=0, obj_num=0, logger=None): logger.warning('Start file %d = %s', file_num, file_name) ignore = output_ignore + list(valid_extra_outputs) - data = builder.buildImages(output, config, file_num, image_num, obj_num, ignore, logger) + data = builder.buildImages(output, config, file_num, image_num, obj_num, ignore) # If any images came back as None, then remove them, since they cannot be written. data = [ im for im in data if im is not None ] @@ -256,7 +254,7 @@ def BuildFile(config, file_num=0, image_num=0, obj_num=0, logger=None): # Go back to file_num as the default index_key. config['index_key'] = 'file_num' - data = builder.addExtraOutputHDUs(config, data, logger) + data = builder.addExtraOutputHDUs(config, data) if 'retry_io' in output: ntries = ParseValue(output,'retry_io',config,int)[0] @@ -265,23 +263,22 @@ def BuildFile(config, file_num=0, image_num=0, obj_num=0, logger=None): else: ntries = 1 - args = (data, file_name, output, config, logger) - RetryIO(builder.writeFile, args, ntries, file_name, logger) + args = (data, file_name, output, config) + RetryIO(builder.writeFile, args, ntries, file_name) logger.debug('file %d: Wrote %s to file %r',file_num,output_type,file_name) - builder.writeExtraOutputs(config, data, logger) + builder.writeExtraOutputs(config, data) t2 = time.time() return file_name, t2-t1 -def GetNFiles(config, logger=None): +def GetNFiles(config): """ Get the number of files that will be made, based on the information in the config dict. Parameters: config: The configuration dict. - logger: If given, a logger object to log progress. [default: None] Returns: the number of files @@ -291,10 +288,10 @@ def GetNFiles(config, logger=None): if output_type not in valid_output_types: raise GalSimConfigValueError("Invalid output.type.", output_type, list(valid_output_types.keys())) - return valid_output_types[output_type].getNFiles(output, config, logger=logger) + return valid_output_types[output_type].getNFiles(output, config) -def GetNImagesForFile(config, file_num, logger=None): +def GetNImagesForFile(config, file_num): """ Get the number of images that will be made for the file number file_num, based on the information in the config dict. @@ -302,7 +299,6 @@ def GetNImagesForFile(config, file_num, logger=None): Parameters: config: The configuration dict. file_num: The current file number. - logger: If given, a logger object to log progress. [default: None] Returns: the number of images @@ -312,10 +308,10 @@ def GetNImagesForFile(config, file_num, logger=None): if output_type not in valid_output_types: raise GalSimConfigValueError("Invalid output.type.", output_type, list(valid_output_types.keys())) - return valid_output_types[output_type].getNImages(output, config, file_num, logger=logger) + return valid_output_types[output_type].getNImages(output, config, file_num) -def GetNObjForFile(config, file_num, image_num, logger=None, approx=False): +def GetNObjForFile(config, file_num, image_num, approx=False): """ Get the number of objects that will be made for each image built as part of the file file_num, which starts at image number image_num, based on the information in the config dict. @@ -324,7 +320,6 @@ def GetNObjForFile(config, file_num, image_num, logger=None, approx=False): config: The configuration dict. file_num: The current file number. image_num: The current image number. - logger: If given, a logger object to log progress. [default: None] approx: Whether an approximate/overestimate is ok [default: False] Returns: @@ -336,10 +331,10 @@ def GetNObjForFile(config, file_num, image_num, logger=None, approx=False): raise GalSimConfigValueError("Invalid output.type.", output_type, list(valid_output_types.keys())) return valid_output_types[output_type].getNObjPerImage(output, config, file_num, image_num, - logger=logger, approx=approx) + approx=approx) -def SetupConfigFileNum(config, file_num, image_num, obj_num, logger=None): +def SetupConfigFileNum(config, file_num, image_num, obj_num): """Do the basic setup of the config dict at the file processing level. Includes: @@ -359,9 +354,7 @@ def SetupConfigFileNum(config, file_num, image_num, obj_num, logger=None): start_obj_num items in the config dict.) image_num: The current image_num. obj_num: The current obj_num. - logger: If given, a logger object to log progress. [default: None] """ - logger = LoggerWrapper(logger) config['file_num'] = file_num config['start_obj_num'] = obj_num config['start_image_num'] = image_num @@ -391,7 +384,7 @@ class OutputBuilder: # A class attribute that sub-classes may override. default_ext = '.fits' - def setup(self, config, base, file_num, logger): + def setup(self, config, base, file_num): """Do any necessary setup at the start of processing a file. The base class just calls SetupConfigRNG, but this provides a hook for sub-classes to @@ -401,12 +394,11 @@ def setup(self, config, base, file_num, logger): config: The configuration dict for the output type. base: The base configuration dict. file_num: The current file_num. - logger: If given, a logger object to log progress. """ - seed = SetupConfigRNG(base, logger=logger) + seed = SetupConfigRNG(base) logger.debug('file %d: seed = %d',file_num,seed) - def getFilename(self, config, base, logger): + def getFilename(self, config, base): """Get the file_name for the current file being worked on. Note that the base class defines a default extension = '.fits'. @@ -415,7 +407,6 @@ def getFilename(self, config, base, logger): Parameters: config: The configuration dict for the output type. base: The base configuration dict. - logger: If given, a logger object to log progress. Returns: the filename to build. @@ -439,7 +430,7 @@ def getFilename(self, config, base, logger): return file_name - def buildImages(self, config, base, file_num, image_num, obj_num, ignore, logger): + def buildImages(self, config, base, file_num, image_num, obj_num, ignore): """Build the images for output. In the base class, this function just calls BuildImage to build the single image to @@ -453,7 +444,6 @@ def buildImages(self, config, base, file_num, image_num, obj_num, ignore, logger obj_num: The current obj_num. ignore: A list of parameters that are allowed to be in config that we can ignore here. i.e. it won't be an error if they are present. - logger: If given, a logger object to log progress. Returns: a list of the images built @@ -463,10 +453,10 @@ def buildImages(self, config, base, file_num, image_num, obj_num, ignore, logger ignore += [ 'file_name', 'dir', 'nfiles' ] CheckAllParams(config, ignore=ignore) - image = BuildImage(base, image_num, obj_num, logger=logger) + image = BuildImage(base, image_num, obj_num) return [ image ] - def getNFiles(self, config, base, logger=None): + def getNFiles(self, config, base): """Returns the number of files to be built. In the base class, this is just output.nfiles. @@ -474,7 +464,6 @@ def getNFiles(self, config, base, logger=None): Parameters: config: The configuration dict for the output field. base: The base configuration dict. - logger: If given, a logger object to log progress. Returns: the number of files to build. @@ -484,7 +473,7 @@ def getNFiles(self, config, base, logger=None): else: return 1 - def getNImages(self, config, base, file_num, logger=None): + def getNImages(self, config, base, file_num): """Returns the number of images to be built for a given ``file_num``. In the base class, we only build a single image, so it returns 1. @@ -493,14 +482,13 @@ def getNImages(self, config, base, file_num, logger=None): config: The configuration dict for the output field. base: The base configuration dict. file_num: The current file number. - logger: If given, a logger object to log progress. Returns: the number of images to build. """ return 1 - def getNObjPerImage(self, config, base, file_num, image_num, logger=None, approx=False): + def getNObjPerImage(self, config, base, file_num, image_num, approx=False): """ Get the number of objects that will be made for each image built as part of the file file_num, which starts at image number image_num, based on the information in the config @@ -511,14 +499,13 @@ def getNObjPerImage(self, config, base, file_num, image_num, logger=None, approx base: The base configuration dict. file_num: The current file number. image_num: The current image number (the first one for this file). - logger: If given, a logger object to log progress. approx: Whether an approximate/overestimate is ok [default: False] Returns: a list of the number of objects in each image [ nobj0, nobj1, nobj2, ... ] """ - nimages = self.getNImages(config, base, file_num, logger=logger) - nobj = [ GetNObjForImage(base, image_num+j, logger=logger, approx=approx) + nimages = self.getNImages(config, base, file_num) + nobj = [ GetNObjForImage(base, image_num+j, approx=approx) for j in range(nimages) ] base['image_num'] = image_num # Make sure this is set back to current image num. return nobj @@ -530,24 +517,23 @@ def canAddHdus(self): """ return True - def addExtraOutputHDUs(self, config, data, logger): + def addExtraOutputHDUs(self, config, data): """If appropriate, add any extra output items that go into HDUs to the data list. Parameters: config: The configuration dict for the output field. data: The data to write. Usually a list of images. - logger: If given, a logger object to log progress. Returns: data (possibly updated with additional items) """ if self.canAddHdus(): - data = AddExtraOutputHDUs(config, data, logger) + data = AddExtraOutputHDUs(config, data) else: - CheckNoExtraOutputHDUs(config, config['output']['type'], logger) + CheckNoExtraOutputHDUs(config, config['output']['type']) return data - def writeFile(self, data, file_name, config, base, logger): + def writeFile(self, data, file_name, config, base): """Write the data to a file. Parameters: @@ -557,19 +543,17 @@ def writeFile(self, data, file_name, config, base, logger): file_name: The file_name to write to. config: The configuration dict for the output field. base: The base configuration dict. - logger: If given, a logger object to log progress. """ writeMulti(data,file_name) - def writeExtraOutputs(self, config, data, logger): + def writeExtraOutputs(self, config, data): """If appropriate, write any extra output items that write their own files. Parameters: config: The configuration dict for the output field. data: The data to write. Usually a list of images. - logger: If given, a logger object to log progress. """ - WriteExtraOutputs(config, data, logger) + WriteExtraOutputs(config, data) def RegisterOutputType(output_type, builder): diff --git a/galsim/config/output_datacube.py b/galsim/config/output_datacube.py index 72a563c1d7..b5681504fd 100644 --- a/galsim/config/output_datacube.py +++ b/galsim/config/output_datacube.py @@ -16,6 +16,7 @@ # and/or other materials provided with the distribution. # +import logging import time from .output import OutputBuilder, RegisterOutputType @@ -26,11 +27,14 @@ from ..errors import GalSimConfigError from ..fits import writeCube +logger = logging.getLogger(__name__) + + class DataCubeBuilder(OutputBuilder): """Builder class for constructing and writing DataCube output types. """ - def buildImages(self, config, base, file_num, image_num, obj_num, ignore, logger): + def buildImages(self, config, base, file_num, image_num, obj_num, ignore): """Build the images A point of attention for DataCubes is that they must all be the same size. @@ -45,12 +49,11 @@ def buildImages(self, config, base, file_num, image_num, obj_num, ignore, logger obj_num: The current obj_num. ignore: A list of parameters that are allowed to be in config that we can ignore here. i.e. it won't be an error if they are present. - logger: If given, a logger object to log progress. Returns: a list of the images built """ - nimages = self.getNImages(config, base, file_num, logger=logger) + nimages = self.getNImages(config, base, file_num) # The above call sets up a default nimages if appropriate. Now, check that there are no # invalid parameters in the config dict. @@ -64,7 +67,7 @@ def buildImages(self, config, base, file_num, image_num, obj_num, ignore, logger # image. t1 = time.time() base1 = CopyConfig(base) - image0 = BuildImage(base1, image_num, obj_num, logger=logger) + image0 = BuildImage(base1, image_num, obj_num) t2 = time.time() # Note: numpy shape is y,x ys, xs = image0.array.shape @@ -78,20 +81,19 @@ def buildImages(self, config, base, file_num, image_num, obj_num, ignore, logger images = [ image0 ] if nimages > 1: - obj_num += GetNObjForImage(base, image_num, logger=logger) - images += BuildImages(nimages-1, base, logger=logger, + obj_num += GetNObjForImage(base, image_num) + images += BuildImages(nimages-1, base, image_num=image_num+1, obj_num=obj_num) return images - def getNImages(self, config, base, file_num, logger=None): + def getNImages(self, config, base, file_num): """Returns the number of images to be built. Parameters: config: The configuration dict for the output field. base: The base configuration dict. file_num: The current file number. - logger: If given, a logger object to log progress. Returns: the number of images to build. @@ -108,7 +110,7 @@ def getNImages(self, config, base, file_num, logger=None): "Attribute output.nimages is required for output.type = MultiFits") return ParseValue(config,'nimages',base,int)[0] - def writeFile(self, data, file_name, config, base, logger): + def writeFile(self, data, file_name, config, base): """Write the data to a file. Parameters: @@ -118,7 +120,6 @@ def writeFile(self, data, file_name, config, base, logger): file_name: The file_name to write to. config: The configuration dict for the output field. base: The base configuration dict. - logger: If given, a logger object to log progress. """ writeCube(data,file_name) diff --git a/galsim/config/output_multifits.py b/galsim/config/output_multifits.py index 53494236f5..41e7fa1fa1 100644 --- a/galsim/config/output_multifits.py +++ b/galsim/config/output_multifits.py @@ -25,11 +25,13 @@ from .value import ParseValue, CheckAllParams from ..errors import GalSimConfigError +logger = logging.getLogger(__name__) + class MultiFitsBuilder(OutputBuilder): """Builder class for constructing and writing MultiFits output types. """ - def buildImages(self, config, base, file_num, image_num, obj_num, ignore, logger): + def buildImages(self, config, base, file_num, image_num, obj_num, ignore): """Build the images Parameters: @@ -40,12 +42,11 @@ def buildImages(self, config, base, file_num, image_num, obj_num, ignore, logger obj_num: The current obj_num. ignore: A list of parameters that are allowed to be in config that we can ignore here. i.e. it won't be an error if they are present. - logger: If given, a logger object to log progress. Returns: a list of the images built """ - nimages = self.getNImages(config, base, file_num, logger=logger) + nimages = self.getNImages(config, base, file_num) # The above call sets up a default nimages if appropriate. Now, check that there are no # invalid parameters in the config dict. @@ -53,9 +54,9 @@ def buildImages(self, config, base, file_num, image_num, obj_num, ignore, logger ignore += [ 'file_name', 'dir', 'nfiles' ] CheckAllParams(config, ignore=ignore, req=req) - return BuildImages(nimages, base, image_num, obj_num, logger=logger) + return BuildImages(nimages, base, image_num, obj_num) - def getNImages(self, config, base, file_num, logger=None): + def getNImages(self, config, base, file_num): """ Get the number of images for a MultiFits file type. @@ -63,7 +64,6 @@ def getNImages(self, config, base, file_num, logger=None): config: The configuration dict for the output field. base: The base configuration dict. file_num: The current file number. - logger: If given, a logger object to log progress. Returns: the number of images diff --git a/galsim/config/photon_ops.py b/galsim/config/photon_ops.py index 068283ea77..db949917be 100644 --- a/galsim/config/photon_ops.py +++ b/galsim/config/photon_ops.py @@ -16,7 +16,9 @@ # and/or other materials provided with the distribution. # -from .util import LoggerWrapper, GetIndex, GetRNG, get_cls_params +import logging + +from .util import GetIndex, GetRNG, get_cls_params from .value import ParseValue, GetAllParams, CheckAllParams, SetDefaultIndex from .input import RegisterInputConnectedType from .sed import BuildSED @@ -26,6 +28,8 @@ from ..photon_array import WavelengthSampler, FRatioAngles, PhotonDCR, Refraction, FocusDepth from ..photon_array import TimeSampler, PupilImageSampler, PupilAnnulusSampler +logger = logging.getLogger(__name__) + # This file handles the construction of photon_ops in config['stamp']['photon_ops']. # This module-level dict will store all the registered photon_op types. @@ -35,7 +39,7 @@ valid_photon_op_types = {} -def BuildPhotonOps(config, key, base, logger=None): +def BuildPhotonOps(config, key, base): """Read the parameters from config[key] (which should be a list) and return a constructed photon_ops as a list. @@ -44,22 +48,20 @@ def BuildPhotonOps(config, key, base, logger=None): (usually base['stamp']) key: The key in the dict for the photon_ops list. base: The base dict of the configuration. - logger: Optionally, provide a logger for logging debug statements. [default: None] Returns: the photon_ops list """ - logger = LoggerWrapper(logger) if not isinstance(config[key], list): raise GalSimConfigError("photon_ops must be a list") photon_ops = config[key] # The list in the config dict ops = [] # List of the actual operators for i in range(len(photon_ops)): - op = BuildPhotonOp(photon_ops, i, base, logger) + op = BuildPhotonOp(photon_ops, i, base) ops.append(op) return ops -def BuildPhotonOp(config, key, base, logger=None): +def BuildPhotonOp(config, key, base): """Read the parameters from config[key] and return a single constructed photon_op object. Parameters: @@ -69,12 +71,10 @@ def BuildPhotonOp(config, key, base, logger=None): things, this is a key into a dict, but here it's normally an integer index into the photon_ops list. base: The base dict of the configuration. - logger: Optionally, provide a logger for logging debug statements. [default: None] Returns: a object that would be valid in a photon_ops list """ - logger = LoggerWrapper(logger) logger.debug('obj %d: Start BuildPhotonOp key = %s',base.get('obj_num',0),key) param = config[key] @@ -110,7 +110,7 @@ def BuildPhotonOp(config, key, base, logger=None): # Need to use a builder. logger.debug('obj %d: Building photon_op type %s', base.get('obj_num',0), op_type) builder = valid_photon_op_types[op_type] - op = builder.buildPhotonOp(param, base, logger) + op = builder.buildPhotonOp(param, base) logger.debug('obj %d: photon_op = %s', base.get('obj_num',0), str(op)) param['current'] = op, False, None, index, index_key @@ -123,7 +123,7 @@ class PhotonOpBuilder: The base class defines the call signatures of the methods that any derived class should follow. """ - def buildPhotonOp(self, config, base, logger): + def buildPhotonOp(self, config, base): """Build the PhotonOp based on the specifications in the config dict. Note: Sub-classes must override this function with a real implementation. @@ -131,7 +131,6 @@ def buildPhotonOp(self, config, base, logger): Parameters: config: The configuration dict for the PhotonOp base: The base configuration dict. - logger: If provided, a logger for logging debug statements. Returns: the constructed PhotonOp object. @@ -149,7 +148,7 @@ class SimplePhotonOpBuilder(PhotonOpBuilder): def __init__(self, init_func): self.init_func = init_func - def getKwargs(self, config, base, logger): + def getKwargs(self, config, base): """Get the kwargs to pass to the build function based on the following attributes of init_func: @@ -168,7 +167,6 @@ def getKwargs(self, config, base, logger): Parameters: config: The configuration dict for the photon_op type. base: The base configuration dict. - logger: If provided, a logger for logging debug statements. Returns: kwargs @@ -176,33 +174,32 @@ def getKwargs(self, config, base, logger): req, opt, single, takes_rng = get_cls_params(self.init_func) kwargs, safe = GetAllParams(config, base, req, opt, single) if takes_rng: # pragma: no cover None of ours have this anymore. But it's still allowed. - kwargs['rng'] = GetRNG(config, base, logger, self.init_func.__name__) + kwargs['rng'] = GetRNG(config, base, self.init_func.__name__) return kwargs - def buildPhotonOp(self, config, base, logger): + def buildPhotonOp(self, config, base): """Build the PhotonOp based on the specifications in the config dict. Parameters: config: The configuration dict for the photon_op type. base: The base configuration dict. - logger: If provided, a logger for logging debug statements. Returns: the constructed PhotonOp object. """ - kwargs = self.getKwargs(config,base,logger) + kwargs = self.getKwargs(config,base) return self.init_func(**kwargs) class WavelengthSamplerBuilder(PhotonOpBuilder): """Build a WavelengthSampler """ # This one needs special handling for sed and bandpass - def buildPhotonOp(self, config, base, logger): + def buildPhotonOp(self, config, base): req, opt, single, takes_rng = get_cls_params(WavelengthSampler) kwargs, safe = GetAllParams(config, base, req, opt, single, ignore=['sed']) if 'sed' not in config: raise GalSimConfigError("sed is required for WavelengthSampler") - sed = BuildSED(config, 'sed', base, logger)[0] + sed = BuildSED(config, 'sed', base)[0] kwargs['sed'] = sed if 'bandpass' not in base: raise GalSimConfigError("bandpass is required for WavelengthSampler") @@ -213,7 +210,7 @@ class PhotonDCRBuilder(PhotonOpBuilder): """Build a PhotonDCR """ # This one needs special handling for obj_coord - def buildPhotonOp(self, config, base, logger): + def buildPhotonOp(self, config, base): req, opt, single, takes_rng = get_cls_params(PhotonDCR) kwargs, safe = GetAllParams(config, base, req, opt, single) if 'sky_pos' in base: @@ -223,7 +220,7 @@ def buildPhotonOp(self, config, base, logger): class ListPhotonOpBuilder(PhotonOpBuilder): """Select a photon_op from a list """ - def buildPhotonOp(self, config, base, logger): + def buildPhotonOp(self, config, base): req = { 'items' : list } opt = { 'index' : int } # Only Check, not Get. We need to handle items a bit differently, since it's a list. diff --git a/galsim/config/process.py b/galsim/config/process.py index 2283e5cf49..701a4dece1 100644 --- a/galsim/config/process.py +++ b/galsim/config/process.py @@ -16,6 +16,7 @@ # and/or other materials provided with the distribution. # +import logging import sys import os import json @@ -28,6 +29,8 @@ from .output import GetNFiles, BuildFiles from ..errors import GalSimValueError +logger = logging.getLogger(__name__) + top_level_fields = ['psf', 'gal', 'stamp', 'image', 'input', 'output', 'eval_variables', 'root', 'modules', 'profile'] @@ -42,7 +45,7 @@ # real file location on disk. valid_templates = {} -def ReadConfig(config_file, file_type=None, logger=None): +def ReadConfig(config_file, file_type=None): """Read in a configuration file and return the corresponding dicts. A YAML file is allowed to define several dicts using multiple documents. The GalSim parser @@ -64,12 +67,10 @@ def ReadConfig(config_file, file_type=None, logger=None): config_file: The name of the configuration file to read. file_type: If given, the type of file to read. [default: None, which mean infer the file type from the extension.] - logger: If given, a logger object to log progress. [default: None] Returns: list of config dicts """ - logger = LoggerWrapper(logger) logger.warning('Reading config file %s', config_file) # Determine the file type from the extension if necessary: if file_type is None: @@ -119,18 +120,16 @@ def ImportModules(config, gdict=None): else: raise -def ProcessTemplate(config, base, logger=None): +def ProcessTemplate(config, base): """If the config dict has a 'template' item, read in the appropriate file and make any requested updates. Parameters: config: The configuration dict. base: The base configuration dict. - logger: If given, a logger object to log progress. [default: None] """ from .value_eval import _GenerateFromEval - logger = LoggerWrapper(logger) if 'template' in config: template_string = config.pop('template') logger.debug("Processing template specified as %s",template_string) @@ -155,7 +154,7 @@ def ProcessTemplate(config, base, logger=None): # Read the config file if appropriate if config_file != '': - template = ReadConfig(config_file, logger=logger)[0] + template = ReadConfig(config_file)[0] ImportModules(template) else: template = base @@ -165,7 +164,7 @@ def ProcessTemplate(config, base, logger=None): template = GetFromConfig(template, field) # In case template has further templates to process, do that now. - ProcessTemplate(template, base=base, logger=logger) + ProcessTemplate(template, base=base) # Copy over the template config into this one. new_params = config.copy() # N.B. Already popped config['template']. @@ -179,29 +178,28 @@ def ProcessTemplate(config, base, logger=None): config['modules'].extend(new_params.pop('modules', [])) # Update the config with the requested changes - UpdateConfig(config, new_params, logger) + UpdateConfig(config, new_params) -def ProcessAllTemplates(config, logger=None, base=None): +def ProcessAllTemplates(config, base=None): """Check through the full config dict and process any fields that have a 'template' item. Parameters: config: The configuration dict. - logger: If given, a logger object to log progress. [default: None] base: The base configuration dict. [default: None] """ if base is None: base = config - ProcessTemplate(config, base, logger) + ProcessTemplate(config, base) for (key, field) in list(config.items()): if isinstance(field, dict): - ProcessAllTemplates(field, logger, base) + ProcessAllTemplates(field, base) elif isinstance(field, list): for item in field: if isinstance(item, dict): - ProcessAllTemplates(item, logger, base) + ProcessAllTemplates(item, base) # This is the main script to process everything in the configuration dict. -def Process(config, logger=None, njobs=1, job=1, new_params=None, except_abort=False): +def Process(config, njobs=1, job=1, new_params=None, except_abort=False): """ Do all processing of the provided configuration dict. In particular, this function handles processing the output field, calling other functions to @@ -216,7 +214,6 @@ def Process(config, logger=None, njobs=1, job=1, new_params=None, except_abort=F Parameters: config: The configuration dict. - logger: If given, a logger object to log progress. [default: None] njobs: The total number of jobs to split the work into. [default: 1] job: Which job should be worked on here (1..njobs). [default: 1] new_params: A dict of new parameter values that should be used to update the config @@ -227,7 +224,6 @@ def Process(config, logger=None, njobs=1, job=1, new_params=None, except_abort=F Returns: the final config dict that was used. """ - logger = LoggerWrapper(logger) if njobs < 1: raise GalSimValueError("Invalid number of jobs",njobs) if job < 1: @@ -242,11 +238,11 @@ def Process(config, logger=None, njobs=1, job=1, new_params=None, except_abort=F ImportModules(config) # Process any template specifications in the dict. - ProcessAllTemplates(config, logger) + ProcessAllTemplates(config) # Update using any new_params that are given: if new_params is not None: - UpdateConfig(config, new_params, logger) + UpdateConfig(config, new_params) # Do this again in case any new modules were added by the templates or command line params. ImportModules(config) @@ -256,14 +252,14 @@ def Process(config, logger=None, njobs=1, job=1, new_params=None, except_abort=F # Warn about any unexpected fields. unexpected = [ k for k in config if k not in top_level_fields and k[0] != '_' ] - if len(unexpected) > 0 and logger: + if len(unexpected) > 0: logger.warning("Warning: config dict contains the following unexpected fields: %s.", unexpected) logger.warning("These fields are not (directly) processed by the config processing.") # Determine how many files we will be processing in total. # Usually, this is just output.nfiles, but different output types may define this differently. - nfiles = GetNFiles(config, logger=logger) + nfiles = GetNFiles(config) logger.debug('nfiles = %d',nfiles) if njobs > 1: @@ -282,7 +278,7 @@ def Process(config, logger=None, njobs=1, job=1, new_params=None, except_abort=F #BuildFiles returns the config dictionary, which can includes stuff added #by custom output types during the run. - config_out = BuildFiles(nfiles, config, file_num=start, logger=logger, + config_out = BuildFiles(nfiles, config, file_num=start, except_abort=except_abort) #Return config_out in case useful return config_out diff --git a/galsim/config/sed.py b/galsim/config/sed.py index 1f87d5a827..add328b7d4 100644 --- a/galsim/config/sed.py +++ b/galsim/config/sed.py @@ -16,9 +16,10 @@ # and/or other materials provided with the distribution. # +import logging + from astropy.units import Quantity, Unit -from .util import LoggerWrapper from .value import ParseValue, GetAllParams, GetIndex from .input import RegisterInputConnectedType from .bandpass import BuildBandpass @@ -26,25 +27,25 @@ from ..sed import SED from ..utilities import basestring, LRU_Cache +logger = logging.getLogger(__name__) + # This module-level dict will store all the registered SED types. # See the RegisterSEDType function at the end of this file. # The keys are the (string) names of the SED types, and the values will be builders that know # how to build the SED object. valid_sed_types = {} -def BuildSED(config, key, base, logger=None): +def BuildSED(config, key, base): """Read the SED parameters from config[key] and return a constructed SED object. Parameters: config: A dict with the configuration information. key: The key name in config indicating which object to build. base: The base dict of the configuration. - logger: Optionally, provide a logger for logging debug statements. [default: None] Returns: (sed, safe) where sed is an SED instance, and safe is whether it is safe to reuse. """ - logger = LoggerWrapper(logger) logger.debug('obj %d: Start BuildSED key = %s',base.get('obj_num',0),key) param = config[key] @@ -77,7 +78,7 @@ def BuildSED(config, key, base, logger=None): raise GalSimConfigValueError("Invalid sed.type.", sed_type, list(valid_sed_types.keys())) logger.debug('obj %d: Building sed type %s', base.get('obj_num',0), sed_type) builder = valid_sed_types[sed_type] - sed, safe = builder.buildSED(param, base, logger) + sed, safe = builder.buildSED(param, base) logger.debug('obj %d: sed = %s', base.get('obj_num',0), sed) param['current'] = sed, safe, SED, index, index_key @@ -90,7 +91,7 @@ class SEDBuilder: The base class defines the call signatures of the methods that any derived class should follow. """ - def buildSED(self, config, base, logger): + def buildSED(self, config, base): """Build the SED based on the specifications in the config dict. Note: Sub-classes must override this function with a real implementation. @@ -98,7 +99,6 @@ def buildSED(self, config, base, logger): Parameters: config: The configuration dict for the SED type. base: The base configuration dict. - logger: If provided, a logger for logging debug statements. Returns: the constructed SED object. @@ -120,18 +120,16 @@ class FileSEDBuilder(SEDBuilder): flux_type (required) Which kind of flux values are in the file Allowed values: flambda, fnu, fphotons, 1 """ - def buildSED(self, config, base, logger): + def buildSED(self, config, base): """Build the SED based on the specifications in the config dict. Parameters: config: The configuration dict for the SED type. base: The base configuration dict. - logger: If provided, a logger for logging debug statements. Returns: the constructed SED object. """ - logger = LoggerWrapper(logger) req = { 'file_name': str, @@ -162,7 +160,7 @@ def buildSED(self, config, base, logger): if norm_flux_density is not None: sed = sed.withFluxDensity(norm_flux_density, wavelength=norm_wavelength) elif norm_flux: - bandpass, safe1 = BuildBandpass(config, 'norm_bandpass', base, logger) + bandpass, safe1 = BuildBandpass(config, 'norm_bandpass', base) sed = sed.withFlux(norm_flux, bandpass=bandpass) safe = safe and safe1 sed = sed.atRedshift(redshift) diff --git a/galsim/config/sensor.py b/galsim/config/sensor.py index d086bd0906..2f8d676e5e 100644 --- a/galsim/config/sensor.py +++ b/galsim/config/sensor.py @@ -18,13 +18,15 @@ import logging -from .util import LoggerWrapper, GetIndex, GetRNG, get_cls_params +from .util import GetIndex, GetRNG, get_cls_params from .value import ParseValue, GetAllParams, CheckAllParams, SetDefaultIndex from .input import RegisterInputConnectedType from ..sensor import Sensor, SiliconSensor from ..errors import GalSimConfigError, GalSimConfigValueError from ..utilities import basestring +logger = logging.getLogger(__name__) + # This file handles the construction of a Sensor in config['image']['sensor']. # This module-level dict will store all the registered sensor types. @@ -34,7 +36,7 @@ valid_sensor_types = {} -def BuildSensor(config, key, base, logger=None): +def BuildSensor(config, key, base): """Read the parameters from config[key] and return a constructed Sensor. Parameters: @@ -42,12 +44,10 @@ def BuildSensor(config, key, base, logger=None): (usually base['image']) key: The key in the dict for the sensor configuration. base: The base dict of the configuration. - logger: Optionally, provide a logger for logging debug statements. [default: None] Returns: a Sensor """ - logger = LoggerWrapper(logger) logger.debug('obj %d: Start BuildSensor key = %s',base.get('obj_num',0),key) param = config[key] @@ -83,7 +83,7 @@ def BuildSensor(config, key, base, logger=None): # Need to use a builder. logger.debug('obj %d: Building sensor type %s', base.get('obj_num',0), sensor_type) builder = valid_sensor_types[sensor_type] - sensor = builder.buildSensor(param, base, logger) + sensor = builder.buildSensor(param, base) logger.debug('obj %d: sensor = %s', base.get('obj_num',0), sensor) param['current'] = sensor, False, None, index, index_key @@ -96,7 +96,7 @@ class SensorBuilder: The base class defines the call signatures of the methods that any derived class should follow. """ - def buildSensor(self, config, base, logger): + def buildSensor(self, config, base): """Build the Sensor based on the specifications in the config dict. Note: Sub-classes must override this function with a real implementation. @@ -104,7 +104,6 @@ def buildSensor(self, config, base, logger): Parameters: config: The configuration dict for the Sensor base: The base configuration dict. - logger: If provided, a logger for logging debug statements. Returns: the constructed Sensor object. @@ -122,7 +121,7 @@ class SimpleSensorBuilder(SensorBuilder): def __init__(self, init_func): self.init_func = init_func - def getKwargs(self, config, base, logger): + def getKwargs(self, config, base): """Get the kwargs to pass to the build function based on the following attributes of init_func: @@ -141,7 +140,6 @@ def getKwargs(self, config, base, logger): Parameters: config: The configuration dict for the sensor type. base: The base configuration dict. - logger: If provided, a logger for logging debug statements. Returns: kwargs @@ -149,27 +147,26 @@ def getKwargs(self, config, base, logger): req, opt, single, takes_rng = get_cls_params(self.init_func) kwargs, safe = GetAllParams(config, base, req, opt, single) if takes_rng: - kwargs['rng'] = GetRNG(config, base, logger, self.init_func.__name__) + kwargs['rng'] = GetRNG(config, base, self.init_func.__name__) return kwargs - def buildSensor(self, config, base, logger): + def buildSensor(self, config, base): """Build the Sensor based on the specifications in the config dict. Parameters: config: The configuration dict for the sensor type. base: The base configuration dict. - logger: If provided, a logger for logging debug statements. Returns: the constructed Sensor object. """ - kwargs = self.getKwargs(config,base,logger) + kwargs = self.getKwargs(config,base) return self.init_func(**kwargs) class ListSensorBuilder(SensorBuilder): """Select a sensor from a list """ - def buildSensor(self, config, base, logger): + def buildSensor(self, config, base): req = { 'items' : list } opt = { 'index' : int } # Only Check, not Get. We need to handle items a bit differently, since it's a list. diff --git a/galsim/config/stamp.py b/galsim/config/stamp.py index f293e8fef9..22a35c241e 100644 --- a/galsim/config/stamp.py +++ b/galsim/config/stamp.py @@ -21,7 +21,7 @@ import math import traceback -from .util import LoggerWrapper, GetRNG, UpdateNProc, MultiProcess, SetupConfigRNG, RemoveCurrent +from .util import GetRNG, UpdateNProc, MultiProcess, SetupConfigRNG, RemoveCurrent from .input import SetupInput from .gsobject import UpdateGSParams, SkipThisObject from .gsobject import BuildGSObject @@ -42,6 +42,8 @@ from ..chromatic import ChromaticObject from ..bandpass import Bandpass +logger = logging.getLogger(__name__) + # This file handles the building of postage stamps to place onto a larger image. # There is only one type of stamp currently, called Basic, which builds a galaxy from # config['gal'] and a PSF from config['psf'] (either but not both of which may be absent), @@ -57,7 +59,7 @@ def BuildStamps(nobjects, config, obj_num=0, - xsize=0, ysize=0, do_noise=True, logger=None): + xsize=0, ysize=0, do_noise=True): """ Build a number of postage stamp images as specified by the config dict. @@ -75,12 +77,10 @@ def BuildStamps(nobjects, config, obj_num=0, automatic sizing.] do_noise: Whether to add noise to the image (according to config['noise']). [default: True] - logger: If given, a logger object to log progress. [default: None] Returns: the tuple (images, current_vars). Both are themselves tuples. """ - logger = LoggerWrapper(logger) logger.debug('image %d: BuildStamps nobjects = %d: obj = %d', config.get('image_num',0),nobjects,obj_num) @@ -92,7 +92,7 @@ def BuildStamps(nobjects, config, obj_num=0, if nobjects > 1 and 'image' in config and 'nproc' in config['image']: nproc = ParseValue(config['image'], 'nproc', config, int)[0] # Update this in case the config value is -1 - nproc = UpdateNProc(nproc, nobjects, config, logger) + nproc = UpdateNProc(nproc, nobjects, config) else: nproc = 1 @@ -111,7 +111,7 @@ def BuildStamps(nobjects, config, obj_num=0, } jobs.append(kwargs) - def done_func(logger, proc, k, result, t): + def done_func(proc, k, result, t): if result[0] is not None: # Note: numpy shape is y,x image = result[0] @@ -121,7 +121,7 @@ def done_func(logger, proc, k, result, t): obj_num = jobs[k]['obj_num'] logger.info(s0 + 'Stamp %d: size = %d x %d, time = %f sec', obj_num, xs, ys, t) - def except_func(logger, proc, k, e, tr): + def except_func(proc, k, e, tr): if proc is None: s0 = '' else: s0 = '%s: '%proc obj_num = jobs[k]['obj_num'] @@ -131,10 +131,10 @@ def except_func(logger, proc, k, e, tr): # Convert to the tasks structure we need for MultiProcess. # Each task is a list of (job, k) tuples. - tasks = MakeStampTasks(config, jobs, logger) + tasks = MakeStampTasks(config, jobs) results = MultiProcess(nproc, config, BuildStamp, tasks, 'stamp', - logger=logger, timeout=timeout, + timeout=timeout, done_func=done_func, except_func=except_func) images, current_vars = zip(*results) @@ -151,7 +151,7 @@ def except_func(logger, proc, k, e, tr): stamp_image_keys = ['offset', 'retry_failures', 'gsparams', 'draw_method', 'dtype', 'n_photons', 'max_extra_noise', 'poisson_flux', 'obj_rng'] -def SetupConfigObjNum(config, obj_num, logger=None): +def SetupConfigObjNum(config, obj_num): """Do the basic setup of the config dict at the stamp (or object) processing level. Includes: @@ -167,9 +167,7 @@ def SetupConfigObjNum(config, obj_num, logger=None): Parameters: config: A configuration dict. obj_num: The current obj_num. - logger: If given, a logger object to log progress. [default: None] """ - logger = LoggerWrapper(logger) config['obj_num'] = obj_num config['index_key'] = 'obj_num' @@ -207,12 +205,12 @@ def SetupConfigObjNum(config, obj_num, logger=None): stamp['draw_method'] = 'auto' # In case this hasn't been done yet. - SetupInput(config, logger) + SetupInput(config) # Mark this as done, so later passes can skip a lot of this. stamp['_done'] = True -def SetupConfigStampSize(config, xsize, ysize, image_pos, world_pos, logger=None): +def SetupConfigStampSize(config, xsize, ysize, image_pos, world_pos): """Do further setup of the config dict at the stamp (or object) processing level reflecting the stamp size and position in either image or world coordinates. @@ -224,12 +222,11 @@ def SetupConfigStampSize(config, xsize, ysize, image_pos, world_pos, logger=None ysize: The size of the stamp in the y-dimension. [may be 0 if unknown] image_pos: The position of the stamp in image coordinates. [may be None] world_pos: The position of the stamp in world coordinates. [may be None] - logger: If given, a logger object to log progress. [default: None] """ stamp = config.get('stamp',{}) stamp_type = stamp.get('type','Basic') builder = valid_stamp_types[stamp_type] - builder.locateStamp(stamp, config, xsize, ysize, image_pos, world_pos, logger) + builder.locateStamp(stamp, config, xsize, ysize, image_pos, world_pos) # Ignore these when parsing the parameters for specific stamp types: @@ -241,7 +238,7 @@ def SetupConfigStampSize(config, xsize, ysize, image_pos, world_pos, logger=None valid_draw_methods = ('auto', 'fft', 'phot', 'real_space', 'no_pixel', 'sb') -def BuildStamp(config, obj_num=0, xsize=0, ysize=0, do_noise=True, logger=None): +def BuildStamp(config, obj_num=0, xsize=0, ysize=0, do_noise=True): """ Build a single stamp image using the given config file @@ -252,15 +249,13 @@ def BuildStamp(config, obj_num=0, xsize=0, ysize=0, do_noise=True, logger=None): ysize: The ysize of the stamp to build (if known). [default: 0] do_noise: Whether to add noise to the image (according to config['noise']). [default: True] - logger: If given, a logger object to log progress. [default: None] Returns: the tuple (image, current_var) """ from .extra import ProcessExtraOutputsForStamp - logger = LoggerWrapper(logger) - SetupConfigObjNum(config, obj_num, logger) + SetupConfigObjNum(config, obj_num) stamp = config['stamp'] stamp_type = stamp['type'] @@ -270,10 +265,10 @@ def BuildStamp(config, obj_num=0, xsize=0, ysize=0, do_noise=True, logger=None): builder = valid_stamp_types[stamp_type] if builder.quickSkip(stamp, config): - ProcessExtraOutputsForStamp(config, True, logger) + ProcessExtraOutputsForStamp(config, True) return None, 0 - builder.setupRNG(stamp, config, logger) + builder.setupRNG(stamp, config) if 'skip_failures' in stamp: skip_failures = ParseValue(stamp, 'skip_failures', config, bool)[0] @@ -308,10 +303,10 @@ def BuildStamp(config, obj_num=0, xsize=0, ysize=0, do_noise=True, logger=None): # Do the necessary initial setup for this stamp type. xsize, ysize, image_pos, world_pos = builder.setup( - stamp, config, xsize, ysize, stamp_ignore, logger) + stamp, config, xsize, ysize, stamp_ignore) # Determine the stamp size and location - builder.locateStamp(stamp, config, xsize, ysize, image_pos, world_pos, logger) + builder.locateStamp(stamp, config, xsize, ysize, image_pos, world_pos) # Get the global gsparams kwargs. Individual objects can add to this. gsparams = {} @@ -327,35 +322,35 @@ def BuildStamp(config, obj_num=0, xsize=0, ysize=0, do_noise=True, logger=None): # Skip is also different from prof = None. # If prof is None, then the user indicated that no object should be # drawn on this stamp, but that a noise image is still desired. - if builder.getSkip(stamp, config, logger): + if builder.getSkip(stamp, config): raise SkipThisObject('') # Build the object to draw - psf = builder.buildPSF(stamp, config, gsparams, logger) - prof = builder.buildProfile(stamp, config, psf, gsparams, logger) + psf = builder.buildPSF(stamp, config, gsparams) + prof = builder.buildProfile(stamp, config, psf, gsparams) # Make an empty image - im = builder.makeStamp(stamp, config, xsize, ysize, logger) + im = builder.makeStamp(stamp, config, xsize, ysize) # Determine which draw method to use - method = builder.getDrawMethod(stamp, config, logger) + method = builder.getDrawMethod(stamp, config) # Determine the net offset (starting from config['stamp_offset'] typically. - offset = builder.getOffset(stamp, config, logger) + offset = builder.getOffset(stamp, config) # At this point, we may want to update whether the object should be skipped # based on other information about the profile and location. - if builder.updateSkip(prof, im, method, offset, stamp, config, logger): + if builder.updateSkip(prof, im, method, offset, stamp, config): raise SkipThisObject('') # Draw the object on the postage stamp - im = builder.draw(prof, im, method, offset, stamp, config, logger) + im = builder.draw(prof, im, method, offset, stamp, config) # Store the final version of the current profile for reference. config['current_prof'] = prof # Update the drawn image according to the SNR if desired. - scale_factor = builder.getSNRScale(im, stamp, config, logger) - im, prof = builder.applySNRScale(im, prof, scale_factor, method, logger) + scale_factor = builder.getSNRScale(im, stamp, config) + im, prof = builder.applySNRScale(im, prof, scale_factor, method) # Set the origin appropriately builder.updateOrigin(stamp, config, im) @@ -366,28 +361,28 @@ def BuildStamp(config, obj_num=0, xsize=0, ysize=0, do_noise=True, logger=None): config['do_noise_in_stamps'] = do_noise # Check if this object should be rejected. - reject = builder.reject(stamp, config, prof, psf, im, logger) + reject = builder.reject(stamp, config, prof, psf, im) if reject: if itry < ntries: logger.warning('Object %d: Rejecting this object and rebuilding', obj_num) - builder.reset(config, logger) + builder.reset(config) continue else: raise GalSimConfigError( "Rejected an object %d times. If this is expected, " "you should specify a larger stamp.retry_failures."%(ntries)) - ProcessExtraOutputsForStamp(config, False, logger) + ProcessExtraOutputsForStamp(config, False) # We always need to do the whiten step here in the stamp processing - current_var = builder.whiten(prof, im, stamp, config, logger) + current_var = builder.whiten(prof, im, stamp, config) if current_var != 0.: logger.debug('obj %d: whitening noise brought current var to %f', config.get('obj_num',0),current_var) # Sometimes, depending on the image type, we go on to do the rest of the noise as well. if do_noise: - im, current_var = builder.addNoise(stamp,config,im,current_var,logger) + im, current_var = builder.addNoise(stamp,config,im,current_var) except SkipThisObject as e: if e.msg != '': @@ -395,8 +390,8 @@ def BuildStamp(config, obj_num=0, xsize=0, ysize=0, do_noise=True, logger=None): logger.info('Skipping object %d %s', obj_num, e.msg) # If xsize, ysize != 0, then this makes a blank stamp for this object. # Otherwise, it's just None here. - im = builder.makeStamp(stamp, config, xsize, ysize, logger) - ProcessExtraOutputsForStamp(config, True, logger) + im = builder.makeStamp(stamp, config, xsize, ysize) + ProcessExtraOutputsForStamp(config, True) return im, 0. except Exception as e: @@ -406,8 +401,8 @@ def BuildStamp(config, obj_num=0, xsize=0, ysize=0, do_noise=True, logger=None): logger.debug('obj %d: Traceback = %s',obj_num,tr) logger.info('Skipping this object') # Now same as SkipThisObject case above. - im = builder.makeStamp(stamp, config, xsize, ysize, logger) - ProcessExtraOutputsForStamp(config, True, logger) + im = builder.makeStamp(stamp, config, xsize, ysize) + ProcessExtraOutputsForStamp(config, True) return im, 0. elif itry >= ntries: # Then this was the last try. Just re-raise the exception. @@ -424,7 +419,7 @@ def BuildStamp(config, obj_num=0, xsize=0, ysize=0, do_noise=True, logger=None): logger.debug('obj %d: Traceback = %s',obj_num,tr) # Need to remove the "current"s from the config dict. Otherwise, # the value generators will do a quick return with the cached value. - builder.reset(config, logger) + builder.reset(config) continue else: @@ -432,7 +427,7 @@ def BuildStamp(config, obj_num=0, xsize=0, ysize=0, do_noise=True, logger=None): return im, current_var -def MakeStampTasks(config, jobs, logger): +def MakeStampTasks(config, jobs): """Turn a list of jobs into a list of tasks. See the doc string for galsim.config.MultiProcess for the meaning of this distinction. @@ -448,17 +443,16 @@ def MakeStampTasks(config, jobs, logger): config: The configuration dict jobs: A list of jobs to split up into tasks. Each job in the list is a dict of parameters that includes 'obj_num'. - logger: A logger object to log progress. Returns: a list of tasks """ stamp = config.get('stamp', {}) stamp_type = stamp.get('type', 'Basic') - return valid_stamp_types[stamp_type].makeTasks(stamp, config, jobs, logger) + return valid_stamp_types[stamp_type].makeTasks(stamp, config, jobs) -def DrawBasic(prof, image, method, offset, config, base, logger, **kwargs): +def DrawBasic(prof, image, method, offset, config, base, **kwargs): """The basic implementation of the draw command This function is provided as a free function, rather than just the base class implementation @@ -476,13 +470,11 @@ def DrawBasic(prof, image, method, offset, config, base, logger, **kwargs): offset: The offset to apply when drawing. config: The configuration dict for the stamp field. base: The base configuration dict. - logger: A logger object to log progress. **kwargs: Any additional kwargs are passed along to the drawImage function. Returns: the resulting image """ - logger = LoggerWrapper(logger) # Setup the kwargs to pass to drawImage # (Start with any additional kwargs given as extra kwargs to DrawBasic and add to it.) kwargs['image'] = image @@ -496,7 +488,7 @@ def DrawBasic(prof, image, method, offset, config, base, logger, **kwargs): # The cadence specified in the noise field is what to use for photon shooting. noise = base.get('image',{}).get('noise',{}) noise = noise if isinstance(noise, dict) else {} - rng = GetRNG(noise, base, logger, "method='phot'") + rng = GetRNG(noise, base, "method='phot'") kwargs['rng'] = rng # Check validity of extra phot options: @@ -538,7 +530,7 @@ def DrawBasic(prof, image, method, offset, config, base, logger, **kwargs): raise GalSimConfigError("Drawing chromatic object requires specifying bandpass") if 'photon_ops' in config: - kwargs['photon_ops'] = BuildPhotonOps(config, 'photon_ops', base, logger) + kwargs['photon_ops'] = BuildPhotonOps(config, 'photon_ops', base) if sensor is not None: sensor.updateRNG(rng) @@ -562,7 +554,7 @@ def DrawBasic(prof, image, method, offset, config, base, logger, **kwargs): logger.debug('obj %d: added_flux = %s',base.get('obj_num',0), image.added_flux) return image -def ParseWorldPos(config, param_name, base, logger): +def ParseWorldPos(config, param_name, base): """A helper function to parse the 'world_pos' value. The world_pos can be specified either as a regular RA, Dec (which in GalSim is known as a @@ -629,7 +621,7 @@ class StampBuilder: It also includes the implementation of the default stamp type: Basic. """ - def setup(self, config, base, xsize, ysize, ignore, logger): + def setup(self, config, base, xsize, ysize, ignore): """ Do the initialization and setup for building a postage stamp. @@ -647,7 +639,6 @@ def setup(self, config, base, xsize, ysize, ignore, logger): ysize: The ysize of the image to build (if known). ignore: A list of parameters that are allowed to be in config that we can ignore here. i.e. it won't be an error if these parameters are present. - logger: A logger object to log progress. Returns: xsize, ysize, image_pos, world_pos @@ -686,9 +677,9 @@ def setup(self, config, base, xsize, ysize, ignore, logger): image_pos = None if 'world_pos' in config: - world_pos = ParseWorldPos(config, 'world_pos', base, logger) + world_pos = ParseWorldPos(config, 'world_pos', base) elif 'world_pos' in image: - world_pos = ParseWorldPos(image, 'world_pos', base, logger) + world_pos = ParseWorldPos(image, 'world_pos', base) else: world_pos = None @@ -707,7 +698,7 @@ def quickSkip(self, config, base): """ return ('quick_skip' in config and ParseValue(config, 'quick_skip', base, bool)[0]) - def setupRNG(self, config, base, logger): + def setupRNG(self, config, base): """Setup the RNG for this object. @param config The configuration dict for the stamp field. @@ -721,10 +712,10 @@ def setupRNG(self, config, base, logger): return # Add 1 to the seed here so the first object has a different rng than the file or image. - seed = SetupConfigRNG(base, seed_offset=1, logger=logger) + seed = SetupConfigRNG(base, seed_offset=1) logger.debug('obj %d: seed = %d',base.get('obj_num',0),seed) - def locateStamp(self, config, base, xsize, ysize, image_pos, world_pos, logger): + def locateStamp(self, config, base, xsize, ysize, image_pos, world_pos): """Determine where and how large the stamp should be. The base class version does the followin: @@ -747,13 +738,11 @@ def locateStamp(self, config, base, xsize, ysize, image_pos, world_pos, logger): @param ysize The size of the stamp in the y-dimension. [may be 0 if unknown] @param image_pos The position of the stamp in image coordinates. [may be None] @param world_pos The position of the stamp in world coordinates. [may be None] - @param logger A logger object to log progress. """ - logger = LoggerWrapper(logger) # Make sure we have a valid wcs in case image-level processing was skipped. if 'wcs' not in base: - base['wcs'] = BuildWCS(base['image'], 'wcs', config, logger) + base['wcs'] = BuildWCS(base['image'], 'wcs', config) wcs = base['wcs'] if xsize: base['stamp_xsize'] = xsize @@ -824,12 +813,11 @@ def locateStamp(self, config, base, xsize, ysize, image_pos, world_pos, logger): if stamp_center: logger.debug('obj %d: stamp_center = %s',base.get('obj_num',0),stamp_center) - def getSkip(self, config, base, logger): + def getSkip(self, config, base): """Initial check of whether to skip this object based on the stamp.skip field. @param config The configuration dict for the stamp field. @param base The base configuration dict. - @param logger A logger object to log progress. @returns skip """ @@ -841,7 +829,7 @@ def getSkip(self, config, base, logger): skip = False return skip - def buildPSF(self, config, base, gsparams, logger): + def buildPSF(self, config, base, gsparams): """Build the PSF object. For the Basic stamp type, this builds a PSF from the base['psf'] dict, if present, @@ -852,14 +840,13 @@ def buildPSF(self, config, base, gsparams, logger): base: The base configuration dict. gsparams: A dict of kwargs to use for a GSParams. More may be added to this list by the galaxy object. - logger: A logger object to log progress. Returns: the PSF """ - return BuildGSObject(base, 'psf', gsparams=gsparams, logger=logger)[0] + return BuildGSObject(base, 'psf', gsparams=gsparams)[0] - def buildProfile(self, config, base, psf, gsparams, logger): + def buildProfile(self, config, base, psf, gsparams): """Build the surface brightness profile (a GSObject) to be drawn. For the Basic stamp type, this builds a galaxy from the base['gal'] dict and convolves @@ -872,12 +859,11 @@ def buildProfile(self, config, base, psf, gsparams, logger): psf: The PSF, if any. This may be None, in which case, no PSF is convolved. gsparams: A dict of kwargs to use for a GSParams. More may be added to this list by the galaxy object. - logger: A logger object to log progress. Returns: the final profile """ - gal = BuildGSObject(base, 'gal', gsparams=gsparams, logger=logger)[0] + gal = BuildGSObject(base, 'gal', gsparams=gsparams)[0] if psf: if gal: @@ -894,7 +880,7 @@ def buildProfile(self, config, base, psf, gsparams, logger): "At least one of gal or psf must be specified in config. " "If you really don't want any object, use gal type = None.") - def makeStamp(self, config, base, xsize, ysize, logger): + def makeStamp(self, config, base, xsize, ysize): """Make the initial empty postage stamp image, if possible. If we don't know xsize, ysize, return None, in which case the stamp will be created @@ -905,7 +891,6 @@ def makeStamp(self, config, base, xsize, ysize, logger): base: The base configuration dict. xsize: The xsize of the image to build (if known). ysize: The ysize of the image to build (if known). - logger: A logger object to log progress. Returns: the image @@ -927,12 +912,11 @@ def makeStamp(self, config, base, xsize, ysize, logger): else: return None - def getDrawMethod(self, config, base, logger): + def getDrawMethod(self, config, base): """Determine the draw method to use. @param config The configuration dict for the stamp field. @param base The base configuration dict. - @param logger A logger object to log progress. @returns method """ @@ -941,7 +925,7 @@ def getDrawMethod(self, config, base, logger): raise GalSimConfigValueError("Invalid draw_method.", method, valid_draw_methods) return method - def getOffset(self, config, base, logger): + def getOffset(self, config, base): """Determine the offset to use. The base class version adds the stamp_offset, which comes from calculations related to @@ -949,7 +933,6 @@ def getOffset(self, config, base, logger): @param config The configuration dict for the stamp field. @param base The base configuration dict. - @param logger A logger object to log progress. @returns offset """ @@ -960,7 +943,7 @@ def getOffset(self, config, base, logger): base['stamp_offset'], offset) return offset - def updateSkip(self, prof, image, method, offset, config, base, logger): + def updateSkip(self, prof, image, method, offset, config, base): """Before drawing the profile, see whether this object can be trivially skipped. The base method checks if the object is completely off the main image, so the @@ -974,7 +957,6 @@ def updateSkip(self, prof, image, method, offset, config, base, logger): offset: The offset to apply when drawing. config: The configuration dict for the stamp field. base: The base configuration dict. - logger: A logger object to log progress. Returns: whether to skip drawing this object. @@ -1005,7 +987,7 @@ def updateSkip(self, prof, image, method, offset, config, base, logger): return False - def draw(self, prof, image, method, offset, config, base, logger): + def draw(self, prof, image, method, offset, config, base): """Draw the profile on the postage stamp image. Parameters: @@ -1015,7 +997,6 @@ def draw(self, prof, image, method, offset, config, base, logger): offset: The offset to apply when drawing. config: The configuration dict for the stamp field. base: The base configuration dict. - logger: A logger object to log progress. Returns: the resulting image @@ -1023,9 +1004,9 @@ def draw(self, prof, image, method, offset, config, base, logger): if prof is None: return image else: - return DrawBasic(prof,image,method,offset,config,base,logger) + return DrawBasic(prof,image,method,offset,config,base) - def whiten(self, prof, image, config, base, logger): + def whiten(self, prof, image, config, base): """If appropriate, whiten the resulting image according to the requested noise profile and the amount of noise originally present in the profile. @@ -1034,7 +1015,6 @@ def whiten(self, prof, image, config, base, logger): image: The image onto which to draw the profile. config: The configuration dict for the stamp field. base: The base configuration dict. - logger: A logger object to log progress. Returns: the variance of the resulting whitened (or symmetrized) image. @@ -1053,7 +1033,7 @@ def whiten(self, prof, image, config, base, logger): raise GalSimConfigError('Only one of whiten or symmetrize is allowed') if whiten or symmetrize: # In case the galaxy was cached, update the rng - rng = GetRNG(noise, base, logger, "whiten") + rng = GetRNG(noise, base, "whiten") prof.noise.rng.reset(rng) if whiten: current_var = prof.noise.whitenImage(image) @@ -1061,7 +1041,7 @@ def whiten(self, prof, image, config, base, logger): current_var = prof.noise.symmetrizeImage(image, symmetrize) return current_var - def getSNRScale(self, image, config, base, logger): + def getSNRScale(self, image, config, base): """Calculate the factor by which to rescale the image based on a desired S/N level. Note: The default implementation does this for the gal or psf field, so if a custom @@ -1072,7 +1052,6 @@ def getSNRScale(self, image, config, base, logger): image: The current image. config: The configuration dict for the stamp field. base: The base configuration dict. - logger: A logger object to log progress. Returns: scale_factor @@ -1111,7 +1090,7 @@ def getSNRScale(self, image, config, base, logger): scale_factor = sn_target / sn_meas return scale_factor - def applySNRScale(self, image, prof, scale_factor, method, logger): + def applySNRScale(self, image, prof, scale_factor, method): """Apply the scale_factor from getSNRScale to the image and profile. The default implementaion just multiplies each of them, but if prof is not a regular @@ -1122,7 +1101,6 @@ def applySNRScale(self, image, prof, scale_factor, method, logger): prof: The profile that was drawn. scale_factor: The factor by which to scale both image and prof. method: The method used by drawImage. - logger: A logger object to log progress. Returns: image, prof (after being properly scaled) @@ -1149,7 +1127,7 @@ def updateOrigin(self, config, base, image): image.setOrigin(base.get('image_origin',PositionI(1,1))) - def reject(self, config, base, prof, psf, image, logger): + def reject(self, config, base, prof, psf, image): """Check to see if this object should be rejected. Parameters: @@ -1158,7 +1136,6 @@ def reject(self, config, base, prof, psf, image, logger): prof: The profile that was drawn. psf: The psf that was used to build the profile. image: The postage stamp image. No noise is on it yet at this point. - logger: A logger object to log progress. Returns: whether to reject this object @@ -1208,13 +1185,12 @@ def reject(self, config, base, prof, psf, image, logger): return True return False - def reset(self, base, logger): + def reset(self, base): """Reset some aspects of the config dict so the object can be rebuilt after rejecting the current object. Parameters: base: The base configuration dict. - logger: A logger object to log progress. """ # Clear current values out of psf, gal, and stamp if they are not safe to reuse. # This means they are either marked as safe or indexed by something other than obj_num. @@ -1222,7 +1198,7 @@ def reset(self, base, logger): if field in base: RemoveCurrent(base[field], keep_safe=True, index_key='obj_num') - def addNoise(self, config, base, image, current_var, logger): + def addNoise(self, config, base, image, current_var): """ Add the sky level and the noise to the stamp. @@ -1234,17 +1210,16 @@ def addNoise(self, config, base, image, current_var, logger): base: The base configuration dict. image: The current image. current_var: The current noise variance present in the image already. - logger: A logger object to log progress. Returns: the new values of image, current_var """ base['current_noise_image'] = base['current_stamp'] AddSky(base,image) - current_var = AddNoise(base,image,current_var,logger) + current_var = AddNoise(base,image,current_var) return image, current_var - def makeTasks(self, config, base, jobs, logger): + def makeTasks(self, config, base, jobs): """Turn a list of jobs into a list of tasks. For the Basic stamp type, there is just one job per task, so the tasks list is just: @@ -1256,7 +1231,6 @@ def makeTasks(self, config, base, jobs, logger): base: The base configuration dict. jobs: A list of jobs to split up into tasks. Each job in the list is a dict of parameters that includes 'obj_num'. - logger: A logger object to log progress. Returns: a list of tasks diff --git a/galsim/config/stamp_ring.py b/galsim/config/stamp_ring.py index d0a1e6794a..249c1d64d0 100644 --- a/galsim/config/stamp_ring.py +++ b/galsim/config/stamp_ring.py @@ -35,7 +35,7 @@ class RingBuilder(StampBuilder): It specializes the setup, buildProfile, reject, and makeTasks functions. """ - def setup(self, config, base, xsize, ysize, ignore, logger): + def setup(self, config, base, xsize, ysize, ignore): """Do the initialization and setup for the Ring type. Most of the work is done by SetupBasic, but we do need to check that the required parameters @@ -48,7 +48,6 @@ def setup(self, config, base, xsize, ysize, ignore, logger): ysize: The ysize of the image to build (if known). ignore: A list of parameters that are allowed to be in config that we can ignore here. i.e. it won't be an error if these parameters are present. - logger: If given, a logger object to log progress. Returns: xsize, ysize, image_pos, world_pos @@ -75,9 +74,9 @@ def setup(self, config, base, xsize, ysize, ignore, logger): # Now go on and do the base class setup. ignore = ignore + list(req) + list(opt) - return super(RingBuilder, self).setup(config, base, xsize, ysize, ignore, logger) + return super(RingBuilder, self).setup(config, base, xsize, ysize, ignore) - def buildProfile(self, config, base, psf, gsparams, logger): + def buildProfile(self, config, base, psf, gsparams): """ Build the object to be drawn. @@ -91,7 +90,6 @@ def buildProfile(self, config, base, psf, gsparams, logger): psf: The PSF, if any. This may be None, in which case, no PSF is convolved. gsparams: A dict of kwargs to use for a GSParams. More may be added to this list by the galaxy object. - logger: If given, a logger object to log progress. Returns: the final profile @@ -104,7 +102,7 @@ def buildProfile(self, config, base, psf, gsparams, logger): if index % num == 0: # Then we are on the first item in the ring, so make it normally. - gal = BuildGSObject(base, 'gal', gsparams=gsparams, logger=logger)[0] + gal = BuildGSObject(base, 'gal', gsparams=gsparams)[0] if gal is None: raise GalSimConfigError( "The gal field must define a valid galaxy for stamp type=Ring.") @@ -121,14 +119,14 @@ def buildProfile(self, config, base, psf, gsparams, logger): gal = gal.rotate(index*dtheta) # Apply any transformations that are given in the stamp field. - gal = TransformObject(gal, config, base, logger)[0] + gal = TransformObject(gal, config, base)[0] if psf is not None: return Convolve(gal,psf) else: return gal - def reject(self, config, base, prof, psf, image, logger): + def reject(self, config, base, prof, psf, image): """Check to see if this object should be rejected. This is the same as base class reject for the first item in the ring. Later items are not @@ -140,18 +138,17 @@ def reject(self, config, base, prof, psf, image, logger): prof: The profile that was drawn. psf: The psf that was used to build the profile. image: The postage stamp image. No noise is on it yet at this point. - logger: If given, a logger object to log progress. Returns: whether the galaxy was rejected. """ index = ParseValue(config, 'index', base, int)[0] if index == 0: - return super(RingBuilder,self).reject(config, base, prof, psf, image, logger) + return super(RingBuilder,self).reject(config, base, prof, psf, image) else: return False - def makeTasks(self, config, base, jobs, logger): + def makeTasks(self, config, base, jobs): """Turn a list of jobs into a list of tasks. For the Ring stamp type, we group jobs into sets of num. If there are extra jobs that @@ -162,7 +159,6 @@ def makeTasks(self, config, base, jobs, logger): base: The base configuration dict. jobs: A list of jobs to split up into tasks. Each job in the list is a dict of parameters that includes 'obj_num'. - logger: If given, a logger object to log progress. Returns: a list of tasks diff --git a/galsim/config/util.py b/galsim/config/util.py index 0c28dfafeb..cb5babcde1 100644 --- a/galsim/config/util.py +++ b/galsim/config/util.py @@ -17,6 +17,7 @@ # import logging +import logging.handlers import copy import sys import time @@ -33,11 +34,23 @@ from ..random import BaseDeviate from ..errors import GalSimConfigError, GalSimConfigValueError, GalSimError, galsim_warn +logger = logging.getLogger(__name__) + max_queue_size = 32767 # This is where multiprocessing.Queue starts to have trouble. # We make it a settable parameter here really for unit testing. # I don't think there is any reason for end users to want to set this. -def MergeConfig(config1, config2, logger=None): + +def log_listener(log_queue): + root = logging.getLogger(__name__) + handler = logging.StreamHandler() + root.addHandler(handler) + + for record in iter(log_queue.get, 'STOP'): + logger = logging.getLogger(record.name) + logger.handle(record) + +def MergeConfig(config1, config2): """ Merge config2 into config1 such that it has all the information from either config1 or config2 including places where both input dicts have some of a field defined. @@ -47,7 +60,6 @@ def MergeConfig(config1, config2, logger=None): For real conflicts (the same value in both cases), config1's value takes precedence """ - logger = LoggerWrapper(logger) for (key, value) in config2.items(): if not key in config1: # If this key isn't in config1 yet, just add it @@ -55,7 +67,7 @@ def MergeConfig(config1, config2, logger=None): elif isinstance(value,dict) and isinstance(config1[key],dict): # If they both have a key, first check if the values are dicts # If they are, just recurse this process and merge those dicts. - MergeConfig(config1[key],value,logger) + MergeConfig(config1[key],value) else: # Otherwise config1 takes precedence logger.info("Not merging key %s from the base config, since the later " @@ -252,89 +264,7 @@ def __init__(self): super(SafeManager, self).__init__(ctx=get_context('fork')) -def GetLoggerProxy(logger): - """Make a proxy for the given logger that can be passed into multiprocessing Processes - and used safely. - - Parameters: - logger: The logger to make a copy of - - Returns: - a proxy for the given logger - """ - if logger: - class LoggerManager(SafeManager): pass - logger_generator = SimpleGenerator(logger) - LoggerManager.register('logger', callable = logger_generator) - logger_manager = LoggerManager() - with single_threaded(): - logger_manager.start() - logger_proxy = logger_manager.logger() - else: - logger_proxy = None - return logger_proxy - -class LoggerWrapper: - """A wrap around a Logger object that checks whether a debug or info or warn call will - actually produce any output before calling the functions. - - This seems like a gratuitous wrapper, and it is if the object being wrapped is a real - Logger object. However, we use it to wrap proxy objects (returned from GetLoggerProxy) - that would otherwise send the arguments of logger.debug(...) calls through a multiprocessing - pipe before (typically) being ignored. Here, we check whether the call will actually - produce any output before calling the functions. - - Parameters: - logger: The logger object to wrap. - """ - def __init__(self, logger): - if isinstance(logger,LoggerWrapper): - self.logger = logger.logger - self.level = logger.level - else: - self.logger = logger - # When multiprocessing, it is faster to check if the level is enabled locally, rather - # than communicating over the pipe to ask the base logger if it isEnabledFor a given - # level. - # If the logger is None, use more than the top logging level (CRITICAL). - self.level = self.logger.getEffectiveLevel() if self.logger else logging.CRITICAL+1 - - def getEffectiveLevel(self): - return self.level - - def __bool__(self): - return self.logger is not None - __nonzero__ = __bool__ - - def debug(self, *args, **kwargs): - if self.logger and self.isEnabledFor(logging.DEBUG): - self.logger.debug(*args, **kwargs) - - def info(self, *args, **kwargs): - if self.logger and self.isEnabledFor(logging.INFO): - self.logger.info(*args, **kwargs) - - def warning(self, *args, **kwargs): - if self.logger and self.isEnabledFor(logging.WARNING): - self.logger.warning(*args, **kwargs) - - def error(self, *args, **kwargs): - if self.logger and self.isEnabledFor(logging.ERROR): - self.logger.error(*args, **kwargs) - - def critical(self, *args, **kwargs): - if self.logger and self.isEnabledFor(logging.CRITICAL): - self.logger.critical(*args, **kwargs) - - def log(self, level, *args, **kwargs): - if self.logger and self.isEnabledFor(level): - self.logger.log(level, *args, **kwargs) - - def isEnabledFor(self, level): - return level >= self.level - - -def UpdateNProc(nproc, ntot, config, logger=None): +def UpdateNProc(nproc, ntot, config): """Update nproc - If nproc < 0, set nproc to ncpu @@ -345,12 +275,10 @@ def UpdateNProc(nproc, ntot, config, logger=None): ntot: The total number of files/images/stamps to do, so the maximum number of processes that would make sense. config: The configuration dict to copy. - logger: If given, a logger object to log progress. [default: None] Returns: the number of processes to use. """ - logger = LoggerWrapper(logger) # First if nproc < 0, update based on ncpu if nproc <= 0: # Try to figure out a good number of processes to use @@ -476,7 +404,7 @@ def PropagateIndexKeyRNGNum(config, index_key=None, rng_num=None, rng_index_key= PropagateIndexKeyRNGNum(config[key], index_key, rng_num, rng_index_key) -def SetupConfigRNG(config, seed_offset=0, logger=None): +def SetupConfigRNG(config, seed_offset=0): """Set up the RNG in the config dict. - Setup config['image']['random_seed'] if necessary @@ -486,15 +414,12 @@ def SetupConfigRNG(config, seed_offset=0, logger=None): config: The configuration dict. seed_offset: An offset to use relative to what config['image']['random_seed'] gives. [default: 0] - logger: If given, a logger object to log progress. [default: None] Returns: the seed used to initialize the RNG. """ from .process import rng_fields - logger = LoggerWrapper(logger) - # If we are starting a new file, clear out the existing rngs. index_key = config['index_key'] if index_key == 'file_num': @@ -620,7 +545,7 @@ def GetFromConfig(config, key): "Unable to parse extended key %s. Field %s is invalid."%(key,k)) from None return value -def SetInConfig(config, key, value, logger=None): +def SetInConfig(config, key, value): """Set the value of a (possibly extended) key in a config dict. If key is a simple string, then this is equivalent to config[key] = value. @@ -643,10 +568,9 @@ def SetInConfig(config, key, value, logger=None): # Then d was really a list. # This has some potentially counter-intuitive consequences, so let the user # know about them. - if logger: - logger.warning("Warning: Removing item %d from %s. "%(k,key[:key.rfind('.')]) + - "Any further adjustments to this list must use the new "+ - "list index values, not the original indices.") + logger.warning("Warning: Removing item %d from %s. "%(k,key[:key.rfind('.')]) + + "Any further adjustments to this list must use the new "+ + "list index values, not the original indices.") del d[k] else: try: @@ -656,7 +580,7 @@ def SetInConfig(config, key, value, logger=None): "Unable to parse extended key %s. Field %s is invalid."%(key,k)) from None -def UpdateConfig(config, new_params, logger=None): +def UpdateConfig(config, new_params): """Update the given config dict with additional parameters/values. Parameters: @@ -666,10 +590,10 @@ def UpdateConfig(config, new_params, logger=None): parsed to update config['gal']['first']['dilate']. """ for key, value in new_params.items(): - SetInConfig(config, key, value, logger) + SetInConfig(config, key, value) -def MultiProcess(nproc, config, job_func, tasks, item, logger=None, timeout=900, +def MultiProcess(nproc, config, job_func, tasks, item, timeout=900, done_func=None, except_func=None, except_abort=True): """A helper function for performing a task using multiprocessing. @@ -695,17 +619,16 @@ def MultiProcess(nproc, config, job_func, tasks, item, logger=None, timeout=900, tasks: A list of tasks to run. Each task is a list of jobs, each of which is a tuple (kwargs, k). item: A string indicating what is being worked on. - logger: If given, a logger object to log progress. [default: None] timeout: How many seconds to allow for each task before timing out. [default: 900] done_func: A function to run upon completion of each job. It will be called as:: - done_func(logger, proc, k, result, t) + done_func(proc, k, result, t) where proc is the process name, k is the index of the job, result is the return value of that job, and t is the time taken. [default: None] except_func: A function to run if an exception is encountered. It will be called as:: - except_func(logger, proc, k, ex, tr) + except_func(proc, k, ex, tr) where proc is the process name, k is the index of the job that failed, ex is the exception caught, and tr is the traceback. [default: None] @@ -724,20 +647,15 @@ def MultiProcess(nproc, config, job_func, tasks, item, logger=None, timeout=900, # The *tasks* can be made up of more than one *job*. Each job involves calling job_func # with the kwargs from the list of jobs. # Each job also carries with it its index in the original list of all jobs. - def worker(task_queue, results_queue, config, logger, initializers, initargs): + def worker(task_queue, results_queue, config, log_queue, initializers, initargs): proc = current_process().name for init, args in zip(initializers, initargs): init(*args) - # The logger object passed in here is a proxy object. This means that all the arguments - # to any logging commands are passed through the pipe to the real Logger object on the - # other end of the pipe. This tends to produce a lot of unnecessary communication, since - # most of those commands don't actually produce any output (e.g. logger.debug(..) commands - # when the logging level is not DEBUG). So it is helpful to wrap this object in a - # LoggerWrapper that checks whether it is worth sending the arguments back to the original - # Logger before calling the functions. - logger = LoggerWrapper(logger) + logger = logging.getLogger(__name__) + handler = logging.handlers.QueueHandler(log_queue) + logger.addHandler(handler) if 'profile' in config and config['profile']: pr = cProfile.Profile() @@ -752,7 +670,6 @@ def worker(task_queue, results_queue, config, logger, initializers, initargs): for kwargs, k in task: t1 = time.time() kwargs['config'] = config - kwargs['logger'] = logger result = job_func(**kwargs) t2 = time.time() results_queue.put( (result, k, t2-t1, proc) ) @@ -789,14 +706,15 @@ def worker(task_queue, results_queue, config, logger, initializers, initargs): if 'profile' in config and config['profile']: logger.info("Starting separate profiling for each of the %d processes.",nproc) - # The logger is not picklable, so we need to make a proxy for it so all the - # processes can emit logging information safely. - logger_proxy = GetLoggerProxy(logger) - # We need to set the number of OpenMP threads to 1 during multiprocessing, otherwise # it may spawn e.g. 64 threads in each of 64 processes, which tends to be bad. with single_threaded(): + # Initialize the log queue and start the listener for logging from multiple processes. + log_queue = Queue() + listener = ctx.Process(target=log_listener, args=(log_queue,)) + listener.start() + # Send the tasks to the task_queue. ntasks = len(tasks) if ntasks > max_queue_size: @@ -828,7 +746,7 @@ def worker(task_queue, results_queue, config, logger, initializers, initargs): # processes, so for the sake of the logging output, we name the processes explicitly. initializers = worker_init_fns initargs = [initargs_fn() for initargs_fn in worker_initargs_fns] - p = Process(target=worker, args=(task_queue, results_queue, config, logger_proxy, + p = Process(target=worker, args=(task_queue, results_queue, config, log_queue, initializers, initargs), name='Process-%d'%(j+1)) p.start() @@ -856,16 +774,17 @@ def worker(task_queue, results_queue, config, logger, initializers, initargs): # t is really the traceback # k is the index for the job that failed if except_func is not None: # pragma: no branch - except_func(logger, proc, k, res, t) + except_func(proc, k, res, t) if except_abort: for j in range(nproc): p_list[j].terminate() + listener.terminate() raise_error = res break else: # The normal case if done_func is not None: # pragma: no branch - done_func(logger, proc, k, res, t) + done_func(proc, k, res, t) results[k] = res except BaseException as e: @@ -886,6 +805,13 @@ def worker(task_queue, results_queue, config, logger, initializers, initargs): # And terminate any jobs that might still be running. for j in range(nproc): p_list[j].terminate() + # Same for logging + while not log_queue.empty(): + try: + log_queue.get_nowait() + except Exception: # pragma: no cover + pass + listener.terminate() raise_error = e finally: @@ -906,6 +832,14 @@ def worker(task_queue, results_queue, config, logger, initializers, initargs): p_list[j].join() task_queue.close() + # Also make sure to stop the log listener with `STOP` and do cleanup. + try: + log_queue.put_nowait('STOP') + except Exception: # pragma: no cover + pass + listener.join() + log_queue.close() + del config['current_nproc'] if raise_error is not None: @@ -918,16 +852,15 @@ def worker(task_queue, results_queue, config, logger, initializers, initargs): try: t1 = time.time() kwargs['config'] = config - kwargs['logger'] = logger result = job_func(**kwargs) t2 = time.time() if done_func is not None: # pragma: no branch - done_func(logger, None, k, result, t2-t1) + done_func(None, k, result, t2-t1) results[k] = result except Exception as e: tr = traceback.format_exc() if except_func is not None: # pragma: no branch - except_func(logger, None, k, e, tr) + except_func(None, k, e, tr) if except_abort: raise @@ -968,22 +901,18 @@ def GetIndex(config, base, is_sequence=False): return index, index_key -def GetRNG(config, base, logger=None, tag=''): +def GetRNG(config, base, tag=''): """Get the appropriate current rng according to whatever the current index_key is. - If a logger is provided, then it will emit a warning if there is no current rng setup. - Parameters: config: The configuration dict for the current item being worked on. base: The base configuration dict. - logger: If given, a logger object to log progress. [default: None] tag: If given, an appropriate name for the current item to use in the warning message. [default: ''] Returns: either the appropriate rng for the current index_key or None """ - logger = LoggerWrapper(logger) if 'rng_index_key' in config: index_key = config['rng_index_key'] if index_key not in valid_index_keys: @@ -1009,7 +938,7 @@ def GetRNG(config, base, logger=None, tag=''): logger.debug("No index_key_rng. Use base[rng]") rng = base.get('rng',None) - if rng is None and logger: + if rng is None: # Only report the warning the first time. rng_tag = tag + '_reported_no_rng' if rng_tag not in base: @@ -1053,7 +982,7 @@ def SetDefaultExt(config, default_ext): _sleep_mult = 1 # 1 second normally, but make it a variable, so I can change it when unit testing. -def RetryIO(func, args, ntries, file_name, logger): +def RetryIO(func, args, ntries, file_name): """A helper function to retry I/O commands """ itry = 0 diff --git a/galsim/config/wcs.py b/galsim/config/wcs.py index 11d7075dd2..9a29903061 100644 --- a/galsim/config/wcs.py +++ b/galsim/config/wcs.py @@ -18,7 +18,7 @@ import logging -from .util import LoggerWrapper, GetIndex, GetRNG, get_cls_params +from .util import GetIndex, GetRNG, get_cls_params from .value import ParseValue, GetAllParams, CheckAllParams, SetDefaultIndex from .input import RegisterInputConnectedType from ..errors import GalSimConfigError, GalSimConfigValueError @@ -30,6 +30,8 @@ from ..celestial import CelestialCoord from ..utilities import basestring +logger = logging.getLogger(__name__) + # This file handles the construction of wcs types in config['image']['wcs']. # This module-level dict will store all the registered wcs types. @@ -39,19 +41,17 @@ valid_wcs_types = {} -def BuildWCS(config, key, base, logger=None): +def BuildWCS(config, key, base): """Read the wcs parameters from config[key] and return a constructed wcs object. Parameters: config: A dict with the configuration information. (usually base['image']) key: The key name in config indicating which object to build. base: The base dict of the configuration. - logger: Optionally, provide a logger for logging debug statements. [default: None] Returns: a BaseWCS instance """ - logger = LoggerWrapper(logger) logger.debug('image %d: Start BuildWCS key = %s',base.get('image_num',0),key) try: @@ -100,7 +100,7 @@ def BuildWCS(config, key, base, logger=None): list(valid_wcs_types.keys())) logger.debug('image %d: Building wcs type %s', base.get('image_num',0), wcs_type) builder = valid_wcs_types[wcs_type] - wcs = builder.buildWCS(param, base, logger) + wcs = builder.buildWCS(param, base) logger.debug('image %d: wcs = %s', base.get('image_num',0), wcs) param['current'] = wcs, False, None, index, index_key @@ -114,7 +114,7 @@ class WCSBuilder: The base class defines the call signatures of the methods that any derived class should follow. """ - def buildWCS(self, config, base, logger): + def buildWCS(self, config, base): """Build the WCS based on the specifications in the config dict. Note: Sub-classes must override this function with a real implementation. @@ -122,7 +122,6 @@ def buildWCS(self, config, base, logger): Parameters: config: The configuration dict for the wcs type. base: The base configuration dict. - logger: If provided, a logger for logging debug statements. Returns: the constructed WCS object. @@ -181,13 +180,12 @@ def getKwargs(self, build_func, config, base): kwargs['rng'] = GetRNG(config, base) return kwargs - def buildWCS(self, config, base, logger): + def buildWCS(self, config, base): """Build the WCS based on the specifications in the config dict. Parameters: config: The configuration dict for the wcs type. base: The base configuration dict. - logger: If provided, a logger for logging debug statements. Returns: the constructed WCS object. @@ -204,14 +202,13 @@ def __init__(self, init_func, origin_init_func): self.init_func = init_func self.origin_init_func = origin_init_func - def buildWCS(self, config, base, logger): + def buildWCS(self, config, base): """Build the WCS based on the specifications in the config dict, using the appropriate type depending on whether an origin is provided. Parameters: config: The configuration dict for the wcs type. base: The base configuration dict. - logger: If provided, a logger for logging debug statements. Returns: the constructed WCS object. @@ -232,13 +229,12 @@ class TanWCSBuilder(WCSBuilder): """ def __init__(self): pass - def buildWCS(self, config, base, logger): + def buildWCS(self, config, base): """Build the TanWCS based on the specifications in the config dict. Parameters: config: The configuration dict for the wcs type. base: The base configuration dict. - logger: If provided, a logger for logging debug statements. Returns: the constructed WCS object. @@ -266,7 +262,7 @@ def buildWCS(self, config, base, logger): class ListWCSBuilder(WCSBuilder): """Select a wcs from a list """ - def buildWCS(self, config, base, logger): + def buildWCS(self, config, base): req = { 'items' : list } opt = { 'index' : int } # Only Check, not Get. We need to handle items a bit differently, since it's a list. diff --git a/galsim/deprecated/randwalk.py b/galsim/deprecated/randwalk.py index ac81f4c8bf..73aca5b304 100644 --- a/galsim/deprecated/randwalk.py +++ b/galsim/deprecated/randwalk.py @@ -26,10 +26,10 @@ def RandomWalk(*args, **kwargs): galsim.RandomWalk = RandomWalk -def BuildRandomWalk(config, base, ignore, gsparams, logger): +def BuildRandomWalk(config, base, ignore, gsparams): from . import depr depr('RandomWalk', 2.2, 'RandomKnots') return galsim.config.gsobject._BuildSimple(galsim.RandomKnots, - config, base, ignore, gsparams, logger) + config, base, ignore, gsparams) RegisterObjectType('RandomWalk', BuildRandomWalk) diff --git a/galsim/des/des_meds.py b/galsim/des/des_meds.py index 5b80743fe0..0c1af3cfe0 100644 --- a/galsim/des/des_meds.py +++ b/galsim/des/des_meds.py @@ -16,6 +16,7 @@ # and/or other materials provided with the distribution. # +import logging import numpy as np import sys import time @@ -30,6 +31,8 @@ from ..config import RegisterOutputType, RegisterExtraOutput from .._pyfits import pyfits +logger = logging.getLogger(__name__) + # these image stamp sizes are available in MEDS format BOX_SIZES = [32,48,64,96,128,192,256] # while creating the meds file, all the data is stored in memory, and then written to disc once all @@ -445,7 +448,7 @@ class MEDSBuilder(OutputBuilder): It requires the use of ``galsim.des`` in the config files ``modules`` section. """ - def buildImages(self, config, base, file_num, image_num, obj_num, ignore, logger): + def buildImages(self, config, base, file_num, image_num, obj_num, ignore): """ Build a meds file as specified in config. @@ -457,7 +460,6 @@ def buildImages(self, config, base, file_num, image_num, obj_num, ignore, logger obj_num: The current obj_num. ignore: A list of parameters that are allowed to be in config that we can ignore here. - logger: If given, a logger object to log progress. Returns: A list of MultiExposureObjects. @@ -479,22 +481,22 @@ def buildImages(self, config, base, file_num, image_num, obj_num, ignore, logger nstamps_per_object = params['nstamps_per_object'] ntot = nobjects * nstamps_per_object - main_images = BuildImages(ntot, base, image_num=image_num, obj_num=obj_num, logger=logger) + main_images = BuildImages(ntot, base, image_num=image_num, obj_num=obj_num) # grab list of offsets for cutout_row/cutout_col. - offsets = GetFinalExtraOutput('meds_get_offset', base, logger) + offsets = GetFinalExtraOutput('meds_get_offset', base) # cutout_row/col is the stamp center (**with the center of the first pixel # being (0,0)**) + offset centers = [0.5*im.array.shape[0]-0.5 for im in main_images] cutout_rows = [c+offset.y for c,offset in zip(centers,offsets)] cutout_cols = [c+offset.x for c,offset in zip(centers,offsets)] - weight_images = GetFinalExtraOutput('weight', base, logger) + weight_images = GetFinalExtraOutput('weight', base) if 'badpix' in config: - badpix_images = GetFinalExtraOutput('badpix', base, logger) + badpix_images = GetFinalExtraOutput('badpix', base) else: badpix_images = None - psf_images = GetFinalExtraOutput('psf', base, logger) + psf_images = GetFinalExtraOutput('psf', base) obj_list = [] for i in range(nobjects): @@ -515,7 +517,7 @@ def buildImages(self, config, base, file_num, image_num, obj_num, ignore, logger return obj_list - def writeFile(self, data, file_name, config, base, logger): + def writeFile(self, data, file_name, config, base): """Write the data to a file. In this case a MEDS file. Parameters: @@ -523,11 +525,10 @@ def writeFile(self, data, file_name, config, base, logger): file_name: The file_name to write to. config: The configuration dict for the output field. base: The base configuration dict. - logger: If given, a logger object to log progress. """ WriteMEDS(data, file_name) - def getNImages(self, config, base, file_num, logger=None): + def getNImages(self, config, base, file_num): # This gets called before starting work on the file, so we can use this opportunity # to make sure that weight and psf processing are turned on. # We just add these as empty dicts, so there is no hdu or file_name parameter, which @@ -558,7 +559,7 @@ class OffsetBuilder(ExtraOutputBuilder): """ # The function to call at the end of building each stamp - def processStamp(self, obj_num, config, base, logger): + def processStamp(self, obj_num, config, base): offset = base['stamp_offset'] stamp = base['stamp'] if 'offset' in stamp: @@ -566,7 +567,7 @@ def processStamp(self, obj_num, config, base, logger): self.scratch[obj_num] = offset # The function to call at the end of building each file to finalize the truth catalog - def finalize(self, config, base, main_data, logger): + def finalize(self, config, base, main_data): offsets_list = [] obj_nums = sorted(self.scratch.keys()) for obj_num in obj_nums: diff --git a/galsim/des/des_psfex.py b/galsim/des/des_psfex.py index 6bb54c945b..454c338204 100644 --- a/galsim/des/des_psfex.py +++ b/galsim/des/des_psfex.py @@ -308,7 +308,7 @@ def _define_xto(self, x): class PSFExLoader(InputLoader): # Allow the user to not provide the image file. In this case, we'll grab the wcs from the # config dict. - def getKwargs(self, config, base, logger): + def getKwargs(self, config, base): req = { 'file_name' : str } opt = { 'dir' : str, 'image_file_name' : str } kwargs, safe = GetAllParams(config, base, req=req, opt=opt) @@ -330,7 +330,7 @@ def getKwargs(self, config, base, logger): # config is a dictionary that includes 'type' plus other items you might want to allow or require. # base is the top level config dictionary where some global variables are stored. # ignore is a list of key words that might be in the config dictionary that you should ignore. -def BuildDES_PSFEx(config, base, ignore, gsparams, logger): +def BuildDES_PSFEx(config, base, ignore, gsparams): """Build a GSObject representing the PSFex model at the correct location in the image in a config-processing context. diff --git a/galsim/des/des_shapelet.py b/galsim/des/des_shapelet.py index 0cff23ee64..9e756f1d41 100644 --- a/galsim/des/des_shapelet.py +++ b/galsim/des/des_shapelet.py @@ -181,7 +181,7 @@ def _definePxy(self, x, min, max): # config is a dictionary that includes 'type' plus other items you might want to allow or require. # base is the top level config dictionary where some global variables are stored. # ignore is a list of key words that might be in the config dictionary that you should ignore. -def BuildDES_Shapelet(config, base, ignore, gsparams, logger): +def BuildDES_Shapelet(config, base, ignore, gsparams): """Build a GSObject representing the shapelet model at the correct location in the image in a config-processing context. diff --git a/galsim/download_cosmos.py b/galsim/download_cosmos.py index cc9cff7876..0c7f8f8791 100644 --- a/galsim/download_cosmos.py +++ b/galsim/download_cosmos.py @@ -27,7 +27,9 @@ from ._version import __version__ as version from .meta_data import share_dir from .utilities import ensure_dir -from .main import make_logger +from .main import setup_logger + +logger = logging.getLogger(__name__) script_name = 'galsim_download_cosmos' @@ -118,7 +120,7 @@ def query_yes_no(question, default="yes"): "(or 'y' or 'n').\n") return valid[choice] -def get_names(args, logger): +def get_names(args): if args.dir is not None: target_dir = os.path.expanduser(args.dir) do_link = not args.nolink @@ -137,7 +139,7 @@ def get_names(args, logger): return url, target, target_dir, link_dir, unpack_dir, do_link -def get_meta(url, args, logger): +def get_meta(url, args): logger.info('') # See how large the file to be downloaded is. @@ -150,7 +152,7 @@ def get_meta(url, args, logger): return meta -def check_existing(target, unpack_dir, meta, args, logger): +def check_existing(target, unpack_dir, meta, args): # Make sure the directory we want to put this file exists. ensure_dir(target) @@ -265,7 +267,7 @@ def check_existing(target, unpack_dir, meta, args, logger): do_download = False return do_download -def download(do_download, url, target, meta, args, logger): +def download(do_download, url, target, meta, args): if not do_download: return logger.info("") u = urlopen(url) @@ -332,7 +334,7 @@ def check_unpack(do_download, unpack_dir, target, args): do_unpack=True return do_unpack -def unpack(do_unpack, target, target_dir, unpack_dir, meta, args, logger): +def unpack(do_unpack, target, target_dir, unpack_dir, meta, args): if not do_unpack: return logger.info("Unpacking the tarball...") with tarfile.open(target) as tar: @@ -387,12 +389,12 @@ def check_remove(do_unpack, target, args): do_remove = True return do_remove -def remove_tarball(do_remove, target, logger): +def remove_tarball(do_remove, target): if do_remove: logger.info("Removing the tarball to save space") os.remove(target) -def make_link(do_link, unpack_dir, link_dir, args, logger): +def make_link(do_link, unpack_dir, link_dir, args): if not do_link: return logger.debug("Linking to %s from %s", unpack_dir, link_dir) if os.path.lexists(link_dir): @@ -431,8 +433,8 @@ def make_link(do_link, unpack_dir, link_dir, args, logger): logger.info("Made link to %s from %s", unpack_dir, link_dir) -def download_cosmos(args, logger): - """The main script given the ArgParsed args and a logger +def download_cosmos(args): + """The main script given the ArgParsed args """ # Give diagnostic about GalSim version logger.debug("GalSim version: %s",version) @@ -447,33 +449,33 @@ def download_cosmos(args, logger): # link_dir is the directory where this would normally have been unpacked. # unpack_dir is the directory that the tarball will unpack into. - url, target, target_dir, link_dir, unpack_dir, do_link = get_names(args, logger) + url, target, target_dir, link_dir, unpack_dir, do_link = get_names(args) - meta = get_meta(url, args, logger) + meta = get_meta(url, args) # Check if the file already exists and if it is the right size - do_download = check_existing(target, unpack_dir, meta, args, logger) + do_download = check_existing(target, unpack_dir, meta, args) # Download the tarball - download(do_download, url, target, meta, args, logger) + download(do_download, url, target, meta, args) # Unpack the tarball do_unpack = check_unpack(do_download, unpack_dir, target, args) - unpack(do_unpack, target, target_dir, unpack_dir, meta, args, logger) + unpack(do_unpack, target, target_dir, unpack_dir, meta, args) # Remove the tarball do_remove = check_remove(do_unpack, target, args) - remove_tarball(do_remove, target, logger) + remove_tarball(do_remove, target) # If we are downloading to an alternate directory, we (usually) link to it from share/galsim - make_link(do_link, unpack_dir, link_dir, args, logger) + make_link(do_link, unpack_dir, link_dir, args) def main(command_args): """The whole process given command-line parameters in their native (non-ArgParse) form. """ args = parse_args(command_args) - logger = make_logger(args) - download_cosmos(args, logger) + setup_logger(args) + download_cosmos(args) def run_main(): """Kick off the process grabbing the command-line parameters from sys.argv diff --git a/galsim/main.py b/galsim/main.py index a60ed70db3..e573070cd2 100644 --- a/galsim/main.py +++ b/galsim/main.py @@ -32,6 +32,8 @@ from .config import ReadConfig, Process from .errors import GalSimError, GalSimValueError, GalSimRangeError +logger = logging.getLogger(__name__) + def parse_args(command_args): """Handle the command line arguments using either argparse (if available) or optparse. """ @@ -106,7 +108,7 @@ def parse_args(command_args): # Return the args return args -def parse_variables(variables, logger): +def parse_variables(variables): """Parse any command-line variables, returning them as a dict """ new_params = {} @@ -141,33 +143,19 @@ def add_modules(config, modules): else: config['modules'].extend(modules) -def make_logger(args): +def setup_logger(args): """Make a logger object according to the command-line specifications. """ - # Make a logger - logger = logging.getLogger('galsim') - # Parse the integer verbosity level from the command line args into a logging_level string logging_levels = { 0: logging.ERROR, 1: logging.WARNING, 2: logging.INFO, 3: logging.DEBUG } level = logging_levels[args.verbosity] - logger.setLevel(level) - - # Setup logging to go to sys.stdout or (if requested) to an output file - if args.log_file is None: - handler = logging.StreamHandler(sys.stdout) - handler.setFormatter(logging.Formatter(args.log_format)) - handler.setLevel(level) - else: - handler = logging.FileHandler(args.log_file, mode='w') - handler.setFormatter(logging.Formatter(args.log_format)) - handler.setLevel(level) - logger.addHandler(handler) - return logger - -def process_config(all_config, args, logger): + + logging.basicConfig(filename=args.log_file, filemode="w", format=args.log_format, level=level, force=True) + +def process_config(all_config, args): """Process the config dict according to the command-line specifications. """ # If requested, load the profiler @@ -183,7 +171,7 @@ def process_config(all_config, args, logger): config['root'] = root # Parse the command-line variables: - new_params = parse_variables(args.variables, logger) + new_params = parse_variables(args.variables) # Add modules to the config['modules'] list add_modules(config, args.module) @@ -197,7 +185,7 @@ def process_config(all_config, args, logger): logger.debug("Process config dict: \n%s", json.dumps(config, indent=4)) # Process the configuration - Process(config, logger, njobs=args.njobs, job=args.job, new_params=new_params, + Process(config, njobs=args.njobs, job=args.job, new_params=new_params, except_abort=args.except_abort) if args.profile: @@ -217,9 +205,9 @@ def main(command_args): """The whole process given command-line parameters in their native (non-ArgParse) form. """ args = parse_args(command_args) - logger = make_logger(args) - all_config = ReadConfig(args.config_file, args.file_type, logger) - process_config(all_config, args, logger) + setup_logger(args) + all_config = ReadConfig(args.config_file, args.file_type) + process_config(all_config, args) def run_main(): """Kick off the process grabbing the command-line parameters from sys.argv diff --git a/galsim/real.py b/galsim/real.py index 8f12ad23a4..c5bb8d662d 100644 --- a/galsim/real.py +++ b/galsim/real.py @@ -18,6 +18,7 @@ __all__ = [ 'RealGalaxy', 'RealGalaxyCatalog', 'ChromaticRealGalaxy', ] +import logging import os import numpy as np import copy @@ -43,6 +44,7 @@ from . import meta_data from ._pyfits import pyfits +logger = logging.getLogger(__name__) HST_area = 45238.93416 # Area of HST primary mirror in cm^2 from Synphot User's Guide. @@ -172,8 +174,6 @@ class RealGalaxy(GSObject): source catalog telescope when creating the `RealGalaxy` (e.g., area_norm=45238.93416 for HST). [default: 1] gsparams: An optional `GSParams` argument. [default: None] - logger: A logger object for output of progress statements if the user wants - them. [default: None] """ _opt_params = { "x_interpolant" : str , "k_interpolant" : str , @@ -193,7 +193,7 @@ class RealGalaxy(GSObject): def __init__(self, real_galaxy_catalog, index=None, id=None, random=False, rng=None, x_interpolant=None, k_interpolant=None, flux=None, flux_rescale=None, - pad_factor=4, noise_pad_size=0, area_norm=1.0, gsparams=None, logger=None): + pad_factor=4, noise_pad_size=0, area_norm=1.0, gsparams=None): if rng is None: rng = BaseDeviate() elif not isinstance(rng, BaseDeviate): @@ -205,8 +205,6 @@ def __init__(self, real_galaxy_catalog, index=None, id=None, random=False, "Cannot supply a flux and a flux rescaling factor.", flux=flux, flux_rescale=flux_rescale) - logger = LoggerWrapper(logger) # So don't need to check `if logger:` all the time. - if isinstance(real_galaxy_catalog, tuple): # Special (undocumented) way to build a RealGalaxy without needing the rgc directly # by providing the things we need from it. Used by COSMOSGalaxy. @@ -532,7 +530,6 @@ class RealGalaxyCatalog: approximately the same total I/O time (assuming you eventually use most of the image files referenced in the catalog), but it is spread over the various calls to `getGalImage` and `getPSFImage`. [default: False] - logger: An optional logger object to log progress. [default: None] """ _opt_params = { 'file_name' : str, 'sample' : str, 'dir' : str, 'preload' : bool } @@ -540,14 +537,12 @@ class RealGalaxyCatalog: # _nobject_only is an intentionally undocumented kwarg that should be used only by # the config structure. It indicates that all we care about is the nobjects parameter. # So skip any other calculations that might normally be necessary on construction. - def __init__(self, file_name=None, sample=None, dir=None, preload=False, logger=None): + def __init__(self, file_name=None, sample=None, dir=None, preload=False): if sample is not None and file_name is not None: raise GalSimIncompatibleValuesError( "Cannot specify both the sample and file_name.", sample=sample, file_name=file_name) - logger = LoggerWrapper(logger) - self.file_name, self.image_dir, self.sample = _parse_files_dirs(file_name, dir, sample) with pyfits.open(self.file_name) as fits: @@ -1016,8 +1011,6 @@ class ChromaticRealGalaxy(ChromaticSum): `ChromaticRealGalaxy` (e.g., area_norm=45238.93416 for HST). [default: 1] gsparams: An optional `GSParams` argument. [default: None] - logger: A logger object for output of progress statements if the user wants - them. [default: None] """ # TODO: SEDs isn't implemented yet in config parser. @@ -1031,15 +1024,13 @@ class ChromaticRealGalaxy(ChromaticSum): _takes_rng = True def __init__(self, real_galaxy_catalogs, index=None, id=None, random=False, rng=None, - gsparams=None, logger=None, **kwargs): + gsparams=None, **kwargs): if rng is None: rng = BaseDeviate() elif not isinstance(rng, BaseDeviate): raise TypeError("The rng provided to ChromaticRealGalaxy is not a BaseDeviate") self.rng = rng - logger = LoggerWrapper(logger) # So don't need to check `if logger:` all the time. - # Get the index to use in the catalog if index is not None: if id is not None or random: @@ -1143,8 +1134,6 @@ def makeFromImages(cls, images, bands, PSFs, xis, **kwargs): `ChromaticRealGalaxy` (e.g., area_norm=45238.93416 for HST). [default: 1] gsparams: An optional `GSParams` argument. [default: None] - logger: A logger object for output of progress statements if the user wants - them. [default: None] """ if not hasattr(PSFs, '__iter__'): @@ -1338,6 +1327,3 @@ def __repr__(self): "area_norm=%r, gsparams=%r)"%(self.catalog_files, self.SEDs, self.index, self._k_interpolant, self._area_norm, self._gsparams)) -# Put this at the bottom to avoid circular import error. -from .config import LoggerWrapper - diff --git a/galsim/roman/roman_config.py b/galsim/roman/roman_config.py index 56ae7adf1d..07b668859e 100644 --- a/galsim/roman/roman_config.py +++ b/galsim/roman/roman_config.py @@ -16,6 +16,8 @@ # and/or other materials provided with the distribution. # +import logging + from . import addReciprocityFailure, applyNonlinearity, applyIPC from . import n_pix, exptime, dark_current, read_noise, gain from . import stray_light_fraction, thermal_backgrounds @@ -32,8 +34,10 @@ from ..random import PoissonDeviate from ..noise import GaussianNoise, PoissonNoise, DeviateNoise +logger = logging.getLogger(__name__) + # RomanPSF object type -def _BuildRomanPSF(config, base, ignore, gsparams, logger): +def _BuildRomanPSF(config, base, ignore, gsparams): req = {} opt = { @@ -80,7 +84,7 @@ def _BuildRomanPSF(config, base, ignore, gsparams, logger): kwargs['extra_aberrations'] = ParseAberrations('extra_aberrations', config, base, 'RomanPSF') - psf = getPSF(wcs=base.get('wcs',None), logger=logger, **kwargs) + psf = getPSF(wcs=base.get('wcs',None), **kwargs) return psf, False RegisterObjectType('RomanPSF', _BuildRomanPSF) @@ -93,13 +97,12 @@ class RomanBandpassBuilder(BandpassBuilder): name (str) The name of the Roman filter to get. (required) """ - def buildBandpass(self, config, base, logger): + def buildBandpass(self, config, base): """Build the Bandpass based on the specifications in the config dict. Parameters: config: The configuration dict for the bandpass type. base: The base configuration dict. - logger: If provided, a logger for logging debug statements. Returns: the constructed Bandpass object. @@ -121,7 +124,7 @@ def buildBandpass(self, config, base, logger): # RomanSCA image type class RomanSCAImageBuilder(ScatteredImageBuilder): - def setup(self, config, base, image_num, obj_num, ignore, logger): + def setup(self, config, base, image_num, obj_num, ignore): """Do the initialization and setup for building the image. This figures out the size that the image will be, but doesn't actually build it yet. @@ -133,7 +136,6 @@ def setup(self, config, base, image_num, obj_num, ignore, logger): obj_num: The first object number in the image. ignore: A list of parameters that are allowed to be in config that we can ignore here. i.e. it won't be an error if these parameters are present. - logger: If given, a logger object to log progress. Returns: xsize, ysize @@ -141,7 +143,7 @@ def setup(self, config, base, image_num, obj_num, ignore, logger): logger.debug('image %d: Building RomanSCA: image, obj = %d,%d', image_num,image_num,obj_num) - self.nobjects = self.getNObj(config, base, image_num, logger=logger) + self.nobjects = self.getNObj(config, base, image_num) logger.debug('image %d: nobj = %d',image_num,self.nobjects) # These are allowed for Scattered, but we don't use them here. @@ -203,7 +205,7 @@ def getBandpass(self, filter_name): self.all_roman_bp = getBandpasses() return self.all_roman_bp[filter_name] - def addNoise(self, image, config, base, image_num, obj_num, current_var, logger): + def addNoise(self, image, config, base, image_num, obj_num, current_var): """Add the final noise to a Scattered image Parameters: @@ -213,7 +215,6 @@ def addNoise(self, image, config, base, image_num, obj_num, current_var, logger) image_num: The current image number. obj_num: The first object number in the image. current_var: The current noise variance in each postage stamps. - logger: If given, a logger object to log progress. """ base['current_noise_image'] = base['current_image'] wcs = base['wcs'] diff --git a/galsim/roman/roman_psfs.py b/galsim/roman/roman_psfs.py index 28a63ece4a..c8f358d187 100644 --- a/galsim/roman/roman_psfs.py +++ b/galsim/roman/roman_psfs.py @@ -16,6 +16,7 @@ # and/or other materials provided with the distribution. # +import logging import numpy as np import os @@ -35,6 +36,7 @@ from .. import fits from .. import meta_data +logger = logging.getLogger(__name__) """ @file roman_psfs.py @@ -59,7 +61,7 @@ def getPSF(SCA, bandpass, SCA_pos=None, pupil_bin=4, wcs=None, n_waves=None, extra_aberrations=None, wavelength=None, gsparams=None, - logger=None, high_accuracy=None, approximate_struts=None): + high_accuracy=None, approximate_struts=None): """Get a single PSF for Roman ST observations. The user must provide the SCA and bandpass; the latter is used when setting up the pupil diff --git a/galsim/utilities.py b/galsim/utilities.py index 6300b2847c..0a7841ed09 100644 --- a/galsim/utilities.py +++ b/galsim/utilities.py @@ -52,6 +52,8 @@ from .position import parse_pos_args from ._utilities import * +logger = logging.getLogger(__name__) + def roll2d(image, shape): """Perform a 2D roll (circular shift) on a supplied 2D NumPy array, conveniently. @@ -1489,12 +1491,11 @@ def nCr(n, r): else: return 0 -def set_omp_threads(num_threads, logger=None): +def set_omp_threads(num_threads): """Set the number of OpenMP threads to use in the C++ layer. :param num_threads: The target number of threads to use (If None or <=0, then try to use the numer of cpus.) - :param logger: If desired, a logger object for logging any warnings here. (default: None) :returns: The number of threads OpenMP reports that it will use. Typically this matches the input, but OpenMP reserves the right not to comply with @@ -1507,12 +1508,10 @@ def set_omp_threads(num_threads, logger=None): # If num_threads is auto, get it from cpu_count if num_threads is None or num_threads <= 0: num_threads = multiprocessing.cpu_count() - if logger: - logger.debug('multiprocessing.cpu_count() = %d',num_threads) + logger.debug('multiprocessing.cpu_count() = %d',num_threads) # Tell OpenMP to use this many threads - if logger: - logger.debug('Telling OpenMP to use %d threads',num_threads) + logger.debug('Telling OpenMP to use %d threads',num_threads) # Cf. comment in get_omp_threads. Do it here too. var = "OMP_PROC_BIND" @@ -1521,13 +1520,12 @@ def set_omp_threads(num_threads, logger=None): num_threads = _galsim.SetOMPThreads(num_threads) # Report back appropriately. - if logger: - logger.debug('OpenMP reports that it will use %d threads',num_threads) - if num_threads > 1: - logger.info('Using %d threads.',num_threads) - elif input_num_threads is not None and input_num_threads != 1: - # Only warn if the user specifically asked for num_threads != 1. - logger.warning("Unable to use multiple threads, since OpenMP is not enabled.") + logger.debug('OpenMP reports that it will use %d threads',num_threads) + if num_threads > 1: + logger.info('Using %d threads.',num_threads) + elif input_num_threads is not None and input_num_threads != 1: + # Only warn if the user specifically asked for num_threads != 1. + logger.warning("Unable to use multiple threads, since OpenMP is not enabled.") return num_threads @@ -1761,39 +1759,6 @@ def f2(*args, **kwargs): return f2 -class CaptureLog: - """A context manager that saves logging output into a string that is accessible for - checking in unit tests. - - After exiting the context, the attribute ``output`` will have the logging output. - - Sample usage: - - >>> with CaptureLog() as cl: - ... cl.logger.info('Do some stuff') - >>> assert cl.output == 'Do some stuff' - - """ - def __init__(self, level=3): - logging_levels = { 0: logging.CRITICAL, - 1: logging.WARNING, - 2: logging.INFO, - 3: logging.DEBUG } - self.logger = logging.getLogger('CaptureLog') - self.logger.setLevel(logging_levels[level]) - self.stream = StringIO() - self.handler = logging.StreamHandler(self.stream) - self.logger.addHandler(self.handler) - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - self.handler.flush() - self.output = self.stream.getvalue().strip() - self.handler.close() - - # Context to make it easier to profile bits of the code class Profile: """A context manager that profiles a snippet of code. diff --git a/tests/config_input_test_modules.py b/tests/config_input_test_modules.py index 044eef6156..2570ac5f53 100644 --- a/tests/config_input_test_modules.py +++ b/tests/config_input_test_modules.py @@ -28,14 +28,12 @@ def _ret_size(size=-1): class InputSizeLoader(galsim.config.InputLoader): - def getKwargs(self, config, base, logger): + def getKwargs(self, config, base): req = {"size": int} kwargs, safe = galsim.config.GetAllParams(config, base, req=req) - if self.takes_logger: - kwargs["logger"] = logger return kwargs, True - def initialize(self, input_objs, num, base, logger): + def initialize(self, input_objs, num, base): if num == 0: base['input_size_0'] = input_objs[0] if all(iobj is not None for iobj in input_objs): diff --git a/tests/galsim_test_helpers.py b/tests/galsim_test_helpers.py index 5b09773d76..f0e62259b8 100644 --- a/tests/galsim_test_helpers.py +++ b/tests/galsim_test_helpers.py @@ -20,7 +20,7 @@ import numpy as np import galsim -from galsim.utilities import check_pickle, check_all_diff, timer, CaptureLog, Profile +from galsim.utilities import check_pickle, check_all_diff, timer, Profile # We used to roll our own versions of these, but numpy.testing has good ones now. from numpy.testing import assert_raises diff --git a/tests/run_examples.py b/tests/run_examples.py index 8d9c2eaab1..a3da8448f0 100644 --- a/tests/run_examples.py +++ b/tests/run_examples.py @@ -84,9 +84,9 @@ def test_demo1(): demo1.main([]) logger = logging.getLogger('galsim') logger.setLevel(logging.WARNING) - config = galsim.config.ReadConfig('demo1.yaml', logger=logger)[0] + config = galsim.config.ReadConfig('demo1.yaml')[0] print('Running demo1.yaml') - galsim.config.Process(config, logger=logger, except_abort=True) + galsim.config.Process(config, except_abort=True) # There is no assert at the end of this one, since they are not expected to be identical # due to the lack of a specified seed. This just checks for syntax errors. @@ -100,9 +100,9 @@ def test_demo2(): demo2.main([]) logger = logging.getLogger('galsim') logger.setLevel(logging.WARNING) - config = galsim.config.ReadConfig('demo2.yaml', file_type='yaml', logger=logger)[0] + config = galsim.config.ReadConfig('demo2.yaml', file_type='yaml')[0] print('Running demo2.yaml') - galsim.config.Process(config, logger=logger, except_abort=True) + galsim.config.Process(config, except_abort=True) assert check_same('output/demo2.fits', 'output_yaml/demo2.fits') @timer @@ -115,9 +115,9 @@ def test_demo3(): demo3.main([]) logger = logging.getLogger('galsim') logger.setLevel(logging.WARNING) - config = galsim.config.ReadConfig('demo3.yaml', logger=logger)[0] + config = galsim.config.ReadConfig('demo3.yaml')[0] print('Running demo3.yaml') - galsim.config.Process(config, logger=logger, except_abort=True) + galsim.config.Process(config, except_abort=True) assert check_same('output/demo3.fits', 'output_yaml/demo3.fits') assert check_same('output/demo3_epsf.fits', 'output_yaml/demo3_epsf.fits') @@ -131,9 +131,9 @@ def test_demo4(): demo4.main([]) logger = logging.getLogger('galsim') logger.setLevel(logging.WARNING) - config = galsim.config.ReadConfig('demo4.yaml', logger=logger)[0] + config = galsim.config.ReadConfig('demo4.yaml')[0] print('Running demo4.yaml') - galsim.config.Process(config, logger=logger, except_abort=True) + galsim.config.Process(config, except_abort=True) assert check_same('output/multi.fits', 'output_yaml/multi.fits') @timer @@ -146,9 +146,9 @@ def test_demo5(): demo5.main([]) logger = logging.getLogger('galsim') logger.setLevel(logging.WARNING) - config = galsim.config.ReadConfig('demo5.yaml', logger=logger)[0] + config = galsim.config.ReadConfig('demo5.yaml')[0] print('Running demo5.yaml') - galsim.config.Process(config, logger=logger, except_abort=True) + galsim.config.Process(config, except_abort=True) assert check_same('output/g08_psf.fits', 'output_yaml/g08_psf.fits') assert check_same('output/g08_gal.fits', 'output_yaml/g08_gal.fits') @@ -162,11 +162,11 @@ def test_demo6(): demo6.main([]) logger = logging.getLogger('galsim') logger.setLevel(logging.WARNING) - configs = galsim.config.ReadConfig('demo6.yaml', logger=logger) + configs = galsim.config.ReadConfig('demo6.yaml') print('Running demo6.yaml pass #1') - galsim.config.Process(configs[0], logger=logger, except_abort=True) + galsim.config.Process(configs[0], except_abort=True) print('Running demo6.yaml pass #2') - galsim.config.Process(configs[1], logger=logger, except_abort=True) + galsim.config.Process(configs[1], except_abort=True) assert check_same('output/psf_real.fits', 'output_yaml/psf_real.fits') assert check_same('output/cube_real.fits', 'output_yaml/cube_real.fits') @@ -182,9 +182,9 @@ def test_demo7(): demo7.main([]) logger = logging.getLogger('galsim') logger.setLevel(logging.WARNING) - config = galsim.config.ReadConfig('demo7.yaml', logger=logger)[0] + config = galsim.config.ReadConfig('demo7.yaml')[0] print('Running demo7.yaml') - galsim.config.Process(config, logger=logger, except_abort=True) + galsim.config.Process(config, except_abort=True) with gzip.open('output/cube_phot.fits.gz', 'rb') as f_in: with open('output/cube_phot.fits', 'wb') as f_out: shutil.copyfileobj(f_in, f_out) @@ -203,11 +203,11 @@ def test_demo8(): demo8.main([]) logger = logging.getLogger('galsim') logger.setLevel(logging.WARNING) - configs = galsim.config.ReadConfig('demo8.yaml', logger=logger) + configs = galsim.config.ReadConfig('demo8.yaml') print('Running demo8.yaml pass #1') - galsim.config.Process(configs[0], logger=logger, except_abort=True) + galsim.config.Process(configs[0], except_abort=True) print('Running demo8.yaml pass #2') - galsim.config.Process(configs[1], logger=logger, except_abort=True) + galsim.config.Process(configs[1], except_abort=True) assert check_same('output/bpd_single.fits', 'output_yaml/bpd_single.fits') assert check_same('output/bpd_multi.fits', 'output_yaml/bpd_multi.fits') @@ -224,17 +224,17 @@ def test_demo9(): demo9.main([]) logger = logging.getLogger('galsim') logger.setLevel(logging.WARNING) - config = galsim.config.ReadConfig('json/demo9.json', logger=logger)[0] + config = galsim.config.ReadConfig('json/demo9.json')[0] print('Running demo9.json') new_params = { 'output.skip' : { 'type' : 'List', 'items' : [0,0,0,0,0,1] } } - galsim.config.Process(config, logger=logger, new_params=new_params, njobs=3, job=1, + galsim.config.Process(config, new_params=new_params, njobs=3, job=1, except_abort=True) - galsim.config.Process(config, logger=logger, new_params=new_params, njobs=3, job=2, + galsim.config.Process(config, new_params=new_params, njobs=3, job=2, except_abort=True) - galsim.config.Process(config, logger=logger, new_params=new_params, njobs=3, job=3, + galsim.config.Process(config, new_params=new_params, njobs=3, job=3, except_abort=True) new_params = { 'output.noclobber' : True } - galsim.config.Process(config, logger=logger, new_params=new_params, except_abort=True) + galsim.config.Process(config, new_params=new_params, except_abort=True) for dir_num in range(1,5): for file_num in range(5): file_name = 'nfw%d/cluster%04d.fits'%(dir_num, file_num) @@ -252,9 +252,9 @@ def test_demo10(): demo10.main([]) logger = logging.getLogger('galsim') logger.setLevel(logging.WARNING) - config = galsim.config.ReadConfig('demo10.yaml', logger=logger)[0] + config = galsim.config.ReadConfig('demo10.yaml')[0] print('Running demo10.yaml') - galsim.config.Process(config, logger=logger, except_abort=True) + galsim.config.Process(config, except_abort=True) assert check_same('output/power_spectrum.fits', 'output_yaml/power_spectrum.fits') @timer @@ -267,9 +267,9 @@ def test_demo11(): demo11.main([]) logger = logging.getLogger('galsim') logger.setLevel(logging.WARNING) - config = galsim.config.ReadConfig('demo11.yaml', logger=logger)[0] + config = galsim.config.ReadConfig('demo11.yaml')[0] print('Running demo11.yaml') - galsim.config.Process(config, logger=logger, except_abort=True) + galsim.config.Process(config, except_abort=True) assert check_same('output/tabulated_power_spectrum.fits.fz', 'output_yaml/tabulated_power_spectrum.fits.fz') @@ -283,13 +283,13 @@ def test_demo12(): demo12.main([]) logger = logging.getLogger('galsim') logger.setLevel(logging.WARNING) - configs = galsim.config.ReadConfig('demo12.yaml', logger=logger) + configs = galsim.config.ReadConfig('demo12.yaml') print('Running demo12.yaml pass #1') - galsim.config.Process(configs[0], logger=logger, except_abort=True) + galsim.config.Process(configs[0], except_abort=True) print('Running demo12.yaml pass #2') - galsim.config.Process(configs[1], logger=logger, except_abort=True) + galsim.config.Process(configs[1], except_abort=True) print('Running demo12.yaml pass #3') - galsim.config.Process(configs[2], logger=logger, except_abort=True) + galsim.config.Process(configs[2], except_abort=True) for part in 'abc': for band in 'ugrizy': assert check_same('output/demo12%s_%s.fits'%(part,band), @@ -305,7 +305,7 @@ def test_demo13(): demo13.main(['--nobj=5', '--filters=Y', '--test']) logger = logging.getLogger('galsim') logger.setLevel(logging.WARNING) - config = galsim.config.ReadConfig('demo13.yaml', logger=logger)[0] + config = galsim.config.ReadConfig('demo13.yaml')[0] del config['input']['cosmos_catalog'][0]['sample'] del config['input']['cosmos_catalog'][1]['sample'] config['image']['nobjects'] = 5 @@ -315,7 +315,7 @@ def test_demo13(): config['input']['cosmos_catalog'][1]['dir'] = 'data' config['input']['cosmos_catalog'][1]['file_name'] = 'real_galaxy_catalog_23.5_example.fits' print('Running demo13.yaml') - galsim.config.Process(config, logger=logger, except_abort=True) + galsim.config.Process(config, except_abort=True) assert check_same('output/demo13_Y106.fits', 'output_yaml/demo13_Y106.fits') @@ -347,32 +347,32 @@ def test_des(): logger.setLevel(logging.WARNING) print('Running draw_psf.yaml') - configs = galsim.config.ReadConfig('draw_psf.yaml', logger=logger) + configs = galsim.config.ReadConfig('draw_psf.yaml') for config in configs: config['output']['nfiles'] = 1 - galsim.config.Process(config, logger=logger, except_abort=True) + galsim.config.Process(config, except_abort=True) assert check_same('output/DECam_00154912_01_psfex_image.fits', 'output_yaml/DECam_00154912_01_psfex_image.fits') assert check_same('output/DECam_00154912_01_fitpsf_image.fits', 'output_yaml/DECam_00154912_01_fitpsf_image.fits') - config = galsim.config.ReadConfig('meds.yaml', logger=logger)[0] + config = galsim.config.ReadConfig('meds.yaml')[0] config['output']['nfiles'] = 1 config['output']['nobjects'] = 100 config['gal']['items'][0]['gal_type'] = 'parametric' config['input']['cosmos_catalog']['file_name'] = '../data/real_galaxy_catalog_23.5_example.fits' del config['input']['cosmos_catalog']['sample'] config['input']['des_wcs']['bad_ccds'] = list(range(2,63)) # All but CCD 1 - galsim.config.Process(config, logger=logger, except_abort=True) + galsim.config.Process(config, except_abort=True) input_cosmos = config['input']['cosmos_catalog'] # Save example COSMOS catalog spec. - config = galsim.config.ReadConfig('blend.yaml', logger=logger)[0] - galsim.config.Process(config, logger=logger, except_abort=True) + config = galsim.config.ReadConfig('blend.yaml')[0] + galsim.config.Process(config, except_abort=True) - config = galsim.config.ReadConfig('blendset.yaml', logger=logger)[0] + config = galsim.config.ReadConfig('blendset.yaml')[0] config['input']['cosmos_catalog'] = input_cosmos config['input']['des_psfex']['file_name']['num'] = 1 - galsim.config.Process(config, logger=logger, except_abort=True) + galsim.config.Process(config, except_abort=True) finally: os.chdir(original_dir) @@ -399,14 +399,14 @@ def test_great3(): p2['input.cosmos_catalog.sample'] = '' for file_name in ['cgc.yaml', 'cgc_psf.yaml', 'rgc.yaml', 'rgc_psf.yaml']: - configs = galsim.config.ReadConfig(file_name, logger=logger) + configs = galsim.config.ReadConfig(file_name) print('Running ',file_name) if 'psf' in file_name: new_params = p1 else: new_params = p2 for config in configs: - galsim.config.Process(config, logger=logger, new_params=new_params, + galsim.config.Process(config, new_params=new_params, except_abort=True) finally: os.chdir(original_dir) diff --git a/tests/test_config_gsobject.py b/tests/test_config_gsobject.py index 66e0c88130..49419710c2 100644 --- a/tests/test_config_gsobject.py +++ b/tests/test_config_gsobject.py @@ -933,7 +933,7 @@ def test_pixel(): gsobject_compare(gal4a, gal4b, conv=galsim.Gaussian(0.1)) @timer -def test_realgalaxy(): +def test_realgalaxy(caplog): """Test various ways to build a RealGalaxy """ # I don't want to gratuitously copy the real_catalog catalog, so use the @@ -1016,11 +1016,9 @@ def test_realgalaxy(): galsim.config.RemoveCurrent(config) # Clear the cached values, so it rebuilds. galsim.config.BuildGSObject(config, 'gal5') - # If there is a logger, there should be a warning message emitted. - with CaptureLog() as cl: - galsim.config.RemoveCurrent(config) - galsim.config.BuildGSObject(config, 'gal5', logger=cl.logger) - assert "No base['rng'] available" in cl.output + galsim.config.RemoveCurrent(config) + galsim.config.BuildGSObject(config, 'gal5') + assert "No base['rng'] available" in caplog.text config['obj_num'] = 5 gal6a = galsim.config.BuildGSObject(config, 'gal6')[0] @@ -1046,9 +1044,11 @@ def test_realgalaxy(): @timer -def test_cosmosgalaxy(): +def test_cosmosgalaxy(caplog): """Test various ways to build a COSMOSGalaxy """ + caplog.set_level(logging.INFO) + # I don't want to gratuitously copy the real_catalog catalog, so use the # version in the examples directory. real_gal_dir = os.path.join('..','examples','data') @@ -1193,26 +1193,26 @@ def test_cosmosgalaxy(): gsobject_compare(gal1a, gal1b, conv=conv) # Check some information passed to the logger about the kind of catalog being built. + caplog.clear() galsim.config.SetupConfigImageNum(config, 0, 0) - with CaptureLog() as cl: - galsim.config.SetupInputsForImage(config, logger=cl.logger) - assert "Using user-specified COSMOSCatalog" in cl.output + galsim.config.SetupInputsForImage(config) + assert "Using user-specified COSMOSCatalog" in caplog.text + caplog.clear() config['input']['cosmos_catalog']['sample'] = 25.2 - with CaptureLog() as cl: - galsim.config.SetupInputsForImage(config, logger=cl.logger) - assert "sample = 25.2" in cl.output + galsim.config.SetupInputsForImage(config) + assert "sample = 25.2" in caplog.text + caplog.clear() config['input']['cosmos_catalog'] = {} - with CaptureLog() as cl: - galsim.config.SetupInputsForImage(config, logger=cl.logger) - assert "Using user-specified" not in cl.output + galsim.config.SetupInputsForImage(config) + assert "Using user-specified" not in caplog.text + caplog.clear() config['gal']['gal_type'] = 'parametric' - with CaptureLog() as cl: - galsim.config.SetupInputsForImage(config, logger=cl.logger) - assert "Using parametric galaxies" in cl.output + galsim.config.SetupInputsForImage(config) + assert "Using parametric galaxies" in caplog.text + caplog.clear() config['gal']['gal_type'] = 'real' - with CaptureLog() as cl: - galsim.config.SetupInputsForImage(config, logger=cl.logger) - assert "Using real galaxies" in cl.output + galsim.config.SetupInputsForImage(config) + assert "Using real galaxies" in caplog.text @timer @@ -1304,24 +1304,20 @@ def COSMOS_ZPhot(config, base, value_type): } } - logger = logging.getLogger('test_cosmos_redshift') - logger.addHandler(logging.StreamHandler(sys.stdout)) - logger.setLevel(logging.DEBUG) - - galsim.config.ProcessInput(config, logger=logger) + galsim.config.ProcessInput(config) galsim.config.SetupConfigImageNum(config, 0, 0) - galsim.config.SetupInputsForImage(config, logger=logger) + galsim.config.SetupInputsForImage(config) cosmos_cat = galsim.COSMOSCatalog( dir=real_gal_dir, file_name=real_gal_cat, use_real=False, preload=True) sample_cat = galsim.GalaxySample( dir=real_gal_dir, file_name=real_gal_cat, use_real=False, preload=True) - galsim.config.SetupConfigObjNum(config, 0, logger=logger) - galsim.config.SetupConfigRNG(config, seed_offset=1, logger=logger) + galsim.config.SetupConfigObjNum(config, 0) + galsim.config.SetupConfigRNG(config, seed_offset=1) image_pos = galsim.config.ParseValue(config['image'], 'image_pos', config, galsim.PositionD)[0] - galsim.config.SetupConfigStampSize(config,32,32,image_pos,None, logger=logger) - gal1a = galsim.config.BuildGSObject(config, 'gal', logger=logger)[0] + galsim.config.SetupConfigStampSize(config,32,32,image_pos,None) + gal1a = galsim.config.BuildGSObject(config, 'gal')[0] print('gal1a = ',gal1a) first_seed = galsim.BaseDeviate(12345).raw() rng = galsim.UniformDeviate(first_seed+1) @@ -1339,13 +1335,13 @@ def COSMOS_ZPhot(config, base, value_type): print('gal1b = ',gal1b) gsobject_compare(gal1a, gal1b) - gal2 = galsim.config.BuildGSObject(config, 'gal2', logger=logger)[0] + gal2 = galsim.config.BuildGSObject(config, 'gal2')[0] gsobject_compare(gal2, gal1b) - gal3 = galsim.config.BuildGSObject(config, 'gal3', logger=logger)[0] + gal3 = galsim.config.BuildGSObject(config, 'gal3')[0] gsobject_compare(gal3, gal1b) @timer -def test_interpolated_image(): +def test_interpolated_image(caplog): """Test various ways to build an InterpolatedImage """ imgdir = 'SBProfile_comparison_images' @@ -1424,17 +1420,16 @@ def test_interpolated_image(): galsim.config.BuildGSObject(config, 'gal6') # If there is a logger, there should be a warning message emitted, but only the first time. - with CaptureLog() as cl: - galsim.config.RemoveCurrent(config) - galsim.config.BuildGSObject(config, 'gal5', logger=cl.logger) - assert "No base['rng'] available" in cl.output - with CaptureLog(level=1) as cl: - galsim.config.RemoveCurrent(config) - galsim.config.BuildGSObject(config, 'gal6', logger=cl.logger) - assert cl.output == '' + galsim.config.RemoveCurrent(config) + galsim.config.BuildGSObject(config, 'gal5') + assert "No base['rng'] available" in caplog.text + caplog.clear() + galsim.config.RemoveCurrent(config) + galsim.config.BuildGSObject(config, 'gal6') + assert caplog.text == '' @timer -def test_add(): +def test_add(caplog): """Test various ways to build a Add """ config = { @@ -1592,10 +1587,9 @@ def test_add(): # If the sum comes out larger than 1, emit a warning config['gal8']['items'][1]['flux'] = 0.9 galsim.config.RemoveCurrent(config) - with CaptureLog() as cl: - galsim.config.BuildGSObject(config, 'gal8', logger=cl.logger) + galsim.config.BuildGSObject(config, 'gal8') assert ("Warning: Automatic flux for the last item in Sum (to make the total flux=1) " + - "resulted in negative flux = -0.200000 for that item") in cl.output + "resulted in negative flux = -0.200000 for that item") in caplog.text with assert_raises(galsim.GalSimConfigError): galsim.config.BuildGSObject(config, 'bad1') @@ -1851,9 +1845,11 @@ def test_list(): @timer -def test_repeat(): +def test_repeat(caplog): """Test use of the repeat option for an object """ + caplog.set_level(logging.DEBUG) + config = { 'rng' : galsim.BaseDeviate(1234), 'gal' : { @@ -1889,10 +1885,9 @@ def test_repeat(): # Also check that the logger reports why it is using the current object config['obj_num'] = 5 - with CaptureLog() as cl: - gal2a = galsim.config.BuildGSObject(config, 'gal', logger=cl.logger)[0] + gal2a = galsim.config.BuildGSObject(config, 'gal')[0] gsobject_compare(gal2a, gal2b) - assert "repeat = 3, index = 5, use current object" in cl.output + assert "repeat = 3, index = 5, use current object" in caplog.text @timer @@ -1934,7 +1929,7 @@ def __init__(self, flux=1., gsparams=None): gsobject_compare(gal3a, gal3b, conv=psf) # Now an equivalent thing, but implemented with a builder rather than a class. - def BuildPseudoDelta(config, base, ignore, gsparams, logger): + def BuildPseudoDelta(config, base, ignore, gsparams): opt = { 'flux' : float } kwargs, safe = galsim.config.GetAllParams(config, base, opt=opt, ignore=ignore) gsparams = galsim.GSParams(**gsparams) # within config, it is passed around as a dict @@ -1955,7 +1950,7 @@ def BuildPseudoDelta(config, base, ignore, gsparams, logger): # This one is potentially more useful. # This is now equivalent to the Eval type, but it started its life as a user type, # so keep it here with a different name. - def EvalGSObject(config, base, ignore, gsparams, logger): + def EvalGSObject(config, base, ignore, gsparams): req = { 'str': str } params, safe = galsim.config.GetAllParams(config, base, req=req, ignore=ignore) return galsim.utilities.math_eval(params['str']).withGSParams(**gsparams), safe @@ -2169,7 +2164,7 @@ def test_sed(): # Base class usage is invalid builder = galsim.config.sed.SEDBuilder() - assert_raises(NotImplementedError, builder.buildSED, config, config, logger=None) + assert_raises(NotImplementedError, builder.buildSED, config, config) diff --git a/tests/test_config_image.py b/tests/test_config_image.py index 5f245f09f5..39ffba3f78 100644 --- a/tests/test_config_image.py +++ b/tests/test_config_image.py @@ -28,7 +28,6 @@ import galsim from galsim_test_helpers import * - @timer def test_single(): """Test the default image type = Single and stamp type = Basic @@ -48,24 +47,6 @@ def test_single(): } } - logger = logging.getLogger('test_single') - logger.addHandler(logging.StreamHandler(sys.stdout)) - logger.setLevel(logging.INFO) - - # Test a little bit of the LoggerWrapper functionality - logger_wrapper = galsim.config.LoggerWrapper(logger) - assert logger_wrapper.level == logger.getEffectiveLevel() - assert logger_wrapper.getEffectiveLevel() == logger.getEffectiveLevel() - assert logger_wrapper.isEnabledFor(logging.WARNING) - assert logger_wrapper.isEnabledFor(logging.CRITICAL) - assert not logger_wrapper.isEnabledFor(logging.DEBUG) - - # smoke test for critical calls - # these are not normally used by galsim so a test here is needed - logger_wrapper.critical("blah blah") - none_logger_wrapper = galsim.config.LoggerWrapper(None) - none_logger_wrapper.critical("blah blah") - im1_list = [] nimages = 6 first_seed = galsim.BaseDeviate(1234).raw() @@ -77,24 +58,24 @@ def test_single(): im1 = gal.drawImage(scale=1) im1_list.append(im1) - im2 = galsim.config.BuildImage(config, obj_num=k, logger=logger) + im2 = galsim.config.BuildImage(config, obj_num=k) np.testing.assert_array_equal(im2.array, im1.array) # Can also use BuildStamp. In this case, it will used the cached value # of sigma, so we don't need to worry about resetting the rng in the config dict. - im3 = galsim.config.BuildStamp(config, obj_num=k, logger=logger)[0] + im3 = galsim.config.BuildStamp(config, obj_num=k)[0] np.testing.assert_array_equal(im3.array, im1.array) # Users can write their own custom stamp builders, in which case they may want # to call DrawBasic directly as part of the draw method in their builder. im4 = galsim.config.DrawBasic(gal, im3.copy(), 'auto', galsim.PositionD(0,0), - config['stamp'], config, logger) + config['stamp'], config) np.testing.assert_array_equal(im4.array, im1.array) # The user is allowed to to add extra kwarg to the end, which would be passed on # to the drawImage command. im5 = galsim.config.DrawBasic(gal, im3.copy(), 'auto', galsim.PositionD(0,0), - config['stamp'], config, logger, + config['stamp'], config, scale=1.0, dtype=np.float32) np.testing.assert_array_equal(im5.array, im1.array) @@ -102,7 +83,7 @@ def test_single(): config['stamp']['scale'] = 1.0 config['stamp']['dtype'] = 'np.float32' im6 = galsim.config.DrawBasic(gal, im3.copy(), 'auto', galsim.PositionD(0,0), - config['stamp'], config, logger) + config['stamp'], config) np.testing.assert_array_equal(im6.array, im1.array) config['stamp'].pop('scale') config['stamp'].pop('dtype') @@ -217,42 +198,38 @@ def test_positions(): } } - logger = logging.getLogger('test_positions') - logger.addHandler(logging.StreamHandler(sys.stdout)) - #logger.setLevel(logging.DEBUG) - gal = galsim.Gaussian(sigma=1.7, flux=100) im1 = gal.drawImage(nx=21, ny=21, scale=1) im1.setCenter(39,43) - im2 = galsim.config.BuildImage(config, logger=logger) + im2 = galsim.config.BuildImage(config) np.testing.assert_array_equal(im2.array, im1.array) assert im2.bounds == im1.bounds # This is really the main test. # image_pos could also be in image config['image']['image_pos'] = config['stamp']['image_pos'] del config['stamp']['image_pos'] - im3 = galsim.config.BuildImage(config, logger=logger) + im3 = galsim.config.BuildImage(config) np.testing.assert_array_equal(im3.array, im1.array) assert im3.bounds == im1.bounds # since our wcs is just a pixel scale, we can also specify world_pos instead. config['stamp']['world_pos'] = config['image']['image_pos'] del config['image']['image_pos'] - im4 = galsim.config.BuildImage(config, logger=logger) + im4 = galsim.config.BuildImage(config) np.testing.assert_array_equal(im4.array, im1.array) assert im4.bounds == im1.bounds # Can also set world_pos in image. config['image']['world_pos'] = config['stamp']['world_pos'] del config['stamp']['world_pos'] - im5 = galsim.config.BuildImage(config, logger=logger) + im5 = galsim.config.BuildImage(config) np.testing.assert_array_equal(im5.array, im1.array) assert im5.bounds == im1.bounds # It is also valid to give both world_pos and image_pos in the image field for Single. config['image']['image_pos'] = config['image']['world_pos'] - im6 = galsim.config.BuildImage(config, logger=logger) + im6 = galsim.config.BuildImage(config) np.testing.assert_array_equal(im6.array, im1.array) assert im6.bounds == im1.bounds @@ -260,26 +237,26 @@ def test_positions(): config['stamp']['image_pos'] = config['image']['image_pos'] del config['image'] del config['stamp']['_done'] - im7, _ = galsim.config.BuildStamp(config, logger=logger) + im7, _ = galsim.config.BuildStamp(config) np.testing.assert_array_equal(im7.array, im1.array) assert im7.bounds == im1.bounds del config['image'] del config['stamp']['_done'] - im8, _ = galsim.config.BuildStamps(1, config, logger=logger) + im8, _ = galsim.config.BuildStamps(1, config) im8 = im8[0] np.testing.assert_array_equal(im8.array, im1.array) assert im8.bounds == im1.bounds del config['image'] del config['stamp']['_done'] - im9 = galsim.config.BuildImage(config, logger=logger) + im9 = galsim.config.BuildImage(config) np.testing.assert_array_equal(im9.array, im1.array) assert im9.bounds == im1.bounds del config['image'] del config['stamp']['_done'] - im10 = galsim.config.BuildImages(1, config, logger=logger) + im10 = galsim.config.BuildImages(1, config) im10 = im10[0] np.testing.assert_array_equal(im10.array, im1.array) assert im10.bounds == im1.bounds @@ -313,7 +290,7 @@ def test_positions(): @timer -def test_phot(): +def test_phot(caplog): """Test draw_method=phot, which has extra allowed kwargs """ config = { @@ -362,9 +339,8 @@ def test_phot(): np.testing.assert_array_equal(im3c.array, im3a.array) # Although with a logger, it should give a warning. - with CaptureLog() as cl: - im3d = galsim.config.BuildImage(config, logger=cl.logger) - assert "ignoring 'max_extra_noise'" in cl.output + im3d = galsim.config.BuildImage(config) + assert "ignoring 'max_extra_noise'" in caplog.text # Without n_photons, it should work. But then, we also need a noise field # So without the noise field, it will raise an exception. @@ -398,9 +374,11 @@ def test_phot(): @timer -def test_reject(): +def test_reject(caplog): """Test various ways that objects can be rejected. """ + caplog.set_level(logging.DEBUG) + # Make a custom function for rejecting COSMOSCatalog objects that use Sersics with n > 2. def HighN(config, base, value_type): # GetCurrentValue returns the constructed 'gal' object for this pass @@ -490,15 +468,8 @@ def HighN(config, base, value_type): galsim.config.ProcessInput(config) orig_config = config.copy() - if False: - logger = logging.getLogger('test_reject') - logger.addHandler(logging.StreamHandler(sys.stdout)) - #logger.setLevel(logging.DEBUG) - else: - logger = galsim.config.LoggerWrapper(None) - nimages = 11 - im_list = galsim.config.BuildStamps(nimages, config, do_noise=False, logger=logger)[0] + im_list = galsim.config.BuildStamps(nimages, config, do_noise=False)[0] # For this particular config, only 3 of them are real images. The others were skipped. # The skipped ones are present in the list, but their flux is 0 fluxes = [im.array.sum(dtype=float) if im is not None else 0 for im in im_list] @@ -506,42 +477,38 @@ def HighN(config, base, value_type): np.testing.assert_almost_equal(fluxes, expected_fluxes, decimal=0) # Check for a few of the logging outputs that explain why things were rejected. - with CaptureLog() as cl: - im_list2 = galsim.config.BuildStamps(nimages, config, do_noise=False, logger=cl.logger)[0] + im_list2 = galsim.config.BuildStamps(nimages, config, do_noise=False)[0] for im1,im2 in zip(im_list, im_list2): assert im1 == im2 - #print(cl.output) # Note: I'm testing for specific error messages here, which could change if we change # the order of operations somewhere. The point here is that we hit at least one of each # kind of skip/rejection/exception that we intended in the config dict above. - assert "obj 8: Skipping because field skip=True" in cl.output - assert "obj 8: Caught SkipThisObject: e = None" in cl.output - assert "Skipping object 8" in cl.output - assert "Object 6: Caught exception index=97 has gone past the number of entries" in cl.output - assert "Object 5: Caught exception inner_radius must be less than radius (1.193147)" in cl.output - assert "Object 4: Caught exception Unable to evaluate string 'math.sqrt(x)'" in cl.output - assert "obj 0: reject evaluated to True" in cl.output - assert "Object 0: Rejecting this object and rebuilding" in cl.output + assert "obj 8: Skipping because field skip=True" in caplog.text + assert "obj 8: Caught SkipThisObject: e = None" in caplog.text + assert "Skipping object 8" in caplog.text + assert "Object 6: Caught exception index=97 has gone past the number of entries" in caplog.text + assert "Object 5: Caught exception inner_radius must be less than radius (1.193147)" in caplog.text + assert "Object 4: Caught exception Unable to evaluate string 'math.sqrt(x)'" in caplog.text + assert "obj 0: reject evaluated to True" in caplog.text + assert "Object 0: Rejecting this object and rebuilding" in caplog.text # This next two can end up with slightly different numerical values depending on numpy version # So use re.search rather than require an exact match. - assert re.search(r"Object 0: Measured flux = 3086.30[0-9]* < 0.95 \* 3265.226572.", cl.output) - assert re.search(r"Object 4: Measured snr = 79.888[0-9]* > 50.0.", cl.output) + assert re.search(r"Object 0: Measured flux = 3086.30[0-9]* < 0.95 \* 3265.226572.", caplog.text) + assert re.search(r"Object 4: Measured snr = 79.888[0-9]* > 50.0.", caplog.text) # 10 is quick skipped, so we don't even get a debug line for it. - assert 'Stamp 10' not in cl.output - assert 'obj 10' not in cl.output + assert 'Stamp 10' not in caplog.text + assert 'obj 10' not in caplog.text # Others all get at least that: for i in range(10): - assert 'Stamp %d'%i in cl.output - assert 'obj %d'%i in cl.output + assert 'Stamp %d'%i in caplog.text + assert 'obj %d'%i in caplog.text # For test coverage to get all branches, do min_snr and max_snr separately. del config['stamp']['max_snr'] config['stamp']['min_snr'] = 26 - with CaptureLog() as cl: - im_list2 = galsim.config.BuildStamps(nimages, config, do_noise=False, logger=cl.logger)[0] - #print(cl.output) - assert re.search(r"Object 6: Measured snr = 25.2741[0-9]* < 26.0.", cl.output) + im_list2 = galsim.config.BuildStamps(nimages, config, do_noise=False)[0] + assert re.search(r"Object 6: Measured snr = 25.2741[0-9]* < 26.0.", caplog.text) # If we lower the number of retries, we'll max out and abort the image config['stamp']['retry_failures'] = 10 @@ -549,13 +516,11 @@ def HighN(config, base, value_type): with assert_raises(galsim.GalSimConfigError): galsim.config.BuildStamps(nimages, config, do_noise=False) try: - with CaptureLog() as cl: - galsim.config.BuildStamps(nimages, config, do_noise=False, logger=cl.logger) + galsim.config.BuildStamps(nimages, config, do_noise=False) except (galsim.GalSimConfigError): pass - #print(cl.output) - assert "Object 0: Too many exceptions/rejections for this object. Aborting." in cl.output - assert "Exception caught when building stamp" in cl.output + assert "Object 0: Too many exceptions/rejections for this object. Aborting." in caplog.text + assert "Exception caught when building stamp" in caplog.text # Even lower, and we get a different limiting error. config['stamp']['retry_failures'] = 4 @@ -563,43 +528,35 @@ def HighN(config, base, value_type): with assert_raises(galsim.GalSimConfigError): galsim.config.BuildStamps(nimages, config, do_noise=False) try: - with CaptureLog() as cl: - galsim.config.BuildStamps(nimages, config, do_noise=False, logger=cl.logger) + galsim.config.BuildStamps(nimages, config, do_noise=False) except (galsim.GalSimConfigError): pass - #print(cl.output) - assert "Rejected an object 5 times. If this is expected," in cl.output - assert "Exception caught when building stamp" in cl.output + assert "Rejected an object 5 times. If this is expected," in caplog.text + assert "Exception caught when building stamp" in caplog.text # We can also do this with BuildImages which runs through a different code path. galsim.config.RemoveCurrent(config) try: - with CaptureLog() as cl: - galsim.config.BuildImages(nimages, config, logger=cl.logger) + galsim.config.BuildImages(nimages, config) except (ValueError,IndexError,galsim.GalSimError): pass - #print(cl.output) - assert "Exception caught when building image" in cl.output + assert "Exception caught when building image" in caplog.text # When in nproc > 1 mode, the error message is slightly different. config['image']['nproc'] = 2 try: - with CaptureLog() as cl: - galsim.config.BuildStamps(nimages, config, do_noise=False, logger=cl.logger) + galsim.config.BuildStamps(nimages, config, do_noise=False) except (ValueError,IndexError,galsim.GalSimError): pass - #print(cl.output) if galsim.config.UpdateNProc(2, nimages, config) > 1: - assert re.search("Process-.: Exception caught when building stamp",cl.output) + assert re.search("Process-.: Exception caught when building stamp",caplog.text) try: - with CaptureLog() as cl: - galsim.config.BuildImages(nimages, config, logger=cl.logger) + galsim.config.BuildImages(nimages, config) except (ValueError,IndexError,galsim.GalSimError): pass - #print(cl.output) if galsim.config.UpdateNProc(2, nimages, config) > 1: - assert re.search("Process-.: Exception caught when building image",cl.output) + assert re.search("Process-.: Exception caught when building image",caplog.text) # Finally, if all images give errors, BuildFiles will not raise an exception, but will just # report that no files were written. @@ -607,18 +564,14 @@ def HighN(config, base, value_type): config['root'] = 'test_reject' # This lets the code generate a file name automatically. del config['stamp']['size'] # Otherwise skipped images will still build an empty image. config = galsim.config.CleanConfig(config) - with CaptureLog() as cl: - galsim.config.BuildFiles(nimages, config, logger=cl.logger) - #print(cl.output) - assert "No files were written. All were either skipped or had errors." in cl.output + galsim.config.BuildFiles(nimages, config) + assert "No files were written. All were either skipped or had errors." in caplog.text # There is a different path if all files raise an exception, rather than are rejected. config['stamp']['type'] = 'hello' config = galsim.config.CleanConfig(config) - with CaptureLog() as cl: - galsim.config.BuildFiles(nimages, config, logger=cl.logger) - #print(cl.output) - assert "No files were written. All were either skipped or had errors." in cl.output + galsim.config.BuildFiles(nimages, config) + assert "No files were written. All were either skipped or had errors." in caplog.text # If we skip all objects, and don't have a definite size for them, then we get to a message # that no stamps were built. @@ -627,39 +580,28 @@ def HighN(config, base, value_type): galsim.config.RemoveCurrent(config) im_list3 = galsim.config.BuildStamps(nimages, config, do_noise=False)[0] assert all (im is None for im in im_list3) - with CaptureLog() as cl: - im_list3 = galsim.config.BuildStamps(nimages, config, do_noise=False, logger=cl.logger)[0] - #print(cl.output) - assert "No stamps were built. All objects were skipped." in cl.output + im_list3 = galsim.config.BuildStamps(nimages, config, do_noise=False)[0] + assert "No stamps were built. All objects were skipped." in caplog.text # Different message if nstamps=0, rather than all failures. - with CaptureLog() as cl: - galsim.config.BuildStamps(0, config, do_noise=False, logger=cl.logger)[0] - assert "No stamps were built, since nstamps == 0." in cl.output + galsim.config.BuildStamps(0, config, do_noise=False)[0] + assert "No stamps were built, since nstamps == 0." in caplog.text # Likewise with BuildImages, but with a slightly different message. - with CaptureLog() as cl: - im_list4 = galsim.config.BuildImages(nimages, config, logger=cl.logger) - assert "No images were built. All were either skipped or had errors." in cl.output + im_list4 = galsim.config.BuildImages(nimages, config) + assert "No images were built. All were either skipped or had errors." in caplog.text # Different message if nimages=0, rather than all failures. - with CaptureLog() as cl: - galsim.config.BuildImages(0, config, logger=cl.logger) - assert "No images were built, since nimages == 0." in cl.output + galsim.config.BuildImages(0, config) + assert "No images were built, since nimages == 0." in caplog.text # And BuildFiles - with CaptureLog() as cl: - galsim.config.BuildFiles(nimages, config, logger=cl.logger) - assert "No files were written. All were either skipped or had errors." in cl.output + galsim.config.BuildFiles(nimages, config) + assert "No files were written. All were either skipped or had errors." in caplog.text # Different message if nfiles=0, rather than all failures. - with CaptureLog() as cl: - galsim.config.BuildFiles(0, config, logger=cl.logger) - assert "No files were made, since nfiles == 0." in cl.output - - # Finally, with a fake logger, this covers the LoggerWrapper functionality. - logger = galsim.config.LoggerWrapper(None) - galsim.config.BuildFiles(nimages, config, logger=logger) + galsim.config.BuildFiles(0, config) + assert "No files were made, since nfiles == 0." in caplog.text # Now go back to the original config, and switch to skip_failures rather than retry. config = orig_config @@ -667,10 +609,10 @@ def HighN(config, base, value_type): # With this and retry_failures, we get an error. with assert_raises(galsim.GalSimConfigValueError): - galsim.config.BuildStamps(nimages, config, do_noise=False, logger=logger) + galsim.config.BuildStamps(nimages, config, do_noise=False) del config['stamp']['retry_failures'] - im_list = galsim.config.BuildStamps(nimages, config, do_noise=False, logger=logger)[0] + im_list = galsim.config.BuildStamps(nimages, config, do_noise=False)[0] fluxes = [im.array.sum(dtype=float) if im is not None else 0 for im in im_list] # Everything gets skipped here. np.testing.assert_almost_equal(fluxes, 0, decimal=0) @@ -681,14 +623,14 @@ def HighN(config, base, value_type): del config['stamp']['max_snr'] del config['stamp']['skip'] del config['stamp']['quick_skip'] - im_list = galsim.config.BuildStamps(nimages, config, do_noise=False, logger=logger)[0] + im_list = galsim.config.BuildStamps(nimages, config, do_noise=False)[0] fluxes = [im.array.sum(dtype=float) if im is not None else 0 for im in im_list] expected_fluxes = [0, 76673, 0, 0, 24074, 0, 0, 9124, 0, 0, 0] np.testing.assert_almost_equal(fluxes, expected_fluxes, decimal=0) @timer -def test_snr(): +def test_snr(caplog): """Test signal-to-noise option for setting the flux """ config = { @@ -742,10 +684,9 @@ def test_snr(): im2b = galsim.config.BuildImage(config) np.testing.assert_array_equal(im2b.array, im2a.array) - with CaptureLog() as cl: - im2c = galsim.config.BuildImage(config, logger=cl.logger) + im2c = galsim.config.BuildImage(config) np.testing.assert_array_equal(im2c.array, im2a.array) - assert 'signal_to_noise calculation is not accurate for draw_method = phot' in cl.output + assert 'signal_to_noise calculation is not accurate for draw_method = phot' in caplog.text @timer @@ -780,8 +721,8 @@ def test_ring(): ring_builder = galsim.config.stamp_ring.RingBuilder() for k in range(6): galsim.config.SetupConfigObjNum(config, k) - ring_builder.setup(config['stamp'], config, None, None, ignore, None) - gal1a = ring_builder.buildProfile(config['stamp'], config, None, {}, None) + ring_builder.setup(config['stamp'], config, None, None, ignore) + gal1a = ring_builder.buildProfile(config['stamp'], config, None, {}) gal1b = gauss.shear(e1=e1_list[k], e2=e2_list[k]) print('gal1a = ',gal1a) print('gal1b = ',gal1b) @@ -848,8 +789,8 @@ def test_ring(): galsim.config.SetupConfigImageNum(config, 0, 0) for k in range(25): galsim.config.SetupConfigObjNum(config, k) - ring_builder.setup(config['stamp'], config, None, None, ignore, None) - gal2a = ring_builder.buildProfile(config['stamp'], config, psf, {}, None) + ring_builder.setup(config['stamp'], config, None, None, ignore) + gal2a = ring_builder.buildProfile(config['stamp'], config, psf, {}) gal2b = disk.rotate(theta = k * 18 * galsim.degrees) gal2b = galsim.Convolve(gal2b,psf) gsobject_compare(gal2a, gal2b) @@ -882,8 +823,8 @@ def test_ring(): for k in range(25): galsim.config.SetupConfigObjNum(config, k) index = k // 4 # make sure we use integer division - ring_builder.setup(config['stamp'], config, None, None, ignore, None) - gal3a = ring_builder.buildProfile(config['stamp'], config, None, {}, None) + ring_builder.setup(config['stamp'], config, None, None, ignore) + gal3a = ring_builder.buildProfile(config['stamp'], config, None, {}) gal3b = sum.rotate(theta = index * 72 * galsim.degrees) gsobject_compare(gal3a, gal3b) @@ -912,9 +853,8 @@ def test_ring(): galsim.config.SetupConfigImageNum(config, 0, 0) galsim.config.SetupConfigObjNum(config, 0) - ring_builder.setup(config['stamp'], config, None, None, ignore, None) - gal4a = ring_builder.buildProfile(config['stamp'], config, None, config['stamp']['gsparams'], - None) + ring_builder.setup(config['stamp'], config, None, None, ignore) + gal4a = ring_builder.buildProfile(config['stamp'], config, None, config['stamp']['gsparams']) gsparams = galsim.GSParams(maxk_threshold=1.e-2, folding_threshold=1.e-2, stepk_minimum_hlr=3) disk = galsim.Exponential(half_light_radius=2, gsparams=gsparams).shear(e2=0.3) bulge = galsim.Sersic(n=3,half_light_radius=1.3, gsparams=gsparams).shear(e1=0.12,e2=-0.08) @@ -1144,9 +1084,9 @@ def test_scattered_noskip(): class NoSkipStampBuilder(galsim.config.StampBuilder): def quickSkip(self, config, base): return False - def getSkip(self, config, base, logger): + def getSkip(self, config, base): return False - def updateSkip(self, prof, image, method, offset, config, base, logger): + def updateSkip(self, prof, image, method, offset, config, base): return False galsim.config.RegisterStampType('NoSkip', NoSkipStampBuilder()) @@ -1204,10 +1144,12 @@ def updateSkip(self, prof, image, method, offset, config, base, logger): @timer -def test_scattered_whiten(): +def test_scattered_whiten(caplog): """Test whitening with the image type Scattered. In particular getting the noise flattened across overlapping stamps and stamps that are partially off the image. """ + caplog.set_level(logging.INFO) + real_gal_dir = os.path.join('..','examples','data') real_gal_cat = 'real_galaxy_catalog_23.5_example.fits' scale = 0.05 @@ -1304,14 +1246,12 @@ def test_scattered_whiten(): np.testing.assert_almost_equal(im2.array, im1.array) # Should give a warning for the objects that fall off the edge - # Note: CaptureLog doesn't work correctly in multiprocessing for some reason. - # I haven't figured out what about the implementation fails, but it prints these - # just fine when using a regular logger with nproc=2. Oh well. + # Note: caplog doesn't work correctly in multiprocessing; + # cf. https://github.com/pytest-dev/pytest/issues/3037. + # It prints these just fine when using a regular logger with nproc=2. Oh well. config['image']['nproc'] = 1 - with CaptureLog() as cl: - im3 = galsim.config.BuildImage(config, logger=cl.logger) - #print(cl.output) - assert "skip drawing object because its image will be entirely off the main image." in cl.output + im3 = galsim.config.BuildImage(config) + assert "skip drawing object because its image will be entirely off the main image." in caplog.text im2 = galsim.config.BuildImage(config) @@ -1659,15 +1599,13 @@ def test_njobs(): } config1 = galsim.config.CopyConfig(config) - logger = logging.getLogger('test_njobs') - logger.addHandler(logging.StreamHandler(sys.stdout)) - galsim.config.Process(config, logger=logger) + galsim.config.Process(config) # Repeat with 2 jobs config = galsim.config.CopyConfig(config1) config['output']['file_name']['root'] = 'test_two_jobs_' - galsim.config.Process(config, njobs=2, job=1, logger=logger) - galsim.config.Process(config, njobs=2, job=2, logger=logger) + galsim.config.Process(config, njobs=2, job=1) + galsim.config.Process(config, njobs=2, job=2) # Check that the images are equal: one00 = galsim.fits.read('test_one_job_00.fits', dir='output') @@ -1685,7 +1623,7 @@ def test_njobs(): config = galsim.config.CopyConfig(config1) config['rng'] = object() with assert_raises(galsim.GalSimConfigError): - galsim.config.ProcessInput(config, logger=logger, safe_only=False) + galsim.config.ProcessInput(config, safe_only=False) @timer @@ -1972,7 +1910,7 @@ def test_wcs(): # Base class usage is invalid builder = galsim.config.wcs.WCSBuilder() - assert_raises(NotImplementedError, builder.buildWCS, config, config, logger=None) + assert_raises(NotImplementedError, builder.buildWCS, config, config) @timer @@ -2062,7 +2000,7 @@ def test_bandpass(): # Base class usage is invalid builder = galsim.config.bandpass.BandpassBuilder() - assert_raises(NotImplementedError, builder.buildBandpass, config, config, logger=None) + assert_raises(NotImplementedError, builder.buildBandpass, config, config) @timer @@ -2078,20 +2016,13 @@ def test_index_key(run_slow): # First generate using the config layer. config = galsim.config.ReadConfig('config_input/index_key.yaml')[0] - if run_slow: - logger = logging.getLogger('test_index_key') - logger.addHandler(logging.StreamHandler(sys.stdout)) - logger.setLevel(logging.DEBUG) - else: - logger = None # Normal sequential config1 = galsim.config.CopyConfig(config) # Note: Using BuildFiles(config1) would normally work, but it has an extra copy internally, # which messes up some of the current checks later. for n in range(nfiles): - galsim.config.BuildFile(config1, file_num=n, image_num=n*nimages, obj_num=n*n_per_file, - logger=logger) + galsim.config.BuildFile(config1, file_num=n, image_num=n*nimages, obj_num=n*n_per_file) images1 = [ galsim.fits.readMulti('output/index_key%02d.fits'%n) for n in range(nfiles) ] if run_slow: @@ -2103,16 +2034,14 @@ def test_index_key(run_slow): config2 = galsim.config.CopyConfig(config) config2['output']['nproc'] = nfiles for n in range(nfiles): - galsim.config.BuildFile(config2, file_num=n, image_num=n*nimages, obj_num=n*n_per_file, - logger=logger) + galsim.config.BuildFile(config2, file_num=n, image_num=n*nimages, obj_num=n*n_per_file) images2 = [ galsim.fits.readMulti('output/index_key%02d.fits'%n) for n in range(nfiles) ] # Multiprocessing images config3 = galsim.config.CopyConfig(config) config3['image']['nproc'] = nfiles for n in range(nfiles): - galsim.config.BuildFile(config3, file_num=n, image_num=n*nimages, obj_num=n*n_per_file, - logger=logger) + galsim.config.BuildFile(config3, file_num=n, image_num=n*nimages, obj_num=n*n_per_file) images3 = [ galsim.fits.readMulti('output/index_key%02d.fits'%n) for n in range(nfiles) ] # New config for each file @@ -2121,8 +2050,7 @@ def test_index_key(run_slow): galsim.config.SetupConfigFileNum(config4[n], n, n*nimages, n*n_per_file) galsim.config.SetupConfigRNG(config4[n]) images4 = [ galsim.config.BuildImages(nimages, config4[n], - image_num=n*nimages, obj_num=n*n_per_file, - logger=logger) + image_num=n*nimages, obj_num=n*n_per_file) for n in range(nfiles) ] # New config for each image @@ -2133,8 +2061,7 @@ def test_index_key(run_slow): images5 = [ [ galsim.config.BuildImage(galsim.config.CopyConfig(config5[n]), image_num=n*nimages+i, - obj_num=n*n_per_file + i*n_per_image, - logger=logger) + obj_num=n*n_per_file + i*n_per_image) for i in range(nimages) ] for n in range(nfiles) ] @@ -2273,13 +2200,9 @@ def test_multirng(run_slow): if run_slow: nimages = 6 ngals = 20 - logger = logging.getLogger('test_multirng') - logger.addHandler(logging.StreamHandler(sys.stdout)) - #logger.setLevel(logging.DEBUG) else: nimages = 3 ngals = 3 - logger = None # First generate using the config layer. config = galsim.config.ReadConfig('config_input/multirng.yaml')[0] @@ -2288,7 +2211,7 @@ def test_multirng(run_slow): config2 = galsim.config.CopyConfig(config) config3 = galsim.config.CopyConfig(config) - images1 = galsim.config.BuildImages(nimages, config1, logger=logger) + images1 = galsim.config.BuildImages(nimages, config1) config2['image']['nproc'] = 6 images2 = galsim.config.BuildImages(nimages, config2) images3 = [ galsim.config.BuildImage(galsim.config.CopyConfig(config), @@ -2403,12 +2326,9 @@ def test_sequential_seeds(run_slow): if run_slow: nimages = 6 ngals = 20 - logger = logging.getLogger('test_sequential_seeds') - logger.addHandler(logging.StreamHandler(sys.stdout)) else: nimages = 3 ngals = 3 - logger = None config = galsim.config.ReadConfig('config_input/sequential_seeds.yaml')[0] config['image']['nobjects'] = ngals @@ -2419,7 +2339,7 @@ def test_sequential_seeds(run_slow): # the same as for the previous image. config1 = galsim.config.CopyConfig(config) config1['eval_variables']['iid'] = n - stamps, current_vars = galsim.config.BuildStamps(ngals, config1, logger=logger) + stamps, current_vars = galsim.config.BuildStamps(ngals, config1) assert len(stamps) == ngals all_stamps.append(stamps) @@ -2431,7 +2351,7 @@ def test_sequential_seeds(run_slow): @timer -def test_template(): +def test_template(caplog): """Test various uses of the template keyword """ # Use the multirng.yaml config file from the above test as a convenient template source @@ -2551,12 +2471,10 @@ def test_template(): config7 = config1.copy() config7['image.random_seed.0'] = "" config7['image.random_seed.0.str'] = '123 + (image_num//3) * @image.nobjects' - with CaptureLog() as cl: - galsim.config.ProcessAllTemplates(config7, cl.logger) + galsim.config.ProcessAllTemplates(config7) assert config7['image']['random_seed'][0]['type'] == 'Eval' assert config7['image']['random_seed'][0]['str'] == '123 + (image_num//3) * @image.nobjects' - print(cl.output) - assert "Removing item 0 from image.random_seed." in cl.output + assert "Removing item 0 from image.random_seed." in caplog.text # Read a template config from a file config8 = galsim.config.ReadConfig('config_input/template.yaml')[0] @@ -2598,9 +2516,11 @@ def test_template(): @timer -def test_variable_cat_size(): +def test_variable_cat_size(caplog): """Test that some automatic nitems calculations work with variable input catalog sizes """ + caplog.set_level(logging.DEBUG) + config = { 'gal': { 'type': 'Gaussian', @@ -2641,20 +2561,16 @@ def test_variable_cat_size(): config1 = galsim.config.CopyConfig(config) # This input isn't safe, so it can't load when doing the safe_only load. - with CaptureLog() as cl: - galsim.config.ProcessInput(config, safe_only=True, logger=cl.logger) - assert "Skip catalog 0, since not safe" in cl.output + galsim.config.ProcessInput(config, safe_only=True) + assert "Skip catalog 0, since not safe" in caplog.text - logger = logging.getLogger('test_variable_input') - logger.addHandler(logging.StreamHandler(sys.stdout)) - #logger.setLevel(logging.DEBUG) cfg_images = [] galsim.config.SetupConfigFileNum(config, 0, 0, 0) - galsim.config.ProcessInput(config, logger=logger) - cfg_images.append(galsim.config.BuildImage(config, 0, 0, logger=logger)) + galsim.config.ProcessInput(config) + cfg_images.append(galsim.config.BuildImage(config, 0, 0)) galsim.config.SetupConfigFileNum(config, 1, 1, 3) - galsim.config.ProcessInput(config, logger=logger) - cfg_images.append(galsim.config.BuildImage(config, 1, 3, logger=logger)) + galsim.config.ProcessInput(config) + cfg_images.append(galsim.config.BuildImage(config, 1, 3)) # Build by hand to compare ref_images = [] @@ -2674,7 +2590,7 @@ def test_variable_cat_size(): assert cfg_images[1] == ref_images[1] # Now run with full Process function - galsim.config.Process(config1, logger=logger) + galsim.config.Process(config1) cfg_images2 = galsim.fits.readMulti('output/test_variable_input.fits') assert cfg_images2[0] == ref_images[0] assert cfg_images2[1] == ref_images[1] @@ -2686,15 +2602,15 @@ class BlendSetBuilder(galsim.config.StampBuilder): GSObject for its "prof". """ - def setup(self, config, base, xsize, ysize, ignore, logger): + def setup(self, config, base, xsize, ysize, ignore): """Do the appropriate setup for a Blend stamp. """ self.ngal = galsim.config.ParseValue(config, 'n_neighbors', base, int)[0] + 1 self.sep = galsim.config.ParseValue(config, 'sep', base, float)[0] ignore = ignore + ['n_neighbors', 'sep'] - return super(self.__class__, self).setup(config, base, xsize, ysize, ignore, logger) + return super(self.__class__, self).setup(config, base, xsize, ysize, ignore) - def buildProfile(self, config, base, psf, gsparams, logger): + def buildProfile(self, config, base, psf, gsparams): """ Build a list of galaxy profiles, each convolved with the psf, to use for the blend image. """ @@ -2703,19 +2619,18 @@ def buildProfile(self, config, base, psf, gsparams, logger): else: self.neighbor_gals = [] for i in range(self.ngal-1): - gal = galsim.config.BuildGSObject(base, 'gal', gsparams=gsparams, logger=logger)[0] + gal = galsim.config.BuildGSObject(base, 'gal', gsparams=gsparams)[0] self.neighbor_gals.append(gal) galsim.config.RemoveCurrent(base['gal'], keep_safe=True) - rng = galsim.config.GetRNG(config, base, logger, 'BlendSet') + rng = galsim.config.GetRNG(config, base, 'BlendSet') ud = galsim.UniformDeviate(rng) self.neighbor_pos = [galsim.PositionI(int(ud()*2*self.sep-self.sep), int(ud()*2*self.sep-self.sep)) for i in range(self.ngal-1)] #print('neighbor positions = ',self.neighbor_pos) - self.main_gal = galsim.config.BuildGSObject(base, 'gal', gsparams=gsparams, - logger=logger)[0] + self.main_gal = galsim.config.BuildGSObject(base, 'gal', gsparams=gsparams)[0] self.profiles = [ self.main_gal ] self.profiles += [ g.shift(p) for g, p in zip(self.neighbor_gals, self.neighbor_pos) ] @@ -2723,7 +2638,7 @@ def buildProfile(self, config, base, psf, gsparams, logger): self.profiles = [ galsim.Convolve(gal, psf) for gal in self.profiles ] return self.profiles - def draw(self, profiles, image, method, offset, config, base, logger): + def draw(self, profiles, image, method, offset, config, base): nx = base['stamp_xsize'] ny = base['stamp_ysize'] wcs = base['wcs'] @@ -2737,8 +2652,7 @@ def draw(self, profiles, image, method, offset, config, base, logger): self.full_images = [] for prof in profiles: im = galsim.ImageF(bounds=bounds, wcs=wcs) - galsim.config.DrawBasic(prof, im, method, offset-im.true_center, config, base, - logger) + galsim.config.DrawBasic(prof, im, method, offset-im.true_center, config, base) self.full_images.append(im) k = base['obj_num'] % self.ngal @@ -3262,7 +3176,7 @@ def test_photon_ops(): with assert_raises(galsim.GalSimConfigError): galsim.config.BuildPhotonOp(config, 'photon_op', config) with assert_raises(NotImplementedError): - galsim.config.photon_ops.PhotonOpBuilder().buildPhotonOp(config,config,None) + galsim.config.photon_ops.PhotonOpBuilder().buildPhotonOp(config,config) del config['photon_ops_orig'][1]['sed'] with assert_raises(galsim.GalSimConfigError): galsim.config.BuildPhotonOp(config['photon_ops_orig'], 1, config) @@ -3439,7 +3353,7 @@ def test_sensor(): with assert_raises(galsim.GalSimConfigError): galsim.config.BuildSensor(config, 'gal', config) with assert_raises(NotImplementedError): - galsim.config.sensor.SensorBuilder().buildSensor(config,config,None) + galsim.config.sensor.SensorBuilder().buildSensor(config,config) config['sensor8']['index'] = 1 with assert_raises(galsim.GalSimConfigError): galsim.config.BuildSensor(config, 'sensor8', config) diff --git a/tests/test_config_input.py b/tests/test_config_input.py index 070a5ad645..3bd013352e 100644 --- a/tests/test_config_input.py +++ b/tests/test_config_input.py @@ -57,25 +57,23 @@ def test_input_init(): }, } - logger = logging.getLogger('test_fits') - logger.addHandler(logging.StreamHandler(sys.stdout)) - logger.setLevel(logging.DEBUG) - first_seed = galsim.BaseDeviate(1234).raw() ud = galsim.UniformDeviate(first_seed + 0 + 1) sigma = ud() + 1. gal = galsim.Gaussian(sigma=sigma, flux=100) im1 = gal.drawImage(scale=1, nx=55, ny=45) - galsim.config.Process(config, logger=logger) + galsim.config.Process(config) file_name = 'output_fits/test_fits.fits' im2 = galsim.fits.read(file_name) np.testing.assert_array_equal(im2.array, im1.array) @timer -def test_approx_nobjects(): +def test_approx_nobjects(caplog): """Test the getApproxNObjects functionality. """ + caplog.set_level(logging.INFO) + class BigCatalog(galsim.Catalog): def getApproxNObjects(self): return 2*self.getNObjects() @@ -177,11 +175,10 @@ def GenerateFromSmallCatalog(config, base, value_type): config3['image']['image_pos']['y']['type'] = 'SmallCatalog' config3['input']['small_catalog'] = galsim.config.CopyConfig(config['input']['catalog']) del config3['input']['catalog'] - with CaptureLog() as cl: - galsim.config.Process(config3, logger=cl.logger) + galsim.config.Process(config3) # The real numbers are 3 and 5, but the small version guesses them to be 1 and 2 respectively. - assert 'Input small_catalog has approximately 1' in cl.output - assert 'Input small_catalog has approximately 2' in cl.output + assert 'Input small_catalog has approximately 1' in caplog.text + assert 'Input small_catalog has approximately 2' in caplog.text images3 = galsim.fits.readMulti('output/test_approx_nobj.fits') assert images3[0] == images1[0] @@ -216,7 +213,7 @@ def getGSScreenShare(self): def getPSF(self): return self.atm.makePSF(500, aper=self.aper) - def BuildAtmPSF(config, base, ignore, gsparams, logger): + def BuildAtmPSF(config, base, ignore, gsparams): atm = galsim.config.GetInputObj('atm_psf', config, base, 'AtmPSF') return atm.getPSF(), True @@ -289,7 +286,7 @@ def BuildAtmPSF(config, base, ignore, gsparams, logger): worker_initargs=galsim.phase_screens.initWorkerArgs) @timer -def test_dependent_inputs(): +def test_dependent_inputs(caplog): """Test inputs that depend on other inputs. imSim has input types that depend on other input types. If these are listed in order, @@ -298,6 +295,8 @@ def test_dependent_inputs(): if it raises an exception that indicates it needs a different input type, try to load that one first. """ + caplog.set_level(logging.INFO) + class Dict1: def __init__(self): self.d = {'a': 1, 'b': 2} @@ -347,9 +346,8 @@ def DepItem(config, base, value_type): }, } } - with CaptureLog() as cl: - galsim.config.ProcessInput(config, cl.logger) - assert 'input seems to depend on' not in cl.output + galsim.config.ProcessInput(config) + assert 'input seems to depend on' not in caplog.text dep = galsim.config.GetInputObj('dep', config, config, 'Dep') assert dep.d == dict(a=1, b=2, c=1, d=2) @@ -366,10 +364,9 @@ def DepItem(config, base, value_type): 'dict2': {}, } } - with CaptureLog() as cl: - galsim.config.ProcessInput(config, cl.logger) - assert 'dep input seems to depend on dict1' in cl.output - assert 'dep input seems to depend on dict2' in cl.output + galsim.config.ProcessInput(config) + assert 'dep input seems to depend on dict1' in caplog.text + assert 'dep input seems to depend on dict2' in caplog.text dep = galsim.config.GetInputObj('dep', config, config, 'Dep') assert dep.d == dict(a=1, b=2, c=1, d=2) @@ -406,30 +403,30 @@ def Dict3Item(config, base, value_type): } } with np.testing.assert_raises(galsim.config.GalSimConfigError): - galsim.config.ProcessInput(config, cl.logger) + galsim.config.ProcessInput(config) # But with safe_only=True, it doesn't raise an exception config = galsim.config.CleanConfig(config) - with CaptureLog() as cl: - galsim.config.ProcessInput(config, cl.logger, safe_only=True) + galsim.config.ProcessInput(config, safe_only=True) - assert 'dep input seems to depend on dict1' in cl.output - assert 'dep input seems to depend on dict2' not in cl.output # Doesn't get to the Dict2Item - assert 'dep input seems to depend on dict3' in cl.output + assert 'dep input seems to depend on dict1' in caplog.text + # assert 'dep input seems to depend on dict2' not in caplog.text # Doesn't get to the Dict2Item + assert 'dep input seems to depend on dict2. Try loading that' in caplog.text # TODO + assert 'dep input seems to depend on dict3' in caplog.text dep = galsim.config.GetInputObj('dep', config, config, 'Dep') assert dep is None # If the dependency graph is circular, make sure we don't get an infinite loop. config = galsim.config.CleanConfig(config) config['input']['dict3']['e'] = {'type': 'DepItem', 'key': 'a'} - galsim.config.ProcessInput(config, cl.logger, safe_only=True) + galsim.config.ProcessInput(config, safe_only=True) dep = galsim.config.GetInputObj('dep', config, config, 'Dep') assert dep is None # Finally, just make sure that if dict3 isn't broken, it all works as expected. config = galsim.config.CleanConfig(config) config['input']['dict3']['e']['type'] = 'Dict1Item' - galsim.config.ProcessInput(config, cl.logger) + galsim.config.ProcessInput(config) dict3 = galsim.config.GetInputObj('dict3', config, config, 'Dict3') dep = galsim.config.GetInputObj('dep', config, config, 'Dep') print('dict3.d = ',dict3.d) diff --git a/tests/test_config_noise.py b/tests/test_config_noise.py index 50c47d1636..206eb151bf 100644 --- a/tests/test_config_noise.py +++ b/tests/test_config_noise.py @@ -19,7 +19,6 @@ import numpy as np import os import sys -import logging import math import galsim @@ -88,7 +87,7 @@ def test_gaussian(): # Base class usage is invalid builder = galsim.config.noise.NoiseBuilder() assert_raises(NotImplementedError, builder.addNoise, config, config, im1a, rng, var, - draw_method='auto', logger=None) + draw_method='auto') assert_raises(NotImplementedError, builder.getNoiseVariance, config, config) @@ -275,18 +274,11 @@ def test_poisson(): def test_ccdnoise(): """Test that the config layer CCD noise adds noise consistent with using a CCDNoise object. """ - import logging - gain = 4 sky = 50 rn = 5 size = 2048 - # Use this to turn on logging, but more info than we noramlly need, so generally leave it off. - #logging.basicConfig(format="%(message)s", level=logging.DEBUG, stream=sys.stdout) - #logger = logging.getLogger() - logger = None - config = {} config['gal'] = { 'type' : 'None' } config['image'] = { @@ -301,7 +293,7 @@ def test_ccdnoise(): 'gain' : gain, 'read_noise' : rn } - image = galsim.config.BuildImage(config,logger=logger) + image = galsim.config.BuildImage(config) print('config-built image: ',np.mean(image.array),np.var(image.array.astype(float))) test_var = np.var(image.array.astype(float)) @@ -346,7 +338,7 @@ def test_ccdnoise(): image2.fill(0) rng.reset(first_seed+1) config['image_num_rng'] = rng - galsim.config.AddNoise(config, image2, current_var=1.e-20, logger=logger) + galsim.config.AddNoise(config, image2, current_var=1.e-20) print('with negligible current_var: ',np.mean(image2.array),np.var(image2.array)) np.testing.assert_almost_equal(np.var(image2.array),test_var, @@ -356,7 +348,7 @@ def test_ccdnoise(): image2.fill(0) gn = galsim.GaussianNoise(rng=rng, sigma=rn/gain) image2.addNoise(gn) - galsim.config.AddNoise(config, image2, current_var=(rn/gain)**2, logger=logger) + galsim.config.AddNoise(config, image2, current_var=(rn/gain)**2) print('current_var == read_noise: ',np.mean(image2.array),np.var(image2.array)) # So far we've done this to very high accuracy, since we've been using the same rng seed, @@ -369,7 +361,7 @@ def test_ccdnoise(): image2.fill(0) gn = galsim.GaussianNoise(rng=rng, sigma=0.5*rn/gain) image2.addNoise(gn) - galsim.config.AddNoise(config, image2, current_var=(0.5*rn/gain)**2, logger=logger) + galsim.config.AddNoise(config, image2, current_var=(0.5*rn/gain)**2) print('current_var < read_noise: ',np.mean(image2.array),np.var(image2.array)) np.testing.assert_almost_equal(np.var(image2.array),test_var, decimal=1, @@ -379,7 +371,7 @@ def test_ccdnoise(): image2.fill(0) gn = galsim.GaussianNoise(rng=rng, sigma=2.*rn/gain) image2.addNoise(gn) - galsim.config.AddNoise(config, image2, current_var=(2.*rn/gain)**2, logger=logger) + galsim.config.AddNoise(config, image2, current_var=(2.*rn/gain)**2) print('current_var > read_noise',np.mean(image2.array),np.var(image2.array)) np.testing.assert_almost_equal(np.var(image2.array),test_var, decimal=1, @@ -513,10 +505,6 @@ def test_ccdnoise_phot(): def test_cosmosnoise(): """Test that the config layer COSMOS noise works with keywords. """ - import logging - - logger = None - pix_scale = 0.03 random_seed = 123 @@ -535,7 +523,7 @@ def test_cosmosnoise(): config['image']['noise'] = { 'type' : 'COSMOS' } - image = galsim.config.BuildStamp(config,logger=logger)[0] + image = galsim.config.BuildStamp(config)[0] # Then make it using explicit kwargs to make sure they are getting passed through properly. config2 = {} @@ -551,7 +539,7 @@ def test_cosmosnoise(): 'file_name' : os.path.join(galsim.meta_data.share_dir,'acs_I_unrot_sci_20_cf.fits'), 'cosmos_scale' : pix_scale } - image2 = galsim.config.BuildStamp(config2,logger=logger)[0] + image2 = galsim.config.BuildStamp(config2)[0] # We used the same RNG and noise file / properties, so should get the same exact noise field. np.testing.assert_allclose( @@ -566,7 +554,7 @@ def test_cosmosnoise(): 'file_name' : os.path.join(galsim.meta_data.share_dir,'acs_I_unrot_sci_20_cf.fits'), 'pixel_scale' : pix_scale } - image3 = galsim.config.BuildStamp(config3,logger=logger)[0] + image3 = galsim.config.BuildStamp(config3)[0] np.testing.assert_allclose(image3.array, image2.array, err_msg='Config Correlated noise not the same as COSMOS') @@ -590,7 +578,7 @@ def test_cosmosnoise(): } config['image']['noise']['whiten'] = True galsim.config.ProcessInput(config) - image3, current_var3 = galsim.config.BuildStamp(config, logger=logger) + image3, current_var3 = galsim.config.BuildStamp(config) print('From BuildStamp, current_var = ',current_var3) # Build the same image by hand to make sure it matches what config drew. diff --git a/tests/test_config_output.py b/tests/test_config_output.py index a462030c21..8ed8eb39e2 100644 --- a/tests/test_config_output.py +++ b/tests/test_config_output.py @@ -35,9 +35,11 @@ @timer -def test_fits(): +def test_fits(caplog): """Test the default output type = Fits """ + caplog.set_level(logging.DEBUG) + # Most of the tests in this file write to the 'output' directory. Here we write to a different # directory and make sure that it properly creates the directory if necessary. if os.path.exists('output_fits'): @@ -59,9 +61,6 @@ def test_fits(): }, } - logger = logging.getLogger('test_fits') - logger.addHandler(logging.StreamHandler(sys.stdout)) - logger.setLevel(logging.DEBUG) config1 = galsim.config.CopyConfig(config) im1_list = [] @@ -74,7 +73,7 @@ def test_fits(): im1 = gal.drawImage(scale=1) im1_list.append(im1) - galsim.config.BuildFile(config, file_num=k, image_num=k, obj_num=k, logger=logger) + galsim.config.BuildFile(config, file_num=k, image_num=k, obj_num=k) file_name = 'output_fits/test_fits_%d.fits'%k im2 = galsim.fits.read(file_name) np.testing.assert_array_equal(im2.array, im1.array) @@ -111,16 +110,14 @@ def test_fits(): # nproc < 0 should automatically determine nproc from ncpu config = galsim.config.CopyConfig(config1) - with CaptureLog() as cl: - galsim.config.Process(config, logger=cl.logger, new_params={'output.nproc' : -1}) - assert 'ncpu = ' in cl.output + galsim.config.Process(config, new_params={'output.nproc' : -1}) + assert 'ncpu = ' in caplog.text # nproc > njobs should drop back to nproc = njobs config = galsim.config.CopyConfig(config1) - with CaptureLog() as cl: - galsim.config.Process(config, logger=cl.logger, new_params={'output.nproc' : 10}) + galsim.config.Process(config, new_params={'output.nproc' : 10}) if galsim.config.UpdateNProc(10, 6, config) > 1: - assert 'There are only 6 jobs to do. Reducing nproc to 6' in cl.output + assert 'There are only 6 jobs to do. Reducing nproc to 6' in caplog.text # There is a feature that we reduce the number of tasks to be < 32767 to avoid problems # with the multiprocessing.Queue overflowing. That 32767 number is a settable paramter, @@ -129,11 +126,9 @@ def test_fits(): galsim.config.util.max_queue_size = 4 config = galsim.config.CopyConfig(config1) config['output']['nproc'] = 2 - with CaptureLog() as cl: - galsim.config.Process(config, logger=cl.logger, new_params={'output.nproc' : 2}) - print(cl.output) + galsim.config.Process(config, new_params={'output.nproc' : 2}) if galsim.config.UpdateNProc(10, 6, config) > 1: - assert 'len(tasks) = 6 is more than max_queue_size = 4' in cl.output + assert 'len(tasks) = 6 is more than max_queue_size = 4' in caplog.text for k in range(nfiles): file_name = 'output_fits/test_fits_%d.fits'%k im2 = galsim.fits.read(file_name) @@ -144,16 +139,9 @@ def test_fits(): # (The single-thread profiling is handled by the galsim executable, which we don't # bother testing here.) config = galsim.config.CopyConfig(config1) - with CaptureLog() as cl: - galsim.config.Process(config, logger=cl.logger, - new_params={'profile':True, 'output.nproc': 4}) - #print(cl.output) - # Unfortunately, the LoggerProxy doesn't really work right with the string logger used - # by CaptureLog. I tried for a while to figure out how to get it to capture the proxied - # logs and couldn't get it working. So this just checks for an info log before the - # multithreading starts. But with a regular logger, there really is profiling output. + galsim.config.Process(config, new_params={'profile':True, 'output.nproc': 4}) if galsim.config.UpdateNProc(10, 6, config) > 1: - assert "Starting separate profiling for each of the" in cl.output + assert "Starting separate profiling for each of the" in caplog.text for p in range(4): pstats_file = f'galsim-Process-{p+1}.pstats' assert os.path.exists(pstats_file) @@ -204,9 +192,8 @@ def test_fits(): # mock this up to make sure we handle it properly (by reverting to nproc = 1. with mock.patch('galsim.config.util.cpu_count', side_effect=RuntimeError()): config = galsim.config.CopyConfig(config1) - with CaptureLog() as cl: - galsim.config.Process(config, logger=cl.logger, new_params={'output.nproc' : -1}) - assert 'Using single process' in cl.output + galsim.config.Process(config, new_params={'output.nproc' : -1}) + assert 'Using single process' in caplog.text @timer @@ -374,7 +361,7 @@ def test_datacube(): @timer -def test_skip(): +def test_skip(caplog): """Test the skip and noclobber options """ config = { @@ -426,11 +413,10 @@ def test_skip(): # Build the ones we skipped using noclobber option del config['output']['skip'] config['output']['noclobber'] = True - with CaptureLog() as cl: - galsim.config.Process(config, logger=cl.logger) - assert "Skipping file 1 = output/test_skip_1.fits because output.noclobber" in cl.output - assert "Skipping file 3 = output/test_skip_3.fits because output.noclobber" in cl.output - assert "Skipping file 5 = output/test_skip_5.fits because output.noclobber" in cl.output + galsim.config.Process(config) + assert "Skipping file 1 = output/test_skip_1.fits because output.noclobber" in caplog.text + assert "Skipping file 3 = output/test_skip_3.fits because output.noclobber" in caplog.text + assert "Skipping file 5 = output/test_skip_5.fits because output.noclobber" in caplog.text for k in range(nfiles): file_name = 'output/test_skip_%d.fits'%k im2 = galsim.fits.read(file_name) @@ -450,10 +436,9 @@ def test_skip(): im2 = galsim.fits.read(file_name) np.testing.assert_array_equal(im2.array, im1_list[k].array) - with CaptureLog() as cl: - galsim.config.Process(config, njobs=3, job=3, logger=cl.logger) - assert "Splitting work into 3 jobs. Doing job 3" in cl.output - assert "Building 2 out of 6 total files: file_num = 4 .. 5" in cl.output + galsim.config.Process(config, njobs=3, job=3) + assert "Splitting work into 3 jobs. Doing job 3" in caplog.text + assert "Building 2 out of 6 total files: file_num = 4 .. 5" in caplog.text # job < 1 or job > njobs is invalid with assert_raises(galsim.GalSimValueError): @@ -466,9 +451,11 @@ def test_skip(): @timer -def test_extra_wt(): +def test_extra_wt(caplog): """Test the extra weight and badpix fields """ + caplog.set_level(logging.DEBUG) + nfiles = 6 config = { 'image' : { @@ -505,9 +492,8 @@ def test_extra_wt(): config['noise'] = { 'type' : 'Poisson', 'sky_level_pixel' : 500 } config['output']['noclobber'] = True galsim.config.RemoveCurrent(config) - with CaptureLog() as cl: - galsim.config.Process(config, logger=cl.logger) - assert 'Not writing weight file 0 = output/test_wt_0.fits' in cl.output + galsim.config.Process(config) + assert 'Not writing weight file 0 = output/test_wt_0.fits' in caplog.text for k in range(nfiles): im = galsim.fits.read('output/test_main_%d.fits'%k) np.testing.assert_equal(im.array, main_im[k].array) @@ -639,24 +625,23 @@ def test_extra_wt(): # If both output.nproc and image.nproc, then only use output.nproc config['image']['nproc' ] = -1 config['image']['nobjects'] = 5 - with CaptureLog() as cl: - galsim.config.Process(config, logger=cl.logger) - #print(cl.output) - #assert 'Already multiprocessing. Ignoring image.nproc' in cl.output - # Note: This doesn't show up because cl.logger doesn't get through the multiprocessing, + galsim.config.Process(config) + # assert 'Already multiprocessing. Ignoring image.nproc' in caplog.text + # Note: This doesn't show up because the log doesn't get through the multiprocessing, # but it does ignore image.nproc > 1. # Do it manually to confirm. config['current_nproc'] = 2 - with CaptureLog() as cl: - nproc = galsim.config.UpdateNProc(2, 5, config, logger=cl.logger) - assert 'Already multiprocessing. Ignoring image.nproc' in cl.output + nproc = galsim.config.UpdateNProc(2, 5, config) + assert 'Already multiprocessing. Ignoring image.nproc' in caplog.text assert nproc == 1 @timer -def test_extra_psf(): +def test_extra_psf(caplog): """Test the extra psf field """ + caplog.set_level(logging.INFO) + nfiles = 6 config = { 'image' : { @@ -848,13 +833,12 @@ def test_extra_psf(): im2 = galsim.fits.read('output_psf/test_psf.fits') np.testing.assert_almost_equal(im2.array, im.array) - with CaptureLog() as cl: - galsim.config.Process(config, logger=cl.logger) - assert "Not writing psf file 1 = output_psf/test_psf.fits because already written" in cl.output - assert "Not writing psf file 2 = output_psf/test_psf.fits because already written" in cl.output - assert "Not writing psf file 3 = output_psf/test_psf.fits because already written" in cl.output - assert "Not writing psf file 4 = output_psf/test_psf.fits because already written" in cl.output - assert "Not writing psf file 5 = output_psf/test_psf.fits because already written" in cl.output + galsim.config.Process(config) + assert "Not writing psf file 1 = output_psf/test_psf.fits because already written" in caplog.text + assert "Not writing psf file 2 = output_psf/test_psf.fits because already written" in caplog.text + assert "Not writing psf file 3 = output_psf/test_psf.fits because already written" in caplog.text + assert "Not writing psf file 4 = output_psf/test_psf.fits because already written" in caplog.text + assert "Not writing psf file 5 = output_psf/test_psf.fits because already written" in caplog.text @timer def test_extra_psf_sn(): @@ -941,7 +925,7 @@ def test_extra_psf_sn(): @timer -def test_extra_truth(): +def test_extra_truth(caplog): """Test the extra truth field """ nobjects = 6 @@ -1077,10 +1061,9 @@ def test_extra_truth(): config['output']['truth']['columns']['beta'] = ( '$0. if @gal.index==0 else (@gal.items.1.ellip).beta') del config['image']['nproc'] - with CaptureLog(level=1) as cl: - with assert_raises(galsim.GalSimConfigError): - galsim.config.Process(config, logger=cl.logger) - assert "beta has type Angle, but previously had type float" in cl.output + with assert_raises(galsim.GalSimConfigError): + galsim.config.Process(config) + assert "beta has type Angle, but previously had type float" in caplog.text config['output']['truth']['columns']['beta'] = ( '$0. if @gal.index==0 else (@gal.items.1.ellip).beta.rad') @@ -1098,9 +1081,11 @@ def test_extra_truth(): assert 'Consider using an explicit value-typed type name like Random_float' in str(e) @timer -def test_retry_io(): +def test_retry_io(caplog): """Test the retry_io option """ + caplog.set_level(logging.DEBUG) + # Make a class that mimics writeMulti, except that it fails about 1/3 of the time. class FlakyWriter: def __init__(self, rng): @@ -1114,13 +1099,13 @@ def writeFile(self, *args, **kwargs): # Now make a copy of Fits and ExtraWeight using this writer. class FlakyFits(galsim.config.OutputBuilder): - def writeFile(self, data, file_name, config, base, logger): + def writeFile(self, data, file_name, config, base): flaky_writer = FlakyWriter(galsim.config.GetRNG(config,base)) flaky_writer.writeFile(data, file_name) galsim.config.RegisterOutputType('FlakyFits', FlakyFits()) class FlakyWeight(galsim.config.extra_weight.WeightBuilder): - def writeFile(self, file_name, config, base, logger): + def writeFile(self, file_name, config, base): flaky_writer = FlakyWriter(galsim.config.GetRNG(config,base)) flaky_writer.writeFile(self.final_data, file_name) galsim.config.RegisterExtraOutput('flaky_weight', FlakyWeight()) @@ -1147,15 +1132,13 @@ def writeFile(self, file_name, config, base, logger): }, } - with CaptureLog() as cl: - galsim.config.Process(config, logger=cl.logger) - #print(cl.output) - assert "File output/test_flaky_fits_0.fits: Caught OSError" in cl.output - assert "This is try 2/6, so sleep for 2 sec and try again." in cl.output - assert "file 0: Wrote FlakyFits to file 'output/test_flaky_fits_0.fits'" in cl.output - assert "File output/test_flaky_wt_4.fits: Caught OSError: " in cl.output - assert "This is try 1/6, so sleep for 1 sec and try again." in cl.output - assert "file 0: Wrote flaky_weight to 'output/test_flaky_wt_0.fits'" in cl.output + galsim.config.Process(config) + assert "File output/test_flaky_fits_0.fits: Caught OSError" in caplog.text + assert "This is try 2/6, so sleep for 2 sec and try again." in caplog.text + assert "file 0: Wrote FlakyFits to file 'output/test_flaky_fits_0.fits'" in caplog.text + assert "File output/test_flaky_wt_4.fits: Caught OSError: " in caplog.text + assert "This is try 1/6, so sleep for 1 sec and try again." in caplog.text + assert "file 0: Wrote flaky_weight to 'output/test_flaky_wt_0.fits'" in caplog.text # Now the regular versions. config2 = galsim.config.CopyConfig(config) @@ -1178,52 +1161,48 @@ def writeFile(self, file_name, config, base, logger): # Without retry_io, it will fail, but keep going del config['output']['retry_io'] galsim.config.RemoveCurrent(config) - with CaptureLog() as cl: - galsim.config.Process(config, logger=cl.logger) - #print(cl.output) - assert "Exception caught for file 0 = output/test_flaky_fits_0.fits" in cl.output - assert "File output/test_flaky_fits_0.fits not written! Continuing on..." in cl.output - assert "file 1: Wrote FlakyFits to file 'output/test_flaky_fits_1.fits'" in cl.output - assert "File 1 = output/test_flaky_fits_1.fits" in cl.output - assert "File 2 = output/test_flaky_fits_2.fits" in cl.output - assert "File 3 = output/test_flaky_fits_3.fits" in cl.output - assert "Exception caught for file 4 = output/test_flaky_fits_4.fits" in cl.output - assert "File output/test_flaky_fits_4.fits not written! Continuing on..." in cl.output - assert "File 5 = output/test_flaky_fits_5.fits" in cl.output + galsim.config.Process(config) + assert "Exception caught for file 0 = output/test_flaky_fits_0.fits" in caplog.text + assert "File output/test_flaky_fits_0.fits not written! Continuing on..." in caplog.text + assert "file 1: Wrote FlakyFits to file 'output/test_flaky_fits_1.fits'" in caplog.text + assert "File 1 = output/test_flaky_fits_1.fits" in caplog.text + assert "File 2 = output/test_flaky_fits_2.fits" in caplog.text + assert "File 3 = output/test_flaky_fits_3.fits" in caplog.text + assert "Exception caught for file 4 = output/test_flaky_fits_4.fits" in caplog.text + assert "File output/test_flaky_fits_4.fits not written! Continuing on..." in caplog.text + assert "File 5 = output/test_flaky_fits_5.fits" in caplog.text # Also works in nproc > 1 mode config['output']['nproc'] = 2 galsim.config.RemoveCurrent(config) - with CaptureLog() as cl: - galsim.config.Process(config, logger=cl.logger) - #print(cl.output) + galsim.config.Process(config) if galsim.config.UpdateNProc(2, nfiles, config) > 1: assert re.search("Process-.: Exception caught for file 0 = output/test_flaky_fits_0.fits", - cl.output) - assert "File output/test_flaky_fits_0.fits not written! Continuing on..." in cl.output - assert re.search("Process-.: File 1 = output/test_flaky_fits_1.fits", cl.output) - assert re.search("Process-.: File 2 = output/test_flaky_fits_2.fits", cl.output) - assert re.search("Process-.: File 3 = output/test_flaky_fits_3.fits", cl.output) + caplog.text) + assert "File output/test_flaky_fits_0.fits not written! Continuing on..." in caplog.text + assert re.search("Process-.: File 1 = output/test_flaky_fits_1.fits", caplog.text) + assert re.search("Process-.: File 2 = output/test_flaky_fits_2.fits", caplog.text) + assert re.search("Process-.: File 3 = output/test_flaky_fits_3.fits", caplog.text) assert re.search("Process-.: Exception caught for file 4 = output/test_flaky_fits_4.fits", - cl.output) - assert "File output/test_flaky_fits_4.fits not written! Continuing on..." in cl.output - assert re.search("Process-.: File 5 = output/test_flaky_fits_5.fits", cl.output) + caplog.text) + assert "File output/test_flaky_fits_4.fits not written! Continuing on..." in caplog.text + assert re.search("Process-.: File 5 = output/test_flaky_fits_5.fits", caplog.text) # But with except_abort = True, it will stop after the first failure del config['output']['nproc'] # Otherwise which file fails in non-deterministic. - with CaptureLog() as cl: - try: - galsim.config.Process(config, logger=cl.logger, except_abort=True) - except OSError as e: - assert str(e) == "p = 0.285159" - #print(cl.output) - assert "File output/test_flaky_fits_0.fits not written." in cl.output + try: + galsim.config.Process(config, except_abort=True) + except OSError as e: + assert str(e) == "p = 0.285159" + assert "File output/test_flaky_fits_0.fits not written." in caplog.text @timer -def test_config(): +def test_config(caplog): """Test that configuration files are read, copied, and merged correctly. """ + caplog.set_level(logging.INFO) + config = { 'gal' : { 'type' : 'Gaussian', 'sigma' : 2.3, 'flux' : { 'type' : 'List', 'items' : [ 100, 500, 1000 ] } }, @@ -1266,10 +1245,9 @@ def test_config(): # Merging identical dicts, should do nothing galsim.config.MergeConfig(config1,config2) assert config == dict(config1) - with CaptureLog() as cl: - galsim.config.MergeConfig(config1,config2,logger=cl.logger) - assert "Not merging key type from the base config" in cl.output - assert "Not merging key items from the base config" in cl.output + galsim.config.MergeConfig(config1,config2) + assert "Not merging key type from the base config" in caplog.text + assert "Not merging key items from the base config" in caplog.text # Merging different configs does something, with the first taking precedence on conflicts del config5['gal'] @@ -1552,10 +1530,7 @@ def test_eval_full_word(): } } - logger = logging.getLogger('test_eval_full_word') - logger.addHandler(logging.StreamHandler(sys.stdout)) - logger.setLevel(logging.DEBUG) - galsim.config.Process(config, logger=logger, except_abort=True) + galsim.config.Process(config, except_abort=True) # First check the truth catalogs data0 = np.genfromtxt('output/test_eval_full_word_0.dat', names=True, deletechars='') @@ -1612,7 +1587,7 @@ def test_eval_full_word(): assert np.max(np.abs(im10.array)) > 200 @timer -def test_timeout(): +def test_timeout(caplog): """Test the timeout option """ config = { @@ -1647,20 +1622,16 @@ def test_timeout(): }, } - logger = logging.getLogger('test_timeout') - logger.addHandler(logging.StreamHandler(sys.stdout)) - #logger.setLevel(logging.DEBUG) - # Single proc: config1 = galsim.config.CopyConfig(config) - galsim.config.Process(config1, logger=logger) + galsim.config.Process(config1) # nproc in output field. config2 = galsim.config.CopyConfig(config) config2['output']['nproc'] = 3 config2['output']['timeout'] = 30 # Still plenty large enough not to timeout. config2['output']['file_name'] = "$'output/test_timeout_nproc1_%d.fits'%file_num" - galsim.config.Process(config2, logger=logger) + galsim.config.Process(config2) for n in range(6): im1 = galsim.fits.read('output/test_timeout_%d.fits'%n) im2 = galsim.fits.read('output/test_timeout_nproc1_%d.fits'%n) @@ -1671,17 +1642,16 @@ def test_timeout(): if platform.python_implementation() != 'PyPy': config2 = galsim.config.CleanConfig(config2) config2['output']['timeout'] = 0.0001 - with CaptureLog() as cl: - with assert_raises(galsim.GalSimError): - galsim.config.Process(config2, logger=cl.logger) - assert 'Multiprocessing timed out waiting for a task to finish.' in cl.output + with assert_raises(galsim.GalSimError): + galsim.config.Process(config2) + assert 'Multiprocessing timed out waiting for a task to finish.' in caplog.text # nproc in image field. config2 = galsim.config.CopyConfig(config) config2['image']['nproc'] = 3 config2['image']['timeout'] = 30 config2['output']['file_name'] = "$'output/test_timeout_nproc2_%d.fits'%file_num" - galsim.config.Process(config2, logger=logger) + galsim.config.Process(config2) for n in range(6): im1 = galsim.fits.read('output/test_timeout_%d.fits'%n) im2 = galsim.fits.read('output/test_timeout_nproc2_%d.fits'%n) @@ -1690,7 +1660,7 @@ def test_timeout(): # If you use BuildImages, it uses the image nproc and timeout specs, but parallelizes # over images rather than stamps. So check that. config2 = galsim.config.CleanConfig(config2) - images = galsim.config.BuildImages(6, config2, logger=logger) + images = galsim.config.BuildImages(6, config2) for n, im in enumerate(images): im1 = galsim.fits.read('output/test_timeout_%d.fits'%n) assert im1 == im @@ -1700,20 +1670,18 @@ def test_timeout(): # This time, it will continue on after each error, but report the error in the log. config2 = galsim.config.CleanConfig(config2) config2['image']['timeout'] = 0.001 - with CaptureLog() as cl: - galsim.config.Process(config2, logger=cl.logger) - assert 'Multiprocessing timed out waiting for a task to finish.' in cl.output + galsim.config.Process(config2) + assert 'Multiprocessing timed out waiting for a task to finish.' in caplog.text # Note: Usually they all fail, and the two lines below are in the logging output, but # it's possible for one of them to finish, so these asserts occasionally fail. - #assert 'File output/test_timeout_nproc2_1.fits not written! Continuing on...' in cl.output - #assert 'No files were written. All were either skipped or had errors.' in cl.output + #assert 'File output/test_timeout_nproc2_1.fits not written! Continuing on...' in caplog.text + #assert 'No files were written. All were either skipped or had errors.' in caplog.text # If you want this to abort, use except_abort=True config2 = galsim.config.CleanConfig(config2) - with CaptureLog() as cl: - with assert_raises(galsim.GalSimError): - galsim.config.Process(config2, logger=cl.logger, except_abort=True) - assert 'Multiprocessing timed out waiting for a task to finish.' in cl.output + with assert_raises(galsim.GalSimError): + galsim.config.Process(config2, except_abort=True) + assert 'Multiprocessing timed out waiting for a task to finish.' in caplog.text @timer def test_direct_extra_output(): diff --git a/tests/test_config_value.py b/tests/test_config_value.py index a3a3df0ba4..3c26e95e7b 100644 --- a/tests/test_config_value.py +++ b/tests/test_config_value.py @@ -16,6 +16,7 @@ # and/or other materials provided with the distribution. # +import logging import numpy as np import astropy.units as u import math @@ -23,11 +24,12 @@ import galsim from galsim_test_helpers import * - @timer -def test_float_value(): +def test_float_value(caplog): """Test various ways to generate a float value """ + caplog.set_level(logging.DEBUG) + halo_mass = 1.e14 halo_conc = 4 halo_z = 0.3 @@ -358,7 +360,7 @@ def test_float_value(): np.testing.assert_almost_equal(sum2, sum([ 0, 1, 2, 4, 6])) # Test NFWHaloMagnification - galsim.config.SetupInputsForImage(config, None) + galsim.config.SetupInputsForImage(config) # Raise an error because no uv_pos with assert_raises(galsim.GalSimConfigError): galsim.config.ParseValue(config,'nfw',config, float) @@ -378,11 +380,9 @@ def test_float_value(): config['uv_pos'] = galsim.PositionD(0.1,0.3) print("strong lensing mag = ",nfw_halo.getMagnification((0.1,0.3), gal_z)) galsim.config.RemoveCurrent(config) - with CaptureLog() as cl: - galsim.config.SetupInputsForImage(config, cl.logger) - nfw2 = galsim.config.ParseValue(config, 'nfw', config, float)[0] - print(cl.output) - assert "Warning: NFWHalo mu = 249.374050 means strong lensing." in cl.output + galsim.config.SetupInputsForImage(config) + nfw2 = galsim.config.ParseValue(config, 'nfw', config, float)[0] + assert "Warning: NFWHalo mu = 249.374050 means strong lensing." in caplog.text np.testing.assert_almost_equal(nfw2, 25.) # Or set a different maximum @@ -397,11 +397,9 @@ def test_float_value(): galsim.config.RemoveCurrent(config) config['uv_pos'] = galsim.PositionD(0.1,0.2) print("very strong lensing mag = ",nfw_halo.getMagnification((0.1,0.2), gal_z)) - with CaptureLog() as cl: - galsim.config.SetupInputsForImage(config, cl.logger) - nfw4 = galsim.config.ParseValue(config, 'nfw', config, float)[0] - print(cl.output) - assert "Warning: NFWHalo mu = -163.631846 means strong lensing." in cl.output + galsim.config.SetupInputsForImage(config) + nfw4 = galsim.config.ParseValue(config, 'nfw', config, float)[0] + assert "Warning: NFWHalo mu = -163.631846 means strong lensing." in caplog.text np.testing.assert_almost_equal(nfw4, 3000.) # Negative max_mu is invalid. @@ -418,7 +416,7 @@ def test_float_value(): config['rng'] = rng.duplicate() ps.buildGrid(grid_spacing=10, ngrid=21, interpolant='linear', rng=rng) print("ps mag = ",ps.getMagnification((0.1,0.2))) - galsim.config.SetupInputsForImage(config, None) + galsim.config.SetupInputsForImage(config) ps1 = galsim.config.ParseValue(config,'ps',config, float)[0] np.testing.assert_almost_equal(ps1, ps.getMagnification((0.1,0.2))) @@ -429,22 +427,18 @@ def test_float_value(): print("strong lensing mag = ",ps.getMagnification((0.1,0.2))) config = galsim.config.CleanConfig(config) config['input']['power_spectrum']['e_power_function'] = '2000 * np.exp(-k**0.2)' - with CaptureLog() as cl: - galsim.config.SetupInputsForImage(config, logger=cl.logger) - ps2a = galsim.config.ParseValue(config,'ps',config, float)[0] - print(cl.output) - assert 'PowerSpectrum mu = -4.335137 means strong lensing. Using mu=25.000000' in cl.output + galsim.config.SetupInputsForImage(config) + ps2a = galsim.config.ParseValue(config,'ps',config, float)[0] + assert 'PowerSpectrum mu = -4.335137 means strong lensing. Using mu=25.000000' in caplog.text np.testing.assert_almost_equal(ps2a, 25.) # Need a different point that happens to have strong lensing, since the PS realization changed. ps.buildGrid(grid_spacing=10, ngrid=21, interpolant='linear', rng=rng) config['uv_pos'] = galsim.PositionD(55,-25) galsim.config.RemoveCurrent(config) - with CaptureLog() as cl: - galsim.config.SetupInputsForImage(config, logger=cl.logger) - ps2b = galsim.config.ParseValue(config, 'ps', config, float)[0] - print(cl.output) - assert "PowerSpectrum mu = 26.746296 means strong lensing. Using mu=25.000000" in cl.output + galsim.config.SetupInputsForImage(config) + ps2b = galsim.config.ParseValue(config, 'ps', config, float)[0] + assert "PowerSpectrum mu = 26.746296 means strong lensing. Using mu=25.000000" in caplog.text np.testing.assert_almost_equal(ps2b, 25.) # Or set a different maximum @@ -465,12 +459,10 @@ def test_float_value(): # Out of bounds results in shear = 0, and a warning. galsim.config.RemoveCurrent(config) config['uv_pos'] = galsim.PositionD(1000,2000) - with CaptureLog() as cl: - galsim.config.SetupInputsForImage(config, cl.logger) - ps2c = galsim.config.ParseValue(config, 'ps', config, float)[0] - print(cl.output) + galsim.config.SetupInputsForImage(config) + ps2c = galsim.config.ParseValue(config, 'ps', config, float)[0] assert ("Extrapolating beyond input range. galsim.PositionD(x=1000.0, y=2000.0) not in " - "galsim.BoundsD") in cl.output + "galsim.BoundsD") in caplog.text np.testing.assert_almost_equal(ps2c, 1.) # Error if no uv_pos @@ -1216,9 +1208,11 @@ def test_angle_value(): @timer -def test_shear_value(): +def test_shear_value(caplog): """Test various ways to generate a Shear value """ + caplog.set_level(logging.DEBUG) + halo_mass = 1.e14 halo_conc = 4 halo_z = 0.3 @@ -1336,7 +1330,7 @@ def test_shear_value(): # Test NFWHaloShear galsim.config.ProcessInput(config) - galsim.config.SetupInputsForImage(config, None) + galsim.config.SetupInputsForImage(config) # Raise an error because no uv_pos with assert_raises(galsim.GalSimConfigError): galsim.config.ParseValue(config, 'nfw', config, galsim.Shear) @@ -1358,11 +1352,9 @@ def test_shear_value(): galsim.config.RemoveCurrent(config) config['uv_pos'] = galsim.PositionD(0.1,0.2) print("strong lensing shear = ",nfw_halo.getShear((0.1,0.2), gal_z)) - with CaptureLog() as cl: - galsim.config.SetupInputsForImage(config, cl.logger) - nfw2a = galsim.config.ParseValue(config, 'nfw', config, galsim.Shear)[0] - print(cl.output) - assert "Warning: NFWHalo shear (g1=1.148773, g2=-1.531697) is invalid." in cl.output + galsim.config.SetupInputsForImage(config) + nfw2a = galsim.config.ParseValue(config, 'nfw', config, galsim.Shear)[0] + assert "Warning: NFWHalo shear (g1=1.148773, g2=-1.531697) is invalid." in caplog.text np.testing.assert_almost_equal((nfw2a.g1, nfw2a.g2), (0,0)) # Test PowerSpectrumShear @@ -1374,7 +1366,7 @@ def test_shear_value(): config['image_xsize'] = config['image_ysize'] = 2000 config['wcs'] = galsim.PixelScale(0.1) config['image_center'] = galsim.PositionD(0,0) - galsim.config.SetupInputsForImage(config, None) + galsim.config.SetupInputsForImage(config) ps1a = galsim.config.ParseValue(config,'ps',config, galsim.Shear)[0] ps1b = ps.getShear((0.1,0.2)) print("ps shear= ",ps1b) @@ -1388,25 +1380,22 @@ def test_shear_value(): print("strong lensing shear = ",ps.getShear((0.1,0.2))) config = galsim.config.CleanConfig(config) config['input']['power_spectrum']['e_power_function'] = '500 * np.exp(-k**0.2)' - galsim.config.SetupInputsForImage(config, None) + galsim.config.SetupInputsForImage(config) ps2b = ps.getShear((0.1,0.2)) print("ps shear= ",ps2b) - with CaptureLog() as cl: - galsim.config.SetupInputsForImage(config, logger=cl.logger) - ps2a = galsim.config.ParseValue(config,'ps',config, galsim.Shear)[0] - assert 'PowerSpectrum shear (g1=-1.626101, g2=0.287082) is invalid. Using shear = 0.' in cl.output + galsim.config.SetupInputsForImage(config) + ps2a = galsim.config.ParseValue(config,'ps',config, galsim.Shear)[0] + assert 'PowerSpectrum shear (g1=-1.626101, g2=0.287082) is invalid. Using shear = 0.' in caplog.text np.testing.assert_almost_equal((ps2a.g1, ps2a.g2), (0,0)) # Out of bounds results in shear = 0, and a warning. galsim.config.RemoveCurrent(config) config['uv_pos'] = galsim.PositionD(1000,2000) - with CaptureLog() as cl: - galsim.config.SetupInputsForImage(config, cl.logger) - ps2c = galsim.config.ParseValue(config, 'ps', config, galsim.Shear)[0] - print(cl.output) + galsim.config.SetupInputsForImage(config) + ps2c = galsim.config.ParseValue(config, 'ps', config, galsim.Shear)[0] assert ("Extrapolating beyond input range. galsim.PositionD(x=1000.0, y=2000.0) not in " "galsim.BoundsD(xmin=-190.00000000000023, xmax=200.00000000000023, " - "ymin=-190.00000000000023, ymax=200.00000000000023)") in cl.output + "ymin=-190.00000000000023, ymax=200.00000000000023)") in caplog.text np.testing.assert_almost_equal((ps2c.g1, ps2c.g2), (0,0)) # Error if no uv_pos @@ -1882,7 +1871,7 @@ def test_eval(): config['image'] = { 'random_seed' : 1234 } rng = galsim.BaseDeviate(galsim.BaseDeviate(1234).raw()) galsim.config.ProcessInput(config) - galsim.config.SetupInputsForImage(config, None) + galsim.config.SetupInputsForImage(config) ps = galsim.PowerSpectrum(e_power_function = lambda k: np.exp(-k**0.2), b_power_function = lambda k: np.exp(-k**1.2)) # ngrid is calculated from the image size by config, which was setup above. diff --git a/tests/test_des.py b/tests/test_des.py index f5316a2804..03a5a0132f 100644 --- a/tests/test_des.py +++ b/tests/test_des.py @@ -326,7 +326,7 @@ def get_offset(obj_num): import logging logging.basicConfig(format="%(message)s", level=logging.WARN, stream=sys.stdout) logger = logging.getLogger('test_meds_config') - galsim.config.BuildFile(galsim.config.CopyConfig(config), logger=logger) + galsim.config.BuildFile(galsim.config.CopyConfig(config)) # Add in badpix and offset so we run both with and without options. config = galsim.config.CleanConfig(config) @@ -336,7 +336,7 @@ def get_offset(obj_num): config['output']['weight'] = {} config['output']['psf'] = {} config['output']['meds_get_offset'] = {} - galsim.config.BuildFile(galsim.config.CopyConfig(config), logger=logger) + galsim.config.BuildFile(galsim.config.CopyConfig(config)) # Scattered image is invalid with MEDS output config = galsim.config.CleanConfig(config) @@ -347,7 +347,7 @@ def get_offset(obj_num): 'size' : stamp_size , } with assert_raises(galsim.GalSimConfigError): - galsim.config.BuildFile(galsim.config.CopyConfig(config), logger=logger) + galsim.config.BuildFile(galsim.config.CopyConfig(config)) # Now repeat, making a separate file for each config = galsim.config.CleanConfig(config) @@ -369,7 +369,7 @@ def get_offset(obj_num): 'stamp_size' : stamp_size, 'random_seed' : seed } - galsim.config.Process(galsim.config.CopyConfig(config), logger=logger) + galsim.config.Process(galsim.config.CopyConfig(config)) try: import meds diff --git a/tests/test_download.py b/tests/test_download.py index 0998b4aec0..7edfdb0444 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -161,12 +161,6 @@ def delay_n(): with assert_raises(ValueError): query_yes_no('', 'invalid') -# Need to call these before each time make_logger is repeated. Else duplicate handles. -def remove_handler(): - logger = logging.getLogger('galsim') - for handler in logger.handlers: - logger.removeHandler(handler) - @timer def test_names(): """Test the get_names function @@ -174,9 +168,7 @@ def test_names(): from galsim.download_cosmos import get_names args = galsim.download_cosmos.parse_args([]) - remove_handler() - logger = galsim.download_cosmos.make_logger(args) - url, target, target_dir, link_dir, unpack_dir, do_link = get_names(args, logger) + url, target, target_dir, link_dir, unpack_dir, do_link = get_names(args) assert url == 'https://zenodo.org/record/3242143/files/COSMOS_25.2_training_sample.tar.gz' assert target_dir == galsim.meta_data.share_dir assert do_link is False @@ -185,7 +177,7 @@ def test_names(): assert link_dir == os.path.join(galsim.meta_data.share_dir, 'COSMOS_25.2_training_sample') args = galsim.download_cosmos.parse_args(['-d','~/share','-s','23.5']) - url, target, target_dir, link_dir, unpack_dir, do_link = get_names(args, logger) + url, target, target_dir, link_dir, unpack_dir, do_link = get_names(args) assert url == 'https://zenodo.org/record/3242143/files/COSMOS_23.5_training_sample.tar.gz' assert target_dir == os.path.expanduser('~/share') assert do_link is True @@ -194,7 +186,7 @@ def test_names(): assert link_dir == os.path.join(galsim.meta_data.share_dir, 'COSMOS_23.5_training_sample') args = galsim.download_cosmos.parse_args(['-d','share','--nolink']) - url, target, target_dir, link_dir, unpack_dir, do_link = get_names(args, logger) + url, target, target_dir, link_dir, unpack_dir, do_link = get_names(args) assert url == 'https://zenodo.org/record/3242143/files/COSMOS_25.2_training_sample.tar.gz' assert target_dir == 'share' assert do_link is False @@ -258,18 +250,16 @@ def test_check(): from galsim.download_cosmos import get_names, get_meta, check_existing args = galsim.download_cosmos.parse_args(['-d','fake_cosmos','-q','-v','3']) - remove_handler() - logger = galsim.download_cosmos.make_logger(args) - url, target, target_dir, link_dir, unpack_dir, do_link = get_names(args, logger) + url, target, target_dir, link_dir, unpack_dir, do_link = get_names(args) # Check get_meta with mock.patch('galsim.download_cosmos.urlopen', fake_urlopen): - meta = get_meta(url, args, logger) + meta = get_meta(url, args) assert meta['Content-Length'] == "728" assert meta['Content-MD5'] == "e05cfe60c037c645d61ac70545cc2a99" # File already exists and is current. - do_download = check_existing(target, unpack_dir, meta, args, logger) + do_download = check_existing(target, unpack_dir, meta, args) assert do_download is False # Some changes imply it's obsolete @@ -280,22 +270,22 @@ def test_check(): meta['X-RateLimit-Remaining'] = "31" meta['Retry-After'] = "120" meta['Set-Cookie'] = "session=2b720f14bdd71a29031a5cb415b391f8" - do_download = check_existing(target, unpack_dir, meta, args, logger) + do_download = check_existing(target, unpack_dir, meta, args) assert do_download is False # Force download anyway args.quiet = False args.force = True - do_download = check_existing(target, unpack_dir, meta, args, logger) + do_download = check_existing(target, unpack_dir, meta, args) assert do_download is True # Ask whether to re-download args.force = False with mock.patch('galsim.download_cosmos.get_input', return_value='y'): - do_download = check_existing(target, unpack_dir, meta, args, logger) + do_download = check_existing(target, unpack_dir, meta, args) assert do_download is True with mock.patch('galsim.download_cosmos.get_input', return_value='n'): - do_download = check_existing(target, unpack_dir, meta, args, logger) + do_download = check_existing(target, unpack_dir, meta, args) assert do_download is False # Meta data is obsolete @@ -303,7 +293,7 @@ def test_check(): meta1['Content-Length'] = "9999" meta1['Content-MD5'] = "f05cfe60c037c645d61ac70545cc2a99" args.quiet = True - do_download = check_existing(target, unpack_dir, meta1, args, logger) + do_download = check_existing(target, unpack_dir, meta1, args) assert do_download is True # If they change the name of the checksum key, we consider it obsolete. @@ -311,73 +301,73 @@ def test_check(): meta2['Content-New-MD5'] = "e05cfe60c037c645d61ac70545cc2a99" del meta2['Content-MD5'] del meta2['ETag'] - do_download = check_existing(target, unpack_dir, meta2, args, logger) + do_download = check_existing(target, unpack_dir, meta2, args) assert do_download is True args.quiet = False args.force = True - do_download = check_existing(target, unpack_dir, meta1, args, logger) + do_download = check_existing(target, unpack_dir, meta1, args) assert do_download is True args.force = False with mock.patch('galsim.download_cosmos.get_input', return_value='y'): - do_download = check_existing(target, unpack_dir, meta1, args, logger) + do_download = check_existing(target, unpack_dir, meta1, args) assert do_download is True with mock.patch('galsim.download_cosmos.get_input', return_value='n'): - do_download = check_existing(target, unpack_dir, meta1, args, logger) + do_download = check_existing(target, unpack_dir, meta1, args) assert do_download is False # Meta data is missing args.quiet = True - do_download = check_existing(target, 'output', meta, args, logger) + do_download = check_existing(target, 'output', meta, args) assert do_download is True # Tarball is present, but wrong size args = galsim.download_cosmos.parse_args(['-d','fake_cosmos','-s','23.5','-q']) - url, target, target_dir, link_dir, unpack_dir, do_link = get_names(args, logger) - do_download = check_existing(target, 'output', meta1, args, logger) + url, target, target_dir, link_dir, unpack_dir, do_link = get_names(args) + do_download = check_existing(target, 'output', meta1, args) assert do_download is True args.quiet = False args.force = True - do_download = check_existing(target, 'output', meta1, args, logger) + do_download = check_existing(target, 'output', meta1, args) assert do_download is True args.force = False with mock.patch('galsim.download_cosmos.get_input', return_value='y'): - do_download = check_existing(target, unpack_dir, meta1, args, logger) + do_download = check_existing(target, unpack_dir, meta1, args) assert do_download is True with mock.patch('galsim.download_cosmos.get_input', return_value='n'): - do_download = check_existing(target, unpack_dir, meta1, args, logger) + do_download = check_existing(target, unpack_dir, meta1, args) assert do_download is False # Tarball is present, and correct size args.quiet = True - do_download = check_existing(target, 'output', meta, args, logger) + do_download = check_existing(target, 'output', meta, args) assert do_download is False args.quiet = False args.force = True - do_download = check_existing(target, 'output', meta, args, logger) + do_download = check_existing(target, 'output', meta, args) assert do_download is True args.force = False with mock.patch('galsim.download_cosmos.get_input', return_value='y'): - do_download = check_existing(target, unpack_dir, meta, args, logger) + do_download = check_existing(target, unpack_dir, meta, args) assert do_download is True with mock.patch('galsim.download_cosmos.get_input', return_value='n'): - do_download = check_existing(target, unpack_dir, meta, args, logger) + do_download = check_existing(target, unpack_dir, meta, args) assert do_download is False # Tarball and unpack_dir both missing args = galsim.download_cosmos.parse_args(['-d','input']) - url, target, target_dir, link_dir, unpack_dir, do_link = get_names(args, logger) - do_download = check_existing(target, unpack_dir, meta, args, logger) + url, target, target_dir, link_dir, unpack_dir, do_link = get_names(args) + do_download = check_existing(target, unpack_dir, meta, args) assert do_download is True @timer -def test_download(): +def test_download(caplog): """Test the download function This one is a little silly. It's almost completely mocked. But we can at least check @@ -386,43 +376,39 @@ def test_download(): from galsim.download_cosmos import get_names, get_meta, download args = galsim.download_cosmos.parse_args(['-d','output','-q']) - remove_handler() - logger = galsim.download_cosmos.make_logger(args) - url, target, target_dir, link_dir, unpack_dir, do_link = get_names(args, logger) + url, target, target_dir, link_dir, unpack_dir, do_link = get_names(args) with mock.patch('galsim.download_cosmos.urlopen', fake_urlopen): - meta = get_meta(url, args, logger) + meta = get_meta(url, args) print('Start download with verbosity = 2') - download(True, url, target, meta, args, logger) + download(True, url, target, meta, args) print('Start download with verbosity = 1') args.verbosity = 1 - download(True, url, target, meta, args, logger) + download(True, url, target, meta, args) print('Start download with verbosity = 3') args.verbosity = 3 - download(True, url, target, meta, args, logger) + download(True, url, target, meta, args) print('Start download with verbosity = 0') args.verbosity = 0 - download(True, url, target, meta, args, logger) + download(True, url, target, meta, args) print("Don't download") - download(False, url, target, meta, args, logger) + download(False, url, target, meta, args) fake_urlopen.err = 'Permission denied' - with CaptureLog() as cl: - assert_raises(OSError, download, True, url, target, meta, args, cl.logger) - assert "Rerun using sudo" in cl.output + assert_raises(OSError, download, True, url, target, meta, args) + assert "Rerun using sudo" in caplog.text fake_urlopen.err = 'Disk quota exceeded' - with CaptureLog() as cl: - assert_raises(OSError, download, True, url, target, meta, args, cl.logger) - assert "You might need to download this in an alternate location" in cl.output + assert_raises(OSError, download, True, url, target, meta, args) + assert "You might need to download this in an alternate location" in caplog.text fake_urlopen.err = 'gack' - assert_raises(OSError, download, True, url, target, meta, args, logger) + assert_raises(OSError, download, True, url, target, meta, args) fake_urlopen.err = None @timer @@ -433,9 +419,7 @@ def test_unpack(): # If we downloaded the file, then we usually want to unpack args = galsim.download_cosmos.parse_args(['-d','fake_cosmos','-s','23.5','-q']) - remove_handler() - logger = galsim.download_cosmos.make_logger(args) - url, target, target_dir, link_dir, unpack_dir, do_link = get_names(args, logger) + url, target, target_dir, link_dir, unpack_dir, do_link = get_names(args) meta = fake_urlopen(url).info() print('unpack_dir = ',unpack_dir) @@ -452,20 +436,20 @@ def test_unpack(): # Now unpack it print('unpack with verbose = 2:') - unpack(True, target, target_dir, unpack_dir, meta, args, logger) + unpack(True, target, target_dir, unpack_dir, meta, args) shutil.rmtree(unpack_dir) print('unpack with verbose = 3:') args.verbosity = 3 - unpack(True, target, target_dir, unpack_dir, meta, args, logger) + unpack(True, target, target_dir, unpack_dir, meta, args) shutil.rmtree(unpack_dir) print('unpack with verbose = 1:') args.verbosity = 1 - unpack(True, target, target_dir, unpack_dir, meta, args, logger) + unpack(True, target, target_dir, unpack_dir, meta, args) print("Don't unpack") - unpack(False, target, target_dir, unpack_dir, meta, args, logger) + unpack(False, target, target_dir, unpack_dir, meta, args) # If it is already unpacked, probably don't unpack it do_unpack = check_unpack(False, unpack_dir, target, args) @@ -502,9 +486,7 @@ def test_remove(): from galsim.download_cosmos import get_names, check_remove, remove_tarball args = galsim.download_cosmos.parse_args(['-d','fake_cosmos','-s','23.5','-q']) - remove_handler() - logger = galsim.download_cosmos.make_logger(args) - url, target, target_dir, link_dir, unpack_dir, do_link = get_names(args, logger) + url, target, target_dir, link_dir, unpack_dir, do_link = get_names(args) # Normally, we remove the tarball if we unpacked it. do_remove = check_remove(True, target, args) @@ -533,9 +515,9 @@ def test_remove(): with open(target1,'w') as f: f.write('blah') assert os.path.isfile(target1) - remove_tarball(False, target1, logger) + remove_tarball(False, target1) assert os.path.isfile(target1) - remove_tarball(True, target1, logger) + remove_tarball(True, target1) assert not os.path.isfile(target1) @@ -546,40 +528,38 @@ def test_link(): from galsim.download_cosmos import get_names, make_link args = galsim.download_cosmos.parse_args(['-d','fake_cosmos','-q']) - remove_handler() - logger = galsim.download_cosmos.make_logger(args) - url, target, target_dir, link_dir, unpack_dir, do_link = get_names(args, logger) + url, target, target_dir, link_dir, unpack_dir, do_link = get_names(args) link_dir = os.path.join('output', 'COSMOS_25.2_training_sample') # If link doesn't exist yet, make it. if os.path.lexists(link_dir): os.unlink(link_dir) assert not os.path.lexists(link_dir) - make_link(True, unpack_dir, link_dir, args, logger) + make_link(True, unpack_dir, link_dir, args) assert os.path.exists(link_dir) assert os.path.islink(link_dir) # If link already exists, remove and remake - make_link(True, unpack_dir, link_dir, args, logger) + make_link(True, unpack_dir, link_dir, args) assert os.path.exists(link_dir) assert os.path.islink(link_dir) # If do_link is False, don't make it os.unlink(link_dir) - make_link(False, unpack_dir, link_dir, args, logger) + make_link(False, unpack_dir, link_dir, args) assert not os.path.exists(link_dir) # If link exists, but is a directory, don't remove it os.mkdir(link_dir) assert os.path.isdir(link_dir) - make_link(True, unpack_dir, link_dir, args, logger) + make_link(True, unpack_dir, link_dir, args) assert os.path.exists(link_dir) assert os.path.isdir(link_dir) assert not os.path.islink(link_dir) # Unless force args.force = True - make_link(True, unpack_dir, link_dir, args, logger) + make_link(True, unpack_dir, link_dir, args) assert os.path.exists(link_dir) assert os.path.islink(link_dir) @@ -589,11 +569,11 @@ def test_link(): os.unlink(link_dir) os.mkdir(link_dir) with mock.patch('galsim.download_cosmos.get_input', return_value='n'): - make_link(True, unpack_dir, link_dir, args, logger) + make_link(True, unpack_dir, link_dir, args) assert os.path.isdir(link_dir) assert not os.path.islink(link_dir) with mock.patch('galsim.download_cosmos.get_input', return_value='y'): - make_link(True, unpack_dir, link_dir, args, logger) + make_link(True, unpack_dir, link_dir, args) assert os.path.islink(link_dir) # If it's a broken link, remove and relink @@ -602,7 +582,7 @@ def test_link(): assert os.path.lexists(link_dir) assert os.path.islink(link_dir) assert not os.path.exists(link_dir) - make_link(True, unpack_dir, link_dir, args, logger) + make_link(True, unpack_dir, link_dir, args) assert os.path.exists(link_dir) assert os.path.islink(link_dir) @@ -612,7 +592,7 @@ def test_link(): args.quiet = True assert os.path.exists(link_dir) assert not os.path.islink(link_dir) - make_link(True, unpack_dir, link_dir, args, logger) + make_link(True, unpack_dir, link_dir, args) assert os.path.exists(link_dir) assert os.path.islink(link_dir) @@ -622,7 +602,7 @@ def test_link(): args.force = True assert os.path.exists(link_dir) assert not os.path.islink(link_dir) - make_link(True, unpack_dir, link_dir, args, logger) + make_link(True, unpack_dir, link_dir, args) assert os.path.exists(link_dir) assert os.path.islink(link_dir) @@ -633,11 +613,11 @@ def test_link(): assert os.path.exists(link_dir) assert not os.path.islink(link_dir) with mock.patch('galsim.download_cosmos.get_input', return_value='n'): - make_link(True, unpack_dir, link_dir, args, logger) + make_link(True, unpack_dir, link_dir, args) assert os.path.exists(link_dir) assert not os.path.islink(link_dir) with mock.patch('galsim.download_cosmos.get_input', return_value='y'): - make_link(True, unpack_dir, link_dir, args, logger) + make_link(True, unpack_dir, link_dir, args) assert os.path.exists(link_dir) assert os.path.islink(link_dir) @@ -655,12 +635,10 @@ def test_full(): with mock.patch('galsim.download_cosmos.share_dir', 'output'), \ mock.patch('galsim.download_cosmos.urlopen', fake_urlopen): - remove_handler() assert not os.path.islink(link_dir1) galsim.download_cosmos.main(['-d','fake_cosmos','-q','-s','23.5','--save']) assert os.path.islink(link_dir1) - remove_handler() assert not os.path.islink(link_dir2) with mock.patch('sys.argv', ['galsim_download_cosmos', '-d', 'fake_cosmos', '-q']): galsim.download_cosmos.run_main() diff --git a/tests/test_main.py b/tests/test_main.py index c45deec0a0..12c3958114 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -25,6 +25,8 @@ import galsim.__main__ from galsim_test_helpers import * +logger = logging.getLogger(__name__) + # This file tests the galsim executable. # Note: Most of the real functional tests are elsewhere. E.g. test_config* # This really just tests the bits related specifically to the functions in galsim/main.py. @@ -149,18 +151,11 @@ def test_args(): galsim.main.parse_args([config_file, '-v', '4']) sys.stderr = sys_stderr -# Need to call these before each time make_logger is repeated. Else duplicate handles. -def remove_handler(): - logger = logging.getLogger('galsim') - for handler in logger.handlers: - logger.removeHandler(handler) - @timer def test_logger(): args = galsim.main.parse_args(['test.yaml']) - remove_handler() - logger = galsim.main.make_logger(args) + galsim.main.setup_logger(args) assert logger.getEffectiveLevel() == logging.WARNING print('handlers = ',logger.handlers) print('This should print...') @@ -172,8 +167,7 @@ def test_logger(): args.verbosity = 3 args.log_file = 'output/test_logger.log' - remove_handler() - logger = galsim.main.make_logger(args) + galsim.main.setup_logger(args) print('handlers = ',logger.handlers) assert logger.getEffectiveLevel() == logging.DEBUG logger.warning("Test warning") @@ -186,8 +180,7 @@ def test_logger(): assert f.readline().strip() == "Test debug" args.verbosity = 0 - remove_handler() - logger = galsim.main.make_logger(args) + galsim.main.setup_logger(args) print('handlers = ',logger.handlers) assert logger.getEffectiveLevel() == logging.ERROR logger.warning("Test warning") @@ -199,8 +192,7 @@ def test_logger(): args.verbosity = 3 args.log_format = '%(levelname)s - %(message)s' - remove_handler() - logger = galsim.main.make_logger(args) + galsim.main.setup_logger(args) print('handlers = ',logger.handlers) assert logger.getEffectiveLevel() == logging.DEBUG logger.warning("Test warning") @@ -214,21 +206,21 @@ def test_logger(): @timer def test_parse_variables(): - logger = logging.getLogger('test_main') + logging.getLogger('test_main') logger.setLevel(logging.ERROR) # Empty list -> empty dict - new_params = galsim.main.parse_variables([], logger) + new_params = galsim.main.parse_variables([]) assert new_params == {} vars = ["output.nfiles=1", "output.dir='.'"] - new_params = galsim.main.parse_variables(vars, logger) + new_params = galsim.main.parse_variables(vars) assert new_params['output.nfiles'] == 1 assert new_params['output.dir'] == '.' # Lists or dicts will be parsed here vars = ["psf={'type':'Gaussian','sigma':0.4}", "output.skip=[0,0,0,0,0,1]"] - new_params = galsim.main.parse_variables(vars, logger) + new_params = galsim.main.parse_variables(vars) assert new_params['psf'] == {'type' : 'Gaussian', 'sigma' : 0.4} assert new_params['output.skip'] == [0,0,0,0,0,1] @@ -236,20 +228,20 @@ def test_parse_variables(): # (Presumably they would give an appropriate error later when they are used if the # string is not a valid value for whatever this is.) vars = ["psf={'type':'Gaussian' : 'sigma':0.4}"] - new_params = galsim.main.parse_variables(vars, logger) + new_params = galsim.main.parse_variables(vars) assert new_params['psf'] == "{'type':'Gaussian' : 'sigma':0.4}" # Missing = is an error vars = ["output.nfiles","1"] - assert_raises(galsim.GalSimError, galsim.main.parse_variables, vars, logger) + assert_raises(galsim.GalSimError, galsim.main.parse_variables, vars) vars = ["output.nfiles-1"] - assert_raises(galsim.GalSimError, galsim.main.parse_variables, vars, logger) + assert_raises(galsim.GalSimError, galsim.main.parse_variables, vars) # Should work correctly if yaml isn't available. # Although it doesn't always parse quite as nicely. E.g. requires ", not ' for string quotes. with mock.patch.dict(sys.modules, {'yaml':None}): vars = ['psf={"type":"Gaussian","sigma":0.4}', 'output.skip=[0,0,0,0,0,1]'] - new_params = galsim.main.parse_variables(vars, logger) + new_params = galsim.main.parse_variables(vars) assert new_params['psf'] == {'type' : 'Gaussian', 'sigma' : 0.4} assert new_params['output.skip'] == [0,0,0,0,0,1] @@ -289,31 +281,27 @@ def test_process(): if os.path.exists(file_name): os.remove(file_name) args = galsim.main.parse_args(['test.yaml','-v','1']) - remove_handler() - logger = galsim.main.make_logger(args) + galsim.main.setup_logger(args) - galsim.main.process_config([config], args, logger) + galsim.main.process_config([config], args) assert os.path.exists(file_name) assert config['root'] == 'test' # This is set automatically args.profile = True print('Should print profile:') - galsim.main.process_config([config], args, logger) + galsim.main.process_config([config], args) assert config['profile'] is True print('Done') - remove_handler() os.remove(file_name) config_file = os.path.join('input','test.yaml') galsim.main.main([config_file,'-v','1']) assert os.path.exists(file_name) with mock.patch('sys.argv', ['galsim', config_file, '-v', '1']): - remove_handler() os.remove(file_name) galsim.main.run_main() assert os.path.exists(file_name) - remove_handler() os.remove(file_name) galsim.__main__.run_main() assert os.path.exists(file_name) diff --git a/tests/test_roman.py b/tests/test_roman.py index 0d949c6729..d91d81ca86 100644 --- a/tests/test_roman.py +++ b/tests/test_roman.py @@ -1038,7 +1038,7 @@ def test_config_sca(): config['image']['ipc'] = True config['image']['read_noise'] = True config['image']['sky_subtract'] = True - im1 = galsim.config.BuildImage(config, obj_num=0, logger=logger) + im1 = galsim.config.BuildImage(config, obj_num=0) sky_level *= (1.0 + galsim.roman.stray_light_fraction) wcs.makeSkyImage(im2, sky_level) im2 += galsim.roman.thermal_backgrounds['H158'] * galsim.roman.exptime @@ -1076,7 +1076,7 @@ def test_config_sca(): del config['image']['sky_subtract'] config['stamp'] = { 'draw_method' : 'phot' } config['image']['bandpass'] = { 'type' : 'RomanBandpass', 'name' : 'H158' } - im1 = galsim.config.BuildImage(config, obj_num=0, logger=logger) + im1 = galsim.config.BuildImage(config, obj_num=0) wcs.makeSkyImage(im2, sky_level) im2 += galsim.roman.thermal_backgrounds['H158'] * galsim.roman.exptime sky_image = im2.copy() diff --git a/tests/test_sensor.py b/tests/test_sensor.py index e37662dc8d..f157e4dc3e 100644 --- a/tests/test_sensor.py +++ b/tests/test_sensor.py @@ -16,6 +16,7 @@ # and/or other materials provided with the distribution. # +import logging import numpy as np import os import sys @@ -1188,9 +1189,10 @@ def test_flat(run_slow): np.testing.assert_allclose(cov02 / counts_total, 0., atol=2*toler) @timer -def test_omp(): +def test_omp(caplog): """Test setting the number of omp threads. """ + caplog.set_level(logging.DEBUG) import multiprocessing # If num_threads <= 0 or None, get num from cpu_count @@ -1215,61 +1217,61 @@ def test_omp(): assert galsim.get_omp_threads() == num_threads # Repeat and check that appropriate messages are emitted - with CaptureLog() as cl: - num_threads = galsim.set_omp_threads(0, logger=cl.logger) - assert "multiprocessing.cpu_count() = " in cl.output - assert "Telling OpenMP to use %s threads"%cpus in cl.output - - with CaptureLog() as cl: - galsim.set_omp_threads(None, logger=cl.logger) - assert "multiprocessing.cpu_count() = " in cl.output - assert "Telling OpenMP to use %s threads"%cpus in cl.output - - with CaptureLog() as cl: - galsim.set_omp_threads(1, logger=cl.logger) - assert "multiprocessing.cpu_count() = " not in cl.output - assert "Telling OpenMP to use 1 threads" in cl.output - assert "Using %s threads"%num_threads not in cl.output - assert "Unable to use multiple threads" not in cl.output - - with CaptureLog() as cl: - galsim.set_omp_threads(2, logger=cl.logger) - assert "multiprocessing.cpu_count() = " not in cl.output - assert "Telling OpenMP to use 2 threads" in cl.output + caplog.clear() + num_threads = galsim.set_omp_threads(0) + assert "multiprocessing.cpu_count() = " in caplog.text + assert "Telling OpenMP to use %s threads"%cpus in caplog.text + + caplog.clear() + galsim.set_omp_threads(None) + assert "multiprocessing.cpu_count() = " in caplog.text + assert "Telling OpenMP to use %s threads"%cpus in caplog.text + + caplog.clear() + galsim.set_omp_threads(1) + assert "multiprocessing.cpu_count() = " not in caplog.text + assert "Telling OpenMP to use 1 threads" in caplog.text + assert "Using %s threads"%num_threads not in caplog.text + assert "Unable to use multiple threads" not in caplog.text + + caplog.clear() + galsim.set_omp_threads(2) + assert "multiprocessing.cpu_count() = " not in caplog.text + assert "Telling OpenMP to use 2 threads" in caplog.text # It's hard to tell what happens in the next step, since we can't control what # galsim._galsim.SetOMPThreads does. It depends on whether OpenMP is enabled and # how many cores are available. So let's mock it up. with mock.patch('galsim.utilities._galsim') as _galsim: # First mock with OpenMP enables and able to use lots of threads + caplog.clear() _galsim.SetOMPThreads = lambda x: x assert galsim.set_omp_threads(20) == 20 - with CaptureLog() as cl: - galsim.set_omp_threads(20, logger=cl.logger) - assert "OpenMP reports that it will use 20 threads" in cl.output - assert "Using 20 threads" in cl.output + galsim.set_omp_threads(20) + assert "OpenMP reports that it will use 20 threads" in caplog.text + assert "Using 20 threads" in caplog.text # Next only 4 threads available + caplog.clear() _galsim.SetOMPThreads = lambda x: 4 if x > 4 else x print(galsim.set_omp_threads(20)) assert galsim.set_omp_threads(20) == 4 - with CaptureLog() as cl: - galsim.set_omp_threads(20, logger=cl.logger) - assert "OpenMP reports that it will use 4 threads" in cl.output - assert "Using 4 threads" in cl.output + galsim.set_omp_threads(20) + assert "OpenMP reports that it will use 4 threads" in caplog.text + assert "Using 4 threads" in caplog.text + caplog.clear() assert galsim.set_omp_threads(2) == 2 - with CaptureLog() as cl: - galsim.set_omp_threads(2, logger=cl.logger) - assert "OpenMP reports that it will use 2 threads" in cl.output + galsim.set_omp_threads(2) + assert "OpenMP reports that it will use 2 threads" in caplog.text # Finally, no OpenMP + caplog.clear() _galsim.SetOMPThreads = lambda x: 1 assert galsim.set_omp_threads(20) == 1 - with CaptureLog() as cl: - galsim.set_omp_threads(20, logger=cl.logger) - assert "OpenMP reports that it will use 1 threads" in cl.output - assert "Unable to use multiple threads" in cl.output + galsim.set_omp_threads(20) + assert "OpenMP reports that it will use 1 threads" in caplog.text + assert "Unable to use multiple threads" in caplog.text # This is really just for coverage. Check that OMP_PROC_BIND gets set properly. with mock.patch('os.environ', {}):