From d6345c6519bebbec161a72cf8cc1dc68a3008c8e Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 26 Jun 2025 15:02:20 -0700 Subject: [PATCH 01/26] add files from NOAO fork --- astroquery/noirlab/__init__.py | 84 +++++++ astroquery/noirlab/core.py | 211 ++++++++++++++++++ astroquery/noirlab/tests/__init__.py | 0 astroquery/noirlab/tests/expected.py | 47 ++++ .../noirlab/tests/test_noirlab_remote.py | 194 ++++++++++++++++ docs/noirlab/noirlab.rst | 158 +++++++++++++ 6 files changed, 694 insertions(+) create mode 100644 astroquery/noirlab/__init__.py create mode 100644 astroquery/noirlab/core.py create mode 100644 astroquery/noirlab/tests/__init__.py create mode 100644 astroquery/noirlab/tests/expected.py create mode 100644 astroquery/noirlab/tests/test_noirlab_remote.py create mode 100644 docs/noirlab/noirlab.rst diff --git a/astroquery/noirlab/__init__.py b/astroquery/noirlab/__init__.py new file mode 100644 index 0000000000..fb878d8255 --- /dev/null +++ b/astroquery/noirlab/__init__.py @@ -0,0 +1,84 @@ +from astropy import config as _config +# Licensed under a 3-clause BSD style license - see LICENSE.rst +"""NSF's OIR Lab Astro Data Archive (Beta) +------------------------------------------ + +The NSF's OIR Lab Astro Data Archive (formerly NOAO Science Archive) +provides access to data taken with more than 40 telescope and +instrument combinations, including those operated in partnership with +the WIYN, SOAR and SMARTS consortia, from semester 2004B to the +present. In addition to raw data, pipeline-reduced data products from +the DECam, Mosaic and NEWFIRM imagers are also available, as well as +advanced data products delivered by teams carrying out surveys and +other large observing programs with NSF OIR Lab facilities. + +Total holdings 9,634,110 V.3 + +====================== ======== ======== + Telescope/Instrument Images Volume +====================== ======== ======== +bok23m-90prime 290070 3% +ct09m-ccd_imager 101671 1% +ct13m-andicam 528124 5% +ct15m-chiron 217063 2% +ct1m-y4kcam 317819 3% +ct4m-arcoiris 49024 0% +ct4m-cosmos 19018 0% +ct4m-decam 4685453 48% +ct4m-mosaic_2 254005 2% +ct4m-newfirm 339731 3% +kp09m-hdi 108835 1% +kp09m-mosaic 1006 0% +kp09m-mosaic_1_1 3533 0% +kp4m-kosmos 8068 0% +kp4m-mosaic 2420 0% +kp4m-mosaic3 563386 5% +kp4m-mosaic_1 52475 0% +kp4m-mosaic_1_1 142351 1% +kp4m-newfirm 962054 9% +soar-goodman 695828 7% +soar-sami 15639 0% +soar-soi 8981 0% +soar-spartan 166821 1% +soar-triplespec 3171 0% +wiyn-bench 33183 0% +wiyn-whirc 64381 0% +====================== ======== ======== + + +ACKNOWLEDGMENT + +This research uses services or data provided by the Astro Data Archive +at NSF's National Optical-Infrared Astronomy Research +Laboratory. NSF's OIR Lab is operated by the Association of +Universities for Research in Astronomy (AURA), Inc. under a +cooperative agreement with the National Science Foundation. + +""" + + +class Conf(_config.ConfigNamespace): + """ + Configuration parameters for `astroquery.noirlab`. + """ + server = _config.ConfigItem( + ['https://astroarchive.noao.edu', + ], + 'Name of the NOIRLAB server to use.' + ) + timeout = _config.ConfigItem( + 30, + 'Time limit for connecting to NOIRLAB server.' + ) + + +conf = Conf() + +from .core import Noirlab, NoirlabClass # noqa + + +__all__ = ['Noirlab', 'NoirlabClass', + 'conf', 'Conf'] + + +# noqa diff --git a/astroquery/noirlab/core.py b/astroquery/noirlab/core.py new file mode 100644 index 0000000000..dfe08edf02 --- /dev/null +++ b/astroquery/noirlab/core.py @@ -0,0 +1,211 @@ +""" +Provide astroquery API access to OIR Lab Astro Data Archive (natica). + +This does DB access through web-services. +""" +import astropy.io.fits as fits +import astropy.table +from ..query import BaseQuery +from ..utils import async_to_sync +from ..utils.class_or_instance import class_or_instance +from . import conf + + +__all__ = ['Noirlab', 'NoirlabClass'] # specifies what to import + + +@async_to_sync +class NoirlabClass(BaseQuery): + + TIMEOUT = conf.timeout + NAT_URL = conf.server + + def __init__(self, which='file'): + """Return object used for searching the NOIRLab Archive. + + Search either Files (which=file) or HDUs (which=hdu). + Files will always be returned. But if which=hdu, + individual HDUs must have RA,DEC fields. Typically this + is only the case with some pipeline processed files. + """ + self._api_version = None + self._adsurl = f'{self.NAT_URL}/api/adv_search' + + if which == 'hdu': + self.siaurl = f'{self.NAT_URL}/api/sia/vohdu' + self._adss_url = f'{self._adsurl}/hasearch' + self._adsc_url = f'{self._adsurl}/core_hdu_fields' + self._adsa_url = f'{self._adsurl}/aux_hdu_fields' + else: + self.siaurl = f'{self.NAT_URL}/api/sia/voimg' + self._adss_url = f'{self._adsurl}/fasearch' + self._adsc_url = f'{self._adsurl}/core_file_fields' + self._adsa_url = f'{self._adsurl}/aux_file_fields' + + super().__init__() + + @property + def api_version(self): + """Return version of Rest API used by this module. + + If the Rest API changes such that the Major version increases, + a new version of this module will likely need to be used. + """ + if self._api_version is None: + response = self._request('GET', + f'{self.NAT_URL}/api/version', + timeout=self.TIMEOUT, + cache=True) + self._api_version = float(response.content) + return self._api_version + + def _validate_version(self): + KNOWN_GOOD_API_VERSION = 2.0 + if (int(self.api_version) - int(KNOWN_GOOD_API_VERSION)) >= 1: + msg = (f'The astroquery.noirlab module is expecting an older ' + f'version of the {self.NAT_URL} API services. ' + f'Please upgrade to latest astroquery. ' + f'Expected version {KNOWN_GOOD_API_VERSION} but got ' + f'{self.api_version} from the API.') + raise Exception(msg) + + def service_metadata(self, cache=True): + """Denotes a Metadata Query: no images are requested; only metadata + should be returned. This feature is described in more detail in: + http://www.ivoa.net/documents/PR/DAL/PR-SIA-1.0-20090521.html#mdquery + """ + url = f'{self.siaurl}?FORMAT=METADATA&format=json' + response = self._request('GET', url, timeout=self.TIMEOUT, cache=cache) + return response.json()[0] + + @class_or_instance + def query_region(self, coordinate, radius=0.1, cache=True): + """Query for NOIRLab observations by region of the sky. + + Given a sky coordinate and radius, returns a `~astropy.table.Table` + of NOIRLab observations. + + Parameters + ---------- + coordinates : str or `~astropy.coordinates` object + The target region which to search. It may be specified as a + string or as the appropriate `~astropy.coordinates` object. + radius : str or `~astropy.units.Quantity` object, optional + Default 0.1 degrees. + The string must be parsable by `~astropy.coordinates.Angle`. The + appropriate `~astropy.units.Quantity` object from + `~astropy.units` may also be used. + + Returns + ------- + response : `~astropy.table.Table` + """ + self._validate_version() + ra, dec = coordinate.to_string('decimal').split() + url = f'{self.siaurl}?POS={ra},{dec}&SIZE={radius}&format=json' + response = self._request('GET', url, + timeout=self.TIMEOUT, + cache=cache) + response.raise_for_status() + return astropy.table.Table(data=response.json()) + + def query_region_async(self, coordinate, radius=0.1, cache=True): + """Query for NOIRLab observations by region of the sky. + + Given a sky coordinate and radius, returns a `~astropy.table.Table` + of NOIRLab observations. + + Parameters + ---------- + coordinates : str or `~astropy.coordinates` object + The target region which to search. It may be specified as a + string or as the appropriate `~astropy.coordinates` object. + radius : str or `~astropy.units.Quantity` object, optional + Default 0.1 degrees. + The string must be parsable by `~astropy.coordinates.Angle`. The + appropriate `~astropy.units.Quantity` object from + `~astropy.units` may also be used. + + Returns + ------- + response : `requests.Response` + """ + self._validate_version() + + ra, dec = coordinate.to_string('decimal').split() + url = f'{self.siaurl}?POS={ra},{dec}&SIZE={radius}&format=json' + response = self._request('GET', url, + timeout=self.TIMEOUT, + cache=cache) + response.raise_for_status() + return response + + def core_fields(self, cache=True): + """List the available CORE fields. CORE fields are faster to search + than AUX fields..""" + response = self._request('GET', self._adsc_url, + timeout=self.TIMEOUT, + cache=cache) + response.raise_for_status() + return response.json() + + def aux_fields(self, instrument, proctype, cache=True): + """List the available AUX fields. AUX fields are ANY fields in the + Archive FITS files that are not core DB fields. These are generally + common to a single Instrument, Proctype combination. AUX fields are + slower to search than CORE fields. Acceptable values for INSTRUMENT and PROCTYPE + are listed in the results of the CATEGORICALS method. + """ + url = f'{self._adsa_url}/{instrument}/{proctype}/' + response = self._request('GET', url, timeout=self.TIMEOUT, cache=cache) + response.raise_for_status() + return response.json() + + def categoricals(self, cache=True): + """List the currently acceptable values for each 'categorical field' + associated with Archive files. A 'categorical field' is one in + which the values are restricted to a specific set. The specific + set may grow over time, but not often. The categorical fields are: + collection, instrument, obs_mode, proc_type, prod_type, site, survey, + telescope. + """ + url = f'{self._adsurl}/cat_lists/?format=json' + response = self._request('GET', url, timeout=self.TIMEOUT, cache=cache) + response.raise_for_status() + return response.json() + + @class_or_instance + def query_metadata(self, qspec, limit=1000, cache=True): + self._validate_version() + url = f'{self._adss_url}/?limit={limit}' + + if qspec is None: + jdata = {"outfields": ["md5sum", ], "search": []} + else: + jdata = qspec + + response = self._request('POST', url, json=jdata, timeout=self.TIMEOUT) + response.raise_for_status() + return astropy.table.Table(rows=response.json()) + + def retrieve(self, fileid, cache=True): + url = f'{self.NAT_URL}/api/retrieve/{fileid}/' + hdul = fits.open(url) + return hdul + + def version(self, cache=False): + url = f'{self.NAT_URL}/api/version/' + response = self._request('GET', url, timeout=self.TIMEOUT, cache=cache) + response.raise_for_status() + return response.json() + + def get_token(self, email, password, cache=True): + url = f'{self.NAT_URL}/api/get_token/' + response = self._request('POST', url, + json={"email": email, "password": password}, + timeout=self.TIMEOUT) + response.raise_for_status() + return response.json() + + +Noirlab = NoirlabClass() diff --git a/astroquery/noirlab/tests/__init__.py b/astroquery/noirlab/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/astroquery/noirlab/tests/expected.py b/astroquery/noirlab/tests/expected.py new file mode 100644 index 0000000000..01a0a6c235 --- /dev/null +++ b/astroquery/noirlab/tests/expected.py @@ -0,0 +1,47 @@ +# flake8: noqa + +query_region_1 = {'4066c45407961eab48eb16bcb9c5b4b7', + '554b2bcb3b2a4353c515d20d9fc29e4e', + '711ce162a7488a72b1629d994264eae7', + 'b9e211760ce74929919a55b225a9860c', + 'f350fff2e9db68f5d3ca74c91f0e374a', + 'f616cc460e51d2aa356ad21d79e22a21'} + +query_region_2 = {'78aed36ad9bfaa561d78845757d0aad6', + '1e04390654cf399781bbacccce78a790', + 'bf1aa2b1a3c6934bba18285572765d6f', + '7977c92b09724fb331b19263819962e4', + 'f990925c8d99a66f642e4463f1dde7f2', + '09535b88fc700227bb47979a670a7d85'} + +service_metadata = {'archive_filename': 'string', + 'date_obs': 'string', + 'dec': 'float', + 'exposure': 'float', + 'filesize': 'int', + 'instrument': 'string', + 'md5sum': 'string', + 'original_filename': 'string', + 'pi': 'string', + 'prop_id': 'string', + 'ra': 'float', + 'release_date': 'string', + 'telescope': 'string', + 'updated': 'string'} + +aux_file_fields = {'AIRMASS': 'str', 'AOS': 'str', 'ASTIG1': 'str', 'ASTIG2': 'str', 'ATTNUM': 'str', 'AVSIG': 'str', 'AVSKY': 'str', 'AZ': 'str', 'BAND': 'str', 'BCAM': 'str', 'BCAMAX': 'str', 'BCAMAY': 'str', 'BCAMAZ': 'str', 'BCAMDX': 'str', 'BCAMDY': 'str', 'BFCFIL': 'str', 'BUNIT': 'str', 'CAMSHUT': 'str', 'CAMSYM': 'str', 'CCDBIN1': 'str', 'CCDBIN2': 'str', 'CCDSEC': 'str', 'CCDSECA': 'str', 'CCDSECB': 'str', 'CENTDEC': 'str', 'CENTRA': 'str', 'CONSTVER': 'str', 'CORN1DEC': 'str', 'CORN1RA': 'str', 'CORN2DEC': 'str', 'CORN2RA': 'str', 'CORN3DEC': 'str', 'CORN3RA': 'str', 'CORN4DEC': 'str', 'CORN4RA': 'str', 'CORNDEC1': 'str', 'CORNDEC2': 'str', 'CORNDEC3': 'str', 'CORNDEC4': 'str', 'CORNRA1': 'str', 'CORNRA2': 'str', 'CORNRA3': 'str', 'CORNRA4': 'str', 'CROSSRA0': 'str', 'DARKTIME': 'str', 'DATASEC': 'str', 'DATASECA': 'str', 'DATASECB': 'str', 'DATE': 'str', 'DATE-OBS': 'str', 'DEC': 'str', 'DES_EXT': 'str', 'DESNCRAY': 'str', 'DESNSTRK': 'str', 'DESREL': 'str', 'DETSIZE': 'str', 'DHEFIRM': 'str', 'DHEINF': 'str', 'DIMM2SEE': 'str', 'DIMMSEE': 'str', 'DODX': 'str', 'DODY': 'str', 'DODZ': 'str', 'DOMEAZ': 'str', 'DOMEFLOR': 'str', 'DOMEHIGH': 'str', 'DOMELOW': 'str', 'DONUTFN1': 'str', 'DONUTFN2': 'str', 'DONUTFN3': 'str', 'DONUTFN4': 'str', 'DONUTFS1': 'str', 'DONUTFS2': 'str', 'DONUTFS3': 'str', 'DONUTFS4': 'str', 'DOXT': 'str', 'DOYT': 'str', 'DTACCOUN': 'str', 'DTACQNAM': 'str', 'DTACQUIS': 'str', 'DTCALDAT': 'str', 'DTCOPYRI': 'str', 'DTINSTRU': 'str', 'DTNSANAM': 'str', 'DTOBSERV': 'str', 'DTPI': 'str', 'DTPIAFFL': 'str', 'DTPROPID': 'str', 'DTQUEUE': 'str', 'DT_RTNAM': 'str', 'DTSITE': 'str', 'DTSTATUS': 'str', 'DTTELESC': 'str', 'DTTITLE': 'str', 'DTUTC': 'str', 'ELLIPTIC': 'str', 'EQUINOX': 'str', 'ERRORS': 'str', 'EUPSPROD': 'str', 'EUPSVER': 'str', 'EXCLUDED': 'str', 'EXPDUR': 'str', 'EXPNUM': 'str', 'EXPREQ': 'str', 'EXPTIME': 'str', 'EXTVER': 'str', 'FADX': 'str', 'FADY': 'str', 'FADZ': 'str', 'FAXT': 'str', 'FAYT': 'str', 'FILENAME': 'str', 'FILTER': 'str', 'FILTPOS': 'str', 'FRGSCALE': 'str', 'FWHM': 'str', 'FZALGOR': 'str', 'FZDTHRSD': 'str', 'FZQMETHD': 'str', 'FZQVALUE': 'str', 'G-CCDNUM': 'str', 'G-FEEDBK': 'str', 'G-FLXVAR': 'str', 'G-LATENC': 'str', 'G-MAXX': 'str', 'G-MAXY': 'str', 'G-MEANX': 'str', 'G-MEANX2': 'str', 'G-MEANXY': 'str', 'G-MEANY': 'str', 'G-MEANY2': 'str', 'G-MODE': 'str', 'G-SEEING': 'str', 'GSKYHOT': 'str', 'GSKYPHOT': 'str', 'GSKYVAR': 'str', 'G-TRANSP': 'str', 'GUIDER': 'str', 'HA': 'str', 'HDRVER': 'str', 'HEX': 'str', 'HUMIDITY': 'str', 'INSTANCE': 'str', 'INSTRUME': 'str', 'IRAF-TLM': 'str', 'LINCFIL': 'str', 'LSKYHOT': 'str', 'LSKYPHOT': 'str', 'LSKYPOW': 'str', 'LSKYVAR': 'str', 'LST': 'str', 'LUTVER': 'str', 'LWTRTEMP': 'str', 'MAGZERO': 'str', 'MAGZP': 'str', 'MAGZPT': 'str', 'MAGZUNC': 'str', 'MAIRTEMP': 'str', 'MANIFEST': 'str', 'MASS2': 'str', 'MJD-END': 'str', 'MJD-OBS': 'str', 'MOONANGL': 'str', 'MSURTEMP': 'str', 'MULTIEXP': 'str', 'MULTIFOC': 'str', 'MULTIID': 'str', 'MULTIROW': 'str', 'MULTITOT': 'str', 'NBLEED': 'str', 'NDONUTS': 'str', 'NEXTEND': 'str', 'NITE': 'str', 'NOTE': 'str', 'NPHTMTCH': 'str', 'NSATPIX': 'str', 'NUM': 'str', 'OBJECT': 'str', 'OBS-ELEV': 'str', 'OBSERVAT': 'str', 'OBSERVER': 'str', 'OBSID': 'str', 'OBS-LAT': 'str', 'OBS-LONG': 'str', 'OBSTYPE': 'str', 'ODATEOBS': 'str', 'OPENSHUT': 'str', 'ORIGIN': 'str', 'OUTTEMP': 'str', 'PHOTFLAG': 'str', 'PHOTREF': 'str', 'PIPELINE': 'str', 'PIXSCAL1': 'str', 'PIXSCAL2': 'str', 'PLARVER': 'str', 'PLDNAME': 'str', 'PLDSID': 'str', 'PLFNAME': 'str', 'PLPROCID': 'str', 'PLQNAME': 'str', 'PLQUEUE': 'str', 'PLVER': 'str', 'PME-TEMP': 'str', 'PMN-TEMP': 'str', 'PMOSTEMP': 'str', 'PMS-TEMP': 'str', 'PMW-TEMP': 'str', 'PRESSURE': 'str', 'PROCTYPE': 'str', 'PRODTYPE': 'str', 'PROGRAM': 'str', 'PROPID': 'str', 'PROPOSER': 'str', 'PUPILAMP': 'str', 'PUPILMAX': 'str', 'PUPILSKY': 'str', 'PUPMAX': 'str', 'PV1_6': 'str', 'PV2_4': 'str', 'RA': 'str', 'RA_CENT': 'str', 'RACMAX': 'str', 'RACMIN': 'str', 'RADECSYS': 'str', 'RADESYS': 'str', 'RADIUS': 'str', 'RADSTD': 'str', 'RECNO': 'str', 'REQNUM': 'str', 'RMCOUNT': 'str', 'SB_ACCOU': 'str', 'SB_DIR1': 'str', 'SB_DIR2': 'str', 'SB_DIR3': 'str', 'SB_HOST': 'str', 'SB_ID': 'str', 'SB_LOCAL': 'str', 'SB_NAME': 'str', 'SB_RECNO': 'str', 'SB_RTNAM': 'str', 'SB_SITE': 'str', 'SCAMPCHI': 'str', 'SCAMPFLG': 'str', 'SCAMPNUM': 'str', 'SCAMPREF': 'str', 'SEQID': 'str', 'SEQNUM': 'str', 'SEQTOT': 'str', 'SEQTYPE': 'str', 'SISPIVER': 'str', 'SKYBRITE': 'str', 'SKYORDER': 'str', 'SKYPC00': 'str', 'SKYPC01': 'str', 'SKYPC02': 'str', 'SKYPC03': 'str', 'SKYSIGMA': 'str', 'SKYSTAT': 'str', 'SKYSUB': 'str', 'SKYSUB01': 'str', 'SKYSUB10': 'str', 'SKYSUBP': 'str', 'SKYUPDAT': 'str', 'SLOT01': 'str', 'SLOT03': 'str', 'SURVEYID': 'str', 'TELDEC': 'str', 'TELEQUIN': 'str', 'TELESCOP': 'str', 'TELFOCUS': 'str', 'TELRA': 'str', 'TELSTAT': 'str', 'TILING': 'str', 'TIME-OBS': 'str', 'TIMESYS': 'str', 'TRACKING': 'str', 'UNITNAME': 'str', 'UPTRTEMP': 'str', 'UTE-TEMP': 'str', 'UTN-TEMP': 'str', 'UTS-TEMP': 'str', 'UTW-TEMP': 'str', 'VALIDA': 'str', 'VALIDB': 'str', 'VSUB': 'str', 'WCSCAL': 'str', 'WINDDIR': 'str', 'WINDSPD': 'str', 'XTALKFIL': 'str', 'ZD': 'str', 'ZPDELDEC': 'str', 'ZPDELRA': 'str'} + +core_file_fields = [{'Field': 'archive_filename', 'Type': 'str'}, {'Field': 'caldat', 'Type': 'datetime64'}, {'Field': 'date_obs_max', 'Type': 'datetime64'}, {'Field': 'date_obs_min', 'Type': 'datetime64'}, {'Field': 'dec_max', 'Type': 'np.float64'}, {'Field': 'dec_min', 'Type': 'np.float64'}, {'Field': 'depth', 'Type': 'np.float64'}, {'Field': 'exposure', 'Type': 'np.float64'}, {'Field': 'filesize', 'Type': 'np.int64'}, {'Field': 'ifilter', 'Type': 'category'}, {'Field': 'instrument', 'Type': 'category'}, {'Field': 'md5sum', 'Type': 'str'}, {'Field': 'obs_mode', 'Type': 'category'}, {'Field': 'obs_type', 'Type': 'category'}, {'Field': 'original_filename', 'Type': 'str'}, {'Field': 'proc_type', 'Type': 'category'}, {'Field': 'prod_type', 'Type': 'category'}, {'Field': 'proposal', 'Type': 'category'}, {'Field': 'ra_max', 'Type': 'np.float64'}, {'Field': 'ra_min', 'Type': 'np.float64'}, {'Field': 'release_date', 'Type': 'datetime64'}, {'Field': 'seeing', 'Type': 'np.float64'}, {'Field': 'site', 'Type': 'category'}, {'Field': 'survey', 'Type': 'category'}, {'Field': 'telescope', 'Type': 'category'}, {'Field': 'updated', 'Type': 'datetime64'}] + +query_file_metadata = [' archive_filename instrument md5sum original_filename proc_type url ', '----------------------------------------------------------------------------- ---------- -------------------------------- ------------------------------------------------------------------------------------- --------- ----------------------------------------------------------------------------', '/net/archive/pipe/20161024/ct4m/2012B-0001/c4d_161025_034649_oki_z_v2.fits.fz decam 0000f0c5015263610a75f0affed377d0 /net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_161025_034649_oki_z_v2.fits.fz skysub https://astroarchive.noao.edu/api/retrieve/0000f0c5015263610a75f0affed377d0/', '/net/archive/pipe/20160928/ct4m/2012B-0001/c4d_160929_090838_ooi_z_v2.fits.fz decam 0001e5a5bcf039ebf0c53a6da32e8888 /net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_160929_090838_ooi_z_v2.fits.fz instcal https://astroarchive.noao.edu/api/retrieve/0001e5a5bcf039ebf0c53a6da32e8888/', '/net/archive/pipe/20160909/ct4m/2012B-0001/c4d_160910_062930_opd_z_v2.fits.fz decam 0003a1c853de73fc1796d2d7d77ca9cd /net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_160910_062930_opd_z_v2.fits.fz resampled https://astroarchive.noao.edu/api/retrieve/0003a1c853de73fc1796d2d7d77ca9cd/'] + +aux_hdu_fields = {'AMPBAL': 'str', 'AMPSECA': 'str', 'AMPSECB': 'str', 'ARAWGAIN': 'str', 'AVSIG': 'str', 'AVSKY': 'str', 'BIASFIL': 'str', 'BLDINTRP': 'str', 'BPM': 'str', 'BPMFIL': 'str', 'CATALOG': 'str', 'CCDNUM': 'str', 'CCDSECA': 'str', 'CCDSECB': 'str', 'CD1_1': 'str', 'CD1_2': 'str', 'CD2_1': 'str', 'CD2_2': 'str', 'CENDEC1': 'str', 'CENRA1': 'str', 'COR1DEC1': 'str', 'COR1RA1': 'str', 'COR2DEC1': 'str', 'COR2RA1': 'str', 'COR3DEC1': 'str', 'COR3RA1': 'str', 'COR4DEC1': 'str', 'COR4RA1': 'str', 'CROSSRA0': 'str', 'CRPIX1': 'str', 'CRPIX2': 'str', 'CRVAL1': 'str', 'CRVAL2': 'str', 'CTYPE1': 'str', 'CTYPE2': 'str', 'CUNIT1': 'str', 'CUNIT2': 'str', 'D0034494': 'str', 'D0034496': 'str', 'D0034497': 'str', 'DATASEC': 'str', 'DATASECA': 'str', 'DATASECB': 'str', 'DATE': 'str', 'DEC': 'str', 'DEC1': 'str', 'DEC13A_2': 'str', 'DEC13B_2': 'str', 'DEC14A_2': 'str', 'DEC15A_2': 'str', 'DEC15B_2': 'str', 'DEC16A_2': 'str', 'DEC16B_2': 'str', 'DEC17A_2': 'str', 'DEC17B_2': 'str', 'DEC18A_2': 'str', 'DEC18B_2': 'str', 'DEC18B_R': 'str', 'DEC18B_S': 'str', 'DEC19A_2': 'str', 'DEC19A_A': 'str', 'DEC19A_D': 'str', 'DEC19A_J': 'str', 'DEC19A_K': 'str', 'DEC19A_L': 'str', 'DEC19A_M': 'str', 'DEC19A_P': 'str', 'DEC19A_S': 'str', 'DEC19A_T': 'str', 'DEC19A_W': 'str', 'DEC19B_2': 'str', 'DEC19B_A': 'str', 'DEC19B_C': 'str', 'DEC19B_D': 'str', 'DEC19B_F': 'str', 'DEC19B_H': 'str', 'DEC19B_I': 'str', 'DEC19B_J': 'str', 'DEC19B_K': 'str', 'DEC19B_L': 'str', 'DEC19B_M': 'str', 'DEC19B_P': 'str', 'DEC19B_R': 'str', 'DEC19B_S': 'str', 'DEC19B_T': 'str', 'DEC19B_W': 'str', 'DEC20A_A': 'str', 'DEC20A_C': 'str', 'DEC20A_F': 'str', 'DEC20A_J': 'str', 'DEC20A_K': 'str', 'DEC20A_L': 'str', 'DEC20A_M': 'str', 'DEC20A_S': 'str', 'DEC20A_T': 'str', 'DEC20B_T': 'str', 'DECALS_2': 'str', 'DECALS_D': 'str', 'DECC1': 'str', 'DECC2': 'str', 'DECC3': 'str', 'DECC4': 'str', 'DEC_CENT': 'str', 'DECCMAX': 'str', 'DECCMIN': 'str', 'DES14B_2': 'str', 'DES15B_2': 'str', 'DES16B_2': 'str', 'DES17A_2': 'str', 'DES17B_2': 'str', 'DES18A_2': 'str', 'DES18B_2': 'str', 'DESBFC': 'str', 'DESBIAS': 'str', 'DESBLEED': 'str', 'DESBPM': 'str', 'DESCRMSK': 'str', 'DESDCXTK': 'str', 'DESDMCR': 'str', 'DES_EXT': 'str', 'DESFIXC': 'str', 'DESFLAT': 'str', 'DESFNAME': 'str', 'DESFRING': 'str', 'DESGAINC': 'str', 'DESILLUM': 'str', 'DESIMMSK': 'str', 'DESLINC': 'str', 'DESNCRAY': 'str', 'DESNSTRK': 'str', 'DESOSCN': 'str', 'DESPHOTF': 'str', 'DESPUPC': 'str', 'DESSAT': 'str', 'DESSKYSB': 'str', 'DESSTAR': 'str', 'DETECTOR': 'str', 'DETPOS': 'str', 'DETSEC': 'str', 'DETSECA': 'str', 'DETSECB': 'str', 'ELLIPTIC': 'str', 'ENG17B_2': 'str', 'ENG18A_2': 'str', 'ENG18B_2': 'str', 'EXPTIME': 'str', 'EXTNAME': 'str', 'EXTVER': 'str', 'FILTER': 'str', 'FIXCFIL': 'str', 'FIXPIX': 'str', 'FIXPIX02': 'str', 'FLATFIL': 'str', 'FLATMEDA': 'str', 'FLATMEDB': 'str', 'FPA': 'str', 'FRGSCALE': 'str', 'FRINGE': 'str', 'FRINGFIL': 'str', 'FWHM': 'str', 'FWHMP1': 'str', 'FZALGOR': 'str', 'FZDTHRSD': 'str', 'FZQMETHD': 'str', 'FZQVALUE': 'str', 'GAINA': 'str', 'GAINB': 'str', 'GCOUNT': 'str', 'ILLCOR': 'str', 'ILLMASK': 'str', 'ILLUMCOR': 'str', 'ILLUMFIL': 'str', 'INHERIT': 'str', 'IRAF-TLM': 'str', 'LAGER_20': 'str', 'LTM1_1': 'str', 'LTM1_2': 'str', 'LTM2_1': 'str', 'LTM2_2': 'str', 'LTV1': 'str', 'LTV2': 'str', 'MAGZERO1': 'str', 'MAGZUNC1': 'str', 'NAXIS1': 'str', 'NAXIS2': 'str', 'NBLEED': 'str', 'NSATPIX': 'str', 'OBJECT': 'str', 'OBJMASK': 'str', 'ORIGIN': 'str', 'PCOUNT': 'str', 'PHOTFLAG': 'str', 'PUPFIL': 'str', 'PUPILAMP': 'str', 'PUPILSKY': 'str', 'PUPMAX': 'str', 'PV1_0': 'str', 'PV1_1': 'str', 'PV1_10': 'str', 'PV1_2': 'str', 'PV1_3': 'str', 'PV1_4': 'str', 'PV1_5': 'str', 'PV1_6': 'str', 'PV1_7': 'str', 'PV1_8': 'str', 'PV1_9': 'str', 'PV2_0': 'str', 'PV2_1': 'str', 'PV2_10': 'str', 'PV2_2': 'str', 'PV2_3': 'str', 'PV2_4': 'str', 'PV2_5': 'str', 'PV2_6': 'str', 'PV2_7': 'str', 'PV2_8': 'str', 'PV2_9': 'str', 'RA': 'str', 'RA1': 'str', 'RAC1': 'str', 'RAC2': 'str', 'RAC3': 'str', 'RAC4': 'str', 'RA_CENT': 'str', 'RACMAX': 'str', 'RACMIN': 'str', 'RDNOISEA': 'str', 'RDNOISEB': 'str', 'REQ13B_2': 'str', 'REQ13B_H': 'str', 'REQ14B_2': 'str', 'REQ14B_K': 'str', 'REQ15B_S': 'str', 'REQ16A_S': 'str', 'REQ18B_B': 'str', 'REQ19A_2': 'str', 'REQ19A_P': 'str', 'SATURATA': 'str', 'SATURATB': 'str', 'SATURATE': 'str', 'SKYBRITE': 'str', 'SKYIM': 'str', 'SKYSBFIL': 'str', 'SKYSIGMA': 'str', 'SKYSUB': 'str', 'SKYSUB00': 'str', 'SKYVARA': 'str', 'SKYVARB': 'str', 'SLOT00': 'str', 'SLOT01': 'str', 'SLOT02': 'str', 'SLOT03': 'str', 'SLOT04': 'str', 'SLOT05': 'str', 'STARFIL': 'str', 'STARMASK': 'str', 'TOO15A_2': 'str', 'TOO15B_2': 'str', 'TOO16A_2': 'str', 'TOO16B_2': 'str', 'TOO17B_2': 'str', 'TOO18A_2': 'str', 'TOO18A_S': 'str', 'TOO18B_2': 'str', 'TOO19A_2': 'str', 'TOO19A_G': 'str', 'TOO19A_K': 'str', 'TOO19A_M': 'str', 'TOO19B_M': 'str', 'TOO20A_M': 'str', 'WAT0_001': 'str', 'WAT1_001': 'str', 'WAT2_001': 'str', 'WCSAXES': 'str', 'WCSDIM': 'str', 'WTMAP': 'str', 'XTENSION': 'str', 'ZDITHER0': 'str'} + +core_hdu_fields = [{'Field': 'boundary', 'Type': 'str'}, {'Field': 'dec', 'Type': 'np.float64'}, {'Field': 'dec_range', 'Type': 'str'}, {'Field': 'fitsfile', 'Type': 'str'}, {'Field': 'fitsfile__archive_filename', 'Type': 'str'}, {'Field': 'fitsfile__caldat', 'Type': 'str'}, {'Field': 'fitsfile__date_obs_max', 'Type': 'str'}, {'Field': 'fitsfile__date_obs_min', 'Type': 'str'}, {'Field': 'fitsfile__dec_max', 'Type': 'str'}, {'Field': 'fitsfile__dec_min', 'Type': 'str'}, {'Field': 'fitsfile__depth', 'Type': 'str'}, {'Field': 'fitsfile__exposure', 'Type': 'str'}, {'Field': 'fitsfile__filesize', 'Type': 'str'}, {'Field': 'fitsfile__ifilter', 'Type': 'str'}, {'Field': 'fitsfile__instrument', 'Type': 'str'}, {'Field': 'fitsfile__md5sum', 'Type': 'str'}, {'Field': 'fitsfile__obs_mode', 'Type': 'str'}, {'Field': 'fitsfile__obs_type', 'Type': 'str'}, {'Field': 'fitsfile__original_filename', 'Type': 'str'}, {'Field': 'fitsfile__proc_type', 'Type': 'str'}, {'Field': 'fitsfile__prod_type', 'Type': 'str'}, {'Field': 'fitsfile__proposal', 'Type': 'str'}, {'Field': 'fitsfile__ra_max', 'Type': 'str'}, {'Field': 'fitsfile__ra_min', 'Type': 'str'}, {'Field': 'fitsfile__release_date', 'Type': 'str'}, {'Field': 'fitsfile__seeing', 'Type': 'str'}, {'Field': 'fitsfile__site', 'Type': 'str'}, {'Field': 'fitsfile__survey', 'Type': 'str'}, {'Field': 'fitsfile__telescope', 'Type': 'str'}, {'Field': 'fitsfile__updated', 'Type': 'str'}, {'Field': 'hdu_idx', 'Type': 'np.int64'}, {'Field': 'id', 'Type': 'str'}, {'Field': 'ra', 'Type': 'np.float64'}, {'Field': 'ra_range', 'Type': 'str'}, {'Field': 'updated', 'Type': 'datetime64'}] + + +query_hdu_metadata = ['AIRMASS fitsfile__archive_filename fitsfile__caldat fitsfile__instrument fitsfile__proc_type', '------- ----------------------------------------------------------------------- ---------------- -------------------- -------------------', ' None /net/archive/mtn/20170814/ct4m/2012B-0001/c4d_170815_051354_ori.fits.fz 2017-08-14 decam raw', ' None /net/archive/mtn/20170814/ct4m/2012B-0001/c4d_170815_051354_ori.fits.fz 2017-08-14 decam raw', ' None /net/archive/mtn/20170814/ct4m/2012B-0001/c4d_170815_051354_ori.fits.fz 2017-08-14 decam raw'] + +categoricals = {'instruments': ['(p)odi', '90prime', 'andicam', 'arcoiris', 'arcon', 'bench', 'ccd_imager', 'chiron', 'cosmos', 'decam', 'echelle', 'falmingos', 'flamingos', 'goodman', 'goodman spectrograph', 'gtcam', 'hdi', 'ice', 'ispi', 'kosmos', 'minimo/ice', 'mop/ice', 'mosaic', 'mosaic3', 'mosaic_1', 'mosaic_1_1', 'mosaic_2', 'newfirm', 'osiris', 'sami', 'soi', 'spartan', 'spartan ir camera', 'triplespec', 'wfc', 'whirc', 'wildfire', 'y4kcam'], 'obsmodes': [], 'proctypes': ['instcal', 'mastercal', 'nota', 'projected', 'raw', 'resampled', 'skysub', 'stacked'], 'prodtypes': ['dqmask', 'expmap', 'graphics (size)', 'image', 'image 2nd version 1', 'image1', 'nota', 'resampled', 'weight', 'wtmap'], 'sites': ['cp', 'ct', 'kp', 'lp'], 'surveys': [], 'telescopes': ['bok23m', 'ct09m', 'ct13m', 'ct15m', 'ct1m', 'ct4m', 'ctlab', 'kp09m', 'kp21m', 'kp35m', 'kp4m', 'kpcf', 'lp25m', 'soar', 'wiyn']} + +retrieve = ['SIMPLE', 'BITPIX', 'NAXIS', 'EXTEND', 'ORIGIN', 'DATE', 'IRAF-TLM', 'OBJECT', 'RAWFILE', 'FILENAME', 'OBJRA', 'OBJDEC', 'OBJEPOCH', '', 'TIMESYS', '', 'OBSERVAT', 'OBS-ELEV', 'OBS-LAT', 'OBS-LONG', 'TELESCOP', 'TELRADEC', 'TELEQUIN', 'TELRA', 'TELDEC', 'HA', 'ZD', 'TELFOCUS', '', 'MOSSIZE', 'NDETS', '', 'OBSERVER', 'PROPOSER', 'PROPID', 'SEQID', 'SEQNUM', '', 'NOHS', 'NOCUTC', 'NOCRSD', 'NOCGID', 'NOCNAME', '', 'DTSITE', 'DTTELESC', 'DTINSTRU', 'DTCALDAT', 'DTUTC', 'DTOBSERV', 'DTPROPID', 'DTPI', 'DTPIAFFL', 'DTTITLE', 'DTCOPYRI', 'DTACQUIS', 'DTACCOUN', 'DTACQNAM', 'DTNSANAM', 'DTSTATUS', 'SB_HOST', 'SB_ACCOU', 'SB_SITE', 'SB_LOCAL', 'SB_DIR1', 'SB_DIR2', 'SB_DIR3', 'SB_RECNO', 'SB_ID', 'SB_NAME', 'RMCOUNT', 'RECNO', 'INSTRUME', 'RSPTGRP', 'RSPGRP', '', 'OBSTYPE', 'PROCTYPE', 'PRODTYPE', 'MIMETYPE', 'EXPTIME', 'FILTER', '', 'RA', 'DEC', 'CENTRA', 'CORN1RA', 'CORN2RA', 'CORN3RA', 'CORN4RA', 'CENTDEC', 'CORN1DEC', 'CORN2DEC', 'CORN3DEC', 'CORN4DEC', 'DATE-OBS', 'TIME-OBS', 'MJD-OBS', 'ST', '', 'CTYPE1', 'CTYPE2', 'CRVAL1', 'CRVAL2', 'CD1_1', 'CD2_2', 'WAT0_001', 'WAT1_001', 'WAT2_001', '', 'DIGAVGS', 'NCOADD', 'FSAMPLE', 'EXPCOADD', 'TITLE', 'DARKFIL', 'DARKINFO', 'LINIMAGE', 'LINCOEFF', 'BUNIT', 'GAIN', 'OEXPTIME', 'MAGZREF', 'PHOTINDX', 'WCSCAL', 'WCSXRMS', 'WCSYRMS', 'MAGZERO', '', 'MAGZSIG', 'MAGZERR', 'MAGZNAV', 'SEEINGP', 'SKYBG', 'SEEING', 'SKYMAG', 'STKBPM', 'OBJBPM', 'CDELT1', 'CDELT2', 'CRPIX1', 'CRPIX2', 'BPM', 'IMCMB001', 'IMCMB002', 'IMCMB003', 'IMCMB004', 'IMCMB005', 'IMCMB006', 'IMCMB007', 'IMCMB008', 'IMCMB009', 'IMCMB010', 'IMCMB011', 'IMCMB012', 'IMCMB013', 'IMCMB014', 'IMCMB015', 'IMCMB016', 'IMCMB017', 'IMCMB018', 'IMCMB019', 'IMCMB020', 'IMCMB021', 'IMCMB022', 'IMCMB023', 'IMCMB024', 'IMCMB025', 'IMCMB026', 'IMCMB027', 'IMCMB028', 'IMCMB029', 'IMCMB030', 'IMCMB031', 'IMCMB032', 'IMCMB033', 'IMCMB034', 'IMCMB035', 'IMCMB036', 'IMCMB037', 'IMCMB038', 'IMCMB039', 'IMCMB040', 'IMCMB041', 'IMCMB042', 'IMCMB043', 'IMCMB044', 'IMCMB045', 'IMCMB046', 'IMCMB047', 'IMCMB048', 'IMCMB049', 'IMCMB050', 'IMCMB051', 'IMCMB052', 'IMCMB053', 'IMCMB054', 'IMCMB055', 'IMCMB056', 'IMCMB057', 'IMCMB058', 'IMCMB059', 'IMCMB060', 'IMCMB061', 'IMCMB062', 'IMCMB063', 'IMCMB064', 'IMCMB065', 'IMCMB066', 'IMCMB067', 'IMCMB068', 'IMCMB069', 'IMCMB070', 'IMCMB071', 'IMCMB072', 'IMCMB073', 'IMCMB074', 'IMCMB075', 'IMCMB076', 'IMCMB077', 'IMCMB078', 'IMCMB079', 'IMCMB080', 'IMCMB081', 'IMCMB082', 'IMCMB083', 'IMCMB084', 'IMCMB085', 'IMCMB086', 'IMCMB087', 'IMCMB088', 'IMCMB089', 'IMCMB090', 'IMCMB091', 'IMCMB092', 'IMCMB093', 'IMCMB094', 'IMCMB095', 'IMCMB096', 'IMCMB097', 'IMCMB098', 'IMCMB099', 'IMCMB100', 'IMCMB101', 'IMCMB102', 'IMCMB103', 'IMCMB104', 'IMCMB105', 'IMCMB106', 'IMCMB107', 'IMCMB108', 'WMETHOD', 'DQAREA', 'DQEXPMAX', 'DQNOIS', 'DQPDPS', 'DQPDAP', 'DQPDPX', 'DQFWHM', 'DQMZ', 'DQSKY', 'DQSKYXGD', 'DQSKYYGD', 'DQMXTIME', 'DQHFTIME', 'DQMXAREA', 'DQHFAREA', 'DQMXFRAC', 'DQHFFRAC', 'DQMXNOIS', 'DQHFNOIS', 'DQMXPDPS', 'DQHFPDPS', 'DQMXPDAP', 'DQHFPDAP', 'DQMXPDPX', 'DQHFPDPX', 'DQSEEMID', 'DQSEESIG', 'DQSEEMIN', 'DQSEEMAX', 'DQSKYMID', 'DQSKYSIG', 'DQSKYMIN', 'DQSKYMAX', 'DQMASK', 'FILTID', 'PHOTBW', 'PHOTFWHM', 'PHOTCLAM', '', 'PIPELINE', 'PLVER', 'EXPMAP', 'ASTRMCAT', 'EFFTIME', 'WCSAXES', 'PLDNAME', '', 'PLQUEUE', 'PLQNAME', 'PLPROCID', 'PLFNAME', 'PLOFNAME', 'DQMFOR', 'PLPROPID', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', 'CHECKSUM', 'DATASUM'] diff --git a/astroquery/noirlab/tests/test_noirlab_remote.py b/astroquery/noirlab/tests/test_noirlab_remote.py new file mode 100644 index 0000000000..9de42a708e --- /dev/null +++ b/astroquery/noirlab/tests/test_noirlab_remote.py @@ -0,0 +1,194 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +# Python library +from __future__ import print_function +# External packages +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.tests.helper import remote_data +# Local packages +from .. import Noirlab +from . import expected as exp + +# performs similar tests as test_module.py, but performs +# the actual HTTP request rather than monkeypatching them. +# should be disabled or enabled at will - use the +# remote_data decorator from astropy: + + +@remote_data +class TestNoirlabClass(object): + + # ############################################################### + # ### (2) SIA; /api/sia/ + # ### + # voimg, vohdu + + def test_service_metadata(self): + """Test compliance with 6.1 of SIA spec v1.0""" + r = Noirlab().service_metadata() + actual = r + print(f'DBG: test_service_metadata={actual}') + expected = exp.service_metadata + assert actual == expected + + def test_query_region_0(self): + """Search FILES using default type (which) selector""" + + c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') + r = Noirlab().query_region(c, radius='0.1') + actual = set(list(r['md5sum'])) + expected = exp.query_region_1 + assert expected.issubset(actual) + + def test_query_region_1(self): + """Search FILES. + Ensure query gets at least the set of files we expect. + Its ok if more files have been added to the remote Archive.""" + + c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') + r = Noirlab(which='file').query_region(c, radius='0.1') + actual = set(list(r['md5sum'])) + expected = exp.query_region_1 + assert expected.issubset(actual) + + def test_query_region_2(self): + """Search HDUs. + Ensure query gets at least the set of files we expect. + Its ok if more files have been added to the remote Archive.""" + + c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') + r = Noirlab(which='hdu').query_region(c, radius='0.07') + actual = set(list(r['md5sum'])) + expected = exp.query_region_2 + assert expected.issubset(actual) + + # ############################################################### + # ### (7) Advanced Search; /api/adv_search/ + # ### + # + # (2) aux_{file,hdu}_fields// + # (2) core_{file,hdu}_fields/ + # [(2) {f,h}adoc JUST LINK to these] + # (2) {f,h}asearch + # cat_list + + # ## + # ## File (default type) + # ## + + def test_aux_file_fields(self): + """List the available AUX FILE fields.""" + r = Noirlab().aux_fields('decam', 'instcal') + actual = r + # Returned results may increase over time. + print(f'DBG: test_aux_file_fields={actual}') + expected = exp.aux_file_fields + assert actual == expected + + def test_core_file_fields(self): + """List the available CORE FILE fields.""" + r = Noirlab().core_fields() + actual = r + print(f'DBG: test_core_file_fields={actual}') + expected = exp.core_file_fields + assert actual == expected + + def test_query_file_metadata(self): + """Search FILE metadata.""" + qspec = { + "outfields": [ + "md5sum", + "archive_filename", + "original_filename", + "instrument", + "proc_type" + ], + "search": [ + ['original_filename', 'c4d_', 'contains'] + ] + } + + r = Noirlab().query_metadata(qspec, limit=3) + actual = r + print(f'DBG: test_query_file_metadata={actual.pformat_all()}') + expected = exp.query_file_metadata + assert actual.pformat_all() == expected + + # ## + # ## HDU + # ## + + def test_aux_hdu_fields(self): + """List the available AUX HDU fields.""" + r = Noirlab(which='hdu').aux_fields('decam', 'instcal') + actual = r + # Returned results may increase over time. + print(f'DBG: test_aux_hdu_fields={actual}') + expected = exp.aux_hdu_fields + assert actual == expected + + def test_core_hdu_fields(self): + """List the available CORE HDU fields.""" + r = Noirlab(which='hdu').core_fields() + actual = r + print(f'DBG: test_core_file_fields={actual}') + expected = exp.core_hdu_fields + assert actual == expected + + def test_query_hdu_metadata(self): + """Search HDU metadata.""" + qspec = { + "outfields": [ + "fitsfile__archive_filename", + "fitsfile__caldat", + "fitsfile__instrument", + "fitsfile__proc_type", + "AIRMASS" # AUX field. Slows search + ], + "search": [ + ["fitsfile__caldat", "2017-08-14", "2017-08-16"], + ["fitsfile__instrument", "decam"], + ["fitsfile__proc_type", "raw"] + ] + } + + r = Noirlab(which='hdu').query_metadata(qspec, limit=3) + actual = r + print(f'DBG: test_query_hdu_metadata={actual.pformat_all()}') + expected = exp.query_hdu_metadata + assert actual.pformat_all() == expected + + # ## + # ## Agnostic + # ## + + def test_categoricals(self): + """List categories.""" + r = Noirlab().categoricals() + actual = r + # Returned results may increase over time. + print(f'DBG: test_categoricals={actual}') + expected = exp.categoricals + assert actual == expected + + # ############################################################## + # ### (3) Other + # get_token + # retrieve/ + # version + + def test_retrieve(self): + hdul = Noirlab().retrieve('f92541fdc566dfebac9e7d75e12b5601') + actual = list(hdul[0].header.keys()) + expected = exp.retrieve + assert actual == expected + + def test_version(self): + r = Noirlab().version() + assert r < 3.0 + + def test_get_token(self): + actual = Noirlab().get_token('nobody@university.edu', '123456789') + expected = {'detail': + 'No active account found with the given credentials'} + assert actual == expected diff --git a/docs/noirlab/noirlab.rst b/docs/noirlab/noirlab.rst new file mode 100644 index 0000000000..3b8534e92a --- /dev/null +++ b/docs/noirlab/noirlab.rst @@ -0,0 +1,158 @@ +.. doctest-skip-all + +.. _astroquery.noirlab: + +************************************ +About the NOIRLab Astro Data Archive +************************************ + +The NOIRLab Astro Data Archive (formerly NOAO Science Archive) +provides access to data taken with more than 40 telescope and +instrument combinations, including those operated in partnership with +the WIYN, SOAR and SMARTS consortia, from semester 2004B to the +present. In addition to raw data, pipeline-reduced data products from +the DECam, Mosaic and NEWFIRM imagers are also available, as well as +advanced data products delivered by teams carrying out surveys and +other large observing programs with NSF OIR Lab facilities. + +For more info about our holdings see the +`NOIRLab Astro Data Archive `_ + +Acknowledgment +============== + +This research uses services or data provided by the Astro Data Archive +at NSF's National Optical-Infrared Astronomy Research +Laboratory. NSF's OIR Lab is operated by the Association of +Universities for Research in Astronomy (AURA), Inc. under a +cooperative agreement with the National Science Foundation. + +************************************** +NOIRLab Queries (`astroquery.noirlab`) +************************************** + +The methods in this module are wrappers around a set of web-services +which can be accessed at +`Rest API documentation `_ +(which is the most +up-to-date info on the web-services). +This data archive is hosted at +`NOIR-CSDC `_. + + +Getting started (SIA) +===================== + +This module supports fetching the table of observation summaries from +the `NOIRLab data archive `_ given +your *Region Of Interest* query values. + +In general, you can query the +`NOIRLab data archive `_ +against full FITS files or against HDUs of those FITS files. Most +users will likely prefer results against full FITS files (the +default). The search will likely be faster when searching files +compared to searching HDUs. If you are trying to retrieve HDU specific +image data from large files that contain many HDUs (such as DECam), +you can reduce your download time considerably by getting only +matching HDUs. + +The results are returned in a `~astropy.table.Table`. The service +can be queried using the :meth:`~astroquery.noirlab.NoirlabClass.query_region`. The +only required argument to this is the target coordinates around which +to query. Specify the coordinates using the appropriate coordinate system from +`astropy.coordinates`. Here is a basic example: + +.. code-block:: python + + >>> from astroquery.noirlab import Noirlab + >>> import astropy.coordinates as coord + >>> from astropy import units as u + >>> from astropy.coordinates import SkyCoord + >>> coord = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') + + >>> noirlab_file = Noirlab(which='file') + >>> results_file = noirlab_file.query_region(coord, radius='0.1') + >>> print(results_file) + archive_filename date_obs ... updated + str71 str10 ... str32 + ----------------------------------------------------------------------- ---------- ... -------------------------------- + string string ... string + /net/archive/mtn/20151027/kp4m/2015B-2001/k4m_151028_085849_ori.fits.fz 2015-10-28 ... 2020-02-09T01:17:17.842642+00:00 + /net/archive/mtn/20151027/kp4m/2015B-2001/k4m_151028_091327_ori.fits.fz 2015-10-28 ... 2020-02-09T01:17:17.875407+00:00 + /net/archive/mtn/20151027/kp4m/2015B-2001/k4m_151028_084112_ori.fits.fz 2015-10-28 ... 2020-02-09T01:17:18.303911+00:00 + /net/archive/mtn/20151120/kp4m/2015B-2001/k4m_151121_033641_ori.fits.fz 2015-11-21 ... 2020-02-09T01:24:35.525861+00:00 + /net/archive/mtn/20151120/kp4m/2015B-2001/k4m_151121_031258_ori.fits.fz 2015-11-21 ... 2020-02-09T01:24:37.873559+00:00 + /net/archive/mtn/20151120/kp4m/2015B-2001/k4m_151121_041031_ori.fits.fz 2015-11-21 ... 2020-02-09T01:24:38.951230+00:00 + +This is an example of searching by HDU. +.. note:: + + Only some instruments have pipeline processing that populates the RA, DEC fields used for this search. + +.. code-block:: python + + >>> noirlab_hdu = Noirlab(which='hdu') + >>> results_hdu = noirlab_hdu.query_region(coord, radius='0.1') + >>> print(results_hdu) + archive_filename caldat date_obs ... ra telescope + str79 str10 str10 ... str8 str6 + ------------------------------------------------------------------------------- ---------- ---------- ... -------- --------- + string string string ... float string + /net/archive/pipe/20180211/kp4m/2016A-0453/k4m_180212_021444_ooi_zd_ls9.fits.fz 2018-02-11 2018-02-12 ... 10.60048 kp4m + /net/archive/pipe/20180211/kp4m/2016A-0453/k4m_180212_030041_ooi_zd_v2.fits.fz 2018-02-11 2018-02-12 ... 10.43286 kp4m + /net/archive/pipe/20151120/kp4m/k4m_151121_031038_ooi_zd_v1.fits.fz 2015-11-20 2015-11-21 ... 10.58226 kp4m + /net/archive/pipe/20180211/kp4m/2016A-0453/k4m_180212_023526_ooi_zd_v2.fits.fz 2018-02-11 2018-02-12 ... 10.58187 kp4m + /net/archive/pipe/20151120/kp4m/k4m_151121_031646_ooi_zd_v1.fits.fz 2015-11-20 2015-11-21 ... 10.56906 kp4m + /net/archive/pipe/20151120/kp4m/k4m_151121_030945_ooi_zd_v1.fits.fz 2015-11-20 2015-11-21 ... 10.58203 kp4m + /net/archive/pipe/20151120/kp4m/k4m_151121_031124_ooi_zd_v1.fits.fz 2015-11-20 2015-11-21 ... 10.58549 kp4m + + +Advanced Search +=============== + +This set of methods supports **arbitrary searches of any fields** +stored in the FITS headers of the Archive. Common fields ("core" +fields) are optimized for search speed. Less common fields ("aux" +fields) will be slower to search. You can search by File or HDU. The +primary method for doing the search in ``query_metadata``. That query +requires a ``JSON`` structure to define the query. We often call this +the *JSON search spec*. Many of the other +methods with this module are here to provide you with the information +you need to construct the ``JSON`` structure. +Summaries of the mechanisms available in the JSON search spec for +`File search `_ +and for `HDU search +`_ +are on the NOIRLab Data Archive website. + +There are three methods who's sole purpose if providing you with +information to help you with the content of your ``JSON`` structure. +They are: + +#. aux_fields() +#. core_fields() +#. categoricals() + +See the Reference/API below for details. The categoricals() method +returns a list of all the "category strings" such as names of +Instruments and Telescopes. The aux/core_fields methods +tell you what fields are available to search. The core fields are +available for all instruments are the search for them is fast. The aux +fields require you to specify instrument and proctype. The set of +fields available is highly dependent on those two fields. The +Instrument determines aux fields in raw files. Proctype determines +what kind of pipeline processing was done. Pipeline processing often +adds important (aux) fields. + + + + + + +Reference/API +============= + + +.. automodapi:: astroquery.noirlab + :no-inheritance-diagram: From e6a7daccdccafafb35d439213c4e9a680780f046 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 26 Jun 2025 15:05:40 -0700 Subject: [PATCH 02/26] add noirlab.rst to index --- docs/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/index.rst b/docs/index.rst index 1f6f5e7850..207f3388ab 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -284,6 +284,7 @@ The following modules have been completed using a common API: nasa_ads/nasa_ads.rst ipac/ned/ned.rst nist/nist.rst + noirlab/noirlab.rst nvas/nvas.rst simbad/simbad.rst skyview/skyview.rst From dd77a3f4559be8260645659acfa7265ff87758fb Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 26 Jun 2025 15:18:28 -0700 Subject: [PATCH 03/26] update change log --- CHANGES.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index e3af10bd20..5da05204a4 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -65,6 +65,12 @@ ipac.irsa in to return all TAP tables, including non-spatial and metadata ones, too. [#3334] +noirlab +^^^^^^^ + +- Restore access to the `NSF NOIRLab `_ + `Astro Data Archive `_ [#3359]. + SIMBAD ^^^^^^ From 9b093d15d380b842f1866ccd80e2ed6770986e35 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 26 Jun 2025 15:38:27 -0700 Subject: [PATCH 04/26] update url and tests --- astroquery/noirlab/__init__.py | 21 ++++----- astroquery/noirlab/core.py | 8 ++-- .../noirlab/tests/test_noirlab_remote.py | 32 +++++++------- docs/noirlab/noirlab.rst | 44 ++++++++----------- 4 files changed, 48 insertions(+), 57 deletions(-) diff --git a/astroquery/noirlab/__init__.py b/astroquery/noirlab/__init__.py index fb878d8255..94b3139f33 100644 --- a/astroquery/noirlab/__init__.py +++ b/astroquery/noirlab/__init__.py @@ -1,7 +1,7 @@ -from astropy import config as _config # Licensed under a 3-clause BSD style license - see LICENSE.rst -"""NSF's OIR Lab Astro Data Archive (Beta) ------------------------------------------- +""" +NSF NOIRLab Astro Data Archive +------------------------------ The NSF's OIR Lab Astro Data Archive (formerly NOAO Science Archive) provides access to data taken with more than 40 telescope and @@ -55,6 +55,7 @@ cooperative agreement with the National Science Foundation. """ +from astropy import config as _config class Conf(_config.ConfigNamespace): @@ -62,23 +63,19 @@ class Conf(_config.ConfigNamespace): Configuration parameters for `astroquery.noirlab`. """ server = _config.ConfigItem( - ['https://astroarchive.noao.edu', + ['https://astroarchive.noirlab.edu', ], - 'Name of the NOIRLAB server to use.' + 'Name of the NSF NOIRLab server to use.' ) timeout = _config.ConfigItem( 30, - 'Time limit for connecting to NOIRLAB server.' + 'Time limit for connecting to NSF NOIRLab server.' ) conf = Conf() -from .core import Noirlab, NoirlabClass # noqa +from .core import NOIRLab, NOIRLabClass # noqa - -__all__ = ['Noirlab', 'NoirlabClass', +__all__ = ['NOIRLab', 'NOIRLabClass', 'conf', 'Conf'] - - -# noqa diff --git a/astroquery/noirlab/core.py b/astroquery/noirlab/core.py index dfe08edf02..91d7d0a5bb 100644 --- a/astroquery/noirlab/core.py +++ b/astroquery/noirlab/core.py @@ -1,5 +1,5 @@ """ -Provide astroquery API access to OIR Lab Astro Data Archive (natica). +Provide astroquery API access to NSF NOIRLab Astro Data Archive. This does DB access through web-services. """ @@ -11,11 +11,11 @@ from . import conf -__all__ = ['Noirlab', 'NoirlabClass'] # specifies what to import +__all__ = ['NOIRLab', 'NOIRLabClass'] # specifies what to import @async_to_sync -class NoirlabClass(BaseQuery): +class NOIRLabClass(BaseQuery): TIMEOUT = conf.timeout NAT_URL = conf.server @@ -208,4 +208,4 @@ def get_token(self, email, password, cache=True): return response.json() -Noirlab = NoirlabClass() +NOIRLab = NOIRLabClass() diff --git a/astroquery/noirlab/tests/test_noirlab_remote.py b/astroquery/noirlab/tests/test_noirlab_remote.py index 9de42a708e..800f759a58 100644 --- a/astroquery/noirlab/tests/test_noirlab_remote.py +++ b/astroquery/noirlab/tests/test_noirlab_remote.py @@ -6,7 +6,7 @@ from astropy.coordinates import SkyCoord from astropy.tests.helper import remote_data # Local packages -from .. import Noirlab +from .. import NOIRLab, NOIRLabClass from . import expected as exp # performs similar tests as test_module.py, but performs @@ -16,7 +16,7 @@ @remote_data -class TestNoirlabClass(object): +class TestNOIRLabClass(object): # ############################################################### # ### (2) SIA; /api/sia/ @@ -25,7 +25,7 @@ class TestNoirlabClass(object): def test_service_metadata(self): """Test compliance with 6.1 of SIA spec v1.0""" - r = Noirlab().service_metadata() + r = NOIRLab.service_metadata() actual = r print(f'DBG: test_service_metadata={actual}') expected = exp.service_metadata @@ -35,7 +35,7 @@ def test_query_region_0(self): """Search FILES using default type (which) selector""" c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') - r = Noirlab().query_region(c, radius='0.1') + r = NOIRLab.query_region(c, radius='0.1') actual = set(list(r['md5sum'])) expected = exp.query_region_1 assert expected.issubset(actual) @@ -46,7 +46,7 @@ def test_query_region_1(self): Its ok if more files have been added to the remote Archive.""" c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') - r = Noirlab(which='file').query_region(c, radius='0.1') + r = NOIRLabClass(which='file').query_region(c, radius='0.1') actual = set(list(r['md5sum'])) expected = exp.query_region_1 assert expected.issubset(actual) @@ -57,7 +57,7 @@ def test_query_region_2(self): Its ok if more files have been added to the remote Archive.""" c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') - r = Noirlab(which='hdu').query_region(c, radius='0.07') + r = NOIRLabClass(which='hdu').query_region(c, radius='0.07') actual = set(list(r['md5sum'])) expected = exp.query_region_2 assert expected.issubset(actual) @@ -78,7 +78,7 @@ def test_query_region_2(self): def test_aux_file_fields(self): """List the available AUX FILE fields.""" - r = Noirlab().aux_fields('decam', 'instcal') + r = NOIRLab.aux_fields('decam', 'instcal') actual = r # Returned results may increase over time. print(f'DBG: test_aux_file_fields={actual}') @@ -87,7 +87,7 @@ def test_aux_file_fields(self): def test_core_file_fields(self): """List the available CORE FILE fields.""" - r = Noirlab().core_fields() + r = NOIRLab.core_fields() actual = r print(f'DBG: test_core_file_fields={actual}') expected = exp.core_file_fields @@ -108,7 +108,7 @@ def test_query_file_metadata(self): ] } - r = Noirlab().query_metadata(qspec, limit=3) + r = NOIRLab.query_metadata(qspec, limit=3) actual = r print(f'DBG: test_query_file_metadata={actual.pformat_all()}') expected = exp.query_file_metadata @@ -120,7 +120,7 @@ def test_query_file_metadata(self): def test_aux_hdu_fields(self): """List the available AUX HDU fields.""" - r = Noirlab(which='hdu').aux_fields('decam', 'instcal') + r = NOIRLabClass(which='hdu').aux_fields('decam', 'instcal') actual = r # Returned results may increase over time. print(f'DBG: test_aux_hdu_fields={actual}') @@ -129,7 +129,7 @@ def test_aux_hdu_fields(self): def test_core_hdu_fields(self): """List the available CORE HDU fields.""" - r = Noirlab(which='hdu').core_fields() + r = NOIRLabClass(which='hdu').core_fields() actual = r print(f'DBG: test_core_file_fields={actual}') expected = exp.core_hdu_fields @@ -152,7 +152,7 @@ def test_query_hdu_metadata(self): ] } - r = Noirlab(which='hdu').query_metadata(qspec, limit=3) + r = NOIRLabClass(which='hdu').query_metadata(qspec, limit=3) actual = r print(f'DBG: test_query_hdu_metadata={actual.pformat_all()}') expected = exp.query_hdu_metadata @@ -164,7 +164,7 @@ def test_query_hdu_metadata(self): def test_categoricals(self): """List categories.""" - r = Noirlab().categoricals() + r = NOIRLab.categoricals() actual = r # Returned results may increase over time. print(f'DBG: test_categoricals={actual}') @@ -178,17 +178,17 @@ def test_categoricals(self): # version def test_retrieve(self): - hdul = Noirlab().retrieve('f92541fdc566dfebac9e7d75e12b5601') + hdul = NOIRLab.retrieve('f92541fdc566dfebac9e7d75e12b5601') actual = list(hdul[0].header.keys()) expected = exp.retrieve assert actual == expected def test_version(self): - r = Noirlab().version() + r = NOIRLab.version() assert r < 3.0 def test_get_token(self): - actual = Noirlab().get_token('nobody@university.edu', '123456789') + actual = NOIRLab.get_token('nobody@university.edu', '123456789') expected = {'detail': 'No active account found with the given credentials'} assert actual == expected diff --git a/docs/noirlab/noirlab.rst b/docs/noirlab/noirlab.rst index 3b8534e92a..a57661ecf4 100644 --- a/docs/noirlab/noirlab.rst +++ b/docs/noirlab/noirlab.rst @@ -2,11 +2,11 @@ .. _astroquery.noirlab: -************************************ -About the NOIRLab Astro Data Archive -************************************ +**************************************** +About the NSF NOIRLab Astro Data Archive +**************************************** -The NOIRLab Astro Data Archive (formerly NOAO Science Archive) +The NSF NOIRLab Astro Data Archive (formerly NOAO Science Archive) provides access to data taken with more than 40 telescope and instrument combinations, including those operated in partnership with the WIYN, SOAR and SMARTS consortia, from semester 2004B to the @@ -15,8 +15,8 @@ the DECam, Mosaic and NEWFIRM imagers are also available, as well as advanced data products delivered by teams carrying out surveys and other large observing programs with NSF OIR Lab facilities. -For more info about our holdings see the -`NOIRLab Astro Data Archive `_ +For more info about our holdings see the +`NSF NOIRLab Astro Data Archive `_ Acknowledgment ============== @@ -31,24 +31,22 @@ cooperative agreement with the National Science Foundation. NOIRLab Queries (`astroquery.noirlab`) ************************************** -The methods in this module are wrappers around a set of web-services -which can be accessed at -`Rest API documentation `_ -(which is the most -up-to-date info on the web-services). -This data archive is hosted at -`NOIR-CSDC `_. +The methods in this module are wrappers around a set of web services +described in the +`Rest API documentation `_. +This data archive is hosted at the +`Community Science and Data Center (CDSC) `_. Getting started (SIA) ===================== This module supports fetching the table of observation summaries from -the `NOIRLab data archive `_ given +the `NOIRLab data archive `_ given your *Region Of Interest* query values. In general, you can query the -`NOIRLab data archive `_ +`NOIRLab data archive `_ against full FITS files or against HDUs of those FITS files. Most users will likely prefer results against full FITS files (the default). The search will likely be faster when searching files @@ -91,7 +89,7 @@ This is an example of searching by HDU. Only some instruments have pipeline processing that populates the RA, DEC fields used for this search. .. code-block:: python - + >>> noirlab_hdu = Noirlab(which='hdu') >>> results_hdu = noirlab_hdu.query_region(coord, radius='0.1') >>> print(results_hdu) @@ -121,19 +119,19 @@ the *JSON search spec*. Many of the other methods with this module are here to provide you with the information you need to construct the ``JSON`` structure. Summaries of the mechanisms available in the JSON search spec for -`File search `_ +`File search `_ and for `HDU search -`_ -are on the NOIRLab Data Archive website. +`_ +are on the NSF NOIRLab Data Archive website. -There are three methods who's sole purpose if providing you with +There are three methods whose sole purpose if providing you with information to help you with the content of your ``JSON`` structure. They are: #. aux_fields() #. core_fields() #. categoricals() - + See the Reference/API below for details. The categoricals() method returns a list of all the "category strings" such as names of Instruments and Telescopes. The aux/core_fields methods @@ -146,10 +144,6 @@ what kind of pipeline processing was done. Pipeline processing often adds important (aux) fields. - - - - Reference/API ============= From e5c647b51a3254bb7116d6ff869d866e97b41011 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 26 Jun 2025 16:02:55 -0700 Subject: [PATCH 05/26] fix remote data import --- astroquery/noirlab/__init__.py | 13 ++++--------- astroquery/noirlab/tests/test_noirlab_remote.py | 6 ++---- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/astroquery/noirlab/__init__.py b/astroquery/noirlab/__init__.py index 94b3139f33..94b1d0da92 100644 --- a/astroquery/noirlab/__init__.py +++ b/astroquery/noirlab/__init__.py @@ -62,15 +62,10 @@ class Conf(_config.ConfigNamespace): """ Configuration parameters for `astroquery.noirlab`. """ - server = _config.ConfigItem( - ['https://astroarchive.noirlab.edu', - ], - 'Name of the NSF NOIRLab server to use.' - ) - timeout = _config.ConfigItem( - 30, - 'Time limit for connecting to NSF NOIRLab server.' - ) + server = _config.ConfigItem(['https://astroarchive.noirlab.edu',], + 'Name of the NSF NOIRLab server to use.') + timeout = _config.ConfigItem(30, + 'Time limit for connecting to NSF NOIRLab server.') conf = Conf() diff --git a/astroquery/noirlab/tests/test_noirlab_remote.py b/astroquery/noirlab/tests/test_noirlab_remote.py index 800f759a58..4f4a6437f8 100644 --- a/astroquery/noirlab/tests/test_noirlab_remote.py +++ b/astroquery/noirlab/tests/test_noirlab_remote.py @@ -1,10 +1,8 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst -# Python library -from __future__ import print_function # External packages +import pytest from astropy import units as u from astropy.coordinates import SkyCoord -from astropy.tests.helper import remote_data # Local packages from .. import NOIRLab, NOIRLabClass from . import expected as exp @@ -15,7 +13,7 @@ # remote_data decorator from astropy: -@remote_data +@pytest.mark.remote_data class TestNOIRLabClass(object): # ############################################################### From a02dbefbd802d1517829e6bd643a09c5fedaff10 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 26 Jun 2025 16:27:28 -0700 Subject: [PATCH 06/26] fix doc errors --- astroquery/noirlab/tests/test_noirlab_remote.py | 4 ++-- docs/noirlab/noirlab.rst | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/astroquery/noirlab/tests/test_noirlab_remote.py b/astroquery/noirlab/tests/test_noirlab_remote.py index 4f4a6437f8..8582b7c4a8 100644 --- a/astroquery/noirlab/tests/test_noirlab_remote.py +++ b/astroquery/noirlab/tests/test_noirlab_remote.py @@ -25,7 +25,7 @@ def test_service_metadata(self): """Test compliance with 6.1 of SIA spec v1.0""" r = NOIRLab.service_metadata() actual = r - print(f'DBG: test_service_metadata={actual}') + # print(f'DBG: test_service_metadata={actual}') expected = exp.service_metadata assert actual == expected @@ -183,7 +183,7 @@ def test_retrieve(self): def test_version(self): r = NOIRLab.version() - assert r < 3.0 + assert r >= 6.0 def test_get_token(self): actual = NOIRLab.get_token('nobody@university.edu', '123456789') diff --git a/docs/noirlab/noirlab.rst b/docs/noirlab/noirlab.rst index a57661ecf4..e05ecf1e2b 100644 --- a/docs/noirlab/noirlab.rst +++ b/docs/noirlab/noirlab.rst @@ -56,20 +56,20 @@ you can reduce your download time considerably by getting only matching HDUs. The results are returned in a `~astropy.table.Table`. The service -can be queried using the :meth:`~astroquery.noirlab.NoirlabClass.query_region`. The +can be queried using the :meth:`~astroquery.noirlab.NOIRLabClass.query_region`. The only required argument to this is the target coordinates around which to query. Specify the coordinates using the appropriate coordinate system from `astropy.coordinates`. Here is a basic example: .. code-block:: python - >>> from astroquery.noirlab import Noirlab + >>> from astroquery.noirlab import NOIRLab >>> import astropy.coordinates as coord >>> from astropy import units as u >>> from astropy.coordinates import SkyCoord >>> coord = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') - >>> noirlab_file = Noirlab(which='file') + >>> noirlab_file = NOIRLab(which='file') >>> results_file = noirlab_file.query_region(coord, radius='0.1') >>> print(results_file) archive_filename date_obs ... updated @@ -90,7 +90,8 @@ This is an example of searching by HDU. .. code-block:: python - >>> noirlab_hdu = Noirlab(which='hdu') + >>> from astroquery.noirlab import NOIRLabClass + >>> noirlab_hdu = NOIRLabClass(which='hdu') >>> results_hdu = noirlab_hdu.query_region(coord, radius='0.1') >>> print(results_hdu) archive_filename caldat date_obs ... ra telescope From bb3bbef22ed128430de58e75c25e45463b55c122 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 26 Jun 2025 17:01:15 -0700 Subject: [PATCH 07/26] add local test --- astroquery/noirlab/tests/expected.py | 3 +- astroquery/noirlab/tests/test_noirlab.py | 41 +++++++++++++++++++ .../noirlab/tests/test_noirlab_remote.py | 14 ++++--- 3 files changed, 51 insertions(+), 7 deletions(-) create mode 100644 astroquery/noirlab/tests/test_noirlab.py diff --git a/astroquery/noirlab/tests/expected.py b/astroquery/noirlab/tests/expected.py index 01a0a6c235..ec55205d5a 100644 --- a/astroquery/noirlab/tests/expected.py +++ b/astroquery/noirlab/tests/expected.py @@ -39,9 +39,10 @@ core_hdu_fields = [{'Field': 'boundary', 'Type': 'str'}, {'Field': 'dec', 'Type': 'np.float64'}, {'Field': 'dec_range', 'Type': 'str'}, {'Field': 'fitsfile', 'Type': 'str'}, {'Field': 'fitsfile__archive_filename', 'Type': 'str'}, {'Field': 'fitsfile__caldat', 'Type': 'str'}, {'Field': 'fitsfile__date_obs_max', 'Type': 'str'}, {'Field': 'fitsfile__date_obs_min', 'Type': 'str'}, {'Field': 'fitsfile__dec_max', 'Type': 'str'}, {'Field': 'fitsfile__dec_min', 'Type': 'str'}, {'Field': 'fitsfile__depth', 'Type': 'str'}, {'Field': 'fitsfile__exposure', 'Type': 'str'}, {'Field': 'fitsfile__filesize', 'Type': 'str'}, {'Field': 'fitsfile__ifilter', 'Type': 'str'}, {'Field': 'fitsfile__instrument', 'Type': 'str'}, {'Field': 'fitsfile__md5sum', 'Type': 'str'}, {'Field': 'fitsfile__obs_mode', 'Type': 'str'}, {'Field': 'fitsfile__obs_type', 'Type': 'str'}, {'Field': 'fitsfile__original_filename', 'Type': 'str'}, {'Field': 'fitsfile__proc_type', 'Type': 'str'}, {'Field': 'fitsfile__prod_type', 'Type': 'str'}, {'Field': 'fitsfile__proposal', 'Type': 'str'}, {'Field': 'fitsfile__ra_max', 'Type': 'str'}, {'Field': 'fitsfile__ra_min', 'Type': 'str'}, {'Field': 'fitsfile__release_date', 'Type': 'str'}, {'Field': 'fitsfile__seeing', 'Type': 'str'}, {'Field': 'fitsfile__site', 'Type': 'str'}, {'Field': 'fitsfile__survey', 'Type': 'str'}, {'Field': 'fitsfile__telescope', 'Type': 'str'}, {'Field': 'fitsfile__updated', 'Type': 'str'}, {'Field': 'hdu_idx', 'Type': 'np.int64'}, {'Field': 'id', 'Type': 'str'}, {'Field': 'ra', 'Type': 'np.float64'}, {'Field': 'ra_range', 'Type': 'str'}, {'Field': 'updated', 'Type': 'datetime64'}] - query_hdu_metadata = ['AIRMASS fitsfile__archive_filename fitsfile__caldat fitsfile__instrument fitsfile__proc_type', '------- ----------------------------------------------------------------------- ---------------- -------------------- -------------------', ' None /net/archive/mtn/20170814/ct4m/2012B-0001/c4d_170815_051354_ori.fits.fz 2017-08-14 decam raw', ' None /net/archive/mtn/20170814/ct4m/2012B-0001/c4d_170815_051354_ori.fits.fz 2017-08-14 decam raw', ' None /net/archive/mtn/20170814/ct4m/2012B-0001/c4d_170815_051354_ori.fits.fz 2017-08-14 decam raw'] categoricals = {'instruments': ['(p)odi', '90prime', 'andicam', 'arcoiris', 'arcon', 'bench', 'ccd_imager', 'chiron', 'cosmos', 'decam', 'echelle', 'falmingos', 'flamingos', 'goodman', 'goodman spectrograph', 'gtcam', 'hdi', 'ice', 'ispi', 'kosmos', 'minimo/ice', 'mop/ice', 'mosaic', 'mosaic3', 'mosaic_1', 'mosaic_1_1', 'mosaic_2', 'newfirm', 'osiris', 'sami', 'soi', 'spartan', 'spartan ir camera', 'triplespec', 'wfc', 'whirc', 'wildfire', 'y4kcam'], 'obsmodes': [], 'proctypes': ['instcal', 'mastercal', 'nota', 'projected', 'raw', 'resampled', 'skysub', 'stacked'], 'prodtypes': ['dqmask', 'expmap', 'graphics (size)', 'image', 'image 2nd version 1', 'image1', 'nota', 'resampled', 'weight', 'wtmap'], 'sites': ['cp', 'ct', 'kp', 'lp'], 'surveys': [], 'telescopes': ['bok23m', 'ct09m', 'ct13m', 'ct15m', 'ct1m', 'ct4m', 'ctlab', 'kp09m', 'kp21m', 'kp35m', 'kp4m', 'kpcf', 'lp25m', 'soar', 'wiyn']} retrieve = ['SIMPLE', 'BITPIX', 'NAXIS', 'EXTEND', 'ORIGIN', 'DATE', 'IRAF-TLM', 'OBJECT', 'RAWFILE', 'FILENAME', 'OBJRA', 'OBJDEC', 'OBJEPOCH', '', 'TIMESYS', '', 'OBSERVAT', 'OBS-ELEV', 'OBS-LAT', 'OBS-LONG', 'TELESCOP', 'TELRADEC', 'TELEQUIN', 'TELRA', 'TELDEC', 'HA', 'ZD', 'TELFOCUS', '', 'MOSSIZE', 'NDETS', '', 'OBSERVER', 'PROPOSER', 'PROPID', 'SEQID', 'SEQNUM', '', 'NOHS', 'NOCUTC', 'NOCRSD', 'NOCGID', 'NOCNAME', '', 'DTSITE', 'DTTELESC', 'DTINSTRU', 'DTCALDAT', 'DTUTC', 'DTOBSERV', 'DTPROPID', 'DTPI', 'DTPIAFFL', 'DTTITLE', 'DTCOPYRI', 'DTACQUIS', 'DTACCOUN', 'DTACQNAM', 'DTNSANAM', 'DTSTATUS', 'SB_HOST', 'SB_ACCOU', 'SB_SITE', 'SB_LOCAL', 'SB_DIR1', 'SB_DIR2', 'SB_DIR3', 'SB_RECNO', 'SB_ID', 'SB_NAME', 'RMCOUNT', 'RECNO', 'INSTRUME', 'RSPTGRP', 'RSPGRP', '', 'OBSTYPE', 'PROCTYPE', 'PRODTYPE', 'MIMETYPE', 'EXPTIME', 'FILTER', '', 'RA', 'DEC', 'CENTRA', 'CORN1RA', 'CORN2RA', 'CORN3RA', 'CORN4RA', 'CENTDEC', 'CORN1DEC', 'CORN2DEC', 'CORN3DEC', 'CORN4DEC', 'DATE-OBS', 'TIME-OBS', 'MJD-OBS', 'ST', '', 'CTYPE1', 'CTYPE2', 'CRVAL1', 'CRVAL2', 'CD1_1', 'CD2_2', 'WAT0_001', 'WAT1_001', 'WAT2_001', '', 'DIGAVGS', 'NCOADD', 'FSAMPLE', 'EXPCOADD', 'TITLE', 'DARKFIL', 'DARKINFO', 'LINIMAGE', 'LINCOEFF', 'BUNIT', 'GAIN', 'OEXPTIME', 'MAGZREF', 'PHOTINDX', 'WCSCAL', 'WCSXRMS', 'WCSYRMS', 'MAGZERO', '', 'MAGZSIG', 'MAGZERR', 'MAGZNAV', 'SEEINGP', 'SKYBG', 'SEEING', 'SKYMAG', 'STKBPM', 'OBJBPM', 'CDELT1', 'CDELT2', 'CRPIX1', 'CRPIX2', 'BPM', 'IMCMB001', 'IMCMB002', 'IMCMB003', 'IMCMB004', 'IMCMB005', 'IMCMB006', 'IMCMB007', 'IMCMB008', 'IMCMB009', 'IMCMB010', 'IMCMB011', 'IMCMB012', 'IMCMB013', 'IMCMB014', 'IMCMB015', 'IMCMB016', 'IMCMB017', 'IMCMB018', 'IMCMB019', 'IMCMB020', 'IMCMB021', 'IMCMB022', 'IMCMB023', 'IMCMB024', 'IMCMB025', 'IMCMB026', 'IMCMB027', 'IMCMB028', 'IMCMB029', 'IMCMB030', 'IMCMB031', 'IMCMB032', 'IMCMB033', 'IMCMB034', 'IMCMB035', 'IMCMB036', 'IMCMB037', 'IMCMB038', 'IMCMB039', 'IMCMB040', 'IMCMB041', 'IMCMB042', 'IMCMB043', 'IMCMB044', 'IMCMB045', 'IMCMB046', 'IMCMB047', 'IMCMB048', 'IMCMB049', 'IMCMB050', 'IMCMB051', 'IMCMB052', 'IMCMB053', 'IMCMB054', 'IMCMB055', 'IMCMB056', 'IMCMB057', 'IMCMB058', 'IMCMB059', 'IMCMB060', 'IMCMB061', 'IMCMB062', 'IMCMB063', 'IMCMB064', 'IMCMB065', 'IMCMB066', 'IMCMB067', 'IMCMB068', 'IMCMB069', 'IMCMB070', 'IMCMB071', 'IMCMB072', 'IMCMB073', 'IMCMB074', 'IMCMB075', 'IMCMB076', 'IMCMB077', 'IMCMB078', 'IMCMB079', 'IMCMB080', 'IMCMB081', 'IMCMB082', 'IMCMB083', 'IMCMB084', 'IMCMB085', 'IMCMB086', 'IMCMB087', 'IMCMB088', 'IMCMB089', 'IMCMB090', 'IMCMB091', 'IMCMB092', 'IMCMB093', 'IMCMB094', 'IMCMB095', 'IMCMB096', 'IMCMB097', 'IMCMB098', 'IMCMB099', 'IMCMB100', 'IMCMB101', 'IMCMB102', 'IMCMB103', 'IMCMB104', 'IMCMB105', 'IMCMB106', 'IMCMB107', 'IMCMB108', 'WMETHOD', 'DQAREA', 'DQEXPMAX', 'DQNOIS', 'DQPDPS', 'DQPDAP', 'DQPDPX', 'DQFWHM', 'DQMZ', 'DQSKY', 'DQSKYXGD', 'DQSKYYGD', 'DQMXTIME', 'DQHFTIME', 'DQMXAREA', 'DQHFAREA', 'DQMXFRAC', 'DQHFFRAC', 'DQMXNOIS', 'DQHFNOIS', 'DQMXPDPS', 'DQHFPDPS', 'DQMXPDAP', 'DQHFPDAP', 'DQMXPDPX', 'DQHFPDPX', 'DQSEEMID', 'DQSEESIG', 'DQSEEMIN', 'DQSEEMAX', 'DQSKYMID', 'DQSKYSIG', 'DQSKYMIN', 'DQSKYMAX', 'DQMASK', 'FILTID', 'PHOTBW', 'PHOTFWHM', 'PHOTCLAM', '', 'PIPELINE', 'PLVER', 'EXPMAP', 'ASTRMCAT', 'EFFTIME', 'WCSAXES', 'PLDNAME', '', 'PLQUEUE', 'PLQNAME', 'PLPROCID', 'PLFNAME', 'PLOFNAME', 'DQMFOR', 'PLPROPID', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', 'CHECKSUM', 'DATASUM'] + +version = '6.0' diff --git a/astroquery/noirlab/tests/test_noirlab.py b/astroquery/noirlab/tests/test_noirlab.py new file mode 100644 index 0000000000..3f1f3ac528 --- /dev/null +++ b/astroquery/noirlab/tests/test_noirlab.py @@ -0,0 +1,41 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +Test astroquery.noirlab, but monkeypatch any HTTP requests. +""" +# External packages +import pytest +# from astropy import units as u +# from astropy.coordinates import SkyCoord +# Local packages +from ...utils.mocks import MockResponse +from .. import NOIRLab, NOIRLabClass +from . import expected as exp + + +@pytest.fixture +def patch_request(request): + def mockreturn(method, url, **kwargs): + # if 'data' in kwargs: + # cmd = kwargs['data']['uquery'] + # else: + # cmd = kwargs['params']['cmd'] + # if 'SpecObjAll' in cmd: + # filename = data_path(DATA_FILES['spectra_id']) + # else: + # filename = data_path(DATA_FILES['images_id']) + + # with open(filename, 'rb') as infile: + # content = infile.read() + if '/version/' in url: + content = exp.version.encode('utf-8') + return MockResponse(content=content, url=url) + + mp = request.getfixturevalue("monkeypatch") + + mp.setattr(NOIRLab, '_request', mockreturn) + return mp + + +def test_version(patch_request): + r = NOIRLab.version() + assert r >= float(exp.version) diff --git a/astroquery/noirlab/tests/test_noirlab_remote.py b/astroquery/noirlab/tests/test_noirlab_remote.py index 8582b7c4a8..4c0e3a2ef4 100644 --- a/astroquery/noirlab/tests/test_noirlab_remote.py +++ b/astroquery/noirlab/tests/test_noirlab_remote.py @@ -1,4 +1,11 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +Performs similar tests as test_noirlab.py, but performs +the actual HTTPS request rather than monkeypatching it. +Enable with *e.g.*:: + + tox -e py310-test-online -- -P noirlab +""" # External packages import pytest from astropy import units as u @@ -7,11 +14,6 @@ from .. import NOIRLab, NOIRLabClass from . import expected as exp -# performs similar tests as test_module.py, but performs -# the actual HTTP request rather than monkeypatching them. -# should be disabled or enabled at will - use the -# remote_data decorator from astropy: - @pytest.mark.remote_data class TestNOIRLabClass(object): @@ -183,7 +185,7 @@ def test_retrieve(self): def test_version(self): r = NOIRLab.version() - assert r >= 6.0 + assert r >= exp.version def test_get_token(self): actual = NOIRLab.get_token('nobody@university.edu', '123456789') From 14834a41d3a2847240f674938a5c04e4f6df74d2 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 26 Jun 2025 22:16:06 -0700 Subject: [PATCH 08/26] adding offline tests --- astroquery/noirlab/__init__.py | 2 +- astroquery/noirlab/tests/expected.py | 988 +++++++++++++++++- astroquery/noirlab/tests/test_noirlab.py | 20 +- .../noirlab/tests/test_noirlab_remote.py | 32 +- 4 files changed, 1004 insertions(+), 38 deletions(-) diff --git a/astroquery/noirlab/__init__.py b/astroquery/noirlab/__init__.py index 94b1d0da92..4990be6cc0 100644 --- a/astroquery/noirlab/__init__.py +++ b/astroquery/noirlab/__init__.py @@ -3,7 +3,7 @@ NSF NOIRLab Astro Data Archive ------------------------------ -The NSF's OIR Lab Astro Data Archive (formerly NOAO Science Archive) +The NSF NOIRLab Astro Data Archive (formerly NOAO Science Archive) provides access to data taken with more than 40 telescope and instrument combinations, including those operated in partnership with the WIYN, SOAR and SMARTS consortia, from semester 2004B to the diff --git a/astroquery/noirlab/tests/expected.py b/astroquery/noirlab/tests/expected.py index ec55205d5a..a46662484d 100644 --- a/astroquery/noirlab/tests/expected.py +++ b/astroquery/noirlab/tests/expected.py @@ -1,4 +1,8 @@ -# flake8: noqa +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +Expected values for online tests. For offline tests, these values +may be used to create mock responses. +""" query_region_1 = {'4066c45407961eab48eb16bcb9c5b4b7', '554b2bcb3b2a4353c515d20d9fc29e4e', @@ -29,20 +33,986 @@ 'telescope': 'string', 'updated': 'string'} -aux_file_fields = {'AIRMASS': 'str', 'AOS': 'str', 'ASTIG1': 'str', 'ASTIG2': 'str', 'ATTNUM': 'str', 'AVSIG': 'str', 'AVSKY': 'str', 'AZ': 'str', 'BAND': 'str', 'BCAM': 'str', 'BCAMAX': 'str', 'BCAMAY': 'str', 'BCAMAZ': 'str', 'BCAMDX': 'str', 'BCAMDY': 'str', 'BFCFIL': 'str', 'BUNIT': 'str', 'CAMSHUT': 'str', 'CAMSYM': 'str', 'CCDBIN1': 'str', 'CCDBIN2': 'str', 'CCDSEC': 'str', 'CCDSECA': 'str', 'CCDSECB': 'str', 'CENTDEC': 'str', 'CENTRA': 'str', 'CONSTVER': 'str', 'CORN1DEC': 'str', 'CORN1RA': 'str', 'CORN2DEC': 'str', 'CORN2RA': 'str', 'CORN3DEC': 'str', 'CORN3RA': 'str', 'CORN4DEC': 'str', 'CORN4RA': 'str', 'CORNDEC1': 'str', 'CORNDEC2': 'str', 'CORNDEC3': 'str', 'CORNDEC4': 'str', 'CORNRA1': 'str', 'CORNRA2': 'str', 'CORNRA3': 'str', 'CORNRA4': 'str', 'CROSSRA0': 'str', 'DARKTIME': 'str', 'DATASEC': 'str', 'DATASECA': 'str', 'DATASECB': 'str', 'DATE': 'str', 'DATE-OBS': 'str', 'DEC': 'str', 'DES_EXT': 'str', 'DESNCRAY': 'str', 'DESNSTRK': 'str', 'DESREL': 'str', 'DETSIZE': 'str', 'DHEFIRM': 'str', 'DHEINF': 'str', 'DIMM2SEE': 'str', 'DIMMSEE': 'str', 'DODX': 'str', 'DODY': 'str', 'DODZ': 'str', 'DOMEAZ': 'str', 'DOMEFLOR': 'str', 'DOMEHIGH': 'str', 'DOMELOW': 'str', 'DONUTFN1': 'str', 'DONUTFN2': 'str', 'DONUTFN3': 'str', 'DONUTFN4': 'str', 'DONUTFS1': 'str', 'DONUTFS2': 'str', 'DONUTFS3': 'str', 'DONUTFS4': 'str', 'DOXT': 'str', 'DOYT': 'str', 'DTACCOUN': 'str', 'DTACQNAM': 'str', 'DTACQUIS': 'str', 'DTCALDAT': 'str', 'DTCOPYRI': 'str', 'DTINSTRU': 'str', 'DTNSANAM': 'str', 'DTOBSERV': 'str', 'DTPI': 'str', 'DTPIAFFL': 'str', 'DTPROPID': 'str', 'DTQUEUE': 'str', 'DT_RTNAM': 'str', 'DTSITE': 'str', 'DTSTATUS': 'str', 'DTTELESC': 'str', 'DTTITLE': 'str', 'DTUTC': 'str', 'ELLIPTIC': 'str', 'EQUINOX': 'str', 'ERRORS': 'str', 'EUPSPROD': 'str', 'EUPSVER': 'str', 'EXCLUDED': 'str', 'EXPDUR': 'str', 'EXPNUM': 'str', 'EXPREQ': 'str', 'EXPTIME': 'str', 'EXTVER': 'str', 'FADX': 'str', 'FADY': 'str', 'FADZ': 'str', 'FAXT': 'str', 'FAYT': 'str', 'FILENAME': 'str', 'FILTER': 'str', 'FILTPOS': 'str', 'FRGSCALE': 'str', 'FWHM': 'str', 'FZALGOR': 'str', 'FZDTHRSD': 'str', 'FZQMETHD': 'str', 'FZQVALUE': 'str', 'G-CCDNUM': 'str', 'G-FEEDBK': 'str', 'G-FLXVAR': 'str', 'G-LATENC': 'str', 'G-MAXX': 'str', 'G-MAXY': 'str', 'G-MEANX': 'str', 'G-MEANX2': 'str', 'G-MEANXY': 'str', 'G-MEANY': 'str', 'G-MEANY2': 'str', 'G-MODE': 'str', 'G-SEEING': 'str', 'GSKYHOT': 'str', 'GSKYPHOT': 'str', 'GSKYVAR': 'str', 'G-TRANSP': 'str', 'GUIDER': 'str', 'HA': 'str', 'HDRVER': 'str', 'HEX': 'str', 'HUMIDITY': 'str', 'INSTANCE': 'str', 'INSTRUME': 'str', 'IRAF-TLM': 'str', 'LINCFIL': 'str', 'LSKYHOT': 'str', 'LSKYPHOT': 'str', 'LSKYPOW': 'str', 'LSKYVAR': 'str', 'LST': 'str', 'LUTVER': 'str', 'LWTRTEMP': 'str', 'MAGZERO': 'str', 'MAGZP': 'str', 'MAGZPT': 'str', 'MAGZUNC': 'str', 'MAIRTEMP': 'str', 'MANIFEST': 'str', 'MASS2': 'str', 'MJD-END': 'str', 'MJD-OBS': 'str', 'MOONANGL': 'str', 'MSURTEMP': 'str', 'MULTIEXP': 'str', 'MULTIFOC': 'str', 'MULTIID': 'str', 'MULTIROW': 'str', 'MULTITOT': 'str', 'NBLEED': 'str', 'NDONUTS': 'str', 'NEXTEND': 'str', 'NITE': 'str', 'NOTE': 'str', 'NPHTMTCH': 'str', 'NSATPIX': 'str', 'NUM': 'str', 'OBJECT': 'str', 'OBS-ELEV': 'str', 'OBSERVAT': 'str', 'OBSERVER': 'str', 'OBSID': 'str', 'OBS-LAT': 'str', 'OBS-LONG': 'str', 'OBSTYPE': 'str', 'ODATEOBS': 'str', 'OPENSHUT': 'str', 'ORIGIN': 'str', 'OUTTEMP': 'str', 'PHOTFLAG': 'str', 'PHOTREF': 'str', 'PIPELINE': 'str', 'PIXSCAL1': 'str', 'PIXSCAL2': 'str', 'PLARVER': 'str', 'PLDNAME': 'str', 'PLDSID': 'str', 'PLFNAME': 'str', 'PLPROCID': 'str', 'PLQNAME': 'str', 'PLQUEUE': 'str', 'PLVER': 'str', 'PME-TEMP': 'str', 'PMN-TEMP': 'str', 'PMOSTEMP': 'str', 'PMS-TEMP': 'str', 'PMW-TEMP': 'str', 'PRESSURE': 'str', 'PROCTYPE': 'str', 'PRODTYPE': 'str', 'PROGRAM': 'str', 'PROPID': 'str', 'PROPOSER': 'str', 'PUPILAMP': 'str', 'PUPILMAX': 'str', 'PUPILSKY': 'str', 'PUPMAX': 'str', 'PV1_6': 'str', 'PV2_4': 'str', 'RA': 'str', 'RA_CENT': 'str', 'RACMAX': 'str', 'RACMIN': 'str', 'RADECSYS': 'str', 'RADESYS': 'str', 'RADIUS': 'str', 'RADSTD': 'str', 'RECNO': 'str', 'REQNUM': 'str', 'RMCOUNT': 'str', 'SB_ACCOU': 'str', 'SB_DIR1': 'str', 'SB_DIR2': 'str', 'SB_DIR3': 'str', 'SB_HOST': 'str', 'SB_ID': 'str', 'SB_LOCAL': 'str', 'SB_NAME': 'str', 'SB_RECNO': 'str', 'SB_RTNAM': 'str', 'SB_SITE': 'str', 'SCAMPCHI': 'str', 'SCAMPFLG': 'str', 'SCAMPNUM': 'str', 'SCAMPREF': 'str', 'SEQID': 'str', 'SEQNUM': 'str', 'SEQTOT': 'str', 'SEQTYPE': 'str', 'SISPIVER': 'str', 'SKYBRITE': 'str', 'SKYORDER': 'str', 'SKYPC00': 'str', 'SKYPC01': 'str', 'SKYPC02': 'str', 'SKYPC03': 'str', 'SKYSIGMA': 'str', 'SKYSTAT': 'str', 'SKYSUB': 'str', 'SKYSUB01': 'str', 'SKYSUB10': 'str', 'SKYSUBP': 'str', 'SKYUPDAT': 'str', 'SLOT01': 'str', 'SLOT03': 'str', 'SURVEYID': 'str', 'TELDEC': 'str', 'TELEQUIN': 'str', 'TELESCOP': 'str', 'TELFOCUS': 'str', 'TELRA': 'str', 'TELSTAT': 'str', 'TILING': 'str', 'TIME-OBS': 'str', 'TIMESYS': 'str', 'TRACKING': 'str', 'UNITNAME': 'str', 'UPTRTEMP': 'str', 'UTE-TEMP': 'str', 'UTN-TEMP': 'str', 'UTS-TEMP': 'str', 'UTW-TEMP': 'str', 'VALIDA': 'str', 'VALIDB': 'str', 'VSUB': 'str', 'WCSCAL': 'str', 'WINDDIR': 'str', 'WINDSPD': 'str', 'XTALKFIL': 'str', 'ZD': 'str', 'ZPDELDEC': 'str', 'ZPDELRA': 'str'} +aux_file_fields = {'AIRMASS': 'str', + 'AOS': 'str', + 'ASTIG1': 'str', + 'ASTIG2': 'str', + 'ATTNUM': 'str', + 'AVSIG': 'str', + 'AVSKY': 'str', + 'AZ': 'str', + 'BAND': 'str', + 'BCAM': 'str', + 'BCAMAX': 'str', + 'BCAMAY': 'str', + 'BCAMAZ': 'str', + 'BCAMDX': 'str', + 'BCAMDY': 'str', + 'BFCFIL': 'str', + 'BUNIT': 'str', + 'CAMSHUT': 'str', + 'CAMSYM': 'str', + 'CCDBIN1': 'str', + 'CCDBIN2': 'str', + 'CCDSEC': 'str', + 'CCDSECA': 'str', + 'CCDSECB': 'str', + 'CENTDEC': 'str', + 'CENTRA': 'str', + 'CONSTVER': 'str', + 'CORN1DEC': 'str', + 'CORN1RA': 'str', + 'CORN2DEC': 'str', + 'CORN2RA': 'str', + 'CORN3DEC': 'str', + 'CORN3RA': 'str', + 'CORN4DEC': 'str', + 'CORN4RA': 'str', + 'CORNDEC1': 'str', + 'CORNDEC2': 'str', + 'CORNDEC3': 'str', + 'CORNDEC4': 'str', + 'CORNRA1': 'str', + 'CORNRA2': 'str', + 'CORNRA3': 'str', + 'CORNRA4': 'str', + 'CROSSRA0': 'str', + 'DARKTIME': 'str', + 'DATASEC': 'str', + 'DATASECA': 'str', + 'DATASECB': 'str', + 'DATE': 'str', + 'DATE-OBS': 'str', + 'DEC': 'str', + 'DES_EXT': 'str', + 'DESNCRAY': 'str', + 'DESNSTRK': 'str', + 'DESREL': 'str', + 'DETSIZE': 'str', + 'DHEFIRM': 'str', + 'DHEINF': 'str', + 'DIMM2SEE': 'str', + 'DIMMSEE': 'str', + 'DODX': 'str', + 'DODY': 'str', + 'DODZ': 'str', + 'DOMEAZ': 'str', + 'DOMEFLOR': 'str', + 'DOMEHIGH': 'str', + 'DOMELOW': 'str', + 'DONUTFN1': 'str', + 'DONUTFN2': 'str', + 'DONUTFN3': 'str', + 'DONUTFN4': 'str', + 'DONUTFS1': 'str', + 'DONUTFS2': 'str', + 'DONUTFS3': 'str', + 'DONUTFS4': 'str', + 'DOXT': 'str', + 'DOYT': 'str', + 'DTACCOUN': 'str', + 'DTACQNAM': 'str', + 'DTACQUIS': 'str', + 'DTCALDAT': 'str', + 'DTCOPYRI': 'str', + 'DTINSTRU': 'str', + 'DTNSANAM': 'str', + 'DTOBSERV': 'str', + 'DTPI': 'str', + 'DTPIAFFL': 'str', + 'DTPROPID': 'str', + 'DTQUEUE': 'str', + 'DT_RTNAM': 'str', + 'DTSITE': 'str', + 'DTSTATUS': 'str', + 'DTTELESC': 'str', + 'DTTITLE': 'str', + 'DTUTC': 'str', + 'ELLIPTIC': 'str', + 'EQUINOX': 'str', + 'ERRORS': 'str', + 'EUPSPROD': 'str', + 'EUPSVER': 'str', + 'EXCLUDED': 'str', + 'EXPDUR': 'str', + 'EXPNUM': 'str', + 'EXPREQ': 'str', + 'EXPTIME': 'str', + 'EXTVER': 'str', + 'FADX': 'str', + 'FADY': 'str', + 'FADZ': 'str', + 'FAXT': 'str', + 'FAYT': 'str', + 'FILENAME': 'str', + 'FILTER': 'str', + 'FILTPOS': 'str', + 'FRGSCALE': 'str', + 'FWHM': 'str', + 'FZALGOR': 'str', + 'FZDTHRSD': 'str', + 'FZQMETHD': 'str', + 'FZQVALUE': 'str', + 'G-CCDNUM': 'str', + 'G-FEEDBK': 'str', + 'G-FLXVAR': 'str', + 'G-LATENC': 'str', + 'G-MAXX': 'str', + 'G-MAXY': 'str', + 'G-MEANX': 'str', + 'G-MEANX2': 'str', + 'G-MEANXY': 'str', + 'G-MEANY': 'str', + 'G-MEANY2': 'str', + 'G-MODE': 'str', + 'G-SEEING': 'str', 'GSKYHOT': 'str', 'GSKYPHOT': 'str', + 'GSKYVAR': 'str', 'G-TRANSP': 'str', 'GUIDER': 'str', + 'HA': 'str', 'HDRVER': 'str', 'HEX': 'str', 'HUMIDITY': 'str', + 'INSTANCE': 'str', 'INSTRUME': 'str', 'IRAF-TLM': 'str', + 'LINCFIL': 'str', 'LSKYHOT': 'str', 'LSKYPHOT': 'str', + 'LSKYPOW': 'str', 'LSKYVAR': 'str', 'LST': 'str', + 'LUTVER': 'str', 'LWTRTEMP': 'str', 'MAGZERO': 'str', + 'MAGZP': 'str', 'MAGZPT': 'str', 'MAGZUNC': 'str', + 'MAIRTEMP': 'str', 'MANIFEST': 'str', 'MASS2': 'str', + 'MJD-END': 'str', 'MJD-OBS': 'str', 'MOONANGL': 'str', + 'MSURTEMP': 'str', 'MULTIEXP': 'str', 'MULTIFOC': 'str', + 'MULTIID': 'str', 'MULTIROW': 'str', 'MULTITOT': 'str', + 'NBLEED': 'str', 'NDONUTS': 'str', 'NEXTEND': 'str', + 'NITE': 'str', 'NOTE': 'str', 'NPHTMTCH': 'str', + 'NSATPIX': 'str', 'NUM': 'str', 'OBJECT': 'str', + 'OBS-ELEV': 'str', 'OBSERVAT': 'str', 'OBSERVER': 'str', + 'OBSID': 'str', 'OBS-LAT': 'str', 'OBS-LONG': 'str', + 'OBSTYPE': 'str', 'ODATEOBS': 'str', 'OPENSHUT': 'str', + 'ORIGIN': 'str', 'OUTTEMP': 'str', 'PHOTFLAG': 'str', + 'PHOTREF': 'str', 'PIPELINE': 'str', 'PIXSCAL1': 'str', + 'PIXSCAL2': 'str', 'PLARVER': 'str', 'PLDNAME': 'str', + 'PLDSID': 'str', 'PLFNAME': 'str', 'PLPROCID': 'str', + 'PLQNAME': 'str', 'PLQUEUE': 'str', 'PLVER': 'str', + 'PME-TEMP': 'str', 'PMN-TEMP': 'str', 'PMOSTEMP': 'str', + 'PMS-TEMP': 'str', 'PMW-TEMP': 'str', 'PRESSURE': 'str', + 'PROCTYPE': 'str', 'PRODTYPE': 'str', 'PROGRAM': 'str', + 'PROPID': 'str', 'PROPOSER': 'str', 'PUPILAMP': 'str', + 'PUPILMAX': 'str', 'PUPILSKY': 'str', 'PUPMAX': 'str', + 'PV1_6': 'str', 'PV2_4': 'str', 'RA': 'str', + 'RA_CENT': 'str', 'RACMAX': 'str', 'RACMIN': 'str', + 'RADECSYS': 'str', 'RADESYS': 'str', 'RADIUS': 'str', + 'RADSTD': 'str', 'RECNO': 'str', 'REQNUM': 'str', + 'RMCOUNT': 'str', 'SB_ACCOU': 'str', 'SB_DIR1': 'str', + 'SB_DIR2': 'str', 'SB_DIR3': 'str', 'SB_HOST': 'str', + 'SB_ID': 'str', 'SB_LOCAL': 'str', 'SB_NAME': 'str', + 'SB_RECNO': 'str', 'SB_RTNAM': 'str', 'SB_SITE': 'str', + 'SCAMPCHI': 'str', 'SCAMPFLG': 'str', 'SCAMPNUM': 'str', + 'SCAMPREF': 'str', 'SEQID': 'str', 'SEQNUM': 'str', + 'SEQTOT': 'str', 'SEQTYPE': 'str', 'SISPIVER': 'str', + 'SKYBRITE': 'str', 'SKYORDER': 'str', 'SKYPC00': 'str', + 'SKYPC01': 'str', 'SKYPC02': 'str', 'SKYPC03': 'str', + 'SKYSIGMA': 'str', 'SKYSTAT': 'str', 'SKYSUB': 'str', + 'SKYSUB01': 'str', 'SKYSUB10': 'str', 'SKYSUBP': 'str', + 'SKYUPDAT': 'str', 'SLOT01': 'str', 'SLOT03': 'str', + 'SURVEYID': 'str', 'TELDEC': 'str', 'TELEQUIN': 'str', + 'TELESCOP': 'str', 'TELFOCUS': 'str', 'TELRA': 'str', + 'TELSTAT': 'str', 'TILING': 'str', 'TIME-OBS': 'str', + 'TIMESYS': 'str', 'TRACKING': 'str', 'UNITNAME': 'str', + 'UPTRTEMP': 'str', 'UTE-TEMP': 'str', 'UTN-TEMP': 'str', + 'UTS-TEMP': 'str', 'UTW-TEMP': 'str', 'VALIDA': 'str', + 'VALIDB': 'str', + 'VSUB': 'str', + 'WCSCAL': 'str', + 'WINDDIR': 'str', + 'WINDSPD': 'str', + 'XTALKFIL': 'str', + 'ZD': 'str', + 'ZPDELDEC': 'str', + 'ZPDELRA': 'str'} -core_file_fields = [{'Field': 'archive_filename', 'Type': 'str'}, {'Field': 'caldat', 'Type': 'datetime64'}, {'Field': 'date_obs_max', 'Type': 'datetime64'}, {'Field': 'date_obs_min', 'Type': 'datetime64'}, {'Field': 'dec_max', 'Type': 'np.float64'}, {'Field': 'dec_min', 'Type': 'np.float64'}, {'Field': 'depth', 'Type': 'np.float64'}, {'Field': 'exposure', 'Type': 'np.float64'}, {'Field': 'filesize', 'Type': 'np.int64'}, {'Field': 'ifilter', 'Type': 'category'}, {'Field': 'instrument', 'Type': 'category'}, {'Field': 'md5sum', 'Type': 'str'}, {'Field': 'obs_mode', 'Type': 'category'}, {'Field': 'obs_type', 'Type': 'category'}, {'Field': 'original_filename', 'Type': 'str'}, {'Field': 'proc_type', 'Type': 'category'}, {'Field': 'prod_type', 'Type': 'category'}, {'Field': 'proposal', 'Type': 'category'}, {'Field': 'ra_max', 'Type': 'np.float64'}, {'Field': 'ra_min', 'Type': 'np.float64'}, {'Field': 'release_date', 'Type': 'datetime64'}, {'Field': 'seeing', 'Type': 'np.float64'}, {'Field': 'site', 'Type': 'category'}, {'Field': 'survey', 'Type': 'category'}, {'Field': 'telescope', 'Type': 'category'}, {'Field': 'updated', 'Type': 'datetime64'}] +core_file_fields = [{'Field': 'archive_filename', 'Type': 'str'}, + {'Field': 'caldat', 'Type': 'datetime64'}, + {'Field': 'date_obs_max', 'Type': 'datetime64'}, + {'Field': 'date_obs_min', 'Type': 'datetime64'}, + {'Field': 'dec_max', 'Type': 'np.float64'}, + {'Field': 'dec_min', 'Type': 'np.float64'}, + {'Field': 'depth', 'Type': 'np.float64'}, + {'Field': 'exposure', 'Type': 'np.float64'}, + {'Field': 'filesize', 'Type': 'np.int64'}, + {'Field': 'ifilter', 'Type': 'category'}, + {'Field': 'instrument', 'Type': 'category'}, + {'Field': 'md5sum', 'Type': 'str'}, + {'Field': 'obs_mode', 'Type': 'category'}, + {'Field': 'obs_type', 'Type': 'category'}, + {'Field': 'original_filename', 'Type': 'str'}, + {'Field': 'proc_type', 'Type': 'category'}, + {'Field': 'prod_type', 'Type': 'category'}, + {'Field': 'proposal', 'Type': 'category'}, + {'Field': 'ra_max', 'Type': 'np.float64'}, + {'Field': 'ra_min', 'Type': 'np.float64'}, + {'Field': 'release_date', 'Type': 'datetime64'}, + {'Field': 'seeing', 'Type': 'np.float64'}, + {'Field': 'site', 'Type': 'category'}, + {'Field': 'survey', 'Type': 'category'}, + {'Field': 'telescope', 'Type': 'category'}, + {'Field': 'updated', 'Type': 'datetime64'}] -query_file_metadata = [' archive_filename instrument md5sum original_filename proc_type url ', '----------------------------------------------------------------------------- ---------- -------------------------------- ------------------------------------------------------------------------------------- --------- ----------------------------------------------------------------------------', '/net/archive/pipe/20161024/ct4m/2012B-0001/c4d_161025_034649_oki_z_v2.fits.fz decam 0000f0c5015263610a75f0affed377d0 /net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_161025_034649_oki_z_v2.fits.fz skysub https://astroarchive.noao.edu/api/retrieve/0000f0c5015263610a75f0affed377d0/', '/net/archive/pipe/20160928/ct4m/2012B-0001/c4d_160929_090838_ooi_z_v2.fits.fz decam 0001e5a5bcf039ebf0c53a6da32e8888 /net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_160929_090838_ooi_z_v2.fits.fz instcal https://astroarchive.noao.edu/api/retrieve/0001e5a5bcf039ebf0c53a6da32e8888/', '/net/archive/pipe/20160909/ct4m/2012B-0001/c4d_160910_062930_opd_z_v2.fits.fz decam 0003a1c853de73fc1796d2d7d77ca9cd /net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_160910_062930_opd_z_v2.fits.fz resampled https://astroarchive.noao.edu/api/retrieve/0003a1c853de73fc1796d2d7d77ca9cd/'] +query_file_metadata = [' archive_filename instrument md5sum original_filename proc_type url ', + '----------------------------------------------------------------------------- ---------- -------------------------------- ------------------------------------------------------------------------------------- --------- ----------------------------------------------------------------------------', + '/net/archive/pipe/20161024/ct4m/2012B-0001/c4d_161025_034649_oki_z_v2.fits.fz decam 0000f0c5015263610a75f0affed377d0 /net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_161025_034649_oki_z_v2.fits.fz skysub https://astroarchive.noao.edu/api/retrieve/0000f0c5015263610a75f0affed377d0/', + '/net/archive/pipe/20160928/ct4m/2012B-0001/c4d_160929_090838_ooi_z_v2.fits.fz decam 0001e5a5bcf039ebf0c53a6da32e8888 /net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_160929_090838_ooi_z_v2.fits.fz instcal https://astroarchive.noao.edu/api/retrieve/0001e5a5bcf039ebf0c53a6da32e8888/', + '/net/archive/pipe/20160909/ct4m/2012B-0001/c4d_160910_062930_opd_z_v2.fits.fz decam 0003a1c853de73fc1796d2d7d77ca9cd /net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_160910_062930_opd_z_v2.fits.fz resampled https://astroarchive.noao.edu/api/retrieve/0003a1c853de73fc1796d2d7d77ca9cd/'] -aux_hdu_fields = {'AMPBAL': 'str', 'AMPSECA': 'str', 'AMPSECB': 'str', 'ARAWGAIN': 'str', 'AVSIG': 'str', 'AVSKY': 'str', 'BIASFIL': 'str', 'BLDINTRP': 'str', 'BPM': 'str', 'BPMFIL': 'str', 'CATALOG': 'str', 'CCDNUM': 'str', 'CCDSECA': 'str', 'CCDSECB': 'str', 'CD1_1': 'str', 'CD1_2': 'str', 'CD2_1': 'str', 'CD2_2': 'str', 'CENDEC1': 'str', 'CENRA1': 'str', 'COR1DEC1': 'str', 'COR1RA1': 'str', 'COR2DEC1': 'str', 'COR2RA1': 'str', 'COR3DEC1': 'str', 'COR3RA1': 'str', 'COR4DEC1': 'str', 'COR4RA1': 'str', 'CROSSRA0': 'str', 'CRPIX1': 'str', 'CRPIX2': 'str', 'CRVAL1': 'str', 'CRVAL2': 'str', 'CTYPE1': 'str', 'CTYPE2': 'str', 'CUNIT1': 'str', 'CUNIT2': 'str', 'D0034494': 'str', 'D0034496': 'str', 'D0034497': 'str', 'DATASEC': 'str', 'DATASECA': 'str', 'DATASECB': 'str', 'DATE': 'str', 'DEC': 'str', 'DEC1': 'str', 'DEC13A_2': 'str', 'DEC13B_2': 'str', 'DEC14A_2': 'str', 'DEC15A_2': 'str', 'DEC15B_2': 'str', 'DEC16A_2': 'str', 'DEC16B_2': 'str', 'DEC17A_2': 'str', 'DEC17B_2': 'str', 'DEC18A_2': 'str', 'DEC18B_2': 'str', 'DEC18B_R': 'str', 'DEC18B_S': 'str', 'DEC19A_2': 'str', 'DEC19A_A': 'str', 'DEC19A_D': 'str', 'DEC19A_J': 'str', 'DEC19A_K': 'str', 'DEC19A_L': 'str', 'DEC19A_M': 'str', 'DEC19A_P': 'str', 'DEC19A_S': 'str', 'DEC19A_T': 'str', 'DEC19A_W': 'str', 'DEC19B_2': 'str', 'DEC19B_A': 'str', 'DEC19B_C': 'str', 'DEC19B_D': 'str', 'DEC19B_F': 'str', 'DEC19B_H': 'str', 'DEC19B_I': 'str', 'DEC19B_J': 'str', 'DEC19B_K': 'str', 'DEC19B_L': 'str', 'DEC19B_M': 'str', 'DEC19B_P': 'str', 'DEC19B_R': 'str', 'DEC19B_S': 'str', 'DEC19B_T': 'str', 'DEC19B_W': 'str', 'DEC20A_A': 'str', 'DEC20A_C': 'str', 'DEC20A_F': 'str', 'DEC20A_J': 'str', 'DEC20A_K': 'str', 'DEC20A_L': 'str', 'DEC20A_M': 'str', 'DEC20A_S': 'str', 'DEC20A_T': 'str', 'DEC20B_T': 'str', 'DECALS_2': 'str', 'DECALS_D': 'str', 'DECC1': 'str', 'DECC2': 'str', 'DECC3': 'str', 'DECC4': 'str', 'DEC_CENT': 'str', 'DECCMAX': 'str', 'DECCMIN': 'str', 'DES14B_2': 'str', 'DES15B_2': 'str', 'DES16B_2': 'str', 'DES17A_2': 'str', 'DES17B_2': 'str', 'DES18A_2': 'str', 'DES18B_2': 'str', 'DESBFC': 'str', 'DESBIAS': 'str', 'DESBLEED': 'str', 'DESBPM': 'str', 'DESCRMSK': 'str', 'DESDCXTK': 'str', 'DESDMCR': 'str', 'DES_EXT': 'str', 'DESFIXC': 'str', 'DESFLAT': 'str', 'DESFNAME': 'str', 'DESFRING': 'str', 'DESGAINC': 'str', 'DESILLUM': 'str', 'DESIMMSK': 'str', 'DESLINC': 'str', 'DESNCRAY': 'str', 'DESNSTRK': 'str', 'DESOSCN': 'str', 'DESPHOTF': 'str', 'DESPUPC': 'str', 'DESSAT': 'str', 'DESSKYSB': 'str', 'DESSTAR': 'str', 'DETECTOR': 'str', 'DETPOS': 'str', 'DETSEC': 'str', 'DETSECA': 'str', 'DETSECB': 'str', 'ELLIPTIC': 'str', 'ENG17B_2': 'str', 'ENG18A_2': 'str', 'ENG18B_2': 'str', 'EXPTIME': 'str', 'EXTNAME': 'str', 'EXTVER': 'str', 'FILTER': 'str', 'FIXCFIL': 'str', 'FIXPIX': 'str', 'FIXPIX02': 'str', 'FLATFIL': 'str', 'FLATMEDA': 'str', 'FLATMEDB': 'str', 'FPA': 'str', 'FRGSCALE': 'str', 'FRINGE': 'str', 'FRINGFIL': 'str', 'FWHM': 'str', 'FWHMP1': 'str', 'FZALGOR': 'str', 'FZDTHRSD': 'str', 'FZQMETHD': 'str', 'FZQVALUE': 'str', 'GAINA': 'str', 'GAINB': 'str', 'GCOUNT': 'str', 'ILLCOR': 'str', 'ILLMASK': 'str', 'ILLUMCOR': 'str', 'ILLUMFIL': 'str', 'INHERIT': 'str', 'IRAF-TLM': 'str', 'LAGER_20': 'str', 'LTM1_1': 'str', 'LTM1_2': 'str', 'LTM2_1': 'str', 'LTM2_2': 'str', 'LTV1': 'str', 'LTV2': 'str', 'MAGZERO1': 'str', 'MAGZUNC1': 'str', 'NAXIS1': 'str', 'NAXIS2': 'str', 'NBLEED': 'str', 'NSATPIX': 'str', 'OBJECT': 'str', 'OBJMASK': 'str', 'ORIGIN': 'str', 'PCOUNT': 'str', 'PHOTFLAG': 'str', 'PUPFIL': 'str', 'PUPILAMP': 'str', 'PUPILSKY': 'str', 'PUPMAX': 'str', 'PV1_0': 'str', 'PV1_1': 'str', 'PV1_10': 'str', 'PV1_2': 'str', 'PV1_3': 'str', 'PV1_4': 'str', 'PV1_5': 'str', 'PV1_6': 'str', 'PV1_7': 'str', 'PV1_8': 'str', 'PV1_9': 'str', 'PV2_0': 'str', 'PV2_1': 'str', 'PV2_10': 'str', 'PV2_2': 'str', 'PV2_3': 'str', 'PV2_4': 'str', 'PV2_5': 'str', 'PV2_6': 'str', 'PV2_7': 'str', 'PV2_8': 'str', 'PV2_9': 'str', 'RA': 'str', 'RA1': 'str', 'RAC1': 'str', 'RAC2': 'str', 'RAC3': 'str', 'RAC4': 'str', 'RA_CENT': 'str', 'RACMAX': 'str', 'RACMIN': 'str', 'RDNOISEA': 'str', 'RDNOISEB': 'str', 'REQ13B_2': 'str', 'REQ13B_H': 'str', 'REQ14B_2': 'str', 'REQ14B_K': 'str', 'REQ15B_S': 'str', 'REQ16A_S': 'str', 'REQ18B_B': 'str', 'REQ19A_2': 'str', 'REQ19A_P': 'str', 'SATURATA': 'str', 'SATURATB': 'str', 'SATURATE': 'str', 'SKYBRITE': 'str', 'SKYIM': 'str', 'SKYSBFIL': 'str', 'SKYSIGMA': 'str', 'SKYSUB': 'str', 'SKYSUB00': 'str', 'SKYVARA': 'str', 'SKYVARB': 'str', 'SLOT00': 'str', 'SLOT01': 'str', 'SLOT02': 'str', 'SLOT03': 'str', 'SLOT04': 'str', 'SLOT05': 'str', 'STARFIL': 'str', 'STARMASK': 'str', 'TOO15A_2': 'str', 'TOO15B_2': 'str', 'TOO16A_2': 'str', 'TOO16B_2': 'str', 'TOO17B_2': 'str', 'TOO18A_2': 'str', 'TOO18A_S': 'str', 'TOO18B_2': 'str', 'TOO19A_2': 'str', 'TOO19A_G': 'str', 'TOO19A_K': 'str', 'TOO19A_M': 'str', 'TOO19B_M': 'str', 'TOO20A_M': 'str', 'WAT0_001': 'str', 'WAT1_001': 'str', 'WAT2_001': 'str', 'WCSAXES': 'str', 'WCSDIM': 'str', 'WTMAP': 'str', 'XTENSION': 'str', 'ZDITHER0': 'str'} +aux_hdu_fields = {'AMPBAL': 'str', 'AMPSECA': 'str', 'AMPSECB': 'str', + 'ARAWGAIN': 'str', 'AVSIG': 'str', 'AVSKY': 'str', + 'BIASFIL': 'str', 'BLDINTRP': 'str', 'BPM': 'str', + 'BPMFIL': 'str', 'CATALOG': 'str', 'CCDNUM': 'str', + 'CCDSECA': 'str', 'CCDSECB': 'str', 'CD1_1': 'str', + 'CD1_2': 'str', 'CD2_1': 'str', 'CD2_2': 'str', + 'CENDEC1': 'str', 'CENRA1': 'str', 'COR1DEC1': 'str', + 'COR1RA1': 'str', 'COR2DEC1': 'str', 'COR2RA1': 'str', + 'COR3DEC1': 'str', 'COR3RA1': 'str', 'COR4DEC1': 'str', + 'COR4RA1': 'str', 'CROSSRA0': 'str', 'CRPIX1': 'str', + 'CRPIX2': 'str', 'CRVAL1': 'str', 'CRVAL2': 'str', + 'CTYPE1': 'str', 'CTYPE2': 'str', 'CUNIT1': 'str', + 'CUNIT2': 'str', 'D0034494': 'str', 'D0034496': 'str', + 'D0034497': 'str', 'DATASEC': 'str', 'DATASECA': 'str', + 'DATASECB': 'str', 'DATE': 'str', 'DEC': 'str', 'DEC1': 'str', + 'DEC13A_2': 'str', 'DEC13B_2': 'str', 'DEC14A_2': 'str', + 'DEC15A_2': 'str', 'DEC15B_2': 'str', 'DEC16A_2': 'str', + 'DEC16B_2': 'str', 'DEC17A_2': 'str', 'DEC17B_2': 'str', + 'DEC18A_2': 'str', 'DEC18B_2': 'str', 'DEC18B_R': 'str', + 'DEC18B_S': 'str', 'DEC19A_2': 'str', 'DEC19A_A': 'str', + 'DEC19A_D': 'str', 'DEC19A_J': 'str', 'DEC19A_K': 'str', + 'DEC19A_L': 'str', 'DEC19A_M': 'str', 'DEC19A_P': 'str', + 'DEC19A_S': 'str', 'DEC19A_T': 'str', 'DEC19A_W': 'str', + 'DEC19B_2': 'str', 'DEC19B_A': 'str', 'DEC19B_C': 'str', + 'DEC19B_D': 'str', 'DEC19B_F': 'str', 'DEC19B_H': 'str', + 'DEC19B_I': 'str', 'DEC19B_J': 'str', 'DEC19B_K': 'str', + 'DEC19B_L': 'str', 'DEC19B_M': 'str', 'DEC19B_P': 'str', + 'DEC19B_R': 'str', 'DEC19B_S': 'str', 'DEC19B_T': 'str', + 'DEC19B_W': 'str', 'DEC20A_A': 'str', 'DEC20A_C': 'str', + 'DEC20A_F': 'str', 'DEC20A_J': 'str', 'DEC20A_K': 'str', + 'DEC20A_L': 'str', 'DEC20A_M': 'str', 'DEC20A_S': 'str', + 'DEC20A_T': 'str', 'DEC20B_T': 'str', 'DECALS_2': 'str', + 'DECALS_D': 'str', 'DECC1': 'str', 'DECC2': 'str', 'DECC3': 'str', + 'DECC4': 'str', 'DEC_CENT': 'str', 'DECCMAX': 'str', + 'DECCMIN': 'str', 'DES14B_2': 'str', 'DES15B_2': 'str', + 'DES16B_2': 'str', 'DES17A_2': 'str', 'DES17B_2': 'str', + 'DES18A_2': 'str', 'DES18B_2': 'str', 'DESBFC': 'str', + 'DESBIAS': 'str', 'DESBLEED': 'str', 'DESBPM': 'str', + 'DESCRMSK': 'str', 'DESDCXTK': 'str', 'DESDMCR': 'str', + 'DES_EXT': 'str', 'DESFIXC': 'str', 'DESFLAT': 'str', + 'DESFNAME': 'str', 'DESFRING': 'str', 'DESGAINC': 'str', + 'DESILLUM': 'str', 'DESIMMSK': 'str', 'DESLINC': 'str', + 'DESNCRAY': 'str', 'DESNSTRK': 'str', 'DESOSCN': 'str', + 'DESPHOTF': 'str', 'DESPUPC': 'str', 'DESSAT': 'str', + 'DESSKYSB': 'str', 'DESSTAR': 'str', 'DETECTOR': 'str', + 'DETPOS': 'str', 'DETSEC': 'str', 'DETSECA': 'str', + 'DETSECB': 'str', 'ELLIPTIC': 'str', 'ENG17B_2': 'str', + 'ENG18A_2': 'str', 'ENG18B_2': 'str', 'EXPTIME': 'str', + 'EXTNAME': 'str', 'EXTVER': 'str', 'FILTER': 'str', + 'FIXCFIL': 'str', 'FIXPIX': 'str', 'FIXPIX02': 'str', + 'FLATFIL': 'str', 'FLATMEDA': 'str', 'FLATMEDB': 'str', + 'FPA': 'str', 'FRGSCALE': 'str', 'FRINGE': 'str', + 'FRINGFIL': 'str', 'FWHM': 'str', 'FWHMP1': 'str', + 'FZALGOR': 'str', 'FZDTHRSD': 'str', 'FZQMETHD': 'str', + 'FZQVALUE': 'str', 'GAINA': 'str', 'GAINB': 'str', + 'GCOUNT': 'str', 'ILLCOR': 'str', 'ILLMASK': 'str', + 'ILLUMCOR': 'str', 'ILLUMFIL': 'str', 'INHERIT': 'str', + 'IRAF-TLM': 'str', 'LAGER_20': 'str', 'LTM1_1': 'str', + 'LTM1_2': 'str', 'LTM2_1': 'str', 'LTM2_2': 'str', + 'LTV1': 'str', 'LTV2': 'str', 'MAGZERO1': 'str', + 'MAGZUNC1': 'str', 'NAXIS1': 'str', 'NAXIS2': 'str', + 'NBLEED': 'str', 'NSATPIX': 'str', 'OBJECT': 'str', + 'OBJMASK': 'str', 'ORIGIN': 'str', 'PCOUNT': 'str', + 'PHOTFLAG': 'str', 'PUPFIL': 'str', 'PUPILAMP': 'str', + 'PUPILSKY': 'str', 'PUPMAX': 'str', 'PV1_0': 'str', + 'PV1_1': 'str', 'PV1_10': 'str', 'PV1_2': 'str', + 'PV1_3': 'str', 'PV1_4': 'str', 'PV1_5': 'str', + 'PV1_6': 'str', 'PV1_7': 'str', 'PV1_8': 'str', 'PV1_9': 'str', + 'PV2_0': 'str', 'PV2_1': 'str', 'PV2_10': 'str', 'PV2_2': 'str', + 'PV2_3': 'str', 'PV2_4': 'str', 'PV2_5': 'str', 'PV2_6': 'str', + 'PV2_7': 'str', 'PV2_8': 'str', 'PV2_9': 'str', 'RA': 'str', + 'RA1': 'str', 'RAC1': 'str', 'RAC2': 'str', 'RAC3': 'str', + 'RAC4': 'str', 'RA_CENT': 'str', 'RACMAX': 'str', + 'RACMIN': 'str', 'RDNOISEA': 'str', 'RDNOISEB': 'str', + 'REQ13B_2': 'str', 'REQ13B_H': 'str', 'REQ14B_2': 'str', + 'REQ14B_K': 'str', 'REQ15B_S': 'str', 'REQ16A_S': 'str', + 'REQ18B_B': 'str', 'REQ19A_2': 'str', 'REQ19A_P': 'str', + 'SATURATA': 'str', 'SATURATB': 'str', 'SATURATE': 'str', + 'SKYBRITE': 'str', 'SKYIM': 'str', 'SKYSBFIL': 'str', + 'SKYSIGMA': 'str', 'SKYSUB': 'str', 'SKYSUB00': 'str', + 'SKYVARA': 'str', 'SKYVARB': 'str', 'SLOT00': 'str', + 'SLOT01': 'str', 'SLOT02': 'str', 'SLOT03': 'str', + 'SLOT04': 'str', 'SLOT05': 'str', 'STARFIL': 'str', + 'STARMASK': 'str', 'TOO15A_2': 'str', 'TOO15B_2': 'str', + 'TOO16A_2': 'str', 'TOO16B_2': 'str', 'TOO17B_2': 'str', + 'TOO18A_2': 'str', 'TOO18A_S': 'str', 'TOO18B_2': 'str', + 'TOO19A_2': 'str', 'TOO19A_G': 'str', 'TOO19A_K': 'str', + 'TOO19A_M': 'str', 'TOO19B_M': 'str', 'TOO20A_M': 'str', + 'WAT0_001': 'str', 'WAT1_001': 'str', 'WAT2_001': 'str', + 'WCSAXES': 'str', 'WCSDIM': 'str', 'WTMAP': 'str', + 'XTENSION': 'str', 'ZDITHER0': 'str'} -core_hdu_fields = [{'Field': 'boundary', 'Type': 'str'}, {'Field': 'dec', 'Type': 'np.float64'}, {'Field': 'dec_range', 'Type': 'str'}, {'Field': 'fitsfile', 'Type': 'str'}, {'Field': 'fitsfile__archive_filename', 'Type': 'str'}, {'Field': 'fitsfile__caldat', 'Type': 'str'}, {'Field': 'fitsfile__date_obs_max', 'Type': 'str'}, {'Field': 'fitsfile__date_obs_min', 'Type': 'str'}, {'Field': 'fitsfile__dec_max', 'Type': 'str'}, {'Field': 'fitsfile__dec_min', 'Type': 'str'}, {'Field': 'fitsfile__depth', 'Type': 'str'}, {'Field': 'fitsfile__exposure', 'Type': 'str'}, {'Field': 'fitsfile__filesize', 'Type': 'str'}, {'Field': 'fitsfile__ifilter', 'Type': 'str'}, {'Field': 'fitsfile__instrument', 'Type': 'str'}, {'Field': 'fitsfile__md5sum', 'Type': 'str'}, {'Field': 'fitsfile__obs_mode', 'Type': 'str'}, {'Field': 'fitsfile__obs_type', 'Type': 'str'}, {'Field': 'fitsfile__original_filename', 'Type': 'str'}, {'Field': 'fitsfile__proc_type', 'Type': 'str'}, {'Field': 'fitsfile__prod_type', 'Type': 'str'}, {'Field': 'fitsfile__proposal', 'Type': 'str'}, {'Field': 'fitsfile__ra_max', 'Type': 'str'}, {'Field': 'fitsfile__ra_min', 'Type': 'str'}, {'Field': 'fitsfile__release_date', 'Type': 'str'}, {'Field': 'fitsfile__seeing', 'Type': 'str'}, {'Field': 'fitsfile__site', 'Type': 'str'}, {'Field': 'fitsfile__survey', 'Type': 'str'}, {'Field': 'fitsfile__telescope', 'Type': 'str'}, {'Field': 'fitsfile__updated', 'Type': 'str'}, {'Field': 'hdu_idx', 'Type': 'np.int64'}, {'Field': 'id', 'Type': 'str'}, {'Field': 'ra', 'Type': 'np.float64'}, {'Field': 'ra_range', 'Type': 'str'}, {'Field': 'updated', 'Type': 'datetime64'}] +core_hdu_fields = [{'Field': 'boundary', 'Type': 'str'}, + {'Field': 'dec', 'Type': 'np.float64'}, + {'Field': 'dec_range', 'Type': 'str'}, + {'Field': 'fitsfile', 'Type': 'str'}, + {'Field': 'fitsfile__archive_filename', 'Type': 'str'}, + {'Field': 'fitsfile__caldat', 'Type': 'str'}, + {'Field': 'fitsfile__date_obs_max', 'Type': 'str'}, + {'Field': 'fitsfile__date_obs_min', 'Type': 'str'}, + {'Field': 'fitsfile__dec_max', 'Type': 'str'}, + {'Field': 'fitsfile__dec_min', 'Type': 'str'}, + {'Field': 'fitsfile__depth', 'Type': 'str'}, + {'Field': 'fitsfile__exposure', 'Type': 'str'}, + {'Field': 'fitsfile__filesize', 'Type': 'str'}, + {'Field': 'fitsfile__ifilter', 'Type': 'str'}, + {'Field': 'fitsfile__instrument', 'Type': 'str'}, + {'Field': 'fitsfile__md5sum', 'Type': 'str'}, + {'Field': 'fitsfile__obs_mode', 'Type': 'str'}, + {'Field': 'fitsfile__obs_type', 'Type': 'str'}, + {'Field': 'fitsfile__original_filename', 'Type': 'str'}, + {'Field': 'fitsfile__proc_type', 'Type': 'str'}, + {'Field': 'fitsfile__prod_type', 'Type': 'str'}, + {'Field': 'fitsfile__proposal', 'Type': 'str'}, + {'Field': 'fitsfile__ra_max', 'Type': 'str'}, + {'Field': 'fitsfile__ra_min', 'Type': 'str'}, + {'Field': 'fitsfile__release_date', 'Type': 'str'}, + {'Field': 'fitsfile__seeing', 'Type': 'str'}, + {'Field': 'fitsfile__site', 'Type': 'str'}, + {'Field': 'fitsfile__survey', 'Type': 'str'}, + {'Field': 'fitsfile__telescope', 'Type': 'str'}, + {'Field': 'fitsfile__updated', 'Type': 'str'}, + {'Field': 'hdu_idx', 'Type': 'np.int64'}, + {'Field': 'id', 'Type': 'str'}, + {'Field': 'ra', 'Type': 'np.float64'}, + {'Field': 'ra_range', 'Type': 'str'}, + {'Field': 'updated', 'Type': 'datetime64'}] -query_hdu_metadata = ['AIRMASS fitsfile__archive_filename fitsfile__caldat fitsfile__instrument fitsfile__proc_type', '------- ----------------------------------------------------------------------- ---------------- -------------------- -------------------', ' None /net/archive/mtn/20170814/ct4m/2012B-0001/c4d_170815_051354_ori.fits.fz 2017-08-14 decam raw', ' None /net/archive/mtn/20170814/ct4m/2012B-0001/c4d_170815_051354_ori.fits.fz 2017-08-14 decam raw', ' None /net/archive/mtn/20170814/ct4m/2012B-0001/c4d_170815_051354_ori.fits.fz 2017-08-14 decam raw'] +query_hdu_metadata = ['AIRMASS fitsfile__archive_filename fitsfile__caldat fitsfile__instrument fitsfile__proc_type', + '------- ----------------------------------------------------------------------- ---------------- -------------------- -------------------', + ' None /net/archive/mtn/20170814/ct4m/2012B-0001/c4d_170815_051354_ori.fits.fz 2017-08-14 decam raw', + ' None /net/archive/mtn/20170814/ct4m/2012B-0001/c4d_170815_051354_ori.fits.fz 2017-08-14 decam raw', + ' None /net/archive/mtn/20170814/ct4m/2012B-0001/c4d_170815_051354_ori.fits.fz 2017-08-14 decam raw'] -categoricals = {'instruments': ['(p)odi', '90prime', 'andicam', 'arcoiris', 'arcon', 'bench', 'ccd_imager', 'chiron', 'cosmos', 'decam', 'echelle', 'falmingos', 'flamingos', 'goodman', 'goodman spectrograph', 'gtcam', 'hdi', 'ice', 'ispi', 'kosmos', 'minimo/ice', 'mop/ice', 'mosaic', 'mosaic3', 'mosaic_1', 'mosaic_1_1', 'mosaic_2', 'newfirm', 'osiris', 'sami', 'soi', 'spartan', 'spartan ir camera', 'triplespec', 'wfc', 'whirc', 'wildfire', 'y4kcam'], 'obsmodes': [], 'proctypes': ['instcal', 'mastercal', 'nota', 'projected', 'raw', 'resampled', 'skysub', 'stacked'], 'prodtypes': ['dqmask', 'expmap', 'graphics (size)', 'image', 'image 2nd version 1', 'image1', 'nota', 'resampled', 'weight', 'wtmap'], 'sites': ['cp', 'ct', 'kp', 'lp'], 'surveys': [], 'telescopes': ['bok23m', 'ct09m', 'ct13m', 'ct15m', 'ct1m', 'ct4m', 'ctlab', 'kp09m', 'kp21m', 'kp35m', 'kp4m', 'kpcf', 'lp25m', 'soar', 'wiyn']} +categoricals = {'instruments': ['(p)odi', + '90prime', + 'andicam', + 'arcoiris', + 'arcon', + 'bench', + 'ccd_imager', + 'chiron', + 'cosmos', + 'decam', + 'echelle', + 'falmingos', + 'flamingos', + 'goodman', + 'goodman spectrograph', + 'gtcam', + 'hdi', + 'ice', + 'ispi', + 'kosmos', + 'minimo/ice', + 'mop/ice', + 'mosaic', + 'mosaic3', + 'mosaic_1', + 'mosaic_1_1', + 'mosaic_2', + 'newfirm', + 'osiris', + 'sami', + 'soi', + 'spartan', + 'spartan ir camera', + 'triplespec', + 'wfc', + 'whirc', + 'wildfire', + 'y4kcam'], + 'obsmodes': [], + 'proctypes': ['instcal', + 'mastercal', + 'nota', + 'projected', + 'raw', + 'resampled', + 'skysub', + 'stacked'], + 'prodtypes': ['dqmask', + 'expmap', + 'graphics (size)', + 'image', + 'image 2nd version 1', + 'image1', + 'nota', + 'resampled', + 'weight', + 'wtmap'], + 'sites': ['cp', + 'ct', + 'kp', + 'lp'], + 'surveys': [], + 'telescopes': ['bok23m', + 'ct09m', + 'ct13m', + 'ct15m', + 'ct1m', + 'ct4m', + 'ctlab', + 'kp09m', + 'kp21m', + 'kp35m', + 'kp4m', + 'kpcf', + 'lp25m', + 'soar', + 'wiyn']} -retrieve = ['SIMPLE', 'BITPIX', 'NAXIS', 'EXTEND', 'ORIGIN', 'DATE', 'IRAF-TLM', 'OBJECT', 'RAWFILE', 'FILENAME', 'OBJRA', 'OBJDEC', 'OBJEPOCH', '', 'TIMESYS', '', 'OBSERVAT', 'OBS-ELEV', 'OBS-LAT', 'OBS-LONG', 'TELESCOP', 'TELRADEC', 'TELEQUIN', 'TELRA', 'TELDEC', 'HA', 'ZD', 'TELFOCUS', '', 'MOSSIZE', 'NDETS', '', 'OBSERVER', 'PROPOSER', 'PROPID', 'SEQID', 'SEQNUM', '', 'NOHS', 'NOCUTC', 'NOCRSD', 'NOCGID', 'NOCNAME', '', 'DTSITE', 'DTTELESC', 'DTINSTRU', 'DTCALDAT', 'DTUTC', 'DTOBSERV', 'DTPROPID', 'DTPI', 'DTPIAFFL', 'DTTITLE', 'DTCOPYRI', 'DTACQUIS', 'DTACCOUN', 'DTACQNAM', 'DTNSANAM', 'DTSTATUS', 'SB_HOST', 'SB_ACCOU', 'SB_SITE', 'SB_LOCAL', 'SB_DIR1', 'SB_DIR2', 'SB_DIR3', 'SB_RECNO', 'SB_ID', 'SB_NAME', 'RMCOUNT', 'RECNO', 'INSTRUME', 'RSPTGRP', 'RSPGRP', '', 'OBSTYPE', 'PROCTYPE', 'PRODTYPE', 'MIMETYPE', 'EXPTIME', 'FILTER', '', 'RA', 'DEC', 'CENTRA', 'CORN1RA', 'CORN2RA', 'CORN3RA', 'CORN4RA', 'CENTDEC', 'CORN1DEC', 'CORN2DEC', 'CORN3DEC', 'CORN4DEC', 'DATE-OBS', 'TIME-OBS', 'MJD-OBS', 'ST', '', 'CTYPE1', 'CTYPE2', 'CRVAL1', 'CRVAL2', 'CD1_1', 'CD2_2', 'WAT0_001', 'WAT1_001', 'WAT2_001', '', 'DIGAVGS', 'NCOADD', 'FSAMPLE', 'EXPCOADD', 'TITLE', 'DARKFIL', 'DARKINFO', 'LINIMAGE', 'LINCOEFF', 'BUNIT', 'GAIN', 'OEXPTIME', 'MAGZREF', 'PHOTINDX', 'WCSCAL', 'WCSXRMS', 'WCSYRMS', 'MAGZERO', '', 'MAGZSIG', 'MAGZERR', 'MAGZNAV', 'SEEINGP', 'SKYBG', 'SEEING', 'SKYMAG', 'STKBPM', 'OBJBPM', 'CDELT1', 'CDELT2', 'CRPIX1', 'CRPIX2', 'BPM', 'IMCMB001', 'IMCMB002', 'IMCMB003', 'IMCMB004', 'IMCMB005', 'IMCMB006', 'IMCMB007', 'IMCMB008', 'IMCMB009', 'IMCMB010', 'IMCMB011', 'IMCMB012', 'IMCMB013', 'IMCMB014', 'IMCMB015', 'IMCMB016', 'IMCMB017', 'IMCMB018', 'IMCMB019', 'IMCMB020', 'IMCMB021', 'IMCMB022', 'IMCMB023', 'IMCMB024', 'IMCMB025', 'IMCMB026', 'IMCMB027', 'IMCMB028', 'IMCMB029', 'IMCMB030', 'IMCMB031', 'IMCMB032', 'IMCMB033', 'IMCMB034', 'IMCMB035', 'IMCMB036', 'IMCMB037', 'IMCMB038', 'IMCMB039', 'IMCMB040', 'IMCMB041', 'IMCMB042', 'IMCMB043', 'IMCMB044', 'IMCMB045', 'IMCMB046', 'IMCMB047', 'IMCMB048', 'IMCMB049', 'IMCMB050', 'IMCMB051', 'IMCMB052', 'IMCMB053', 'IMCMB054', 'IMCMB055', 'IMCMB056', 'IMCMB057', 'IMCMB058', 'IMCMB059', 'IMCMB060', 'IMCMB061', 'IMCMB062', 'IMCMB063', 'IMCMB064', 'IMCMB065', 'IMCMB066', 'IMCMB067', 'IMCMB068', 'IMCMB069', 'IMCMB070', 'IMCMB071', 'IMCMB072', 'IMCMB073', 'IMCMB074', 'IMCMB075', 'IMCMB076', 'IMCMB077', 'IMCMB078', 'IMCMB079', 'IMCMB080', 'IMCMB081', 'IMCMB082', 'IMCMB083', 'IMCMB084', 'IMCMB085', 'IMCMB086', 'IMCMB087', 'IMCMB088', 'IMCMB089', 'IMCMB090', 'IMCMB091', 'IMCMB092', 'IMCMB093', 'IMCMB094', 'IMCMB095', 'IMCMB096', 'IMCMB097', 'IMCMB098', 'IMCMB099', 'IMCMB100', 'IMCMB101', 'IMCMB102', 'IMCMB103', 'IMCMB104', 'IMCMB105', 'IMCMB106', 'IMCMB107', 'IMCMB108', 'WMETHOD', 'DQAREA', 'DQEXPMAX', 'DQNOIS', 'DQPDPS', 'DQPDAP', 'DQPDPX', 'DQFWHM', 'DQMZ', 'DQSKY', 'DQSKYXGD', 'DQSKYYGD', 'DQMXTIME', 'DQHFTIME', 'DQMXAREA', 'DQHFAREA', 'DQMXFRAC', 'DQHFFRAC', 'DQMXNOIS', 'DQHFNOIS', 'DQMXPDPS', 'DQHFPDPS', 'DQMXPDAP', 'DQHFPDAP', 'DQMXPDPX', 'DQHFPDPX', 'DQSEEMID', 'DQSEESIG', 'DQSEEMIN', 'DQSEEMAX', 'DQSKYMID', 'DQSKYSIG', 'DQSKYMIN', 'DQSKYMAX', 'DQMASK', 'FILTID', 'PHOTBW', 'PHOTFWHM', 'PHOTCLAM', '', 'PIPELINE', 'PLVER', 'EXPMAP', 'ASTRMCAT', 'EFFTIME', 'WCSAXES', 'PLDNAME', '', 'PLQUEUE', 'PLQNAME', 'PLPROCID', 'PLFNAME', 'PLOFNAME', 'DQMFOR', 'PLPROPID', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', 'CHECKSUM', 'DATASUM'] +retrieve = ['SIMPLE', + 'BITPIX', + 'NAXIS', + 'EXTEND', + 'ORIGIN', + 'DATE', + 'IRAF-TLM', + 'OBJECT', + 'RAWFILE', + 'FILENAME', + 'OBJRA', + 'OBJDEC', + 'OBJEPOCH', + '', + 'TIMESYS', + '', + 'OBSERVAT', + 'OBS-ELEV', + 'OBS-LAT', + 'OBS-LONG', + 'TELESCOP', + 'TELRADEC', + 'TELEQUIN', + 'TELRA', + 'TELDEC', + 'HA', + 'ZD', + 'TELFOCUS', + '', + 'MOSSIZE', + 'NDETS', + '', + 'OBSERVER', + 'PROPOSER', + 'PROPID', + 'SEQID', + 'SEQNUM', + '', + 'NOHS', + 'NOCUTC', + 'NOCRSD', + 'NOCGID', + 'NOCNAME', + '', + 'DTSITE', + 'DTTELESC', + 'DTINSTRU', + 'DTCALDAT', + 'DTUTC', + 'DTOBSERV', + 'DTPROPID', + 'DTPI', + 'DTPIAFFL', + 'DTTITLE', + 'DTCOPYRI', + 'DTACQUIS', + 'DTACCOUN', + 'DTACQNAM', + 'DTNSANAM', + 'DTSTATUS', + 'SB_HOST', + 'SB_ACCOU', + 'SB_SITE', + 'SB_LOCAL', + 'SB_DIR1', + 'SB_DIR2', + 'SB_DIR3', + 'SB_RECNO', + 'SB_ID', + 'SB_NAME', + 'RMCOUNT', + 'RECNO', + 'INSTRUME', + 'RSPTGRP', + 'RSPGRP', + '', + 'OBSTYPE', + 'PROCTYPE', + 'PRODTYPE', + 'MIMETYPE', + 'EXPTIME', + 'FILTER', + '', + 'RA', + 'DEC', + 'CENTRA', + 'CORN1RA', + 'CORN2RA', + 'CORN3RA', + 'CORN4RA', + 'CENTDEC', + 'CORN1DEC', + 'CORN2DEC', + 'CORN3DEC', + 'CORN4DEC', + 'DATE-OBS', + 'TIME-OBS', + 'MJD-OBS', + 'ST', + '', + 'CTYPE1', + 'CTYPE2', + 'CRVAL1', + 'CRVAL2', + 'CD1_1', + 'CD2_2', + 'WAT0_001', + 'WAT1_001', + 'WAT2_001', + '', + 'DIGAVGS', + 'NCOADD', + 'FSAMPLE', + 'EXPCOADD', + 'TITLE', + 'DARKFIL', + 'DARKINFO', + 'LINIMAGE', + 'LINCOEFF', + 'BUNIT', + 'GAIN', + 'OEXPTIME', + 'MAGZREF', + 'PHOTINDX', + 'WCSCAL', + 'WCSXRMS', + 'WCSYRMS', + 'MAGZERO', + '', + 'MAGZSIG', + 'MAGZERR', + 'MAGZNAV', + 'SEEINGP', + 'SKYBG', + 'SEEING', + 'SKYMAG', + 'STKBPM', + 'OBJBPM', + 'CDELT1', + 'CDELT2', + 'CRPIX1', + 'CRPIX2', + 'BPM', + 'IMCMB001', + 'IMCMB002', + 'IMCMB003', + 'IMCMB004', + 'IMCMB005', + 'IMCMB006', + 'IMCMB007', + 'IMCMB008', + 'IMCMB009', + 'IMCMB010', + 'IMCMB011', + 'IMCMB012', + 'IMCMB013', + 'IMCMB014', + 'IMCMB015', + 'IMCMB016', + 'IMCMB017', + 'IMCMB018', + 'IMCMB019', + 'IMCMB020', + 'IMCMB021', + 'IMCMB022', + 'IMCMB023', + 'IMCMB024', + 'IMCMB025', + 'IMCMB026', + 'IMCMB027', + 'IMCMB028', + 'IMCMB029', + 'IMCMB030', + 'IMCMB031', + 'IMCMB032', + 'IMCMB033', + 'IMCMB034', + 'IMCMB035', + 'IMCMB036', + 'IMCMB037', + 'IMCMB038', + 'IMCMB039', + 'IMCMB040', + 'IMCMB041', + 'IMCMB042', + 'IMCMB043', + 'IMCMB044', + 'IMCMB045', + 'IMCMB046', + 'IMCMB047', + 'IMCMB048', + 'IMCMB049', + 'IMCMB050', + 'IMCMB051', + 'IMCMB052', + 'IMCMB053', + 'IMCMB054', + 'IMCMB055', + 'IMCMB056', + 'IMCMB057', + 'IMCMB058', + 'IMCMB059', + 'IMCMB060', + 'IMCMB061', + 'IMCMB062', + 'IMCMB063', + 'IMCMB064', + 'IMCMB065', + 'IMCMB066', + 'IMCMB067', + 'IMCMB068', + 'IMCMB069', + 'IMCMB070', + 'IMCMB071', + 'IMCMB072', + 'IMCMB073', + 'IMCMB074', + 'IMCMB075', + 'IMCMB076', + 'IMCMB077', + 'IMCMB078', + 'IMCMB079', + 'IMCMB080', + 'IMCMB081', + 'IMCMB082', + 'IMCMB083', + 'IMCMB084', + 'IMCMB085', + 'IMCMB086', + 'IMCMB087', + 'IMCMB088', + 'IMCMB089', + 'IMCMB090', + 'IMCMB091', + 'IMCMB092', + 'IMCMB093', + 'IMCMB094', + 'IMCMB095', + 'IMCMB096', + 'IMCMB097', + 'IMCMB098', + 'IMCMB099', + 'IMCMB100', + 'IMCMB101', + 'IMCMB102', + 'IMCMB103', + 'IMCMB104', + 'IMCMB105', + 'IMCMB106', + 'IMCMB107', + 'IMCMB108', + 'WMETHOD', + 'DQAREA', + 'DQEXPMAX', + 'DQNOIS', + 'DQPDPS', + 'DQPDAP', + 'DQPDPX', + 'DQFWHM', + 'DQMZ', + 'DQSKY', + 'DQSKYXGD', + 'DQSKYYGD', + 'DQMXTIME', + 'DQHFTIME', + 'DQMXAREA', + 'DQHFAREA', + 'DQMXFRAC', + 'DQHFFRAC', + 'DQMXNOIS', + 'DQHFNOIS', + 'DQMXPDPS', + 'DQHFPDPS', + 'DQMXPDAP', + 'DQHFPDAP', + 'DQMXPDPX', + 'DQHFPDPX', + 'DQSEEMID', + 'DQSEESIG', + 'DQSEEMIN', + 'DQSEEMAX', + 'DQSKYMID', + 'DQSKYSIG', + 'DQSKYMIN', + 'DQSKYMAX', + 'DQMASK', + 'FILTID', + 'PHOTBW', + 'PHOTFWHM', + 'PHOTCLAM', + '', + 'PIPELINE', + 'PLVER', + 'EXPMAP', + 'ASTRMCAT', + 'EFFTIME', + 'WCSAXES', + 'PLDNAME', + '', + 'PLQUEUE', + 'PLQNAME', + 'PLPROCID', + 'PLFNAME', + 'PLOFNAME', + 'DQMFOR', + 'PLPROPID', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + 'CHECKSUM', + 'DATASUM'] version = '6.0' + +get_token = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzUxMDg0MTE5LCJpYXQiOjE3NTA5OTc3MTksImp0aSI6ImNjYzBmZGJlOTlmNTQ1NDVhMDRjYTU0NGM2Yjc4ZjdmIiwidXNlcl9pZCI6MTMwNzIsImVtYWlsIjoibm9ib2R5QHVuaXZlcnNpdHkuZWR1In0.YZdfyOQUlqiBO48a3y720YDOu9_AWGwEFdiWJ3ML_pg' diff --git a/astroquery/noirlab/tests/test_noirlab.py b/astroquery/noirlab/tests/test_noirlab.py index 3f1f3ac528..5e1d349c6a 100644 --- a/astroquery/noirlab/tests/test_noirlab.py +++ b/astroquery/noirlab/tests/test_noirlab.py @@ -8,26 +8,17 @@ # from astropy.coordinates import SkyCoord # Local packages from ...utils.mocks import MockResponse -from .. import NOIRLab, NOIRLabClass +from .. import NOIRLab #, NOIRLabClass from . import expected as exp @pytest.fixture def patch_request(request): def mockreturn(method, url, **kwargs): - # if 'data' in kwargs: - # cmd = kwargs['data']['uquery'] - # else: - # cmd = kwargs['params']['cmd'] - # if 'SpecObjAll' in cmd: - # filename = data_path(DATA_FILES['spectra_id']) - # else: - # filename = data_path(DATA_FILES['images_id']) - - # with open(filename, 'rb') as infile: - # content = infile.read() if '/version/' in url: content = exp.version.encode('utf-8') + elif '/get_token/' in url: + content = f'"{exp.get_token}"'.encode('utf-8') return MockResponse(content=content, url=url) mp = request.getfixturevalue("monkeypatch") @@ -39,3 +30,8 @@ def mockreturn(method, url, **kwargs): def test_version(patch_request): r = NOIRLab.version() assert r >= float(exp.version) + + +def test_get_token(patch_request): + actual = NOIRLab.get_token('nobody@university.edu', '123456') + assert actual == exp.get_token diff --git a/astroquery/noirlab/tests/test_noirlab_remote.py b/astroquery/noirlab/tests/test_noirlab_remote.py index 4c0e3a2ef4..a05b80f8b2 100644 --- a/astroquery/noirlab/tests/test_noirlab_remote.py +++ b/astroquery/noirlab/tests/test_noirlab_remote.py @@ -25,7 +25,7 @@ class TestNOIRLabClass(object): def test_service_metadata(self): """Test compliance with 6.1 of SIA spec v1.0""" - r = NOIRLab.service_metadata() + r = NOIRLab().service_metadata() actual = r # print(f'DBG: test_service_metadata={actual}') expected = exp.service_metadata @@ -35,7 +35,7 @@ def test_query_region_0(self): """Search FILES using default type (which) selector""" c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') - r = NOIRLab.query_region(c, radius='0.1') + r = NOIRLab().query_region(c, radius='0.1') actual = set(list(r['md5sum'])) expected = exp.query_region_1 assert expected.issubset(actual) @@ -78,7 +78,7 @@ def test_query_region_2(self): def test_aux_file_fields(self): """List the available AUX FILE fields.""" - r = NOIRLab.aux_fields('decam', 'instcal') + r = NOIRLab().aux_fields('decam', 'instcal') actual = r # Returned results may increase over time. print(f'DBG: test_aux_file_fields={actual}') @@ -87,7 +87,7 @@ def test_aux_file_fields(self): def test_core_file_fields(self): """List the available CORE FILE fields.""" - r = NOIRLab.core_fields() + r = NOIRLab().core_fields() actual = r print(f'DBG: test_core_file_fields={actual}') expected = exp.core_file_fields @@ -108,9 +108,9 @@ def test_query_file_metadata(self): ] } - r = NOIRLab.query_metadata(qspec, limit=3) + r = NOIRLab().query_metadata(qspec, limit=3) actual = r - print(f'DBG: test_query_file_metadata={actual.pformat_all()}') + # print(f'DBG: test_query_file_metadata={actual.pformat_all()}') expected = exp.query_file_metadata assert actual.pformat_all() == expected @@ -123,7 +123,7 @@ def test_aux_hdu_fields(self): r = NOIRLabClass(which='hdu').aux_fields('decam', 'instcal') actual = r # Returned results may increase over time. - print(f'DBG: test_aux_hdu_fields={actual}') + # print(f'DBG: test_aux_hdu_fields={actual}') expected = exp.aux_hdu_fields assert actual == expected @@ -131,7 +131,7 @@ def test_core_hdu_fields(self): """List the available CORE HDU fields.""" r = NOIRLabClass(which='hdu').core_fields() actual = r - print(f'DBG: test_core_file_fields={actual}') + # print(f'DBG: test_core_file_fields={actual}') expected = exp.core_hdu_fields assert actual == expected @@ -154,7 +154,7 @@ def test_query_hdu_metadata(self): r = NOIRLabClass(which='hdu').query_metadata(qspec, limit=3) actual = r - print(f'DBG: test_query_hdu_metadata={actual.pformat_all()}') + # print(f'DBG: test_query_hdu_metadata={actual.pformat_all()}') expected = exp.query_hdu_metadata assert actual.pformat_all() == expected @@ -164,7 +164,7 @@ def test_query_hdu_metadata(self): def test_categoricals(self): """List categories.""" - r = NOIRLab.categoricals() + r = NOIRLab().categoricals() actual = r # Returned results may increase over time. print(f'DBG: test_categoricals={actual}') @@ -178,17 +178,17 @@ def test_categoricals(self): # version def test_retrieve(self): - hdul = NOIRLab.retrieve('f92541fdc566dfebac9e7d75e12b5601') + hdul = NOIRLab().retrieve('f92541fdc566dfebac9e7d75e12b5601') actual = list(hdul[0].header.keys()) expected = exp.retrieve assert actual == expected def test_version(self): - r = NOIRLab.version() + r = NOIRLab().version() assert r >= exp.version def test_get_token(self): - actual = NOIRLab.get_token('nobody@university.edu', '123456789') - expected = {'detail': - 'No active account found with the given credentials'} - assert actual == expected + actual = NOIRLab().get_token('nobody@university.edu', '123456789') + # expected = {'detail': + # 'No active account found with the given credentials'} + assert actual == exp.get_token From 13d76868ef9e287d8508822e9857b3adffd35733 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Fri, 27 Jun 2025 11:17:15 -0700 Subject: [PATCH 09/26] ongoing test refactoring --- astroquery/noirlab/core.py | 94 +- astroquery/noirlab/tests/expected.py | 852 +++++------------- astroquery/noirlab/tests/test_noirlab.py | 51 +- .../noirlab/tests/test_noirlab_remote.py | 358 ++++---- 4 files changed, 524 insertions(+), 831 deletions(-) diff --git a/astroquery/noirlab/core.py b/astroquery/noirlab/core.py index 91d7d0a5bb..f9d8e4c38c 100644 --- a/astroquery/noirlab/core.py +++ b/astroquery/noirlab/core.py @@ -7,6 +7,7 @@ import astropy.table from ..query import BaseQuery from ..utils import async_to_sync +from ..exceptions import RemoteServiceError from ..utils.class_or_instance import class_or_instance from . import conf @@ -16,22 +17,23 @@ @async_to_sync class NOIRLabClass(BaseQuery): - + """Search functionality for the NSF NOIRLab Astro Data Archive. + + Parameters + ---------- + hdu : :class:`bool`, optional + If ``True``, search HDUs in files. THe HDUs must have RA, DEC header + keywords. This is not guaranteed for all files. + The default is to just search for files. + """ TIMEOUT = conf.timeout NAT_URL = conf.server - def __init__(self, which='file'): - """Return object used for searching the NOIRLab Archive. - - Search either Files (which=file) or HDUs (which=hdu). - Files will always be returned. But if which=hdu, - individual HDUs must have RA,DEC fields. Typically this - is only the case with some pipeline processed files. - """ + def __init__(self, hdu=False): self._api_version = None self._adsurl = f'{self.NAT_URL}/api/adv_search' - if which == 'hdu': + if hdu: self.siaurl = f'{self.NAT_URL}/api/sia/vohdu' self._adss_url = f'{self._adsurl}/hasearch' self._adsc_url = f'{self._adsurl}/core_hdu_fields' @@ -46,33 +48,29 @@ def __init__(self, which='file'): @property def api_version(self): - """Return version of Rest API used by this module. + """Return version of REST API used by this module. - If the Rest API changes such that the Major version increases, + If the REST API changes such that the major version increases, a new version of this module will likely need to be used. """ if self._api_version is None: - response = self._request('GET', - f'{self.NAT_URL}/api/version', - timeout=self.TIMEOUT, - cache=True) - self._api_version = float(response.content) + self._api_version = float(self.version()) return self._api_version def _validate_version(self): - KNOWN_GOOD_API_VERSION = 2.0 + KNOWN_GOOD_API_VERSION = 6.0 if (int(self.api_version) - int(KNOWN_GOOD_API_VERSION)) >= 1: msg = (f'The astroquery.noirlab module is expecting an older ' f'version of the {self.NAT_URL} API services. ' f'Please upgrade to latest astroquery. ' f'Expected version {KNOWN_GOOD_API_VERSION} but got ' f'{self.api_version} from the API.') - raise Exception(msg) + raise RemoteServiceError(msg) def service_metadata(self, cache=True): """Denotes a Metadata Query: no images are requested; only metadata - should be returned. This feature is described in more detail in: - http://www.ivoa.net/documents/PR/DAL/PR-SIA-1.0-20090521.html#mdquery + should be returned. This feature is described in more detail in: + http://www.ivoa.net/documents/PR/DAL/PR-SIA-1.0-20090521.html#mdquery """ url = f'{self.siaurl}?FORMAT=METADATA&format=json' response = self._request('GET', url, timeout=self.TIMEOUT, cache=cache) @@ -98,7 +96,7 @@ def query_region(self, coordinate, radius=0.1, cache=True): Returns ------- - response : `~astropy.table.Table` + `~astropy.table.Table` """ self._validate_version() ra, dec = coordinate.to_string('decimal').split() @@ -128,7 +126,7 @@ def query_region_async(self, coordinate, radius=0.1, cache=True): Returns ------- - response : `requests.Response` + `requests.Response` """ self._validate_version() @@ -184,26 +182,66 @@ def query_metadata(self, qspec, limit=1000, cache=True): else: jdata = qspec - response = self._request('POST', url, json=jdata, timeout=self.TIMEOUT) + response = self._request('POST', url, json=jdata, + timeout=self.TIMEOUT, cache=cache) response.raise_for_status() return astropy.table.Table(rows=response.json()) - def retrieve(self, fileid, cache=True): + def retrieve(self, fileid): + """Simply fetch a file by `fileid`. + + Parameters + ---------- + fileid : :class:`str` + The MD5 ID of the file. + + Returns + ------- + :class:`~astropy.io.fits.HDUlist` + The open FITS file. This object should be ``.close()``d when done. + """ url = f'{self.NAT_URL}/api/retrieve/{fileid}/' - hdul = fits.open(url) - return hdul + hdulist = fits.open(url) + return hdulist def version(self, cache=False): + """Return the version of the REST API. + + Parameters + ---------- + cache : :class:`bool`, optional + If ``True`` cache the result locally. + + Returns + ------- + :class:`float` + The API version as a number. + """ url = f'{self.NAT_URL}/api/version/' response = self._request('GET', url, timeout=self.TIMEOUT, cache=cache) response.raise_for_status() return response.json() def get_token(self, email, password, cache=True): + """Get an access token to use with proprietary data. + + Parameters + ---------- + email : :class:`str` + Email for account access. + password : :class:`str` + Password associated with `email`. *Please* never hard-code your + password *anywhere*. + + Returns + ------- + :class:`str` + The access token as a string. + """ url = f'{self.NAT_URL}/api/get_token/' response = self._request('POST', url, json={"email": email, "password": password}, - timeout=self.TIMEOUT) + timeout=self.TIMEOUT, cache=cache) response.raise_for_status() return response.json() diff --git a/astroquery/noirlab/tests/expected.py b/astroquery/noirlab/tests/expected.py index a46662484d..f1f7f53377 100644 --- a/astroquery/noirlab/tests/expected.py +++ b/astroquery/noirlab/tests/expected.py @@ -1,4 +1,5 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst +# flake8: noqa """ Expected values for online tests. For offline tests, these values may be used to create mock responses. @@ -18,20 +19,10 @@ 'f990925c8d99a66f642e4463f1dde7f2', '09535b88fc700227bb47979a670a7d85'} -service_metadata = {'archive_filename': 'string', - 'date_obs': 'string', - 'dec': 'float', - 'exposure': 'float', - 'filesize': 'int', - 'instrument': 'string', - 'md5sum': 'string', - 'original_filename': 'string', - 'pi': 'string', - 'prop_id': 'string', - 'ra': 'float', - 'release_date': 'string', - 'telescope': 'string', - 'updated': 'string'} +service_metadata = [{'ParameterName': 'INPUT:DATE', + 'DataType': 'char', + 'DefaultValue': 'NA', + 'Doc': 'NA'}] aux_file_fields = {'AIRMASS': 'str', 'AOS': 'str', @@ -165,56 +156,157 @@ 'G-MEANY': 'str', 'G-MEANY2': 'str', 'G-MODE': 'str', - 'G-SEEING': 'str', 'GSKYHOT': 'str', 'GSKYPHOT': 'str', - 'GSKYVAR': 'str', 'G-TRANSP': 'str', 'GUIDER': 'str', - 'HA': 'str', 'HDRVER': 'str', 'HEX': 'str', 'HUMIDITY': 'str', - 'INSTANCE': 'str', 'INSTRUME': 'str', 'IRAF-TLM': 'str', - 'LINCFIL': 'str', 'LSKYHOT': 'str', 'LSKYPHOT': 'str', - 'LSKYPOW': 'str', 'LSKYVAR': 'str', 'LST': 'str', - 'LUTVER': 'str', 'LWTRTEMP': 'str', 'MAGZERO': 'str', - 'MAGZP': 'str', 'MAGZPT': 'str', 'MAGZUNC': 'str', - 'MAIRTEMP': 'str', 'MANIFEST': 'str', 'MASS2': 'str', - 'MJD-END': 'str', 'MJD-OBS': 'str', 'MOONANGL': 'str', - 'MSURTEMP': 'str', 'MULTIEXP': 'str', 'MULTIFOC': 'str', - 'MULTIID': 'str', 'MULTIROW': 'str', 'MULTITOT': 'str', - 'NBLEED': 'str', 'NDONUTS': 'str', 'NEXTEND': 'str', - 'NITE': 'str', 'NOTE': 'str', 'NPHTMTCH': 'str', - 'NSATPIX': 'str', 'NUM': 'str', 'OBJECT': 'str', - 'OBS-ELEV': 'str', 'OBSERVAT': 'str', 'OBSERVER': 'str', - 'OBSID': 'str', 'OBS-LAT': 'str', 'OBS-LONG': 'str', - 'OBSTYPE': 'str', 'ODATEOBS': 'str', 'OPENSHUT': 'str', - 'ORIGIN': 'str', 'OUTTEMP': 'str', 'PHOTFLAG': 'str', - 'PHOTREF': 'str', 'PIPELINE': 'str', 'PIXSCAL1': 'str', - 'PIXSCAL2': 'str', 'PLARVER': 'str', 'PLDNAME': 'str', - 'PLDSID': 'str', 'PLFNAME': 'str', 'PLPROCID': 'str', - 'PLQNAME': 'str', 'PLQUEUE': 'str', 'PLVER': 'str', - 'PME-TEMP': 'str', 'PMN-TEMP': 'str', 'PMOSTEMP': 'str', - 'PMS-TEMP': 'str', 'PMW-TEMP': 'str', 'PRESSURE': 'str', - 'PROCTYPE': 'str', 'PRODTYPE': 'str', 'PROGRAM': 'str', - 'PROPID': 'str', 'PROPOSER': 'str', 'PUPILAMP': 'str', - 'PUPILMAX': 'str', 'PUPILSKY': 'str', 'PUPMAX': 'str', - 'PV1_6': 'str', 'PV2_4': 'str', 'RA': 'str', - 'RA_CENT': 'str', 'RACMAX': 'str', 'RACMIN': 'str', - 'RADECSYS': 'str', 'RADESYS': 'str', 'RADIUS': 'str', - 'RADSTD': 'str', 'RECNO': 'str', 'REQNUM': 'str', - 'RMCOUNT': 'str', 'SB_ACCOU': 'str', 'SB_DIR1': 'str', - 'SB_DIR2': 'str', 'SB_DIR3': 'str', 'SB_HOST': 'str', - 'SB_ID': 'str', 'SB_LOCAL': 'str', 'SB_NAME': 'str', - 'SB_RECNO': 'str', 'SB_RTNAM': 'str', 'SB_SITE': 'str', - 'SCAMPCHI': 'str', 'SCAMPFLG': 'str', 'SCAMPNUM': 'str', - 'SCAMPREF': 'str', 'SEQID': 'str', 'SEQNUM': 'str', - 'SEQTOT': 'str', 'SEQTYPE': 'str', 'SISPIVER': 'str', - 'SKYBRITE': 'str', 'SKYORDER': 'str', 'SKYPC00': 'str', - 'SKYPC01': 'str', 'SKYPC02': 'str', 'SKYPC03': 'str', - 'SKYSIGMA': 'str', 'SKYSTAT': 'str', 'SKYSUB': 'str', - 'SKYSUB01': 'str', 'SKYSUB10': 'str', 'SKYSUBP': 'str', - 'SKYUPDAT': 'str', 'SLOT01': 'str', 'SLOT03': 'str', - 'SURVEYID': 'str', 'TELDEC': 'str', 'TELEQUIN': 'str', - 'TELESCOP': 'str', 'TELFOCUS': 'str', 'TELRA': 'str', - 'TELSTAT': 'str', 'TILING': 'str', 'TIME-OBS': 'str', - 'TIMESYS': 'str', 'TRACKING': 'str', 'UNITNAME': 'str', - 'UPTRTEMP': 'str', 'UTE-TEMP': 'str', 'UTN-TEMP': 'str', - 'UTS-TEMP': 'str', 'UTW-TEMP': 'str', 'VALIDA': 'str', + 'G-SEEING': 'str', + 'GSKYHOT': 'str', + 'GSKYPHOT': 'str', + 'GSKYVAR': 'str', + 'G-TRANSP': 'str', + 'GUIDER': 'str', + 'HA': 'str', + 'HDRVER': 'str', + 'HEX': 'str', + 'HUMIDITY': 'str', + 'INSTANCE': 'str', + 'INSTRUME': 'str', + 'IRAF-TLM': 'str', + 'LINCFIL': 'str', + 'LSKYHOT': 'str', + 'LSKYPHOT': 'str', + 'LSKYPOW': 'str', + 'LSKYVAR': 'str', + 'LST': 'str', + 'LUTVER': 'str', + 'LWTRTEMP': 'str', + 'MAGZERO': 'str', + 'MAGZP': 'str', + 'MAGZPT': 'str', + 'MAGZUNC': 'str', + 'MAIRTEMP': 'str', + 'MANIFEST': 'str', + 'MASS2': 'str', + 'MJD-END': 'str', + 'MJD-OBS': 'str', + 'MOONANGL': 'str', + 'MSURTEMP': 'str', + 'MULTIEXP': 'str', + 'MULTIFOC': 'str', + 'MULTIID': 'str', + 'MULTIROW': 'str', + 'MULTITOT': 'str', + 'NBLEED': 'str', + 'NDONUTS': 'str', + 'NEXTEND': 'str', + 'NITE': 'str', + 'NOTE': 'str', + 'NPHTMTCH': 'str', + 'NSATPIX': 'str', + 'NUM': 'str', + 'OBJECT': 'str', + 'OBS-ELEV': 'str', + 'OBSERVAT': 'str', + 'OBSERVER': 'str', + 'OBSID': 'str', + 'OBS-LAT': 'str', + 'OBS-LONG': 'str', + 'OBSTYPE': 'str', + 'ODATEOBS': 'str', + 'OPENSHUT': 'str', + 'ORIGIN': 'str', + 'OUTTEMP': 'str', + 'PHOTFLAG': 'str', + 'PHOTREF': 'str', + 'PIPELINE': 'str', + 'PIXSCAL1': 'str', + 'PIXSCAL2': 'str', + 'PLARVER': 'str', + 'PLDNAME': 'str', + 'PLDSID': 'str', + 'PLFNAME': 'str', + 'PLPROCID': 'str', + 'PLQNAME': 'str', + 'PLQUEUE': 'str', + 'PLVER': 'str', + 'PME-TEMP': 'str', + 'PMN-TEMP': 'str', + 'PMOSTEMP': 'str', + 'PMS-TEMP': 'str', + 'PMW-TEMP': 'str', + 'PRESSURE': 'str', + 'PROCTYPE': 'str', + 'PRODTYPE': 'str', + 'PROGRAM': 'str', + 'PROPID': 'str', + 'PROPOSER': 'str', + 'PUPILAMP': 'str', + 'PUPILMAX': 'str', + 'PUPILSKY': 'str', + 'PUPMAX': 'str', + 'PV1_6': 'str', + 'PV2_4': 'str', + 'RA': 'str', + 'RA_CENT': 'str', + 'RACMAX': 'str', + 'RACMIN': 'str', + 'RADECSYS': 'str', + 'RADESYS': 'str', + 'RADIUS': 'str', + 'RADSTD': 'str', + 'RECNO': 'str', + 'REQNUM': 'str', + 'RMCOUNT': 'str', + 'SB_ACCOU': 'str', + 'SB_DIR1': 'str', + 'SB_DIR2': 'str', + 'SB_DIR3': 'str', + 'SB_HOST': 'str', + 'SB_ID': 'str', + 'SB_LOCAL': 'str', + 'SB_NAME': 'str', + 'SB_RECNO': 'str', + 'SB_RTNAM': 'str', + 'SB_SITE': 'str', + 'SCAMPCHI': 'str', + 'SCAMPFLG': 'str', + 'SCAMPNUM': 'str', + 'SCAMPREF': 'str', + 'SEQID': 'str', + 'SEQNUM': 'str', + 'SEQTOT': 'str', + 'SEQTYPE': 'str', + 'SISPIVER': 'str', + 'SKYBRITE': 'str', + 'SKYORDER': 'str', + 'SKYPC00': 'str', + 'SKYPC01': 'str', + 'SKYPC02': 'str', + 'SKYPC03': 'str', + 'SKYSIGMA': 'str', + 'SKYSTAT': 'str', + 'SKYSUB': 'str', + 'SKYSUB01': 'str', + 'SKYSUB10': 'str', + 'SKYSUBP': 'str', + 'SKYUPDAT': 'str', + 'SLOT01': 'str', + 'SLOT03': 'str', + 'SURVEYID': 'str', + 'TELDEC': 'str', + 'TELEQUIN': 'str', + 'TELESCOP': 'str', + 'TELFOCUS': 'str', + 'TELRA': 'str', + 'TELSTAT': 'str', + 'TILING': 'str', + 'TIME-OBS': 'str', + 'TIMESYS': 'str', + 'TRACKING': 'str', + 'UNITNAME': 'str', + 'UPTRTEMP': 'str', + 'UTE-TEMP': 'str', + 'UTN-TEMP': 'str', + 'UTS-TEMP': 'str', + 'UTW-TEMP': 'str', + 'VALIDA': 'str', 'VALIDB': 'str', 'VSUB': 'str', 'WCSCAL': 'str', @@ -399,17 +491,30 @@ 'arcon', 'bench', 'ccd_imager', + 'ccd_spec', 'chiron', 'cosmos', + 'cpapir', 'decam', 'echelle', 'falmingos', 'flamingos', + 'ghts_blue', + 'ghts_blue_imager', + 'ghts_red', + 'ghts_red_imager', 'goodman', 'goodman spectrograph', 'gtcam', 'hdi', + 'hlsp_bok23m', + 'hlsp_decam', + 'hlsp_ir_imager', + 'hlsp_mosaic', + 'hlsp_mosaic3', + 'hlsp_mosaic_2', 'ice', + 'ir_imager', 'ispi', 'kosmos', 'minimo/ice', @@ -421,6 +526,7 @@ 'mosaic_2', 'newfirm', 'osiris', + 'pfccd', 'sami', 'soi', 'spartan', @@ -430,7 +536,56 @@ 'whirc', 'wildfire', 'y4kcam'], - 'obsmodes': [], + 'obsmodes': ['acq', + 'imaging', + 'sos_slit', + 'spec', + 'stare'], + 'obstypes': ['acq', + 'acquire', + 'arc', + 'bias', + 'calibration', + 'calibration or comparison', + 'comp', + 'comparison', + 'dark', + 'dflat', + 'dome flat', + 'dome or projector flat', + 'domeflat', + 'expose', + 'flat', + 'focus', + 'fringe', + 'fringecor', + 'guider', + 'illum', + 'illumcor', + 'illumination calibration', + 'junk', + 'lampflat', + 'none', + 'nota', + 'obj', + 'object', + 'phot flat', + 'photometric standard', + 'projector', + 'projector flat', + 'pupil', + 'red', + 'remap', + 'sflat', + 'sky', + 'sky flat', + 'skyflat', + 'spectrum', + 'standard', + 'std', + 'test', + 'unknown', + 'zero'], 'proctypes': ['instcal', 'mastercal', 'nota', @@ -438,15 +593,29 @@ 'raw', 'resampled', 'skysub', - 'stacked'], - 'prodtypes': ['dqmask', + 'stacked', + 'starsubtracted'], + 'prodtypes': ['chi2', + 'depth', + 'dqmask', 'expmap', + 'flag', + 'galdepth', 'graphics (size)', 'image', 'image 2nd version 1', 'image1', + 'invvar', + 'maskbits', + 'model', + 'ncomb', + 'nexp', 'nota', + 'nrej', + 'psfsize', 'resampled', + 'rms', + 'sigma', 'weight', 'wtmap'], 'sites': ['cp', @@ -470,549 +639,12 @@ 'soar', 'wiyn']} -retrieve = ['SIMPLE', - 'BITPIX', - 'NAXIS', - 'EXTEND', - 'ORIGIN', - 'DATE', - 'IRAF-TLM', - 'OBJECT', - 'RAWFILE', - 'FILENAME', - 'OBJRA', - 'OBJDEC', - 'OBJEPOCH', - '', - 'TIMESYS', - '', - 'OBSERVAT', - 'OBS-ELEV', - 'OBS-LAT', - 'OBS-LONG', - 'TELESCOP', - 'TELRADEC', - 'TELEQUIN', - 'TELRA', - 'TELDEC', - 'HA', - 'ZD', - 'TELFOCUS', - '', - 'MOSSIZE', - 'NDETS', - '', - 'OBSERVER', - 'PROPOSER', - 'PROPID', - 'SEQID', - 'SEQNUM', - '', - 'NOHS', - 'NOCUTC', - 'NOCRSD', - 'NOCGID', - 'NOCNAME', - '', - 'DTSITE', - 'DTTELESC', - 'DTINSTRU', - 'DTCALDAT', - 'DTUTC', - 'DTOBSERV', - 'DTPROPID', - 'DTPI', - 'DTPIAFFL', - 'DTTITLE', - 'DTCOPYRI', - 'DTACQUIS', - 'DTACCOUN', - 'DTACQNAM', - 'DTNSANAM', - 'DTSTATUS', - 'SB_HOST', - 'SB_ACCOU', - 'SB_SITE', - 'SB_LOCAL', - 'SB_DIR1', - 'SB_DIR2', - 'SB_DIR3', - 'SB_RECNO', - 'SB_ID', - 'SB_NAME', - 'RMCOUNT', - 'RECNO', - 'INSTRUME', - 'RSPTGRP', - 'RSPGRP', - '', - 'OBSTYPE', - 'PROCTYPE', - 'PRODTYPE', - 'MIMETYPE', - 'EXPTIME', - 'FILTER', - '', - 'RA', - 'DEC', - 'CENTRA', - 'CORN1RA', - 'CORN2RA', - 'CORN3RA', - 'CORN4RA', - 'CENTDEC', - 'CORN1DEC', - 'CORN2DEC', - 'CORN3DEC', - 'CORN4DEC', - 'DATE-OBS', - 'TIME-OBS', - 'MJD-OBS', - 'ST', - '', - 'CTYPE1', - 'CTYPE2', - 'CRVAL1', - 'CRVAL2', - 'CD1_1', - 'CD2_2', - 'WAT0_001', - 'WAT1_001', - 'WAT2_001', - '', - 'DIGAVGS', - 'NCOADD', - 'FSAMPLE', - 'EXPCOADD', - 'TITLE', - 'DARKFIL', - 'DARKINFO', - 'LINIMAGE', - 'LINCOEFF', - 'BUNIT', - 'GAIN', - 'OEXPTIME', - 'MAGZREF', - 'PHOTINDX', - 'WCSCAL', - 'WCSXRMS', - 'WCSYRMS', - 'MAGZERO', - '', - 'MAGZSIG', - 'MAGZERR', - 'MAGZNAV', - 'SEEINGP', - 'SKYBG', - 'SEEING', - 'SKYMAG', - 'STKBPM', - 'OBJBPM', - 'CDELT1', - 'CDELT2', - 'CRPIX1', - 'CRPIX2', - 'BPM', - 'IMCMB001', - 'IMCMB002', - 'IMCMB003', - 'IMCMB004', - 'IMCMB005', - 'IMCMB006', - 'IMCMB007', - 'IMCMB008', - 'IMCMB009', - 'IMCMB010', - 'IMCMB011', - 'IMCMB012', - 'IMCMB013', - 'IMCMB014', - 'IMCMB015', - 'IMCMB016', - 'IMCMB017', - 'IMCMB018', - 'IMCMB019', - 'IMCMB020', - 'IMCMB021', - 'IMCMB022', - 'IMCMB023', - 'IMCMB024', - 'IMCMB025', - 'IMCMB026', - 'IMCMB027', - 'IMCMB028', - 'IMCMB029', - 'IMCMB030', - 'IMCMB031', - 'IMCMB032', - 'IMCMB033', - 'IMCMB034', - 'IMCMB035', - 'IMCMB036', - 'IMCMB037', - 'IMCMB038', - 'IMCMB039', - 'IMCMB040', - 'IMCMB041', - 'IMCMB042', - 'IMCMB043', - 'IMCMB044', - 'IMCMB045', - 'IMCMB046', - 'IMCMB047', - 'IMCMB048', - 'IMCMB049', - 'IMCMB050', - 'IMCMB051', - 'IMCMB052', - 'IMCMB053', - 'IMCMB054', - 'IMCMB055', - 'IMCMB056', - 'IMCMB057', - 'IMCMB058', - 'IMCMB059', - 'IMCMB060', - 'IMCMB061', - 'IMCMB062', - 'IMCMB063', - 'IMCMB064', - 'IMCMB065', - 'IMCMB066', - 'IMCMB067', - 'IMCMB068', - 'IMCMB069', - 'IMCMB070', - 'IMCMB071', - 'IMCMB072', - 'IMCMB073', - 'IMCMB074', - 'IMCMB075', - 'IMCMB076', - 'IMCMB077', - 'IMCMB078', - 'IMCMB079', - 'IMCMB080', - 'IMCMB081', - 'IMCMB082', - 'IMCMB083', - 'IMCMB084', - 'IMCMB085', - 'IMCMB086', - 'IMCMB087', - 'IMCMB088', - 'IMCMB089', - 'IMCMB090', - 'IMCMB091', - 'IMCMB092', - 'IMCMB093', - 'IMCMB094', - 'IMCMB095', - 'IMCMB096', - 'IMCMB097', - 'IMCMB098', - 'IMCMB099', - 'IMCMB100', - 'IMCMB101', - 'IMCMB102', - 'IMCMB103', - 'IMCMB104', - 'IMCMB105', - 'IMCMB106', - 'IMCMB107', - 'IMCMB108', - 'WMETHOD', - 'DQAREA', - 'DQEXPMAX', - 'DQNOIS', - 'DQPDPS', - 'DQPDAP', - 'DQPDPX', - 'DQFWHM', - 'DQMZ', - 'DQSKY', - 'DQSKYXGD', - 'DQSKYYGD', - 'DQMXTIME', - 'DQHFTIME', - 'DQMXAREA', - 'DQHFAREA', - 'DQMXFRAC', - 'DQHFFRAC', - 'DQMXNOIS', - 'DQHFNOIS', - 'DQMXPDPS', - 'DQHFPDPS', - 'DQMXPDAP', - 'DQHFPDAP', - 'DQMXPDPX', - 'DQHFPDPX', - 'DQSEEMID', - 'DQSEESIG', - 'DQSEEMIN', - 'DQSEEMAX', - 'DQSKYMID', - 'DQSKYSIG', - 'DQSKYMIN', - 'DQSKYMAX', - 'DQMASK', - 'FILTID', - 'PHOTBW', - 'PHOTFWHM', - 'PHOTCLAM', - '', - 'PIPELINE', - 'PLVER', - 'EXPMAP', - 'ASTRMCAT', - 'EFFTIME', - 'WCSAXES', - 'PLDNAME', - '', - 'PLQUEUE', - 'PLQNAME', - 'PLPROCID', - 'PLFNAME', - 'PLOFNAME', - 'DQMFOR', - 'PLPROPID', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - 'CHECKSUM', - 'DATASUM'] +retrieve = {'DATE': '2009-09-06T20:13:15', + 'TELESCOP': 'KPNO 4.0 meter telescope', + 'INSTRUME': 'newfirm', + 'CHECKSUM': '7GBmAGAm7GAmAGAm', + 'DATASUM': '0'} version = '6.0' -get_token = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzUxMDg0MTE5LCJpYXQiOjE3NTA5OTc3MTksImp0aSI6ImNjYzBmZGJlOTlmNTQ1NDVhMDRjYTU0NGM2Yjc4ZjdmIiwidXNlcl9pZCI6MTMwNzIsImVtYWlsIjoibm9ib2R5QHVuaXZlcnNpdHkuZWR1In0.YZdfyOQUlqiBO48a3y720YDOu9_AWGwEFdiWJ3ML_pg' +get_token = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9' diff --git a/astroquery/noirlab/tests/test_noirlab.py b/astroquery/noirlab/tests/test_noirlab.py index 5e1d349c6a..41cf5e8cbe 100644 --- a/astroquery/noirlab/tests/test_noirlab.py +++ b/astroquery/noirlab/tests/test_noirlab.py @@ -2,34 +2,55 @@ """ Test astroquery.noirlab, but monkeypatch any HTTP requests. """ -# External packages +import json import pytest # from astropy import units as u # from astropy.coordinates import SkyCoord -# Local packages from ...utils.mocks import MockResponse -from .. import NOIRLab #, NOIRLabClass +from .. import NOIRLab # , NOIRLabClass from . import expected as exp +def mock_content(method, url, **kwargs): + if 'FORMAT=METADATA' in url: + content = json.dumps(exp.service_metadata) + elif '/cat_lists/' in url: + content = json.dumps(exp.categoricals) + elif '/version/' in url: + content = exp.version + elif '/get_token/' in url: + content = f'"{exp.get_token}"' + return MockResponse(content=content.encode('utf-8'), url=url) + + @pytest.fixture -def patch_request(request): - def mockreturn(method, url, **kwargs): - if '/version/' in url: - content = exp.version.encode('utf-8') - elif '/get_token/' in url: - content = f'"{exp.get_token}"'.encode('utf-8') - return MockResponse(content=content, url=url) +def patch_request(monkeypatch): + monkeypatch.setattr(NOIRLab, '_request', mock_content) + return monkeypatch + - mp = request.getfixturevalue("monkeypatch") +def test_service_metadata(patch_request): + """Test compliance with 6.1 of SIA spec v1.0. + """ + actual = NOIRLab().service_metadata() + assert actual == exp.service_metadata[0] - mp.setattr(NOIRLab, '_request', mockreturn) - return mp + +def test_categoricals(patch_request): + """List categories. + """ + actual = NOIRLab().categoricals() + assert actual == exp.categoricals def test_version(patch_request): - r = NOIRLab.version() - assert r >= float(exp.version) + actual = NOIRLab.version() + assert actual >= float(exp.version) + + +def test_api_version(patch_request): + actual = NOIRLab.api_version + assert actual >= float(exp.version) def test_get_token(patch_request): diff --git a/astroquery/noirlab/tests/test_noirlab_remote.py b/astroquery/noirlab/tests/test_noirlab_remote.py index a05b80f8b2..75fccf92fd 100644 --- a/astroquery/noirlab/tests/test_noirlab_remote.py +++ b/astroquery/noirlab/tests/test_noirlab_remote.py @@ -6,189 +6,191 @@ tox -e py310-test-online -- -P noirlab """ -# External packages import pytest from astropy import units as u from astropy.coordinates import SkyCoord -# Local packages from .. import NOIRLab, NOIRLabClass from . import expected as exp +# (2) SIA; /api/sia/ +# voimg, vohdu + +@pytest.mark.remote_data +def test_service_metadata(): + """Test compliance with 6.1 of SIA spec v1.0. + """ + actual = NOIRLab().service_metadata() + assert actual == exp.service_metadata[0] + + +@pytest.mark.skip(reason='old API') +@pytest.mark.remote_data +def test_query_region_0(): + """Search FILES.""" + + c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') + r = NOIRLab().query_region(c, radius='0.1') + actual = set(list(r['md5sum'])) + expected = exp.query_region_1 + assert expected.issubset(actual) + + +@pytest.mark.skip(reason='old API') +@pytest.mark.remote_data +def test_query_region_1(): + """Search FILES. + Ensure query gets at least the set of files we expect. + Its ok if more files have been added to the remote Archive.""" + + c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') + r = NOIRLabClass().query_region(c, radius='0.1') + actual = set(list(r['md5sum'])) + expected = exp.query_region_1 + assert expected.issubset(actual) + + +@pytest.mark.skip(reason='old API') +@pytest.mark.remote_data +def test_query_region_2(): + """Search HDUs. + Ensure query gets at least the set of files we expect. + Its ok if more files have been added to the remote Archive.""" + + c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') + r = NOIRLabClass(hdu=True).query_region(c, radius='0.07') + actual = set(list(r['md5sum'])) + expected = exp.query_region_2 + assert expected.issubset(actual) + + +# (7) Advanced Search; /api/adv_search/ +# +# (2) aux_{file,hdu}_fields// +# (2) core_{file,hdu}_fields/ +# [(2) {f,h}adoc JUST LINK to these] +# (2) {f,h}asearch +# cat_list +# +# File (default type) +# +@pytest.mark.skip(reason='old API') +@pytest.mark.remote_data +def test_aux_file_fields(): + """List the available AUX FILE fields.""" + actual = NOIRLab().aux_fields('decam', 'instcal') + assert actual == exp.aux_file_fields + + +@pytest.mark.skip(reason='old API') +@pytest.mark.remote_data +def test_core_file_fields(): + """List the available CORE FILE fields.""" + actual = NOIRLab().core_fields() + assert actual == exp.core_file_fields + + +@pytest.mark.skip(reason='old API') +@pytest.mark.remote_data +def test_query_file_metadata(): + """Search FILE metadata.""" + qspec = { + "outfields": [ + "md5sum", + "archive_filename", + "original_filename", + "instrument", + "proc_type" + ], + "search": [ + ['original_filename', 'c4d_', 'contains'] + ] + } + + actual = NOIRLab().query_metadata(qspec, limit=3) + assert actual.pformat_all() == exp.query_file_metadata + + +# +# HDU search +# +@pytest.mark.skip(reason='old API') +@pytest.mark.remote_data +def test_aux_hdu_fields(): + """List the available AUX HDU fields.""" + actual = NOIRLabClass(hdu=True).aux_fields('decam', 'instcal') + assert actual == exp.aux_hdu_fields + + +@pytest.mark.skip(reason='old API') +@pytest.mark.remote_data +def test_core_hdu_fields(): + """List the available CORE HDU fields.""" + actual = NOIRLabClass(hdu=True).core_fields() + assert actual == exp.core_hdu_fields + + +@pytest.mark.skip(reason='old API') +@pytest.mark.remote_data +def test_query_hdu_metadata(): + """Search HDU metadata.""" + qspec = { + "outfields": [ + "fitsfile__archive_filename", + "fitsfile__caldat", + "fitsfile__instrument", + "fitsfile__proc_type", + "AIRMASS" # AUX field. Slows search + ], + "search": [ + ["fitsfile__caldat", "2017-08-14", "2017-08-16"], + ["fitsfile__instrument", "decam"], + ["fitsfile__proc_type", "raw"] + ] + } + + actual = NOIRLabClass(hdu=True).query_metadata(qspec, limit=3) + assert actual.pformat_all() == exp.query_hdu_metadata + + +# +# Agnostic +# +@pytest.mark.remote_data +def test_categoricals(): + """List categories. + """ + actual = NOIRLab().categoricals() + assert actual == exp.categoricals + + +# Other tests: +# get_token +# retrieve/ +# version +# +@pytest.mark.remote_data +def test_retrieve(): + hdulist = NOIRLab().retrieve('f92541fdc566dfebac9e7d75e12b5601') + for key in exp.retrieve: + assert key in hdulist[0].header + assert hdulist[0].header[key] == exp.retrieve[key] + hdulist.close() + + +@pytest.mark.remote_data +def test_version(): + actual = NOIRLab().version() + assert actual >= float(exp.version) + + +@pytest.mark.remote_data +def test_api_version(): + actual = NOIRLab().api_version + assert actual >= float(exp.version) + + @pytest.mark.remote_data -class TestNOIRLabClass(object): - - # ############################################################### - # ### (2) SIA; /api/sia/ - # ### - # voimg, vohdu - - def test_service_metadata(self): - """Test compliance with 6.1 of SIA spec v1.0""" - r = NOIRLab().service_metadata() - actual = r - # print(f'DBG: test_service_metadata={actual}') - expected = exp.service_metadata - assert actual == expected - - def test_query_region_0(self): - """Search FILES using default type (which) selector""" - - c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') - r = NOIRLab().query_region(c, radius='0.1') - actual = set(list(r['md5sum'])) - expected = exp.query_region_1 - assert expected.issubset(actual) - - def test_query_region_1(self): - """Search FILES. - Ensure query gets at least the set of files we expect. - Its ok if more files have been added to the remote Archive.""" - - c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') - r = NOIRLabClass(which='file').query_region(c, radius='0.1') - actual = set(list(r['md5sum'])) - expected = exp.query_region_1 - assert expected.issubset(actual) - - def test_query_region_2(self): - """Search HDUs. - Ensure query gets at least the set of files we expect. - Its ok if more files have been added to the remote Archive.""" - - c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') - r = NOIRLabClass(which='hdu').query_region(c, radius='0.07') - actual = set(list(r['md5sum'])) - expected = exp.query_region_2 - assert expected.issubset(actual) - - # ############################################################### - # ### (7) Advanced Search; /api/adv_search/ - # ### - # - # (2) aux_{file,hdu}_fields// - # (2) core_{file,hdu}_fields/ - # [(2) {f,h}adoc JUST LINK to these] - # (2) {f,h}asearch - # cat_list - - # ## - # ## File (default type) - # ## - - def test_aux_file_fields(self): - """List the available AUX FILE fields.""" - r = NOIRLab().aux_fields('decam', 'instcal') - actual = r - # Returned results may increase over time. - print(f'DBG: test_aux_file_fields={actual}') - expected = exp.aux_file_fields - assert actual == expected - - def test_core_file_fields(self): - """List the available CORE FILE fields.""" - r = NOIRLab().core_fields() - actual = r - print(f'DBG: test_core_file_fields={actual}') - expected = exp.core_file_fields - assert actual == expected - - def test_query_file_metadata(self): - """Search FILE metadata.""" - qspec = { - "outfields": [ - "md5sum", - "archive_filename", - "original_filename", - "instrument", - "proc_type" - ], - "search": [ - ['original_filename', 'c4d_', 'contains'] - ] - } - - r = NOIRLab().query_metadata(qspec, limit=3) - actual = r - # print(f'DBG: test_query_file_metadata={actual.pformat_all()}') - expected = exp.query_file_metadata - assert actual.pformat_all() == expected - - # ## - # ## HDU - # ## - - def test_aux_hdu_fields(self): - """List the available AUX HDU fields.""" - r = NOIRLabClass(which='hdu').aux_fields('decam', 'instcal') - actual = r - # Returned results may increase over time. - # print(f'DBG: test_aux_hdu_fields={actual}') - expected = exp.aux_hdu_fields - assert actual == expected - - def test_core_hdu_fields(self): - """List the available CORE HDU fields.""" - r = NOIRLabClass(which='hdu').core_fields() - actual = r - # print(f'DBG: test_core_file_fields={actual}') - expected = exp.core_hdu_fields - assert actual == expected - - def test_query_hdu_metadata(self): - """Search HDU metadata.""" - qspec = { - "outfields": [ - "fitsfile__archive_filename", - "fitsfile__caldat", - "fitsfile__instrument", - "fitsfile__proc_type", - "AIRMASS" # AUX field. Slows search - ], - "search": [ - ["fitsfile__caldat", "2017-08-14", "2017-08-16"], - ["fitsfile__instrument", "decam"], - ["fitsfile__proc_type", "raw"] - ] - } - - r = NOIRLabClass(which='hdu').query_metadata(qspec, limit=3) - actual = r - # print(f'DBG: test_query_hdu_metadata={actual.pformat_all()}') - expected = exp.query_hdu_metadata - assert actual.pformat_all() == expected - - # ## - # ## Agnostic - # ## - - def test_categoricals(self): - """List categories.""" - r = NOIRLab().categoricals() - actual = r - # Returned results may increase over time. - print(f'DBG: test_categoricals={actual}') - expected = exp.categoricals - assert actual == expected - - # ############################################################## - # ### (3) Other - # get_token - # retrieve/ - # version - - def test_retrieve(self): - hdul = NOIRLab().retrieve('f92541fdc566dfebac9e7d75e12b5601') - actual = list(hdul[0].header.keys()) - expected = exp.retrieve - assert actual == expected - - def test_version(self): - r = NOIRLab().version() - assert r >= exp.version - - def test_get_token(self): - actual = NOIRLab().get_token('nobody@university.edu', '123456789') - # expected = {'detail': - # 'No active account found with the given credentials'} - assert actual == exp.get_token +def test_get_token(): + actual = NOIRLab().get_token('nobody@university.edu', '123456') + assert actual.split('.')[0] == exp.get_token From f09035784da1d7663222d99a56c48b545961c3b4 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Fri, 27 Jun 2025 11:30:25 -0700 Subject: [PATCH 10/26] fix test patching --- astroquery/noirlab/tests/test_noirlab.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astroquery/noirlab/tests/test_noirlab.py b/astroquery/noirlab/tests/test_noirlab.py index 41cf5e8cbe..336e31995d 100644 --- a/astroquery/noirlab/tests/test_noirlab.py +++ b/astroquery/noirlab/tests/test_noirlab.py @@ -32,14 +32,14 @@ def patch_request(monkeypatch): def test_service_metadata(patch_request): """Test compliance with 6.1 of SIA spec v1.0. """ - actual = NOIRLab().service_metadata() + actual = NOIRLab.service_metadata() assert actual == exp.service_metadata[0] def test_categoricals(patch_request): """List categories. """ - actual = NOIRLab().categoricals() + actual = NOIRLab.categoricals() assert actual == exp.categoricals From f8fc5cf1ad9eb417e71640155f56f2541c14c250 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Fri, 27 Jun 2025 14:09:21 -0700 Subject: [PATCH 11/26] adding more tests --- astroquery/noirlab/core.py | 20 +- astroquery/noirlab/tests/expected.py | 343 +----------------- astroquery/noirlab/tests/test_noirlab.py | 24 ++ .../noirlab/tests/test_noirlab_remote.py | 112 +++--- 4 files changed, 110 insertions(+), 389 deletions(-) diff --git a/astroquery/noirlab/core.py b/astroquery/noirlab/core.py index f9d8e4c38c..74bd8c4fd2 100644 --- a/astroquery/noirlab/core.py +++ b/astroquery/noirlab/core.py @@ -139,8 +139,20 @@ def query_region_async(self, coordinate, radius=0.1, cache=True): return response def core_fields(self, cache=True): - """List the available CORE fields. CORE fields are faster to search - than AUX fields..""" + """List the available CORE fields. + + CORE fields are faster to search than AUX fields. + + Parameters + ---------- + cache : :class:`bool`, optional + If ``True`` cache the result locally. + + Returns + ------- + :class:`list` + A list of field descriptions, each a :class:`dict`. + """ response = self._request('GET', self._adsc_url, timeout=self.TIMEOUT, cache=cache) @@ -174,7 +186,7 @@ def categoricals(self, cache=True): @class_or_instance def query_metadata(self, qspec, limit=1000, cache=True): - self._validate_version() + # self._validate_version() url = f'{self._adss_url}/?limit={limit}' if qspec is None: @@ -198,7 +210,7 @@ def retrieve(self, fileid): Returns ------- :class:`~astropy.io.fits.HDUlist` - The open FITS file. This object should be ``.close()``d when done. + The open FITS file. Call ``.close()`` on this object when done. """ url = f'{self.NAT_URL}/api/retrieve/{fileid}/' hdulist = fits.open(url) diff --git a/astroquery/noirlab/tests/expected.py b/astroquery/noirlab/tests/expected.py index f1f7f53377..845baf0523 100644 --- a/astroquery/noirlab/tests/expected.py +++ b/astroquery/noirlab/tests/expected.py @@ -5,6 +5,11 @@ may be used to create mock responses. """ +service_metadata = [{'ParameterName': 'INPUT:DATE', + 'DataType': 'char', + 'DefaultValue': 'NA', + 'Doc': 'NA'}] + query_region_1 = {'4066c45407961eab48eb16bcb9c5b4b7', '554b2bcb3b2a4353c515d20d9fc29e4e', '711ce162a7488a72b1629d994264eae7', @@ -19,330 +24,22 @@ 'f990925c8d99a66f642e4463f1dde7f2', '09535b88fc700227bb47979a670a7d85'} -service_metadata = [{'ParameterName': 'INPUT:DATE', - 'DataType': 'char', - 'DefaultValue': 'NA', - 'Doc': 'NA'}] - -aux_file_fields = {'AIRMASS': 'str', - 'AOS': 'str', - 'ASTIG1': 'str', - 'ASTIG2': 'str', - 'ATTNUM': 'str', - 'AVSIG': 'str', - 'AVSKY': 'str', - 'AZ': 'str', - 'BAND': 'str', - 'BCAM': 'str', - 'BCAMAX': 'str', - 'BCAMAY': 'str', - 'BCAMAZ': 'str', - 'BCAMDX': 'str', - 'BCAMDY': 'str', - 'BFCFIL': 'str', - 'BUNIT': 'str', - 'CAMSHUT': 'str', - 'CAMSYM': 'str', - 'CCDBIN1': 'str', - 'CCDBIN2': 'str', - 'CCDSEC': 'str', - 'CCDSECA': 'str', - 'CCDSECB': 'str', - 'CENTDEC': 'str', - 'CENTRA': 'str', - 'CONSTVER': 'str', - 'CORN1DEC': 'str', - 'CORN1RA': 'str', - 'CORN2DEC': 'str', - 'CORN2RA': 'str', - 'CORN3DEC': 'str', - 'CORN3RA': 'str', - 'CORN4DEC': 'str', - 'CORN4RA': 'str', - 'CORNDEC1': 'str', - 'CORNDEC2': 'str', - 'CORNDEC3': 'str', - 'CORNDEC4': 'str', - 'CORNRA1': 'str', - 'CORNRA2': 'str', - 'CORNRA3': 'str', - 'CORNRA4': 'str', - 'CROSSRA0': 'str', - 'DARKTIME': 'str', - 'DATASEC': 'str', - 'DATASECA': 'str', - 'DATASECB': 'str', - 'DATE': 'str', - 'DATE-OBS': 'str', - 'DEC': 'str', - 'DES_EXT': 'str', - 'DESNCRAY': 'str', - 'DESNSTRK': 'str', - 'DESREL': 'str', - 'DETSIZE': 'str', - 'DHEFIRM': 'str', - 'DHEINF': 'str', - 'DIMM2SEE': 'str', - 'DIMMSEE': 'str', - 'DODX': 'str', - 'DODY': 'str', - 'DODZ': 'str', - 'DOMEAZ': 'str', - 'DOMEFLOR': 'str', - 'DOMEHIGH': 'str', - 'DOMELOW': 'str', - 'DONUTFN1': 'str', - 'DONUTFN2': 'str', - 'DONUTFN3': 'str', - 'DONUTFN4': 'str', - 'DONUTFS1': 'str', - 'DONUTFS2': 'str', - 'DONUTFS3': 'str', - 'DONUTFS4': 'str', - 'DOXT': 'str', - 'DOYT': 'str', - 'DTACCOUN': 'str', - 'DTACQNAM': 'str', - 'DTACQUIS': 'str', - 'DTCALDAT': 'str', - 'DTCOPYRI': 'str', - 'DTINSTRU': 'str', - 'DTNSANAM': 'str', - 'DTOBSERV': 'str', - 'DTPI': 'str', - 'DTPIAFFL': 'str', - 'DTPROPID': 'str', - 'DTQUEUE': 'str', - 'DT_RTNAM': 'str', - 'DTSITE': 'str', - 'DTSTATUS': 'str', - 'DTTELESC': 'str', - 'DTTITLE': 'str', - 'DTUTC': 'str', - 'ELLIPTIC': 'str', - 'EQUINOX': 'str', - 'ERRORS': 'str', - 'EUPSPROD': 'str', - 'EUPSVER': 'str', - 'EXCLUDED': 'str', - 'EXPDUR': 'str', - 'EXPNUM': 'str', - 'EXPREQ': 'str', - 'EXPTIME': 'str', - 'EXTVER': 'str', - 'FADX': 'str', - 'FADY': 'str', - 'FADZ': 'str', - 'FAXT': 'str', - 'FAYT': 'str', - 'FILENAME': 'str', - 'FILTER': 'str', - 'FILTPOS': 'str', - 'FRGSCALE': 'str', - 'FWHM': 'str', - 'FZALGOR': 'str', - 'FZDTHRSD': 'str', - 'FZQMETHD': 'str', - 'FZQVALUE': 'str', - 'G-CCDNUM': 'str', - 'G-FEEDBK': 'str', - 'G-FLXVAR': 'str', - 'G-LATENC': 'str', - 'G-MAXX': 'str', - 'G-MAXY': 'str', - 'G-MEANX': 'str', - 'G-MEANX2': 'str', - 'G-MEANXY': 'str', - 'G-MEANY': 'str', - 'G-MEANY2': 'str', - 'G-MODE': 'str', - 'G-SEEING': 'str', - 'GSKYHOT': 'str', - 'GSKYPHOT': 'str', - 'GSKYVAR': 'str', - 'G-TRANSP': 'str', - 'GUIDER': 'str', - 'HA': 'str', - 'HDRVER': 'str', - 'HEX': 'str', - 'HUMIDITY': 'str', - 'INSTANCE': 'str', - 'INSTRUME': 'str', - 'IRAF-TLM': 'str', - 'LINCFIL': 'str', - 'LSKYHOT': 'str', - 'LSKYPHOT': 'str', - 'LSKYPOW': 'str', - 'LSKYVAR': 'str', - 'LST': 'str', - 'LUTVER': 'str', - 'LWTRTEMP': 'str', - 'MAGZERO': 'str', - 'MAGZP': 'str', - 'MAGZPT': 'str', - 'MAGZUNC': 'str', - 'MAIRTEMP': 'str', - 'MANIFEST': 'str', - 'MASS2': 'str', - 'MJD-END': 'str', - 'MJD-OBS': 'str', - 'MOONANGL': 'str', - 'MSURTEMP': 'str', - 'MULTIEXP': 'str', - 'MULTIFOC': 'str', - 'MULTIID': 'str', - 'MULTIROW': 'str', - 'MULTITOT': 'str', - 'NBLEED': 'str', - 'NDONUTS': 'str', - 'NEXTEND': 'str', - 'NITE': 'str', - 'NOTE': 'str', - 'NPHTMTCH': 'str', - 'NSATPIX': 'str', - 'NUM': 'str', - 'OBJECT': 'str', - 'OBS-ELEV': 'str', - 'OBSERVAT': 'str', - 'OBSERVER': 'str', - 'OBSID': 'str', - 'OBS-LAT': 'str', - 'OBS-LONG': 'str', - 'OBSTYPE': 'str', - 'ODATEOBS': 'str', - 'OPENSHUT': 'str', - 'ORIGIN': 'str', - 'OUTTEMP': 'str', - 'PHOTFLAG': 'str', - 'PHOTREF': 'str', - 'PIPELINE': 'str', - 'PIXSCAL1': 'str', - 'PIXSCAL2': 'str', - 'PLARVER': 'str', - 'PLDNAME': 'str', - 'PLDSID': 'str', - 'PLFNAME': 'str', - 'PLPROCID': 'str', - 'PLQNAME': 'str', - 'PLQUEUE': 'str', - 'PLVER': 'str', - 'PME-TEMP': 'str', - 'PMN-TEMP': 'str', - 'PMOSTEMP': 'str', - 'PMS-TEMP': 'str', - 'PMW-TEMP': 'str', - 'PRESSURE': 'str', - 'PROCTYPE': 'str', - 'PRODTYPE': 'str', - 'PROGRAM': 'str', - 'PROPID': 'str', - 'PROPOSER': 'str', - 'PUPILAMP': 'str', - 'PUPILMAX': 'str', - 'PUPILSKY': 'str', - 'PUPMAX': 'str', - 'PV1_6': 'str', - 'PV2_4': 'str', - 'RA': 'str', - 'RA_CENT': 'str', - 'RACMAX': 'str', - 'RACMIN': 'str', - 'RADECSYS': 'str', - 'RADESYS': 'str', - 'RADIUS': 'str', - 'RADSTD': 'str', - 'RECNO': 'str', - 'REQNUM': 'str', - 'RMCOUNT': 'str', - 'SB_ACCOU': 'str', - 'SB_DIR1': 'str', - 'SB_DIR2': 'str', - 'SB_DIR3': 'str', - 'SB_HOST': 'str', - 'SB_ID': 'str', - 'SB_LOCAL': 'str', - 'SB_NAME': 'str', - 'SB_RECNO': 'str', - 'SB_RTNAM': 'str', - 'SB_SITE': 'str', - 'SCAMPCHI': 'str', - 'SCAMPFLG': 'str', - 'SCAMPNUM': 'str', - 'SCAMPREF': 'str', - 'SEQID': 'str', - 'SEQNUM': 'str', - 'SEQTOT': 'str', - 'SEQTYPE': 'str', - 'SISPIVER': 'str', - 'SKYBRITE': 'str', - 'SKYORDER': 'str', - 'SKYPC00': 'str', - 'SKYPC01': 'str', - 'SKYPC02': 'str', - 'SKYPC03': 'str', - 'SKYSIGMA': 'str', - 'SKYSTAT': 'str', - 'SKYSUB': 'str', - 'SKYSUB01': 'str', - 'SKYSUB10': 'str', - 'SKYSUBP': 'str', - 'SKYUPDAT': 'str', - 'SLOT01': 'str', - 'SLOT03': 'str', - 'SURVEYID': 'str', - 'TELDEC': 'str', - 'TELEQUIN': 'str', - 'TELESCOP': 'str', - 'TELFOCUS': 'str', - 'TELRA': 'str', - 'TELSTAT': 'str', - 'TILING': 'str', - 'TIME-OBS': 'str', - 'TIMESYS': 'str', - 'TRACKING': 'str', - 'UNITNAME': 'str', - 'UPTRTEMP': 'str', - 'UTE-TEMP': 'str', - 'UTN-TEMP': 'str', - 'UTS-TEMP': 'str', - 'UTW-TEMP': 'str', - 'VALIDA': 'str', - 'VALIDB': 'str', - 'VSUB': 'str', - 'WCSCAL': 'str', - 'WINDDIR': 'str', - 'WINDSPD': 'str', - 'XTALKFIL': 'str', - 'ZD': 'str', - 'ZPDELDEC': 'str', - 'ZPDELRA': 'str'} +core_file_fields = [{'Field': 'archive_filename', 'Type': 'str', 'Desc': 'Filename assigned by the Archive'}, + {'Field': 'caldat', 'Type': 'datetime64', 'Desc': 'The local calendar date of the telescope, at the start of PM observing.'}, + {'Field': 'dateobs_center', 'Type': 'datetime64', 'Desc': 'DATE-OBS midpoint of range'}, + {'Field': 'dateobs_max', 'Type': 'datetime64', 'Desc': 'DATE-OBS min,max'}, + {'Field': 'dateobs_min', 'Type': 'datetime64', 'Desc': 'DATE-OBS min,max'},] -core_file_fields = [{'Field': 'archive_filename', 'Type': 'str'}, - {'Field': 'caldat', 'Type': 'datetime64'}, - {'Field': 'date_obs_max', 'Type': 'datetime64'}, - {'Field': 'date_obs_min', 'Type': 'datetime64'}, - {'Field': 'dec_max', 'Type': 'np.float64'}, - {'Field': 'dec_min', 'Type': 'np.float64'}, - {'Field': 'depth', 'Type': 'np.float64'}, - {'Field': 'exposure', 'Type': 'np.float64'}, - {'Field': 'filesize', 'Type': 'np.int64'}, - {'Field': 'ifilter', 'Type': 'category'}, - {'Field': 'instrument', 'Type': 'category'}, - {'Field': 'md5sum', 'Type': 'str'}, - {'Field': 'obs_mode', 'Type': 'category'}, - {'Field': 'obs_type', 'Type': 'category'}, - {'Field': 'original_filename', 'Type': 'str'}, - {'Field': 'proc_type', 'Type': 'category'}, - {'Field': 'prod_type', 'Type': 'category'}, - {'Field': 'proposal', 'Type': 'category'}, - {'Field': 'ra_max', 'Type': 'np.float64'}, - {'Field': 'ra_min', 'Type': 'np.float64'}, - {'Field': 'release_date', 'Type': 'datetime64'}, - {'Field': 'seeing', 'Type': 'np.float64'}, - {'Field': 'site', 'Type': 'category'}, - {'Field': 'survey', 'Type': 'category'}, - {'Field': 'telescope', 'Type': 'category'}, - {'Field': 'updated', 'Type': 'datetime64'}] +aux_file_fields = [{'Field': 'AIRMASS', 'Type': 'str', 'Desc': 'airmass at approx. start of exposure'}, + {'Field': 'AOS', 'Type': 'str', 'Desc': 'AOS data available if true'}, + {'Field': 'ASTIG1', 'Type': 'str', 'Desc': '4MAPS correction 1'}, + {'Field': 'ASTIG2', 'Type': 'str', 'Desc': '4MAPS correction 2'}, + {'Field': 'ASTRMREF', 'Type': 'str', 'Desc': 'Astrometric ref. catalog'}, + {'Field': 'ATTNUM', 'Type': 'str', 'Desc': ''}, + {'Field': 'AVSIG', 'Type': 'str', 'Desc': ''}, + {'Field': 'AVSKY', 'Type': 'str', 'Desc': ''}, + {'Field': 'AZ', 'Type': 'str', 'Desc': ''}, + {'Field': 'BAND', 'Type': 'str', 'Desc': ''}] query_file_metadata = [' archive_filename instrument md5sum original_filename proc_type url ', '----------------------------------------------------------------------------- ---------- -------------------------------- ------------------------------------------------------------------------------------- --------- ----------------------------------------------------------------------------', diff --git a/astroquery/noirlab/tests/test_noirlab.py b/astroquery/noirlab/tests/test_noirlab.py index 336e31995d..ae8bd7a8ec 100644 --- a/astroquery/noirlab/tests/test_noirlab.py +++ b/astroquery/noirlab/tests/test_noirlab.py @@ -14,6 +14,10 @@ def mock_content(method, url, **kwargs): if 'FORMAT=METADATA' in url: content = json.dumps(exp.service_metadata) + elif '/core_file_fields' in url: + content = json.dumps(exp.core_file_fields) + elif '/aux_file_fields' in url: + content = json.dumps(exp.aux_file_fields) elif '/cat_lists/' in url: content = json.dumps(exp.categoricals) elif '/version/' in url: @@ -36,6 +40,20 @@ def test_service_metadata(patch_request): assert actual == exp.service_metadata[0] +def test_core_file_fields(patch_request): + """List the available CORE FILE fields. + """ + actual = NOIRLab.core_fields() + assert actual == exp.core_file_fields + + +def test_aux_file_fields(patch_request): + """List the available AUX FILE fields. + """ + actual = NOIRLab.aux_fields('decam', 'instcal') + assert actual == exp.aux_file_fields + + def test_categoricals(patch_request): """List categories. """ @@ -44,15 +62,21 @@ def test_categoricals(patch_request): def test_version(patch_request): + """Test the API version. + """ actual = NOIRLab.version() assert actual >= float(exp.version) def test_api_version(patch_request): + """Test the API version as a property. + """ actual = NOIRLab.api_version assert actual >= float(exp.version) def test_get_token(patch_request): + """Test token retrieval. + """ actual = NOIRLab.get_token('nobody@university.edu', '123456') assert actual == exp.get_token diff --git a/astroquery/noirlab/tests/test_noirlab_remote.py b/astroquery/noirlab/tests/test_noirlab_remote.py index 75fccf92fd..db401dd443 100644 --- a/astroquery/noirlab/tests/test_noirlab_remote.py +++ b/astroquery/noirlab/tests/test_noirlab_remote.py @@ -74,39 +74,41 @@ def test_query_region_2(): # # File (default type) # -@pytest.mark.skip(reason='old API') +@pytest.mark.remote_data +def test_core_file_fields(): + """List the available CORE FILE fields. + """ + actual = NOIRLab().core_fields() + assert actual[:5] == exp.core_file_fields + + @pytest.mark.remote_data def test_aux_file_fields(): - """List the available AUX FILE fields.""" + """List the available AUX FILE fields. + """ actual = NOIRLab().aux_fields('decam', 'instcal') - assert actual == exp.aux_file_fields + assert actual[:10] == exp.aux_file_fields -@pytest.mark.skip(reason='old API') @pytest.mark.remote_data -def test_core_file_fields(): - """List the available CORE FILE fields.""" - actual = NOIRLab().core_fields() - assert actual == exp.core_file_fields +def test_categoricals(): + """List categories. + """ + actual = NOIRLab().categoricals() + assert actual == exp.categoricals @pytest.mark.skip(reason='old API') @pytest.mark.remote_data def test_query_file_metadata(): - """Search FILE metadata.""" - qspec = { - "outfields": [ - "md5sum", - "archive_filename", - "original_filename", - "instrument", - "proc_type" - ], - "search": [ - ['original_filename', 'c4d_', 'contains'] - ] - } - + """Search FILE metadata. + """ + qspec = {"outfields": ["md5sum", + "archive_filename", + "original_filename", + "instrument", + "proc_type"], + "search": [['original_filename', 'c4d_', 'contains']]} actual = NOIRLab().query_metadata(qspec, limit=3) assert actual.pformat_all() == exp.query_file_metadata @@ -114,61 +116,41 @@ def test_query_file_metadata(): # # HDU search # -@pytest.mark.skip(reason='old API') -@pytest.mark.remote_data -def test_aux_hdu_fields(): - """List the available AUX HDU fields.""" - actual = NOIRLabClass(hdu=True).aux_fields('decam', 'instcal') - assert actual == exp.aux_hdu_fields - - @pytest.mark.skip(reason='old API') @pytest.mark.remote_data def test_core_hdu_fields(): - """List the available CORE HDU fields.""" + """List the available CORE HDU fields. + """ actual = NOIRLabClass(hdu=True).core_fields() assert actual == exp.core_hdu_fields @pytest.mark.skip(reason='old API') @pytest.mark.remote_data -def test_query_hdu_metadata(): - """Search HDU metadata.""" - qspec = { - "outfields": [ - "fitsfile__archive_filename", - "fitsfile__caldat", - "fitsfile__instrument", - "fitsfile__proc_type", - "AIRMASS" # AUX field. Slows search - ], - "search": [ - ["fitsfile__caldat", "2017-08-14", "2017-08-16"], - ["fitsfile__instrument", "decam"], - ["fitsfile__proc_type", "raw"] - ] - } - - actual = NOIRLabClass(hdu=True).query_metadata(qspec, limit=3) - assert actual.pformat_all() == exp.query_hdu_metadata +def test_aux_hdu_fields(): + """List the available AUX HDU fields. + """ + actual = NOIRLabClass(hdu=True).aux_fields('decam', 'instcal') + assert actual == exp.aux_hdu_fields -# -# Agnostic -# +@pytest.mark.skip(reason='old API') @pytest.mark.remote_data -def test_categoricals(): - """List categories. +def test_query_hdu_metadata(): + """Search HDU metadata. """ - actual = NOIRLab().categoricals() - assert actual == exp.categoricals + qspec = {"outfields": ["fitsfile__archive_filename", + "fitsfile__caldat", + "fitsfile__instrument", + "fitsfile__proc_type", + "AIRMASS"], # AUX field. Slows search + "search": [["fitsfile__caldat", "2017-08-14", "2017-08-16"], + ["fitsfile__instrument", "decam"], + ["fitsfile__proc_type", "raw"]]} + actual = NOIRLabClass(hdu=True).query_metadata(qspec, limit=3) + assert actual.pformat_all() == exp.query_hdu_metadata -# Other tests: -# get_token -# retrieve/ -# version -# @pytest.mark.remote_data def test_retrieve(): hdulist = NOIRLab().retrieve('f92541fdc566dfebac9e7d75e12b5601') @@ -180,17 +162,23 @@ def test_retrieve(): @pytest.mark.remote_data def test_version(): + """Test the API version. + """ actual = NOIRLab().version() assert actual >= float(exp.version) @pytest.mark.remote_data def test_api_version(): + """Test the API version as a property. + """ actual = NOIRLab().api_version assert actual >= float(exp.version) @pytest.mark.remote_data def test_get_token(): + """Test token retrieval. + """ actual = NOIRLab().get_token('nobody@university.edu', '123456') assert actual.split('.')[0] == exp.get_token From 6a70c4600be0226bf7c9be82465e481bc2670dd4 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Fri, 27 Jun 2025 14:15:15 -0700 Subject: [PATCH 12/26] fix docstring --- astroquery/noirlab/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astroquery/noirlab/core.py b/astroquery/noirlab/core.py index 74bd8c4fd2..e27df79538 100644 --- a/astroquery/noirlab/core.py +++ b/astroquery/noirlab/core.py @@ -200,7 +200,7 @@ def query_metadata(self, qspec, limit=1000, cache=True): return astropy.table.Table(rows=response.json()) def retrieve(self, fileid): - """Simply fetch a file by `fileid`. + """Simply fetch a file by MD5 ID. Parameters ---------- @@ -209,7 +209,7 @@ def retrieve(self, fileid): Returns ------- - :class:`~astropy.io.fits.HDUlist` + :class:`~astropy.io.fits.HDUList` The open FITS file. Call ``.close()`` on this object when done. """ url = f'{self.NAT_URL}/api/retrieve/{fileid}/' From 6eaacb6456456d13a4d863ce7a90d39fba3e4359 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Tue, 8 Jul 2025 13:51:57 -0400 Subject: [PATCH 13/26] adding more tests --- astroquery/noirlab/core.py | 6 +- astroquery/noirlab/tests/expected.py | 147 +++--------------- astroquery/noirlab/tests/test_noirlab.py | 42 ++++- .../noirlab/tests/test_noirlab_remote.py | 20 +-- 4 files changed, 66 insertions(+), 149 deletions(-) diff --git a/astroquery/noirlab/core.py b/astroquery/noirlab/core.py index e27df79538..b60374abe8 100644 --- a/astroquery/noirlab/core.py +++ b/astroquery/noirlab/core.py @@ -35,12 +35,12 @@ def __init__(self, hdu=False): if hdu: self.siaurl = f'{self.NAT_URL}/api/sia/vohdu' - self._adss_url = f'{self._adsurl}/hasearch' + self._adss_url = f'{self._adsurl}/find?rectype=hdu' self._adsc_url = f'{self._adsurl}/core_hdu_fields' self._adsa_url = f'{self._adsurl}/aux_hdu_fields' else: self.siaurl = f'{self.NAT_URL}/api/sia/voimg' - self._adss_url = f'{self._adsurl}/fasearch' + self._adss_url = f'{self._adsurl}/find?rectype=file' self._adsc_url = f'{self._adsurl}/core_file_fields' self._adsa_url = f'{self._adsurl}/aux_file_fields' @@ -187,7 +187,7 @@ def categoricals(self, cache=True): @class_or_instance def query_metadata(self, qspec, limit=1000, cache=True): # self._validate_version() - url = f'{self._adss_url}/?limit={limit}' + url = f'{self._adss_url}&limit={limit}' if qspec is None: jdata = {"outfields": ["md5sum", ], "search": []} diff --git a/astroquery/noirlab/tests/expected.py b/astroquery/noirlab/tests/expected.py index 845baf0523..1c7e1942ac 100644 --- a/astroquery/noirlab/tests/expected.py +++ b/astroquery/noirlab/tests/expected.py @@ -47,133 +47,28 @@ '/net/archive/pipe/20160928/ct4m/2012B-0001/c4d_160929_090838_ooi_z_v2.fits.fz decam 0001e5a5bcf039ebf0c53a6da32e8888 /net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_160929_090838_ooi_z_v2.fits.fz instcal https://astroarchive.noao.edu/api/retrieve/0001e5a5bcf039ebf0c53a6da32e8888/', '/net/archive/pipe/20160909/ct4m/2012B-0001/c4d_160910_062930_opd_z_v2.fits.fz decam 0003a1c853de73fc1796d2d7d77ca9cd /net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_160910_062930_opd_z_v2.fits.fz resampled https://astroarchive.noao.edu/api/retrieve/0003a1c853de73fc1796d2d7d77ca9cd/'] -aux_hdu_fields = {'AMPBAL': 'str', 'AMPSECA': 'str', 'AMPSECB': 'str', - 'ARAWGAIN': 'str', 'AVSIG': 'str', 'AVSKY': 'str', - 'BIASFIL': 'str', 'BLDINTRP': 'str', 'BPM': 'str', - 'BPMFIL': 'str', 'CATALOG': 'str', 'CCDNUM': 'str', - 'CCDSECA': 'str', 'CCDSECB': 'str', 'CD1_1': 'str', - 'CD1_2': 'str', 'CD2_1': 'str', 'CD2_2': 'str', - 'CENDEC1': 'str', 'CENRA1': 'str', 'COR1DEC1': 'str', - 'COR1RA1': 'str', 'COR2DEC1': 'str', 'COR2RA1': 'str', - 'COR3DEC1': 'str', 'COR3RA1': 'str', 'COR4DEC1': 'str', - 'COR4RA1': 'str', 'CROSSRA0': 'str', 'CRPIX1': 'str', - 'CRPIX2': 'str', 'CRVAL1': 'str', 'CRVAL2': 'str', - 'CTYPE1': 'str', 'CTYPE2': 'str', 'CUNIT1': 'str', - 'CUNIT2': 'str', 'D0034494': 'str', 'D0034496': 'str', - 'D0034497': 'str', 'DATASEC': 'str', 'DATASECA': 'str', - 'DATASECB': 'str', 'DATE': 'str', 'DEC': 'str', 'DEC1': 'str', - 'DEC13A_2': 'str', 'DEC13B_2': 'str', 'DEC14A_2': 'str', - 'DEC15A_2': 'str', 'DEC15B_2': 'str', 'DEC16A_2': 'str', - 'DEC16B_2': 'str', 'DEC17A_2': 'str', 'DEC17B_2': 'str', - 'DEC18A_2': 'str', 'DEC18B_2': 'str', 'DEC18B_R': 'str', - 'DEC18B_S': 'str', 'DEC19A_2': 'str', 'DEC19A_A': 'str', - 'DEC19A_D': 'str', 'DEC19A_J': 'str', 'DEC19A_K': 'str', - 'DEC19A_L': 'str', 'DEC19A_M': 'str', 'DEC19A_P': 'str', - 'DEC19A_S': 'str', 'DEC19A_T': 'str', 'DEC19A_W': 'str', - 'DEC19B_2': 'str', 'DEC19B_A': 'str', 'DEC19B_C': 'str', - 'DEC19B_D': 'str', 'DEC19B_F': 'str', 'DEC19B_H': 'str', - 'DEC19B_I': 'str', 'DEC19B_J': 'str', 'DEC19B_K': 'str', - 'DEC19B_L': 'str', 'DEC19B_M': 'str', 'DEC19B_P': 'str', - 'DEC19B_R': 'str', 'DEC19B_S': 'str', 'DEC19B_T': 'str', - 'DEC19B_W': 'str', 'DEC20A_A': 'str', 'DEC20A_C': 'str', - 'DEC20A_F': 'str', 'DEC20A_J': 'str', 'DEC20A_K': 'str', - 'DEC20A_L': 'str', 'DEC20A_M': 'str', 'DEC20A_S': 'str', - 'DEC20A_T': 'str', 'DEC20B_T': 'str', 'DECALS_2': 'str', - 'DECALS_D': 'str', 'DECC1': 'str', 'DECC2': 'str', 'DECC3': 'str', - 'DECC4': 'str', 'DEC_CENT': 'str', 'DECCMAX': 'str', - 'DECCMIN': 'str', 'DES14B_2': 'str', 'DES15B_2': 'str', - 'DES16B_2': 'str', 'DES17A_2': 'str', 'DES17B_2': 'str', - 'DES18A_2': 'str', 'DES18B_2': 'str', 'DESBFC': 'str', - 'DESBIAS': 'str', 'DESBLEED': 'str', 'DESBPM': 'str', - 'DESCRMSK': 'str', 'DESDCXTK': 'str', 'DESDMCR': 'str', - 'DES_EXT': 'str', 'DESFIXC': 'str', 'DESFLAT': 'str', - 'DESFNAME': 'str', 'DESFRING': 'str', 'DESGAINC': 'str', - 'DESILLUM': 'str', 'DESIMMSK': 'str', 'DESLINC': 'str', - 'DESNCRAY': 'str', 'DESNSTRK': 'str', 'DESOSCN': 'str', - 'DESPHOTF': 'str', 'DESPUPC': 'str', 'DESSAT': 'str', - 'DESSKYSB': 'str', 'DESSTAR': 'str', 'DETECTOR': 'str', - 'DETPOS': 'str', 'DETSEC': 'str', 'DETSECA': 'str', - 'DETSECB': 'str', 'ELLIPTIC': 'str', 'ENG17B_2': 'str', - 'ENG18A_2': 'str', 'ENG18B_2': 'str', 'EXPTIME': 'str', - 'EXTNAME': 'str', 'EXTVER': 'str', 'FILTER': 'str', - 'FIXCFIL': 'str', 'FIXPIX': 'str', 'FIXPIX02': 'str', - 'FLATFIL': 'str', 'FLATMEDA': 'str', 'FLATMEDB': 'str', - 'FPA': 'str', 'FRGSCALE': 'str', 'FRINGE': 'str', - 'FRINGFIL': 'str', 'FWHM': 'str', 'FWHMP1': 'str', - 'FZALGOR': 'str', 'FZDTHRSD': 'str', 'FZQMETHD': 'str', - 'FZQVALUE': 'str', 'GAINA': 'str', 'GAINB': 'str', - 'GCOUNT': 'str', 'ILLCOR': 'str', 'ILLMASK': 'str', - 'ILLUMCOR': 'str', 'ILLUMFIL': 'str', 'INHERIT': 'str', - 'IRAF-TLM': 'str', 'LAGER_20': 'str', 'LTM1_1': 'str', - 'LTM1_2': 'str', 'LTM2_1': 'str', 'LTM2_2': 'str', - 'LTV1': 'str', 'LTV2': 'str', 'MAGZERO1': 'str', - 'MAGZUNC1': 'str', 'NAXIS1': 'str', 'NAXIS2': 'str', - 'NBLEED': 'str', 'NSATPIX': 'str', 'OBJECT': 'str', - 'OBJMASK': 'str', 'ORIGIN': 'str', 'PCOUNT': 'str', - 'PHOTFLAG': 'str', 'PUPFIL': 'str', 'PUPILAMP': 'str', - 'PUPILSKY': 'str', 'PUPMAX': 'str', 'PV1_0': 'str', - 'PV1_1': 'str', 'PV1_10': 'str', 'PV1_2': 'str', - 'PV1_3': 'str', 'PV1_4': 'str', 'PV1_5': 'str', - 'PV1_6': 'str', 'PV1_7': 'str', 'PV1_8': 'str', 'PV1_9': 'str', - 'PV2_0': 'str', 'PV2_1': 'str', 'PV2_10': 'str', 'PV2_2': 'str', - 'PV2_3': 'str', 'PV2_4': 'str', 'PV2_5': 'str', 'PV2_6': 'str', - 'PV2_7': 'str', 'PV2_8': 'str', 'PV2_9': 'str', 'RA': 'str', - 'RA1': 'str', 'RAC1': 'str', 'RAC2': 'str', 'RAC3': 'str', - 'RAC4': 'str', 'RA_CENT': 'str', 'RACMAX': 'str', - 'RACMIN': 'str', 'RDNOISEA': 'str', 'RDNOISEB': 'str', - 'REQ13B_2': 'str', 'REQ13B_H': 'str', 'REQ14B_2': 'str', - 'REQ14B_K': 'str', 'REQ15B_S': 'str', 'REQ16A_S': 'str', - 'REQ18B_B': 'str', 'REQ19A_2': 'str', 'REQ19A_P': 'str', - 'SATURATA': 'str', 'SATURATB': 'str', 'SATURATE': 'str', - 'SKYBRITE': 'str', 'SKYIM': 'str', 'SKYSBFIL': 'str', - 'SKYSIGMA': 'str', 'SKYSUB': 'str', 'SKYSUB00': 'str', - 'SKYVARA': 'str', 'SKYVARB': 'str', 'SLOT00': 'str', - 'SLOT01': 'str', 'SLOT02': 'str', 'SLOT03': 'str', - 'SLOT04': 'str', 'SLOT05': 'str', 'STARFIL': 'str', - 'STARMASK': 'str', 'TOO15A_2': 'str', 'TOO15B_2': 'str', - 'TOO16A_2': 'str', 'TOO16B_2': 'str', 'TOO17B_2': 'str', - 'TOO18A_2': 'str', 'TOO18A_S': 'str', 'TOO18B_2': 'str', - 'TOO19A_2': 'str', 'TOO19A_G': 'str', 'TOO19A_K': 'str', - 'TOO19A_M': 'str', 'TOO19B_M': 'str', 'TOO20A_M': 'str', - 'WAT0_001': 'str', 'WAT1_001': 'str', 'WAT2_001': 'str', - 'WCSAXES': 'str', 'WCSDIM': 'str', 'WTMAP': 'str', - 'XTENSION': 'str', 'ZDITHER0': 'str'} -core_hdu_fields = [{'Field': 'boundary', 'Type': 'str'}, - {'Field': 'dec', 'Type': 'np.float64'}, - {'Field': 'dec_range', 'Type': 'str'}, - {'Field': 'fitsfile', 'Type': 'str'}, - {'Field': 'fitsfile__archive_filename', 'Type': 'str'}, - {'Field': 'fitsfile__caldat', 'Type': 'str'}, - {'Field': 'fitsfile__date_obs_max', 'Type': 'str'}, - {'Field': 'fitsfile__date_obs_min', 'Type': 'str'}, - {'Field': 'fitsfile__dec_max', 'Type': 'str'}, - {'Field': 'fitsfile__dec_min', 'Type': 'str'}, - {'Field': 'fitsfile__depth', 'Type': 'str'}, - {'Field': 'fitsfile__exposure', 'Type': 'str'}, - {'Field': 'fitsfile__filesize', 'Type': 'str'}, - {'Field': 'fitsfile__ifilter', 'Type': 'str'}, - {'Field': 'fitsfile__instrument', 'Type': 'str'}, - {'Field': 'fitsfile__md5sum', 'Type': 'str'}, - {'Field': 'fitsfile__obs_mode', 'Type': 'str'}, - {'Field': 'fitsfile__obs_type', 'Type': 'str'}, - {'Field': 'fitsfile__original_filename', 'Type': 'str'}, - {'Field': 'fitsfile__proc_type', 'Type': 'str'}, - {'Field': 'fitsfile__prod_type', 'Type': 'str'}, - {'Field': 'fitsfile__proposal', 'Type': 'str'}, - {'Field': 'fitsfile__ra_max', 'Type': 'str'}, - {'Field': 'fitsfile__ra_min', 'Type': 'str'}, - {'Field': 'fitsfile__release_date', 'Type': 'str'}, - {'Field': 'fitsfile__seeing', 'Type': 'str'}, - {'Field': 'fitsfile__site', 'Type': 'str'}, - {'Field': 'fitsfile__survey', 'Type': 'str'}, - {'Field': 'fitsfile__telescope', 'Type': 'str'}, - {'Field': 'fitsfile__updated', 'Type': 'str'}, - {'Field': 'hdu_idx', 'Type': 'np.int64'}, - {'Field': 'id', 'Type': 'str'}, - {'Field': 'ra', 'Type': 'np.float64'}, - {'Field': 'ra_range', 'Type': 'str'}, - {'Field': 'updated', 'Type': 'datetime64'}] +core_hdu_fields = [{'Field': 'dec_center', 'Type': 'np.float64', 'Desc': 'DEC center from pipeline processing'}, + {'Field': 'dec_max', 'Type': 'np.float64', 'Desc': 'DEC min,max range of HDU'}, + {'Field': 'dec_min', 'Type': 'np.float64', 'Desc': 'DEC min,max range of HDU'}, + {'Field': 'fitsfile', 'Type': 'str', 'Desc': ''}, + {'Field': 'hdu_idx', 'Type': 'np.int64', 'Desc': ''}, + {'Field': 'id', 'Type': 'str', 'Desc': ''}, + {'Field': 'ra_center', 'Type': 'np.float64', 'Desc': 'RA center from pipeline processing'}, + {'Field': 'ra_max', 'Type': 'np.float64', 'Desc': 'RA min,max range of HDU'}, + {'Field': 'ra_min', 'Type': 'np.float64', 'Desc': 'RA min,max range of HDU'}, + {'Field': 'updated', 'Type': 'datetime64', 'Desc': 'When Hdu created/updated'}] + +aux_hdu_fields = [{'Field': 'AMPMTCH', 'Type': 'str', 'Desc': ''}, + {'Field': 'ARAWGAIN', 'Type': 'str', 'Desc': '[e/adu] Average raw gain'}, + {'Field': 'AVSIG', 'Type': 'str', 'Desc': ''}, + {'Field': 'AVSKY', 'Type': 'str', 'Desc': ''}, + {'Field': 'BIASFIL', 'Type': 'str', 'Desc': 'Bias'}, + {'Field': 'BLDINTRP', 'Type': 'str', 'Desc': ''}, + {'Field': 'BPM', 'Type': 'str', 'Desc': ''}, + {'Field': 'BPMFIL', 'Type': 'str', 'Desc': 'BPM file used to build mask'}, + {'Field': 'CCDNUM', 'Type': 'str', 'Desc': ''}, + {'Field': 'CD1_1', 'Type': 'str', 'Desc': 'Coordinate matrix'}] query_hdu_metadata = ['AIRMASS fitsfile__archive_filename fitsfile__caldat fitsfile__instrument fitsfile__proc_type', '------- ----------------------------------------------------------------------- ---------------- -------------------- -------------------', diff --git a/astroquery/noirlab/tests/test_noirlab.py b/astroquery/noirlab/tests/test_noirlab.py index ae8bd7a8ec..f7ff566d7d 100644 --- a/astroquery/noirlab/tests/test_noirlab.py +++ b/astroquery/noirlab/tests/test_noirlab.py @@ -7,10 +7,13 @@ # from astropy import units as u # from astropy.coordinates import SkyCoord from ...utils.mocks import MockResponse -from .. import NOIRLab # , NOIRLabClass +from .. import NOIRLab, NOIRLabClass from . import expected as exp +NOIRLabHDU = NOIRLabClass(hdu=True) + + def mock_content(method, url, **kwargs): if 'FORMAT=METADATA' in url: content = json.dumps(exp.service_metadata) @@ -18,6 +21,14 @@ def mock_content(method, url, **kwargs): content = json.dumps(exp.core_file_fields) elif '/aux_file_fields' in url: content = json.dumps(exp.aux_file_fields) + elif '/find?rectype=file' in url: + content = json.dumps(exp.query_file_metadata) + elif '/core_hdu_fields' in url: + content = json.dumps(exp.core_hdu_fields) + elif '/aux_hdu_fields' in url: + content = json.dumps(exp.aux_hdu_fields) + elif '/find?rectype=hdu' in url: + content = json.dumps(exp.query_hdu_metadata) elif '/cat_lists/' in url: content = json.dumps(exp.categoricals) elif '/version/' in url: @@ -30,6 +41,7 @@ def mock_content(method, url, **kwargs): @pytest.fixture def patch_request(monkeypatch): monkeypatch.setattr(NOIRLab, '_request', mock_content) + monkeypatch.setattr(NOIRLabHDU, '_request', mock_content) return monkeypatch @@ -54,6 +66,34 @@ def test_aux_file_fields(patch_request): assert actual == exp.aux_file_fields +@pytest.mark.skip(reason='WIP') +def test_query_file_metadata(patch_request): + """Search FILE metadata. + """ + qspec = {"outfields": ["md5sum", + "archive_filename", + "original_filename", + "instrument", + "proc_type"], + "search": [['original_filename', 'c4d_', 'contains']]} + actual = NOIRLab.query_metadata(qspec, limit=3) + assert actual.pformat_all() == exp.query_file_metadata + + +def test_core_hdu_fields(patch_request): + """List the available CORE HDU fields. + """ + actual = NOIRLabHDU.core_fields() + assert actual == exp.core_hdu_fields + + +def test_aux_hdu_fields(patch_request): + """List the available AUX HDU fields. + """ + actual = NOIRLabHDU.aux_fields('decam', 'instcal') + assert actual == exp.aux_hdu_fields + + def test_categoricals(patch_request): """List categories. """ diff --git a/astroquery/noirlab/tests/test_noirlab_remote.py b/astroquery/noirlab/tests/test_noirlab_remote.py index db401dd443..2833cdbc77 100644 --- a/astroquery/noirlab/tests/test_noirlab_remote.py +++ b/astroquery/noirlab/tests/test_noirlab_remote.py @@ -13,9 +13,6 @@ from . import expected as exp -# (2) SIA; /api/sia/ -# voimg, vohdu - @pytest.mark.remote_data def test_service_metadata(): """Test compliance with 6.1 of SIA spec v1.0. @@ -64,16 +61,6 @@ def test_query_region_2(): assert expected.issubset(actual) -# (7) Advanced Search; /api/adv_search/ -# -# (2) aux_{file,hdu}_fields// -# (2) core_{file,hdu}_fields/ -# [(2) {f,h}adoc JUST LINK to these] -# (2) {f,h}asearch -# cat_list -# -# File (default type) -# @pytest.mark.remote_data def test_core_file_fields(): """List the available CORE FILE fields. @@ -113,10 +100,6 @@ def test_query_file_metadata(): assert actual.pformat_all() == exp.query_file_metadata -# -# HDU search -# -@pytest.mark.skip(reason='old API') @pytest.mark.remote_data def test_core_hdu_fields(): """List the available CORE HDU fields. @@ -125,13 +108,12 @@ def test_core_hdu_fields(): assert actual == exp.core_hdu_fields -@pytest.mark.skip(reason='old API') @pytest.mark.remote_data def test_aux_hdu_fields(): """List the available AUX HDU fields. """ actual = NOIRLabClass(hdu=True).aux_fields('decam', 'instcal') - assert actual == exp.aux_hdu_fields + assert actual[:10] == exp.aux_hdu_fields @pytest.mark.skip(reason='old API') From 2f8816e861bfc34f5399c06f2efac4d58d431089 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Wed, 9 Jul 2025 10:59:48 -0400 Subject: [PATCH 14/26] activate additional tests and add placeholders --- astroquery/noirlab/core.py | 11 ++-- astroquery/noirlab/tests/expected.py | 41 ++++++++++-- astroquery/noirlab/tests/test_noirlab.py | 63 +++++++++++++++++-- .../noirlab/tests/test_noirlab_remote.py | 15 ++--- 4 files changed, 109 insertions(+), 21 deletions(-) diff --git a/astroquery/noirlab/core.py b/astroquery/noirlab/core.py index b60374abe8..c1cf1e9e4f 100644 --- a/astroquery/noirlab/core.py +++ b/astroquery/noirlab/core.py @@ -35,12 +35,12 @@ def __init__(self, hdu=False): if hdu: self.siaurl = f'{self.NAT_URL}/api/sia/vohdu' - self._adss_url = f'{self._adsurl}/find?rectype=hdu' + self._adss_url = f'{self._adsurl}/find/?rectype=hdu' self._adsc_url = f'{self._adsurl}/core_hdu_fields' self._adsa_url = f'{self._adsurl}/aux_hdu_fields' else: self.siaurl = f'{self.NAT_URL}/api/sia/voimg' - self._adss_url = f'{self._adsurl}/find?rectype=file' + self._adss_url = f'{self._adsurl}/find/?rectype=file' self._adsc_url = f'{self._adsurl}/core_file_fields' self._adsa_url = f'{self._adsurl}/aux_file_fields' @@ -105,7 +105,8 @@ def query_region(self, coordinate, radius=0.1, cache=True): timeout=self.TIMEOUT, cache=cache) response.raise_for_status() - return astropy.table.Table(data=response.json()) + # return astropy.table.Table(data=response.json()) + return response.json() def query_region_async(self, coordinate, radius=0.1, cache=True): """Query for NOIRLab observations by region of the sky. @@ -197,7 +198,9 @@ def query_metadata(self, qspec, limit=1000, cache=True): response = self._request('POST', url, json=jdata, timeout=self.TIMEOUT, cache=cache) response.raise_for_status() - return astropy.table.Table(rows=response.json()) + j = response.json() + # The first entry contains metadata. + return astropy.table.Table(rows=j[1:]) def retrieve(self, fileid): """Simply fetch a file by MD5 ID. diff --git a/astroquery/noirlab/tests/expected.py b/astroquery/noirlab/tests/expected.py index 1c7e1942ac..9a4a720ee2 100644 --- a/astroquery/noirlab/tests/expected.py +++ b/astroquery/noirlab/tests/expected.py @@ -41,12 +41,43 @@ {'Field': 'AZ', 'Type': 'str', 'Desc': ''}, {'Field': 'BAND', 'Type': 'str', 'Desc': ''}] -query_file_metadata = [' archive_filename instrument md5sum original_filename proc_type url ', - '----------------------------------------------------------------------------- ---------- -------------------------------- ------------------------------------------------------------------------------------- --------- ----------------------------------------------------------------------------', - '/net/archive/pipe/20161024/ct4m/2012B-0001/c4d_161025_034649_oki_z_v2.fits.fz decam 0000f0c5015263610a75f0affed377d0 /net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_161025_034649_oki_z_v2.fits.fz skysub https://astroarchive.noao.edu/api/retrieve/0000f0c5015263610a75f0affed377d0/', - '/net/archive/pipe/20160928/ct4m/2012B-0001/c4d_160929_090838_ooi_z_v2.fits.fz decam 0001e5a5bcf039ebf0c53a6da32e8888 /net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_160929_090838_ooi_z_v2.fits.fz instcal https://astroarchive.noao.edu/api/retrieve/0001e5a5bcf039ebf0c53a6da32e8888/', - '/net/archive/pipe/20160909/ct4m/2012B-0001/c4d_160910_062930_opd_z_v2.fits.fz decam 0003a1c853de73fc1796d2d7d77ca9cd /net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_160910_062930_opd_z_v2.fits.fz resampled https://astroarchive.noao.edu/api/retrieve/0003a1c853de73fc1796d2d7d77ca9cd/'] +query_file_metadata = ['proc_type original_filename md5sum instrument archive_filename ', + '--------- ------------------------------------------------------------------------------------- -------------------------------- ---------- -----------------------------------------------------------------------------', + ' instcal /net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_161005_063408_ood_r_v2.fits.fz bd70774014d8c32d0fc8f9933dcc5785 decam /net/archive/pipe/20161004/ct4m/2012B-0001/c4d_161005_063408_ood_r_v2.fits.fz', + 'resampled /net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_180125_005333_opd_i_v1.fits.fz 39a5e8c1d0c1ee721f1a753367f5f7c5 decam /net/archive/pipe/20180124/ct4m/2012B-0001/c4d_180125_005333_opd_i_v1.fits.fz', + 'resampled /net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_160930_070518_opi_r_v2.fits.fz 9860ceed19ac461a5435055e68060c6c decam /net/archive/pipe/20160929/ct4m/2012B-0001/c4d_160930_070518_opi_r_v2.fits.fz'] +query_file_meta_raw = [{'META': {'endpoint': 'adv_search/find'}, + 'PARAMETERS': {'rectype': 'file', + 'limit': 3, + 'default_limit': 1000, + 'default_offset': 0, + 'default_sort': 'md5sum', + 'oldest': None, + 'previd': None, + 'last': 3, + 'json_payload': {'outfields': ['md5sum', 'archive_filename', 'original_filename', 'instrument', 'proc_type'], + 'search': [['original_filename', 'c4d_', 'contains']]}}, + 'HEADER': {'proc_type': 'category', + 'original_filename': 'str', + 'md5sum': 'str', + 'instrument': 'category', + 'archive_filename': 'str'}}, + {'proc_type': 'instcal', + 'original_filename': '/net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_161005_063408_ood_r_v2.fits.fz', + 'md5sum': 'bd70774014d8c32d0fc8f9933dcc5785', + 'instrument': 'decam', + 'archive_filename': '/net/archive/pipe/20161004/ct4m/2012B-0001/c4d_161005_063408_ood_r_v2.fits.fz'}, + {'proc_type': 'resampled', + 'original_filename': '/net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_180125_005333_opd_i_v1.fits.fz', + 'md5sum': '39a5e8c1d0c1ee721f1a753367f5f7c5', + 'instrument': 'decam', + 'archive_filename': '/net/archive/pipe/20180124/ct4m/2012B-0001/c4d_180125_005333_opd_i_v1.fits.fz'}, + {'proc_type': 'resampled', + 'original_filename': '/net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_160930_070518_opi_r_v2.fits.fz', + 'md5sum': '9860ceed19ac461a5435055e68060c6c', + 'instrument': 'decam', + 'archive_filename': '/net/archive/pipe/20160929/ct4m/2012B-0001/c4d_160930_070518_opi_r_v2.fits.fz'}] core_hdu_fields = [{'Field': 'dec_center', 'Type': 'np.float64', 'Desc': 'DEC center from pipeline processing'}, {'Field': 'dec_max', 'Type': 'np.float64', 'Desc': 'DEC min,max range of HDU'}, diff --git a/astroquery/noirlab/tests/test_noirlab.py b/astroquery/noirlab/tests/test_noirlab.py index f7ff566d7d..5ac52f655e 100644 --- a/astroquery/noirlab/tests/test_noirlab.py +++ b/astroquery/noirlab/tests/test_noirlab.py @@ -4,8 +4,8 @@ """ import json import pytest -# from astropy import units as u -# from astropy.coordinates import SkyCoord +from astropy import units as u +from astropy.coordinates import SkyCoord from ...utils.mocks import MockResponse from .. import NOIRLab, NOIRLabClass from . import expected as exp @@ -21,8 +21,8 @@ def mock_content(method, url, **kwargs): content = json.dumps(exp.core_file_fields) elif '/aux_file_fields' in url: content = json.dumps(exp.aux_file_fields) - elif '/find?rectype=file' in url: - content = json.dumps(exp.query_file_metadata) + elif '/find/?rectype=file' in url: + content = json.dumps(exp.query_file_meta_raw) elif '/core_hdu_fields' in url: content = json.dumps(exp.core_hdu_fields) elif '/aux_hdu_fields' in url: @@ -52,6 +52,44 @@ def test_service_metadata(patch_request): assert actual == exp.service_metadata[0] +@pytest.mark.skip(reason='WIP') +def test_query_region_0(): + """Search FILES. + """ + c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') + r = NOIRLab.query_region(c, radius='0.1') + actual = set(list(r['md5sum'])) + expected = exp.query_region_1 + assert expected.issubset(actual) + + +@pytest.mark.skip(reason='WIP') +def test_query_region_1(): + """Search FILES. + + Ensure query gets at least the set of files we expect. + It is OK if more files have been added to the remote Archive. + """ + c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') + r = NOIRLab.query_region(c, radius='0.1') + actual = set(list(r['md5sum'])) + expected = exp.query_region_1 + assert expected.issubset(actual) + + +@pytest.mark.skip(reason='WIP') +def test_query_region_2(): + """Search HDUs. + + Ensure query gets at least the set of files we expect. + Its ok if more files have been added to the remote Archive. + """ + c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') + r = NOIRLabHDU.query_region(c, radius='0.07') + actual = set(list(r['md5sum'])) + expected = exp.query_region_2 + assert expected.issubset(actual) + def test_core_file_fields(patch_request): """List the available CORE FILE fields. """ @@ -66,7 +104,6 @@ def test_aux_file_fields(patch_request): assert actual == exp.aux_file_fields -@pytest.mark.skip(reason='WIP') def test_query_file_metadata(patch_request): """Search FILE metadata. """ @@ -94,6 +131,22 @@ def test_aux_hdu_fields(patch_request): assert actual == exp.aux_hdu_fields +@pytest.mark.skip(reason='WIP') +def test_query_hdu_metadata(patch_request): + """Search HDU metadata. + """ + qspec = {"outfields": ["fitsfile__archive_filename", + "fitsfile__caldat", + "fitsfile__instrument", + "fitsfile__proc_type", + "AIRMASS"], # AUX field. Slows search + "search": [["fitsfile__caldat", "2017-08-14", "2017-08-16"], + ["fitsfile__instrument", "decam"], + ["fitsfile__proc_type", "raw"]]} + actual = NOIRLabHDU.query_metadata(qspec, limit=3) + assert actual.pformat_all() == exp.query_hdu_metadata + + def test_categoricals(patch_request): """List categories. """ diff --git a/astroquery/noirlab/tests/test_noirlab_remote.py b/astroquery/noirlab/tests/test_noirlab_remote.py index 2833cdbc77..cd5ca9c59f 100644 --- a/astroquery/noirlab/tests/test_noirlab_remote.py +++ b/astroquery/noirlab/tests/test_noirlab_remote.py @@ -24,8 +24,8 @@ def test_service_metadata(): @pytest.mark.skip(reason='old API') @pytest.mark.remote_data def test_query_region_0(): - """Search FILES.""" - + """Search FILES. + """ c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') r = NOIRLab().query_region(c, radius='0.1') actual = set(list(r['md5sum'])) @@ -37,9 +37,10 @@ def test_query_region_0(): @pytest.mark.remote_data def test_query_region_1(): """Search FILES. - Ensure query gets at least the set of files we expect. - Its ok if more files have been added to the remote Archive.""" + Ensure query gets at least the set of files we expect. + It is OK if more files have been added to the remote Archive. + """ c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') r = NOIRLabClass().query_region(c, radius='0.1') actual = set(list(r['md5sum'])) @@ -51,9 +52,10 @@ def test_query_region_1(): @pytest.mark.remote_data def test_query_region_2(): """Search HDUs. - Ensure query gets at least the set of files we expect. - Its ok if more files have been added to the remote Archive.""" + Ensure query gets at least the set of files we expect. + Its ok if more files have been added to the remote Archive. + """ c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') r = NOIRLabClass(hdu=True).query_region(c, radius='0.07') actual = set(list(r['md5sum'])) @@ -85,7 +87,6 @@ def test_categoricals(): assert actual == exp.categoricals -@pytest.mark.skip(reason='old API') @pytest.mark.remote_data def test_query_file_metadata(): """Search FILE metadata. From 714c2f0f8e2b199849b0cccb30954b62bb20a03a Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Wed, 9 Jul 2025 11:43:55 -0400 Subject: [PATCH 15/26] fix style issue --- astroquery/noirlab/tests/test_noirlab.py | 1 + astroquery/noirlab/tests/test_noirlab_remote.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/astroquery/noirlab/tests/test_noirlab.py b/astroquery/noirlab/tests/test_noirlab.py index 5ac52f655e..8753385b31 100644 --- a/astroquery/noirlab/tests/test_noirlab.py +++ b/astroquery/noirlab/tests/test_noirlab.py @@ -90,6 +90,7 @@ def test_query_region_2(): expected = exp.query_region_2 assert expected.issubset(actual) + def test_core_file_fields(patch_request): """List the available CORE FILE fields. """ diff --git a/astroquery/noirlab/tests/test_noirlab_remote.py b/astroquery/noirlab/tests/test_noirlab_remote.py index cd5ca9c59f..4f0b25fc9c 100644 --- a/astroquery/noirlab/tests/test_noirlab_remote.py +++ b/astroquery/noirlab/tests/test_noirlab_remote.py @@ -98,7 +98,9 @@ def test_query_file_metadata(): "proc_type"], "search": [['original_filename', 'c4d_', 'contains']]} actual = NOIRLab().query_metadata(qspec, limit=3) - assert actual.pformat_all() == exp.query_file_metadata + # .pformat_all() is deprecated, use pformat instead. + # assert actual.pformat_all() == exp.query_file_metadata + assert actual.pformat() == exp.query_file_metadata @pytest.mark.remote_data From 25f879f17867cd2a3325a57a65015b2aa271c760 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Wed, 9 Jul 2025 12:36:20 -0400 Subject: [PATCH 16/26] working on doc strings --- astroquery/noirlab/core.py | 106 +++++++++++++++++++++++++++++-------- 1 file changed, 85 insertions(+), 21 deletions(-) diff --git a/astroquery/noirlab/core.py b/astroquery/noirlab/core.py index c1cf1e9e4f..dd07902510 100644 --- a/astroquery/noirlab/core.py +++ b/astroquery/noirlab/core.py @@ -58,6 +58,8 @@ def api_version(self): return self._api_version def _validate_version(self): + """Ensure the API is compatible with the code. + """ KNOWN_GOOD_API_VERSION = 6.0 if (int(self.api_version) - int(KNOWN_GOOD_API_VERSION)) >= 1: msg = (f'The astroquery.noirlab module is expecting an older ' @@ -68,9 +70,21 @@ def _validate_version(self): raise RemoteServiceError(msg) def service_metadata(self, cache=True): - """Denotes a Metadata Query: no images are requested; only metadata - should be returned. This feature is described in more detail in: - http://www.ivoa.net/documents/PR/DAL/PR-SIA-1.0-20090521.html#mdquery + """A SIA metadata query: no images are requested; only metadata + should be returned. + + This feature is described in more detail in: + https://www.ivoa.net/documents/PR/DAL/PR-SIA-1.0-20090521.html#mdquery + + Parameters + ---------- + cache : :class:`bool`, optional + If ``True`` cache the result locally. + + Returns + ------- + :class:`dict` + A dictionary containing SIA metadata. """ url = f'{self.siaurl}?FORMAT=METADATA&format=json' response = self._request('GET', url, timeout=self.TIMEOUT, cache=cache) @@ -85,10 +99,10 @@ def query_region(self, coordinate, radius=0.1, cache=True): Parameters ---------- - coordinates : str or `~astropy.coordinates` object + coordinates : :class:`str` or `~astropy.coordinates` object The target region which to search. It may be specified as a string or as the appropriate `~astropy.coordinates` object. - radius : str or `~astropy.units.Quantity` object, optional + radius : :class:`str` or `~astropy.units.Quantity` object, optional Default 0.1 degrees. The string must be parsable by `~astropy.coordinates.Angle`. The appropriate `~astropy.units.Quantity` object from @@ -96,7 +110,8 @@ def query_region(self, coordinate, radius=0.1, cache=True): Returns ------- - `~astropy.table.Table` + :class:`~astropy.table.Table` + A table containing the results. """ self._validate_version() ra, dec = coordinate.to_string('decimal').split() @@ -116,18 +131,21 @@ def query_region_async(self, coordinate, radius=0.1, cache=True): Parameters ---------- - coordinates : str or `~astropy.coordinates` object + coordinates : :clas:`str` or `~astropy.coordinates` object The target region which to search. It may be specified as a string or as the appropriate `~astropy.coordinates` object. - radius : str or `~astropy.units.Quantity` object, optional + radius : :clas:`str` or `~astropy.units.Quantity` object, optional Default 0.1 degrees. The string must be parsable by `~astropy.coordinates.Angle`. The appropriate `~astropy.units.Quantity` object from `~astropy.units` may also be used. + cache : :class:`bool`, optional + If ``True`` cache the result locally. Returns ------- - `requests.Response` + :class:`~requests.Response` + Response object. """ self._validate_version() @@ -161,11 +179,28 @@ def core_fields(self, cache=True): return response.json() def aux_fields(self, instrument, proctype, cache=True): - """List the available AUX fields. AUX fields are ANY fields in the - Archive FITS files that are not core DB fields. These are generally - common to a single Instrument, Proctype combination. AUX fields are - slower to search than CORE fields. Acceptable values for INSTRUMENT and PROCTYPE - are listed in the results of the CATEGORICALS method. + """List the available AUX fields. + + AUX fields are any fields in the Archive FITS files that are not + CORE DB fields. These are generally common to a single instrument, + proctype combination. AUX fields are slower to search than CORE fields. + Acceptable values for `instrument` and `proctype` are listed in the + results of the :meth:`astroquery.noirlab.core.NOIRLabClass.categoricals` + method. + + Parameters + ---------- + instrument : :class:`str` + The specific instrument, *e.g.* '90prime' or 'decam'. + proctype : :class:`str` + A description of the type of image, *e.g.* 'raw' or 'instcal'. + cache : :class:`bool`, optional + If ``True`` cache the result locally. + + Returns + ------- + :class:`list` + A list of field descriptions, each a :class:`dict`. """ url = f'{self._adsa_url}/{instrument}/{proctype}/' response = self._request('GET', url, timeout=self.TIMEOUT, cache=cache) @@ -174,11 +209,22 @@ def aux_fields(self, instrument, proctype, cache=True): def categoricals(self, cache=True): """List the currently acceptable values for each 'categorical field' - associated with Archive files. A 'categorical field' is one in - which the values are restricted to a specific set. The specific - set may grow over time, but not often. The categorical fields are: - collection, instrument, obs_mode, proc_type, prod_type, site, survey, - telescope. + associated with Archive files. + + A 'categorical field' is one in which the values are restricted to a + specific set. The specific set may grow over time, but not often. + The categorical fields are: ``instrument``, ``obsmode``, ``obstype``, + ``proctype``, ``prodtype``, ``site``, ``survey``, ``telescope``. + + Parameters + ---------- + cache : :class:`bool`, optional + If ``True`` cache the result locally. + + Returns + ------- + :class:`dict` + A dictionary containing the category metadata. """ url = f'{self._adsurl}/cat_lists/?format=json' response = self._request('GET', url, timeout=self.TIMEOUT, cache=cache) @@ -186,8 +232,24 @@ def categoricals(self, cache=True): return response.json() @class_or_instance - def query_metadata(self, qspec, limit=1000, cache=True): - # self._validate_version() + def query_metadata(self, qspec=None, limit=1000, cache=True): + """Query the archive database for details on available files. + + Paramters + --------- + qspec : :class:`dict`, optional + The query that will be passed to the API. + limit : :class:`int`, optional + The number of results to return, default 1000. + cache : :class:`bool`, optional + If ``True`` cache the result locally. + + Returns + ------- + :class:`~astropy.table.Table` + A Table containing the results. + """ + self._validate_version() url = f'{self._adss_url}&limit={limit}' if qspec is None: @@ -247,6 +309,8 @@ def get_token(self, email, password, cache=True): password : :class:`str` Password associated with `email`. *Please* never hard-code your password *anywhere*. + cache : :class:`bool`, optional + If ``True`` cache the result locally. Returns ------- From 298a69f39f5438d57e24024d38c2e207a318eed3 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 10 Jul 2025 12:28:51 -0400 Subject: [PATCH 17/26] fix pformat issues --- astroquery/noirlab/tests/test_noirlab.py | 4 ++-- astroquery/noirlab/tests/test_noirlab_remote.py | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/astroquery/noirlab/tests/test_noirlab.py b/astroquery/noirlab/tests/test_noirlab.py index 8753385b31..f650482181 100644 --- a/astroquery/noirlab/tests/test_noirlab.py +++ b/astroquery/noirlab/tests/test_noirlab.py @@ -115,7 +115,7 @@ def test_query_file_metadata(patch_request): "proc_type"], "search": [['original_filename', 'c4d_', 'contains']]} actual = NOIRLab.query_metadata(qspec, limit=3) - assert actual.pformat_all() == exp.query_file_metadata + assert actual.pformat(max_width=-1) == exp.query_file_metadata def test_core_hdu_fields(patch_request): @@ -145,7 +145,7 @@ def test_query_hdu_metadata(patch_request): ["fitsfile__instrument", "decam"], ["fitsfile__proc_type", "raw"]]} actual = NOIRLabHDU.query_metadata(qspec, limit=3) - assert actual.pformat_all() == exp.query_hdu_metadata + assert actual.pformat(max_width=-1) == exp.query_hdu_metadata def test_categoricals(patch_request): diff --git a/astroquery/noirlab/tests/test_noirlab_remote.py b/astroquery/noirlab/tests/test_noirlab_remote.py index 4f0b25fc9c..734e5bb0e2 100644 --- a/astroquery/noirlab/tests/test_noirlab_remote.py +++ b/astroquery/noirlab/tests/test_noirlab_remote.py @@ -98,9 +98,7 @@ def test_query_file_metadata(): "proc_type"], "search": [['original_filename', 'c4d_', 'contains']]} actual = NOIRLab().query_metadata(qspec, limit=3) - # .pformat_all() is deprecated, use pformat instead. - # assert actual.pformat_all() == exp.query_file_metadata - assert actual.pformat() == exp.query_file_metadata + assert actual.pformat(max_width=-1) == exp.query_file_metadata @pytest.mark.remote_data @@ -133,7 +131,7 @@ def test_query_hdu_metadata(): ["fitsfile__instrument", "decam"], ["fitsfile__proc_type", "raw"]]} actual = NOIRLabClass(hdu=True).query_metadata(qspec, limit=3) - assert actual.pformat_all() == exp.query_hdu_metadata + assert actual.pformat(max_width=-1) == exp.query_hdu_metadata @pytest.mark.remote_data From c70367335ae78e6c415d93c0c6b5109303d48a52 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 10 Jul 2025 12:36:46 -0400 Subject: [PATCH 18/26] update docs --- astroquery/noirlab/__init__.py | 77 +++++++++++----------------------- astroquery/noirlab/core.py | 6 +-- 2 files changed, 27 insertions(+), 56 deletions(-) diff --git a/astroquery/noirlab/__init__.py b/astroquery/noirlab/__init__.py index 4990be6cc0..9bf811ee5a 100644 --- a/astroquery/noirlab/__init__.py +++ b/astroquery/noirlab/__init__.py @@ -1,59 +1,30 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst """ NSF NOIRLab Astro Data Archive ------------------------------- - -The NSF NOIRLab Astro Data Archive (formerly NOAO Science Archive) -provides access to data taken with more than 40 telescope and -instrument combinations, including those operated in partnership with -the WIYN, SOAR and SMARTS consortia, from semester 2004B to the -present. In addition to raw data, pipeline-reduced data products from -the DECam, Mosaic and NEWFIRM imagers are also available, as well as -advanced data products delivered by teams carrying out surveys and -other large observing programs with NSF OIR Lab facilities. - -Total holdings 9,634,110 V.3 - -====================== ======== ======== - Telescope/Instrument Images Volume -====================== ======== ======== -bok23m-90prime 290070 3% -ct09m-ccd_imager 101671 1% -ct13m-andicam 528124 5% -ct15m-chiron 217063 2% -ct1m-y4kcam 317819 3% -ct4m-arcoiris 49024 0% -ct4m-cosmos 19018 0% -ct4m-decam 4685453 48% -ct4m-mosaic_2 254005 2% -ct4m-newfirm 339731 3% -kp09m-hdi 108835 1% -kp09m-mosaic 1006 0% -kp09m-mosaic_1_1 3533 0% -kp4m-kosmos 8068 0% -kp4m-mosaic 2420 0% -kp4m-mosaic3 563386 5% -kp4m-mosaic_1 52475 0% -kp4m-mosaic_1_1 142351 1% -kp4m-newfirm 962054 9% -soar-goodman 695828 7% -soar-sami 15639 0% -soar-soi 8981 0% -soar-spartan 166821 1% -soar-triplespec 3171 0% -wiyn-bench 33183 0% -wiyn-whirc 64381 0% -====================== ======== ======== - - -ACKNOWLEDGMENT - -This research uses services or data provided by the Astro Data Archive -at NSF's National Optical-Infrared Astronomy Research -Laboratory. NSF's OIR Lab is operated by the Association of -Universities for Research in Astronomy (AURA), Inc. under a -cooperative agreement with the National Science Foundation. - +============================== + +Overview +-------- + +The NOIRLab Astro Data Archive (formerly NOAO Science Archive) provides +access to data taken with more than 40 telescope and instrument combinations, +including those operated in partnership with the WIYN, SOAR and SMARTS +consortia, from semester 2004B to the present. In addition to raw data, +pipeline-reduced data products from the DECam, Mosaic and NEWFIRM imagers +are also available, as well as advanced data products delivered by teams +carrying out surveys and other large observing programs with NSF NOIRLab +facilities. + +A detailed list of holdings in the archive is available at +https://astroarchive.noirlab.edu/about/. + +Acknowledgment +-------------- + +This research uses services or data provided by the Astro Data Archive at +NSF's NOIRLab. NOIRLab is operated by the Association of Universities for +Research in Astronomy (AURA), Inc. under a cooperative agreement with the +National Science Foundation. """ from astropy import config as _config diff --git a/astroquery/noirlab/core.py b/astroquery/noirlab/core.py index dd07902510..55997ef76e 100644 --- a/astroquery/noirlab/core.py +++ b/astroquery/noirlab/core.py @@ -131,10 +131,10 @@ def query_region_async(self, coordinate, radius=0.1, cache=True): Parameters ---------- - coordinates : :clas:`str` or `~astropy.coordinates` object + coordinates : :class:`str` or `~astropy.coordinates` object The target region which to search. It may be specified as a string or as the appropriate `~astropy.coordinates` object. - radius : :clas:`str` or `~astropy.units.Quantity` object, optional + radius : :class:`str` or `~astropy.units.Quantity` object, optional Default 0.1 degrees. The string must be parsable by `~astropy.coordinates.Angle`. The appropriate `~astropy.units.Quantity` object from @@ -184,7 +184,7 @@ def aux_fields(self, instrument, proctype, cache=True): AUX fields are any fields in the Archive FITS files that are not CORE DB fields. These are generally common to a single instrument, proctype combination. AUX fields are slower to search than CORE fields. - Acceptable values for `instrument` and `proctype` are listed in the + Acceptable values for ``instrument`` and ``proctype`` are listed in the results of the :meth:`astroquery.noirlab.core.NOIRLabClass.categoricals` method. From 038a5f6dfb54cd1608ceef631496398792fa911e Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 10 Jul 2025 13:07:31 -0400 Subject: [PATCH 19/26] add test coverage for some corner cases --- astroquery/noirlab/core.py | 2 ++ astroquery/noirlab/tests/expected.py | 15 ++++++++++++ astroquery/noirlab/tests/test_noirlab.py | 23 ++++++++++++++++++- .../noirlab/tests/test_noirlab_remote.py | 8 +++++++ 4 files changed, 47 insertions(+), 1 deletion(-) diff --git a/astroquery/noirlab/core.py b/astroquery/noirlab/core.py index 55997ef76e..7fcfd09c05 100644 --- a/astroquery/noirlab/core.py +++ b/astroquery/noirlab/core.py @@ -107,6 +107,8 @@ def query_region(self, coordinate, radius=0.1, cache=True): The string must be parsable by `~astropy.coordinates.Angle`. The appropriate `~astropy.units.Quantity` object from `~astropy.units` may also be used. + cache : :class:`bool`, optional + If ``True`` cache the result locally. Returns ------- diff --git a/astroquery/noirlab/tests/expected.py b/astroquery/noirlab/tests/expected.py index 9a4a720ee2..16ffaf445e 100644 --- a/astroquery/noirlab/tests/expected.py +++ b/astroquery/noirlab/tests/expected.py @@ -47,6 +47,14 @@ 'resampled /net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_180125_005333_opd_i_v1.fits.fz 39a5e8c1d0c1ee721f1a753367f5f7c5 decam /net/archive/pipe/20180124/ct4m/2012B-0001/c4d_180125_005333_opd_i_v1.fits.fz', 'resampled /net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_160930_070518_opi_r_v2.fits.fz 9860ceed19ac461a5435055e68060c6c decam /net/archive/pipe/20160929/ct4m/2012B-0001/c4d_160930_070518_opi_r_v2.fits.fz'] +query_file_metadata_minimal = [' md5sum ', + '--------------------------------', + '97f18cde19976fe167484003f1aca94c', + '09325eb851e81df518042af1d882b9c5', + 'd4bf24fdfa22385ce748304dceee0d97', + '9d81bef7e7073878e0aa33e63ac8a8c9', + '4938fe8591ea4ac88332bbf2e6e2cb11'] + query_file_meta_raw = [{'META': {'endpoint': 'adv_search/find'}, 'PARAMETERS': {'rectype': 'file', 'limit': 3, @@ -79,6 +87,13 @@ 'instrument': 'decam', 'archive_filename': '/net/archive/pipe/20160929/ct4m/2012B-0001/c4d_160930_070518_opi_r_v2.fits.fz'}] +query_file_meta_raw_minimal = [{'META': {'endpoint': 'adv_search/find'},}, + {'md5sum': '97f18cde19976fe167484003f1aca94c',}, + {'md5sum': '09325eb851e81df518042af1d882b9c5',}, + {'md5sum': 'd4bf24fdfa22385ce748304dceee0d97',}, + {'md5sum': '9d81bef7e7073878e0aa33e63ac8a8c9',}, + {'md5sum': '4938fe8591ea4ac88332bbf2e6e2cb11',}] + core_hdu_fields = [{'Field': 'dec_center', 'Type': 'np.float64', 'Desc': 'DEC center from pipeline processing'}, {'Field': 'dec_max', 'Type': 'np.float64', 'Desc': 'DEC min,max range of HDU'}, {'Field': 'dec_min', 'Type': 'np.float64', 'Desc': 'DEC min,max range of HDU'}, diff --git a/astroquery/noirlab/tests/test_noirlab.py b/astroquery/noirlab/tests/test_noirlab.py index f650482181..9d76f131ef 100644 --- a/astroquery/noirlab/tests/test_noirlab.py +++ b/astroquery/noirlab/tests/test_noirlab.py @@ -7,6 +7,7 @@ from astropy import units as u from astropy.coordinates import SkyCoord from ...utils.mocks import MockResponse +from ...exceptions import RemoteServiceError from .. import NOIRLab, NOIRLabClass from . import expected as exp @@ -21,8 +22,10 @@ def mock_content(method, url, **kwargs): content = json.dumps(exp.core_file_fields) elif '/aux_file_fields' in url: content = json.dumps(exp.aux_file_fields) - elif '/find/?rectype=file' in url: + elif '/find/?rectype=file&limit=3' in url: content = json.dumps(exp.query_file_meta_raw) + elif '/find/?rectype=file&limit=5' in url: + content = json.dumps(exp.query_file_meta_raw_minimal) elif '/core_hdu_fields' in url: content = json.dumps(exp.core_hdu_fields) elif '/aux_hdu_fields' in url: @@ -118,6 +121,13 @@ def test_query_file_metadata(patch_request): assert actual.pformat(max_width=-1) == exp.query_file_metadata +def test_query_file_metadata_minimal_input(patch_request): + """Search FILE metadata with minimum input parameters. + """ + actual = NOIRLab.query_metadata(qspec=None, limit=5) + assert actual.pformat(max_width=-1) == exp.query_file_metadata_minimal + + def test_core_hdu_fields(patch_request): """List the available CORE HDU fields. """ @@ -169,6 +179,17 @@ def test_api_version(patch_request): assert actual >= float(exp.version) +def test__validate_version(patch_request): + """Check exception raised by outdated API version. + """ + actual_api = NOIRLab.api_version + NOIRLab._api_version = 9.8 + with pytest.raises(RemoteServiceError) as e: + NOIRLab._validate_version() + assert e.value.args[0] == 'The astroquery.noirlab module is expecting an older version of the https://astroarchive.noirlab.edu API services. Please upgrade to latest astroquery. Expected version 6.0 but got 9.8 from the API.' + NOIRLab._api_version = actual_api + + def test_get_token(patch_request): """Test token retrieval. """ diff --git a/astroquery/noirlab/tests/test_noirlab_remote.py b/astroquery/noirlab/tests/test_noirlab_remote.py index 734e5bb0e2..e4711179c7 100644 --- a/astroquery/noirlab/tests/test_noirlab_remote.py +++ b/astroquery/noirlab/tests/test_noirlab_remote.py @@ -101,6 +101,14 @@ def test_query_file_metadata(): assert actual.pformat(max_width=-1) == exp.query_file_metadata +@pytest.mark.remote_data +def test_query_file_metadata_minimal_input(): + """Search FILE metadata with minimum input parameters. + """ + actual = NOIRLab().query_metadata(qspec=None, limit=5) + assert actual.pformat(max_width=-1) == exp.query_file_metadata_minimal + + @pytest.mark.remote_data def test_core_hdu_fields(): """List the available CORE HDU fields. From 7929ebaad04fa3b925f7cd867344149a259b2e96 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 10 Jul 2025 13:19:25 -0400 Subject: [PATCH 20/26] fix style & add placeholders --- astroquery/noirlab/tests/test_noirlab.py | 13 ++++++++++++- astroquery/noirlab/tests/test_noirlab_remote.py | 9 +++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/astroquery/noirlab/tests/test_noirlab.py b/astroquery/noirlab/tests/test_noirlab.py index 9d76f131ef..fcef147e1c 100644 --- a/astroquery/noirlab/tests/test_noirlab.py +++ b/astroquery/noirlab/tests/test_noirlab.py @@ -158,6 +158,14 @@ def test_query_hdu_metadata(patch_request): assert actual.pformat(max_width=-1) == exp.query_hdu_metadata +@pytest.mark.skip(reason='WIP') +def test_query_hdu_metadata_minimal_input(patch_request): + """Search HDU metadata with minimum input parameters. + """ + actual = NOIRLabHDU.query_metadata(qspec=None, limit=3) + assert actual.pformat(max_width=-1) == exp.query_hdu_metadata_minimal + + def test_categoricals(patch_request): """List categories. """ @@ -186,7 +194,10 @@ def test__validate_version(patch_request): NOIRLab._api_version = 9.8 with pytest.raises(RemoteServiceError) as e: NOIRLab._validate_version() - assert e.value.args[0] == 'The astroquery.noirlab module is expecting an older version of the https://astroarchive.noirlab.edu API services. Please upgrade to latest astroquery. Expected version 6.0 but got 9.8 from the API.' + assert e.value.args[0] == ('The astroquery.noirlab module is expecting an older ' + 'version of the https://astroarchive.noirlab.edu API services. ' + 'Please upgrade to latest astroquery. ' + 'Expected version 6.0 but got 9.8 from the API.') NOIRLab._api_version = actual_api diff --git a/astroquery/noirlab/tests/test_noirlab_remote.py b/astroquery/noirlab/tests/test_noirlab_remote.py index e4711179c7..274628efb6 100644 --- a/astroquery/noirlab/tests/test_noirlab_remote.py +++ b/astroquery/noirlab/tests/test_noirlab_remote.py @@ -125,6 +125,15 @@ def test_aux_hdu_fields(): assert actual[:10] == exp.aux_hdu_fields +@pytest.mark.skip(reason='old API') +@pytest.mark.remote_data +def test_query_hdu_metadata_minimal_input(): + """Search HDU metadata with minimum input parameters. + """ + actual = NOIRLabClass(hdu=True).query_metadata(qspec=None, limit=5) + assert actual.pformat(max_width=-1) == exp.query_hdu_metadata_minimal + + @pytest.mark.skip(reason='old API') @pytest.mark.remote_data def test_query_hdu_metadata(): From e0c421f8f2866453828c96dd5c487f5158895f60 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 10 Jul 2025 13:24:20 -0400 Subject: [PATCH 21/26] tweak formatting --- astroquery/noirlab/tests/expected.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/astroquery/noirlab/tests/expected.py b/astroquery/noirlab/tests/expected.py index 16ffaf445e..337bcd15f4 100644 --- a/astroquery/noirlab/tests/expected.py +++ b/astroquery/noirlab/tests/expected.py @@ -87,7 +87,10 @@ 'instrument': 'decam', 'archive_filename': '/net/archive/pipe/20160929/ct4m/2012B-0001/c4d_160930_070518_opi_r_v2.fits.fz'}] -query_file_meta_raw_minimal = [{'META': {'endpoint': 'adv_search/find'},}, +query_file_meta_raw_minimal = [{'META': {'endpoint': 'adv_search/find'}, + 'PARAMETERS': {'rectype': 'file', + 'limit': 5}, + 'HEADER': {'md5sum': 'str'}}, {'md5sum': '97f18cde19976fe167484003f1aca94c',}, {'md5sum': '09325eb851e81df518042af1d882b9c5',}, {'md5sum': 'd4bf24fdfa22385ce748304dceee0d97',}, From f63e0912a9aa292c73030d183c399e1df9330417 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 10 Jul 2025 13:42:23 -0400 Subject: [PATCH 22/26] update and sync docs --- astroquery/noirlab/__init__.py | 4 +- astroquery/noirlab/core.py | 4 +- docs/noirlab/noirlab.rst | 69 +++++++++++++++++----------------- 3 files changed, 38 insertions(+), 39 deletions(-) diff --git a/astroquery/noirlab/__init__.py b/astroquery/noirlab/__init__.py index 9bf811ee5a..88e248e09f 100644 --- a/astroquery/noirlab/__init__.py +++ b/astroquery/noirlab/__init__.py @@ -15,8 +15,8 @@ carrying out surveys and other large observing programs with NSF NOIRLab facilities. -A detailed list of holdings in the archive is available at -https://astroarchive.noirlab.edu/about/. +For more info about our holdings see the +`NSF NOIRLab Astro Data Archive `_. Acknowledgment -------------- diff --git a/astroquery/noirlab/core.py b/astroquery/noirlab/core.py index 7fcfd09c05..f04e82d20e 100644 --- a/astroquery/noirlab/core.py +++ b/astroquery/noirlab/core.py @@ -237,8 +237,8 @@ def categoricals(self, cache=True): def query_metadata(self, qspec=None, limit=1000, cache=True): """Query the archive database for details on available files. - Paramters - --------- + Parameters + ---------- qspec : :class:`dict`, optional The query that will be passed to the API. limit : :class:`int`, optional diff --git a/docs/noirlab/noirlab.rst b/docs/noirlab/noirlab.rst index e05ecf1e2b..654d6e8e0f 100644 --- a/docs/noirlab/noirlab.rst +++ b/docs/noirlab/noirlab.rst @@ -6,26 +6,25 @@ About the NSF NOIRLab Astro Data Archive **************************************** -The NSF NOIRLab Astro Data Archive (formerly NOAO Science Archive) -provides access to data taken with more than 40 telescope and -instrument combinations, including those operated in partnership with -the WIYN, SOAR and SMARTS consortia, from semester 2004B to the -present. In addition to raw data, pipeline-reduced data products from -the DECam, Mosaic and NEWFIRM imagers are also available, as well as -advanced data products delivered by teams carrying out surveys and -other large observing programs with NSF OIR Lab facilities. +The NOIRLab Astro Data Archive (formerly NOAO Science Archive) provides +access to data taken with more than 40 telescope and instrument combinations, +including those operated in partnership with the WIYN, SOAR and SMARTS +consortia, from semester 2004B to the present. In addition to raw data, +pipeline-reduced data products from the DECam, Mosaic and NEWFIRM imagers +are also available, as well as advanced data products delivered by teams +carrying out surveys and other large observing programs with NSF NOIRLab +facilities. For more info about our holdings see the -`NSF NOIRLab Astro Data Archive `_ +`NSF NOIRLab Astro Data Archive `_. Acknowledgment ============== -This research uses services or data provided by the Astro Data Archive -at NSF's National Optical-Infrared Astronomy Research -Laboratory. NSF's OIR Lab is operated by the Association of -Universities for Research in Astronomy (AURA), Inc. under a -cooperative agreement with the National Science Foundation. +This research uses services or data provided by the Astro Data Archive at +NSF's NOIRLab. NOIRLab is operated by the Association of Universities for +Research in Astronomy (AURA), Inc. under a cooperative agreement with the +National Science Foundation. ************************************** NOIRLab Queries (`astroquery.noirlab`) @@ -33,7 +32,7 @@ NOIRLab Queries (`astroquery.noirlab`) The methods in this module are wrappers around a set of web services described in the -`Rest API documentation `_. +`REST API documentation `_. This data archive is hosted at the `Community Science and Data Center (CDSC) `_. @@ -110,44 +109,44 @@ This is an example of searching by HDU. Advanced Search =============== -This set of methods supports **arbitrary searches of any fields** +This set of methods supports *arbitrary searches of any fields* stored in the FITS headers of the Archive. Common fields ("core" fields) are optimized for search speed. Less common fields ("aux" fields) will be slower to search. You can search by File or HDU. The -primary method for doing the search in ``query_metadata``. That query -requires a ``JSON`` structure to define the query. We often call this -the *JSON search spec*. Many of the other -methods with this module are here to provide you with the information -you need to construct the ``JSON`` structure. +primary method for doing the search is +:meth:`~astroquery.noirlab.NOIRLabClass.query_metadata`. That query +requires a JSON_ structure to define the query. We often call this +the *JSON search spec*. Additional methods in this module +provide information needed to construct the JSON_ structure. Summaries of the mechanisms available in the JSON search spec for `File search `_ and for `HDU search `_ are on the NSF NOIRLab Data Archive website. -There are three methods whose sole purpose if providing you with -information to help you with the content of your ``JSON`` structure. -They are: +These methods provide information needed to fill in a JSON_ query structure: -#. aux_fields() -#. core_fields() -#. categoricals() +#. :meth:`~astroquery.noirlab.NOIRLabClass.aux_fields` +#. :meth:`~astroquery.noirlab.NOIRLabClass.core_fields` +#. :meth:`~astroquery.noirlab.NOIRLabClass.categoricals` -See the Reference/API below for details. The categoricals() method +See the Reference/API below for details. The +:meth:`~astroquery.noirlab.NOIRLabClass.categoricals` method returns a list of all the "category strings" such as names of -Instruments and Telescopes. The aux/core_fields methods +Instruments and Telescopes. The :meth:`~astroquery.noirlab.NOIRLabClass.aux_fields` +and :meth:`~astroquery.noirlab.NOIRLabClass.core_fields` methods tell you what fields are available to search. The core fields are -available for all instruments are the search for them is fast. The aux -fields require you to specify instrument and proctype. The set of -fields available is highly dependent on those two fields. The -Instrument determines aux fields in raw files. Proctype determines +available for all instruments; searching on these fields is optimized +for speed. The aux fields require you to specify instrument and proctype. +The set of aux fields available is highly dependent on those two fields. The +instrument determines aux fields in raw files. Proctype determines what kind of pipeline processing was done. Pipeline processing often -adds important (aux) fields. +adds important additional aux fields. +.. _JSON: https://www.json.org/json-en.html Reference/API ============= - .. automodapi:: astroquery.noirlab :no-inheritance-diagram: From 29b2437228fa0d5225d4ccd63063b364875aced4 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 17 Jul 2025 12:32:58 -0700 Subject: [PATCH 23/26] activate hdu metadata tests --- astroquery/noirlab/core.py | 16 ++- astroquery/noirlab/tests/expected.py | 102 ++++++++++++------ astroquery/noirlab/tests/test_noirlab.py | 31 +++--- .../noirlab/tests/test_noirlab_remote.py | 35 +++--- 4 files changed, 109 insertions(+), 75 deletions(-) diff --git a/astroquery/noirlab/core.py b/astroquery/noirlab/core.py index f04e82d20e..b570897287 100644 --- a/astroquery/noirlab/core.py +++ b/astroquery/noirlab/core.py @@ -234,13 +234,20 @@ def categoricals(self, cache=True): return response.json() @class_or_instance - def query_metadata(self, qspec=None, limit=1000, cache=True): + def query_metadata(self, qspec=None, sort=None, limit=1000, cache=True): """Query the archive database for details on available files. + `qspec` should minimally contain a list of output columns and a list of + search parameters, which could be empty. For example:: + + qspec = {"outfields": ["md5sum", ], "search": []} + Parameters ---------- qspec : :class:`dict`, optional The query that will be passed to the API. + sort : :class:`str`, optional + Sort the results on one of the columns in `qspec`. limit : :class:`int`, optional The number of results to return, default 1000. cache : :class:`bool`, optional @@ -253,6 +260,9 @@ def query_metadata(self, qspec=None, limit=1000, cache=True): """ self._validate_version() url = f'{self._adss_url}&limit={limit}' + if sort: + # TODO: write a test for this, which may involve refactoring async versus sync. + url += f'&sort={sort}' if qspec is None: jdata = {"outfields": ["md5sum", ], "search": []} @@ -264,7 +274,9 @@ def query_metadata(self, qspec=None, limit=1000, cache=True): response.raise_for_status() j = response.json() # The first entry contains metadata. - return astropy.table.Table(rows=j[1:]) + names = list(j[0]['HEADER'].keys()) + rows = [[row[n] for n in names] for row in j[1:]] + return astropy.table.Table(names=names, rows=rows) def retrieve(self, fileid): """Simply fetch a file by MD5 ID. diff --git a/astroquery/noirlab/tests/expected.py b/astroquery/noirlab/tests/expected.py index 337bcd15f4..1204f2887d 100644 --- a/astroquery/noirlab/tests/expected.py +++ b/astroquery/noirlab/tests/expected.py @@ -41,19 +41,19 @@ {'Field': 'AZ', 'Type': 'str', 'Desc': ''}, {'Field': 'BAND', 'Type': 'str', 'Desc': ''}] -query_file_metadata = ['proc_type original_filename md5sum instrument archive_filename ', - '--------- ------------------------------------------------------------------------------------- -------------------------------- ---------- -----------------------------------------------------------------------------', - ' instcal /net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_161005_063408_ood_r_v2.fits.fz bd70774014d8c32d0fc8f9933dcc5785 decam /net/archive/pipe/20161004/ct4m/2012B-0001/c4d_161005_063408_ood_r_v2.fits.fz', - 'resampled /net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_180125_005333_opd_i_v1.fits.fz 39a5e8c1d0c1ee721f1a753367f5f7c5 decam /net/archive/pipe/20180124/ct4m/2012B-0001/c4d_180125_005333_opd_i_v1.fits.fz', - 'resampled /net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_160930_070518_opi_r_v2.fits.fz 9860ceed19ac461a5435055e68060c6c decam /net/archive/pipe/20160929/ct4m/2012B-0001/c4d_160930_070518_opi_r_v2.fits.fz'] +query_file_metadata = [' md5sum archive_filename original_filename instrument proc_type', + '-------------------------------- ----------------------------------------------------------------------------- ------------------------------------------------------------------------------------- ---------- ---------', + '0000f0c5015263610a75f0affed377d0 /net/archive/pipe/20161024/ct4m/2012B-0001/c4d_161025_034649_oki_z_v2.fits.fz /net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_161025_034649_oki_z_v2.fits.fz decam skysub', + '0001e5a5bcf039ebf0c53a6da32e8888 /net/archive/pipe/20160928/ct4m/2012B-0001/c4d_160929_090838_ooi_z_v2.fits.fz /net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_160929_090838_ooi_z_v2.fits.fz decam instcal', + '0003a1c853de73fc1796d2d7d77ca9cd /net/archive/pipe/20160909/ct4m/2012B-0001/c4d_160910_062930_opd_z_v2.fits.fz /net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_160910_062930_opd_z_v2.fits.fz decam resampled'] query_file_metadata_minimal = [' md5sum ', '--------------------------------', - '97f18cde19976fe167484003f1aca94c', - '09325eb851e81df518042af1d882b9c5', - 'd4bf24fdfa22385ce748304dceee0d97', - '9d81bef7e7073878e0aa33e63ac8a8c9', - '4938fe8591ea4ac88332bbf2e6e2cb11'] + '0000004ab27d9e427bb93c640b358633', + '0000032cfbe72cc162eaec4c0a9ce6ec', + '0000041587917c9cb234c6116b396394', + '000005365aac087b505ce636c8ca573a', + '0000065ee227de201686b54706ba58e9'] query_file_meta_raw = [{'META': {'endpoint': 'adv_search/find'}, 'PARAMETERS': {'rectype': 'file', @@ -66,36 +66,36 @@ 'last': 3, 'json_payload': {'outfields': ['md5sum', 'archive_filename', 'original_filename', 'instrument', 'proc_type'], 'search': [['original_filename', 'c4d_', 'contains']]}}, - 'HEADER': {'proc_type': 'category', + 'HEADER': {'md5sum': 'str', + 'archive_filename': 'str', 'original_filename': 'str', - 'md5sum': 'str', 'instrument': 'category', - 'archive_filename': 'str'}}, - {'proc_type': 'instcal', - 'original_filename': '/net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_161005_063408_ood_r_v2.fits.fz', - 'md5sum': 'bd70774014d8c32d0fc8f9933dcc5785', + 'proc_type': 'category'}}, + {'proc_type': 'skysub', + 'original_filename': '/net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_161025_034649_oki_z_v2.fits.fz', + 'md5sum': '0000f0c5015263610a75f0affed377d0', 'instrument': 'decam', - 'archive_filename': '/net/archive/pipe/20161004/ct4m/2012B-0001/c4d_161005_063408_ood_r_v2.fits.fz'}, - {'proc_type': 'resampled', - 'original_filename': '/net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_180125_005333_opd_i_v1.fits.fz', - 'md5sum': '39a5e8c1d0c1ee721f1a753367f5f7c5', + 'archive_filename': '/net/archive/pipe/20161024/ct4m/2012B-0001/c4d_161025_034649_oki_z_v2.fits.fz'}, + {'proc_type': 'instcal', + 'original_filename': '/net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_160929_090838_ooi_z_v2.fits.fz', + 'md5sum': '0001e5a5bcf039ebf0c53a6da32e8888', 'instrument': 'decam', - 'archive_filename': '/net/archive/pipe/20180124/ct4m/2012B-0001/c4d_180125_005333_opd_i_v1.fits.fz'}, + 'archive_filename': '/net/archive/pipe/20160928/ct4m/2012B-0001/c4d_160929_090838_ooi_z_v2.fits.fz'}, {'proc_type': 'resampled', - 'original_filename': '/net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_160930_070518_opi_r_v2.fits.fz', - 'md5sum': '9860ceed19ac461a5435055e68060c6c', + 'original_filename': '/net/decdata1/deccp/NHPPS_DATA/DCP/Final/Submitted/c4d_160910_062930_opd_z_v2.fits.fz', + 'md5sum': '0003a1c853de73fc1796d2d7d77ca9cd', 'instrument': 'decam', - 'archive_filename': '/net/archive/pipe/20160929/ct4m/2012B-0001/c4d_160930_070518_opi_r_v2.fits.fz'}] + 'archive_filename': '/net/archive/pipe/20160909/ct4m/2012B-0001/c4d_160910_062930_opd_z_v2.fits.fz'}] query_file_meta_raw_minimal = [{'META': {'endpoint': 'adv_search/find'}, 'PARAMETERS': {'rectype': 'file', 'limit': 5}, 'HEADER': {'md5sum': 'str'}}, - {'md5sum': '97f18cde19976fe167484003f1aca94c',}, - {'md5sum': '09325eb851e81df518042af1d882b9c5',}, - {'md5sum': 'd4bf24fdfa22385ce748304dceee0d97',}, - {'md5sum': '9d81bef7e7073878e0aa33e63ac8a8c9',}, - {'md5sum': '4938fe8591ea4ac88332bbf2e6e2cb11',}] + {'md5sum': '0000004ab27d9e427bb93c640b358633',}, + {'md5sum': '0000032cfbe72cc162eaec4c0a9ce6ec',}, + {'md5sum': '0000041587917c9cb234c6116b396394',}, + {'md5sum': '000005365aac087b505ce636c8ca573a',}, + {'md5sum': '0000065ee227de201686b54706ba58e9',}] core_hdu_fields = [{'Field': 'dec_center', 'Type': 'np.float64', 'Desc': 'DEC center from pipeline processing'}, {'Field': 'dec_max', 'Type': 'np.float64', 'Desc': 'DEC min,max range of HDU'}, @@ -119,11 +119,45 @@ {'Field': 'CCDNUM', 'Type': 'str', 'Desc': ''}, {'Field': 'CD1_1', 'Type': 'str', 'Desc': 'Coordinate matrix'}] -query_hdu_metadata = ['AIRMASS fitsfile__archive_filename fitsfile__caldat fitsfile__instrument fitsfile__proc_type', - '------- ----------------------------------------------------------------------- ---------------- -------------------- -------------------', - ' None /net/archive/mtn/20170814/ct4m/2012B-0001/c4d_170815_051354_ori.fits.fz 2017-08-14 decam raw', - ' None /net/archive/mtn/20170814/ct4m/2012B-0001/c4d_170815_051354_ori.fits.fz 2017-08-14 decam raw', - ' None /net/archive/mtn/20170814/ct4m/2012B-0001/c4d_170815_051354_ori.fits.fz 2017-08-14 decam raw'] +query_hdu_metadata = ['EXPTIME md5sum caldat archive_filename hdu:EQUINOX instrument proc_type AIRMASS', + '------- -------------------------------- ---------- ----------------------------------------------------------------------- ----------- ---------- --------- -------', + ' 90.0 004a631d6f5493e8adeb7123255380cb 2017-08-15 /net/archive/mtn/20170815/ct4m/2017B-0110/c4d_170816_095757_ori.fits.fz 2000.0 decam raw 1.04', + ' 90.0 004a631d6f5493e8adeb7123255380cb 2017-08-15 /net/archive/mtn/20170815/ct4m/2017B-0110/c4d_170816_095757_ori.fits.fz 2000.0 decam raw 1.04', + ' 90.0 004a631d6f5493e8adeb7123255380cb 2017-08-15 /net/archive/mtn/20170815/ct4m/2017B-0110/c4d_170816_095757_ori.fits.fz 2000.0 decam raw 1.04'] + +query_hdu_metadata_raw = [{'META': {'endpoint': 'adv_search/find'}, + 'PARAMETERS': {'rectype': 'hdu', + 'limit': 3, + 'sort': 'md5sum', + 'default_limit': 1000, + 'default_offset': 0, + 'default_sort': 'fitsfile__md5sum,hdu_idx', + 'oldest': None, + 'previd': None, + 'last': 3, + 'json_payload': {'outfields': ['md5sum', 'archive_filename', 'caldat', 'instrument', 'proc_type', 'AIRMASS', 'EXPTIME'], + 'search': [['instrument', 'decam'], ['proc_type', 'raw'], ['caldat', '2017-08-14', '2017-08-16']]}}, + 'HEADER': {'EXPTIME': 'str', 'md5sum': 'str', 'caldat': 'datetime64', 'archive_filename': 'str', 'instrument': 'category', 'proc_type': 'category', 'AIRMASS': 'np.float64'}}, + {'EXPTIME': 90.0, 'md5sum': '004a631d6f5493e8adeb7123255380cb', 'caldat': '2017-08-15', 'archive_filename': '/net/archive/mtn/20170815/ct4m/2017B-0110/c4d_170816_095757_ori.fits.fz', 'instrument': 'decam', 'proc_type': 'raw', 'AIRMASS': 1.04}, + {'EXPTIME': 90.0, 'md5sum': '004a631d6f5493e8adeb7123255380cb', 'caldat': '2017-08-15', 'archive_filename': '/net/archive/mtn/20170815/ct4m/2017B-0110/c4d_170816_095757_ori.fits.fz', 'instrument': 'decam', 'proc_type': 'raw', 'AIRMASS': 1.04}, + {'EXPTIME': 90.0, 'md5sum': '004a631d6f5493e8adeb7123255380cb', 'caldat': '2017-08-15', 'archive_filename': '/net/archive/mtn/20170815/ct4m/2017B-0110/c4d_170816_095757_ori.fits.fz', 'instrument': 'decam', 'proc_type': 'raw', 'AIRMASS': 1.04}] + +query_hdu_metadata_raw = [{'META': {'endpoint': 'adv_search/find'}, + 'PARAMETERS': {'rectype': 'hdu', + 'limit': 3, + 'sort': 'md5sum', + 'default_limit': 1000, + 'default_offset': 0, + 'default_sort': 'fitsfile__md5sum,hdu_idx', + 'oldest': None, + 'previd': None, + 'last': 3, + 'json_payload': {'outfields': ['md5sum', 'archive_filename', 'caldat', 'instrument', 'proc_type', 'AIRMASS', 'EXPTIME', 'hdu:EQUINOX'], + 'search': [['instrument', 'decam'], ['proc_type', 'raw'], ['caldat', '2017-08-14', '2017-08-16']]}}, + 'HEADER': {'EXPTIME': 'str', 'md5sum': 'str', 'caldat': 'datetime64', 'archive_filename': 'str', 'hdu:EQUINOX': 'str', 'instrument': 'category', 'proc_type': 'category', 'AIRMASS': 'np.float64'}}, + {'hdu:EQUINOX': 2000.0, 'EXPTIME': 90.0, 'md5sum': '004a631d6f5493e8adeb7123255380cb', 'caldat': '2017-08-15', 'archive_filename': '/net/archive/mtn/20170815/ct4m/2017B-0110/c4d_170816_095757_ori.fits.fz', 'instrument': 'decam', 'proc_type': 'raw', 'AIRMASS': 1.04}, + {'hdu:EQUINOX': 2000.0, 'EXPTIME': 90.0, 'md5sum': '004a631d6f5493e8adeb7123255380cb', 'caldat': '2017-08-15', 'archive_filename': '/net/archive/mtn/20170815/ct4m/2017B-0110/c4d_170816_095757_ori.fits.fz', 'instrument': 'decam', 'proc_type': 'raw', 'AIRMASS': 1.04}, + {'hdu:EQUINOX': 2000.0, 'EXPTIME': 90.0, 'md5sum': '004a631d6f5493e8adeb7123255380cb', 'caldat': '2017-08-15', 'archive_filename': '/net/archive/mtn/20170815/ct4m/2017B-0110/c4d_170816_095757_ori.fits.fz', 'instrument': 'decam', 'proc_type': 'raw', 'AIRMASS': 1.04}] categoricals = {'instruments': ['(p)odi', '90prime', diff --git a/astroquery/noirlab/tests/test_noirlab.py b/astroquery/noirlab/tests/test_noirlab.py index fcef147e1c..db198d090b 100644 --- a/astroquery/noirlab/tests/test_noirlab.py +++ b/astroquery/noirlab/tests/test_noirlab.py @@ -26,6 +26,8 @@ def mock_content(method, url, **kwargs): content = json.dumps(exp.query_file_meta_raw) elif '/find/?rectype=file&limit=5' in url: content = json.dumps(exp.query_file_meta_raw_minimal) + elif '/find/?rectype=hdu&limit=3' in url: + content = json.dumps(exp.query_hdu_metadata_raw) elif '/core_hdu_fields' in url: content = json.dumps(exp.core_hdu_fields) elif '/aux_hdu_fields' in url: @@ -142,30 +144,23 @@ def test_aux_hdu_fields(patch_request): assert actual == exp.aux_hdu_fields -@pytest.mark.skip(reason='WIP') def test_query_hdu_metadata(patch_request): """Search HDU metadata. """ - qspec = {"outfields": ["fitsfile__archive_filename", - "fitsfile__caldat", - "fitsfile__instrument", - "fitsfile__proc_type", - "AIRMASS"], # AUX field. Slows search - "search": [["fitsfile__caldat", "2017-08-14", "2017-08-16"], - ["fitsfile__instrument", "decam"], - ["fitsfile__proc_type", "raw"]]} - actual = NOIRLabHDU.query_metadata(qspec, limit=3) + qspec = {"outfields": ["md5sum", + "archive_filename", + "caldat", + "instrument", + "proc_type", + "EXPTIME", + "AIRMASS"], + "search": [["caldat", "2017-08-14", "2017-08-16"], + ["instrument", "decam"], + ["proc_type", "raw"]]} + actual = NOIRLabHDU.query_metadata(qspec, sort='md5sum', limit=3) assert actual.pformat(max_width=-1) == exp.query_hdu_metadata -@pytest.mark.skip(reason='WIP') -def test_query_hdu_metadata_minimal_input(patch_request): - """Search HDU metadata with minimum input parameters. - """ - actual = NOIRLabHDU.query_metadata(qspec=None, limit=3) - assert actual.pformat(max_width=-1) == exp.query_hdu_metadata_minimal - - def test_categoricals(patch_request): """List categories. """ diff --git a/astroquery/noirlab/tests/test_noirlab_remote.py b/astroquery/noirlab/tests/test_noirlab_remote.py index 274628efb6..772cffa95b 100644 --- a/astroquery/noirlab/tests/test_noirlab_remote.py +++ b/astroquery/noirlab/tests/test_noirlab_remote.py @@ -97,7 +97,7 @@ def test_query_file_metadata(): "instrument", "proc_type"], "search": [['original_filename', 'c4d_', 'contains']]} - actual = NOIRLab().query_metadata(qspec, limit=3) + actual = NOIRLab().query_metadata(qspec, sort='md5sum', limit=3) assert actual.pformat(max_width=-1) == exp.query_file_metadata @@ -105,7 +105,7 @@ def test_query_file_metadata(): def test_query_file_metadata_minimal_input(): """Search FILE metadata with minimum input parameters. """ - actual = NOIRLab().query_metadata(qspec=None, limit=5) + actual = NOIRLab().query_metadata(qspec=None, sort='md5sum', limit=5) assert actual.pformat(max_width=-1) == exp.query_file_metadata_minimal @@ -125,29 +125,22 @@ def test_aux_hdu_fields(): assert actual[:10] == exp.aux_hdu_fields -@pytest.mark.skip(reason='old API') -@pytest.mark.remote_data -def test_query_hdu_metadata_minimal_input(): - """Search HDU metadata with minimum input parameters. - """ - actual = NOIRLabClass(hdu=True).query_metadata(qspec=None, limit=5) - assert actual.pformat(max_width=-1) == exp.query_hdu_metadata_minimal - - -@pytest.mark.skip(reason='old API') @pytest.mark.remote_data def test_query_hdu_metadata(): """Search HDU metadata. """ - qspec = {"outfields": ["fitsfile__archive_filename", - "fitsfile__caldat", - "fitsfile__instrument", - "fitsfile__proc_type", - "AIRMASS"], # AUX field. Slows search - "search": [["fitsfile__caldat", "2017-08-14", "2017-08-16"], - ["fitsfile__instrument", "decam"], - ["fitsfile__proc_type", "raw"]]} - actual = NOIRLabClass(hdu=True).query_metadata(qspec, limit=3) + qspec = {"outfields": ["md5sum", + "archive_filename", + "caldat", + "instrument", + "proc_type", + "EXPTIME", + "AIRMASS", + "hdu:EQUINOX"], + "search": [["caldat", "2017-08-14", "2017-08-16"], + ["instrument", "decam"], + ["proc_type", "raw"]]} + actual = NOIRLabClass(hdu=True).query_metadata(qspec, sort='md5sum', limit=3) assert actual.pformat(max_width=-1) == exp.query_hdu_metadata From 3894c1680fd1af06b2009fc090037185a5946d2c Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 17 Jul 2025 16:21:11 -0700 Subject: [PATCH 24/26] all tests now active --- astroquery/noirlab/core.py | 150 +++++++++++------- astroquery/noirlab/tests/test_noirlab.py | 102 +++++------- .../noirlab/tests/test_noirlab_remote.py | 93 ++++------- docs/noirlab/noirlab.rst | 86 ++++++---- 4 files changed, 211 insertions(+), 220 deletions(-) diff --git a/astroquery/noirlab/core.py b/astroquery/noirlab/core.py index b570897287..672e4516dc 100644 --- a/astroquery/noirlab/core.py +++ b/astroquery/noirlab/core.py @@ -8,7 +8,7 @@ from ..query import BaseQuery from ..utils import async_to_sync from ..exceptions import RemoteServiceError -from ..utils.class_or_instance import class_or_instance +# from ..utils.class_or_instance import class_or_instance from . import conf @@ -18,32 +18,12 @@ @async_to_sync class NOIRLabClass(BaseQuery): """Search functionality for the NSF NOIRLab Astro Data Archive. - - Parameters - ---------- - hdu : :class:`bool`, optional - If ``True``, search HDUs in files. THe HDUs must have RA, DEC header - keywords. This is not guaranteed for all files. - The default is to just search for files. """ TIMEOUT = conf.timeout NAT_URL = conf.server - def __init__(self, hdu=False): + def __init__(self): self._api_version = None - self._adsurl = f'{self.NAT_URL}/api/adv_search' - - if hdu: - self.siaurl = f'{self.NAT_URL}/api/sia/vohdu' - self._adss_url = f'{self._adsurl}/find/?rectype=hdu' - self._adsc_url = f'{self._adsurl}/core_hdu_fields' - self._adsa_url = f'{self._adsurl}/aux_hdu_fields' - else: - self.siaurl = f'{self.NAT_URL}/api/sia/voimg' - self._adss_url = f'{self._adsurl}/find/?rectype=file' - self._adsc_url = f'{self._adsurl}/core_file_fields' - self._adsa_url = f'{self._adsurl}/aux_file_fields' - super().__init__() @property @@ -69,7 +49,60 @@ def _validate_version(self): f'{self.api_version} from the API.') raise RemoteServiceError(msg) - def service_metadata(self, cache=True): + def _sia_url(self, hdu=False): + """Return the URL for SIA queries. + + Parameters + ---------- + hdu : :class:`bool`, optional + If ``True`` return the URL for HDU-based queries. + + Returns + ------- + :class:`str` + The query URL. + """ + return f'{self.NAT_URL}/api/sia/vohdu' if hdu else f'{self.NAT_URL}/api/sia/voimg' + + def _fields_url(self, hdu=False, aux=False): + """Return the URL for metadata queries. + + Parameters + ---------- + hdu : :class:`bool`, optional + If ``True`` return the URL for HDU-based queries. + aux : :class:`bool`, optional + If ``True`` return metadata on AUX fields. + + Returns + ------- + :class:`str` + The query URL. + """ + file = 'hdu' if hdu else 'file' + core = 'aux' if aux else 'core' + return f'{self.NAT_URL}/api/adv_search/{core}_{file}_fields' + + def _response_to_table(self, response_json): + """Convert a JSON response to a :class:`~astropy.table.Table`. + + Parameters + ---------- + response_json : :class:`list` + A query response formatted as a list of objects. The query + metadata is the first item in the list. + + Returns + ------- + :class:`~astropy.table.Table` + The converted response. The column ordering will match the + ordering of the `HEADER` metadata. + """ + names = list(response_json[0]['HEADER'].keys()) + rows = [[row[n] for n in names] for row in response_json[1:]] + return astropy.table.Table(names=names, rows=rows) + + def service_metadata(self, hdu=False, cache=True): """A SIA metadata query: no images are requested; only metadata should be returned. @@ -78,6 +111,8 @@ def service_metadata(self, cache=True): Parameters ---------- + hdu : :class:`bool`, optional + If ``True`` return the URL for HDU-based queries. cache : :class:`bool`, optional If ``True`` cache the result locally. @@ -86,12 +121,11 @@ def service_metadata(self, cache=True): :class:`dict` A dictionary containing SIA metadata. """ - url = f'{self.siaurl}?FORMAT=METADATA&format=json' + url = f'{self._sia_url(hdu=hdu)}?FORMAT=METADATA&format=json' response = self._request('GET', url, timeout=self.TIMEOUT, cache=cache) - return response.json()[0] + return response.json() - @class_or_instance - def query_region(self, coordinate, radius=0.1, cache=True): + def query_region(self, coordinate, radius=0.1, hdu=False, cache=True): """Query for NOIRLab observations by region of the sky. Given a sky coordinate and radius, returns a `~astropy.table.Table` @@ -99,7 +133,7 @@ def query_region(self, coordinate, radius=0.1, cache=True): Parameters ---------- - coordinates : :class:`str` or `~astropy.coordinates` object + coordinate : :class:`str` or `~astropy.coordinates` object The target region which to search. It may be specified as a string or as the appropriate `~astropy.coordinates` object. radius : :class:`str` or `~astropy.units.Quantity` object, optional @@ -107,6 +141,8 @@ def query_region(self, coordinate, radius=0.1, cache=True): The string must be parsable by `~astropy.coordinates.Angle`. The appropriate `~astropy.units.Quantity` object from `~astropy.units` may also be used. + hdu : :class:`bool`, optional + If ``True`` return the URL for HDU-based queries. cache : :class:`bool`, optional If ``True`` cache the result locally. @@ -115,17 +151,11 @@ def query_region(self, coordinate, radius=0.1, cache=True): :class:`~astropy.table.Table` A table containing the results. """ - self._validate_version() - ra, dec = coordinate.to_string('decimal').split() - url = f'{self.siaurl}?POS={ra},{dec}&SIZE={radius}&format=json' - response = self._request('GET', url, - timeout=self.TIMEOUT, - cache=cache) + response = self.query_region_async(coordinate, radius=radius, hdu=hdu, cache=cache) response.raise_for_status() - # return astropy.table.Table(data=response.json()) - return response.json() + return self._response_to_table(response.json()) - def query_region_async(self, coordinate, radius=0.1, cache=True): + def query_region_async(self, coordinate, radius=0.1, hdu=False, cache=True): """Query for NOIRLab observations by region of the sky. Given a sky coordinate and radius, returns a `~astropy.table.Table` @@ -133,7 +163,7 @@ def query_region_async(self, coordinate, radius=0.1, cache=True): Parameters ---------- - coordinates : :class:`str` or `~astropy.coordinates` object + coordinate : :class:`str` or `~astropy.coordinates` object The target region which to search. It may be specified as a string or as the appropriate `~astropy.coordinates` object. radius : :class:`str` or `~astropy.units.Quantity` object, optional @@ -141,6 +171,8 @@ def query_region_async(self, coordinate, radius=0.1, cache=True): The string must be parsable by `~astropy.coordinates.Angle`. The appropriate `~astropy.units.Quantity` object from `~astropy.units` may also be used. + hdu : :class:`bool`, optional + If ``True`` return the URL for HDU-based queries. cache : :class:`bool`, optional If ``True`` cache the result locally. @@ -150,22 +182,21 @@ def query_region_async(self, coordinate, radius=0.1, cache=True): Response object. """ self._validate_version() - ra, dec = coordinate.to_string('decimal').split() - url = f'{self.siaurl}?POS={ra},{dec}&SIZE={radius}&format=json' - response = self._request('GET', url, - timeout=self.TIMEOUT, - cache=cache) - response.raise_for_status() + url = f'{self._sia_url(hdu=hdu)}?POS={ra},{dec}&SIZE={radius}&VERB=3&format=json' + response = self._request('GET', url, timeout=self.TIMEOUT, cache=cache) + # response.raise_for_status() return response - def core_fields(self, cache=True): - """List the available CORE fields. + def core_fields(self, hdu=False, cache=True): + """List the available CORE fields for file or HDU searches. CORE fields are faster to search than AUX fields. Parameters ---------- + hdu : :class:`bool`, optional + If ``True`` return the fields for HDU-based queries. cache : :class:`bool`, optional If ``True`` cache the result locally. @@ -174,13 +205,12 @@ def core_fields(self, cache=True): :class:`list` A list of field descriptions, each a :class:`dict`. """ - response = self._request('GET', self._adsc_url, - timeout=self.TIMEOUT, - cache=cache) + url = self._fields_url(hdu=hdu, aux=False) + response = self._request('GET', url, timeout=self.TIMEOUT, cache=cache) response.raise_for_status() return response.json() - def aux_fields(self, instrument, proctype, cache=True): + def aux_fields(self, instrument, proctype, hdu=False, cache=True): """List the available AUX fields. AUX fields are any fields in the Archive FITS files that are not @@ -196,6 +226,8 @@ def aux_fields(self, instrument, proctype, cache=True): The specific instrument, *e.g.* '90prime' or 'decam'. proctype : :class:`str` A description of the type of image, *e.g.* 'raw' or 'instcal'. + hdu : :class:`bool`, optional + If ``True`` return the fields for HDU-based queries. cache : :class:`bool`, optional If ``True`` cache the result locally. @@ -204,7 +236,7 @@ def aux_fields(self, instrument, proctype, cache=True): :class:`list` A list of field descriptions, each a :class:`dict`. """ - url = f'{self._adsa_url}/{instrument}/{proctype}/' + url = f'{self._fields_url(hdu=hdu, aux=True)}/{instrument}/{proctype}/' response = self._request('GET', url, timeout=self.TIMEOUT, cache=cache) response.raise_for_status() return response.json() @@ -228,13 +260,12 @@ def categoricals(self, cache=True): :class:`dict` A dictionary containing the category metadata. """ - url = f'{self._adsurl}/cat_lists/?format=json' + url = f'{self.NAT_URL}/api/adv_search/cat_lists/?format=json' response = self._request('GET', url, timeout=self.TIMEOUT, cache=cache) response.raise_for_status() return response.json() - @class_or_instance - def query_metadata(self, qspec=None, sort=None, limit=1000, cache=True): + def query_metadata(self, qspec=None, sort=None, limit=1000, hdu=False, cache=True): """Query the archive database for details on available files. `qspec` should minimally contain a list of output columns and a list of @@ -250,6 +281,8 @@ def query_metadata(self, qspec=None, sort=None, limit=1000, cache=True): Sort the results on one of the columns in `qspec`. limit : :class:`int`, optional The number of results to return, default 1000. + hdu : :class:`bool`, optional + If ``True`` return the URL for HDU-based queries. cache : :class:`bool`, optional If ``True`` cache the result locally. @@ -259,7 +292,8 @@ def query_metadata(self, qspec=None, sort=None, limit=1000, cache=True): A Table containing the results. """ self._validate_version() - url = f'{self._adss_url}&limit={limit}' + file = 'hdu' if hdu else 'file' + url = f'{self.NAT_URL}/api/adv_search/find/?rectype={file}&limit={limit}' if sort: # TODO: write a test for this, which may involve refactoring async versus sync. url += f'&sort={sort}' @@ -272,11 +306,7 @@ def query_metadata(self, qspec=None, sort=None, limit=1000, cache=True): response = self._request('POST', url, json=jdata, timeout=self.TIMEOUT, cache=cache) response.raise_for_status() - j = response.json() - # The first entry contains metadata. - names = list(j[0]['HEADER'].keys()) - rows = [[row[n] for n in names] for row in j[1:]] - return astropy.table.Table(names=names, rows=rows) + return self._response_to_table(response.json()) def retrieve(self, fileid): """Simply fetch a file by MD5 ID. diff --git a/astroquery/noirlab/tests/test_noirlab.py b/astroquery/noirlab/tests/test_noirlab.py index db198d090b..b807ba40aa 100644 --- a/astroquery/noirlab/tests/test_noirlab.py +++ b/astroquery/noirlab/tests/test_noirlab.py @@ -8,16 +8,19 @@ from astropy.coordinates import SkyCoord from ...utils.mocks import MockResponse from ...exceptions import RemoteServiceError -from .. import NOIRLab, NOIRLabClass +from .. import NOIRLab from . import expected as exp -NOIRLabHDU = NOIRLabClass(hdu=True) - - def mock_content(method, url, **kwargs): if 'FORMAT=METADATA' in url: content = json.dumps(exp.service_metadata) + elif '/sia/voimg' in url: + raw_json = [{'HEADER': {'md5sum': 'str'}}] + [{'md5sum': m} for m in exp.query_region_1] + content = json.dumps(raw_json) + elif '/sia/vohdu' in url: + raw_json = [{'HEADER': {'md5sum': 'str'}}] + [{'md5sum': m} for m in exp.query_region_2] + content = json.dumps(raw_json) elif '/core_file_fields' in url: content = json.dumps(exp.core_file_fields) elif '/aux_file_fields' in url: @@ -46,68 +49,54 @@ def mock_content(method, url, **kwargs): @pytest.fixture def patch_request(monkeypatch): monkeypatch.setattr(NOIRLab, '_request', mock_content) - monkeypatch.setattr(NOIRLabHDU, '_request', mock_content) return monkeypatch -def test_service_metadata(patch_request): +@pytest.mark.parametrize('hdu', [(False,), (True,)]) +def test_service_metadata(patch_request, hdu): """Test compliance with 6.1 of SIA spec v1.0. """ - actual = NOIRLab.service_metadata() - assert actual == exp.service_metadata[0] - - -@pytest.mark.skip(reason='WIP') -def test_query_region_0(): - """Search FILES. - """ - c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') - r = NOIRLab.query_region(c, radius='0.1') - actual = set(list(r['md5sum'])) - expected = exp.query_region_1 - assert expected.issubset(actual) + actual = NOIRLab.service_metadata(hdu=hdu) + assert actual[0] == exp.service_metadata[0] -@pytest.mark.skip(reason='WIP') -def test_query_region_1(): - """Search FILES. +@pytest.mark.parametrize('hdu,radius', [(False, '0.1'), (True, '0.07')]) +def test_query_region(patch_request, hdu, radius): + """Search a region. Ensure query gets at least the set of files we expect. It is OK if more files have been added to the remote Archive. """ c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') - r = NOIRLab.query_region(c, radius='0.1') - actual = set(list(r['md5sum'])) - expected = exp.query_region_1 - assert expected.issubset(actual) - - -@pytest.mark.skip(reason='WIP') -def test_query_region_2(): - """Search HDUs. - - Ensure query gets at least the set of files we expect. - Its ok if more files have been added to the remote Archive. - """ - c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') - r = NOIRLabHDU.query_region(c, radius='0.07') - actual = set(list(r['md5sum'])) - expected = exp.query_region_2 + r = NOIRLab().query_region(c, radius=radius, hdu=hdu) + actual = set(r['md5sum'].tolist()) + if hdu: + expected = exp.query_region_2 + else: + expected = exp.query_region_1 assert expected.issubset(actual) -def test_core_file_fields(patch_request): - """List the available CORE FILE fields. +@pytest.mark.parametrize('hdu', [(False,), (True,)]) +def test_core_fields(patch_request, hdu): + """List the available CORE fields. """ - actual = NOIRLab.core_fields() - assert actual == exp.core_file_fields + actual = NOIRLab.core_fields(hdu=hdu) + if hdu: + assert actual == exp.core_hdu_fields + else: + assert actual == exp.core_file_fields -def test_aux_file_fields(patch_request): - """List the available AUX FILE fields. +@pytest.mark.parametrize('hdu', [(False,), (True,)]) +def test_aux_fields(patch_request, hdu): + """List the available AUX fields. """ - actual = NOIRLab.aux_fields('decam', 'instcal') - assert actual == exp.aux_file_fields + actual = NOIRLab.aux_fields('decam', 'instcal', hdu=hdu) + if hdu: + assert actual == exp.aux_hdu_fields + else: + assert actual == exp.aux_file_fields def test_query_file_metadata(patch_request): @@ -130,20 +119,6 @@ def test_query_file_metadata_minimal_input(patch_request): assert actual.pformat(max_width=-1) == exp.query_file_metadata_minimal -def test_core_hdu_fields(patch_request): - """List the available CORE HDU fields. - """ - actual = NOIRLabHDU.core_fields() - assert actual == exp.core_hdu_fields - - -def test_aux_hdu_fields(patch_request): - """List the available AUX HDU fields. - """ - actual = NOIRLabHDU.aux_fields('decam', 'instcal') - assert actual == exp.aux_hdu_fields - - def test_query_hdu_metadata(patch_request): """Search HDU metadata. """ @@ -153,11 +128,12 @@ def test_query_hdu_metadata(patch_request): "instrument", "proc_type", "EXPTIME", - "AIRMASS"], + "AIRMASS", + "hdu:EQUINOX"], "search": [["caldat", "2017-08-14", "2017-08-16"], ["instrument", "decam"], ["proc_type", "raw"]]} - actual = NOIRLabHDU.query_metadata(qspec, sort='md5sum', limit=3) + actual = NOIRLab.query_metadata(qspec, sort='md5sum', limit=3, hdu=True) assert actual.pformat(max_width=-1) == exp.query_hdu_metadata diff --git a/astroquery/noirlab/tests/test_noirlab_remote.py b/astroquery/noirlab/tests/test_noirlab_remote.py index 772cffa95b..30f773967b 100644 --- a/astroquery/noirlab/tests/test_noirlab_remote.py +++ b/astroquery/noirlab/tests/test_noirlab_remote.py @@ -9,74 +9,59 @@ import pytest from astropy import units as u from astropy.coordinates import SkyCoord -from .. import NOIRLab, NOIRLabClass +from .. import NOIRLab from . import expected as exp @pytest.mark.remote_data -def test_service_metadata(): +@pytest.mark.parametrize('hdu', [(False,), (True,)]) +def test_service_metadata(hdu): """Test compliance with 6.1 of SIA spec v1.0. """ - actual = NOIRLab().service_metadata() - assert actual == exp.service_metadata[0] + actual = NOIRLab().service_metadata(hdu=hdu) + assert actual[0] == exp.service_metadata[0] -@pytest.mark.skip(reason='old API') @pytest.mark.remote_data -def test_query_region_0(): - """Search FILES. - """ - c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') - r = NOIRLab().query_region(c, radius='0.1') - actual = set(list(r['md5sum'])) - expected = exp.query_region_1 - assert expected.issubset(actual) - - -@pytest.mark.skip(reason='old API') -@pytest.mark.remote_data -def test_query_region_1(): - """Search FILES. +@pytest.mark.parametrize('hdu,radius', [(False, '0.1'), (True, '0.07')]) +def test_query_region(hdu, radius): + """Search a region. Ensure query gets at least the set of files we expect. It is OK if more files have been added to the remote Archive. """ c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') - r = NOIRLabClass().query_region(c, radius='0.1') - actual = set(list(r['md5sum'])) - expected = exp.query_region_1 - assert expected.issubset(actual) - - -@pytest.mark.skip(reason='old API') -@pytest.mark.remote_data -def test_query_region_2(): - """Search HDUs. - - Ensure query gets at least the set of files we expect. - Its ok if more files have been added to the remote Archive. - """ - c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') - r = NOIRLabClass(hdu=True).query_region(c, radius='0.07') - actual = set(list(r['md5sum'])) - expected = exp.query_region_2 + r = NOIRLab().query_region(c, radius=radius, hdu=hdu) + actual = set(r['md5sum'].tolist()) + if hdu: + expected = exp.query_region_2 + else: + expected = exp.query_region_1 assert expected.issubset(actual) @pytest.mark.remote_data -def test_core_file_fields(): - """List the available CORE FILE fields. +@pytest.mark.parametrize('hdu', [(False,), (True,)]) +def test_core_fields(hdu): + """List the available CORE fields. """ - actual = NOIRLab().core_fields() - assert actual[:5] == exp.core_file_fields + actual = NOIRLab().core_fields(hdu=hdu) + if hdu: + assert actual == exp.core_hdu_fields + else: + assert actual[:5] == exp.core_file_fields @pytest.mark.remote_data -def test_aux_file_fields(): - """List the available AUX FILE fields. +@pytest.mark.parametrize('hdu', [(False,), (True,)]) +def test_aux_fields(hdu): + """List the available AUX fields. """ - actual = NOIRLab().aux_fields('decam', 'instcal') - assert actual[:10] == exp.aux_file_fields + actual = NOIRLab().aux_fields('decam', 'instcal', hdu=hdu) + if hdu: + assert actual[:10] == exp.aux_hdu_fields + else: + assert actual[:10] == exp.aux_file_fields @pytest.mark.remote_data @@ -109,22 +94,6 @@ def test_query_file_metadata_minimal_input(): assert actual.pformat(max_width=-1) == exp.query_file_metadata_minimal -@pytest.mark.remote_data -def test_core_hdu_fields(): - """List the available CORE HDU fields. - """ - actual = NOIRLabClass(hdu=True).core_fields() - assert actual == exp.core_hdu_fields - - -@pytest.mark.remote_data -def test_aux_hdu_fields(): - """List the available AUX HDU fields. - """ - actual = NOIRLabClass(hdu=True).aux_fields('decam', 'instcal') - assert actual[:10] == exp.aux_hdu_fields - - @pytest.mark.remote_data def test_query_hdu_metadata(): """Search HDU metadata. @@ -140,7 +109,7 @@ def test_query_hdu_metadata(): "search": [["caldat", "2017-08-14", "2017-08-16"], ["instrument", "decam"], ["proc_type", "raw"]]} - actual = NOIRLabClass(hdu=True).query_metadata(qspec, sort='md5sum', limit=3) + actual = NOIRLab().query_metadata(qspec, sort='md5sum', limit=3, hdu=True) assert actual.pformat(max_width=-1) == exp.query_hdu_metadata diff --git a/docs/noirlab/noirlab.rst b/docs/noirlab/noirlab.rst index 654d6e8e0f..12f544929a 100644 --- a/docs/noirlab/noirlab.rst +++ b/docs/noirlab/noirlab.rst @@ -1,5 +1,3 @@ -.. doctest-skip-all - .. _astroquery.noirlab: **************************************** @@ -60,50 +58,68 @@ only required argument to this is the target coordinates around which to query. Specify the coordinates using the appropriate coordinate system from `astropy.coordinates`. Here is a basic example: -.. code-block:: python +.. doctest-remote-data:: >>> from astroquery.noirlab import NOIRLab - >>> import astropy.coordinates as coord >>> from astropy import units as u >>> from astropy.coordinates import SkyCoord >>> coord = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') - - >>> noirlab_file = NOIRLab(which='file') - >>> results_file = noirlab_file.query_region(coord, radius='0.1') - >>> print(results_file) - archive_filename date_obs ... updated - str71 str10 ... str32 - ----------------------------------------------------------------------- ---------- ... -------------------------------- - string string ... string - /net/archive/mtn/20151027/kp4m/2015B-2001/k4m_151028_085849_ori.fits.fz 2015-10-28 ... 2020-02-09T01:17:17.842642+00:00 - /net/archive/mtn/20151027/kp4m/2015B-2001/k4m_151028_091327_ori.fits.fz 2015-10-28 ... 2020-02-09T01:17:17.875407+00:00 - /net/archive/mtn/20151027/kp4m/2015B-2001/k4m_151028_084112_ori.fits.fz 2015-10-28 ... 2020-02-09T01:17:18.303911+00:00 - /net/archive/mtn/20151120/kp4m/2015B-2001/k4m_151121_033641_ori.fits.fz 2015-11-21 ... 2020-02-09T01:24:35.525861+00:00 - /net/archive/mtn/20151120/kp4m/2015B-2001/k4m_151121_031258_ori.fits.fz 2015-11-21 ... 2020-02-09T01:24:37.873559+00:00 - /net/archive/mtn/20151120/kp4m/2015B-2001/k4m_151121_041031_ori.fits.fz 2015-11-21 ... 2020-02-09T01:24:38.951230+00:00 + >>> results_file = NOIRLab.query_region(coord, radius='0.1') + >>> mosaic_filter = results_file['instrument'] == 'mosaic3' + >>> print(results_file[mosaic_filter][['archive_filename', 'ra_center', 'dec_center']]) + archive_filename ra_center dec_center + ----------------------------------------------------------------------- ------------------ ----------------- + /net/archive/mtn/20151120/kp4m/2015B-2001/k4m_151121_041031_ori.fits.fz 10.579374999999999 41.19416666666666 + /net/archive/mtn/20151120/kp4m/2015B-2001/k4m_151121_031258_ori.fits.fz 10.579374999999999 41.19416666666666 + /net/archive/mtn/20151027/kp4m/2015B-2001/k4m_151028_085849_ori.fits.fz 10.672041666666665 41.24944166666667 + /net/archive/mtn/20151027/kp4m/2015B-2001/k4m_151028_084112_ori.fits.fz 10.672041666666665 41.24944166666667 + /net/archive/mtn/20151120/kp4m/2015B-2001/k4m_151121_033641_ori.fits.fz 10.579374999999999 41.19416666666666 + /net/archive/mtn/20151027/kp4m/2015B-2001/k4m_151028_091327_ori.fits.fz 10.672041666666665 41.24944166666667 This is an example of searching by HDU. + .. note:: Only some instruments have pipeline processing that populates the RA, DEC fields used for this search. -.. code-block:: python - - >>> from astroquery.noirlab import NOIRLabClass - >>> noirlab_hdu = NOIRLabClass(which='hdu') - >>> results_hdu = noirlab_hdu.query_region(coord, radius='0.1') - >>> print(results_hdu) - archive_filename caldat date_obs ... ra telescope - str79 str10 str10 ... str8 str6 - ------------------------------------------------------------------------------- ---------- ---------- ... -------- --------- - string string string ... float string - /net/archive/pipe/20180211/kp4m/2016A-0453/k4m_180212_021444_ooi_zd_ls9.fits.fz 2018-02-11 2018-02-12 ... 10.60048 kp4m - /net/archive/pipe/20180211/kp4m/2016A-0453/k4m_180212_030041_ooi_zd_v2.fits.fz 2018-02-11 2018-02-12 ... 10.43286 kp4m - /net/archive/pipe/20151120/kp4m/k4m_151121_031038_ooi_zd_v1.fits.fz 2015-11-20 2015-11-21 ... 10.58226 kp4m - /net/archive/pipe/20180211/kp4m/2016A-0453/k4m_180212_023526_ooi_zd_v2.fits.fz 2018-02-11 2018-02-12 ... 10.58187 kp4m - /net/archive/pipe/20151120/kp4m/k4m_151121_031646_ooi_zd_v1.fits.fz 2015-11-20 2015-11-21 ... 10.56906 kp4m - /net/archive/pipe/20151120/kp4m/k4m_151121_030945_ooi_zd_v1.fits.fz 2015-11-20 2015-11-21 ... 10.58203 kp4m - /net/archive/pipe/20151120/kp4m/k4m_151121_031124_ooi_zd_v1.fits.fz 2015-11-20 2015-11-21 ... 10.58549 kp4m +.. doctest-remote-data:: + + >>> results_hdu = NOIRLab.query_region(coord, radius='0.1', hdu=True) + >>> mosaic_hdu_filter = results_hdu['instrument'] == 'mosaic3' + >>> print(results_hdu[mosaic_hdu_filter][['archive_filename', 'hdu_idx']]) + archive_filename hdu_idx + ------------------------------------------------------------------------------- ------- + /net/archive/pipe/20180211/kp4m/2016A-0453/k4m_180212_030259_ooi_zd_v2.fits.fz 0 + /net/archive/pipe/20180211/kp4m/2016A-0453/k4m_180212_030259_ooi_zd_v2.fits.fz 1 + /net/archive/pipe/20180211/kp4m/2016A-0453/k4m_180212_021444_ooi_zd_ls9.fits.fz 2 + /net/archive/pipe/20151120/kp4m/2015B-2001/k4m_151121_031124_ooi_zd_v1.fits.fz 3 + /net/archive/pipe/20151120/kp4m/2015B-2001/k4m_151121_031819_ooi_zd_v1.fits.fz 0 + /net/archive/pipe/20151120/kp4m/2015B-2001/k4m_151121_031426_ooi_zd_v1.fits.fz 0 + /net/archive/pipe/20151120/kp4m/2015B-2001/k4m_151121_031426_ooi_zd_v1.fits.fz 1 + /net/archive/pipe/20151120/kp4m/2015B-2001/k4m_151121_031038_ooi_zd_v1.fits.fz 2 + /net/archive/pipe/20151120/kp4m/2015B-2001/k4m_151121_031038_ooi_zd_v1.fits.fz 3 + /net/archive/pipe/20151120/kp4m/2015B-2001/k4m_151121_031646_ooi_zd_v1.fits.fz 0 + /net/archive/pipe/20180211/kp4m/2016A-0453/k4m_180212_030259_ooi_zd_ls9.fits.fz 0 + /net/archive/pipe/20180211/kp4m/2016A-0453/k4m_180212_030259_ooi_zd_ls9.fits.fz 1 + /net/archive/pipe/20151120/kp4m/2015B-2001/k4m_151121_030855_ooi_zd_v1.fits.fz 2 + /net/archive/pipe/20151120/kp4m/2015B-2001/k4m_151121_031731_ooi_zd_v1.fits.fz 0 + /net/archive/pipe/20180211/kp4m/2016A-0453/k4m_180212_023526_ooi_zd_v2.fits.fz 1 + /net/archive/pipe/20180211/kp4m/2016A-0453/k4m_180212_030041_ooi_zd_ls9.fits.fz 2 + /net/archive/pipe/20180211/kp4m/2016A-0453/k4m_180212_030041_ooi_zd_ls9.fits.fz 3 + /net/archive/pipe/20180206/kp4m/2016A-0453/k4m_180207_024709_ooi_zd_v2.fits.fz 2 + /net/archive/pipe/20180211/kp4m/2016A-0453/k4m_180212_021444_ooi_zd_v2.fits.fz 2 + /net/archive/pipe/20151120/kp4m/2015B-2001/k4m_151121_031342_ooi_zd_v1.fits.fz 0 + /net/archive/pipe/20151120/kp4m/2015B-2001/k4m_151121_031342_ooi_zd_v1.fits.fz 1 + /net/archive/pipe/20180211/kp4m/2016A-0453/k4m_180212_030041_ooi_zd_v2.fits.fz 2 + /net/archive/pipe/20180211/kp4m/2016A-0453/k4m_180212_030041_ooi_zd_v2.fits.fz 3 + /net/archive/pipe/20151120/kp4m/2015B-2001/k4m_151121_030945_ooi_zd_v1.fits.fz 2 + /net/archive/pipe/20151120/kp4m/2015B-2001/k4m_151121_030809_ooi_zd_v1.fits.fz 2 + /net/archive/pipe/20180211/kp4m/2016A-0453/k4m_180212_023526_ooi_zd_ls9.fits.fz 1 + /net/archive/pipe/20151120/kp4m/2015B-2001/k4m_151121_031511_ooi_zd_v1.fits.fz 1 + /net/archive/pipe/20180206/kp4m/2016A-0453/k4m_180207_024709_ooi_zd_ls9.fits.fz 2 + /net/archive/pipe/20151120/kp4m/2015B-2001/k4m_151121_031904_ooi_zd_v1.fits.fz 0 + /net/archive/pipe/20151120/kp4m/2015B-2001/k4m_151121_030717_ooi_zd_v1.fits.fz 2 + /net/archive/pipe/20151120/kp4m/2015B-2001/k4m_151121_031558_ooi_zd_v1.fits.fz 1 Advanced Search From afa21af7c5e8822c96f8b240d3eab06e96157e22 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 17 Jul 2025 16:26:19 -0700 Subject: [PATCH 25/26] fix remote data error --- astroquery/noirlab/tests/test_noirlab.py | 2 +- docs/noirlab/noirlab.rst | 26 ++++++++++++------------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/astroquery/noirlab/tests/test_noirlab.py b/astroquery/noirlab/tests/test_noirlab.py index b807ba40aa..ef6bfa1b09 100644 --- a/astroquery/noirlab/tests/test_noirlab.py +++ b/astroquery/noirlab/tests/test_noirlab.py @@ -68,7 +68,7 @@ def test_query_region(patch_request, hdu, radius): It is OK if more files have been added to the remote Archive. """ c = SkyCoord(ra=10.625*u.degree, dec=41.2*u.degree, frame='icrs') - r = NOIRLab().query_region(c, radius=radius, hdu=hdu) + r = NOIRLab.query_region(c, radius=radius, hdu=hdu) actual = set(r['md5sum'].tolist()) if hdu: expected = exp.query_region_2 diff --git a/docs/noirlab/noirlab.rst b/docs/noirlab/noirlab.rst index 12f544929a..0e99f909aa 100644 --- a/docs/noirlab/noirlab.rst +++ b/docs/noirlab/noirlab.rst @@ -1,8 +1,18 @@ .. _astroquery.noirlab: -**************************************** +************************************** +NOIRLab Queries (`astroquery.noirlab`) +************************************** + +The methods in this module are wrappers around a set of web services +described in the +`REST API documentation `_. +This data archive is hosted at the +`Community Science and Data Center (CDSC) `_. + + About the NSF NOIRLab Astro Data Archive -**************************************** +======================================== The NOIRLab Astro Data Archive (formerly NOAO Science Archive) provides access to data taken with more than 40 telescope and instrument combinations, @@ -17,23 +27,13 @@ For more info about our holdings see the `NSF NOIRLab Astro Data Archive `_. Acknowledgment -============== +~~~~~~~~~~~~~~ This research uses services or data provided by the Astro Data Archive at NSF's NOIRLab. NOIRLab is operated by the Association of Universities for Research in Astronomy (AURA), Inc. under a cooperative agreement with the National Science Foundation. -************************************** -NOIRLab Queries (`astroquery.noirlab`) -************************************** - -The methods in this module are wrappers around a set of web services -described in the -`REST API documentation `_. -This data archive is hosted at the -`Community Science and Data Center (CDSC) `_. - Getting started (SIA) ===================== From 708dbb444d8c1e3743331c09bb3be8ba4d258128 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 17 Jul 2025 16:38:33 -0700 Subject: [PATCH 26/26] fix doc compilation errors --- astroquery/noirlab/__init__.py | 27 ++------------------------- astroquery/noirlab/core.py | 5 +++-- docs/noirlab/noirlab.rst | 3 +++ 3 files changed, 8 insertions(+), 27 deletions(-) diff --git a/astroquery/noirlab/__init__.py b/astroquery/noirlab/__init__.py index 88e248e09f..5061407a79 100644 --- a/astroquery/noirlab/__init__.py +++ b/astroquery/noirlab/__init__.py @@ -1,30 +1,7 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst """ -NSF NOIRLab Astro Data Archive -============================== - -Overview --------- - -The NOIRLab Astro Data Archive (formerly NOAO Science Archive) provides -access to data taken with more than 40 telescope and instrument combinations, -including those operated in partnership with the WIYN, SOAR and SMARTS -consortia, from semester 2004B to the present. In addition to raw data, -pipeline-reduced data products from the DECam, Mosaic and NEWFIRM imagers -are also available, as well as advanced data products delivered by teams -carrying out surveys and other large observing programs with NSF NOIRLab -facilities. - -For more info about our holdings see the -`NSF NOIRLab Astro Data Archive `_. - -Acknowledgment --------------- - -This research uses services or data provided by the Astro Data Archive at -NSF's NOIRLab. NOIRLab is operated by the Association of Universities for -Research in Astronomy (AURA), Inc. under a cooperative agreement with the -National Science Foundation. +NSF NOIRLab Astro Data Archive Query Tool +----------------------------------------- """ from astropy import config as _config diff --git a/astroquery/noirlab/core.py b/astroquery/noirlab/core.py index 672e4516dc..3ef83209a7 100644 --- a/astroquery/noirlab/core.py +++ b/astroquery/noirlab/core.py @@ -1,3 +1,4 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst """ Provide astroquery API access to NSF NOIRLab Astro Data Archive. @@ -268,7 +269,7 @@ def categoricals(self, cache=True): def query_metadata(self, qspec=None, sort=None, limit=1000, hdu=False, cache=True): """Query the archive database for details on available files. - `qspec` should minimally contain a list of output columns and a list of + ``qspec`` should minimally contain a list of output columns and a list of search parameters, which could be empty. For example:: qspec = {"outfields": ["md5sum", ], "search": []} @@ -278,7 +279,7 @@ def query_metadata(self, qspec=None, sort=None, limit=1000, hdu=False, cache=Tru qspec : :class:`dict`, optional The query that will be passed to the API. sort : :class:`str`, optional - Sort the results on one of the columns in `qspec`. + Sort the results on one of the columns in ``qspec``. limit : :class:`int`, optional The number of results to return, default 1000. hdu : :class:`bool`, optional diff --git a/docs/noirlab/noirlab.rst b/docs/noirlab/noirlab.rst index 0e99f909aa..71d0372943 100644 --- a/docs/noirlab/noirlab.rst +++ b/docs/noirlab/noirlab.rst @@ -4,6 +4,9 @@ NOIRLab Queries (`astroquery.noirlab`) ************************************** +Introduction +============ + The methods in this module are wrappers around a set of web services described in the `REST API documentation `_.