From 632d7cdc61a810dbc69e7dfa366c0b3340518c36 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Thu, 4 Jun 2020 14:40:29 -0700 Subject: [PATCH 01/25] update manifest for new cdirs formalism at NERSC; add a directory to host Tycho files; some minor formatting fixes --- etc/desitarget.module | 12 +++++++----- py/desitarget/gaiamatch.py | 1 + py/desitarget/uratmatch.py | 1 + 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/etc/desitarget.module b/etc/desitarget.module index 226e110fa..6660192af 100644 --- a/etc/desitarget.module +++ b/etc/desitarget.module @@ -76,8 +76,10 @@ setenv [string toupper $product] $PRODUCT_DIR # # Add any non-standard Module code below this point. # -setenv GAIA_DIR /project/projectdirs/desi/target/gaia_dr2 -setenv CMX_DIR /project/projectdirs/desi/target/cmx_files -setenv SCND_DIR /project/projectdirs/desi/target/secondary -setenv URAT_DIR /project/projectdirs/desi/target/urat_dr1 -setenv TARG_DIR /project/projectdirs/desi/target/catalogs \ No newline at end of file +setenv TARG_DIR /global/cfs/cdirs/desi/target/catalogs +setenv DUST_DIR /global/cfs/cdirs/cosmo/data/dust/v0_1 +setenv CMX_DIR /global/cfs/cdirs/desi/target/cmx_files +setenv SCND_DIR /global/cfs/cdirs/desi/target/secondary +setenv GAIA_DIR /global/cfs/cdirs/desi/target/gaia_dr2 +setenv URAT_DIR /global/cfs/cdirs/desi/target/urat_dr1 +setenv TYCHO_DIR /global/cfs/cdirs/desi/target/tycho_dr2 diff --git a/py/desitarget/gaiamatch.py b/py/desitarget/gaiamatch.py index 54e53a447..f7cdec737 100644 --- a/py/desitarget/gaiamatch.py +++ b/py/desitarget/gaiamatch.py @@ -664,6 +664,7 @@ def find_gaia_files(objs, neighbors=True, radec=False): # ADM which flavor of RA/Dec was passed. if radec: ra, dec = objs + dec = np.array(dec) else: ra, dec = objs["RA"], objs["DEC"] diff --git a/py/desitarget/uratmatch.py b/py/desitarget/uratmatch.py index fe3c001d3..b5b3a6a20 100644 --- a/py/desitarget/uratmatch.py +++ b/py/desitarget/uratmatch.py @@ -535,6 +535,7 @@ def find_urat_files(objs, neighbors=True, radec=False): # ADM which flavor of RA/Dec was passed. if radec: ra, dec = objs + dec = np.array(dec) else: ra, dec = objs["RA"], objs["DEC"] From 2028d4b19b8c54794d77980e356ea53133c9ed94 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 5 Jun 2020 11:31:50 -0700 Subject: [PATCH 02/25] new module to create HEALPixel-split Tycho files and match to Tycho objects --- py/desitarget/tychomatch.py | 395 ++++++++++++++++++++++++++++++++++++ py/desitarget/uratmatch.py | 2 +- 2 files changed, 396 insertions(+), 1 deletion(-) create mode 100644 py/desitarget/tychomatch.py diff --git a/py/desitarget/tychomatch.py b/py/desitarget/tychomatch.py new file mode 100644 index 000000000..6edca11d3 --- /dev/null +++ b/py/desitarget/tychomatch.py @@ -0,0 +1,395 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +# -*- coding: utf-8 -*- +""" +===================== +desitarget.tychomatch +===================== + +Useful Tycho catalog matching and manipulation routines. +""" +import os +import numpy as np +import fitsio +import requests +import pickle +from datetime import datetime + +from pkg_resources import resource_filename +from time import time +from astropy.io import ascii +from glob import glob +import healpy as hp + +from desitarget.internal import sharedmem +from desimodel.footprint import radec2pix +from desitarget.geomask import add_hp_neighbors, radec_match_to + +# ADM set up the DESI default logger +from desiutil.log import get_logger +log = get_logger() + +# ADM start the clock +start = time() + +# ADM columns contained in our version of the Tycho fits files. +tychodatamodel = np.array([], dtype=[ + ('TYC1', '>i2'), ('TYC2', '>i2'), ('TYC3', '|u1'), + ('RA', '>f8'), ('DEC', '>f8'), + ('MEAN_RA', '>f8'), ('MEAN_DEC', '>f8'), + ('SIGMA_RA', '>f4'), ('SIGMA_DEC', '>f4'), + # ADM these are converted to be in mas/yr for consistency with Gaia. + ('PM_RA', '>f4'), ('PM_DEC', '>f4'), + ('SIGMA_PM_RA', '>f4'), ('SIGMA_PM_DEC', '>f4'), + ('EPOCH_RA', '>f4'), ('EPOCH_DEC', '>f4'), + ('MAG_BT', '>f4'), ('MAG_VT', '>f4'), ('MAG_HP', '>f4'), ('ISGALAXY', '|u1'), + ('JMAG', '>f4'), ('HMAG', '>f4'), ('KMAG', '>f4'), ('ZGUESS', '>f4') +]) + +def _get_tycho_dir(): + """Convenience function to grab the Tycho environment variable. + + Returns + ------- + :class:`str` + The directory stored in the $TYCHO_DIR environment variable. + """ + # ADM check that the $TYCHO_DIR environment variable is set. + tychodir = os.environ.get('TYCHO_DIR') + if tychodir is None: + msg = "Set $TYCHO_DIR environment variable!" + log.critical(msg) + raise ValueError(msg) + + return tychodir + + +def _get_tycho_nside(): + """Grab the HEALPixel nside to be used throughout this module. + + Returns + ------- + :class:`int` + The HEALPixel nside number for Tycho file creation and retrieval. + """ + nside = 4 + + return nside + + +def grab_tycho(cosmodir="/global/cfs/cdirs/cosmo/staging/tycho2/"): + """Retrieve the cosmo versions of the Tycho files at NERSC. + + Parameters + ---------- + cosmodir : :class:`str` + The NERSC directory that hosts the Tycho files. + + Returns + ------- + Nothing + But the Tycho fits file, README are written to $TYCHO_DIR/fits. + + Notes + ----- + - The environment variable $TYCHO_DIR must be set. + - The fits file is "cleaned up" to conform to DESI Data Systems + standards (e.g. all columns are converted to upper-case). + """ + # ADM check that the TYCHO_DIR is set and retrieve it. + tychodir = _get_tycho_dir() + + # ADM construct the directory to which to write files. + fitsdir = os.path.join(tychodir, 'fits') + # ADM the directory better be empty for the copy! + if os.path.exists(fitsdir): + if len(os.listdir(fitsdir)) > 0: + msg = "{} should be empty to get TYCHO FITS file!".format(fitsdir) + log.critical(msg) + raise ValueError(msg) + # ADM make the directory, if needed. + else: + log.info('Making TYCHO directory for storing FITS files') + os.makedirs(fitsdir) + + # ADM the actual name of the Tycho file and the associated README. + tychofn = "tycho2.kd.fits" + cosmofile = os.path.join(cosmodir, tychofn) + rfile = os.path.join(cosmodir, "README") + + # ADM the associated output files. + outfile = os.path.join(fitsdir, tychofn) + routfile = os.path.join(fitsdir, "README") + + # ADM read in the Tycho file and header in upper-case. + objs, hdr = fitsio.read(cosmofile, header=True, upper=True) + nobjs = len(objs) + done = np.zeros(nobjs, dtype=tychodatamodel.dtype) + for col in tychodatamodel.dtype.names: + # ADM proper motions need converted to mas/yr. + if "PM" in col: + done[col] = objs[col]*1000 + else: + done[col] = objs[col] + + # ADM add some information to the header + copydate = datetime.utcnow().isoformat(timespec='seconds') + hdr["COSMODIR"] = cosmodir + hdr["COPYDATE"] = copydate + + # ADM write the data. + fitsio.write(outfile, done, extname='TYCHOFITS', header=hdr) + + # ADM also update the README. + msg = "\nCopied from: {}\non: {}\nthe specific file being: {}\n".format( + cosmodir, copydate, cosmofile) + with open(rfile) as f: + readme = f.read() + with open(routfile, 'w') as f: + f.write(readme+msg) + + log.info('Wrote Tycho FITS file...t={:.1f}s'.format(time()-start)) + + return + + +def tycho_fits_to_healpix(): + """Convert files in $TYCHO_DIR/fits to files in $TYCHO_DIR/healpix. + + Returns + ------- + Nothing + But the archived Tycho FITS files in $TYCHO_DIR/fits are + rearranged by HEALPixel in the directory $TYCHO_DIR/healpix. + The HEALPixel sense is nested with nside=_get_tycho_nside(), and + each file in $TYCHO_DIR/healpix is called healpix-xxxxx.fits, + where xxxxx corresponds to the HEALPixel number. + + Notes + ----- + - The environment variable $TYCHO_DIR must be set. + """ + # ADM the resolution at which the Tycho HEALPix files are stored. + nside = _get_tycho_nside() + npix = hp.nside2npix(nside) + + # ADM check that the TYCHO_DIR is set. + tychodir = _get_tycho_dir() + + # ADM construct the directories for reading/writing files. + fitsdir = os.path.join(tychodir, "fits") + tychofn = os.path.join(fitsdir, "tycho2.kd.fits") + hpxdir = os.path.join(tychodir, "healpix") + + # ADM make sure the output directory is empty. + if os.path.exists(hpxdir): + if len(os.listdir(hpxdir)) > 0: + msg = "{} must be empty to make Tycho HEALPix files!".format(hpxdir) + log.critical(msg) + raise ValueError(msg) + # ADM make the output directory, if needed. + else: + log.info("Making Tycho directory for storing HEALPix files") + os.makedirs(hpxdir) + + # ADM read in the Tycho file and assing Tycho objects to HEALPixels. + objs, allhdr = fitsio.read(tychofn, header=True, upper=True) + pix = radec2pix(nside, objs["RA"], objs["DEC"]) + + # ADM loop through the pixels and write out the files. + for pixnum in range(npix): + # ADM construct the name of the output file. + outfilename = "healpix-{:05d}.fits".format(pixnum) + outfile = os.path.join(hpxdir, outfilename) + # ADM update the header with new information. + hdr = dict(allhdr).copy() + hdr["HPXNSIDE"] = nside + hdr["HPXNEST"] = True + hdr["HPXDATE"] = datetime.utcnow().isoformat(timespec='seconds') + + # ADM determine which objects are in this pixel and write out. + done = objs[pix == pixnum] + + fitsio.write(outfile, done, extname="TYCHOHPX", header=hdr) + + log.info('Wrote Tycho HEALPix files...t={:.1f}s'.format(time()-start)) + + return + + +def make_tycho_files(): + """Make the HEALPix-split Tycho files in one fell swoop. + + Returns + ------- + Nothing + But produces: + - A FITS file with appropriate header and columns from + `tychodatamodel`, and a README in $TYCHO_DIR/fits. + - FITS files reorganized by HEALPixel in $TYCHO_DIR/healpix. + + The HEALPixel sense is nested with nside=_get_tycho_nside(), and + each file in $TYCHO_DIR/healpix is called healpix-xxxxx.fits, + where xxxxx corresponds to the HEALPixel number. + + Notes + ----- + - The environment variable $TYCHO_DIR must be set. + """ + t0 = time() + log.info('Begin making Tycho files...t={:.1f}s'.format(time()-t0)) + + # ADM check that the TYCHO_DIR is set. + tychodir = _get_tycho_dir() + + # ADM a quick check that the fits and healpix directories are empty + # ADM before embarking on the slower parts of the code. + fitsdir = os.path.join(tychodir, 'fits') + hpxdir = os.path.join(tychodir, 'healpix') + for direc in [fitsdir, hpxdir]: + if os.path.exists(direc): + if len(os.listdir(direc)) > 0: + msg = "{} should be empty to make Tycho files!".format(direc) + log.critical(msg) + raise ValueError(msg) + + grab_tycho() + log.info('Copied Tycho FITS file from cosmo...t={:.1f}s'.format(time()-t0)) + + tycho_fits_to_healpix() + log.info('Rearranged FITS files by HEALPixel...t={:.1f}s'.format(time()-t0)) + + return + + +def find_tycho_files(objs, neighbors=True, radec=False): + """Find full paths to Tycho healpix files for objects by RA/Dec. + + Parameters + ---------- + objs : :class:`~numpy.ndarray` + Array of objects. Must contain the columns "RA" and "DEC". + neighbors : :class:`bool`, optional, defaults to ``True`` + Also return all pixels that touch the files of interest + to prevent edge effects (e.g. if a Tycho source is 1 arcsec + away from a primary source and so in an adjacent pixel). + radec : :class:`bool`, optional, defaults to ``False`` + If ``True`` then the passed `objs` is an [RA, Dec] list + instead of a rec array that contains "RA" and "DEC". + + Returns + ------- + :class:`list` + A list of all Tycho files to read to account for objects at + the passed locations. + + Notes + ----- + - The environment variable $TYCHO_DIR must be set. + """ + # ADM the resolution at which the Tycho HEALPix files are stored. + nside = _get_tycho_nside() + + # ADM check that the TYCHO_DIR is set and retrieve it. + tychodir = _get_tycho_dir() + hpxdir = os.path.join(tychodir, 'healpix') + + # ADM which flavor of RA/Dec was passed. + if radec: + ra, dec = objs + dec = np.array(dec) + else: + ra, dec = objs["RA"], objs["DEC"] + + # ADM convert RA/Dec to co-latitude and longitude in radians. + theta, phi = np.radians(90-dec), np.radians(ra) + + # ADM retrieve the pixels in which the locations lie. + pixnum = hp.ang2pix(nside, theta, phi, nest=True) + # ADM if neighbors was sent, then retrieve all pixels that touch each + # ADM pixel covered by the input RA/Dec, to prevent edge effects. + if neighbors: + pixnum = add_hp_neighbors(nside, pixnum) + + # ADM reformat file names in the Tycho healpix format. + tychofiles = [os.path.join(hpxdir, 'healpix-{:05d}.fits'.format(pn)) + for pn in pixnum] + + # ADM restrict to only files/HEALPixels actually covered by Tycho. + tychofiles = [fn for fn in tychofiles if os.path.exists(fn)] + + return tychofiles + + +def match_to_tycho(objs, matchrad=1., radec=False): + """Match objects to Tycho healpixel files. + + Parameters + ---------- + objs : :class:`~numpy.ndarray` + Must contain at least "RA" and "DEC". + matchrad : :class:`float`, optional, defaults to 1 arcsec + The radius at which to match in arcseconds. + radec : :class:`bool`, optional, defaults to ``False`` + If ``True`` then the passed `objs` is an [RA, Dec] list instead of + a rec array. + + Returns + ------- + :class:`~numpy.ndarray` + The matching Tycho information for each object. The returned + format is as for desitarget.tychomatch.tychodatamodel with + an extra column "TYCHO_SEP" which is the matching distance + in ARCSECONDS. + + Notes + ----- + - For objects with NO match in Tycho, the "TYC1", "TYC2" and + "TYCHO_SEP" columns are -1, and other columns are zero. + - Retrieves the CLOSEST match to Tycho for each passed object. + - Because this reads in HEALPixel split files, it's (far) faster + for objects that are clumped rather than widely distributed. + """ + # ADM parse whether a structure or coordinate list was passed. + if radec: + ra, dec = objs + else: + ra, dec = objs["RA"], objs["DEC"] + + # ADM set up an array of Tycho information for the output. + nobjs = len(ra) + done = np.zeros(nobjs, dtype=tychodatamodel.dtype) + + # ADM objects without matches should have TYC1/2/3, TYCHO_SEP of -1. + for col in "TYC1", "TYC2": + done[col] = -1 + tycho_sep = np.zeros(nobjs) - 1 + + # ADM determine which Tycho files need to be scraped. + tychofiles = find_tycho_files([ra, dec], radec=True) + nfiles = len(tychofiles) + + # ADM catch the case of no matches to Tycho. + if nfiles > 0: + # ADM loop through the Tycho files and find matches. + for ifn, fn in enumerate(tychofiles): + if ifn % 500 == 0 and ifn > 0: + log.info('{}/{} files; {:.1f} total mins elapsed' + .format(ifn, nfiles, (time()-start)/60.)) + tycho = fitsio.read(fn) + idtycho, idobjs, dist = radec_match_to( + [tycho["RA"], tycho["DEC"]], [ra, dec], + sep=matchrad, radec=True, return_sep=True) + + # ADM update matches whenever we have a CLOSER match. + ii = (tycho_sep[idobjs] == -1) | (tycho_sep[idobjs] > dist) + done[idobjs[ii]] = tycho[idtycho[ii]] + tycho_sep[idobjs[ii]] = dist[ii] + + # ADM add the separation distances to the output array. + dt = tychodatamodel.dtype.descr + [("TYCHO_SEP", ">f4")] + output = np.zeros(nobjs, dtype=dt) + for col in tychodatamodel.dtype.names: + output[col] = done[col] + output["TYCHO_SEP"] = tycho_sep + + return output diff --git a/py/desitarget/uratmatch.py b/py/desitarget/uratmatch.py index b5b3a6a20..a33f20e2e 100644 --- a/py/desitarget/uratmatch.py +++ b/py/desitarget/uratmatch.py @@ -577,7 +577,7 @@ def match_to_urat(objs, matchrad=1., radec=False): :class:`~numpy.ndarray` The matching URAT information for each object. The returned format is as for desitarget.uratmatch.uratdatamodel with - and extra column "URAT_SEP" which is the matching distance + an extra column "URAT_SEP" which is the matching distance in ARCSECONDS. Notes From 4ca4beec670712b57dfb420888d43463724cad63 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 5 Jun 2020 15:48:17 -0700 Subject: [PATCH 03/25] deprecate the collect_ and model_ functions in brightmask. I think I was the only one using them. Instead we'll use Gaia/Tycho to build masks --- py/desitarget/brightmask.py | 265 +----------------------------------- 1 file changed, 7 insertions(+), 258 deletions(-) diff --git a/py/desitarget/brightmask.py b/py/desitarget/brightmask.py index 252257fcc..97c4a123c 100644 --- a/py/desitarget/brightmask.py +++ b/py/desitarget/brightmask.py @@ -43,6 +43,13 @@ infac = 0.5 nearfac = 1./infac +maskdatamodel = np.array([], dtype=[ + ('RA', '>f8'), ('DEC', '>f8'), + ('REF_CAT', '|S2'), ('REF_ID', '>i8'), + ('IN_RADIUS', '>f4'), ('NEAR_RADIUS', '>f4'), + ('E1', '>f4'), ('E2', '>f4'), ('TYPE', '|S3') +]) + def _rexlike(rextype): """If the object is REX (a round exponential galaxy)""" @@ -96,264 +103,6 @@ def max_objid_bricks(targs): return dict(ordered[maxind]) -def collect_bright_stars(bands, maglim, numproc=4, - rootdirname='/global/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1', - outfilename=None): - """Extract a structure from the sweeps containing only bright stars in a given band to a given magnitude limit. - - Parameters - ---------- - bands : :class:`str` - A magnitude band from the sweeps, e.g., "G", "R", "Z". - Can pass multiple bands as string, e.g. "GRZ", in which case maglim has to be a - list of the same length as the string. - maglim : :class:`float` - The upper limit in that magnitude band for which to assemble a list of bright stars. - Can pass a list of magnitude limits, in which case bands has to be a string of the - same length (e.g., "GRZ" for [12.3,12.7,12.6] - numproc : :class:`int`, optional - Number of processes over which to parallelize - rootdirname : :class:`str`, optional, defaults to dr3 - Root directory containing either sweeps or tractor files...e.g. for dr3 this might be - /global/project/projectdirs/cosmo/data/legacysurvey/dr3/sweep/dr3.1 - outfilename : :class:`str`, optional, defaults to not writing anything to file - (FITS) File name to which to write the output structure of bright stars - - Returns - ------- - :class:`recarray` - The structure of bright stars from the sweeps limited in the passed band(s) to the - passed maglim(s). - """ - # ADM set up default logger. - from desiutil.log import get_logger - log = get_logger() - - # ADM this is just a special case of collect_bright_sources. - sourcestruc = collect_bright_sources(bands, maglim, - numproc=numproc, - rootdirname=rootdirname, outfilename=None) - # ADM check if a source is unresolved. - psflike = _psflike(sourcestruc["TYPE"]) - wstar = np.where(psflike) - if len(wstar[0]) > 0: - done = sourcestruc[wstar] - if outfilename is not None: - fitsio.write(outfilename, done, clobber=True) - return done - else: - log.error('No PSF-like objects brighter than {} in {} in files in {}' - .format(str(maglim), bands, rootdirname)) - return -1 - - -def collect_bright_sources(bands, maglim, numproc=4, - rootdirname='/global/project/projectdirs/cosmo/data/legacysurvey/dr5/sweep/5.0', - outfilename=None): - """Extract a structure from the sweeps containing all bright sources in a given band to a given magnitude limit. - - Parameters - ---------- - bands : :class:`str` - A magnitude band from the sweeps, e.g., "G", "R", "Z". - Can pass multiple bands as string, e.g. "GRZ", in which case maglim has to be a - list of the same length as the string. - maglim : :class:`float` - The upper limit in that magnitude band for which to assemble a list of bright sources. - Can pass a list of magnitude limits, in which case bands has to be a string of the - same length (e.g., "GRZ" for [12.3,12.7,12.6]. - numproc : :class:`int`, optional - Number of processes over which to parallelize. - rootdirname : :class:`str`, optional, defaults to dr5 - Root directory containing either sweeps or tractor files...e.g. for dr5 this might be - /global/project/projectdirs/cosmo/data/legacysurvey/dr5/sweep/dr5.0. - outfilename : :class:`str`, optional, defaults to not writing anything to file - (FITS) File name to which to write the output structure of bright sources. - - Returns - ------- - :class:`recarray` - The structure of bright sources from the sweeps limited in the passed band(s) to the - passed maglim(s). - """ - # ADM set up default logger. - from desiutil.log import get_logger - log = get_logger() - - # ADM use io.py to retrieve list of sweeps or tractor files. - infiles = io.list_sweepfiles(rootdirname) - if len(infiles) == 0: - infiles = io.list_tractorfiles(rootdirname) - if len(infiles) == 0: - raise IOError('No sweep or tractor files found in {}'.format(rootdirname)) - - # ADM force the input maglim to be a list (in case a single value was passed). - if isinstance(maglim, int) or isinstance(maglim, float): - maglim = [maglim] - - # ADM set bands to uppercase if passed as lower case. - bands = bands.upper() - # ADM the band names as a flux array instead of a string. - bandnames = np.array(["FLUX_"+band for band in bands]) - - if len(bandnames) != len(maglim): - raise IOError('bands has to be the same length as maglim and {} does not equal {}' - .format(len(bands), len(maglim))) - - # ADM change input magnitude(s) to a flux to test against. - fluxlim = 10.**((22.5-np.array(maglim))/2.5) - - # ADM parallel formalism from this step forward is stolen from cuts.select_targets. - - # ADM function to grab the bright sources from a given file. - def _get_bright_sources(filename): - """Retrieves bright sources from a sweeps/Tractor file""" - objs = io.read_tractor(filename) - # ADM write the fluxes as an array instead of as named columns. - - # ADM Retain rows for which ANY band is brighter than maglim. - ok = np.zeros(objs[bandnames[0]].shape, dtype=bool) - for i, bandname in enumerate(bandnames): - ok |= (objs[bandname] > fluxlim[i]) - - w = np.where(ok) - if len(w[0]) > 0: - return objs[w] - - # ADM counter for how many files have been processed. - # ADM critical to use np.ones because a numpy scalar allows in place modifications. - # c.f https://www.python.org/dev/peps/pep-3104/ - totfiles = np.ones((), dtype='i8')*len(infiles) - nfiles = np.ones((), dtype='i8') - t0 = time() - log.info('Collecting bright sources from sweeps...') - - def _update_status(result): - """wrapper function for the critical reduction operation, - that occurs on the main parallel process.""" - if nfiles % 25 == 0: - elapsed = time() - t0 - rate = nfiles / elapsed - log.info('{}/{} files; {:.1f} files/sec; {:.1f} total mins elapsed' - .format(nfiles, totfiles, rate, elapsed/60.)) - nfiles[...] += 1 # this is an in-place modification - return result - - # ADM did we ask to parallelize, or not? - if numproc > 1: - pool = sharedmem.MapReduce(np=numproc) - with pool: - sourcestruc = pool.map(_get_bright_sources, infiles, reduce=_update_status) - else: - sourcestruc = [] - for file in infiles: - sourcestruc.append(_update_status(_get_bright_sources(file))) - - # ADM note that if there were no bright sources in a file then - # ADM the _get_bright_sources function will have returned NoneTypes - # ADM so we need to filter those out. - sourcestruc = [x for x in sourcestruc if x is not None] - if len(sourcestruc) == 0: - raise IOError('There are no sources brighter than {} in {} in files in {} with which to make a mask' - .format(str(maglim), bands, rootdirname)) - # ADM concatenate all of the output recarrays. - sourcestruc = np.hstack(sourcestruc) - - # ADM if the name of a file for output is passed, then write to it. - if outfilename is not None: - fitsio.write(outfilename, sourcestruc, clobber=True) - - return sourcestruc - - -def model_bright_stars(band, instarfile, - rootdirname='/global/project/projectdirs/cosmo/data/legacysurvey/dr3.1/'): - - """Build a dictionary of the fraction of bricks containing a star of a given - magnitude in a given band as function of Galactic l and b. - - Parameters - ---------- - band : :class:`str` - A magnitude band from the sweeps, e.g., "G", "R", "Z". - instarfile : :class:`str` - File of bright objects in (e.g.) sweeps, created by collect_bright_stars. - rootdirname : :class:`str`, optional, defaults to dr3 - Root directory for a data release...e.g. for dr3 this would be - /global/project/projectdirs/cosmo/data/legacysurvey/dr3.1/. - - Returns - ------- - :class:`dictionary` - dictionary of the fraction of bricks containing a star of a given - magnitude in a given band as function of Galactic l Keys are mag - bin CENTERS, values are arrays running from 0->1 to 359->360. - :class:`dictionary` - dictionary of the fraction of bricks containing a star of a given - magnitude in a given band as function of Galactic b. Keys are mag - bin CENTERS, values are arrays running from -90->-89 to 89->90. - - Notes - ----- - - converts using coordinates of the brick center, so is an approximation. - - """ - # ADM histogram bin edges in Galactic coordinates at resolution of 1 degree. - lbinedges = np.arange(361) - bbinedges = np.arange(-90, 91) - - # ADM set band to uppercase if passed as lower case. - band = band.upper() - - # ADM read in the bright object file. - fx = fitsio.FITS(instarfile) - objs = fx[1].read() - # ADM convert fluxes in band of interest for each object to magnitudes. - mags = 22.5-2.5*np.log10(objs["FLUX_"+band]) - # ADM Galactic l and b for each object of interest. - c = SkyCoord(objs["RA"]*u.degree, objs["DEC"]*u.degree, frame='icrs') - lobjs = c.galactic.l.degree - bobjs = c.galactic.b.degree - - # ADM construct histogram bin edges in magnitude in passed band. - magstep = 0.1 - magmin = -1.5 # ADM magnitude of Sirius to 1 d.p. - magmax = np.max(mags) - magbinedges = np.arange(np.rint((magmax-magmin)/magstep))*magstep+magmin - - # ADM read in the data-release specific brick information file. - fx = fitsio.FITS(glob(rootdirname+'/survey-bricks-dr*.fits.gz')[0], upper=True) - bricks = fx[1].read(columns=['RA', 'DEC']) - - # ADM convert RA/Dec of the brick center to Galatic coordinates and - # ADM build a histogram of the number of bins at each coordinate. - # ADM using the center is imperfect, so this is approximate at best. - c = SkyCoord(bricks["RA"]*u.degree, bricks["DEC"]*u.degree, frame='icrs') - lbrick = c.galactic.l.degree - bbrick = c.galactic.b.degree - lhistobrick = (np.histogram(lbrick, bins=lbinedges))[0] - bhistobrick = (np.histogram(bbrick, bins=bbinedges))[0] - - # ADM loop through the magnitude bins and populate a dictionary - # ADM of the number of stars in this magnitude range per brick. - ldict, bdict = {}, {} - for mag in magbinedges: - key = "{:.2f}".format(mag+(0.5*magstep)) - # ADM range in magnitude. - w = np.where((mags >= mag) & (mags < mag+magstep)) - if len(w[0]): - # ADM histograms of numbers of objects in l, b. - lhisto = (np.histogram(lobjs[w], bins=lbinedges))[0] - bhisto = (np.histogram(bobjs[w], bins=bbinedges))[0] - # ADM fractions of objects in l, b per brick. - lfrac = np.where(lhistobrick > 0, lhisto/lhistobrick, 0) - bfrac = np.where(bhistobrick > 0, bhisto/bhistobrick, 0) - # ADM populate the dictionaries. - ldict[key], bdict[key] = lfrac, bfrac - - return ldict, bdict - - def make_bright_star_mask(bands, maglim, numproc=4, rootdirname='/global/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1', infilename=None, outfilename=None): From f40729757e7360d1be321bad84820d8366a9932e Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 5 Jun 2020 17:34:00 -0700 Subject: [PATCH 04/25] documentation for new function to make bright star masks from Tycho/Gaia; standardize healpix- file format throughout desitarget --- etc/desitarget.module | 1 + py/desitarget/brightmask.py | 55 ++++++++++++++++++++++++++++++++++--- py/desitarget/gaiamatch.py | 13 ++++----- py/desitarget/io.py | 18 ++++++++++++ py/desitarget/tychomatch.py | 5 ++-- py/desitarget/uratmatch.py | 5 ++-- 6 files changed, 80 insertions(+), 17 deletions(-) diff --git a/etc/desitarget.module b/etc/desitarget.module index 6660192af..95100526a 100644 --- a/etc/desitarget.module +++ b/etc/desitarget.module @@ -80,6 +80,7 @@ setenv TARG_DIR /global/cfs/cdirs/desi/target/catalogs setenv DUST_DIR /global/cfs/cdirs/cosmo/data/dust/v0_1 setenv CMX_DIR /global/cfs/cdirs/desi/target/cmx_files setenv SCND_DIR /global/cfs/cdirs/desi/target/secondary +setenc MASK_DIR /global/cfs/cdirs/desi/target/masks setenv GAIA_DIR /global/cfs/cdirs/desi/target/gaia_dr2 setenv URAT_DIR /global/cfs/cdirs/desi/target/urat_dr1 setenv TYCHO_DIR /global/cfs/cdirs/desi/target/tycho_dr2 diff --git a/py/desitarget/brightmask.py b/py/desitarget/brightmask.py index 97c4a123c..4dfd03b01 100644 --- a/py/desitarget/brightmask.py +++ b/py/desitarget/brightmask.py @@ -39,14 +39,16 @@ import matplotlib.pyplot as plt # noqa: E402 # ADM factor by which the "in" radius is smaller than the "near" radius -# ADM and vice-versa. infac = 0.5 +# ADM and vice-versa. nearfac = 1./infac +# ADM size of the bright star mask (NEAR_RADIUS) in arcseconds. +masksize = 5. maskdatamodel = np.array([], dtype=[ ('RA', '>f8'), ('DEC', '>f8'), - ('REF_CAT', '|S2'), ('REF_ID', '>i8'), - ('IN_RADIUS', '>f4'), ('NEAR_RADIUS', '>f4'), + ('REF_CAT', '|S2'), ('REF_ID', '>i8'), ('REF_MAG', '>f4'), + ('URAT_ID', '>i8'), ('IN_RADIUS', '>f4'), ('NEAR_RADIUS', '>f4'), ('E1', '>f4'), ('E2', '>f4'), ('TYPE', '|S3') ]) @@ -103,7 +105,52 @@ def max_objid_bricks(targs): return dict(ordered[maxind]) -def make_bright_star_mask(bands, maglim, numproc=4, +def make_bright_star_mask(maglim, numproc=1, epoch=2023.0): + """Make a bright star mask using Tycho, Gaia and URAT. + + Parameters + ---------- + maglim : :class:`float` + Faintest magnitude at which to make the mask (e.g. 12). This mag + is interpreted as G-band for Gaia and, in order of preference, VT + then HP then BT for Tycho (not every Tycho source has each band). + numproc : :class:`int`, optional, defaults to 1 + Number of processes over which to parallelize. + epoch : :class:`float` + The mask is built at this epoch. Not all sources have proper + motions from every survey, so proper motions are used, in order + of preference, from Gaia, URAT, then Tycho. + + Returns + ------- + :class:`recarray` + - The bright source mask in the form of `maskdatamodel.dtype`: + - `REF_CAT` is `"T2"` for Tycho, `"G2"` for Gaia, `"U2"` for + Gaia positions with URAT proper motions. + - `REF_ID` is `Tyc1`*1,000,000+`Tyc2`*10+`Tyc3` for Tycho2; + `"sourceid"` for Gaia-DR2 and Gaia-DR2 with URAT. + - `REF_MAG` is, in order of preference, G-band for Gaia, VT + then HP then BT for Tycho. + - `URAT_ID` contains the URAT reference number for Gaia objects + that use the URAT proper motion (objects with `REF_CAT=U2`). + - The radii are in ARCSECONDS. + - `E1` and `E2` are placeholders for ellipticity components, and + are set to 0 for Gaia and Tycho sources. + - `TYPE` is always `PSF` for star-like objects. + + Notes + ----- + - `IN_RADIUS` corresponds to the `IN_BRIGHT_OBJECT` bit in + `data/targetmask.yaml` (related to `NEAR_RADIUS` by `nearfac`). + - `NEAR_RADIUS` corresponds to the `NEAR_BRIGHT_OBJECT` bit in + `data/targetmask.yaml` (currently fixed at `masksize`). + - The correct mask size for DESI is an open question. + - The `GAIA_DIR`, `URAT_DIR`, `TYCHO_DIR` and `MASK_DIR` + environment variables must be set. + """ + + +def make_bright_star_mask_old(bands, maglim, numproc=4, rootdirname='/global/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1', infilename=None, outfilename=None): """Make a bright star mask from a structure of bright stars drawn from the sweeps. diff --git a/py/desitarget/gaiamatch.py b/py/desitarget/gaiamatch.py index f7cdec737..ba4457e4a 100644 --- a/py/desitarget/gaiamatch.py +++ b/py/desitarget/gaiamatch.py @@ -421,7 +421,7 @@ def _write_hpx_fits(pixlist): else: done = np.hstack([done, objs[pix == pixnum]]) # ADM construct the name of the output file. - outfilename = 'healpix-{:05d}.fits'.format(pixnum) + outfilename = io.hpx_filename(pixnum) outfile = os.path.join(hpxdir, outfilename) # ADM write out the file. hdr = fitsio.FITSHDR() @@ -679,7 +679,7 @@ def find_gaia_files(objs, neighbors=True, radec=False): pixnum = add_hp_neighbors(nside, pixnum) # ADM reformat in the Gaia healpix format used by desitarget. - gaiafiles = [os.path.join(hpxdir, 'healpix-{:05d}.fits'.format(pn)) for pn in pixnum] + gaiafiles = [os.path.join(hpxdir, io.hpx_filename(pn)) for pn in pixnum] return gaiafiles @@ -727,7 +727,7 @@ def find_gaia_files_hp(nside, pixlist, neighbors=True): pixnum = add_hp_neighbors(filenside, pixnum) # ADM reformat in the Gaia healpix format used by desitarget. - gaiafiles = [os.path.join(hpxdir, 'healpix-{:05d}.fits'.format(pn)) for pn in pixnum] + gaiafiles = [os.path.join(hpxdir, io.hpx_filename(pn)) for pn in pixnum] return gaiafiles @@ -775,7 +775,7 @@ def find_gaia_files_box(gaiabounds, neighbors=True): pixnum = add_hp_neighbors(nside, pixnum) # ADM reformat in the Gaia healpix format used by desitarget. - gaiafiles = [os.path.join(hpxdir, 'healpix-{:05d}.fits'.format(pn)) for pn in pixnum] + gaiafiles = [os.path.join(hpxdir, io.hpx_filename(pn)) for pn in pixnum] return gaiafiles @@ -822,8 +822,7 @@ def find_gaia_files_beyond_gal_b(mingalb, neighbors=True): pixnum = add_hp_neighbors(nside, pixnum) # ADM reformat in the Gaia healpix format used by desitarget. - gaiafiles = [os.path.join(hpxdir, 'healpix-{:05d}.fits'.format(pn)) - for pn in pixnum] + gaiafiles = [os.path.join(hpxdir, io.hpx_filename(pn)) for pn in pixnum] return gaiafiles @@ -872,7 +871,7 @@ def find_gaia_files_tiles(tiles=None, neighbors=True): pixnum = add_hp_neighbors(nside, pixnum) # ADM reformat in the Gaia healpix format used by desitarget. - gaiafiles = [os.path.join(hpxdir, 'healpix-{:05d}.fits'.format(pn)) for pn in pixnum] + gaiafiles = [os.path.join(hpxdir, io.hpx_filename(pn)) for pn in pixnum] return gaiafiles diff --git a/py/desitarget/io.py b/py/desitarget/io.py index 72ba41ad4..8d175de63 100644 --- a/py/desitarget/io.py +++ b/py/desitarget/io.py @@ -2294,3 +2294,21 @@ def check_both_set(hpxlist, nside): .format(hpxlist, nside) log.critical(msg) raise ValueError(msg) + + +def hpx_filename(hpx): + """Return the standard name for HEALPixel-split input files + + Parameters + ---------- + hpx : :class:`str` or `int` + A HEALPixel integer. + + Returns + ------- + :class: `str` + Filename in the format used throughout desitarget for + HEALPixel-split input databases. + """ + + return 'healpix-{:05d}.fits'.format(hpx) diff --git a/py/desitarget/tychomatch.py b/py/desitarget/tychomatch.py index 6edca11d3..fe5a8a1e7 100644 --- a/py/desitarget/tychomatch.py +++ b/py/desitarget/tychomatch.py @@ -198,7 +198,7 @@ def tycho_fits_to_healpix(): # ADM loop through the pixels and write out the files. for pixnum in range(npix): # ADM construct the name of the output file. - outfilename = "healpix-{:05d}.fits".format(pixnum) + outfilename = io.hpx_filename(pixnum) outfile = os.path.join(hpxdir, outfilename) # ADM update the header with new information. hdr = dict(allhdr).copy() @@ -311,8 +311,7 @@ def find_tycho_files(objs, neighbors=True, radec=False): pixnum = add_hp_neighbors(nside, pixnum) # ADM reformat file names in the Tycho healpix format. - tychofiles = [os.path.join(hpxdir, 'healpix-{:05d}.fits'.format(pn)) - for pn in pixnum] + tychofiles = [os.path.join(hpxdir, io.hpx_filename(pn)) for pn in pixnum] # ADM restrict to only files/HEALPixels actually covered by Tycho. tychofiles = [fn for fn in tychofiles if os.path.exists(fn)] diff --git a/py/desitarget/uratmatch.py b/py/desitarget/uratmatch.py index a33f20e2e..bebf3e013 100644 --- a/py/desitarget/uratmatch.py +++ b/py/desitarget/uratmatch.py @@ -393,7 +393,7 @@ def _write_hpx_fits(pixlist): else: done = np.hstack([done, objs[pix == pixnum]]) # ADM construct the name of the output file. - outfilename = 'healpix-{:05d}.fits'.format(pixnum) + outfilename = io.hpx_filename(pixnum) outfile = os.path.join(hpxdir, outfilename) # ADM write out the file. hdr = fitsio.FITSHDR() @@ -550,8 +550,7 @@ def find_urat_files(objs, neighbors=True, radec=False): pixnum = add_hp_neighbors(nside, pixnum) # ADM reformat file names in the URAT healpix format. - uratfiles = [os.path.join(hpxdir, 'healpix-{:05d}.fits'.format(pn)) - for pn in pixnum] + uratfiles = [os.path.join(hpxdir, io.hpx_filename(pn)) for pn in pixnum] # ADM restrict to only files/HEALPixels actually covered by URAT. uratfiles = [fn for fn in uratfiles if os.path.exists(fn)] From 06a7a069ef00a5b4f6f593cde0d7c122733ea24a Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Mon, 8 Jun 2020 17:47:59 -0700 Subject: [PATCH 05/25] extensive new functionality to; read Tycho files by HEALPixel; shift coordinates based on proper motions; build a bright star mask from Gaia and Tycho --- py/desitarget/brightmask.py | 187 ++++++++++++++++++++++++++++-------- py/desitarget/gaiamatch.py | 30 +++--- py/desitarget/geomask.py | 50 +++++++++- py/desitarget/randoms.py | 4 +- py/desitarget/tychomatch.py | 79 ++++++++++++--- py/desitarget/uratmatch.py | 9 ++ 6 files changed, 293 insertions(+), 66 deletions(-) diff --git a/py/desitarget/brightmask.py b/py/desitarget/brightmask.py index 4dfd03b01..7033e23c1 100644 --- a/py/desitarget/brightmask.py +++ b/py/desitarget/brightmask.py @@ -11,28 +11,40 @@ .. _`the DR5 sweeps`: http://legacysurvey.org/dr5/files/#sweep-catalogs .. _`Legacy Surveys catalogs`: http://legacysurvey.org/dr5/catalogs/ """ -from __future__ import (absolute_import, division) from time import time -import numpy as np import fitsio import healpy as hp import os import re -import numpy.lib.recfunctions as rfn from glob import glob + +import numpy as np +import numpy.lib.recfunctions as rfn + from astropy.coordinates import SkyCoord from astropy import units as u + from matplotlib.patches import Polygon from matplotlib.collections import PatchCollection -from . import __version__ as desitarget_version + from desitarget import io from desitarget.internal import sharedmem from desitarget.targetmask import desi_mask, targetid_mask from desitarget.targets import encode_targetid +from desitarget.gaiamatch import find_gaia_files from desitarget.geomask import circles, cap_area, circle_boundaries from desitarget.geomask import ellipses, ellipse_boundary, is_in_ellipse +from desitarget.geomask import radec_match_to, rewind_coords from desitarget.cuts import _psflike +from desitarget.tychomatch import get_tycho_dir, get_tycho_nside +from desitarget.tychomatch import find_tycho_files_hp +from desitarget.gfa import add_urat_pms + from desiutil import depend, brick +# ADM set up default logger +from desiutil.log import get_logger +log = get_logger() + # ADM fake the matplotlib display so it doesn't die on allocated nodes. import matplotlib matplotlib.use('Agg') @@ -56,10 +68,6 @@ def _rexlike(rextype): """If the object is REX (a round exponential galaxy)""" - # ADM set up default logger. - from desiutil.log import get_logger - log = get_logger() - # ADM explicitly checking for an empty input. if rextype is None: log.error("NoneType submitted to _rexlike function") @@ -105,17 +113,24 @@ def max_objid_bricks(targs): return dict(ordered[maxind]) -def make_bright_star_mask(maglim, numproc=1, epoch=2023.0): - """Make a bright star mask using Tycho, Gaia and URAT. +def make_bright_star_mask_in_hp(nside, pixnum, + maglim=12., matchrad=1., epoch=2023.0): + """Make a bright star mask in a HEALPixel using Tycho, Gaia and URAT. Parameters ---------- - maglim : :class:`float` - Faintest magnitude at which to make the mask (e.g. 12). This mag - is interpreted as G-band for Gaia and, in order of preference, VT + nside : :class:`int` + (NESTED) HEALPixel nside. + pixnum : :class:`int` + A single HEALPixel number. + maglim : :class:`float`, optional, defaults to 12. + Faintest magnitude at which to make the mask. This magnitude is + interpreted as G-band for Gaia and, in order of preference, VT then HP then BT for Tycho (not every Tycho source has each band). - numproc : :class:`int`, optional, defaults to 1 - Number of processes over which to parallelize. + matchrad : :class:`int`, optional, defaults to 1. + Tycho sources that match a Gaia source at this separation in + ARCSECONDS are NOT included in the output mask. The matching is + performed rigorously, accounting for Gaia proper motions. epoch : :class:`float` The mask is built at this epoch. Not all sources have proper motions from every survey, so proper motions are used, in order @@ -125,14 +140,13 @@ def make_bright_star_mask(maglim, numproc=1, epoch=2023.0): ------- :class:`recarray` - The bright source mask in the form of `maskdatamodel.dtype`: - - `REF_CAT` is `"T2"` for Tycho, `"G2"` for Gaia, `"U2"` for - Gaia positions with URAT proper motions. + - `REF_CAT` is `"T2"` for Tycho and `"G2"` for Gaia. - `REF_ID` is `Tyc1`*1,000,000+`Tyc2`*10+`Tyc3` for Tycho2; `"sourceid"` for Gaia-DR2 and Gaia-DR2 with URAT. - `REF_MAG` is, in order of preference, G-band for Gaia, VT then HP then BT for Tycho. - `URAT_ID` contains the URAT reference number for Gaia objects - that use the URAT proper motion (objects with `REF_CAT=U2`). + that use the URAT proper motion, or -1 otherwise. - The radii are in ARCSECONDS. - `E1` and `E2` are placeholders for ellipticity components, and are set to 0 for Gaia and Tycho sources. @@ -140,14 +154,122 @@ def make_bright_star_mask(maglim, numproc=1, epoch=2023.0): Notes ----- - - `IN_RADIUS` corresponds to the `IN_BRIGHT_OBJECT` bit in - `data/targetmask.yaml` (related to `NEAR_RADIUS` by `nearfac`). - - `NEAR_RADIUS` corresponds to the `NEAR_BRIGHT_OBJECT` bit in - `data/targetmask.yaml` (currently fixed at `masksize`). + - `IN_RADIUS` (`NEAR_RADIUS`) corresponds to `IN_BRIGHT_OBJECT` + (`NEAR_BRIGHT_OBJECT`) in `data/targetmask.yaml`. These radii + are set in the function `desitarget.brightmask.radius()`. - The correct mask size for DESI is an open question. - - The `GAIA_DIR`, `URAT_DIR`, `TYCHO_DIR` and `MASK_DIR` - environment variables must be set. + - The `GAIA_DIR`, `URAT_DIR` and `TYCHO_DIR` environment + variables must be set. """ + # ADM start the clock. + t0 = time() + + # ADM read in the Tycho files. + tychofns = find_tycho_files_hp(nside, pixnum, neighbors=False) + tychoobjs = [] + for fn in tychofns: + tychoobjs.append(fitsio.read(fn, ext='TYCHOHPX')) + tychoobjs = np.concatenate(tychoobjs) + # ADM create the Tycho reference magnitude, which is VT then HP + # ADM then BT in order of preference. + tychomag = tychoobjs["MAG_VT"].copy() + tychomag[tychomag == 0] = tychoobjs["MAG_HP"][tychomag == 0] + tychomag[tychomag == 0] = tychoobjs["MAG_BT"][tychomag == 0] + # ADM discard any Tycho objects below the input magnitude limit. + ii = tychomag < maglim + tychomag, tychoobjs = tychomag[ii], tychoobjs[ii] + log.info('Read {} (mag < {}) Tycho objects...t = {:.1f} mins'.format( + np.sum(ii), maglim, (time()-t0)/60)) + + # ADM read in the associated Gaia files. Also grab + # ADM neighboring pixels to prevent edge effects. + gaiafns = find_gaia_files(tychoobjs, neighbors=True) + gaiaobjs = [] + cols = 'SOURCE_ID', 'RA', 'DEC', 'PHOT_G_MEAN_MAG', 'PMRA', 'PMDEC' + for fn in gaiafns: + gaiaobjs.append(fitsio.read(fn, ext='GAIAHPX', columns=cols)) + gaiaobjs = np.concatenate(gaiaobjs) + gaiaobjs = rfn.rename_fields(gaiaobjs, {"SOURCE_ID":"REF_ID"}) + # ADM limit Gaia objects to 3 magnitudes fainter than the passed + # ADM limit. This leaves some (!) leeway when matching to Tycho. + gaiaobjs = gaiaobjs[gaiaobjs['PHOT_G_MEAN_MAG'] < maglim + 3] + log.info('Read {} (G < {}) Gaia sources...t = {:.1f} mins'.format( + len(gaiaobjs), maglim + 3, (time()-t0)/60)) + + # ADM substitute URAT where Gaia proper motions don't exist. + ii = ((np.isnan(gaiaobjs["PMRA"]) | (gaiaobjs["PMRA"] == 0)) & + (np.isnan(gaiaobjs["PMDEC"]) | (gaiaobjs["PMDEC"] == 0))) + log.info('Add URAT for {} Gaia objects with no PMs...t = {:.1f} mins'.format( + np.sum(ii), (time()-t0)/60)) + urat = add_urat_pms(gaiaobjs[ii], numproc=1) + log.info('Found an additional {} URAT objects...t = {:.1f} mins'.format( + np.sum(urat["URAT_ID"] != -1), (time()-t0)/60)) + for col in "PMRA", "PMDEC": + gaiaobjs[col][ii] = urat[col] + # ADM need to track the URATID to track which objects have + # ADM substituted proper motions. + uratid = np.zeros_like(gaiaobjs["REF_ID"])-1 + uratid[ii] = urat["URAT_ID"] + + # ADM match to remove Tycho objects already in Gaia. Prefer the more + # ADM accurate Gaia proper motions. Note, however, that Tycho epochs + # ADM can differ from the mean (1991.5) by as as much as 0.86 years, + # ADM so a star with a proper motion as large as Barnard's Star + # ADM (10.3 arcsec) can be off by a significant margin (~10"). + margin = 10. + ra, dec = rewind_coords(gaiaobjs["RA"], gaiaobjs["DEC"], + gaiaobjs["PMRA"], gaiaobjs["PMDEC"]) + # ADM match Gaia to Tycho with a suitable margin. + log.info('Match Gaia to Tycho with margin={}"...t = {:.1f} mins'.format( + margin, (time()-t0)/60)) + igaia, itycho = radec_match_to([ra, dec], + [tychoobjs["RA"], tychoobjs["DEC"]], + sep=margin, radec=True) + log.info('{} matches. Refining at 1"...t = {:.1f} mins'.format( + len(itycho), (time()-t0)/60)) + # ADM match Gaia to Tycho at the more exact reference epoch. + epoch_ra = tychoobjs[itycho]["EPOCH_RA"] + epoch_dec = tychoobjs[itycho]["EPOCH_DEC"] + # ADM some of the Tycho epochs aren't populated. + epoch_ra[epoch_ra == 0], epoch_dec[epoch_dec == 0] = 1991.5, 1991.5 + ra, dec = rewind_coords(gaiaobjs["RA"][igaia], gaiaobjs["DEC"][igaia], + gaiaobjs["PMRA"][igaia], gaiaobjs["PMDEC"][igaia], + epochpast=epoch_ra, epochpastdec=epoch_dec) + _, refined = radec_match_to([ra, dec], [tychoobjs["RA"][itycho], + tychoobjs["DEC"][itycho]], radec=True) + # ADM retain Tycho objects that DON'T match Gaia. + keep = np.ones(len(tychoobjs), dtype='bool') + keep[itycho[refined]] = False + tychokeep, tychomag = tychoobjs[keep], tychomag[keep] + log.info('Kept {} Tycho sources with no Gaia match...t = {:.1f} mins'.format( + len(tychokeep), (time()-t0)/60)) + + # ADM now we're done matching to Gaia, limit Gaia to the passed + # ADM magnitude limit and to the HEALPixel boundary of interest. + theta, phi = np.radians(90-gaiaobjs["DEC"]), np.radians(gaiaobjs["RA"]) + gaiahpx = hp.ang2pix(nside, theta, phi, nest=True) + ii = (gaiahpx == pixnum) & (gaiaobjs['PHOT_G_MEAN_MAG'] < maglim) + gaiakeep, uratid = gaiaobjs[ii], uratid[ii] + + # ADM finally, format according to the mask data model... + gaiamask = np.zeros(len(gaiakeep), dtype=maskdatamodel.dtype) + tychomask = np.zeros(len(tychokeep), dtype=maskdatamodel.dtype) + for col in "RA", "DEC": + gaiamask[col] = gaiakeep[col] + tychomask[col] = tychokeep[col] + gaiamask["REF_ID"] = gaiakeep["REF_ID"] + # ADM take care to rigorously convert to int64. + tychomask["REF_ID"] = tychokeep["TYC1"] + tychomask["REF_ID"] *= int(1e6) + tychomask["REF_ID"] += tychokeep["TYC2"]*10 + tychokeep["TYC3"] + gaiamask["REF_CAT"], tychomask["REF_CAT"] = 'G2', 'T2' + gaiamask["REF_MAG"] = gaiakeep['PHOT_G_MEAN_MAG'] + tychomask["REF_MAG"] = tychomag + gaiamask["URAT_ID"], tychomask["URAT_ID"] = uratid, -1 + gaiamask["TYPE"], tychomask["TYPE"] = 'PSF', 'PSF' + mask = np.concatenate([gaiamask, tychomask]) + + return mask def make_bright_star_mask_old(bands, maglim, numproc=4, @@ -202,10 +324,6 @@ def make_bright_star_mask_old(bands, maglim, numproc=4, and half that radius to set the ``IN_RADIUS``. We convert this from arcminutes to arcseconds. - It's an open question as to what the correct radii are for DESI observations. """ - # ADM set up default logger - from desiutil.log import get_logger - log = get_logger() - # ADM this is just a special case of make_bright_source_mask sourcemask = make_bright_source_mask(bands, maglim, numproc=numproc, rootdirname=rootdirname, @@ -386,10 +504,6 @@ def plot_mask(mask, limits=None, radius="IN_RADIUS", show=True): ------- Nothing """ - # ADM set up the default log. - from desiutil.log import get_logger, DEBUG - log = get_logger(DEBUG) - # ADM make this work even for a single mask. mask = np.atleast_1d(mask) @@ -460,13 +574,8 @@ def is_in_bright_mask(targs, sourcemask, inonly=False): near_mask : array_like. ``True`` for array entries that correspond to a target that is NEAR a mask. """ - t0 = time() - # ADM set up default logger. - from desiutil.log import get_logger - log = get_logger() - # ADM initialize an array of all False (nothing is yet in a mask). in_mask = np.zeros(len(targs), dtype=bool) near_mask = np.zeros(len(targs), dtype=bool) @@ -829,10 +938,6 @@ def mask_targets(targs, inmaskfile=None, nside=None, bands="GRZ", maglim=[10, 10 - Runs in about 10 minutes for 20M targets and 50k masks (roughly maglim=10). """ - # ADM set up default logger. - from desiutil.log import get_logger - log = get_logger() - t0 = time() if inmaskfile is None and outfilename is None: diff --git a/py/desitarget/gaiamatch.py b/py/desitarget/gaiamatch.py index ba4457e4a..36a35da6f 100644 --- a/py/desitarget/gaiamatch.py +++ b/py/desitarget/gaiamatch.py @@ -64,7 +64,7 @@ ]) -def _get_gaia_dir(): +def get_gaia_dir(): """Convenience function to grab the Gaia environment variable. Returns @@ -178,7 +178,7 @@ def scrape_gaia(url="http://cdn.gea.esac.esa.int/Gaia/gdr2/gaia_source/csv/", nf - Runs in about 26 hours for 60,000 Gaia files. """ # ADM check that the GAIA_DIR is set and retrieve it. - gaiadir = _get_gaia_dir() + gaiadir = get_gaia_dir() # ADM construct the directory to which to write files. csvdir = os.path.join(gaiadir, 'csv') @@ -252,7 +252,7 @@ def gaia_csv_to_fits(numproc=4): nside = _get_gaia_nside() # ADM check that the GAIA_DIR is set. - gaiadir = _get_gaia_dir() + gaiadir = get_gaia_dir() log.info("running on {} processors".format(numproc)) # ADM construct the directories for reading/writing files. @@ -376,7 +376,7 @@ def gaia_fits_to_healpix(numproc=4): nside = _get_gaia_nside() # ADM check that the GAIA_DIR is set. - gaiadir = _get_gaia_dir() + gaiadir = get_gaia_dir() # ADM construct the directories for reading/writing files. fitsdir = os.path.join(gaiadir, 'fits') @@ -496,7 +496,7 @@ def make_gaia_files(numproc=4, download=False): log.info('Begin making Gaia files...t={:.1f}s'.format(time()-t0)) # ADM check that the GAIA_DIR is set. - gaiadir = _get_gaia_dir() + gaiadir = get_gaia_dir() # ADM a quick check that the fits and healpix directories are empty # ADM before embarking on the slower parts of the code. @@ -658,7 +658,7 @@ def find_gaia_files(objs, neighbors=True, radec=False): nside = _get_gaia_nside() # ADM check that the GAIA_DIR is set and retrieve it. - gaiadir = _get_gaia_dir() + gaiadir = get_gaia_dir() hpxdir = os.path.join(gaiadir, 'healpix') # ADM which flavor of RA/Dec was passed. @@ -673,6 +673,14 @@ def find_gaia_files(objs, neighbors=True, radec=False): # ADM retrieve the pixels in which the locations lie. pixnum = hp.ang2pix(nside, theta, phi, nest=True) + + # ADM retrieve only the UNIQUE pixel numbers. It's possible that only + # ADM one pixel was produced, so guard against pixnum being non-iterable. + if not isinstance(pixnum, np.integer): + pixnum = list(set(pixnum)) + else: + pixnum = [pixnum] + # ADM if neighbors was sent, then retrieve all pixels that touch each # ADM pixel covered by the provided locations, to prevent edge effects... if neighbors: @@ -712,7 +720,7 @@ def find_gaia_files_hp(nside, pixlist, neighbors=True): filenside = _get_gaia_nside() # ADM check that the GAIA_DIR is set and retrieve it. - gaiadir = _get_gaia_dir() + gaiadir = get_gaia_dir() hpxdir = os.path.join(gaiadir, 'healpix') # ADM work with pixlist as an array. @@ -763,7 +771,7 @@ def find_gaia_files_box(gaiabounds, neighbors=True): nside = _get_gaia_nside() # ADM check that the GAIA_DIR is set and retrieve it. - gaiadir = _get_gaia_dir() + gaiadir = get_gaia_dir() hpxdir = os.path.join(gaiadir, 'healpix') # ADM determine the pixels that touch the box. @@ -810,7 +818,7 @@ def find_gaia_files_beyond_gal_b(mingalb, neighbors=True): nside = _get_gaia_nside() # ADM check that the GAIA_DIR is set and retrieve it. - gaiadir = _get_gaia_dir() + gaiadir = get_gaia_dir() hpxdir = os.path.join(gaiadir, 'healpix') # ADM determine the pixels beyond mingalb. @@ -858,7 +866,7 @@ def find_gaia_files_tiles(tiles=None, neighbors=True): nside = _get_gaia_nside() # ADM check that the GAIA_DIR is set and retrieve it. - gaiadir = _get_gaia_dir() + gaiadir = get_gaia_dir() hpxdir = os.path.join(gaiadir, 'healpix') # ADM determine the pixels that touch the tiles. @@ -1068,7 +1076,7 @@ def write_gaia_matches(infiles, numproc=4, outdir="."): - The environment variable $GAIA_DIR must be set. """ # ADM check that the GAIA_DIR is set and retrieve it. - gaiadir = _get_gaia_dir() + gaiadir = get_gaia_dir() # ADM convert a single file, if passed to a list of files. if isinstance(infiles, str): diff --git a/py/desitarget/geomask.py b/py/desitarget/geomask.py index de684580a..92b9c0261 100644 --- a/py/desitarget/geomask.py +++ b/py/desitarget/geomask.py @@ -5,7 +5,7 @@ desitarget.geomask ================== -Utility functions for restricting targets to regions on the sky +Utility functions for geometry on the sky, masking, etc. """ from __future__ import (absolute_import, division) @@ -835,6 +835,8 @@ def add_hp_neighbors(nside, pixnum): ------- :class:`list` The passed list of pixels with all neighbors added to the list. + Only unique pixels are returned, so any duplicate integers in + the passed `pixnum` are removed. Notes ----- @@ -1493,6 +1495,52 @@ def radec_match_to(matchto, objs, sep=1., radec=False, return_sep=False): return idmatchto[ii], idobjs[ii] +def rewind_coords(ranow, decnow, pmra, pmdec, epochnow=2015.5, + epochpast=1991.5, epochpastdec=None): + """Shift coordinates into the past based on proper motions. + + Parameters + ---------- + ranow : :class:`flt` or `~numpy.ndarray` + Right Ascension (degrees) at "current" epoch. + decnow : :class:`flt` or `~numpy.ndarray` + Declination (degrees) at "current" epoch. + pmra : :class:`flt` or `~numpy.ndarray` + Proper motion in RA (mas/yr). + pmdec : :class:`flt` or `~numpy.ndarray` + Proper motion in Dec (mas/yr). + epochnow : :class:`flt` or `~numpy.ndarray`, optional + The "current" epoch (years). Defaults to Gaia DR2 (2015.5). + epochpast : :class:`flt` or `~numpy.ndarray`, optional + Epoch in the past (years). Defaults to Tycho DR2 mean (1991.5). + epochpastdec : :class:`flt` or `~numpy.ndarray`, optional + If passed and not ``None`` then epochpast is interpreted as the + epoch of the RA and this is interpreted as the epoch of the Dec. + + Returns + ------- + :class:`~numpy.ndarray` + Right Ascension in the past (degrees). + :class:`~numpy.ndarray` + Declination in the past (degrees). + + Notes + ----- + - All output RAs will be in the range 0 < RA < 360o. + """ + # ADM allow for different RA/Dec coordinates. + if epochpastdec is None: + epochpastdec = epochpast + + # ADM rewind coordinates. + cosdec = np.cos(np.deg2rad(decnow)) + ra = ranow - ((epochnow-epochpast) * pmra / 3600. / 1000. / cosdec) + dec = decnow - ((epochnow-epochpastdec) * pmdec / 3600. / 1000.) + + # ADM %360. is to deal with wraparound bugs. + return ra%360., dec + + def shares_hp(nside, objs1, objs2, radec=False): """Check if arrays of objects occupy the same HEALPixels. diff --git a/py/desitarget/randoms.py b/py/desitarget/randoms.py index f8ceb09f6..2fbc04335 100644 --- a/py/desitarget/randoms.py +++ b/py/desitarget/randoms.py @@ -17,7 +17,7 @@ import photutils from glob import glob, iglob -from desitarget.gaiamatch import _get_gaia_dir +from desitarget.gaiamatch import get_gaia_dir from desitarget.geomask import bundle_bricks, box_area from desitarget.targets import resolve, main_cmx_or_sv, finalize from desitarget.skyfibers import get_brick_info @@ -864,7 +864,7 @@ def stellar_density(nside=256): - The environment variable $GAIA_DIR must be set. """ # ADM check that the GAIA_DIR is set and retrieve it. - gaiadir = _get_gaia_dir() + gaiadir = get_gaia_dir() hpdir = os.path.join(gaiadir, 'healpix') # ADM the number of pixels and the pixel area at nside. diff --git a/py/desitarget/tychomatch.py b/py/desitarget/tychomatch.py index fe5a8a1e7..63dadcb64 100644 --- a/py/desitarget/tychomatch.py +++ b/py/desitarget/tychomatch.py @@ -20,9 +20,10 @@ from glob import glob import healpy as hp +from desitarget import io from desitarget.internal import sharedmem from desimodel.footprint import radec2pix -from desitarget.geomask import add_hp_neighbors, radec_match_to +from desitarget.geomask import add_hp_neighbors, radec_match_to, nside2nside # ADM set up the DESI default logger from desiutil.log import get_logger @@ -45,7 +46,7 @@ ('JMAG', '>f4'), ('HMAG', '>f4'), ('KMAG', '>f4'), ('ZGUESS', '>f4') ]) -def _get_tycho_dir(): +def get_tycho_dir(): """Convenience function to grab the Tycho environment variable. Returns @@ -63,7 +64,7 @@ def _get_tycho_dir(): return tychodir -def _get_tycho_nside(): +def get_tycho_nside(): """Grab the HEALPixel nside to be used throughout this module. Returns @@ -96,7 +97,7 @@ def grab_tycho(cosmodir="/global/cfs/cdirs/cosmo/staging/tycho2/"): standards (e.g. all columns are converted to upper-case). """ # ADM check that the TYCHO_DIR is set and retrieve it. - tychodir = _get_tycho_dir() + tychodir = get_tycho_dir() # ADM construct the directory to which to write files. fitsdir = os.path.join(tychodir, 'fits') @@ -160,7 +161,7 @@ def tycho_fits_to_healpix(): Nothing But the archived Tycho FITS files in $TYCHO_DIR/fits are rearranged by HEALPixel in the directory $TYCHO_DIR/healpix. - The HEALPixel sense is nested with nside=_get_tycho_nside(), and + The HEALPixel sense is nested with nside=get_tycho_nside(), and each file in $TYCHO_DIR/healpix is called healpix-xxxxx.fits, where xxxxx corresponds to the HEALPixel number. @@ -169,11 +170,11 @@ def tycho_fits_to_healpix(): - The environment variable $TYCHO_DIR must be set. """ # ADM the resolution at which the Tycho HEALPix files are stored. - nside = _get_tycho_nside() + nside = get_tycho_nside() npix = hp.nside2npix(nside) # ADM check that the TYCHO_DIR is set. - tychodir = _get_tycho_dir() + tychodir = get_tycho_dir() # ADM construct the directories for reading/writing files. fitsdir = os.path.join(tychodir, "fits") @@ -227,7 +228,7 @@ def make_tycho_files(): `tychodatamodel`, and a README in $TYCHO_DIR/fits. - FITS files reorganized by HEALPixel in $TYCHO_DIR/healpix. - The HEALPixel sense is nested with nside=_get_tycho_nside(), and + The HEALPixel sense is nested with nside=get_tycho_nside(), and each file in $TYCHO_DIR/healpix is called healpix-xxxxx.fits, where xxxxx corresponds to the HEALPixel number. @@ -239,7 +240,7 @@ def make_tycho_files(): log.info('Begin making Tycho files...t={:.1f}s'.format(time()-t0)) # ADM check that the TYCHO_DIR is set. - tychodir = _get_tycho_dir() + tychodir = get_tycho_dir() # ADM a quick check that the fits and healpix directories are empty # ADM before embarking on the slower parts of the code. @@ -287,10 +288,10 @@ def find_tycho_files(objs, neighbors=True, radec=False): - The environment variable $TYCHO_DIR must be set. """ # ADM the resolution at which the Tycho HEALPix files are stored. - nside = _get_tycho_nside() + nside = get_tycho_nside() # ADM check that the TYCHO_DIR is set and retrieve it. - tychodir = _get_tycho_dir() + tychodir = get_tycho_dir() hpxdir = os.path.join(tychodir, 'healpix') # ADM which flavor of RA/Dec was passed. @@ -305,6 +306,14 @@ def find_tycho_files(objs, neighbors=True, radec=False): # ADM retrieve the pixels in which the locations lie. pixnum = hp.ang2pix(nside, theta, phi, nest=True) + + # ADM retrieve only the UNIQUE pixel numbers. It's possible that only + # ADM one pixel was produced, so guard against pixnum being non-iterable. + if not isinstance(pixnum, np.integer): + pixnum = list(set(pixnum)) + else: + pixnum = [pixnum] + # ADM if neighbors was sent, then retrieve all pixels that touch each # ADM pixel covered by the input RA/Dec, to prevent edge effects. if neighbors: @@ -319,6 +328,54 @@ def find_tycho_files(objs, neighbors=True, radec=False): return tychofiles +def find_tycho_files_hp(nside, pixlist, neighbors=True): + """Find full paths to Tycho healpix files in a set of HEALPixels. + + Parameters + ---------- + nside : :class:`int` + (NESTED) HEALPixel nside. + pixlist : :class:`list` or `int` + A set of HEALPixels at `nside`. + neighbors : :class:`bool`, optional, defaults to ``True`` + Also return files corresponding to all neighbors that touch the + pixels in `pixlist` to prevent edge effects (e.g. a Tycho source + is 1 arcsec outside of `pixlist` and so in an adjacent pixel). + + Returns + ------- + :class:`list` + A list of all Tycho files that need to be read in to account for + objects in the passed list of pixels. + + Notes + ----- + - The environment variable $TYCHO_DIR must be set. + """ + # ADM the resolution at which the healpix files are stored. + filenside = get_tycho_nside() + + # ADM check that the TYCHO_DIR is set and retrieve it. + tychodir = get_tycho_dir() + hpxdir = os.path.join(tychodir, 'healpix') + + # ADM work with pixlist as an array. + pixlist = np.atleast_1d(pixlist) + + # ADM determine the pixels that touch the passed pixlist. + pixnum = nside2nside(nside, filenside, pixlist) + + # ADM if neighbors was sent, then retrieve all pixels that touch each + # ADM pixel covered by the provided locations, to prevent edge effects... + if neighbors: + pixnum = add_hp_neighbors(filenside, pixnum) + + # ADM reformat in the healpix format used by desitarget. + tychofiles = [os.path.join(hpxdir, io.hpx_filename(pn)) for pn in pixnum] + + return tychofiles + + def match_to_tycho(objs, matchrad=1., radec=False): """Match objects to Tycho healpixel files. diff --git a/py/desitarget/uratmatch.py b/py/desitarget/uratmatch.py index bebf3e013..d8ddad255 100644 --- a/py/desitarget/uratmatch.py +++ b/py/desitarget/uratmatch.py @@ -24,6 +24,7 @@ from desitarget.internal import sharedmem from desimodel.footprint import radec2pix from desitarget.geomask import add_hp_neighbors, radec_match_to +from desitarget import io # ADM set up the DESI default logger from desiutil.log import get_logger @@ -544,6 +545,14 @@ def find_urat_files(objs, neighbors=True, radec=False): # ADM retrieve the pixels in which the locations lie. pixnum = hp.ang2pix(nside, theta, phi, nest=True) + + # ADM retrieve only the UNIQUE pixel numbers. It's possible that only + # ADM one pixel was produced, so guard against pixnum being non-iterable. + if not isinstance(pixnum, np.integer): + pixnum = list(set(pixnum)) + else: + pixnum = [pixnum] + # ADM if neighbors was sent, then retrieve all pixels that touch each # ADM pixel covered by the provided locations, to prevent edge effects... if neighbors: From 0233494bd122dcbba3d896754aab273e51c7a9f9 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 9 Jun 2020 12:57:02 -0700 Subject: [PATCH 06/25] add function to calculate the radius-magnitude relation --- py/desitarget/brightmask.py | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/py/desitarget/brightmask.py b/py/desitarget/brightmask.py index 7033e23c1..777c648bf 100644 --- a/py/desitarget/brightmask.py +++ b/py/desitarget/brightmask.py @@ -50,13 +50,6 @@ matplotlib.use('Agg') import matplotlib.pyplot as plt # noqa: E402 -# ADM factor by which the "in" radius is smaller than the "near" radius -infac = 0.5 -# ADM and vice-versa. -nearfac = 1./infac -# ADM size of the bright star mask (NEAR_RADIUS) in arcseconds. -masksize = 5. - maskdatamodel = np.array([], dtype=[ ('RA', '>f8'), ('DEC', '>f8'), ('REF_CAT', '|S2'), ('REF_ID', '>i8'), ('REF_MAG', '>f4'), @@ -65,6 +58,32 @@ ]) +def radius(mag): + """The relation used to set the radius of bright star masks. + + Parameters + ---------- + mag : :class:`flt` or :class:`recarray` + Magnitude. Typically, in order of preference, G-band for Gaia + or VT then HP then BT for Tycho. + + Returns + ------- + :class:`recarray` + The `IN_RADIUS`, corresponding to the `IN_BRIGHT_OBJECT` bit + in `data/targetmask.yaml`. + :class:`recarray` + The `NEAR_RADIUS`, corresponding to the `NEAR_BRIGHT_OBJECT` bit + in data/targetmask.yaml`. + """ + # ADM mask all sources with mag < 12 at 5 arcsecs. + nearrad = (mag < 12.) * 5 + # ADM the IN_RADIUS is half the near radius. + inrad = nearrad/2. + + return inrad, nearrad + + def _rexlike(rextype): """If the object is REX (a round exponential galaxy)""" From ac68d0d438046003279d019299f836608ef23a1a Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 9 Jun 2020 13:10:52 -0700 Subject: [PATCH 07/25] implement radius-magnitude function; deprecate old bright star/source mask code --- py/desitarget/brightmask.py | 215 +----------------------------------- 1 file changed, 3 insertions(+), 212 deletions(-) diff --git a/py/desitarget/brightmask.py b/py/desitarget/brightmask.py index 777c648bf..ac7a0b30f 100644 --- a/py/desitarget/brightmask.py +++ b/py/desitarget/brightmask.py @@ -58,7 +58,7 @@ ]) -def radius(mag): +def radii(mag): """The relation used to set the radius of bright star masks. Parameters @@ -287,221 +287,12 @@ def make_bright_star_mask_in_hp(nside, pixnum, gaiamask["URAT_ID"], tychomask["URAT_ID"] = uratid, -1 gaiamask["TYPE"], tychomask["TYPE"] = 'PSF', 'PSF' mask = np.concatenate([gaiamask, tychomask]) + # ADM ...and add the mask radii. + mask["IN_RADIUS"], mask["NEAR_RADIUS"] = radii(mask["REF_MAG"]) return mask -def make_bright_star_mask_old(bands, maglim, numproc=4, - rootdirname='/global/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1', - infilename=None, outfilename=None): - """Make a bright star mask from a structure of bright stars drawn from the sweeps. - - Parameters - ---------- - bands : :class:`str` - A magnitude band from the sweeps, e.g., "G", "R", "Z". - Can pass multiple bands as string, e.g. ``"GRZ"``, in which case maglim has to be a - list of the same length as the string. - maglim : :class:`float` - The upper limit in that magnitude band for which to assemble a list of bright stars. - Can pass a list of magnitude limits, in which case bands has to be a string of the - same length (e.g., ``"GRZ"`` for [12.3,12.7,12.6]). - numproc : :class:`int`, optional - Number of processes over which to parallelize. - rootdirname : :class:`str`, optional, defaults to dr3 - Root directory containing either sweeps or tractor files...e.g. for dr3 this might be - /global/project/projectdirs/cosmo/data/legacysurvey/dr3/sweep/dr3.1. This is only - used if ``infilename`` is not passed. - infilename : :class:`str`, optional, - if this exists, then the list of bright stars is read in from the file of this name - if this is not passed, then code defaults to deriving the recarray of bright stars - from ``rootdirname`` via a call to ``collect_bright_stars``. - outfilename : :class:`str`, optional, defaults to not writing anything to file - (FITS) File name to which to write the output bright star mask. - - Returns - ------- - :class:`recarray` - - The bright source mask in the form ``RA``, ``DEC``, ``TARGETID``, - ``IN_RADIUS``, ``NEAR_RADIUS``, ``E1``, ``E2``, ``TYPE`` - (may also be written to file if "outfilename" is passed). - - TARGETID is as calculated in :mod:`desitarget.targets.encode_targetid`. - - The radii are in ARCSECONDS (they default to equivalents of half-light radii for ellipses). - - `E1` and `E2` are ellipticity components, which are 0 for unresolved objects. - - `TYPE` is always `PSF` for star-like objects. This is taken from the sweeps files, see, e.g.: - http://legacysurvey.org/dr5/files/#sweep-catalogs. - - Notes - ----- - - ``IN_RADIUS`` is a smaller radius that corresponds to the ``IN_BRIGHT_OBJECT`` bit in - ``data/targetmask.yaml`` (and is in ARCSECONDS). - - ``NEAR_RADIUS`` is a radius that corresponds to the ``NEAR_BRIGHT_OBJECT`` bit in - ``data/targetmask.yaml`` (and is in ARCSECONDS). - - Currently uses the radius-as-a-function-of-B-mag for Tycho stars from the BOSS mask - (in every band) to set the ``NEAR_RADIUS``: - R = (0.0802B*B - 1.860B + 11.625) (see Eqn. 9 of https://arxiv.org/pdf/1203.6594.pdf) - and half that radius to set the ``IN_RADIUS``. We convert this from arcminutes to arcseconds. - - It's an open question as to what the correct radii are for DESI observations. - """ - # ADM this is just a special case of make_bright_source_mask - sourcemask = make_bright_source_mask(bands, maglim, - numproc=numproc, rootdirname=rootdirname, - infilename=infilename, outfilename=None) - # ADM check if a source is unresolved. - psflike = _psflike(sourcemask["TYPE"]) - wstar = np.where(psflike) - if len(wstar[0]) > 0: - done = sourcemask[wstar] - if outfilename is not None: - fitsio.write(outfilename, done, clobber=True) - return done - else: - log.error('No PSF-like objects brighter than {} in {}' - .format(str(maglim), bands)) - return -1 - - -def make_bright_source_mask(bands, maglim, numproc=4, - rootdirname='/global/project/projectdirs/cosmo/data/legacysurvey/dr5/sweep/5.0', - infilename=None, outfilename=None): - """Make a mask of bright sources from a structure of bright sources drawn from the sweeps. - - Parameters - ---------- - bands : :class:`str` - A magnitude band from the sweeps, e.g., "G", "R", "Z". - Can pass multiple bands as string, e.g. ``"GRZ"``, in which case maglim has to be a - list of the same length as the string. - maglim : :class:`float` - The upper limit in that magnitude band for which to assemble a list of bright sources. - Can pass a list of magnitude limits, in which case bands has to be a string of the - same length (e.g., ``"GRZ"`` for [12.3,12.7,12.6]). - numproc : :class:`int`, optional - Number of processes over which to parallelize. - rootdirname : :class:`str`, optional, defaults to dr3 - Root directory containing either sweeps or tractor files...e.g. for dr5 this might be - ``/global/project/projectdirs/cosmo/data/legacysurvey/dr5/sweep/dr5.0``. This is only - used if ``infilename`` is not passed. - infilename : :class:`str`, optional, - if this exists, then the list of bright sources is read in from the file of this name. - if this is not passed, then code defaults to deriving the recarray of bright sources - from ``rootdirname`` via a call to ``collect_bright_sources``. - outfilename : :class:`str`, optional, defaults to not writing anything to file - (FITS) File name to which to write the output bright source mask. - - Returns - ------- - :class:`recarray` - - The bright source mask in the form ``RA`, ``DEC``, ``TARGETID``, ``IN_RADIUS``, - ``NEAR_RADIUS``, ``E1``, ``E2``, ``TYPE`` - (may also be written to file if ``outfilename`` is passed). - - ``TARGETID`` is as calculated in :mod:`desitarget.targets.encode_targetid`. - - The radii are in ARCSECONDS (they default to equivalents of half-light radii for ellipses). - - ``E1`` and ``E2`` are the ellipticity components as defined at the bottom of, e.g.: - http://legacysurvey.org/dr5/catalogs/. - - ``TYPE`` is the ``TYPE`` from the sweeps files, see, e.g.: - http://legacysurvey.org/dr5/files/#sweep-catalogs. - - Notes - ----- - - ``IN_RADIUS`` is a smaller radius that corresponds to the ``IN_BRIGHT_OBJECT`` bit in - ``data/targetmask.yaml`` (and is in ARCSECONDS). - - ``NEAR_RADIUS`` is a radius that corresponds to the ``NEAR_BRIGHT_OBJECT`` bit in - ``data/targetmask.yaml`` (and is in ARCSECONDS). - - Currently uses the radius-as-a-function-of-B-mag for Tycho stars from the BOSS mask - (in every band) to set the ``NEAR_RADIUS``: - R = (0.0802B*B - 1.860B + 11.625) (see Eqn. 9 of https://arxiv.org/pdf/1203.6594.pdf) - and half that radius to set the ``IN_RADIUS``. We convert this from arcminutes to arcseconds. - - It's an open question as to what the correct radii are for DESI observations. - """ - - # ADM set bands to uppercase if passed as lower case. - bands = bands.upper() - # ADM the band names and nobs columns as arrays instead of strings. - bandnames = np.array(["FLUX_"+band for band in bands]) - nobsnames = np.array(["NOBS_"+band for band in bands]) - - # ADM force the input maglim to be a list (in case a single value was passed). - if isinstance(maglim, int) or isinstance(maglim, float): - maglim = [maglim] - - if len(bandnames) != len(maglim): - msg = "bands has to be the same length as maglim and {} does not equal {}".format( - len(bandnames), len(maglim)) - raise IOError(msg) - - # ADM change input magnitude(s) to a flux to test against. - fluxlim = 10.**((22.5-np.array(maglim))/2.5) - - if infilename is not None: - objs = io.read_tractor(infilename) - else: - objs = collect_bright_sources(bands, maglim, numproc, rootdirname, outfilename) - - # ADM write the fluxes and bands as arrays instead of named columns - - # ADM limit to the passed faint limit. - ok = np.zeros(objs[bandnames[0]].shape, dtype=bool) - fluxes = np.zeros((len(ok), len(bandnames)), dtype=objs[bandnames[0]].dtype) - for i, (bandname, nobsname) in enumerate(zip(bandnames, nobsnames)): - fluxes[:, i] = objs[bandname].copy() - # ADM set any observations with NOBS = 0 to have small flux - # so glitches don't end up as bright object masks. - fluxes[objs[nobsname] == 0, i] = 0.0 - ok |= (fluxes[:, i] > fluxlim[i]) - - w = np.where(ok) - - fluxes = fluxes[w] - objs = objs[w] - - # ADM grab the (GRZ) magnitudes for observations - # ADM and record only the largest flux (smallest magnitude). - fluxmax = np.max(fluxes, axis=1) - mags = 22.5-2.5*np.log10(fluxmax) - - # ADM each object's TYPE. - objtype = objs["TYPE"] - - # ADM calculate the TARGETID. - targetid = encode_targetid(objid=objs['OBJID'], brickid=objs['BRICKID'], release=objs['RELEASE']) - - # ADM first set the shape parameters assuming everything is an exponential - # ADM this will correctly assign e1, e2 of 0 to things with zero shape. - in_radius = objs['SHAPEEXP_R'] - e1 = objs['SHAPEEXP_E1'] - e2 = objs['SHAPEEXP_E2'] - # ADM now to account for deVaucouleurs objects, or things that are dominated by - # ADM deVaucouleurs profiles, update objects with a larger "DEV" than "EXP" radius. - wdev = np.where(objs['SHAPEDEV_R'] > objs['SHAPEEXP_R']) - if len(wdev[0]) > 0: - in_radius[wdev] = objs[wdev]['SHAPEDEV_R'] - e1[wdev] = objs[wdev]['SHAPEDEV_E1'] - e2[wdev] = objs[wdev]['SHAPEDEV_E2'] - # ADM finally use the Tycho radius (see the notes above) for PSF or star-like objects. - # ADM More consideration will be needed to derive correct numbers for this for DESI!!! - # ADM this calculation was for "near" Tycho objects and was in arcmin, so we convert - # ADM it to arcsec and multiply it by infac (see the top of the module). - tycho_in_radius = infac*(0.0802*mags*mags - 1.860*mags + 11.625)*60. - wpsf = np.where(_psflike(objtype)) - in_radius[wpsf] = tycho_in_radius[wpsf] - - # ADM set "near" as a multiple of "in" radius using the factor at the top of the code. - near_radius = in_radius*nearfac - - # ADM create an output recarray that is just RA, Dec, TARGETID and the radius. - done = objs[['RA', 'DEC']].copy() - done = rfn.append_fields(done, ["TARGETID", "IN_RADIUS", "NEAR_RADIUS", "E1", "E2", "TYPE"], - [targetid, in_radius, near_radius, e1, e2, objtype], - usemask=False, - dtypes=['>i8', '>f4', '>f4', '>f4', '>f4', '|S4']) - - if outfilename is not None: - fitsio.write(outfilename, done, clobber=True) - - return done - - def plot_mask(mask, limits=None, radius="IN_RADIUS", show=True): """Make a plot of a mask and either display it or retain the plot object for over-plotting. From b806c5fa556f143581a78141492cdfe32fb64503 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 9 Jun 2020 16:28:28 -0700 Subject: [PATCH 08/25] make mask at a specific input epoch; fix int64-type bug in calculating the REF_ID for Tycho --- py/desitarget/brightmask.py | 68 ++++++++++++++++++++++++------------- py/desitarget/geomask.py | 24 +++++++++++-- 2 files changed, 66 insertions(+), 26 deletions(-) diff --git a/py/desitarget/brightmask.py b/py/desitarget/brightmask.py index ac7a0b30f..52f2f57fd 100644 --- a/py/desitarget/brightmask.py +++ b/py/desitarget/brightmask.py @@ -51,7 +51,7 @@ import matplotlib.pyplot as plt # noqa: E402 maskdatamodel = np.array([], dtype=[ - ('RA', '>f8'), ('DEC', '>f8'), + ('RA', '>f8'), ('DEC', '>f8'), ('PMRA', '>f4'), ('PMDEC', '>f4'), ('REF_CAT', '|S2'), ('REF_ID', '>i8'), ('REF_MAG', '>f4'), ('URAT_ID', '>i8'), ('IN_RADIUS', '>f4'), ('NEAR_RADIUS', '>f4'), ('E1', '>f4'), ('E2', '>f4'), ('TYPE', '|S3') @@ -132,8 +132,8 @@ def max_objid_bricks(targs): return dict(ordered[maxind]) -def make_bright_star_mask_in_hp(nside, pixnum, - maglim=12., matchrad=1., epoch=2023.0): +def make_bright_star_mask_in_hp(nside, pixnum, gaiaepoch=2015.5, + maglim=12., matchrad=1., maskepoch=2023.0): """Make a bright star mask in a HEALPixel using Tycho, Gaia and URAT. Parameters @@ -142,6 +142,9 @@ def make_bright_star_mask_in_hp(nside, pixnum, (NESTED) HEALPixel nside. pixnum : :class:`int` A single HEALPixel number. + gaiaepoch: :class:`float`, optional, defaults to Gaia DR2 (2015.5) + The epoch of the Gaia observations. Should be 2015.5 unless we + move beyond Gaia DR2. maglim : :class:`float`, optional, defaults to 12. Faintest magnitude at which to make the mask. This magnitude is interpreted as G-band for Gaia and, in order of preference, VT @@ -150,7 +153,7 @@ def make_bright_star_mask_in_hp(nside, pixnum, Tycho sources that match a Gaia source at this separation in ARCSECONDS are NOT included in the output mask. The matching is performed rigorously, accounting for Gaia proper motions. - epoch : :class:`float` + maskepoch : :class:`float` The mask is built at this epoch. Not all sources have proper motions from every survey, so proper motions are used, in order of preference, from Gaia, URAT, then Tycho. @@ -197,8 +200,8 @@ def make_bright_star_mask_in_hp(nside, pixnum, # ADM discard any Tycho objects below the input magnitude limit. ii = tychomag < maglim tychomag, tychoobjs = tychomag[ii], tychoobjs[ii] - log.info('Read {} (mag < {}) Tycho objects...t = {:.1f} mins'.format( - np.sum(ii), maglim, (time()-t0)/60)) + log.info('Read {} (mag < {}) Tycho objects (pix={})...t={:.1f} mins'.format( + np.sum(ii), maglim, pixnum, (time()-t0)/60)) # ADM read in the associated Gaia files. Also grab # ADM neighboring pixels to prevent edge effects. @@ -212,17 +215,17 @@ def make_bright_star_mask_in_hp(nside, pixnum, # ADM limit Gaia objects to 3 magnitudes fainter than the passed # ADM limit. This leaves some (!) leeway when matching to Tycho. gaiaobjs = gaiaobjs[gaiaobjs['PHOT_G_MEAN_MAG'] < maglim + 3] - log.info('Read {} (G < {}) Gaia sources...t = {:.1f} mins'.format( - len(gaiaobjs), maglim + 3, (time()-t0)/60)) + log.info('Read {} (G < {}) Gaia sources (pix={})...t={:.1f} mins'.format( + len(gaiaobjs), maglim+3, pixnum, (time()-t0)/60)) # ADM substitute URAT where Gaia proper motions don't exist. ii = ((np.isnan(gaiaobjs["PMRA"]) | (gaiaobjs["PMRA"] == 0)) & (np.isnan(gaiaobjs["PMDEC"]) | (gaiaobjs["PMDEC"] == 0))) - log.info('Add URAT for {} Gaia objects with no PMs...t = {:.1f} mins'.format( - np.sum(ii), (time()-t0)/60)) + log.info('Add URAT for {} Gaia objects with no PMs (pix={})...t={:.1f} mins' + .format(np.sum(ii), pixnum, (time()-t0)/60)) urat = add_urat_pms(gaiaobjs[ii], numproc=1) - log.info('Found an additional {} URAT objects...t = {:.1f} mins'.format( - np.sum(urat["URAT_ID"] != -1), (time()-t0)/60)) + log.info('Found an additional {} URAT objects (pix={})...t={:.1f} mins' + .format(np.sum(urat["URAT_ID"] != -1), pixnum, (time()-t0)/60)) for col in "PMRA", "PMDEC": gaiaobjs[col][ii] = urat[col] # ADM need to track the URATID to track which objects have @@ -237,15 +240,16 @@ def make_bright_star_mask_in_hp(nside, pixnum, # ADM (10.3 arcsec) can be off by a significant margin (~10"). margin = 10. ra, dec = rewind_coords(gaiaobjs["RA"], gaiaobjs["DEC"], - gaiaobjs["PMRA"], gaiaobjs["PMDEC"]) + gaiaobjs["PMRA"], gaiaobjs["PMDEC"], + epochnow=gaiaepoch) # ADM match Gaia to Tycho with a suitable margin. - log.info('Match Gaia to Tycho with margin={}"...t = {:.1f} mins'.format( - margin, (time()-t0)/60)) + log.info('Match Gaia to Tycho with margin={}" (pix={})...t={:.1f} mins' + .format(margin, pixnum, (time()-t0)/60)) igaia, itycho = radec_match_to([ra, dec], [tychoobjs["RA"], tychoobjs["DEC"]], sep=margin, radec=True) - log.info('{} matches. Refining at 1"...t = {:.1f} mins'.format( - len(itycho), (time()-t0)/60)) + log.info('{} matches. Refining at 1" (pix={})...t={:.1f} mins'.format( + len(itycho), pixnum, (time()-t0)/60)) # ADM match Gaia to Tycho at the more exact reference epoch. epoch_ra = tychoobjs[itycho]["EPOCH_RA"] epoch_dec = tychoobjs[itycho]["EPOCH_DEC"] @@ -253,6 +257,7 @@ def make_bright_star_mask_in_hp(nside, pixnum, epoch_ra[epoch_ra == 0], epoch_dec[epoch_dec == 0] = 1991.5, 1991.5 ra, dec = rewind_coords(gaiaobjs["RA"][igaia], gaiaobjs["DEC"][igaia], gaiaobjs["PMRA"][igaia], gaiaobjs["PMDEC"][igaia], + epochnow = gaiaepoch, epochpast=epoch_ra, epochpastdec=epoch_dec) _, refined = radec_match_to([ra, dec], [tychoobjs["RA"][itycho], tychoobjs["DEC"][itycho]], radec=True) @@ -260,8 +265,8 @@ def make_bright_star_mask_in_hp(nside, pixnum, keep = np.ones(len(tychoobjs), dtype='bool') keep[itycho[refined]] = False tychokeep, tychomag = tychoobjs[keep], tychomag[keep] - log.info('Kept {} Tycho sources with no Gaia match...t = {:.1f} mins'.format( - len(tychokeep), (time()-t0)/60)) + log.info('Kept {} Tycho sources with no Gaia match (pix={})...t={:.1f} mins' + .format(len(tychokeep), pixnum, (time()-t0)/60)) # ADM now we're done matching to Gaia, limit Gaia to the passed # ADM magnitude limit and to the HEALPixel boundary of interest. @@ -269,18 +274,35 @@ def make_bright_star_mask_in_hp(nside, pixnum, gaiahpx = hp.ang2pix(nside, theta, phi, nest=True) ii = (gaiahpx == pixnum) & (gaiaobjs['PHOT_G_MEAN_MAG'] < maglim) gaiakeep, uratid = gaiaobjs[ii], uratid[ii] + log.info('Mask also comprises {} Gaia sources (pix={})...t={:.1f} mins' + .format(len(gaiakeep), pixnum, (time()-t0)/60)) + + # ADM move the coordinates forwards to the input mask epoch. + epoch_ra, epoch_dec = tychokeep["EPOCH_RA"], tychokeep["EPOCH_DEC"] + # ADM some of the Tycho epochs aren't populated. + epoch_ra[epoch_ra == 0], epoch_dec[epoch_dec == 0] = 1991.5, 1991.5 + ra, dec = rewind_coords( + tychokeep["RA"], tychokeep["DEC"], tychokeep["PM_RA"], tychokeep["PM_DEC"], + epochnow=epoch_ra, epochnowdec=epoch_dec, epochpast=maskepoch) + tychokeep["RA"], tychokeep["DEC"] = ra, dec + ra, dec = rewind_coords( + gaiakeep["RA"], gaiakeep["DEC"], gaiakeep["PMRA"], gaiakeep["PMDEC"], + epochnow=gaiaepoch, epochpast=maskepoch) + gaiakeep["RA"], gaiakeep["DEC"] = ra, dec # ADM finally, format according to the mask data model... gaiamask = np.zeros(len(gaiakeep), dtype=maskdatamodel.dtype) tychomask = np.zeros(len(tychokeep), dtype=maskdatamodel.dtype) for col in "RA", "DEC": gaiamask[col] = gaiakeep[col] + gaiamask["PM"+col] = gaiakeep["PM"+col] tychomask[col] = tychokeep[col] + tychomask["PM"+col] = tychokeep["PM_"+col] gaiamask["REF_ID"] = gaiakeep["REF_ID"] - # ADM take care to rigorously convert to int64. - tychomask["REF_ID"] = tychokeep["TYC1"] - tychomask["REF_ID"] *= int(1e6) - tychomask["REF_ID"] += tychokeep["TYC2"]*10 + tychokeep["TYC3"] + # ADM take care to rigorously convert to int64 for Tycho. + tychomask["REF_ID"] = tychokeep["TYC1"].astype('int64')*int(1e6) + \ + tychokeep["TYC2"].astype('int64')*10 + \ + tychokeep["TYC3"] gaiamask["REF_CAT"], tychomask["REF_CAT"] = 'G2', 'T2' gaiamask["REF_MAG"] = gaiakeep['PHOT_G_MEAN_MAG'] tychomask["REF_MAG"] = tychomag diff --git a/py/desitarget/geomask.py b/py/desitarget/geomask.py index 92b9c0261..8b2f72a7b 100644 --- a/py/desitarget/geomask.py +++ b/py/desitarget/geomask.py @@ -1495,7 +1495,8 @@ def radec_match_to(matchto, objs, sep=1., radec=False, return_sep=False): return idmatchto[ii], idobjs[ii] -def rewind_coords(ranow, decnow, pmra, pmdec, epochnow=2015.5, +def rewind_coords(ranow, decnow, pmra, pmdec, + epochnow=2015.5, epochnowdec=None, epochpast=1991.5, epochpastdec=None): """Shift coordinates into the past based on proper motions. @@ -1511,6 +1512,9 @@ def rewind_coords(ranow, decnow, pmra, pmdec, epochnow=2015.5, Proper motion in Dec (mas/yr). epochnow : :class:`flt` or `~numpy.ndarray`, optional The "current" epoch (years). Defaults to Gaia DR2 (2015.5). + epochnowdec : :class:`flt` or `~numpy.ndarray`, optional + If passed and not ``None`` then epochnow is interpreted as the + epoch of the RA and this is interpreted as the epoch of the Dec. epochpast : :class:`flt` or `~numpy.ndarray`, optional Epoch in the past (years). Defaults to Tycho DR2 mean (1991.5). epochpastdec : :class:`flt` or `~numpy.ndarray`, optional @@ -1527,15 +1531,29 @@ def rewind_coords(ranow, decnow, pmra, pmdec, epochnow=2015.5, Notes ----- - All output RAs will be in the range 0 < RA < 360o. + - Only called "rewind_coords" to correspond to the default + `epochnow` > `epochpast`. "fast forwarding" works fine, too, + i.e., you can pass `epochpast` > `epochnow` to move coordinates + to a future epoch. + - Inaccurate to ~0.1" for motions as high as ~10"/yr (Barnard's + Star) after ~200 years because of the simplified cosdec term. """ # ADM allow for different RA/Dec coordinates. + if epochnowdec is None: + epochnowdec = epochnow if epochpastdec is None: epochpastdec = epochpast - # ADM rewind coordinates. + # ADM enforce "double-type" precision for RA/Dec floats. + if isinstance(ranow, float): + ranow = np.array([ranow], dtype='f8') + if isinstance(decnow, float): + decnow = np.array([decnow], dtype='f8') + + # ADM "rewind" coordinates. cosdec = np.cos(np.deg2rad(decnow)) ra = ranow - ((epochnow-epochpast) * pmra / 3600. / 1000. / cosdec) - dec = decnow - ((epochnow-epochpastdec) * pmdec / 3600. / 1000.) + dec = decnow - ((epochnowdec-epochpastdec) * pmdec / 3600. / 1000.) # ADM %360. is to deal with wraparound bugs. return ra%360., dec From 52a23f75405f5969df1ff945309a624f81daf4f9 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Wed, 10 Jun 2020 09:33:41 -0700 Subject: [PATCH 09/25] typos and code style --- etc/desitarget.module | 2 +- py/desitarget/brightmask.py | 14 +++++++++----- py/desitarget/geomask.py | 4 ++-- py/desitarget/tychomatch.py | 3 ++- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/etc/desitarget.module b/etc/desitarget.module index 95100526a..77c047e2f 100644 --- a/etc/desitarget.module +++ b/etc/desitarget.module @@ -80,7 +80,7 @@ setenv TARG_DIR /global/cfs/cdirs/desi/target/catalogs setenv DUST_DIR /global/cfs/cdirs/cosmo/data/dust/v0_1 setenv CMX_DIR /global/cfs/cdirs/desi/target/cmx_files setenv SCND_DIR /global/cfs/cdirs/desi/target/secondary -setenc MASK_DIR /global/cfs/cdirs/desi/target/masks +setenv MASK_DIR /global/cfs/cdirs/desi/target/masks setenv GAIA_DIR /global/cfs/cdirs/desi/target/gaia_dr2 setenv URAT_DIR /global/cfs/cdirs/desi/target/urat_dr1 setenv TYCHO_DIR /global/cfs/cdirs/desi/target/tycho_dr2 diff --git a/py/desitarget/brightmask.py b/py/desitarget/brightmask.py index 52f2f57fd..9394293ed 100644 --- a/py/desitarget/brightmask.py +++ b/py/desitarget/brightmask.py @@ -43,13 +43,14 @@ from desiutil import depend, brick # ADM set up default logger from desiutil.log import get_logger -log = get_logger() # ADM fake the matplotlib display so it doesn't die on allocated nodes. import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt # noqa: E402 +log = get_logger() + maskdatamodel = np.array([], dtype=[ ('RA', '>f8'), ('DEC', '>f8'), ('PMRA', '>f4'), ('PMDEC', '>f4'), ('REF_CAT', '|S2'), ('REF_ID', '>i8'), ('REF_MAG', '>f4'), @@ -173,6 +174,10 @@ def make_bright_star_mask_in_hp(nside, pixnum, gaiaepoch=2015.5, - `E1` and `E2` are placeholders for ellipticity components, and are set to 0 for Gaia and Tycho sources. - `TYPE` is always `PSF` for star-like objects. + - Note that the mask is based on objects in the pixel AT THEIR + NATIVE EPOCH *NOT* AT THE INPUT `maskepoch`. It is therefore + possible for locations in the output mask to be just beyond + the boundaries of the input pixel. Notes ----- @@ -211,7 +216,7 @@ def make_bright_star_mask_in_hp(nside, pixnum, gaiaepoch=2015.5, for fn in gaiafns: gaiaobjs.append(fitsio.read(fn, ext='GAIAHPX', columns=cols)) gaiaobjs = np.concatenate(gaiaobjs) - gaiaobjs = rfn.rename_fields(gaiaobjs, {"SOURCE_ID":"REF_ID"}) + gaiaobjs = rfn.rename_fields(gaiaobjs, {"SOURCE_ID": "REF_ID"}) # ADM limit Gaia objects to 3 magnitudes fainter than the passed # ADM limit. This leaves some (!) leeway when matching to Tycho. gaiaobjs = gaiaobjs[gaiaobjs['PHOT_G_MEAN_MAG'] < maglim + 3] @@ -257,7 +262,7 @@ def make_bright_star_mask_in_hp(nside, pixnum, gaiaepoch=2015.5, epoch_ra[epoch_ra == 0], epoch_dec[epoch_dec == 0] = 1991.5, 1991.5 ra, dec = rewind_coords(gaiaobjs["RA"][igaia], gaiaobjs["DEC"][igaia], gaiaobjs["PMRA"][igaia], gaiaobjs["PMDEC"][igaia], - epochnow = gaiaepoch, + epochnow=gaiaepoch, epochpast=epoch_ra, epochpastdec=epoch_dec) _, refined = radec_match_to([ra, dec], [tychoobjs["RA"][itycho], tychoobjs["DEC"][itycho]], radec=True) @@ -301,8 +306,7 @@ def make_bright_star_mask_in_hp(nside, pixnum, gaiaepoch=2015.5, gaiamask["REF_ID"] = gaiakeep["REF_ID"] # ADM take care to rigorously convert to int64 for Tycho. tychomask["REF_ID"] = tychokeep["TYC1"].astype('int64')*int(1e6) + \ - tychokeep["TYC2"].astype('int64')*10 + \ - tychokeep["TYC3"] + tychokeep["TYC2"].astype('int64')*10 + tychokeep["TYC3"] gaiamask["REF_CAT"], tychomask["REF_CAT"] = 'G2', 'T2' gaiamask["REF_MAG"] = gaiakeep['PHOT_G_MEAN_MAG'] tychomask["REF_MAG"] = tychomag diff --git a/py/desitarget/geomask.py b/py/desitarget/geomask.py index 8b2f72a7b..333ad338b 100644 --- a/py/desitarget/geomask.py +++ b/py/desitarget/geomask.py @@ -1555,8 +1555,8 @@ def rewind_coords(ranow, decnow, pmra, pmdec, ra = ranow - ((epochnow-epochpast) * pmra / 3600. / 1000. / cosdec) dec = decnow - ((epochnowdec-epochpastdec) * pmdec / 3600. / 1000.) - # ADM %360. is to deal with wraparound bugs. - return ra%360., dec + # ADM % 360. is to deal with wraparound bugs. + return ra % 360., dec def shares_hp(nside, objs1, objs2, radec=False): diff --git a/py/desitarget/tychomatch.py b/py/desitarget/tychomatch.py index 63dadcb64..7a31e1a90 100644 --- a/py/desitarget/tychomatch.py +++ b/py/desitarget/tychomatch.py @@ -46,6 +46,7 @@ ('JMAG', '>f4'), ('HMAG', '>f4'), ('KMAG', '>f4'), ('ZGUESS', '>f4') ]) + def get_tycho_dir(): """Convenience function to grab the Tycho environment variable. @@ -147,7 +148,7 @@ def grab_tycho(cosmodir="/global/cfs/cdirs/cosmo/staging/tycho2/"): readme = f.read() with open(routfile, 'w') as f: f.write(readme+msg) - + log.info('Wrote Tycho FITS file...t={:.1f}s'.format(time()-start)) return From 0061a089960a5d77c984029497217acae7eb36df Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Wed, 10 Jun 2020 13:31:14 -0700 Subject: [PATCH 10/25] parallelize making the mask --- py/desitarget/brightmask.py | 196 +++++++++++++++++++++++++----------- 1 file changed, 140 insertions(+), 56 deletions(-) diff --git a/py/desitarget/brightmask.py b/py/desitarget/brightmask.py index 9394293ed..5bc1b19d3 100644 --- a/py/desitarget/brightmask.py +++ b/py/desitarget/brightmask.py @@ -8,8 +8,6 @@ .. _`Tech Note 2346`: https://desi.lbl.gov/DocDB/cgi-bin/private/ShowDocument?docid=2346 .. _`Tech Note 2348`: https://desi.lbl.gov/DocDB/cgi-bin/private/ShowDocument?docid=2348 -.. _`the DR5 sweeps`: http://legacysurvey.org/dr5/files/#sweep-catalogs -.. _`Legacy Surveys catalogs`: http://legacysurvey.org/dr5/catalogs/ """ from time import time import fitsio @@ -133,7 +131,7 @@ def max_objid_bricks(targs): return dict(ordered[maxind]) -def make_bright_star_mask_in_hp(nside, pixnum, gaiaepoch=2015.5, +def make_bright_star_mask_in_hp(nside, pixnum, verbose=True, gaiaepoch=2015.5, maglim=12., matchrad=1., maskepoch=2023.0): """Make a bright star mask in a HEALPixel using Tycho, Gaia and URAT. @@ -143,50 +141,19 @@ def make_bright_star_mask_in_hp(nside, pixnum, gaiaepoch=2015.5, (NESTED) HEALPixel nside. pixnum : :class:`int` A single HEALPixel number. - gaiaepoch: :class:`float`, optional, defaults to Gaia DR2 (2015.5) - The epoch of the Gaia observations. Should be 2015.5 unless we - move beyond Gaia DR2. - maglim : :class:`float`, optional, defaults to 12. - Faintest magnitude at which to make the mask. This magnitude is - interpreted as G-band for Gaia and, in order of preference, VT - then HP then BT for Tycho (not every Tycho source has each band). - matchrad : :class:`int`, optional, defaults to 1. - Tycho sources that match a Gaia source at this separation in - ARCSECONDS are NOT included in the output mask. The matching is - performed rigorously, accounting for Gaia proper motions. - maskepoch : :class:`float` - The mask is built at this epoch. Not all sources have proper - motions from every survey, so proper motions are used, in order - of preference, from Gaia, URAT, then Tycho. + verbose : :class:`bool` + If ``True`` then log informational messages. Returns ------- :class:`recarray` - - The bright source mask in the form of `maskdatamodel.dtype`: - - `REF_CAT` is `"T2"` for Tycho and `"G2"` for Gaia. - - `REF_ID` is `Tyc1`*1,000,000+`Tyc2`*10+`Tyc3` for Tycho2; - `"sourceid"` for Gaia-DR2 and Gaia-DR2 with URAT. - - `REF_MAG` is, in order of preference, G-band for Gaia, VT - then HP then BT for Tycho. - - `URAT_ID` contains the URAT reference number for Gaia objects - that use the URAT proper motion, or -1 otherwise. - - The radii are in ARCSECONDS. - - `E1` and `E2` are placeholders for ellipticity components, and - are set to 0 for Gaia and Tycho sources. - - `TYPE` is always `PSF` for star-like objects. - - Note that the mask is based on objects in the pixel AT THEIR - NATIVE EPOCH *NOT* AT THE INPUT `maskepoch`. It is therefore - possible for locations in the output mask to be just beyond - the boundaries of the input pixel. + The bright star mask in the form of `maskdatamodel.dtype`. Notes ----- - - `IN_RADIUS` (`NEAR_RADIUS`) corresponds to `IN_BRIGHT_OBJECT` - (`NEAR_BRIGHT_OBJECT`) in `data/targetmask.yaml`. These radii - are set in the function `desitarget.brightmask.radius()`. - - The correct mask size for DESI is an open question. - - The `GAIA_DIR`, `URAT_DIR` and `TYCHO_DIR` environment - variables must be set. + - Runs in a a minute or so for a typical nside=4 pixel. + - See :func:`~desitarget.brightmask.make_bright_star_mask` for + descriptions of the output mask and the other input parameters. """ # ADM start the clock. t0 = time() @@ -205,8 +172,9 @@ def make_bright_star_mask_in_hp(nside, pixnum, gaiaepoch=2015.5, # ADM discard any Tycho objects below the input magnitude limit. ii = tychomag < maglim tychomag, tychoobjs = tychomag[ii], tychoobjs[ii] - log.info('Read {} (mag < {}) Tycho objects (pix={})...t={:.1f} mins'.format( - np.sum(ii), maglim, pixnum, (time()-t0)/60)) + if verbose: + log.info('Read {} (mag < {}) Tycho objects (pix={})...t={:.1f} mins'. + format(np.sum(ii), maglim, pixnum, (time()-t0)/60)) # ADM read in the associated Gaia files. Also grab # ADM neighboring pixels to prevent edge effects. @@ -220,17 +188,20 @@ def make_bright_star_mask_in_hp(nside, pixnum, gaiaepoch=2015.5, # ADM limit Gaia objects to 3 magnitudes fainter than the passed # ADM limit. This leaves some (!) leeway when matching to Tycho. gaiaobjs = gaiaobjs[gaiaobjs['PHOT_G_MEAN_MAG'] < maglim + 3] - log.info('Read {} (G < {}) Gaia sources (pix={})...t={:.1f} mins'.format( - len(gaiaobjs), maglim+3, pixnum, (time()-t0)/60)) + if verbose: + log.info('Read {} (G < {}) Gaia sources (pix={})...t={:.1f} mins'.format( + len(gaiaobjs), maglim+3, pixnum, (time()-t0)/60)) # ADM substitute URAT where Gaia proper motions don't exist. ii = ((np.isnan(gaiaobjs["PMRA"]) | (gaiaobjs["PMRA"] == 0)) & (np.isnan(gaiaobjs["PMDEC"]) | (gaiaobjs["PMDEC"] == 0))) - log.info('Add URAT for {} Gaia objects with no PMs (pix={})...t={:.1f} mins' - .format(np.sum(ii), pixnum, (time()-t0)/60)) + if verbose: + log.info('Add URAT for {} Gaia objs with no PMs (pix={})...t={:.1f} mins' + .format(np.sum(ii), pixnum, (time()-t0)/60)) urat = add_urat_pms(gaiaobjs[ii], numproc=1) - log.info('Found an additional {} URAT objects (pix={})...t={:.1f} mins' - .format(np.sum(urat["URAT_ID"] != -1), pixnum, (time()-t0)/60)) + if verbose: + log.info('Found an additional {} URAT objects (pix={})...t={:.1f} mins' + .format(np.sum(urat["URAT_ID"] != -1), pixnum, (time()-t0)/60)) for col in "PMRA", "PMDEC": gaiaobjs[col][ii] = urat[col] # ADM need to track the URATID to track which objects have @@ -248,13 +219,15 @@ def make_bright_star_mask_in_hp(nside, pixnum, gaiaepoch=2015.5, gaiaobjs["PMRA"], gaiaobjs["PMDEC"], epochnow=gaiaepoch) # ADM match Gaia to Tycho with a suitable margin. - log.info('Match Gaia to Tycho with margin={}" (pix={})...t={:.1f} mins' - .format(margin, pixnum, (time()-t0)/60)) + if verbose: + log.info('Match Gaia to Tycho with margin={}" (pix={})...t={:.1f} mins' + .format(margin, pixnum, (time()-t0)/60)) igaia, itycho = radec_match_to([ra, dec], [tychoobjs["RA"], tychoobjs["DEC"]], sep=margin, radec=True) - log.info('{} matches. Refining at 1" (pix={})...t={:.1f} mins'.format( - len(itycho), pixnum, (time()-t0)/60)) + if verbose: + log.info('{} matches. Refining at 1" (pix={})...t={:.1f} mins'.format( + len(itycho), pixnum, (time()-t0)/60)) # ADM match Gaia to Tycho at the more exact reference epoch. epoch_ra = tychoobjs[itycho]["EPOCH_RA"] epoch_dec = tychoobjs[itycho]["EPOCH_DEC"] @@ -270,8 +243,9 @@ def make_bright_star_mask_in_hp(nside, pixnum, gaiaepoch=2015.5, keep = np.ones(len(tychoobjs), dtype='bool') keep[itycho[refined]] = False tychokeep, tychomag = tychoobjs[keep], tychomag[keep] - log.info('Kept {} Tycho sources with no Gaia match (pix={})...t={:.1f} mins' - .format(len(tychokeep), pixnum, (time()-t0)/60)) + if verbose: + log.info('Kept {} Tychos with no Gaia match (pix={})...t={:.1f} mins' + .format(len(tychokeep), pixnum, (time()-t0)/60)) # ADM now we're done matching to Gaia, limit Gaia to the passed # ADM magnitude limit and to the HEALPixel boundary of interest. @@ -279,8 +253,9 @@ def make_bright_star_mask_in_hp(nside, pixnum, gaiaepoch=2015.5, gaiahpx = hp.ang2pix(nside, theta, phi, nest=True) ii = (gaiahpx == pixnum) & (gaiaobjs['PHOT_G_MEAN_MAG'] < maglim) gaiakeep, uratid = gaiaobjs[ii], uratid[ii] - log.info('Mask also comprises {} Gaia sources (pix={})...t={:.1f} mins' - .format(len(gaiakeep), pixnum, (time()-t0)/60)) + if verbose: + log.info('Mask also comprises {} Gaia sources (pix={})...t={:.1f} mins' + .format(len(gaiakeep), pixnum, (time()-t0)/60)) # ADM move the coordinates forwards to the input mask epoch. epoch_ra, epoch_dec = tychokeep["EPOCH_RA"], tychokeep["EPOCH_DEC"] @@ -316,6 +291,115 @@ def make_bright_star_mask_in_hp(nside, pixnum, gaiaepoch=2015.5, # ADM ...and add the mask radii. mask["IN_RADIUS"], mask["NEAR_RADIUS"] = radii(mask["REF_MAG"]) + if verbose: + log.info("Done making mask...(pix={})...t={:.1f} mins".format( + pixnum, (time()-t0)/60.)) + + return mask + + +def make_bright_star_mask(maglim=12., matchrad=1., numproc=16, + maskepoch=2023.0, gaiaepoch=2015.5): + """Make an all-sky bright star mask using Tycho, Gaia and URAT. + + Parameters + ---------- + maglim : :class:`float`, optional, defaults to 12. + Faintest magnitude at which to make the mask. This magnitude is + interpreted as G-band for Gaia and, in order of preference, VT + then HP then BT for Tycho (not every Tycho source has each band). + matchrad : :class:`int`, optional, defaults to 1. + Tycho sources that match a Gaia source at this separation in + ARCSECONDS are NOT included in the output mask. The matching is + performed rigorously, accounting for Gaia proper motions. + numproc : :class:`int`, optional, defaults to 16. + Number of processes over which to parallelize + maskepoch : :class:`float` + The mask is built at this epoch. Not all sources have proper + motions from every survey, so proper motions are used, in order + of preference, from Gaia, URAT, then Tycho. + gaiaepoch: :class:`float`, optional, defaults to Gaia DR2 (2015.5) + The epoch of the Gaia observations. Should be 2015.5 unless we + move beyond Gaia DR2. + + Returns + ------- + :class:`recarray` + - The bright star mask in the form of `maskdatamodel.dtype`: + - `REF_CAT` is `"T2"` for Tycho and `"G2"` for Gaia. + - `REF_ID` is `Tyc1`*1,000,000+`Tyc2`*10+`Tyc3` for Tycho2; + `"sourceid"` for Gaia-DR2 and Gaia-DR2 with URAT. + - `REF_MAG` is, in order of preference, G-band for Gaia, VT + then HP then BT for Tycho. + - `URAT_ID` contains the URAT reference number for Gaia objects + that use the URAT proper motion, or -1 otherwise. + - The radii are in ARCSECONDS. + - `E1` and `E2` are placeholders for ellipticity components, and + are set to 0 for Gaia and Tycho sources. + - `TYPE` is always `PSF` for star-like objects. + - Note that the mask is based on objects in the pixel AT THEIR + NATIVE EPOCH *NOT* AT THE INPUT `maskepoch`. It is therefore + possible for locations in the output mask to be just beyond + the boundaries of the input pixel. + + Notes + ----- + - Takes about 20 minutes to run parallelized at `numproc`=16 for + `maglim`=12 + - `IN_RADIUS` (`NEAR_RADIUS`) corresponds to `IN_BRIGHT_OBJECT` + (`NEAR_BRIGHT_OBJECT`) in `data/targetmask.yaml`. These radii + are set in the function `desitarget.brightmask.radius()`. + - The correct mask size for DESI is an open question. + - The `GAIA_DIR`, `URAT_DIR` and `TYCHO_DIR` environment + variables must be set. + """ + log.info("running on {} processors".format(numproc)) + + # ADM grab the nside of the Tycho files, which is a reasonable + # ADM resolution for bright stars. + nside = get_tycho_nside() + npixels = hp.nside2npix(nside) + + # ADM array of HEALPixels over which to parallelize... + pixels = np.arange(npixels) + # ADM ...shuffle for better balance across nodes (as there are more + # ADM stars in certain regions of the sky where pixels adjoin). + np.random.shuffle(pixels) + + # ADM the common function that is actually parallelized across. + def _make_bright_star_mx(pixnum): + """returns bright star mask in one HEALPixel""" + return make_bright_star_mask_in_hp( + nside, pixnum, maglim=maglim, matchrad=matchrad, + gaiaepoch=gaiaepoch, maskepoch=maskepoch, verbose=False) + + # ADM this is just to count pixels in _update_status. + npix = np.zeros((), dtype='i8') + t0 = time() + + def _update_status(result): + """wrap key reduction operation on the main parallel process""" + if npix % 10 == 0 and npix > 0: + rate = (time() - t0) / npix + log.info('{}/{} HEALPixels; {:.1f} secs/pixel...t = {:.1f} mins'. + format(npix, npixels, rate, (time()-t0)/60.)) + npix[...] += 1 + return result + + # ADM Parallel process across HEALPixels. + if numproc > 1: + pool = sharedmem.MapReduce(np=numproc) + with pool: + mask = pool.map(_make_bright_star_mx, pixels, reduce=_update_status) + else: + mask = list() + for pixel in pixels: + mask.append(_update_status(_make_bright_star_mx(pixel))) + + mask = np.concatenate(mask) + + log.info("Done making mask...t = {:.1f} mins".format((time()-t0)/60.)) + return mask From cbe5d76c93ae6270cee5403b23df22d5a150fe04 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Wed, 10 Jun 2020 14:03:28 -0700 Subject: [PATCH 11/25] update binary executable (without writing, yet) --- bin/make_bright_mask | 47 ++++++++++++++++++++----------------- bin/supplement_skies | 4 ++-- py/desitarget/brightmask.py | 18 ++++++++++++++ 3 files changed, 45 insertions(+), 24 deletions(-) diff --git a/bin/make_bright_mask b/bin/make_bright_mask index d3d9872a1..4ff370267 100755 --- a/bin/make_bright_mask +++ b/bin/make_bright_mask @@ -6,44 +6,47 @@ import os, sys import numpy as np from desitarget import io -from desitarget.brightmask import make_bright_source_mask +from desitarget.brightmask import make_bright_star_mask, get_mask_dir #import warnings #warnings.simplefilter('error') import multiprocessing -nproc = multiprocessing.cpu_count() // 2 +nproc = multiprocessing.cpu_count() // 4 from desiutil.log import get_logger log = get_logger() from argparse import ArgumentParser ap = ArgumentParser() -ap.add_argument("src", help="root directory with tractor/sweeps files from which to build mask") -ap.add_argument("dest", help='Output file name to which to write mask') -ap.add_argument("--bands", - help='Bands to use to build the mask (e.g. GRZ)', - default="GRZ") +ap.add_argument("--maskdir", + help='Output directory to which to write mask [defaults to the $MASK_DIR environment variable]') ap.add_argument("--maglim", - help='Magnitude limits for building the mask (e.g. 10,10,10)', - default="10,10,10") -### ap.add_argument('-b', "--bricklist", help='filename with list of bricknames to include') + help="Magnitude limit for building the mask [defaults to 12]", + default=12) +ap.add_argument("--matchrad", + help="Tycho stars are discarded if they match a Gaia star at this radius", + default=12) ap.add_argument("--numproc", type=int, - help='number of concurrent processes to use [{}]'.format(nproc), - default=nproc) + help="number of concurrent processes to use [{}]".format(nproc), + default=nproc) +ap.add_argument("--maskepoch", + help="Stars are shifted (by just proper motion) to this epoch to make the mask [2023.0]", + default=2023.) +ap.add_argument("--gaiaepoch", + help="Epoch of Gaia observations [2015.5]", + default=2015.) ns = ap.parse_args() -infiles = io.list_sweepfiles(ns.src) -if len(infiles) == 0: - infiles = io.list_tractorfiles(ns.src) -if len(infiles) == 0: - log.critical('no sweep or tractor files found') - sys.exit(1) -maglim = [ float(ml) for ml in ns.maglim.split(',') ] +# ADM if the MASK_DIR directory was passed, set it... +maskdir = ns.maskdir +if maskdir is None: + # ADM ...otherwise retrieve it from the environment variable. + maskdir = get_mask_dir() -sourcemask = make_bright_source_mask(ns.bands,maglim,numproc=ns.numproc, - rootdirname=ns.src,outfilename=ns.dest) +mask = make_bright_star_mask(maglim=ns.maglim, matchrad=ns.matchrad, numproc=ns.numproc, + maskepoch=ns.maskepoch, gaiaepoch=ns.gaiaepoch) -log.info('wrote a file of {} masks to {}'.format(len(sourcemask), ns.dest)) +log.info('wrote a file of {} masks to {}'.format(len(mask), ns.maskdir)) diff --git a/bin/supplement_skies b/bin/supplement_skies index 679944081..d1ba2b563 100755 --- a/bin/supplement_skies +++ b/bin/supplement_skies @@ -81,8 +81,8 @@ if ns.bundlefiles is not None: gaiadir = ns.gaiadir if gaiadir is None: # ADM ...otherwise retrieve it from the environment variable. - from desitarget.gaiamatch import _get_gaia_dir - gaiadir = _get_gaia_dir() + from desitarget.gaiamatch import get_gaia_dir + gaiadir = get_gaia_dir() # ADM if needed, determine the minimum density of sky fibers to generate. nskiespersqdeg = ns.nskiespersqdeg diff --git a/py/desitarget/brightmask.py b/py/desitarget/brightmask.py index 5bc1b19d3..a52467c44 100644 --- a/py/desitarget/brightmask.py +++ b/py/desitarget/brightmask.py @@ -57,6 +57,24 @@ ]) +def get_mask_dir(): + """Convenience function to grab the MASK_DIR environment variable. + + Returns + ------- + :class:`str` + The directory stored in the $TYCHO_DIR environment variable. + """ + # ADM check that the $MASK_DIR environment variable is set. + maskdir = os.environ.get('MASK_DIR') + if maskdir is None: + msg = "Set $MASK_DIR environment variable!" + log.critical(msg) + raise ValueError(msg) + + return maskdir + + def radii(mag): """The relation used to set the radius of bright star masks. From 21fc90c5a539440a0187c4d6386ebe3c8eb9da4f Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Wed, 10 Jun 2020 18:36:18 -0700 Subject: [PATCH 12/25] add functionality to write mask files in a standardized manner --- bin/make_bright_mask | 41 ++++++++----- py/desitarget/brightmask.py | 19 ++---- py/desitarget/io.py | 114 ++++++++++++++++++++++++++++++++++-- py/desitarget/uratmatch.py | 14 ++--- 4 files changed, 146 insertions(+), 42 deletions(-) diff --git a/bin/make_bright_mask b/bin/make_bright_mask index 4ff370267..07fdd9858 100755 --- a/bin/make_bright_mask +++ b/bin/make_bright_mask @@ -1,18 +1,20 @@ #!/usr/bin/env python -from __future__ import print_function, division - import os, sys import numpy as np from desitarget import io from desitarget.brightmask import make_bright_star_mask, get_mask_dir +from desitarget.tychomatch import get_tycho_nside, get_tycho_dir +from desitarget.uratmatch import get_urat_dir +from desitarget.gaiamatch import get_gaia_dir #import warnings #warnings.simplefilter('error') import multiprocessing -nproc = multiprocessing.cpu_count() // 4 +nproc = multiprocessing.cpu_count() // 2 +nside = get_tycho_nside() from desiutil.log import get_logger log = get_logger() @@ -21,21 +23,23 @@ from argparse import ArgumentParser ap = ArgumentParser() ap.add_argument("--maskdir", help='Output directory to which to write mask [defaults to the $MASK_DIR environment variable]') -ap.add_argument("--maglim", +ap.add_argument("--maglim", type=float, help="Magnitude limit for building the mask [defaults to 12]", - default=12) -ap.add_argument("--matchrad", + default=12.) +ap.add_argument("--matchrad", type=float, help="Tycho stars are discarded if they match a Gaia star at this radius", - default=12) + default=1.) +ap.add_argument("--nside", type=int, + help="Write output files in HEALPixels at this nside [{}]".format(nside)) ap.add_argument("--numproc", type=int, help="number of concurrent processes to use [{}]".format(nproc), default=nproc) -ap.add_argument("--maskepoch", +ap.add_argument("--maskepoch", type=float, help="Stars are shifted (by just proper motion) to this epoch to make the mask [2023.0]", - default=2023.) -ap.add_argument("--gaiaepoch", + default=2023.0) +ap.add_argument("--gaiaepoch", type=float, help="Epoch of Gaia observations [2015.5]", - default=2015.) + default=2015.5) ns = ap.parse_args() @@ -45,8 +49,17 @@ if maskdir is None: # ADM ...otherwise retrieve it from the environment variable. maskdir = get_mask_dir() -mask = make_bright_star_mask(maglim=ns.maglim, matchrad=ns.matchrad, numproc=ns.numproc, - maskepoch=ns.maskepoch, gaiaepoch=ns.gaiaepoch) +mask = make_bright_star_mask( + maglim=ns.maglim, matchrad=ns.matchrad, numproc=ns.numproc, + maskepoch=ns.maskepoch, gaiaepoch=ns.gaiaepoch) + +# ADM extra header keywords for the output fits file. +extra = {k: v for k, v in zip( + ["matchrad", "gaiaepoc", "gaiadir", "tychodir", "uratdir"], + [ns.matchrad, ns.gaiaepoch, get_gaia_dir(), get_tycho_dir(), get_urat_dir()])} -log.info('wrote a file of {} masks to {}'.format(len(mask), ns.maskdir)) +# ADM write the mask file(s). +nmasks, mxdir = io.write_masks(maskdir, mask, maglim=ns.maglim, + maskepoch=ns.maskepoch, nside=nside, extra=extra) +log.info('wrote {} total masks to files in {}'.format(nmasks, mxdir)) diff --git a/py/desitarget/brightmask.py b/py/desitarget/brightmask.py index a52467c44..96ae28c43 100644 --- a/py/desitarget/brightmask.py +++ b/py/desitarget/brightmask.py @@ -316,7 +316,7 @@ def make_bright_star_mask_in_hp(nside, pixnum, verbose=True, gaiaepoch=2015.5, return mask -def make_bright_star_mask(maglim=12., matchrad=1., numproc=16, +def make_bright_star_mask(maglim=12., matchrad=1., numproc=32, maskepoch=2023.0, gaiaepoch=2015.5): """Make an all-sky bright star mask using Tycho, Gaia and URAT. @@ -362,8 +362,7 @@ def make_bright_star_mask(maglim=12., matchrad=1., numproc=16, Notes ----- - - Takes about 20 minutes to run parallelized at `numproc`=16 for - `maglim`=12 + - Runs in about 20 minutes for `numproc`=32 and `maglim`=12. - `IN_RADIUS` (`NEAR_RADIUS`) corresponds to `IN_BRIGHT_OBJECT` (`NEAR_BRIGHT_OBJECT`) in `data/targetmask.yaml`. These radii are set in the function `desitarget.brightmask.radius()`. @@ -499,8 +498,7 @@ def is_in_bright_mask(targs, sourcemask, inonly=False): A recarray of targets as made by, e.g., :mod:`desitarget.cuts.select_targets`. sourcemask : :class:`recarray` A recarray containing a mask as made by, e.g., - :mod:`desitarget.brightmask.make_bright_star_mask` or - :mod:`desitarget.brightmask.make_bright_source_mask`. + :mod:`desitarget.brightmask.make_bright_star_mask` inonly : :class:`boolean`, optional, defaults to False If True, then only calculate the in_mask return but not the near_mask return, which is about a factor of 2 faster. @@ -827,7 +825,6 @@ def set_target_bits(targs, sourcemask): def mask_targets(targs, inmaskfile=None, nside=None, bands="GRZ", maglim=[10, 10, 10], numproc=4, - rootdirname='/global/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1', outfilename=None, drbricks=None): """Add bits for if objects are in a bright mask, and SAFE (BADSKY) locations, to a target set. @@ -854,9 +851,6 @@ def mask_targets(targs, inmaskfile=None, nside=None, bands="GRZ", maglim=[10, 10 same length (e.g., "GRZ" for [12.3,12.7,12.6]) numproc : :class:`int`, optional Number of processes over which to parallelize - rootdirname : :class:`str`, optional, defaults to dr3 - Root directory containing either sweeps or tractor files...e.g. for dr3 this might be - /global/project/projectdirs/cosmo/data/legacysurvey/dr3/sweep/dr3.1 outfilename : :class:`str`, optional, defaults to not writing anything to file (FITS) File name to which to write the output mask ONE OF outfilename or inmaskfile MUST BE PASSED @@ -887,12 +881,7 @@ def mask_targets(targs, inmaskfile=None, nside=None, bands="GRZ", maglim=[10, 10 raise ValueError("{} doesn't exist".format(targs)) targs = fitsio.read(targs) - # ADM check if a file for the bright source mask was passed, if not then create it. - if inmaskfile is None: - sourcemask = make_bright_source_mask(bands, maglim, numproc=numproc, - rootdirname=rootdirname, outfilename=outfilename) - else: - sourcemask = fitsio.read(inmaskfile) + sourcemask = fitsio.read(inmaskfile) ntargsin = len(targs) log.info('Number of targets {}...t={:.1f}s'.format(ntargsin, time()-t0)) diff --git a/py/desitarget/io.py b/py/desitarget/io.py index 8d175de63..b3c0ffb55 100644 --- a/py/desitarget/io.py +++ b/py/desitarget/io.py @@ -439,7 +439,6 @@ def write_targets(targdir, data, indir=None, indir2=None, nchunks=None, The number of targets that were written to file. :class:`str` The name of the file to which targets were written. - """ # ADM create header. hdr = fitsio.FITSHDR() @@ -483,7 +482,7 @@ def write_targets(targdir, data, indir=None, indir2=None, nchunks=None, resolve=resolve, supp=supp) ntargs = len(data) - # ADM die immediately if there are no targets to write. + # ADM die if there are no targets to write. if ntargs == 0: return ntargs, filename @@ -818,6 +817,12 @@ def write_skies(targdir, data, indir=None, indir2=None, supp=False, mock : :class:`bool`, optional, defaults to ``False``. If ``True`` then construct the file path for mock sky target catalogs. + Returns + ------- + :class:`int` + The number of skies that were written to file. + :class:`str` + The name of the file to which skies were written. """ nskies = len(data) @@ -936,6 +941,13 @@ def write_gfas(targdir, data, indir=None, indir2=None, nside=None, extra : :class:`dict`, optional If passed (and not None), write these extra dictionary keys and values to the output header. + + Returns + ------- + :class:`int` + The number of gfas that were written to file. + :class:`str` + The name of the file to which gfas were written. """ # ADM if passed, use the indir to determine the Data Release # ADM integer and string for the input targets. @@ -1043,6 +1055,13 @@ def write_randoms(targdir, data, indir=None, hdr=None, nside=None, supp=False, extra : :class:`dict`, optional If passed (and not ``None``), write these extra dictionary keys and values to the output header. + + Returns + ------- + :class:`int` + The number of randoms that were written to file. + :class:`str` + The name of the file to which randoms were written. """ # ADM create header to include versions, etc. If a `hdr` was # ADM passed, then use it, if not then create a new header. @@ -1101,7 +1120,7 @@ def write_randoms(targdir, data, indir=None, hdr=None, nside=None, supp=False, region=region, seed=seed, nohp=True) nrands = len(data) - # ADM die immediately if there are no targets to write. + # ADM die if there are no targets to write. if nrands == 0: return nrands, filename @@ -1127,6 +1146,77 @@ def write_randoms(targdir, data, indir=None, hdr=None, nside=None, supp=False, return nrands, filename +def write_masks(maskdir, data, + maglim=None, maskepoch=None, nside=None, extra=None): + """Write a catalogue of masks and associated pixel-level info. + + Parameters + ---------- + maskdir : :class:`str` + Path to output mask directory (the file names are built + on-the-fly from other inputs). + data : :class:`~numpy.ndarray` + Array of masks to write to file. Must contain at least the + columns "RA" and "DEC". + maglim : :class:`float`, optional, defaults to ``None`` + Magnitude limit to which the mask was made. + maskepoch : :class:`float`, optional, defaults to ``None`` + Epoch at which the mask was made. + nside: :class:`int`, defaults to not splitting by HEALPixel. + The HEALPix nside at which to write the output files. + extra : :class:`dict`, optional + If passed (and not ``None``), write these extra dictionary keys + and values to the output header. + + Returns + ------- + :class:`int` + The total number of masks that were written. + :class:`str` + The name of the directory to which masks were written. + """ + # ADM create header to include versions, etc. + hdr = fitsio.FITSHDR() + depend.setdep(hdr, 'desitarget', desitarget_version) + depend.setdep(hdr, 'desitarget-git', gitversion()) + # ADM add the magnitude and epoch to the header. + hdr["MAGLIM"] = maglim + hdr["MASKEPOC"] = maskepoch + # ADM add the extra dictionary to the header. + if extra is not None: + for key in extra: + hdr[key] = extra[key] + # ADM add the HEALPixel information to the header. + hdr["FILENSID"] = nside + hdr["FILENEST"] = True + + nmasks = len(data) + # ADM die if there are no masks to write. + if nmasks == 0: + return nmasks, None + + # ADM write across HEAPixels at input nside. + npix = hp.nside2npix(nside) + theta, phi = np.radians(90-data["DEC"]), np.radians(data["RA"]) + hpx = hp.ang2pix(nside, theta, phi, nest=True) + for pix in range(npix): + outdata = data[hpx==pix] + outhdr = dict(hdr).copy() + outhdr["FILEHPX"] = pix + # ADM construct the output file name. + fn = find_target_files(maskdir, flavor="masks", + hp=pix, maglim=maglim, epoch=maskepoch) + # ADM create necessary directory, if it doesn't exist. + os.makedirs(os.path.dirname(fn), exist_ok=True) + # ADM write the output file. + if len(outdata) > 0: + fitsio.write( + fn, outdata, extname='MASKS', header=outhdr, clobber=True) + log.info('wrote {} masks to {}'.format(len(outdata), fn)) + + return nmasks, os.path.dirname(fn) + + def is_sky_dir_official(skydirname): """Check a sky file or directory has the correct HEALPixel structure. @@ -1694,7 +1784,8 @@ def _get_targ_dir(): def find_target_files(targdir, dr=None, flavor="targets", survey="main", obscon=None, hp=None, nside=None, resolve=True, supp=False, - mock=False, nohp=False, seed=None, region=None): + mock=False, nohp=False, seed=None, region=None, + maglim=None, epoch=None): """Build the name of an output target file (or directory). Parameters @@ -1704,7 +1795,7 @@ def find_target_files(targdir, dr=None, flavor="targets", survey="main", dr : :class:`str` or :class:`int`, optional, defaults to "X" Name of a Legacy Surveys Data Release (e.g. 8) flavor : :class:`str`, optional, defaults to `targets` - Options include "skies", "gfas", "targets", "randoms". + Options include "skies", "gfas", "targets", "randoms", "masks". survey : :class:`str`, optional, defaults to `main` Options include "main", "cmx", "svX" (where X is 1, 2 etc.). Only relevant if `flavor` is "targets". @@ -1734,6 +1825,12 @@ def find_target_files(targdir, dr=None, flavor="targets", survey="main", region : :class:`int`, optional If `region` is not ``None``, then it is added to the directory name after `resolve`. Only relevant if `flavor` is "randoms". + maglim : :class:`float`, optional + Magnitude limit to which the mask was made. Only relevant if + `flavor` is "masks". Must be passed if `flavor` is "masks". + epoch : :class:`float` + Epoch at which the mask was made. Only relevant if `flavor` is + "masks". Must be passed if `flavor` is "masks". Returns ------- @@ -1760,7 +1857,7 @@ def find_target_files(targdir, dr=None, flavor="targets", survey="main", if mock: allowed_flavor = ["targets", "truth", "sky"] else: - allowed_flavor = ["targets", "skies", "gfas", "randoms"] + allowed_flavor = ["targets", "skies", "gfas", "randoms", "masks"] if flavor not in allowed_flavor: msg = "flavor must be {}, not {}".format(' or '.join(allowed_flavor), flavor) log.critical(msg) @@ -1806,6 +1903,9 @@ def find_target_files(targdir, dr=None, flavor="targets", survey="main", surv = survey[0:2] prefix = flavor fn = os.path.join(targdir, flavor) + if flavor == "masks": + maskdir = "maglim-{}-epoch-{}".format(maglim, epoch) + fn = os.path.join(targdir, maskdir) if flavor == "targets": if surv in ["cmx", "sv"]: @@ -1831,6 +1931,8 @@ def find_target_files(targdir, dr=None, flavor="targets", survey="main", if hp is not None: hpstr = ",".join([str(pix) for pix in np.atleast_1d(hp)]) backend = "{}-{}-hp-{}.fits".format(prefix, drstr, hpstr) + if flavor == "masks": + backend = "{}-hp-{}.fits".format(prefix, hpstr) fn = os.path.join(fn, backend) else: if nohp: diff --git a/py/desitarget/uratmatch.py b/py/desitarget/uratmatch.py index d8ddad255..a311aa1f5 100644 --- a/py/desitarget/uratmatch.py +++ b/py/desitarget/uratmatch.py @@ -43,7 +43,7 @@ ]) -def _get_urat_dir(): +def get_urat_dir(): """Convenience function to grab the URAT environment variable. Returns @@ -96,7 +96,7 @@ def scrape_urat(url="http://cdsarc.u-strasbg.fr/ftp/I/329/URAT1/v12/", - Runs in about 50 minutes for 575 URAT files. """ # ADM check that the URAT_DIR is set and retrieve it. - uratdir = _get_urat_dir() + uratdir = get_urat_dir() # ADM construct the directory to which to write files. bindir = os.path.join(uratdir, 'binary') @@ -161,7 +161,7 @@ def urat_binary_to_csv(): - Runs in about 40 minutes for 575 files. """ # ADM check that the URAT_DIR is set. - uratdir = _get_urat_dir() + uratdir = get_urat_dir() # ADM a quick check that the csv directory is empty before writing. csvdir = os.path.join(uratdir, 'csv') @@ -221,7 +221,7 @@ def urat_csv_to_fits(numproc=5): nside = _get_urat_nside() # ADM check that the URAT_DIR is set. - uratdir = _get_urat_dir() + uratdir = get_urat_dir() log.info("running on {} processors".format(numproc)) # ADM construct the directories for reading/writing files. @@ -349,7 +349,7 @@ def urat_fits_to_healpix(numproc=5): nside = _get_urat_nside() # ADM check that the URAT_DIR is set. - uratdir = _get_urat_dir() + uratdir = get_urat_dir() # ADM construct the directories for reading/writing files. fitsdir = os.path.join(uratdir, 'fits') @@ -470,7 +470,7 @@ def make_urat_files(numproc=5, download=False): log.info('Begin making URAT files...t={:.1f}s'.format(time()-t0)) # ADM check that the URAT_DIR is set. - uratdir = _get_urat_dir() + uratdir = get_urat_dir() # ADM a quick check that the fits and healpix directories are empty # ADM before embarking on the slower parts of the code. @@ -530,7 +530,7 @@ def find_urat_files(objs, neighbors=True, radec=False): nside = _get_urat_nside() # ADM check that the URAT_DIR is set and retrieve it. - uratdir = _get_urat_dir() + uratdir = get_urat_dir() hpxdir = os.path.join(uratdir, 'healpix') # ADM which flavor of RA/Dec was passed. From 050aa6fdbb4816637a711b0e34d2a14b13adcb6d Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Thu, 11 Jun 2020 09:45:34 -0700 Subject: [PATCH 13/25] allow read_targets() functions to also read MASKS files --- py/desitarget/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/desitarget/io.py b/py/desitarget/io.py index b3c0ffb55..297736d04 100644 --- a/py/desitarget/io.py +++ b/py/desitarget/io.py @@ -1971,7 +1971,7 @@ def read_target_files(filename, columns=None, rows=None, header=False, """ start = time() # ADM start with some checking that this is a target file. - targtypes = "TARGETS", "GFA_TARGETS", "SKY_TARGETS" + targtypes = "TARGETS", "GFA_TARGETS", "SKY_TARGETS", "MASKS" # ADM read in the FITS extention info. f = fitsio.FITS(filename) if len(f) != 2: From 85f39b7673b3aa7dbea9cef6a84f85ee7b69f10b Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 12 Jun 2020 12:26:57 -0700 Subject: [PATCH 14/25] update matching-and-masking functions to use new Gaia/Tycho masks; optimize to run in pixel boundaries; update sloppy code style --- py/desitarget/brightmask.py | 388 ++++++++++++++++++------------------ py/desitarget/geomask.py | 23 ++- 2 files changed, 209 insertions(+), 202 deletions(-) diff --git a/py/desitarget/brightmask.py b/py/desitarget/brightmask.py index 96ae28c43..b8fe47893 100644 --- a/py/desitarget/brightmask.py +++ b/py/desitarget/brightmask.py @@ -32,7 +32,7 @@ from desitarget.gaiamatch import find_gaia_files from desitarget.geomask import circles, cap_area, circle_boundaries from desitarget.geomask import ellipses, ellipse_boundary, is_in_ellipse -from desitarget.geomask import radec_match_to, rewind_coords +from desitarget.geomask import radec_match_to, rewind_coords, add_hp_neighbors from desitarget.cuts import _psflike from desitarget.tychomatch import get_tycho_dir, get_tycho_nside from desitarget.tychomatch import find_tycho_files_hp @@ -94,9 +94,9 @@ def radii(mag): in data/targetmask.yaml`. """ # ADM mask all sources with mag < 12 at 5 arcsecs. - nearrad = (mag < 12.) * 5 - # ADM the IN_RADIUS is half the near radius. - inrad = nearrad/2. + inrad = (mag < 12.) * 5. + # ADM the NEAR_RADIUS is twice the IN_RADIUS. + nearrad = inrad*2. return inrad, nearrad @@ -421,21 +421,19 @@ def _update_status(result): def plot_mask(mask, limits=None, radius="IN_RADIUS", show=True): - """Make a plot of a mask and either display it or retain the plot object for over-plotting. + """Plot a mask or masks. Parameters ---------- mask : :class:`recarray` - A mask constructed by ``make_bright_source_mask`` - (or read in from file in the ``make_bright_source_mask`` format). + A mask, as constructed by, e.g. :func:`make_bright_star_mask()`. limits : :class:`list`, optional - The RA/Dec limits of the plot in the form [ramin, ramax, decmin, decmax]. + RA/Dec plot limits in the form [ramin, ramax, decmin, decmax]. radius : :class: `str`, optional - Which mask radius to plot (``IN_RADIUS`` or ``NEAR_RADIUS``). Both can be plotted - by calling this function twice with show=False and then with ``over=True``. + Which mask radius to plot (``IN_RADIUS`` or ``NEAR_RADIUS``). show : :class:`boolean` - If ``True``, then display the plot, Otherwise, just execute the plot commands - so it can be added to, shown or saved to file later. + If ``True``, then display the plot, Otherwise, just execute the + plot commands so it can be added to or saved to file later. Returns ------- @@ -463,20 +461,25 @@ def plot_mask(mask, limits=None, radius="IN_RADIUS", show=True): tol = 3.*np.max(mask[radius])/3600. # ADM the np.min/np.max combinations are to guard against people # ADM passing flipped RAs (so RA increases to the east). - w = np.where((mask["RA"] > np.min(limits[:2])-tol) & (mask["RA"] < np.max(limits[:2])+tol) & - (mask["DEC"] > np.min(limits[-2:])-tol) & (mask["DEC"] < np.max(limits[-2:])+tol)) - if len(w[0]) == 0: - log.error('No mask entries within specified limits ({})'.format(limits)) + ii = ((mask["RA"] > np.min(limits[:2])-tol) & + (mask["RA"] < np.max(limits[:2])+tol) & + (mask["DEC"] > np.min(limits[-2:])-tol) & + (mask["DEC"] < np.max(limits[-2:])+tol)) + if np.sum(ii) == 0: + msg = 'No mask entries within specified limits ({})'.format(limits) + log.error(msg) + raise ValueError(msg) else: - mask = mask[w] + mask = mask[ii] # ADM create ellipse polygons for each entry in the mask and # ADM make a list of matplotlib patches for them. patches = [] for i, ellipse in enumerate(mask): # ADM create points on the ellipse boundary. - ras, decs = ellipse_boundary(ellipse["RA"], ellipse["DEC"], ellipse[radius], - ellipse["E1"], ellipse["E2"]) + ras, decs = ellipse_boundary( + ellipse["RA"], ellipse["DEC"], + ellipse[radius], ellipse["E1"], ellipse["E2"]) polygon = Polygon(np.array(list(zip(ras, decs))), True) patches.append(polygon) @@ -490,38 +493,46 @@ def plot_mask(mask, limits=None, radius="IN_RADIUS", show=True): def is_in_bright_mask(targs, sourcemask, inonly=False): - """Determine whether a set of targets is in a bright source mask. + """Determine whether a set of targets is in a bright star mask. Parameters ---------- targs : :class:`recarray` - A recarray of targets as made by, e.g., :mod:`desitarget.cuts.select_targets`. + A recarray of targets, skies etc., as made by, e.g., + :func:`desitarget.cuts.select_targets()`. sourcemask : :class:`recarray` A recarray containing a mask as made by, e.g., - :mod:`desitarget.brightmask.make_bright_star_mask` + :func:`desitarget.brightmask.make_bright_star_mask()` inonly : :class:`boolean`, optional, defaults to False - If True, then only calculate the in_mask return but not the near_mask return, - which is about a factor of 2 faster. + If ``True``, then only calculate the `in_mask` return but not + the `near_mask` return, which is about a factor of 2 faster. Returns ------- - in_mask : array_like. - ``True`` for array entries that correspond to a target that is IN a mask. - near_mask : array_like. - ``True`` for array entries that correspond to a target that is NEAR a mask. + :class:`list` + [`in_mask`, `near_mask`] where `in_mask` (`near_mask`) is a + boolean array that is ``True`` for `targs` that are IN (NEAR) a + mask. If `inonly` is ``True`` then this is just [`in_mask`]. + :class: `list` + [`used_in_mask`, `used_near_mask`] where `used_in_mask` + (`used_near_mask`) is a boolean array that is ``True`` for masks + in `sourcemask` that contain a target at the IN (NEAR) radius. + If `inonly` is ``True`` then this is just [`used_in_mask`]. """ t0 = time() - # ADM initialize an array of all False (nothing is yet in a mask). + # ADM initialize arrays of all False (nothing is yet in a mask). in_mask = np.zeros(len(targs), dtype=bool) near_mask = np.zeros(len(targs), dtype=bool) + used_in_mask = np.zeros(len(sourcemask), dtype=bool) + used_near_mask = np.zeros(len(sourcemask), dtype=bool) - # ADM turn the coordinates of the masks and the targets into SkyCoord objects. + # ADM turn the mask and target coordinates into SkyCoord objects. ctargs = SkyCoord(targs["RA"]*u.degree, targs["DEC"]*u.degree) cmask = SkyCoord(sourcemask["RA"]*u.degree, sourcemask["DEC"]*u.degree) - # ADM this is the largest search radius we should need to consider - # ADM in the future an obvious speed up is to split on radius + # ADM this is the largest search radius we should need to consider. + # ADM In the future an obvious speed up is to split on radius # ADM as large radii are rarer but take longer. maxrad = max(sourcemask["IN_RADIUS"])*u.arcsec if not inonly: @@ -534,12 +545,13 @@ def is_in_bright_mask(targs, sourcemask, inonly=False): # ADM catch the case where nothing fell in a mask. if len(idmask) == 0: if inonly: - return in_mask - return in_mask, near_mask + return [in_mask], [used_in_mask] + return [in_mask, near_mask], [used_in_mask, used_near_mask] - # ADM need to differentiate targets that are in ellipse-on-the-sky masks - # ADM from targets that are in circle-on-the-sky masks. - rex_or_psf = _rexlike(sourcemask[idmask]["TYPE"]) | _psflike(sourcemask[idmask]["TYPE"]) + # ADM need to differentiate targets that are in ellipse-on-the-sky + # ADM masks from targets that are in circle-on-the-sky masks. + rex_or_psf = _rexlike(sourcemask[idmask]["TYPE"]) | _psflike( + sourcemask[idmask]["TYPE"]) w_ellipse = np.where(~rex_or_psf) # ADM only continue if there are any elliptical masks. @@ -547,11 +559,11 @@ def is_in_bright_mask(targs, sourcemask, inonly=False): idelltargs = idtargs[w_ellipse] idellmask = idmask[w_ellipse] - log.info('Testing {} total targets against {} total elliptical masks...t={:.1f}s' + log.info('Testing {} targets against {} elliptical masks...t={:.1f}s' .format(len(set(idelltargs)), len(set(idellmask)), time()-t0)) - # ADM to speed the calculation, make a dictionary of which targets (the - # ADM values) are associated with each mask (the keys). + # ADM to speed the calculation, make a dictionary of which + # ADM targets (the values) associate with each mask (the keys). targidineachmask = {} # ADM first initiate a list for each relevant key (mask ID). for maskid in set(idellmask): @@ -561,58 +573,64 @@ def is_in_bright_mask(targs, sourcemask, inonly=False): for index, targid in enumerate(idelltargs): targidineachmask[idellmask[index]].append(targid) - # ADM loop through the masks and determine which relevant points occupy - # ADM them for both the IN_RADIUS and the NEAR_RADIUS. + # ADM loop through the masks and determine which relevant points + # ADM occupy them for both the IN_RADIUS and the NEAR_RADIUS. for maskid in targidineachmask: targids = targidineachmask[maskid] ellras, elldecs = targs[targids]["RA"], targs[targids]["DEC"] mask = sourcemask[maskid] - # ADM Refine True/False for being in a mask based on the elliptical masks. - in_mask[targids] |= is_in_ellipse(ellras, elldecs, mask["RA"], mask["DEC"], - mask["IN_RADIUS"], mask["E1"], mask["E2"]) + # ADM Refine being in a mask based on the elliptical masks. + in_ell = is_in_ellipse( + ellras, elldecs, mask["RA"], mask["DEC"], + mask["IN_RADIUS"], mask["E1"], mask["E2"]) + in_mask[targids] |= in_ell + used_in_mask[maskid] |= np.any(in_ell) if not inonly: - near_mask[targids] |= is_in_ellipse(ellras, elldecs, - mask["RA"], mask["DEC"], - mask["NEAR_RADIUS"], - mask["E1"], mask["E2"]) + in_ell = is_in_ellipse(ellras, elldecs, + mask["RA"], mask["DEC"], + mask["NEAR_RADIUS"], + mask["E1"], mask["E2"]) + near_mask[targids] |= in_ell + used_near_mask[maskid] |= np.any(in_ell) log.info('Done with elliptical masking...t={:1f}s'.format(time()-t0)) - # ADM finally, record targets that were in a circles-on-the-sky mask, which + # ADM Finally, record targets in a circles-on-the-sky mask, which # ADM trumps any information about just being in an elliptical mask. - # ADM find angular separations less than the mask radius for circle masks - # ADM matches that meet these criteria are in a circle mask (at least one). - w_in = np.where((d2d.arcsec < sourcemask[idmask]["IN_RADIUS"]) & rex_or_psf) + # ADM Find separations less than the mask radius for circle masks + # ADM matches meeting these criteria are in at least one circle mask. + w_in = (d2d.arcsec < sourcemask[idmask]["IN_RADIUS"]) & (rex_or_psf) in_mask[idtargs[w_in]] = True + used_in_mask[idmask[w_in]] = True if not inonly: - w_near = np.where((d2d.arcsec < sourcemask[idmask]["NEAR_RADIUS"]) & rex_or_psf) + w_near = (d2d.arcsec < sourcemask[idmask]["NEAR_RADIUS"]) & (rex_or_psf) near_mask[idtargs[w_near]] = True - return in_mask, near_mask + used_near_mask[idmask[w_near]] = True + return [in_mask, near_mask], [used_in_mask, used_near_mask] - return in_mask + return [in_mask], [used_in_mask] def is_bright_source(targs, sourcemask): - """Determine whether any of a set of targets are, themselves, a bright source mask. + """Determine whether targets are, themselves, a bright source mask. Parameters ---------- targs : :class:`recarray` - A recarray of targets as made by, e.g., :mod:`desitarget.cuts.select_targets`. + Targets as made by, e.g., :func:`desitarget.cuts.select_targets()`. sourcemask : :class:`recarray` A recarray containing a bright source mask as made by, e.g., - :mod:`desitarget.brightmask.make_bright_star_mask` or - :mod:`desitarget.brightmask.make_bright_source_mask`. + :func:`desitarget.brightmask.make_bright_star_mask()` Returns ------- is_mask : array_like - True for array entries that correspond to targets that are, themselves, a mask. - + ``True`` for `targs` that are, themselves, a mask. """ - # ADM initialize an array of all False (nothing yet has been shown to correspond to a mask). + # ADM initialize an array of all False (nothing yet has been shown + # ADM to correspond to a mask). is_mask = np.zeros(len(targs), dtype=bool) # ADM calculate the TARGETID for the targets. @@ -620,29 +638,28 @@ def is_bright_source(targs, sourcemask): brickid=targs['BRICKID'], release=targs['RELEASE']) - # ADM super-fast set-based look-up of which TARGETIDs are matches between the masks and the targets. + # ADM super-fast set-based look-up of which TARGETIDs are match + # ADM between the masks and the targets. matches = set(sourcemask["TARGETID"]).intersection(set(targetid)) - # ADM determine the indexes of the targets that have a TARGETID in matches. + # ADM indexes of the targets that have a TARGETID in matches. w_mask = [index for index, item in enumerate(targetid) if item in matches] - # ADM w_mask now contains the target indices that match to a bright mask on TARGETID. + # ADM w_mask now holds target indices that match a mask on TARGETID. is_mask[w_mask] = True return is_mask def generate_safe_locations(sourcemask, Nperradius=1): - """Given a bright source mask, generate SAFE (BADSKY) locations at its periphery. + """Given a mask, generate SAFE (BADSKY) locations at its periphery. Parameters ---------- sourcemask : :class:`recarray` A recarray containing a bright mask as made by, e.g., - :mod:`desitarget.brightmask.make_bright_star_mask` or - :mod:`desitarget.brightmask.make_bright_source_mask`. - Nperradius : :class:`int`, optional, defaults to 1 per arcsec of radius - The number of safe locations to generate scaled by the radius of each mask - in ARCSECONDS (i.e. the number of positions per arcsec of radius). + :func:`desitarget.brightmask.make_bright_star_mask()` + Nperradius : :class:`int`, optional, defaults to 1. + Number of safe locations to make per arcsec radius of each mask. Returns ------- @@ -688,51 +705,47 @@ def generate_safe_locations(sourcemask, Nperradius=1): sourcemask[w]["DEC"], radius[w], sourcemask[w]["E1"], sourcemask[w]["E2"], Nsafe[w]) - ras, decs = np.concatenate((ras, ellras)), np.concatenate((decs, elldecs)) + ras = np.concatenate((ras, ellras)) + decs = np.concatenate((decs, elldecs)) return ras, decs -def append_safe_targets(targs, sourcemask, nside=None, drbricks=None): - """Append targets at SAFE (BADSKY) locations to target list, set bits in TARGETID and DESI_TARGET. +def get_safe_targets(targs, sourcemask): + """Get SAFE (BADSKY) locations for targs, set TARGETID/DESI_TARGET. Parameters ---------- targs : :class:`~numpy.ndarray` - A recarray of targets as made by, e.g. :mod:`desitarget.cuts.select_targets`. - nside : :class:`integer` - The HEALPix nside used throughout the DESI data model. + Targets made by, e.g. :func:`desitarget.cuts.select_targets()`. sourcemask : :class:`~numpy.ndarray` - A recarray containing a bright source mask as made by, e.g. - :mod:`desitarget.brightmask.make_bright_star_mask` or - :mod:`desitarget.brightmask.make_bright_source_mask`. - drbricks : :class:`~numpy.ndarray`, optional - A rec array containing at least the "release", "ra", "dec" and "nobjs" columns from a survey bricks file. - This is typically used for testing only. + A bright source mask as made by, e.g. + :func:`desitarget.brightmask.make_bright_star_mask()`. Returns ------- - The original recarray of targets (targs) is returned with additional SAFE (BADSKY) targets appended to it. + :class:`~numpy.ndarray` + SAFE (BADSKY) locations for `targs` with the same data model as + for `targs`. Notes ----- - - See `Tech Note 2346`_ for more on the SAFE (BADSKY) locations. - - See `Tech Note 2348`_ for more on setting the SKY bit in TARGETID. - - Currently hard-coded to create an additional 1 safe location per arcsec of mask radius. - The correct number per radial element (Nperradius) for DESI is an open question. + - `Tech Note 2346`_ details SAFE (BADSKY) locations. + - `Tech Note 2348`_ details setting the SKY bit in TARGETID. + - Hard-coded to create 1 safe location per arcsec of mask radius. + The correct number (Nperradius) for DESI is an open question. """ - # ADM Number of safe locations per radial arcsec of each mask. Nperradius = 1 - # ADM generate SAFE locations at the periphery of the masks appropriate to a density of Nperradius. + # ADM grab SAFE locations around masks at a density of Nperradius. ra, dec = generate_safe_locations(sourcemask, Nperradius) - # ADM duplicate the targs rec array with a number of rows equal to the generated safe locations. + # ADM duplicate targs data model for safe locations. nrows = len(ra) safes = np.zeros(nrows, dtype=targs.dtype) - # ADM populate the safes recarray with the RA/Dec of the SAFE locations. + # ADM populate the safes with the RA/Dec of the SAFE locations. safes["RA"] = ra safes["DEC"] = dec @@ -744,76 +757,64 @@ def append_safe_targets(targs, sourcemask, nside=None, drbricks=None): safes["BRICKID"] = b.brickid(safes["RA"], safes["DEC"]) safes["BRICKNAME"] = b.brickname(safes["RA"], safes["DEC"]) - # ADM get the string version of the data release (to find directories for brick information). - drint = np.max(targs['RELEASE']//1000) - # ADM check the targets all have the same release. - checker = np.min(targs['RELEASE']//1000) - if drint != checker: - raise IOError('Objects from multiple data releases in same input numpy array?!') - drstring = 'dr'+str(drint) - - # ADM now add the OBJIDs, ensuring they start higher than any other OBJID in the DR - # ADM read in the Data Release bricks file. - if drbricks is None: - rootdir = "/project/projectdirs/cosmo/data/legacysurvey/"+drstring.strip()+"/" - drbricks = fitsio.read(rootdir+"survey-bricks-"+drstring.strip()+".fits.gz") - # ADM the BRICK IDs that are populated for this DR. - drbrickids = b.brickid(drbricks["ra"], drbricks["dec"]) - # ADM the maximum possible BRICKID at bricksize=0.25. - brickmax = 662174 - # ADM create a histogram of how many SAFE/BADSKY objects are in each brick. - hsafes = np.histogram(safes["BRICKID"], range=[0, brickmax+1], bins=brickmax+1)[0] - # ADM create a histogram of how many objects are in each brick in this DR. - hnobjs = np.zeros(len(hsafes), dtype=int) - hnobjs[drbrickids] = drbricks["nobjs"] - # ADM make each OBJID for a SAFE/BADSKY +1 higher than any other OBJID in the DR. - safes["BRICK_OBJID"] = hnobjs[safes["BRICKID"]] + 1 - # ADM sort the safes array on BRICKID. - safes = safes[safes["BRICKID"].argsort()] - # ADM remove zero entries from the histogram of BRICKIDs in safes, for speed. - hsafes = hsafes[np.where(hsafes > 0)] - # ADM the count by which to augment each OBJID to make unique OBJIDs for safes. - objsadd = np.hstack([np.arange(i) for i in hsafes]) - # ADM finalize the OBJID for each SAFE target. - safes["BRICK_OBJID"] += objsadd - - # ADM finally, update the TARGETID with the OBJID, the BRICKID, and the fact these are skies. + # ADM now add OBJIDs, counting backwards from the maximum possible + # ADM OBJID to ensure no duplicateion of TARGETIDs for real targets. + maxobjid = 2**targetid_mask.OBJID.nbits - 1 + sortid = np.argsort(safes["BRICKID"]) + _, cnts = np.unique(safes["BRICKID"], return_counts=True) + brickids = np.concatenate([np.arange(i) for i in cnts]) + safes["BRICK_OBJID"][sortid] = brickids + + # ADM finally, update the TARGETID. safes["TARGETID"] = encode_targetid(objid=safes['BRICK_OBJID'], brickid=safes['BRICKID'], sky=1) # ADM return the input targs with the SAFE targets appended. - return np.hstack([targs, safes]) + return safes -def set_target_bits(targs, sourcemask): +def set_target_bits(targs, sourcemask, return_masks=False): """Apply bright source mask to targets, return desi_target array. Parameters ---------- targs : :class:`recarray` - A recarray of targets as made by, e.g., :mod:`desitarget.cuts.select_targets`. + Targets as made by, e.g., :func:`desitarget.cuts.select_targets()`. sourcemask : :class:`recarray` A recarray containing a bright source mask as made by, e.g. :mod:`desitarget.brightmask.make_bright_star_mask` or :mod:`desitarget.brightmask.make_bright_source_mask`. + return_masks : :class:`bool` + If ``True`` also return boolean arrays of which of the + masks in `sourcemask` contain a target. + Returns ------- - an ndarray of the updated desi_target bit that includes bright source information. + :class:`recarray` + `DESI_TARGET` column updates with bright source information bits. + :class:`list`, only returned if `return_masks` is ``True`` + [`used_in_mask`, `used_near_mask`] where `used_in_mask` + (`used_near_mask`) is a boolean array that is ``True`` for masks + in `sourcemask` that contain a target at the IN (NEAR) radius. Notes ----- - - Sets ``IN_BRIGHT_OBJECT`` and ``NEAR_BRIGHT_OBJECT`` via matches to - circular and/or elliptical masks. - - Sets BRIGHT_OBJECT via an index match on TARGETID - (defined as in :mod:`desitarget.targets.encode_targetid`). + - Sets ``IN_BRIGHT_OBJECT`` and ``NEAR_BRIGHT_OBJECT`` via + matches to circular and/or elliptical masks. + - Sets ``BRIGHT_OBJECT`` via an index match on ``TARGETID`` + (defined as in :func:`desitarget.targets.encode_targetid()`). See :mod:`desitarget.targetmask` for the definition of each bit. """ + if "TARGETID" in sourcemask.dtype.names: + bright_object = is_bright_source(targs, sourcemask) + else: + bright_object = 0 - bright_object = is_bright_source(targs, sourcemask) - in_bright_object, near_bright_object = is_in_bright_mask(targs, sourcemask) + intargs, inmasks = is_in_bright_mask(targs, sourcemask) + in_bright_object, near_bright_object = intargs desi_target = targs["DESI_TARGET"].copy() @@ -821,90 +822,95 @@ def set_target_bits(targs, sourcemask): desi_target |= in_bright_object * desi_mask.IN_BRIGHT_OBJECT desi_target |= near_bright_object * desi_mask.NEAR_BRIGHT_OBJECT + if return_masks: + return desi_target, inmasks return desi_target -def mask_targets(targs, inmaskfile=None, nside=None, bands="GRZ", maglim=[10, 10, 10], numproc=4, - outfilename=None, drbricks=None): - """Add bits for if objects are in a bright mask, and SAFE (BADSKY) locations, to a target set. +def mask_targets(targs, inmaskdir, nside=2, pixlist=None): + """Add bits for if objects occupy masks, and SAFE (BADSKY) locations. Parameters ---------- targs : :class:`str` or `~numpy.ndarray` - A recarray of targets created by :mod:`desitarget.cuts.select_targets` OR a filename of - a file that contains such a set of targets - inmaskfile : :class:`str`, optional - An input bright source mask created by, e.g. - :mod:`desitarget.brightmask.make_bright_star_mask` or - :mod:`desitarget.brightmask.make_bright_source_mask` - If None, defaults to making the bright mask from scratch - The next 5 parameters are only relevant to making the bright mask from scratch - nside : :class:`integer` - The HEALPix nside used throughout the DESI data model - bands : :class:`str` - A magnitude band from the sweeps, e.g., "G", "R", "Z". - Can pass multiple bands as string, e.g. "GRZ", in which case maglim has to be a - list of the same length as the string - maglim : :class:`float` - The upper limit in that magnitude band for which to assemble a list of bright sources. - Can pass a list of magnitude limits, in which case bands has to be a string of the - same length (e.g., "GRZ" for [12.3,12.7,12.6]) - numproc : :class:`int`, optional - Number of processes over which to parallelize - outfilename : :class:`str`, optional, defaults to not writing anything to file - (FITS) File name to which to write the output mask ONE OF outfilename or - inmaskfile MUST BE PASSED - drbricks : :class:`~numpy.ndarray`, optional - A rec array containing at least the "release", "ra", "dec" and "nobjs" columns from a survey bricks file - This is typically used for testing only. + An array of targets/skies etc. created by, e.g., + :func:`desitarget.cuts.select_targets()` OR the filename of a + file that contains such a set of targets/skies, etc. + inmaskdir : :class:`str`, optional + An input bright star mask file or HEALPixel-split directory as + made by :func:`desitarget.brightmask.make_bright_star_mask()` + nside : :class:`int`, optional, defaults to 2 + The nside at which the targets were generated. If the mask is + a HEALPixel-split directory, then this helps to perform more + efficient masking as only the subset of masks that are in + pixels containing `targs` at this `nside` will be considered + (together with neighboring pixels to account for edge effects). + pixlist : :class:`list` or `int`, optional + A set of HEALPixels corresponding to the `targs`. Only the subset + of masks in HEALPixels in `pixlist` at `nside` will be considered + (together with neighboring pixels to account for edge effects). + If ``None``, then the pixels touched by `targs` is derived from + from `targs` itself. Returns ------- :class:`~numpy.ndarray` - the input targets with the DESI_TARGET column updated to reflect the BRIGHT_OBJECT bits - and SAFE (BADSKY) sky locations added around the perimeter of the bright source mask. + Input targets with the `DESI_TARGET` column updated to reflect + the `BRIGHT_OBJECT` bits and SAFE (`BADSKY`) sky locations added + around the perimeter of the mask. Notes ----- - - See `Tech Note 2346`_ for more details about SAFE (BADSKY) locations. - - Runs in about 10 minutes for 20M targets and 50k masks (roughly maglim=10). + - `Tech Note 2346`_ details SAFE (BADSKY) locations. """ - t0 = time() - if inmaskfile is None and outfilename is None: - raise IOError('One of inmaskfile or outfilename must be passed') - - # ADM Check if targs is a filename or the structure itself. + # ADM Check if targs is a file name or the structure itself. if isinstance(targs, str): if not os.path.exists(targs): raise ValueError("{} doesn't exist".format(targs)) targs = fitsio.read(targs) - sourcemask = fitsio.read(inmaskfile) - - ntargsin = len(targs) - log.info('Number of targets {}...t={:.1f}s'.format(ntargsin, time()-t0)) - log.info('Number of masks {}...t={:.1f}s'.format(len(sourcemask), time()-t0)) + # ADM determine which pixels are occupied by targets. + if pixlist is None: + theta, phi = np.radians(90-targs["DEC"]), np.radians(targs["RA"]) + pixlist = list(set(hp.ang2pix(nside, theta, phi, nest=True))) + else: + # ADM in case an integer was passed. + pixlist = np.atleast_1d(pixlist) + pixlist = add_hp_neighbors(nside, pixlist) - # ADM generate SAFE locations and add them to the target list. - targs = append_safe_targets(targs, sourcemask, nside=nside, drbricks=drbricks) + # ADM read in the (potentially HEALPixel-split) mask. + sourcemask = io.read_targets_in_hp(inmaskdir, nside, pixlist) - log.info('Generated {} SAFE (BADSKY) locations...t={:.1f}s'.format(len(targs)-ntargsin, time()-t0)) + ntargs = len(targs) + log.info('Total number of masks {}'.format(len(sourcemask))) + log.info('Total number of targets {}...t={:.1f}s'.format(ntargs, time()-t0)) # ADM update the bits depending on whether targets are in a mask. - dt = set_target_bits(targs, sourcemask) - done = targs.copy() - done["DESI_TARGET"] = dt + # ADM also grab masks that contain or are near a target. + dt, mx = set_target_bits(targs, sourcemask, return_masks=True) + targs["DESI_TARGET"] = dt + inmasks, nearmasks = mx + + # ADM generate SAFE locations for masks that contain a target. + safes = get_safe_targets(targs, sourcemask[inmasks]) + # ADM update the bits for the safe locations depending on whether + # ADM they're in a mask. + safes["DESI_TARGET"] = set_target_bits(safes, sourcemask) + # ADM combine the targets and safe locations. + done = np.concatenate([targs, safes]) + + log.info('Generated {} SAFE (BADSKY) locations...t={:.1f}s'.format( + len(done)-ntargs, time()-t0)) # ADM remove any SAFE locations that are in bright masks (because they aren't really safe). - w = np.where(((done["DESI_TARGET"] & desi_mask.BAD_SKY) == 0) | - ((done["DESI_TARGET"] & desi_mask.IN_BRIGHT_OBJECT) == 0)) - if len(w[0]) > 0: - done = done[w] + ii = (((done["DESI_TARGET"] & desi_mask.BAD_SKY) == 0) | + ((done["DESI_TARGET"] & desi_mask.IN_BRIGHT_OBJECT) == 0)) + done = done[ii] log.info("...of these, {} SAFE (BADSKY) locations aren't in masks...t={:.1f}s" - .format(len(done)-ntargsin, time()-t0)) + .format(len(done)-ntargs, time()-t0)) log.info('Finishing up...t={:.1f}s'.format(time()-t0)) diff --git a/py/desitarget/geomask.py b/py/desitarget/geomask.py index 333ad338b..e9536fe20 100644 --- a/py/desitarget/geomask.py +++ b/py/desitarget/geomask.py @@ -580,27 +580,28 @@ def circle_boundaries(RAcens, DECcens, r, nloc): The Declinations of nloc equally space locations on the periphery of the mask """ - - # ADM the radius of each mask in degrees with a 0.1% kick to get things beyond the mask edges + # ADM radius in degrees with a 0.1% kick to push beyond the edge. radius = 1.001*r/3600. - # ADM determine nloc Dec offsets equally spaced around the perimeter for each mask - offdec = [rad*np.sin(np.arange(ns)*2*np.pi/ns) for ns, rad in zip(nloc, radius)] + # ADM nloc Dec offsets equally spaced around the circle perimeter. + offdec = np.array([rad*np.sin(np.arange(ns)*2*np.pi/ns) + for ns, rad in zip(nloc, radius)]).transpose() - # ADM use offsets to determine DEC positions + # ADM use offsets to determine DEC positions. decs = DECcens + offdec - # ADM determine the offsets in RA at these Decs given the mask center Dec - offrapos = [sphere_circle_ra_off(th, cen, declocs) for th, cen, declocs in zip(radius, DECcens, decs)] + # ADM offsets in RA at these Decs given the mask center Dec. + offrapos = [sphere_circle_ra_off(th, cen, declocs) + for th, cen, declocs in zip(radius, DECcens, decs.transpose())] - # ADM determine which of the RA offsets are in the positive direction + # ADM determine which RA offsets are in the positive direction. sign = [np.sign(np.cos(np.arange(ns)*2*np.pi/ns)) for ns in nloc] - # ADM determine the RA offsets with the appropriate sign and add them to the RA of each mask - offra = [o*s for o, s in zip(offrapos, sign)] + # ADM add RA offsets with the right sign to the the circle center. + offra = np.array([o*s for o, s in zip(offrapos, sign)]).transpose() ras = RAcens + offra - # ADM have to turn the generated locations into 1-D arrays before returning them + # ADM return the results as 1-D arrays. return np.hstack(ras), np.hstack(decs) From f4ab58cc3f7d807ed8d4030c55880005b55df8f2 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 12 Jun 2020 16:46:35 -0700 Subject: [PATCH 15/25] update the select_skies executable to mask for bright stars by default --- bin/select_skies | 55 ++++++++++++++++++++++++++++++++++--- py/desitarget/brightmask.py | 4 ++- 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/bin/select_skies b/bin/select_skies index 890e4bd95..94b85b6a3 100755 --- a/bin/select_skies +++ b/bin/select_skies @@ -8,9 +8,11 @@ from desitarget import io from desitarget.io import check_both_set from desitarget.geomask import bundle_bricks from desitarget.targets import resolve +from desitarget.brightmask import mask_targets import numpy as np import healpy as hp +from glob import glob #import warnings #warnings.simplefilter('error') @@ -58,10 +60,26 @@ ap.add_argument("--brickspersec", type=float, help="estimate of bricks completed per second by the (parallelized) code. Used with `bundlebricks` to guess run times (defaults to 1.6)", default=1.6) ap.add_argument("--numproc", type=int, - help='number of concurrent processes to use [{}]'.format(nproc), + help="number of concurrent processes to use [{}]".format(nproc), default=nproc) +ap.add_argument("--nomasking", action='store_true', + help="Masking occurs by default. If this is set, do NOT use a bright star mask to mask the sky locations") +ap.add_argument("--maskdir", + help="Name of the specific directory (or file) containing the bright star mask (defaults to the most recent directory in $MASK_DIR)", + default=None) ns = ap.parse_args() +do_mask = not(ns.nomasking) +# ADM build the list of command line arguments as +# ADM bundlefiles potentially needs to know about them. +extra = " --numproc {}".format(ns.numproc) +nsdict = vars(ns) +for nskey in "nskiespersqdeg", "bands", "apertures", "nomasking", "maskdir": + if isinstance(nsdict[nskey], bool): + if nsdict[nskey]: + extra += " --{}".format(nskey) + elif nsdict[nskey] is not None: + extra += " --{} {}".format(nskey, nsdict[nskey]) for indir in ns.surveydir, ns.surveydir2: if indir is not None: @@ -106,8 +124,9 @@ if ns.bundlebricks is not None: # ADM pixnum only contains unique bricks, need to add duplicates. allpixnum = np.concatenate([np.zeros(cnt, dtype=int)+pix for cnt, pix in zip(cnts.astype(int), pixnum)]) - bundle_bricks(allpixnum, ns.bundlebricks, ns.nside, prefix='skies', - gather=False, surveydirs=drdirs, brickspersec=ns.brickspersec) + bundle_bricks( + allpixnum, ns.bundlebricks, ns.nside, prefix='skies', extra=extra, + gather=False, surveydirs=drdirs, brickspersec=ns.brickspersec) else: log.info("running on {} processors".format(ns.numproc)) # ADM formally writing pixelized files requires both the nside @@ -139,11 +158,39 @@ else: else: skies = resolved + # ADM mask the sky locations using a bright star mask. + if do_mask: + if ns.maskdir is not None: + maskdir = ns.maskdir + else: + # ADM default to finding the most recent mask directory. + try: + md = os.environ["MASK_DIR"] + except KeyError: + msg = "either pass maskdir or set $MASK_DIR!" + log.error(msg) + raise IOError(msg) + # ADM a fairly exhaustive list of possible mask directories. + mds = glob(os.path.join(md, "*maglim*")) + \ + glob(os.path.join(md, "*/*maglim*")) + \ + glob(os.path.join(md, "*/*/*maglim*")) + if len(mds)==0: + msg = "no mask directories found (maybe pass --nomasking?)!" + log.error(msg) + raise IOError(msg) + maskdir = max(mds, key=os.path.getctime) + + skies = mask_targets(skies, maskdir, nside=ns.nside, pixlist=pixlist) + + # ADM extra header keywords for the output fits file. + extra = {k: v for k, v in zip(["masked"], + [do_mask])} + # ADM this correctly records the apertures in the output file header # ADM as well as adding HEALPixel information. nskies, outfile = io.write_skies(ns.dest, skies, indir=ns.surveydir, indir2=ns.surveydir2, nside=nside, - apertures_arcsec=apertures, + apertures_arcsec=apertures, extra=extra, nskiespersqdeg=nskiespersqdeg, nsidefile=ns.nside, hpxlist=pixlist) diff --git a/py/desitarget/brightmask.py b/py/desitarget/brightmask.py index b8fe47893..e09d2f250 100644 --- a/py/desitarget/brightmask.py +++ b/py/desitarget/brightmask.py @@ -432,7 +432,7 @@ def plot_mask(mask, limits=None, radius="IN_RADIUS", show=True): radius : :class: `str`, optional Which mask radius to plot (``IN_RADIUS`` or ``NEAR_RADIUS``). show : :class:`boolean` - If ``True``, then display the plot, Otherwise, just execute the + If ``True``, then display the plot, Otherwise, just execute the plot commands so it can be added to or saved to file later. Returns @@ -878,6 +878,8 @@ def mask_targets(targs, inmaskdir, nside=2, pixlist=None): else: # ADM in case an integer was passed. pixlist = np.atleast_1d(pixlist) + log.info("Masking using masks in {} at nside={} in HEALPixels={}".format( + inmaskdir, nside, pixlist)) pixlist = add_hp_neighbors(nside, pixlist) # ADM read in the (potentially HEALPixel-split) mask. From e53f5753afa040a07bcb6dd4ce4ffeedae5fbde5 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 12 Jun 2020 17:12:24 -0700 Subject: [PATCH 16/25] minor code style and logging changes --- py/desitarget/brightmask.py | 2 +- py/desitarget/io.py | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/py/desitarget/brightmask.py b/py/desitarget/brightmask.py index e09d2f250..1f8ec6512 100644 --- a/py/desitarget/brightmask.py +++ b/py/desitarget/brightmask.py @@ -848,7 +848,7 @@ def mask_targets(targs, inmaskdir, nside=2, pixlist=None): pixlist : :class:`list` or `int`, optional A set of HEALPixels corresponding to the `targs`. Only the subset of masks in HEALPixels in `pixlist` at `nside` will be considered - (together with neighboring pixels to account for edge effects). + (together with neighboring pixels to account for edge effects). If ``None``, then the pixels touched by `targs` is derived from from `targs` itself. diff --git a/py/desitarget/io.py b/py/desitarget/io.py index 297736d04..b674220a9 100644 --- a/py/desitarget/io.py +++ b/py/desitarget/io.py @@ -1200,7 +1200,7 @@ def write_masks(maskdir, data, theta, phi = np.radians(90-data["DEC"]), np.radians(data["RA"]) hpx = hp.ang2pix(nside, theta, phi, nest=True) for pix in range(npix): - outdata = data[hpx==pix] + outdata = data[hpx == pix] outhdr = dict(hdr).copy() outhdr["FILEHPX"] = pix # ADM construct the output file name. @@ -1948,7 +1948,7 @@ def find_target_files(targdir, dr=None, flavor="targets", survey="main", def read_target_files(filename, columns=None, rows=None, header=False, - downsample=None, verbose=True): + downsample=None, verbose=False): """Wrapper to cycle through allowed extensions to read target files. Parameters @@ -2006,7 +2006,7 @@ def read_target_files(filename, columns=None, rows=None, header=False, def read_targets_in_hp(hpdirname, nside, pixlist, columns=None, - header=False, downsample=None): + header=False, downsample=None, verbose=False): """Read in targets in a set of HEALPixels. Parameters @@ -2029,6 +2029,8 @@ def read_targets_in_hp(hpdirname, nside, pixlist, columns=None, If not `None`, downsample targets by (roughly) this value, e.g. for `downsample=10` a set of 900 targets would have ~90 random targets returned. + verbose : :class:`bool`, optional, defaults to ``False`` + Passed to :func:`read_target_files()`. Returns ------- @@ -2064,7 +2066,7 @@ def read_targets_in_hp(hpdirname, nside, pixlist, columns=None, fn0 = list(filedict.values())[0] notargs, nohdr = read_target_files( fn0, columns=columnscopy, rows=0, header=True, - downsample=downsample, verbose=False) + downsample=downsample, verbose=verbose) notargs = np.zeros(0, dtype=notargs.dtype) # ADM change the passed pixels to the nside of the file schema. @@ -2081,8 +2083,9 @@ def read_targets_in_hp(hpdirname, nside, pixlist, columns=None, targets = [] start = time() for infile in infiles: - targs, hdr = read_target_files(infile, columns=columnscopy, - header=True, downsample=downsample) + targs, hdr = read_target_files( + infile, columns=columnscopy, header=True, + downsample=downsample, verbose=verbose) targets.append(targs) # ADM if targets is empty, return no targets. if len(targets) == 0: @@ -2093,8 +2096,9 @@ def read_targets_in_hp(hpdirname, nside, pixlist, columns=None, targets = np.concatenate(targets) # ADM ...otherwise just read in the targets. else: - targets, hdr = read_target_files(hpdirname, columns=columnscopy, - header=True, downsample=downsample) + targets, hdr = read_target_files( + hpdirname, columns=columnscopy, header=True, + downsample=downsample, verbose=verbose) # ADM restrict the targets to the actual requested HEALPixels... ii = is_in_hp(targets, nside, pixlist) From 11c98d7cb3a50ceb35c3d7d7593354e24d68a851 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Sun, 14 Jun 2020 10:10:32 -0700 Subject: [PATCH 17/25] catch a corner case when no safe locations are found --- py/desitarget/brightmask.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/py/desitarget/brightmask.py b/py/desitarget/brightmask.py index 1f8ec6512..9897eeffd 100644 --- a/py/desitarget/brightmask.py +++ b/py/desitarget/brightmask.py @@ -735,7 +735,7 @@ def get_safe_targets(targs, sourcemask): - Hard-coded to create 1 safe location per arcsec of mask radius. The correct number (Nperradius) for DESI is an open question. """ - # ADM Number of safe locations per radial arcsec of each mask. + # ADM number of safe locations per radial arcsec of each mask. Nperradius = 1 # ADM grab SAFE locations around masks at a density of Nperradius. @@ -744,6 +744,9 @@ def get_safe_targets(targs, sourcemask): # ADM duplicate targs data model for safe locations. nrows = len(ra) safes = np.zeros(nrows, dtype=targs.dtype) + # ADM return early if there are no safe locations. + if nrows == 0: + return safes # ADM populate the safes with the RA/Dec of the SAFE locations. safes["RA"] = ra From 1764d9f91a13e745c390683c22ae74eabc48a46d Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Mon, 15 Jun 2020 12:39:43 -0700 Subject: [PATCH 18/25] update the supplement_skies executable to mask for bright stars by default --- bin/select_skies | 23 ++-------------------- bin/supplement_skies | 25 ++++++++++++++++++++---- py/desitarget/brightmask.py | 38 ++++++++++++++++++++++++++++++++++++- 3 files changed, 60 insertions(+), 26 deletions(-) diff --git a/bin/select_skies b/bin/select_skies index 94b85b6a3..7f71994b6 100755 --- a/bin/select_skies +++ b/bin/select_skies @@ -8,7 +8,7 @@ from desitarget import io from desitarget.io import check_both_set from desitarget.geomask import bundle_bricks from desitarget.targets import resolve -from desitarget.brightmask import mask_targets +from desitarget.brightmask import mask_targets, get_recent_mask_dir import numpy as np import healpy as hp @@ -160,26 +160,7 @@ else: # ADM mask the sky locations using a bright star mask. if do_mask: - if ns.maskdir is not None: - maskdir = ns.maskdir - else: - # ADM default to finding the most recent mask directory. - try: - md = os.environ["MASK_DIR"] - except KeyError: - msg = "either pass maskdir or set $MASK_DIR!" - log.error(msg) - raise IOError(msg) - # ADM a fairly exhaustive list of possible mask directories. - mds = glob(os.path.join(md, "*maglim*")) + \ - glob(os.path.join(md, "*/*maglim*")) + \ - glob(os.path.join(md, "*/*/*maglim*")) - if len(mds)==0: - msg = "no mask directories found (maybe pass --nomasking?)!" - log.error(msg) - raise IOError(msg) - maskdir = max(mds, key=os.path.getctime) - + maskdir = get_recent_mask_dir(ns.maskdir) skies = mask_targets(skies, maskdir, nside=ns.nside, pixlist=pixlist) # ADM extra header keywords for the output fits file. diff --git a/bin/supplement_skies b/bin/supplement_skies index d1ba2b563..ec691a83d 100755 --- a/bin/supplement_skies +++ b/bin/supplement_skies @@ -3,6 +3,7 @@ from desitarget.skyfibers import supplement_skies, density_of_sky_fibers from desitarget import io from desitarget.geomask import bundle_bricks, shares_hp, pixarea2nside +from desitarget.brightmask import mask_targets, get_recent_mask_dir import os, sys import numpy as np @@ -56,8 +57,14 @@ ap.add_argument("--mingalb", type=float, ap.add_argument("--radius", type=float, help="Radius at which to avoid (all) Gaia sources (arcseconds; defaults to [2])", default=2.) +ap.add_argument("--nomasking", action='store_true', + help="Masking occurs by default. If this is set, do NOT use Tycho+Gaia+URAT bright star mask to mask the sky locations") +ap.add_argument("--maskdir", + help="Name of the specific directory (or file) containing the bright star mask (defaults to the most recent directory in $MASK_DIR)", + default=None) ns = ap.parse_args() +do_mask = not(ns.nomasking) # ADM check the input sky file is in the "official" format. official = io.is_sky_dir_official(ns.skydir) @@ -96,8 +103,12 @@ extra = " --numproc {}".format(ns.numproc) extra += " --nskiespersqdeg {}".format(nskiespersqdeg) extra += " --gaiadir {}".format(gaiadir) nsdict = vars(ns) -for nskey in "mindec", "mingalb", "radius": - extra += " --{} {}".format(nskey, nsdict[nskey]) +for nskey in "mindec", "mingalb", "radius", "nomasking", "maskdir": + if isinstance(nsdict[nskey], bool): + if nsdict[nskey]: + extra += " --{}".format(nskey) + elif nsdict[nskey] is not None: + extra += " --{} {}".format(nskey, nsdict[nskey]) # ADM only proceed if we're not writing a slurm script. if ns.bundlefiles is None: @@ -135,6 +146,12 @@ if ns.bundlefiles is None: pixlist=pixlist, mindec=ns.mindec, mingalb=ns.mingalb) + # ADM mask the supplemental sky locations using a bright star mask. + if do_mask: + maskdir = get_recent_mask_dir(ns.maskdir) + supp_skies = mask_targets(supp_skies, maskdir, + nside=ns.nside, pixlist=pixlist) + # ADM remove supplemental skies that share HEALPixels with skies. nside_resol = pixarea2nside(1./18000)//2 ii, _ = shares_hp(nside_resol, supp_skies, skies) @@ -142,8 +159,8 @@ if ns.bundlefiles is None: log.info("Removed {} supp skies that matched skies".format(np.sum(ii))) # ADM extra header keywords for the output fits file. - extra = {k: v for k, v in zip(["radius", "mindec", "mingalb"], - [ns.radius, ns.mindec, ns.mingalb])} + extra = {k: v for k, v in zip(["radius", "mindec", "mingalb", "masked"], + [ns.radius, ns.mindec, ns.mingalb, do_mask])} nskies, outfile = io.write_skies(ns.dest, supp_skies, supp=True, indir=gaiadir, nside=nside, nskiespersqdeg=nskiespersqdeg, diff --git a/py/desitarget/brightmask.py b/py/desitarget/brightmask.py index 9897eeffd..50201bc03 100644 --- a/py/desitarget/brightmask.py +++ b/py/desitarget/brightmask.py @@ -63,7 +63,7 @@ def get_mask_dir(): Returns ------- :class:`str` - The directory stored in the $TYCHO_DIR environment variable. + The directory stored in the $MASK_DIR environment variable. """ # ADM check that the $MASK_DIR environment variable is set. maskdir = os.environ.get('MASK_DIR') @@ -75,6 +75,42 @@ def get_mask_dir(): return maskdir +def get_recent_mask_dir(input_dir=None): + """Grab the most recent sub-directory of masks in MASK_DIR. + + Parameters + ---------- + input_dir : :class:`str`, optional, defaults to ``None`` + If passed and not ``None``, then this is returned as the output. + + Returns + ------- + :class:`str` + If `input_dir` is not ``None``, then the most recently created + sub-directory (with the appropriate format for a mask directory) + in $MASK_DIR is returned. + """ + if input_dir is not None: + return input_dir + else: + # ADM glob the most recent mask directory. + try: + md = os.environ["MASK_DIR"] + except KeyError: + msg = "pass a mask directory, turn off masking, or set $MASK_DIR!" + log.error(msg) + raise IOError(msg) + # ADM a fairly exhaustive list of possible mask directories. + mds = glob(os.path.join(md, "*maglim*")) + \ + glob(os.path.join(md, "*/*maglim*")) + \ + glob(os.path.join(md, "*/*/*maglim*")) + if len(mds)==0: + msg = "no mask sub-directories found in {}".format(md) + log.error(msg) + raise IOError(msg) + return max(mds, key=os.path.getctime) + + def radii(mag): """The relation used to set the radius of bright star masks. From d4e17282c83d60237d32b85ba56593a26a7823f6 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Mon, 15 Jun 2020 12:55:31 -0700 Subject: [PATCH 19/25] remove file of unit tests that are completely deprecated --- py/desitarget/brightmask.py | 6 +- py/desitarget/test/test_brightstar.py | 79 --------------------------- 2 files changed, 3 insertions(+), 82 deletions(-) delete mode 100644 py/desitarget/test/test_brightstar.py diff --git a/py/desitarget/brightmask.py b/py/desitarget/brightmask.py index 50201bc03..5b858b886 100644 --- a/py/desitarget/brightmask.py +++ b/py/desitarget/brightmask.py @@ -102,9 +102,9 @@ def get_recent_mask_dir(input_dir=None): raise IOError(msg) # ADM a fairly exhaustive list of possible mask directories. mds = glob(os.path.join(md, "*maglim*")) + \ - glob(os.path.join(md, "*/*maglim*")) + \ - glob(os.path.join(md, "*/*/*maglim*")) - if len(mds)==0: + glob(os.path.join(md, "*/*maglim*")) + \ + glob(os.path.join(md, "*/*/*maglim*")) + if len(mds) == 0: msg = "no mask sub-directories found in {}".format(md) log.error(msg) raise IOError(msg) diff --git a/py/desitarget/test/test_brightstar.py b/py/desitarget/test/test_brightstar.py deleted file mode 100644 index 47c85debb..000000000 --- a/py/desitarget/test/test_brightstar.py +++ /dev/null @@ -1,79 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst -# -*- coding: utf-8 -*- -"""Old tests of desitarget.brightmask (deprecated). -""" -import unittest -from pkg_resources import resource_filename -import os.path -import fitsio -import numpy as np - -# ADM this is an old set of tests from when the brightmask -# ADM module only worked with circular sources and was -# ADM therefore called brightstar instead of brightmask -from desitarget import brightmask as brightstar -# ADM these remain useful tests to increase coverage, though -from desitarget.targetmask import desi_mask, targetid_mask - - -class TestBRIGHTSTAR(unittest.TestCase): - - def setUp(self): - # ADM some locations of output test files - self.testbsfile = 'bs.fits' - self.testmaskfile = 'bsmask.fits' - - # ADM some locations of input files - self.bsdatadir = resource_filename('desitarget.test', 't2') - self.datadir = resource_filename('desitarget.test', 't') - - def tearDown(self): - # ADM remove any existing bright star files in this directory - if os.path.exists(self.testmaskfile): - os.remove(self.testmaskfile) - if os.path.exists(self.testbsfile): - os.remove(self.testbsfile) - - @unittest.skip('This test is deprecated.') - def test_collect_bright_stars(self): - """Test the collection of bright stars from the sweeps - """ - # ADM collect the bright stars from the sweeps in the data directory and write to file... - bs1 = brightstar.collect_bright_stars('grz', [9, 9, 9], - rootdirname=self.bsdatadir, outfilename=self.testbsfile) - # ADM ...and read in the file that was written - bs2 = fitsio.read(self.testbsfile) - # ADM the created collection of objects from the sweeps should be the same as the read-in file - bs1ids = bs1['BRICKID'].astype(np.int64)*1000000 + bs1['OBJID'] - bs2ids = bs2['BRICKID'].astype(np.int64)*1000000 + bs2['OBJID'] - self.assertTrue(np.all(bs1ids == bs2ids)) - - @unittest.skip('This test is deprecated.') - def test_make_bright_star_mask(self): - """Test the construction of a bright star mask - """ - # ADM create a collection of bright stars and write to file - bs1 = brightstar.collect_bright_stars('grz', [23, 23, 23], - rootdirname=self.bsdatadir, outfilename=self.testbsfile) - # ADM create a bright star mask from the collection of bright stars and write to file... - mask = brightstar.make_bright_star_mask('grz', [23, 23, 23], - infilename=self.testbsfile, outfilename=self.testmaskfile) - # ADM ...and read it back in - mask1 = fitsio.read(self.testmaskfile) - # ADM create the bright star mask from scratch - mask2 = brightstar.make_bright_star_mask('grz', [23, 23, 23], - rootdirname=self.bsdatadir) - # ADM the created-from-scratch mask should be the same as the read-in mask - self.assertTrue(np.all(mask1["TARGETID"] == mask2["TARGETID"])) - - -if __name__ == '__main__': - unittest.main() - - -def test_suite(): - """Allows testing of only this module with the command: - - python setup.py test -m desitarget.test.test_brightstar - """ - return unittest.defaultTestLoader.loadTestsFromName(__name__) From 1f6ef31c3054e6c2149fad38c920b77dcad24a6e Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Mon, 15 Jun 2020 13:45:15 -0700 Subject: [PATCH 20/25] begin updating unit tests, fix bug where Tycho objects were bring read in at file resolution instead of in the passed pixel --- py/desitarget/brightmask.py | 7 ++-- py/desitarget/test/test_brightmask.py | 46 +++++---------------------- 2 files changed, 13 insertions(+), 40 deletions(-) diff --git a/py/desitarget/brightmask.py b/py/desitarget/brightmask.py index 5b858b886..82c6f1736 100644 --- a/py/desitarget/brightmask.py +++ b/py/desitarget/brightmask.py @@ -223,8 +223,11 @@ def make_bright_star_mask_in_hp(nside, pixnum, verbose=True, gaiaepoch=2015.5, tychomag = tychoobjs["MAG_VT"].copy() tychomag[tychomag == 0] = tychoobjs["MAG_HP"][tychomag == 0] tychomag[tychomag == 0] = tychoobjs["MAG_BT"][tychomag == 0] - # ADM discard any Tycho objects below the input magnitude limit. - ii = tychomag < maglim + # ADM discard any Tycho objects below the input magnitude limit + # ADM and outside of the HEALPixels of interest. + theta, phi = np.radians(90-tychoobjs["DEC"]), np.radians(tychoobjs["RA"]) + tychohpx = hp.ang2pix(nside, theta, phi, nest=True) + ii = (tychohpx == pixnum) & (tychomag < maglim) tychomag, tychoobjs = tychomag[ii], tychoobjs[ii] if verbose: log.info('Read {} (mag < {}) Tycho objects (pix={})...t={:.1f} mins'. diff --git a/py/desitarget/test/test_brightmask.py b/py/desitarget/test/test_brightmask.py index 32f3802a6..c47b81e36 100644 --- a/py/desitarget/test/test_brightmask.py +++ b/py/desitarget/test/test_brightmask.py @@ -53,14 +53,7 @@ def setUp(self): self.unmasktargs = rfn.append_fields(unmasktargs, "BRICK_OBJID", zeros, usemask=False, dtypes='>i4') self.unmasktargs["BRICK_OBJID"] = self.unmasktargs["OBJID"] - # ADM set up brick information for just the brick with brickID 330368 (bricksize=0.25) - self.drbricks = np.zeros(1, dtype=[('ra', '>f8'), ('dec', '>f8'), ('nobjs', '>i2')]) - self.drbricks["ra"] = 0.125 - self.drbricks["dec"] = 0.0 - self.drbricks["nobjs"] = 1000 - # ADM invent a mask with differing radii (1' and 20') and declinations - # ADM some elliptical components, and some different formats for TYPE self.mask = np.zeros(3, dtype=[('RA', '>f8'), ('DEC', '>f8'), ('IN_RADIUS', '>f4'), ('E1', '>f4'), ('E2', '>f4'), ('TYPE', 'S4')]) self.mask["DEC"] = [0, 70, 35] @@ -78,19 +71,6 @@ def tearDown(self): if os.path.exists(self.testbsfile): os.remove(self.testbsfile) - def test_collect_bright_sources(self): - """Test the collection of bright sources from the sweeps - """ - # ADM collect the bright sources from the sweeps in the data directory and write to file... - bs1 = brightmask.collect_bright_sources('grz', [9, 9, 9], - rootdirname=self.bsdatadir, outfilename=self.testbsfile) - # ADM ...and read in the file that was written - bs2 = fitsio.read(self.testbsfile) - # ADM the created collection of objects from the sweeps should be the same as the read-in file - bs1ids = bs1['BRICKID'].astype(np.int64)*1000000 + bs1['OBJID'] - bs2ids = bs2['BRICKID'].astype(np.int64)*1000000 + bs2['OBJID'] - self.assertTrue(np.all(bs1ids == bs2ids)) - def test_make_bright_source_mask(self): """Test the construction of a bright source mask """ @@ -111,9 +91,8 @@ def test_mask_targets(self): """Test that targets in masks are flagged as being in masks """ # ADM mask the targets, creating the mask - targs = brightmask.mask_targets(self.masktargs, bands="RZ", maglim=[8, 10], numproc=1, - rootdirname=self.bsdatadir, outfilename=self.testmaskfile, - drbricks=self.drbricks) + targs = brightmask.mask_targets(self.masktargs, maglim=[8, 10], numproc=1, + rootdirname=self.bsdatadir, outfilename=self.testmaskfile) self.assertTrue(np.any(targs["DESI_TARGET"] != 0)) def test_non_mask_targets(self): @@ -125,8 +104,8 @@ def test_non_mask_targets(self): mask = brightmask.make_bright_source_mask('RZ', [8, 10], rootdirname=self.bsdatadir, outfilename=self.testmaskfile) # ADM mask the targets, reading in the mask - targs = brightmask.mask_targets(self.testtargfile, - inmaskfile=self.testmaskfile, drbricks=self.drbricks) + targs = brightmask.mask_targets(self.testtargfile, inmaskfile=self.testmaskfile) + # ADM none of the targets should have been masked self.assertTrue(np.all((targs["DESI_TARGET"] == 0) | ((targs["DESI_TARGET"] & desi_mask.BAD_SKY) != 0))) @@ -134,7 +113,8 @@ def test_safe_locations(self): """Test that SAFE/BADSKY locations are equidistant from mask centers """ # ADM append SAFE (BADSKY) locations around the perimeter of the mask - targs = brightmask.append_safe_targets(self.unmasktargs, self.mask, drbricks=self.drbricks) + safes = brightmask.get_safe_targets(self.unmasktargs, self.mask) + targs = np.concatenate([self.unmasktargs, safes]) # ADM restrict to just SAFE (BADSKY) locations skybitset = ((targs["TARGETID"] & targetid_mask.SKY) != 0) safes = targs[np.where(skybitset)] @@ -153,7 +133,8 @@ def test_targetid(self): """Test SKY/RELEASE/BRICKID/OBJID are set correctly in TARGETID and DESI_TARGET for SAFE/BADSKY locations """ # ADM append SAFE (BADSKY) locations around the periphery of the mask - targs = brightmask.append_safe_targets(self.unmasktargs, self.mask, drbricks=self.drbricks) + safes = brightmask.get_safe_targets(self.unmasktargs, self.mask) + targs = np.concatenate([self.unmasktargs, safes]) # ADM first check that the SKY bit and BADSKY bits are appropriately set skybitset = ((targs["TARGETID"] & targetid_mask.SKY) != 0) @@ -180,17 +161,6 @@ def test_targetid(self): self.assertEqual(drbitset, drbitshould) self.assertEqual(drbitset, 0) - # ADM check that the OBJIDs proceed from "nobjs" in self.drbricks - rmostbit = targetid_mask.OBJID.bitnum - lmostbit = targetid_mask.OBJID.bitnum + targetid_mask.OBJID.nbits - # ADM guard against the fact that when written the rmostbit for OBJID is 0 - if rmostbit == 0: - objidset = np.array([int(bintargid[-lmostbit:], 2) for bintargid in bintargids]) - else: - objidset = np.array([int(bintargid[-lmostbit:-rmostbit], 2) for bintargid in bintargids]) - objidshould = self.drbricks["nobjs"]+np.arange(len(objidset))+1 - self.assertTrue(np.all(objidset == objidshould)) - # ADM finally check that the BRICKIDs are all 330368 rmostbit = targetid_mask.BRICKID.bitnum lmostbit = targetid_mask.BRICKID.bitnum + targetid_mask.BRICKID.nbits From 6adb2ad0094ae5d698a3547170fc2dbd21ccddf4 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Mon, 15 Jun 2020 19:31:23 -0700 Subject: [PATCH 21/25] keep updating unit tests; catch some corner cases in mask creation --- py/desitarget/brightmask.py | 14 ++- py/desitarget/gfa.py | 11 +- py/desitarget/test/make_testgaia.py | 36 +++++- .../test/t4/healpix/healpix-04669.fits | 2 +- .../test/t4/healpix/healpix-04671.fits | 2 +- .../test/t4/healpix/healpix-04751.fits | 2 +- .../test/t4/healpix/healpix-04761.fits | 2 +- .../test/t4/healpix/healpix-12284.fits | 2 +- py/desitarget/test/test_brightmask.py | 104 ++++++++++-------- 9 files changed, 116 insertions(+), 59 deletions(-) diff --git a/py/desitarget/brightmask.py b/py/desitarget/brightmask.py index 82c6f1736..d43a28585 100644 --- a/py/desitarget/brightmask.py +++ b/py/desitarget/brightmask.py @@ -239,7 +239,9 @@ def make_bright_star_mask_in_hp(nside, pixnum, verbose=True, gaiaepoch=2015.5, gaiaobjs = [] cols = 'SOURCE_ID', 'RA', 'DEC', 'PHOT_G_MEAN_MAG', 'PMRA', 'PMDEC' for fn in gaiafns: - gaiaobjs.append(fitsio.read(fn, ext='GAIAHPX', columns=cols)) + if os.path.exists(fn): + gaiaobjs.append(fitsio.read(fn, ext='GAIAHPX', columns=cols)) + gaiaobjs = np.concatenate(gaiaobjs) gaiaobjs = rfn.rename_fields(gaiaobjs, {"SOURCE_ID": "REF_ID"}) # ADM limit Gaia objects to 3 magnitudes fainter than the passed @@ -255,6 +257,7 @@ def make_bright_star_mask_in_hp(nside, pixnum, verbose=True, gaiaepoch=2015.5, if verbose: log.info('Add URAT for {} Gaia objs with no PMs (pix={})...t={:.1f} mins' .format(np.sum(ii), pixnum, (time()-t0)/60)) + urat = add_urat_pms(gaiaobjs[ii], numproc=1) if verbose: log.info('Found an additional {} URAT objects (pix={})...t={:.1f} mins' @@ -285,6 +288,7 @@ def make_bright_star_mask_in_hp(nside, pixnum, verbose=True, gaiaepoch=2015.5, if verbose: log.info('{} matches. Refining at 1" (pix={})...t={:.1f} mins'.format( len(itycho), pixnum, (time()-t0)/60)) + # ADM match Gaia to Tycho at the more exact reference epoch. epoch_ra = tychoobjs[itycho]["EPOCH_RA"] epoch_dec = tychoobjs[itycho]["EPOCH_DEC"] @@ -294,8 +298,12 @@ def make_bright_star_mask_in_hp(nside, pixnum, verbose=True, gaiaepoch=2015.5, gaiaobjs["PMRA"][igaia], gaiaobjs["PMDEC"][igaia], epochnow=gaiaepoch, epochpast=epoch_ra, epochpastdec=epoch_dec) - _, refined = radec_match_to([ra, dec], [tychoobjs["RA"][itycho], - tychoobjs["DEC"][itycho]], radec=True) + # ADM catch the corner case where there are no initial matches. + if ra > 0: + _, refined = radec_match_to([ra, dec], [tychoobjs["RA"][itycho], + tychoobjs["DEC"][itycho]], radec=True) + else: + refined = np.array([], dtype='int') # ADM retain Tycho objects that DON'T match Gaia. keep = np.ones(len(tychoobjs), dtype='bool') keep[itycho[refined]] = False diff --git a/py/desitarget/gfa.py b/py/desitarget/gfa.py index f69bcf73c..2f49c630a 100644 --- a/py/desitarget/gfa.py +++ b/py/desitarget/gfa.py @@ -430,9 +430,14 @@ def _update_status(result): for splitobj in splitobjs: urats.append(_update_status(_get_urat_matches(splitobj))) - # ADM remember to grab the REFIDs as well as the URAT matches. - refids = np.concatenate(np.array(urats)[:, 1]) - urats = np.concatenate(np.array(urats)[:, 0]) + # ADM remember to grab the REFIDs as well as the URAT matches...and + # ADM to catch the corner case where objects occupy only one pixel. + if len(urats) == 1: + refids = urats[0][1] + urats = urats[0][0] + else: + refids = np.concatenate(np.array(urats)[:, 1]) + urats = np.concatenate(np.array(urats)[:, 0]) # ADM sort the output to match the input, on REF_ID. ii = np.zeros_like(refids) diff --git a/py/desitarget/test/make_testgaia.py b/py/desitarget/test/make_testgaia.py index d195599c3..f1e814f78 100644 --- a/py/desitarget/test/make_testgaia.py +++ b/py/desitarget/test/make_testgaia.py @@ -7,6 +7,8 @@ from time import time from pkg_resources import resource_filename from desitarget.gaiamatch import find_gaia_files +from desitarget.tychomatch import find_tycho_files +from desitarget.uratmatch import find_urat_files from desitarget import io start = time() @@ -17,21 +19,47 @@ tractorfiles = sorted(io.list_tractorfiles(datadir)) sweepfiles = sorted(io.list_sweepfiles(datadir)) -# ADM read in each of the relevant Gaia files. +# ADM read in relevant Gaia files. gaiafiles = [] for fn in sweepfiles + tractorfiles: objs = fitsio.read(fn, columns=["RA", "DEC"]) gaiafiles.append(find_gaia_files(objs, neighbors=False)) -gaiafiles = np.unique(gaiafiles) +gaiafiles = np.unique(np.concatenate(gaiafiles)) # ADM loop through the Gaia files and write out some rows # ADM to the "t4" unit test directory. +tychofiles, uratfiles = [], [] if not os.path.exists("t4"): os.makedirs(os.path.join("t4", "healpix")) for fn in gaiafiles: - objs = fitsio.read(fn) + objs, hdr = fitsio.read(fn, 1, header=True) outfile = os.path.join("t4", "healpix", os.path.basename(fn)) - fitsio.write(outfile, objs[:25], clobber=True) + fitsio.write(outfile, objs[:25], header=hdr, clobber=True, extname="GAIAHPX") + # ADM find some Tycho and URAT files that accompany the Gaia files. + tychofiles.append(find_tycho_files(objs[:25], neighbors=False)) + uratfiles.append(find_urat_files(objs[:25], neighbors=False)) print("writing {}".format(outfile)) +tychofiles = np.unique(np.concatenate(tychofiles)) +uratfiles = np.unique(np.concatenate(uratfiles)) + +# ADM loop through the Gaia files and write out accompanying Tycho +# ADM and URAT objects. +for direc, fns, ext in zip(["tycho", "urat"], + [tychofiles, uratfiles], + ["TYCHOHPX", "URATHPX"]): + outdir = os.path.join("t4", direc) + if not os.path.exists(outdir): + os.makedirs(os.path.join("t4", direc, "healpix")) + for fn in fns: + objs, hdr = fitsio.read(fn, 1, header=True) + outfile = os.path.join("t4", direc, "healpix", os.path.basename(fn)) + s = set(gaiafiles) + # ADM ensure a match with objects in the Gaia files. + ii = np.array( + [len(set(find_gaia_files(i, neighbors=False)).intersection(s)) > 0 + for i in objs]) + fitsio.write(outfile, objs[ii][:25], + clobber=True, header=hdr, extname=ext) + print("writing {}".format(outfile)) print('Done...t={:.2f}s'.format(time()-start)) diff --git a/py/desitarget/test/t4/healpix/healpix-04669.fits b/py/desitarget/test/t4/healpix/healpix-04669.fits index 36b86120a..1584f3544 100644 --- a/py/desitarget/test/t4/healpix/healpix-04669.fits +++ b/py/desitarget/test/t4/healpix/healpix-04669.fits @@ -1,4 +1,4 @@ -SIMPLE = T / file does conform to FITS standard BITPIX = 16 / number of bits per data pixel NAXIS = 0 / number of data axes EXTEND = T / FITS dataset may contain extensions COMMENT FITS (Flexible Image Transport System) format is defined in 'AstronomyCOMMENT and Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H END XTENSION= 'BINTABLE' / binary table extension BITPIX = 8 / 8-bit bytes NAXIS = 2 / 2-dimensional binary table NAXIS1 = 88 / width of table in bytes NAXIS2 = 25 / number of rows in table PCOUNT = 0 / size of special data area GCOUNT = 1 / one data group (required keyword) TFIELDS = 21 / number of fields in each row TTYPE1 = 'SOURCE_ID' / label for field 1 TFORM1 = 'K ' / data format of field: 8-byte INTEGER TTYPE2 = 'REF_CAT ' / label for field 2 TFORM2 = '2A ' / data format of field: ASCII Character TTYPE3 = 'RA ' / label for field 3 TFORM3 = 'D ' / data format of field: 8-byte DOUBLE TTYPE4 = 'DEC ' / label for field 4 TFORM4 = 'D ' / data format of field: 8-byte DOUBLE TTYPE5 = 'PHOT_G_MEAN_MAG' / label for field 5 TFORM5 = 'E ' / data format of field: 4-byte REAL TTYPE6 = 'PHOT_G_MEAN_FLUX_OVER_ERROR' / label for field 6 TFORM6 = 'E ' / data format of field: 4-byte REAL TTYPE7 = 'PHOT_BP_MEAN_MAG' / label for field 7 TFORM7 = 'E ' / data format of field: 4-byte REAL TTYPE8 = 'PHOT_BP_MEAN_FLUX_OVER_ERROR' / label for field 8 TFORM8 = 'E ' / data format of field: 4-byte REAL TTYPE9 = 'PHOT_RP_MEAN_MAG' / label for field 9 TFORM9 = 'E ' / data format of field: 4-byte REAL TTYPE10 = 'PHOT_RP_MEAN_FLUX_OVER_ERROR' / label for field 10 TFORM10 = 'E ' / data format of field: 4-byte REAL TTYPE11 = 'PHOT_BP_RP_EXCESS_FACTOR' / label for field 11 TFORM11 = 'E ' / data format of field: 4-byte REAL TTYPE12 = 'ASTROMETRIC_EXCESS_NOISE' / label for field 12 TFORM12 = 'E ' / data format of field: 4-byte REAL TTYPE13 = 'DUPLICATED_SOURCE' / label for field 13 TFORM13 = 'L ' / data format of field: 1-byte LOGICAL TTYPE14 = 'ASTROMETRIC_SIGMA5D_MAX' / label for field 14 TFORM14 = 'E ' / data format of field: 4-byte REAL TTYPE15 = 'ASTROMETRIC_PARAMS_SOLVED' / label for field 15 TFORM15 = 'B ' / data format of field TZERO15 = -128 / offset for signed bytes TSCAL15 = 1 / data are not scaled TTYPE16 = 'PARALLAX' / label for field 16 TFORM16 = 'E ' / data format of field: 4-byte REAL TTYPE17 = 'PARALLAX_ERROR' / label for field 17 TFORM17 = 'E ' / data format of field: 4-byte REAL TTYPE18 = 'PMRA ' / label for field 18 TFORM18 = 'E ' / data format of field: 4-byte REAL TTYPE19 = 'PMRA_ERROR' / label for field 19 TFORM19 = 'E ' / data format of field: 4-byte REAL TTYPE20 = 'PMDEC ' / label for field 20 TFORM20 = 'E ' / data format of field: 4-byte REAL TTYPE21 = 'PMDEC_ERROR' / label for field 21 TFORM21 = 'E ' / data format of field: 4-byte REAL END $~G2@u$:gAWC#'A[E@^APlBI?6>F?끉?? +SIMPLE = T / file does conform to FITS standard BITPIX = 16 / number of bits per data pixel NAXIS = 0 / number of data axes EXTEND = T / FITS dataset may contain extensions COMMENT FITS (Flexible Image Transport System) format is defined in 'AstronomyCOMMENT and Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H END XTENSION= 'BINTABLE' / binary table extension BITPIX = 8 / 8-bit bytes NAXIS = 2 / 2-dimensional binary table NAXIS1 = 88 / width of table in bytes NAXIS2 = 25 / number of rows in table PCOUNT = 0 / size of special data area GCOUNT = 1 / one data group (required keyword) TFIELDS = 21 / number of fields in each row TTYPE1 = 'SOURCE_ID' / label for field 1 TFORM1 = 'K ' / data format of field: 8-byte INTEGER TTYPE2 = 'REF_CAT ' / label for field 2 TFORM2 = '2A ' / data format of field: ASCII Character TTYPE3 = 'RA ' / label for field 3 TFORM3 = 'D ' / data format of field: 8-byte DOUBLE TTYPE4 = 'DEC ' / label for field 4 TFORM4 = 'D ' / data format of field: 8-byte DOUBLE TTYPE5 = 'PHOT_G_MEAN_MAG' / label for field 5 TFORM5 = 'E ' / data format of field: 4-byte REAL TTYPE6 = 'PHOT_G_MEAN_FLUX_OVER_ERROR' / label for field 6 TFORM6 = 'E ' / data format of field: 4-byte REAL TTYPE7 = 'PHOT_BP_MEAN_MAG' / label for field 7 TFORM7 = 'E ' / data format of field: 4-byte REAL TTYPE8 = 'PHOT_BP_MEAN_FLUX_OVER_ERROR' / label for field 8 TFORM8 = 'E ' / data format of field: 4-byte REAL TTYPE9 = 'PHOT_RP_MEAN_MAG' / label for field 9 TFORM9 = 'E ' / data format of field: 4-byte REAL TTYPE10 = 'PHOT_RP_MEAN_FLUX_OVER_ERROR' / label for field 10 TFORM10 = 'E ' / data format of field: 4-byte REAL TTYPE11 = 'PHOT_BP_RP_EXCESS_FACTOR' / label for field 11 TFORM11 = 'E ' / data format of field: 4-byte REAL TTYPE12 = 'ASTROMETRIC_EXCESS_NOISE' / label for field 12 TFORM12 = 'E ' / data format of field: 4-byte REAL TTYPE13 = 'DUPLICATED_SOURCE' / label for field 13 TFORM13 = 'L ' / data format of field: 1-byte LOGICAL TTYPE14 = 'ASTROMETRIC_SIGMA5D_MAX' / label for field 14 TFORM14 = 'E ' / data format of field: 4-byte REAL TTYPE15 = 'ASTROMETRIC_PARAMS_SOLVED' / label for field 15 TFORM15 = 'B ' / data format of field TZERO15 = -128 / offset for signed bytes TSCAL15 = 1 / data are not scaled TTYPE16 = 'PARALLAX' / label for field 16 TFORM16 = 'E ' / data format of field: 4-byte REAL TTYPE17 = 'PARALLAX_ERROR' / label for field 17 TFORM17 = 'E ' / data format of field: 4-byte REAL TTYPE18 = 'PMRA ' / label for field 18 TFORM18 = 'E ' / data format of field: 4-byte REAL TTYPE19 = 'PMRA_ERROR' / label for field 19 TFORM19 = 'E ' / data format of field: 4-byte REAL TTYPE20 = 'PMDEC ' / label for field 20 TFORM20 = 'E ' / data format of field: 4-byte REAL TTYPE21 = 'PMDEC_ERROR' / label for field 21 TFORM21 = 'E ' / data format of field: 4-byte REAL EXTNAME = 'GAIAHPX ' / name of this binary table extension HPXNSIDE= 32 HPXNEST = T END $~G2@u$:gAWC#'A[E@^APlBI?6>F?끉?? )?ƚ?~?-$zG2@u+"BS h,A[Cs A@4A#BG?>?mF? ؟?oE>{j@[հ? )D?$z@ G2@u*NǕ (fAC6AA (kAAq?SF?9fOg>V О?<H2??kJ$z $G2@u3aj diff --git a/py/desitarget/test/t4/healpix/healpix-04671.fits b/py/desitarget/test/t4/healpix/healpix-04671.fits index f372f0f1e..bca298b83 100644 --- a/py/desitarget/test/t4/healpix/healpix-04671.fits +++ b/py/desitarget/test/t4/healpix/healpix-04671.fits @@ -1,4 +1,4 @@ -SIMPLE = T / file does conform to FITS standard BITPIX = 16 / number of bits per data pixel NAXIS = 0 / number of data axes EXTEND = T / FITS dataset may contain extensions COMMENT FITS (Flexible Image Transport System) format is defined in 'AstronomyCOMMENT and Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H END XTENSION= 'BINTABLE' / binary table extension BITPIX = 8 / 8-bit bytes NAXIS = 2 / 2-dimensional binary table NAXIS1 = 88 / width of table in bytes NAXIS2 = 25 / number of rows in table PCOUNT = 0 / size of special data area GCOUNT = 1 / one data group (required keyword) TFIELDS = 21 / number of fields in each row TTYPE1 = 'SOURCE_ID' / label for field 1 TFORM1 = 'K ' / data format of field: 8-byte INTEGER TTYPE2 = 'REF_CAT ' / label for field 2 TFORM2 = '2A ' / data format of field: ASCII Character TTYPE3 = 'RA ' / label for field 3 TFORM3 = 'D ' / data format of field: 8-byte DOUBLE TTYPE4 = 'DEC ' / label for field 4 TFORM4 = 'D ' / data format of field: 8-byte DOUBLE TTYPE5 = 'PHOT_G_MEAN_MAG' / label for field 5 TFORM5 = 'E ' / data format of field: 4-byte REAL TTYPE6 = 'PHOT_G_MEAN_FLUX_OVER_ERROR' / label for field 6 TFORM6 = 'E ' / data format of field: 4-byte REAL TTYPE7 = 'PHOT_BP_MEAN_MAG' / label for field 7 TFORM7 = 'E ' / data format of field: 4-byte REAL TTYPE8 = 'PHOT_BP_MEAN_FLUX_OVER_ERROR' / label for field 8 TFORM8 = 'E ' / data format of field: 4-byte REAL TTYPE9 = 'PHOT_RP_MEAN_MAG' / label for field 9 TFORM9 = 'E ' / data format of field: 4-byte REAL TTYPE10 = 'PHOT_RP_MEAN_FLUX_OVER_ERROR' / label for field 10 TFORM10 = 'E ' / data format of field: 4-byte REAL TTYPE11 = 'PHOT_BP_RP_EXCESS_FACTOR' / label for field 11 TFORM11 = 'E ' / data format of field: 4-byte REAL TTYPE12 = 'ASTROMETRIC_EXCESS_NOISE' / label for field 12 TFORM12 = 'E ' / data format of field: 4-byte REAL TTYPE13 = 'DUPLICATED_SOURCE' / label for field 13 TFORM13 = 'L ' / data format of field: 1-byte LOGICAL TTYPE14 = 'ASTROMETRIC_SIGMA5D_MAX' / label for field 14 TFORM14 = 'E ' / data format of field: 4-byte REAL TTYPE15 = 'ASTROMETRIC_PARAMS_SOLVED' / label for field 15 TFORM15 = 'B ' / data format of field TZERO15 = -128 / offset for signed bytes TSCAL15 = 1 / data are not scaled TTYPE16 = 'PARALLAX' / label for field 16 TFORM16 = 'E ' / data format of field: 4-byte REAL TTYPE17 = 'PARALLAX_ERROR' / label for field 17 TFORM17 = 'E ' / data format of field: 4-byte REAL TTYPE18 = 'PMRA ' / label for field 18 TFORM18 = 'E ' / data format of field: 4-byte REAL TTYPE19 = 'PMRA_ERROR' / label for field 19 TFORM19 = 'E ' / data format of field: 4-byte REAL TTYPE20 = 'PMDEC ' / label for field 20 TFORM20 = 'E ' / data format of field: 4-byte REAL TTYPE21 = 'PMDEC_ERROR' / label for field 21 TFORM21 = 'E ' / data format of field: 4-byte REAL END $~G2@u$4/?KAo!B>pA,@ A@AyT??3F@@Qn?dU?O ^?=$N +SIMPLE = T / file does conform to FITS standard BITPIX = 16 / number of bits per data pixel NAXIS = 0 / number of data axes EXTEND = T / FITS dataset may contain extensions COMMENT FITS (Flexible Image Transport System) format is defined in 'AstronomyCOMMENT and Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H END XTENSION= 'BINTABLE' / binary table extension BITPIX = 8 / 8-bit bytes NAXIS = 2 / 2-dimensional binary table NAXIS1 = 88 / width of table in bytes NAXIS2 = 25 / number of rows in table PCOUNT = 0 / size of special data area GCOUNT = 1 / one data group (required keyword) TFIELDS = 21 / number of fields in each row TTYPE1 = 'SOURCE_ID' / label for field 1 TFORM1 = 'K ' / data format of field: 8-byte INTEGER TTYPE2 = 'REF_CAT ' / label for field 2 TFORM2 = '2A ' / data format of field: ASCII Character TTYPE3 = 'RA ' / label for field 3 TFORM3 = 'D ' / data format of field: 8-byte DOUBLE TTYPE4 = 'DEC ' / label for field 4 TFORM4 = 'D ' / data format of field: 8-byte DOUBLE TTYPE5 = 'PHOT_G_MEAN_MAG' / label for field 5 TFORM5 = 'E ' / data format of field: 4-byte REAL TTYPE6 = 'PHOT_G_MEAN_FLUX_OVER_ERROR' / label for field 6 TFORM6 = 'E ' / data format of field: 4-byte REAL TTYPE7 = 'PHOT_BP_MEAN_MAG' / label for field 7 TFORM7 = 'E ' / data format of field: 4-byte REAL TTYPE8 = 'PHOT_BP_MEAN_FLUX_OVER_ERROR' / label for field 8 TFORM8 = 'E ' / data format of field: 4-byte REAL TTYPE9 = 'PHOT_RP_MEAN_MAG' / label for field 9 TFORM9 = 'E ' / data format of field: 4-byte REAL TTYPE10 = 'PHOT_RP_MEAN_FLUX_OVER_ERROR' / label for field 10 TFORM10 = 'E ' / data format of field: 4-byte REAL TTYPE11 = 'PHOT_BP_RP_EXCESS_FACTOR' / label for field 11 TFORM11 = 'E ' / data format of field: 4-byte REAL TTYPE12 = 'ASTROMETRIC_EXCESS_NOISE' / label for field 12 TFORM12 = 'E ' / data format of field: 4-byte REAL TTYPE13 = 'DUPLICATED_SOURCE' / label for field 13 TFORM13 = 'L ' / data format of field: 1-byte LOGICAL TTYPE14 = 'ASTROMETRIC_SIGMA5D_MAX' / label for field 14 TFORM14 = 'E ' / data format of field: 4-byte REAL TTYPE15 = 'ASTROMETRIC_PARAMS_SOLVED' / label for field 15 TFORM15 = 'B ' / data format of field TZERO15 = -128 / offset for signed bytes TSCAL15 = 1 / data are not scaled TTYPE16 = 'PARALLAX' / label for field 16 TFORM16 = 'E ' / data format of field: 4-byte REAL TTYPE17 = 'PARALLAX_ERROR' / label for field 17 TFORM17 = 'E ' / data format of field: 4-byte REAL TTYPE18 = 'PMRA ' / label for field 18 TFORM18 = 'E ' / data format of field: 4-byte REAL TTYPE19 = 'PMRA_ERROR' / label for field 19 TFORM19 = 'E ' / data format of field: 4-byte REAL TTYPE20 = 'PMDEC ' / label for field 20 TFORM20 = 'E ' / data format of field: 4-byte REAL TTYPE21 = 'PMDEC_ERROR' / label for field 21 TFORM21 = 'E ' / data format of field: 4-byte REAL EXTNAME = 'GAIAHPX ' / name of this binary table extension HPXNSIDE= 32 HPXNEST = T END $~G2@u$4/?KAo!B>pA,@ A@AyT??3F@@Qn?dU?O ^?=$N G2@u (8G-ATCˋAOAA"B?J>՗F?8侟>=?FC? C?$(dG2@u :ȒAnDTIKAB4A=C ?F>H]?>A> >$G2@u -tYzM@TA=CnAAZAiAv?F>݄ob|ޗ>b>">$~X|G2@ul'˰~AC] AoAAAl? F?JW&?h ?gW?P$e diff --git a/py/desitarget/test/t4/healpix/healpix-04751.fits b/py/desitarget/test/t4/healpix/healpix-04751.fits index 342c3ba79..79ae15e36 100644 --- a/py/desitarget/test/t4/healpix/healpix-04751.fits +++ b/py/desitarget/test/t4/healpix/healpix-04751.fits @@ -1,4 +1,4 @@ -SIMPLE = T / file does conform to FITS standard BITPIX = 16 / number of bits per data pixel NAXIS = 0 / number of data axes EXTEND = T / FITS dataset may contain extensions COMMENT FITS (Flexible Image Transport System) format is defined in 'AstronomyCOMMENT and Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H END XTENSION= 'BINTABLE' / binary table extension BITPIX = 8 / 8-bit bytes NAXIS = 2 / 2-dimensional binary table NAXIS1 = 88 / width of table in bytes NAXIS2 = 25 / number of rows in table PCOUNT = 0 / size of special data area GCOUNT = 1 / one data group (required keyword) TFIELDS = 21 / number of fields in each row TTYPE1 = 'SOURCE_ID' / label for field 1 TFORM1 = 'K ' / data format of field: 8-byte INTEGER TTYPE2 = 'REF_CAT ' / label for field 2 TFORM2 = '2A ' / data format of field: ASCII Character TTYPE3 = 'RA ' / label for field 3 TFORM3 = 'D ' / data format of field: 8-byte DOUBLE TTYPE4 = 'DEC ' / label for field 4 TFORM4 = 'D ' / data format of field: 8-byte DOUBLE TTYPE5 = 'PHOT_G_MEAN_MAG' / label for field 5 TFORM5 = 'E ' / data format of field: 4-byte REAL TTYPE6 = 'PHOT_G_MEAN_FLUX_OVER_ERROR' / label for field 6 TFORM6 = 'E ' / data format of field: 4-byte REAL TTYPE7 = 'PHOT_BP_MEAN_MAG' / label for field 7 TFORM7 = 'E ' / data format of field: 4-byte REAL TTYPE8 = 'PHOT_BP_MEAN_FLUX_OVER_ERROR' / label for field 8 TFORM8 = 'E ' / data format of field: 4-byte REAL TTYPE9 = 'PHOT_RP_MEAN_MAG' / label for field 9 TFORM9 = 'E ' / data format of field: 4-byte REAL TTYPE10 = 'PHOT_RP_MEAN_FLUX_OVER_ERROR' / label for field 10 TFORM10 = 'E ' / data format of field: 4-byte REAL TTYPE11 = 'PHOT_BP_RP_EXCESS_FACTOR' / label for field 11 TFORM11 = 'E ' / data format of field: 4-byte REAL TTYPE12 = 'ASTROMETRIC_EXCESS_NOISE' / label for field 12 TFORM12 = 'E ' / data format of field: 4-byte REAL TTYPE13 = 'DUPLICATED_SOURCE' / label for field 13 TFORM13 = 'L ' / data format of field: 1-byte LOGICAL TTYPE14 = 'ASTROMETRIC_SIGMA5D_MAX' / label for field 14 TFORM14 = 'E ' / data format of field: 4-byte REAL TTYPE15 = 'ASTROMETRIC_PARAMS_SOLVED' / label for field 15 TFORM15 = 'B ' / data format of field TZERO15 = -128 / offset for signed bytes TSCAL15 = 1 / data are not scaled TTYPE16 = 'PARALLAX' / label for field 16 TFORM16 = 'E ' / data format of field: 4-byte REAL TTYPE17 = 'PARALLAX_ERROR' / label for field 17 TFORM17 = 'E ' / data format of field: 4-byte REAL TTYPE18 = 'PMRA ' / label for field 18 TFORM18 = 'E ' / data format of field: 4-byte REAL TTYPE19 = 'PMRA_ERROR' / label for field 19 TFORM19 = 'E ' / data format of field: 4-byte REAL TTYPE20 = 'PMDEC ' / label for field 20 TFORM20 = 'E ' / data format of field: 4-byte REAL TTYPE21 = 'PMDEC_ERROR' / label for field 21 TFORM21 = 'E ' / data format of field: 4-byte REAL END %,G2@tlT[|A BRA`A >AA?1^FAH[%G2@tau+] 3AC} +SIMPLE = T / file does conform to FITS standard BITPIX = 16 / number of bits per data pixel NAXIS = 0 / number of data axes EXTEND = T / FITS dataset may contain extensions COMMENT FITS (Flexible Image Transport System) format is defined in 'AstronomyCOMMENT and Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H END XTENSION= 'BINTABLE' / binary table extension BITPIX = 8 / 8-bit bytes NAXIS = 2 / 2-dimensional binary table NAXIS1 = 88 / width of table in bytes NAXIS2 = 25 / number of rows in table PCOUNT = 0 / size of special data area GCOUNT = 1 / one data group (required keyword) TFIELDS = 21 / number of fields in each row TTYPE1 = 'SOURCE_ID' / label for field 1 TFORM1 = 'K ' / data format of field: 8-byte INTEGER TTYPE2 = 'REF_CAT ' / label for field 2 TFORM2 = '2A ' / data format of field: ASCII Character TTYPE3 = 'RA ' / label for field 3 TFORM3 = 'D ' / data format of field: 8-byte DOUBLE TTYPE4 = 'DEC ' / label for field 4 TFORM4 = 'D ' / data format of field: 8-byte DOUBLE TTYPE5 = 'PHOT_G_MEAN_MAG' / label for field 5 TFORM5 = 'E ' / data format of field: 4-byte REAL TTYPE6 = 'PHOT_G_MEAN_FLUX_OVER_ERROR' / label for field 6 TFORM6 = 'E ' / data format of field: 4-byte REAL TTYPE7 = 'PHOT_BP_MEAN_MAG' / label for field 7 TFORM7 = 'E ' / data format of field: 4-byte REAL TTYPE8 = 'PHOT_BP_MEAN_FLUX_OVER_ERROR' / label for field 8 TFORM8 = 'E ' / data format of field: 4-byte REAL TTYPE9 = 'PHOT_RP_MEAN_MAG' / label for field 9 TFORM9 = 'E ' / data format of field: 4-byte REAL TTYPE10 = 'PHOT_RP_MEAN_FLUX_OVER_ERROR' / label for field 10 TFORM10 = 'E ' / data format of field: 4-byte REAL TTYPE11 = 'PHOT_BP_RP_EXCESS_FACTOR' / label for field 11 TFORM11 = 'E ' / data format of field: 4-byte REAL TTYPE12 = 'ASTROMETRIC_EXCESS_NOISE' / label for field 12 TFORM12 = 'E ' / data format of field: 4-byte REAL TTYPE13 = 'DUPLICATED_SOURCE' / label for field 13 TFORM13 = 'L ' / data format of field: 1-byte LOGICAL TTYPE14 = 'ASTROMETRIC_SIGMA5D_MAX' / label for field 14 TFORM14 = 'E ' / data format of field: 4-byte REAL TTYPE15 = 'ASTROMETRIC_PARAMS_SOLVED' / label for field 15 TFORM15 = 'B ' / data format of field TZERO15 = -128 / offset for signed bytes TSCAL15 = 1 / data are not scaled TTYPE16 = 'PARALLAX' / label for field 16 TFORM16 = 'E ' / data format of field: 4-byte REAL TTYPE17 = 'PARALLAX_ERROR' / label for field 17 TFORM17 = 'E ' / data format of field: 4-byte REAL TTYPE18 = 'PMRA ' / label for field 18 TFORM18 = 'E ' / data format of field: 4-byte REAL TTYPE19 = 'PMRA_ERROR' / label for field 19 TFORM19 = 'E ' / data format of field: 4-byte REAL TTYPE20 = 'PMDEC ' / label for field 20 TFORM20 = 'E ' / data format of field: 4-byte REAL TTYPE21 = 'PMDEC_ERROR' / label for field 21 TFORM21 = 'E ' / data format of field: 4-byte REAL EXTNAME = 'GAIAHPX ' / name of this binary table extension HPXNSIDE= 32 HPXNEST = T END %,G2@tlT[|A BRA`A >AA?1^FAH[%G2@tau+] 3AC} "A;AHA8An`?=>ZNF?J[?xl>{[?(g?9A%=G2@th4&SigA{D_AK[BkrAC =?f>< F>!?[>)7AV>n>T% (G2@tenaB~ADJSwANBWAB?Q>F>W> >/>Je4of>[%A G2@t^WC9-}]A6[ChA A&rAB?9F>?\!> J?>c>%1XG2@teZvt 6pABkA;r@Ur]As@?AhF@=% G2@tfӕ7 EAuDgA0hB¨AhC6:9?F>J>>=8տ(> >%L%hG2@tb͢o%OAC+`AmHAGY^AuMB`5?>~F?B@`;\?\A~>"̓?g%5( diff --git a/py/desitarget/test/t4/healpix/healpix-04761.fits b/py/desitarget/test/t4/healpix/healpix-04761.fits index d81227766..3c7718ca8 100644 --- a/py/desitarget/test/t4/healpix/healpix-04761.fits +++ b/py/desitarget/test/t4/healpix/healpix-04761.fits @@ -1,4 +1,4 @@ -SIMPLE = T / file does conform to FITS standard BITPIX = 16 / number of bits per data pixel NAXIS = 0 / number of data axes EXTEND = T / FITS dataset may contain extensions COMMENT FITS (Flexible Image Transport System) format is defined in 'AstronomyCOMMENT and Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H END XTENSION= 'BINTABLE' / binary table extension BITPIX = 8 / 8-bit bytes NAXIS = 2 / 2-dimensional binary table NAXIS1 = 88 / width of table in bytes NAXIS2 = 25 / number of rows in table PCOUNT = 0 / size of special data area GCOUNT = 1 / one data group (required keyword) TFIELDS = 21 / number of fields in each row TTYPE1 = 'SOURCE_ID' / label for field 1 TFORM1 = 'K ' / data format of field: 8-byte INTEGER TTYPE2 = 'REF_CAT ' / label for field 2 TFORM2 = '2A ' / data format of field: ASCII Character TTYPE3 = 'RA ' / label for field 3 TFORM3 = 'D ' / data format of field: 8-byte DOUBLE TTYPE4 = 'DEC ' / label for field 4 TFORM4 = 'D ' / data format of field: 8-byte DOUBLE TTYPE5 = 'PHOT_G_MEAN_MAG' / label for field 5 TFORM5 = 'E ' / data format of field: 4-byte REAL TTYPE6 = 'PHOT_G_MEAN_FLUX_OVER_ERROR' / label for field 6 TFORM6 = 'E ' / data format of field: 4-byte REAL TTYPE7 = 'PHOT_BP_MEAN_MAG' / label for field 7 TFORM7 = 'E ' / data format of field: 4-byte REAL TTYPE8 = 'PHOT_BP_MEAN_FLUX_OVER_ERROR' / label for field 8 TFORM8 = 'E ' / data format of field: 4-byte REAL TTYPE9 = 'PHOT_RP_MEAN_MAG' / label for field 9 TFORM9 = 'E ' / data format of field: 4-byte REAL TTYPE10 = 'PHOT_RP_MEAN_FLUX_OVER_ERROR' / label for field 10 TFORM10 = 'E ' / data format of field: 4-byte REAL TTYPE11 = 'PHOT_BP_RP_EXCESS_FACTOR' / label for field 11 TFORM11 = 'E ' / data format of field: 4-byte REAL TTYPE12 = 'ASTROMETRIC_EXCESS_NOISE' / label for field 12 TFORM12 = 'E ' / data format of field: 4-byte REAL TTYPE13 = 'DUPLICATED_SOURCE' / label for field 13 TFORM13 = 'L ' / data format of field: 1-byte LOGICAL TTYPE14 = 'ASTROMETRIC_SIGMA5D_MAX' / label for field 14 TFORM14 = 'E ' / data format of field: 4-byte REAL TTYPE15 = 'ASTROMETRIC_PARAMS_SOLVED' / label for field 15 TFORM15 = 'B ' / data format of field TZERO15 = -128 / offset for signed bytes TSCAL15 = 1 / data are not scaled TTYPE16 = 'PARALLAX' / label for field 16 TFORM16 = 'E ' / data format of field: 4-byte REAL TTYPE17 = 'PARALLAX_ERROR' / label for field 17 TFORM17 = 'E ' / data format of field: 4-byte REAL TTYPE18 = 'PMRA ' / label for field 18 TFORM18 = 'E ' / data format of field: 4-byte REAL TTYPE19 = 'PMRA_ERROR' / label for field 19 TFORM19 = 'E ' / data format of field: 4-byte REAL TTYPE20 = 'PMDEC ' / label for field 20 TFORM20 = 'E ' / data format of field: 4-byte REAL TTYPE21 = 'PMDEC_ERROR' / label for field 21 TFORM21 = 'E ' / data format of field: 4-byte REAL END %'PG2@t};ؿ4Al=B!A@=A6AF,?5XFBnR*%8OG2@t L1?\wA]BoL AףA A@A7?x@7FD6%0{G2@tj3&UkEbABA%@AzA9=?gF@%3XXG2@t0~z?P™ABeAƽ@EAgkA*9?FA(%2PFG2@t:9m}?2AIB.FA %3z G2@tr?J]^ACAZBNAB2?F?{'?> 0l?X0?7-c%3G2@tL"l AYCݖAcAI~A1Bs;?ֹ>5F?MWc? n@N?@x?'%38G2@tu1?|YeA4KB,FAoǃ%3/G2@t[眫?@[A C(TAaA A3BQ#?%F?N%?E>3|@:>>?n>*%3πG2@tC?{`}A#C A\@paAxAr}??`F@5΃%2@؀G2@tjyf?iya,ASBwA>@tAcA,]?F@%3XG2@tvs?S ADmA]BAA€C?W=.F>b>)'_>]%A,>1h,Y>P89%3*KG2@t{i׬OgA}B#Ae@4AFAs?d:F@{?]ZW??=@%3 G2@t44c^حkDADaD7AǜB{AC9?T>9F>?<>^,U@y>@2>%38 G2@t' +SIMPLE = T / file does conform to FITS standard BITPIX = 16 / number of bits per data pixel NAXIS = 0 / number of data axes EXTEND = T / FITS dataset may contain extensions COMMENT FITS (Flexible Image Transport System) format is defined in 'AstronomyCOMMENT and Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H END XTENSION= 'BINTABLE' / binary table extension BITPIX = 8 / 8-bit bytes NAXIS = 2 / 2-dimensional binary table NAXIS1 = 88 / width of table in bytes NAXIS2 = 25 / number of rows in table PCOUNT = 0 / size of special data area GCOUNT = 1 / one data group (required keyword) TFIELDS = 21 / number of fields in each row TTYPE1 = 'SOURCE_ID' / label for field 1 TFORM1 = 'K ' / data format of field: 8-byte INTEGER TTYPE2 = 'REF_CAT ' / label for field 2 TFORM2 = '2A ' / data format of field: ASCII Character TTYPE3 = 'RA ' / label for field 3 TFORM3 = 'D ' / data format of field: 8-byte DOUBLE TTYPE4 = 'DEC ' / label for field 4 TFORM4 = 'D ' / data format of field: 8-byte DOUBLE TTYPE5 = 'PHOT_G_MEAN_MAG' / label for field 5 TFORM5 = 'E ' / data format of field: 4-byte REAL TTYPE6 = 'PHOT_G_MEAN_FLUX_OVER_ERROR' / label for field 6 TFORM6 = 'E ' / data format of field: 4-byte REAL TTYPE7 = 'PHOT_BP_MEAN_MAG' / label for field 7 TFORM7 = 'E ' / data format of field: 4-byte REAL TTYPE8 = 'PHOT_BP_MEAN_FLUX_OVER_ERROR' / label for field 8 TFORM8 = 'E ' / data format of field: 4-byte REAL TTYPE9 = 'PHOT_RP_MEAN_MAG' / label for field 9 TFORM9 = 'E ' / data format of field: 4-byte REAL TTYPE10 = 'PHOT_RP_MEAN_FLUX_OVER_ERROR' / label for field 10 TFORM10 = 'E ' / data format of field: 4-byte REAL TTYPE11 = 'PHOT_BP_RP_EXCESS_FACTOR' / label for field 11 TFORM11 = 'E ' / data format of field: 4-byte REAL TTYPE12 = 'ASTROMETRIC_EXCESS_NOISE' / label for field 12 TFORM12 = 'E ' / data format of field: 4-byte REAL TTYPE13 = 'DUPLICATED_SOURCE' / label for field 13 TFORM13 = 'L ' / data format of field: 1-byte LOGICAL TTYPE14 = 'ASTROMETRIC_SIGMA5D_MAX' / label for field 14 TFORM14 = 'E ' / data format of field: 4-byte REAL TTYPE15 = 'ASTROMETRIC_PARAMS_SOLVED' / label for field 15 TFORM15 = 'B ' / data format of field TZERO15 = -128 / offset for signed bytes TSCAL15 = 1 / data are not scaled TTYPE16 = 'PARALLAX' / label for field 16 TFORM16 = 'E ' / data format of field: 4-byte REAL TTYPE17 = 'PARALLAX_ERROR' / label for field 17 TFORM17 = 'E ' / data format of field: 4-byte REAL TTYPE18 = 'PMRA ' / label for field 18 TFORM18 = 'E ' / data format of field: 4-byte REAL TTYPE19 = 'PMRA_ERROR' / label for field 19 TFORM19 = 'E ' / data format of field: 4-byte REAL TTYPE20 = 'PMDEC ' / label for field 20 TFORM20 = 'E ' / data format of field: 4-byte REAL TTYPE21 = 'PMDEC_ERROR' / label for field 21 TFORM21 = 'E ' / data format of field: 4-byte REAL EXTNAME = 'GAIAHPX ' / name of this binary table extension HPXNSIDE= 32 HPXNEST = T END %'PG2@t};ؿ4Al=B!A@=A6AF,?5XFBnR*%8OG2@t L1?\wA]BoL AףA A@A7?x@7FD6%0{G2@tj3&UkEbABA%@AzA9=?gF@%3XXG2@t0~z?P™ABeAƽ@EAgkA*9?FA(%2PFG2@t:9m}?2AIB.FA %3z G2@tr?J]^ACAZBNAB2?F?{'?> 0l?X0?7-c%3G2@tL"l AYCݖAcAI~A1Bs;?ֹ>5F?MWc? n@N?@x?'%38G2@tu1?|YeA4KB,FAoǃ%3/G2@t[眫?@[A C(TAaA A3BQ#?%F?N%?E>3|@:>>?n>*%3πG2@tC?{`}A#C A\@paAxAr}??`F@5΃%2@؀G2@tjyf?iya,ASBwA>@tAcA,]?F@%3XG2@tvs?S ADmA]BAA€C?W=.F>b>)'_>]%A,>1h,Y>P89%3*KG2@t{i׬OgA}B#Ae@4AFAs?d:F@{?]ZW??=@%3 G2@t44c^حkDADaD7AǜB{AC9?T>9F>?<>^,U@y>@2>%38 G2@t' ?ꂂYAgC5ALBKAB?F> @>Yx> ؆>%3`xG2@tB 1?ѕ^yHAoCA~A5AAi?aF@Hѩ%3~ }G2@tA^Hl?MUvtAFB9Aa@AvA!?Q`?T;F@c%3 G2@tܼ,6HAd E.AhC3A\C6?F=tӟ?B==5=O )=I6W%3$G2@t.?ƖIqAC#FAv@&A+Br??F?I*@%w7?kA;6?ccl?%3o'G2@tdž?9*ABx?AAAA?PFE4:%3(3G2@t̼|,-ĞA7CA4A1A0B{=A? ?bF?%3HXG2@tnǿsrABfRAi@BAVAbB??F@$%3 ]G2@t}C?CArFBA)Y@jA A?ߒF@*%2G2@t?TtpAC AAHi4A:A4 ?#F??!Ρ?O J??@d?%3p0 diff --git a/py/desitarget/test/t4/healpix/healpix-12284.fits b/py/desitarget/test/t4/healpix/healpix-12284.fits index e548334bc..b8540fadf 100644 --- a/py/desitarget/test/t4/healpix/healpix-12284.fits +++ b/py/desitarget/test/t4/healpix/healpix-12284.fits @@ -1,4 +1,4 @@ -SIMPLE = T / file does conform to FITS standard BITPIX = 16 / number of bits per data pixel NAXIS = 0 / number of data axes EXTEND = T / FITS dataset may contain extensions COMMENT FITS (Flexible Image Transport System) format is defined in 'AstronomyCOMMENT and Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H END XTENSION= 'BINTABLE' / binary table extension BITPIX = 8 / 8-bit bytes NAXIS = 2 / 2-dimensional binary table NAXIS1 = 88 / width of table in bytes NAXIS2 = 25 / number of rows in table PCOUNT = 0 / size of special data area GCOUNT = 1 / one data group (required keyword) TFIELDS = 21 / number of fields in each row TTYPE1 = 'SOURCE_ID' / label for field 1 TFORM1 = 'K ' / data format of field: 8-byte INTEGER TTYPE2 = 'REF_CAT ' / label for field 2 TFORM2 = '2A ' / data format of field: ASCII Character TTYPE3 = 'RA ' / label for field 3 TFORM3 = 'D ' / data format of field: 8-byte DOUBLE TTYPE4 = 'DEC ' / label for field 4 TFORM4 = 'D ' / data format of field: 8-byte DOUBLE TTYPE5 = 'PHOT_G_MEAN_MAG' / label for field 5 TFORM5 = 'E ' / data format of field: 4-byte REAL TTYPE6 = 'PHOT_G_MEAN_FLUX_OVER_ERROR' / label for field 6 TFORM6 = 'E ' / data format of field: 4-byte REAL TTYPE7 = 'PHOT_BP_MEAN_MAG' / label for field 7 TFORM7 = 'E ' / data format of field: 4-byte REAL TTYPE8 = 'PHOT_BP_MEAN_FLUX_OVER_ERROR' / label for field 8 TFORM8 = 'E ' / data format of field: 4-byte REAL TTYPE9 = 'PHOT_RP_MEAN_MAG' / label for field 9 TFORM9 = 'E ' / data format of field: 4-byte REAL TTYPE10 = 'PHOT_RP_MEAN_FLUX_OVER_ERROR' / label for field 10 TFORM10 = 'E ' / data format of field: 4-byte REAL TTYPE11 = 'PHOT_BP_RP_EXCESS_FACTOR' / label for field 11 TFORM11 = 'E ' / data format of field: 4-byte REAL TTYPE12 = 'ASTROMETRIC_EXCESS_NOISE' / label for field 12 TFORM12 = 'E ' / data format of field: 4-byte REAL TTYPE13 = 'DUPLICATED_SOURCE' / label for field 13 TFORM13 = 'L ' / data format of field: 1-byte LOGICAL TTYPE14 = 'ASTROMETRIC_SIGMA5D_MAX' / label for field 14 TFORM14 = 'E ' / data format of field: 4-byte REAL TTYPE15 = 'ASTROMETRIC_PARAMS_SOLVED' / label for field 15 TFORM15 = 'B ' / data format of field TZERO15 = -128 / offset for signed bytes TSCAL15 = 1 / data are not scaled TTYPE16 = 'PARALLAX' / label for field 16 TFORM16 = 'E ' / data format of field: 4-byte REAL TTYPE17 = 'PARALLAX_ERROR' / label for field 17 TFORM17 = 'E ' / data format of field: 4-byte REAL TTYPE18 = 'PMRA ' / label for field 18 TFORM18 = 'E ' / data format of field: 4-byte REAL TTYPE19 = 'PMRA_ERROR' / label for field 19 TFORM19 = 'E ' / data format of field: 4-byte REAL TTYPE20 = 'PMDEC ' / label for field 20 TFORM20 = 'E ' / data format of field: 4-byte REAL TTYPE21 = 'PMDEC_ERROR' / label for field 21 TFORM21 = 'E ' / data format of field: 4-byte REAL END _PL_G2@s抠u·lyA +SIMPLE = T / file does conform to FITS standard BITPIX = 16 / number of bits per data pixel NAXIS = 0 / number of data axes EXTEND = T / FITS dataset may contain extensions COMMENT FITS (Flexible Image Transport System) format is defined in 'AstronomyCOMMENT and Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H END XTENSION= 'BINTABLE' / binary table extension BITPIX = 8 / 8-bit bytes NAXIS = 2 / 2-dimensional binary table NAXIS1 = 88 / width of table in bytes NAXIS2 = 25 / number of rows in table PCOUNT = 0 / size of special data area GCOUNT = 1 / one data group (required keyword) TFIELDS = 21 / number of fields in each row TTYPE1 = 'SOURCE_ID' / label for field 1 TFORM1 = 'K ' / data format of field: 8-byte INTEGER TTYPE2 = 'REF_CAT ' / label for field 2 TFORM2 = '2A ' / data format of field: ASCII Character TTYPE3 = 'RA ' / label for field 3 TFORM3 = 'D ' / data format of field: 8-byte DOUBLE TTYPE4 = 'DEC ' / label for field 4 TFORM4 = 'D ' / data format of field: 8-byte DOUBLE TTYPE5 = 'PHOT_G_MEAN_MAG' / label for field 5 TFORM5 = 'E ' / data format of field: 4-byte REAL TTYPE6 = 'PHOT_G_MEAN_FLUX_OVER_ERROR' / label for field 6 TFORM6 = 'E ' / data format of field: 4-byte REAL TTYPE7 = 'PHOT_BP_MEAN_MAG' / label for field 7 TFORM7 = 'E ' / data format of field: 4-byte REAL TTYPE8 = 'PHOT_BP_MEAN_FLUX_OVER_ERROR' / label for field 8 TFORM8 = 'E ' / data format of field: 4-byte REAL TTYPE9 = 'PHOT_RP_MEAN_MAG' / label for field 9 TFORM9 = 'E ' / data format of field: 4-byte REAL TTYPE10 = 'PHOT_RP_MEAN_FLUX_OVER_ERROR' / label for field 10 TFORM10 = 'E ' / data format of field: 4-byte REAL TTYPE11 = 'PHOT_BP_RP_EXCESS_FACTOR' / label for field 11 TFORM11 = 'E ' / data format of field: 4-byte REAL TTYPE12 = 'ASTROMETRIC_EXCESS_NOISE' / label for field 12 TFORM12 = 'E ' / data format of field: 4-byte REAL TTYPE13 = 'DUPLICATED_SOURCE' / label for field 13 TFORM13 = 'L ' / data format of field: 1-byte LOGICAL TTYPE14 = 'ASTROMETRIC_SIGMA5D_MAX' / label for field 14 TFORM14 = 'E ' / data format of field: 4-byte REAL TTYPE15 = 'ASTROMETRIC_PARAMS_SOLVED' / label for field 15 TFORM15 = 'B ' / data format of field TZERO15 = -128 / offset for signed bytes TSCAL15 = 1 / data are not scaled TTYPE16 = 'PARALLAX' / label for field 16 TFORM16 = 'E ' / data format of field: 4-byte REAL TTYPE17 = 'PARALLAX_ERROR' / label for field 17 TFORM17 = 'E ' / data format of field: 4-byte REAL TTYPE18 = 'PMRA ' / label for field 18 TFORM18 = 'E ' / data format of field: 4-byte REAL TTYPE19 = 'PMRA_ERROR' / label for field 19 TFORM19 = 'E ' / data format of field: 4-byte REAL TTYPE20 = 'PMDEC ' / label for field 20 TFORM20 = 'E ' / data format of field: 4-byte REAL TTYPE21 = 'PMDEC_ERROR' / label for field 21 TFORM21 = 'E ' / data format of field: 4-byte REAL EXTNAME = 'GAIAHPX ' / name of this binary table extension HPXNSIDE= 32 HPXNEST = T END _PL_G2@s抠u·lyA CBAq@ݻAAOS2?@^F@ûAD.AAA C*??F?)@%>T,>@,>ɓ_"G2@s ‰T2 AQBA @I+A@U?@d;FF_iрG2@sG.BhMABqkALY"FH_iG2@s ZtlXAQnCAz@A~A%?zF@Oj|@ DI?@_>@ _QG2@sncAfC©A AAjAE\?F?m-J>"?;"LF>>,u>6@k? QJ>_YYG2@seYAAB6AF@3_!G2@sx81WAB@sFI& _G2@s7#}A?B魲A@A@?F@$qZ+?LJ diff --git a/py/desitarget/test/test_brightmask.py b/py/desitarget/test/test_brightmask.py index c47b81e36..8b4c89783 100644 --- a/py/desitarget/test/test_brightmask.py +++ b/py/desitarget/test/test_brightmask.py @@ -10,6 +10,8 @@ import numpy.lib.recfunctions as rfn from astropy.coordinates import SkyCoord from astropy import units as u +from glob import glob +import healpy as hp from desitarget import brightmask, io from desitarget.targetmask import desi_mask, targetid_mask @@ -19,17 +21,32 @@ class TestBRIGHTMASK(unittest.TestCase): - def setUp(self): - # ADM some locations of output test files - self.testbsfile = 'bs.fits' - self.testmaskfile = 'bsmask.fits' - self.testtargfile = 'bstargs.fits' - - # ADM some locations of input files - self.bsdatadir = resource_filename('desitarget.test', 't2') - self.datadir = resource_filename('desitarget.test', 't') - maskablefile = self.bsdatadir + '/sweep-190m005-200p000.fits' - unmaskablefile = self.datadir + '/sweep-320m005-330p000.fits' + @classmethod + def setUpClass(cls): + # ADM set up the necessary environment variables. + cls.gaiadir_orig = os.getenv("GAIA_DIR") + os.environ["GAIA_DIR"] = resource_filename('desitarget.test', 't4') + cls.tychodir_orig = os.getenv("TYCHO_DIR") + os.environ["TYCHO_DIR"] = resource_filename('desitarget.test', 't4/tycho') + cls.uratdir_orig = os.getenv("URAT_DIR") + os.environ["URAT_DIR"] = resource_filename('desitarget.test', 't4/urat') + + # ADM some locations of input files. + cls.bsdatadir = resource_filename('desitarget.test', 't2') + cls.datadir = resource_filename('desitarget.test', 't') + maskablefile = cls.bsdatadir + '/sweep-190m005-200p000.fits' + unmaskablefile = cls.datadir + '/sweep-320m005-330p000.fits' + + # ADM allowed HEALPixels in the Tycho directory. + pixnum = [] + fns = glob(os.path.join(os.environ["TYCHO_DIR"], 'healpix', '*fits')) + for fn in fns: + data, hdr = fitsio.read(fn, "TYCHOHPX", header=True) + nside = hdr["HPXNSIDE"] + theta, phi = np.radians(90-data["DEC"]), np.radians(data["RA"]) + pixnum.append(list(set(hp.ang2pix(nside, theta, phi, nest=True)))) + cls.pixnum = [i for eachlist in pixnum for i in eachlist] + cls.nside = nside # ADM read in the "maskable targets" (targets that ARE in masks) masktargs = fitsio.read(maskablefile) @@ -39,8 +56,8 @@ def setUp(self): [zeros, zeros], usemask=False, dtypes='>i8') # ADM As the sweeps file is also doubling as a targets file, we have to duplicate the # ADM column "BRICK_OBJID" and include it as the new column "OBJID" - self.masktargs = rfn.append_fields(masktargs, "BRICK_OBJID", zeros, usemask=False, dtypes='>i4') - self.masktargs["BRICK_OBJID"] = self.masktargs["OBJID"] + cls.masktargs = rfn.append_fields(masktargs, "BRICK_OBJID", zeros, usemask=False, dtypes='>i4') + cls.masktargs["BRICK_OBJID"] = cls.masktargs["OBJID"] # ADM read in the "unmaskable targets" (targets that are NOT in masks) unmasktargs = fitsio.read(unmaskablefile) @@ -50,41 +67,40 @@ def setUp(self): [zeros, zeros], usemask=False, dtypes='>i8') # ADM As the sweeps file is also doubling as a targets file, we have to duplicate the # ADM column "BRICK_OBJID" and include it as the new column "OBJID" - self.unmasktargs = rfn.append_fields(unmasktargs, "BRICK_OBJID", zeros, usemask=False, dtypes='>i4') - self.unmasktargs["BRICK_OBJID"] = self.unmasktargs["OBJID"] + cls.unmasktargs = rfn.append_fields(unmasktargs, "BRICK_OBJID", zeros, usemask=False, dtypes='>i4') + cls.unmasktargs["BRICK_OBJID"] = cls.unmasktargs["OBJID"] # ADM invent a mask with differing radii (1' and 20') and declinations - self.mask = np.zeros(3, dtype=[('RA', '>f8'), ('DEC', '>f8'), ('IN_RADIUS', '>f4'), - ('E1', '>f4'), ('E2', '>f4'), ('TYPE', 'S4')]) - self.mask["DEC"] = [0, 70, 35] - self.mask["IN_RADIUS"] = [1, 20, 10] - self.mask["E1"] = [0., 0., -0.3] - self.mask["E2"] = [0., 0., 0.5] - self.mask["TYPE"] = ['REX', b'PSF ', 'EXP '] - - def tearDown(self): + cls.mask = np.zeros(3, dtype=[('RA', '>f8'), ('DEC', '>f8'), ('IN_RADIUS', '>f4'), + ('E1', '>f4'), ('E2', '>f4'), ('TYPE', 'S4')]) + cls.mask["DEC"] = [0, 70, 35] + cls.mask["IN_RADIUS"] = [1, 20, 10] + cls.mask["E1"] = [0., 0., -0.3] + cls.mask["E2"] = [0., 0., 0.5] + cls.mask["TYPE"] = ['REX', b'PSF ', 'EXP '] + + @classmethod + def tearDownClass(cls): # ADM remove any existing test files in this directory - if os.path.exists(self.testmaskfile): - os.remove(self.testmaskfile) - if os.path.exists(self.testtargfile): - os.remove(self.testtargfile) - if os.path.exists(self.testbsfile): - os.remove(self.testbsfile) - - def test_make_bright_source_mask(self): - """Test the construction of a bright source mask + if os.path.exists(cls.testmaskfile): + os.remove(cls.testmaskfile) + if os.path.exists(cls.testtargfile): + os.remove(cls.testtargfile) + if os.path.exists(cls.testbsfile): + os.remove(cls.testbsfile) + # ADM reset the environment variables. + if cls.gaiadir_orig is not None: + os.environ["GAIA_DIR"] = cls.gaiadir_orig + if cls.tychodir_orig is not None: + os.environ["TYCHO_DIR"] = cls.tychodir_orig + if cls.uratdir_orig is not None: + os.environ["URAT_DIR"] = cls.uratdir_orig + + def test_make_bright_star_mask(self): + """Test the construction of a bright star mask. """ - # ADM create a collection of bright sources and write to file - bs1 = brightmask.collect_bright_sources('grz', [9, 9, 9], - rootdirname=self.bsdatadir, outfilename=self.testbsfile) - # ADM create a bright source mask from the collection of bright sources and write to file... - mask = brightmask.make_bright_source_mask('grz', [9, 9, 9], - infilename=self.testbsfile, outfilename=self.testmaskfile) - # ADM ...and read it back in - mask1 = fitsio.read(self.testmaskfile) - # ADM create the bright source mask from scratch - mask2 = brightmask.make_bright_source_mask('grz', [9, 9, 9], rootdirname=self.bsdatadir) - # ADM the created-from-scratch mask should be the same as the read-in mask + mask = make_bright_star_mask_in_hp(cls.nside, cls.pixnum[0], maglim=20.) + self.assertTrue(np.all(mask1["TARGETID"] == mask2["TARGETID"])) def test_mask_targets(self): From caafd8039b1ab6a74de47949d78129b406fa7c33 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Mon, 15 Jun 2020 19:35:13 -0700 Subject: [PATCH 22/25] add new unit test files for building masks from Tycho/Gaia/URAT --- py/desitarget/test/t4/tycho/healpix/healpix-00072.fits | 6 ++++++ py/desitarget/test/t4/tycho/healpix/healpix-00074.fits | 7 +++++++ py/desitarget/test/t4/tycho/healpix/healpix-00191.fits | 8 ++++++++ py/desitarget/test/t4/urat/healpix/healpix-04669.fits | 2 ++ py/desitarget/test/t4/urat/healpix/healpix-04671.fits | 5 +++++ py/desitarget/test/t4/urat/healpix/healpix-04751.fits | 5 +++++ py/desitarget/test/t4/urat/healpix/healpix-04761.fits | 2 ++ py/desitarget/test/t4/urat/healpix/healpix-12284.fits | 1 + 8 files changed, 36 insertions(+) create mode 100644 py/desitarget/test/t4/tycho/healpix/healpix-00072.fits create mode 100644 py/desitarget/test/t4/tycho/healpix/healpix-00074.fits create mode 100644 py/desitarget/test/t4/tycho/healpix/healpix-00191.fits create mode 100644 py/desitarget/test/t4/urat/healpix/healpix-04669.fits create mode 100644 py/desitarget/test/t4/urat/healpix/healpix-04671.fits create mode 100644 py/desitarget/test/t4/urat/healpix/healpix-04751.fits create mode 100644 py/desitarget/test/t4/urat/healpix/healpix-04761.fits create mode 100644 py/desitarget/test/t4/urat/healpix/healpix-12284.fits diff --git a/py/desitarget/test/t4/tycho/healpix/healpix-00072.fits b/py/desitarget/test/t4/tycho/healpix/healpix-00072.fits new file mode 100644 index 000000000..d6d1abab1 --- /dev/null +++ b/py/desitarget/test/t4/tycho/healpix/healpix-00072.fits @@ -0,0 +1,6 @@ +SIMPLE = T / file does conform to FITS standard BITPIX = 16 / number of bits per data pixel NAXIS = 0 / number of data axes EXTEND = T / FITS dataset may contain extensions COMMENT FITS (Flexible Image Transport System) format is defined in 'AstronomyCOMMENT and Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H END XTENSION= 'BINTABLE' / binary table extension BITPIX = 8 / 8-bit bytes NAXIS = 2 / 2-dimensional binary table NAXIS1 = 98 / width of table in bytes NAXIS2 = 25 / number of rows in table PCOUNT = 0 / size of special data area GCOUNT = 1 / one data group (required keyword) TFIELDS = 23 / number of fields in each row TTYPE1 = 'TYC1 ' / label for field 1 TFORM1 = 'I ' / data format of field: 2-byte INTEGER TTYPE2 = 'TYC2 ' / label for field 2 TFORM2 = 'I ' / data format of field: 2-byte INTEGER TTYPE3 = 'TYC3 ' / label for field 3 TFORM3 = 'B ' / data format of field: BYTE TTYPE4 = 'RA ' / label for field 4 TFORM4 = 'D ' / data format of field: 8-byte DOUBLE TTYPE5 = 'DEC ' / label for field 5 TFORM5 = 'D ' / data format of field: 8-byte DOUBLE TTYPE6 = 'MEAN_RA ' / label for field 6 TFORM6 = 'D ' / data format of field: 8-byte DOUBLE TTYPE7 = 'MEAN_DEC' / label for field 7 TFORM7 = 'D ' / data format of field: 8-byte DOUBLE TTYPE8 = 'SIGMA_RA' / label for field 8 TFORM8 = 'E ' / data format of field: 4-byte REAL TTYPE9 = 'SIGMA_DEC' / label for field 9 TFORM9 = 'E ' / data format of field: 4-byte REAL TTYPE10 = 'PM_RA ' / label for field 10 TFORM10 = 'E ' / data format of field: 4-byte REAL TTYPE11 = 'PM_DEC ' / label for field 11 TFORM11 = 'E ' / data format of field: 4-byte REAL TTYPE12 = 'SIGMA_PM_RA' / label for field 12 TFORM12 = 'E ' / data format of field: 4-byte REAL TTYPE13 = 'SIGMA_PM_DEC' / label for field 13 TFORM13 = 'E ' / data format of field: 4-byte REAL TTYPE14 = 'EPOCH_RA' / label for field 14 TFORM14 = 'E ' / data format of field: 4-byte REAL TTYPE15 = 'EPOCH_DEC' / label for field 15 TFORM15 = 'E ' / data format of field: 4-byte REAL TTYPE16 = 'MAG_BT ' / label for field 16 TFORM16 = 'E ' / data format of field: 4-byte REAL TTYPE17 = 'MAG_VT ' / label for field 17 TFORM17 = 'E ' / data format of field: 4-byte REAL TTYPE18 = 'MAG_HP ' / label for field 18 TFORM18 = 'E ' / data format of field: 4-byte REAL TTYPE19 = 'ISGALAXY' / label for field 19 TFORM19 = 'B ' / data format of field: BYTE TTYPE20 = 'JMAG ' / label for field 20 TFORM20 = 'E ' / data format of field: 4-byte REAL TTYPE21 = 'HMAG ' / label for field 21 TFORM21 = 'E ' / data format of field: 4-byte REAL TTYPE22 = 'KMAG ' / label for field 22 TFORM22 = 'E ' / data format of field: 4-byte REAL TTYPE23 = 'ZGUESS ' / label for field 23 TFORM23 = 'E ' / data format of field: 4-byte REAL EXTNAME = 'TYCHOHPX' / name of this binary table extension ORIGIN = 'ESO-QFITS' DATE = '2019-04-11T07:08:51' AN_FILE = 'TAGALONG' COSMODIR= '/global/cfs/cdirs/cosmo/staging/tycho2/' COPYDATE= '2020-06-06T00:38:32' HPXNSIDE= 4 HPXNEST = T HPXDATE = '2020-06-06T00:38:44' END j@uthM@u AA'mA@\@\Aq@uo6S%]1X@ub6SPu27Ԣ7A(ff@@ffDDAH`A>bBBBq$@uy:mbY@u/GA7V:8 )33@@DRDATAA^A%CACA`BA5͟q@u8Zn X7 z7FzAEVA4A"uAvA?}A0q@uof@^@uKf8~7b]7v홚fg?33?33DD)A@u6C&@uzm&ea~6vd7ʉˆ?33?33DDA1"A"AA+AA%A?q2@u^+Uw@u]dž!AB7 +77"?ff?ffDDA<bA$(A@+@^At@uT}V.@u/727lh@@DDAL1A;VAȴAA A)=q/@uZCh [bu9@uZŗ[66E7$A{33??ٙDHDA.A'|A7A#AA!q@u t^ y@u t5T\7Q-8 @lfg@33@DDAGABABBBt@u! +';-R@u!%/)|7?8 S"?ff;33@ff@ffDDAR=qA;lAjA=qA A*ߤq#@u!,-@u!VT,#S78A34K33@@ffDDARIAEpBBBt@u!su?Kt@u!Ģ0uc6#7N7XxA 33@,@@DDAO%A.A +A@=qAJtA@u"b#ig@u"bGj"7%8A?L@33@DDAISA=qA#AAmA3'^@teQihXY@te򳽇47;7YEff??DqD3A3wA+A^AyA+A$c^#@thzB BI@thJZK7:*7}S33??ٙDDA6-A- +=A A|AjA,{^@tiR*o?Ew@tie}4D]7xq7Afff33?ff?33D3DA@{A4`A%A 5?AA1QY@tQrGϠf@tQ|_R 77d@ff33@ff@ffD +DAC?}A3AAA%A$XYL@tR="% }6@tRaur"{BBBY@t`8ŗy!laR@t`8d+7j=8=33@@DDAP(AG#A JAhA33A1Y@tZXM%Ӥ,@tZQ06as6&YBR@33??DDAA?}AffA~@|AںY@tX9˳r3m_&@tX9bAmq p77&?@9@FfgDDA;A3mA A$A^5A-Y@t_Ïa\˅J@t_Qb27GD82)A33x@y@ffDDAUXAChsA(bNA DAA8}Y@t_@u6ξ@t_ĿЊ7C8#XG@ə?@s33@ffDDATA>9ArAlAA0eY@tTT^M[@tTT㙍/R77'Ŭff ?fff?DDA,rA!A^A tA +\A&Y@tY-P8~v~@tYLɁ+u+7 F7.KAP@34?ff?ffD{DA4A#SA +=@@AY0@tZe|Y@tZTBfe777+j3333??D{DA7FA!wA;d@@홚A5Yp@tX[!f@tXJ+n97g'7@33?33?33D{DHA9wA1 +=AIAffA^A+Y@tX?ܜ#&6@tX?*𙆿&$Q7]$7baA(??ٙDDA>A5BBBY1@tSBֹ@tSB-3'87]7A3433?ٙ?ٙD{DAF\A5xAAGA /A(~ \ No newline at end of file diff --git a/py/desitarget/test/t4/tycho/healpix/healpix-00191.fits b/py/desitarget/test/t4/tycho/healpix/healpix-00191.fits new file mode 100644 index 000000000..2691e3ac0 --- /dev/null +++ b/py/desitarget/test/t4/tycho/healpix/healpix-00191.fits @@ -0,0 +1,8 @@ +SIMPLE = T / file does conform to FITS standard BITPIX = 16 / number of bits per data pixel NAXIS = 0 / number of data axes EXTEND = T / FITS dataset may contain extensions COMMENT FITS (Flexible Image Transport System) format is defined in 'AstronomyCOMMENT and Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H END XTENSION= 'BINTABLE' / binary table extension BITPIX = 8 / 8-bit bytes NAXIS = 2 / 2-dimensional binary table NAXIS1 = 98 / width of table in bytes NAXIS2 = 25 / number of rows in table PCOUNT = 0 / size of special data area GCOUNT = 1 / one data group (required keyword) TFIELDS = 23 / number of fields in each row TTYPE1 = 'TYC1 ' / label for field 1 TFORM1 = 'I ' / data format of field: 2-byte INTEGER TTYPE2 = 'TYC2 ' / label for field 2 TFORM2 = 'I ' / data format of field: 2-byte INTEGER TTYPE3 = 'TYC3 ' / label for field 3 TFORM3 = 'B ' / data format of field: BYTE TTYPE4 = 'RA ' / label for field 4 TFORM4 = 'D ' / data format of field: 8-byte DOUBLE TTYPE5 = 'DEC ' / label for field 5 TFORM5 = 'D ' / data format of field: 8-byte DOUBLE TTYPE6 = 'MEAN_RA ' / label for field 6 TFORM6 = 'D ' / data format of field: 8-byte DOUBLE TTYPE7 = 'MEAN_DEC' / label for field 7 TFORM7 = 'D ' / data format of field: 8-byte DOUBLE TTYPE8 = 'SIGMA_RA' / label for field 8 TFORM8 = 'E ' / data format of field: 4-byte REAL TTYPE9 = 'SIGMA_DEC' / label for field 9 TFORM9 = 'E ' / data format of field: 4-byte REAL TTYPE10 = 'PM_RA ' / label for field 10 TFORM10 = 'E ' / data format of field: 4-byte REAL TTYPE11 = 'PM_DEC ' / label for field 11 TFORM11 = 'E ' / data format of field: 4-byte REAL TTYPE12 = 'SIGMA_PM_RA' / label for field 12 TFORM12 = 'E ' / data format of field: 4-byte REAL TTYPE13 = 'SIGMA_PM_DEC' / label for field 13 TFORM13 = 'E ' / data format of field: 4-byte REAL TTYPE14 = 'EPOCH_RA' / label for field 14 TFORM14 = 'E ' / data format of field: 4-byte REAL TTYPE15 = 'EPOCH_DEC' / label for field 15 TFORM15 = 'E ' / data format of field: 4-byte REAL TTYPE16 = 'MAG_BT ' / label for field 16 TFORM16 = 'E ' / data format of field: 4-byte REAL TTYPE17 = 'MAG_VT ' / label for field 17 TFORM17 = 'E ' / data format of field: 4-byte REAL TTYPE18 = 'MAG_HP ' / label for field 18 TFORM18 = 'E ' / data format of field: 4-byte REAL TTYPE19 = 'ISGALAXY' / label for field 19 TFORM19 = 'B ' / data format of field: BYTE TTYPE20 = 'JMAG ' / label for field 20 TFORM20 = 'E ' / data format of field: 4-byte REAL TTYPE21 = 'HMAG ' / label for field 21 TFORM21 = 'E ' / data format of field: 4-byte REAL TTYPE22 = 'KMAG ' / label for field 22 TFORM22 = 'E ' / data format of field: 4-byte REAL TTYPE23 = 'ZGUESS ' / label for field 23 TFORM23 = 'E ' / data format of field: 4-byte REAL EXTNAME = 'TYCHOHPX' / name of this binary table extension ORIGIN = 'ESO-QFITS' DATE = '2019-04-11T07:08:51' AN_FILE = 'TAGALONG' COSMODIR= '/global/cfs/cdirs/cosmo/staging/tycho2/' COPYDATE= '2020-06-06T00:38:32' HPXNSIDE= 4 HPXNEST = T HPXDATE = '2020-06-06T00:38:50' END C)@sCİN:}O@sCAyGN_f 7b]77a@ @33DD)A6A,AAAA(QC @sud”@sr7 7$@?33?33DHD)A-A#AA PA DAȴC/@suk%A@suq7a85g^ff33@l@DDASA=/AA `A +A)CCg@srdHр@srٙcGø66K@ff@33??DDA!AnAA^A|A`Cy@s2P/*@s</x e7A7:i@33ff??D=DHA-CA%mA JAAIA!gC@sw@s'aB]M87xq7Bn@ @33DRD)A:DA1pA +=AA33A)CZ@s#e\~}^{@su;\&o7y7B@&ff@,D=DRAGFA6AA#AƨAA0]Co@s҇-W@suDM167^LQ7 33@ @D)D=A?%A'CAD@@1A|C]@sVk|" +@s"U\L77B#34Afg@ff@33DDA;A0ffAdZA A+A(K^C@saG=fWh@sa~ifU877ЌAp@l@33@ DDfA<5?A9PBBBC@s +s@sCr77‘A$33@33@ DDA?A9XBBBC@swU%@sw3OY77[B +@@&ffDDA@1'A6BBBC@sH(C SЌ@sc|Q4.{7(7A134@33@ D +DA>bNA3mA"MAjA%A/C@s1@sH$6K06wAff??DDA)A @"@r\@e@ɄM?1@spmL wk#@sڋQW w㤪6%6K0B-332ff??D)DRAA@j@@PAp?@sn27 rvj@sawiC ȧ77ř,33.fg?33@DDA;hsA:nBBB? @s^s$ y9|]@s^ d8 e8S; @,@333DDADbNAS7LBBB?=@sS<= ׽:@sS~ nׇ88 ff<@ff@ DDAA/A=xA%7LA 1A`A4~(?@s{™ )@sAC N7sd7g Rff?33@DD3A<ȴA*bNAVAAffA#tT?6@s' 3s@sf^. 3<}ߪ7R7qg@33333@ff@ DDA: A(Av@;@A"h?$@s~ ,@s [ ,7xq7'ŬFfg ?ٙ?ffDDA9AO@M@ӶF@GAȴ?@st\ dI@st X36o6@ a?33?DqDA(DAOA`A ZA +jA8?@sG* ^j@sԂ j?M7,n7@u{0`AAAAAA`4@AL?@uknAAAAAA@əALA@u!mU6P%A~@333@ffALO@u1~AAAAAAB]33A(@33ALP@u !)A9X= CAu<oAq;=x陚@ffALR@uAx=CAyV=ěApj>c33@ALW@u3֚dSm@AAAAAAs339@ALX@u;gT_Q AAAAAA X@ALY@u=s;,AAAAAA@@ffALZ@uGB؄Lgp A;=LAq`B=qAkG=@?@ffAL[@uN"v*k@AAAAAAA|@əAL\@uQUpEAAAAAAB33A#33AL^@uZ.\!0A)#=\)A#7L=wA!=mhAffy@AL_@u[O`AAAAAAB \ No newline at end of file diff --git a/py/desitarget/test/t4/urat/healpix/healpix-04671.fits b/py/desitarget/test/t4/urat/healpix/healpix-04671.fits new file mode 100644 index 000000000..7faaea65d --- /dev/null +++ b/py/desitarget/test/t4/urat/healpix/healpix-04671.fits @@ -0,0 +1,5 @@ +SIMPLE = T / file does conform to FITS standard BITPIX = 16 / number of bits per data pixel NAXIS = 0 / number of data axes EXTEND = T / FITS dataset may contain extensions COMMENT FITS (Flexible Image Transport System) format is defined in 'AstronomyCOMMENT and Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H END XTENSION= 'BINTABLE' / binary table extension BITPIX = 8 / 8-bit bytes NAXIS = 2 / 2-dimensional binary table NAXIS1 = 60 / width of table in bytes NAXIS2 = 25 / number of rows in table PCOUNT = 0 / size of special data area GCOUNT = 1 / one data group (required keyword) TFIELDS = 12 / number of fields in each row TTYPE1 = 'URAT_ID ' / label for field 1 TFORM1 = 'K ' / data format of field: 8-byte INTEGER TTYPE2 = 'RA ' / label for field 2 TFORM2 = 'D ' / data format of field: 8-byte DOUBLE TTYPE3 = 'DEC ' / label for field 3 TFORM3 = 'D ' / data format of field: 8-byte DOUBLE TTYPE4 = 'APASS_G_MAG' / label for field 4 TFORM4 = 'E ' / data format of field: 4-byte REAL TTYPE5 = 'APASS_G_MAG_ERROR' / label for field 5 TFORM5 = 'E ' / data format of field: 4-byte REAL TTYPE6 = 'APASS_R_MAG' / label for field 6 TFORM6 = 'E ' / data format of field: 4-byte REAL TTYPE7 = 'APASS_R_MAG_ERROR' / label for field 7 TFORM7 = 'E ' / data format of field: 4-byte REAL TTYPE8 = 'APASS_I_MAG' / label for field 8 TFORM8 = 'E ' / data format of field: 4-byte REAL TTYPE9 = 'APASS_I_MAG_ERROR' / label for field 9 TFORM9 = 'E ' / data format of field: 4-byte REAL TTYPE10 = 'PMRA ' / label for field 10 TFORM10 = 'E ' / data format of field: 4-byte REAL TTYPE11 = 'PMDEC ' / label for field 11 TFORM11 = 'E ' / data format of field: 4-byte REAL TTYPE12 = 'PM_ERROR' / label for field 12 TFORM12 = 'E ' / data format of field: 4-byte REAL EXTNAME = 'URATHPX ' / name of this binary table extension HPXNSIDE= 32 HPXNEST = T END AK@u*|@AAAAAAAffAffAK&@un(l g@AF=PA&=aGASoA33<@AK)@um9-0DAAAAAAA+33AK*@uܞ/#qbAAAAAAAi@@AK/@uVpAAAAAAAI@ffAK2@u0zQf| +bAK#; +A>EPBB-@ffAK3@u1Dkj=ʹA[<tAGT<ěA=X> Ci™@ffAK6@uCՕi.m{EAAAAAAAD@AK;@ue}2&TR@A+<49XAH=wAA9@AK>@uoOX Zf'AAAAAAAVff@AKC@u^KAAAAAAA@AKF@uڤM4bAAAAAAA4?ff@AKG@uj qA?}ixAoAAAffff@AKH@u/DDRAAAAAAA33{33@əAKJ@uQ*AAAAAAA33@33AKK@uȭ;8?" )`AAAAA?}-AH@AKM@uA!AoA󶺃oAzVoAff 33@AKO@u.%׈AAAAAABffA33@ٙAKP@u +FlO_`AAAAAAAAKS@u#3X{GpK@AAAAAA>AAKT@u,QG:L@AAAAAAB/332ff@AKV@u7ɚ$QAAAAAABAKW@u:s AAAAAA@0@33AKX@u;1L +D"> AAAAAA? @ٙAKY@uEy^yw AAAAAAA AA \ No newline at end of file diff --git a/py/desitarget/test/t4/urat/healpix/healpix-04751.fits b/py/desitarget/test/t4/urat/healpix/healpix-04751.fits new file mode 100644 index 000000000..2ebb5add3 --- /dev/null +++ b/py/desitarget/test/t4/urat/healpix/healpix-04751.fits @@ -0,0 +1,5 @@ +SIMPLE = T / file does conform to FITS standard BITPIX = 16 / number of bits per data pixel NAXIS = 0 / number of data axes EXTEND = T / FITS dataset may contain extensions COMMENT FITS (Flexible Image Transport System) format is defined in 'AstronomyCOMMENT and Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H END XTENSION= 'BINTABLE' / binary table extension BITPIX = 8 / 8-bit bytes NAXIS = 2 / 2-dimensional binary table NAXIS1 = 60 / width of table in bytes NAXIS2 = 25 / number of rows in table PCOUNT = 0 / size of special data area GCOUNT = 1 / one data group (required keyword) TFIELDS = 12 / number of fields in each row TTYPE1 = 'URAT_ID ' / label for field 1 TFORM1 = 'K ' / data format of field: 8-byte INTEGER TTYPE2 = 'RA ' / label for field 2 TFORM2 = 'D ' / data format of field: 8-byte DOUBLE TTYPE3 = 'DEC ' / label for field 3 TFORM3 = 'D ' / data format of field: 8-byte DOUBLE TTYPE4 = 'APASS_G_MAG' / label for field 4 TFORM4 = 'E ' / data format of field: 4-byte REAL TTYPE5 = 'APASS_G_MAG_ERROR' / label for field 5 TFORM5 = 'E ' / data format of field: 4-byte REAL TTYPE6 = 'APASS_R_MAG' / label for field 6 TFORM6 = 'E ' / data format of field: 4-byte REAL TTYPE7 = 'APASS_R_MAG_ERROR' / label for field 7 TFORM7 = 'E ' / data format of field: 4-byte REAL TTYPE8 = 'APASS_I_MAG' / label for field 8 TFORM8 = 'E ' / data format of field: 4-byte REAL TTYPE9 = 'APASS_I_MAG_ERROR' / label for field 9 TFORM9 = 'E ' / data format of field: 4-byte REAL TTYPE10 = 'PMRA ' / label for field 10 TFORM10 = 'E ' / data format of field: 4-byte REAL TTYPE11 = 'PMDEC ' / label for field 11 TFORM11 = 'E ' / data format of field: 4-byte REAL TTYPE12 = 'PM_ERROR' / label for field 12 TFORM12 = 'E ' / data format of field: 4-byte REAL EXTNAME = 'URATHPX ' / name of this binary table extension HPXNSIDE= 32 HPXNEST = T END A6i@t]&^*@AAAAAA?333Fff@ffA6k@t]@8?P%AAAAAA?ffvff@A6l@t]A@oJA}`B=ixAp;ěAl~o@ffs33@A6u@t]#]:Y AzẃoAx źoAq33o@fff33@A6y@t]!q +^N+AAAAAA?ٙ@A6{@t]Ǚ[ѹkAAAAAA +@ٙA6|@t]\Yy#X `AAAAAAA@A6~@t]$; @AAAAAAff@A6@t]yѫp AAAAAAAffA6@t^%O-Y 2 Ax<tAoff@əA6@t^,EAAAAAAAA33@əA6@t^Wj(kvAAAAAAffl@A6@t^DAGAAAAAA@333@33A6@t^`S]4AAAAAA@A@33A6@t^0n# AAAAAA@@@A6@t^uk-E@AAAAAA33@@33A6@t^hAAAAAAA6@A6@t^#+2AAAAAAA+33@@A6@t^jރO0yu AAAAAA?A{33AA6@t^P?ʰ@AtrL&ff@˘@t.< wWrAAAAAA;33 @˘@t2<~Ml{5*8AAAAAA8@˘@tD(MvAAAAAA@fff33A333˘@t >AAAAAA@A0@ff˘@t62AAAAAAA@陚˘@t[Q\sAe/=-A[-=hsA]>@@˘@t\οaAAAAAAAffx@ə˘@tZѿfʘA~ff<# +Aw7=OA|oA@ff@˘@tq[BEA!oA=qoAA@ff33@˘@tLI%<҄A\=-AU=AY`B>@ @ff˘@tLD|ҍ]AAAAAA+33LA˘@t1ZXfAAAAAAA@˘@t$MKQPAAAAAAAff@˘@t&Rℕs9AAAAAA33@˘@t-ŠIAAAAAA33@ff˘@tH-m߿(DAAAAAABff@˘@tYr=Ҷ&AAAAAAff?ff@33˘@tf;ЗrAAAAAAff@ \ No newline at end of file diff --git a/py/desitarget/test/t4/urat/healpix/healpix-12284.fits b/py/desitarget/test/t4/urat/healpix/healpix-12284.fits new file mode 100644 index 000000000..01ef9760b --- /dev/null +++ b/py/desitarget/test/t4/urat/healpix/healpix-12284.fits @@ -0,0 +1 @@ +SIMPLE = T / file does conform to FITS standard BITPIX = 16 / number of bits per data pixel NAXIS = 0 / number of data axes EXTEND = T / FITS dataset may contain extensions COMMENT FITS (Flexible Image Transport System) format is defined in 'AstronomyCOMMENT and Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H END XTENSION= 'BINTABLE' / binary table extension BITPIX = 8 / 8-bit bytes NAXIS = 2 / 2-dimensional binary table NAXIS1 = 60 / width of table in bytes NAXIS2 = 25 / number of rows in table PCOUNT = 0 / size of special data area GCOUNT = 1 / one data group (required keyword) TFIELDS = 12 / number of fields in each row TTYPE1 = 'URAT_ID ' / label for field 1 TFORM1 = 'K ' / data format of field: 8-byte INTEGER TTYPE2 = 'RA ' / label for field 2 TFORM2 = 'D ' / data format of field: 8-byte DOUBLE TTYPE3 = 'DEC ' / label for field 3 TFORM3 = 'D ' / data format of field: 8-byte DOUBLE TTYPE4 = 'APASS_G_MAG' / label for field 4 TFORM4 = 'E ' / data format of field: 4-byte REAL TTYPE5 = 'APASS_G_MAG_ERROR' / label for field 5 TFORM5 = 'E ' / data format of field: 4-byte REAL TTYPE6 = 'APASS_R_MAG' / label for field 6 TFORM6 = 'E ' / data format of field: 4-byte REAL TTYPE7 = 'APASS_R_MAG_ERROR' / label for field 7 TFORM7 = 'E ' / data format of field: 4-byte REAL TTYPE8 = 'APASS_I_MAG' / label for field 8 TFORM8 = 'E ' / data format of field: 4-byte REAL TTYPE9 = 'APASS_I_MAG_ERROR' / label for field 9 TFORM9 = 'E ' / data format of field: 4-byte REAL TTYPE10 = 'PMRA ' / label for field 10 TFORM10 = 'E ' / data format of field: 4-byte REAL TTYPE11 = 'PMDEC ' / label for field 11 TFORM11 = 'E ' / data format of field: 4-byte REAL TTYPE12 = 'PM_ERROR' / label for field 12 TFORM12 = 'E ' / data format of field: 4-byte REAL EXTNAME = 'URATHPX ' / name of this binary table extension HPXNSIDE= 32 HPXNEST = T END ab@sbn AAAAAAff33@ffad@sT9-AAAAAABae@scrVsAAAAAA A&ffag@s*wNΚ AAAAAAA33AATas@sA=wA=7L@;`Bl@a@s!c~:; AAAAAABa@sTdz AAAAAAAC33ff@ffa@sYO"ָ AAAAAA@əff@a@sd+iAAAAAAA33@a@shb`w!y\YAAAAAA+33K33@ffa@s[7wAAAAAA33@a@sҫJ AAAAAA?ff@33a@s-JsK۠AAAAAAffS33Aa@seS>AAAAAAB H@ffa@s=MךA5`BoA@a@s!|6AAAAAA@33@ffa@s:߉_`AAAAAA@ffa@sj@AAAAAABa@sm%=4@A%Ao?fff33@33a@s޸ !-w AAAAAAB33ffA 33a@sUCMAAAAAA@&ff@a@sآtڵ`AAAAAA33陚@a@sathAAAAAAAX@a@sIJq AAAAAAlA,a@sr06AAAAAAAd&ff@ \ No newline at end of file From 6d000dc33b8f95bd21dce324deea0aa9638dd147 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 16 Jun 2020 15:33:37 -0700 Subject: [PATCH 23/25] finish updating unit tests --- py/desitarget/brightmask.py | 34 ++- py/desitarget/io.py | 45 ++-- py/desitarget/test/make_testdata.py | 10 +- py/desitarget/test/make_testgaia.py | 2 +- .../test/t2/sweep-190m005-200p000.fits | 4 - py/desitarget/test/test_brightmask.py | 239 ++++++++++++------ 6 files changed, 213 insertions(+), 121 deletions(-) delete mode 100644 py/desitarget/test/t2/sweep-190m005-200p000.fits diff --git a/py/desitarget/brightmask.py b/py/desitarget/brightmask.py index d43a28585..ecc78bee6 100644 --- a/py/desitarget/brightmask.py +++ b/py/desitarget/brightmask.py @@ -299,7 +299,7 @@ def make_bright_star_mask_in_hp(nside, pixnum, verbose=True, gaiaepoch=2015.5, epochnow=gaiaepoch, epochpast=epoch_ra, epochpastdec=epoch_dec) # ADM catch the corner case where there are no initial matches. - if ra > 0: + if ra.size > 0: _, refined = radec_match_to([ra, dec], [tychoobjs["RA"][itycho], tychoobjs["DEC"][itycho]], radec=True) else: @@ -364,7 +364,8 @@ def make_bright_star_mask_in_hp(nside, pixnum, verbose=True, gaiaepoch=2015.5, def make_bright_star_mask(maglim=12., matchrad=1., numproc=32, - maskepoch=2023.0, gaiaepoch=2015.5): + maskepoch=2023.0, gaiaepoch=2015.5, + nside=None, pixels=None): """Make an all-sky bright star mask using Tycho, Gaia and URAT. Parameters @@ -383,9 +384,17 @@ def make_bright_star_mask(maglim=12., matchrad=1., numproc=32, The mask is built at this epoch. Not all sources have proper motions from every survey, so proper motions are used, in order of preference, from Gaia, URAT, then Tycho. - gaiaepoch: :class:`float`, optional, defaults to Gaia DR2 (2015.5) + gaiaepoch : :class:`float`, optional, defaults to Gaia DR2 (2015.5) The epoch of the Gaia observations. Should be 2015.5 unless we move beyond Gaia DR2. + nside : :class:`int`, optional, defaults to ``None`` + If passed, create a mask only in nested HEALPixels in `pixels` + at this `nside`. Otherwise, run for the whole sky. If `nside` + is passed then `pixels` must be passed too. + pixels :class:`list`, optional, defaults to ``None`` + If passed, create a mask only in nested HEALPixels at `nside` for + pixel integers in `pixels`. Otherwise, run for the whole sky. If + `pixels` is passed then `nside` must be passed too. Returns ------- @@ -409,7 +418,7 @@ def make_bright_star_mask(maglim=12., matchrad=1., numproc=32, Notes ----- - - Runs in about 20 minutes for `numproc`=32 and `maglim`=12. + - Runs (all-sky) in ~20 minutes for `numproc`=32 and `maglim`=12. - `IN_RADIUS` (`NEAR_RADIUS`) corresponds to `IN_BRIGHT_OBJECT` (`NEAR_BRIGHT_OBJECT`) in `data/targetmask.yaml`. These radii are set in the function `desitarget.brightmask.radius()`. @@ -419,15 +428,18 @@ def make_bright_star_mask(maglim=12., matchrad=1., numproc=32, """ log.info("running on {} processors".format(numproc)) + # ADM check if HEALPixel parameters have been correctly sent. + io.check_both_set(pixels, nside) + # ADM grab the nside of the Tycho files, which is a reasonable # ADM resolution for bright stars. - nside = get_tycho_nside() - npixels = hp.nside2npix(nside) - - # ADM array of HEALPixels over which to parallelize... - pixels = np.arange(npixels) - # ADM ...shuffle for better balance across nodes (as there are more - # ADM stars in certain regions of the sky where pixels adjoin). + if nside is None: + nside = get_tycho_nside() + npixels = hp.nside2npix(nside) + # ADM array of HEALPixels over which to parallelize... + pixels = np.arange(npixels) + # ADM ...shuffle for better balance across nodes (as there are + # ADM more stars in regions of the sky where pixels adjoin). np.random.shuffle(pixels) # ADM the common function that is actually parallelized across. diff --git a/py/desitarget/io.py b/py/desitarget/io.py index b674220a9..1830d91c1 100644 --- a/py/desitarget/io.py +++ b/py/desitarget/io.py @@ -1180,8 +1180,10 @@ def write_masks(maskdir, data, depend.setdep(hdr, 'desitarget', desitarget_version) depend.setdep(hdr, 'desitarget-git', gitversion()) # ADM add the magnitude and epoch to the header. - hdr["MAGLIM"] = maglim - hdr["MASKEPOC"] = maskepoch + if maglim is not None: + hdr["MAGLIM"] = maglim + if maskepoch is not None: + hdr["MASKEPOC"] = maskepoch # ADM add the extra dictionary to the header. if extra is not None: for key in extra: @@ -1196,23 +1198,32 @@ def write_masks(maskdir, data, return nmasks, None # ADM write across HEAPixels at input nside. - npix = hp.nside2npix(nside) - theta, phi = np.radians(90-data["DEC"]), np.radians(data["RA"]) - hpx = hp.ang2pix(nside, theta, phi, nest=True) - for pix in range(npix): - outdata = data[hpx == pix] - outhdr = dict(hdr).copy() - outhdr["FILEHPX"] = pix - # ADM construct the output file name. - fn = find_target_files(maskdir, flavor="masks", - hp=pix, maglim=maglim, epoch=maskepoch) - # ADM create necessary directory, if it doesn't exist. + if nside is not None: + npix = hp.nside2npix(nside) + theta, phi = np.radians(90-data["DEC"]), np.radians(data["RA"]) + hpx = hp.ang2pix(nside, theta, phi, nest=True) + for pix in range(npix): + outdata = data[hpx == pix] + outhdr = dict(hdr).copy() + outhdr["FILEHPX"] = pix + # ADM construct the output file name. + fn = find_target_files(maskdir, flavor="masks", + hp=pix, maglim=maglim, epoch=maskepoch) + # ADM create necessary directory, if it doesn't exist. + os.makedirs(os.path.dirname(fn), exist_ok=True) + # ADM write the output file. + if len(outdata) > 0: + fitsio.write( + fn, outdata, extname='MASKS', header=outhdr, clobber=True) + log.info('wrote {} masks to {}'.format(len(outdata), fn)) + else: + fn = find_target_files(maskdir, flavor="masks", hp="X", + maglim=maglim, epoch=maskepoch) os.makedirs(os.path.dirname(fn), exist_ok=True) - # ADM write the output file. - if len(outdata) > 0: + if len(data) > 0: fitsio.write( - fn, outdata, extname='MASKS', header=outhdr, clobber=True) - log.info('wrote {} masks to {}'.format(len(outdata), fn)) + fn, data, extname='MASKS', header=hdr, clobber=True) + log.info('wrote {} masks to {}'.format(len(data), fn)) return nmasks, os.path.dirname(fn) diff --git a/py/desitarget/test/make_testdata.py b/py/desitarget/test/make_testdata.py index 43c2dbdae..b20827bd8 100644 --- a/py/desitarget/test/make_testdata.py +++ b/py/desitarget/test/make_testdata.py @@ -123,13 +123,13 @@ # ADM adding a file to make a mask for bright stars # ADM this should go in its own directory /t2 (others are in t1) -filepath = '{}/sweep-{}.fits'.format(sweepdir, '190m005-200p000') -data, hdr = read_tractor(filepath, header=True) +# filepath = '{}/sweep-{}.fits'.format(sweepdir, '190m005-200p000') +# data, hdr = read_tractor(filepath, header=True) # ADM the "CONTINUE" comment keyword is not yet implemented # ADM in fitsio, so delete it to prevent fitsio barfing on headers -hdr.delete("CONTINUE") -keep = np.where(data["FLUX_Z"] > 100000) -fitsio.write('t2/'+basename(filepath), data[keep], header=hdr, clobber=True) +# hdr.delete("CONTINUE") +# keep = np.where(data["FLUX_Z"] > 100000) +# fitsio.write('t2/'+basename(filepath), data[keep], header=hdr, clobber=True) # ADM adding a fake pixel weight map sysdic = _load_systematics() diff --git a/py/desitarget/test/make_testgaia.py b/py/desitarget/test/make_testgaia.py index f1e814f78..9d6c147a0 100644 --- a/py/desitarget/test/make_testgaia.py +++ b/py/desitarget/test/make_testgaia.py @@ -55,7 +55,7 @@ outfile = os.path.join("t4", direc, "healpix", os.path.basename(fn)) s = set(gaiafiles) # ADM ensure a match with objects in the Gaia files. - ii = np.array( + ii = np.array( [len(set(find_gaia_files(i, neighbors=False)).intersection(s)) > 0 for i in objs]) fitsio.write(outfile, objs[ii][:25], diff --git a/py/desitarget/test/t2/sweep-190m005-200p000.fits b/py/desitarget/test/t2/sweep-190m005-200p000.fits deleted file mode 100644 index 004881151..000000000 --- a/py/desitarget/test/t2/sweep-190m005-200p000.fits +++ /dev/null @@ -1,4 +0,0 @@ -SIMPLE = T / file does conform to FITS standard BITPIX = 16 / number of bits per data pixel NAXIS = 0 / number of data axes EXTEND = T / FITS dataset may contain extensions COMMENT FITS (Flexible Image Transport System) format is defined in 'AstronomyCOMMENT and Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H END XTENSION= 'BINTABLE' / binary table extension BITPIX = 8 / 8-bit bytes NAXIS = 2 / 2-dimensional binary table NAXIS1 = 383 / width of table in bytes NAXIS2 = 4 / number of rows in table PCOUNT = 0 / size of special data area GCOUNT = 1 / one data group (required keyword) TFIELDS = 96 / number of fields in each row TTYPE1 = 'RELEASE ' / label for field 1 TFORM1 = 'I ' / data format of field: 2-byte INTEGER TTYPE2 = 'BRICKID ' / label for field 2 TFORM2 = 'J ' / data format of field: 4-byte INTEGER TTYPE3 = 'BRICKNAME' / label for field 3 TFORM3 = '8A ' / data format of field: ASCII Character TTYPE4 = 'OBJID ' / label for field 4 TFORM4 = 'J ' / data format of field: 4-byte INTEGER TTYPE5 = 'TYPE ' / label for field 5 TFORM5 = '4A ' / data format of field: ASCII Character TTYPE6 = 'RA ' / label for field 6 TFORM6 = 'D ' / data format of field: 8-byte DOUBLE TTYPE7 = 'RA_IVAR ' / label for field 7 TFORM7 = 'E ' / data format of field: 4-byte REAL TTYPE8 = 'DEC ' / label for field 8 TFORM8 = 'D ' / data format of field: 8-byte DOUBLE TTYPE9 = 'DEC_IVAR' / label for field 9 TFORM9 = 'E ' / data format of field: 4-byte REAL TTYPE10 = 'DCHISQ ' / label for field 10 TFORM10 = '5E ' / data format of field: 4-byte REAL TTYPE11 = 'EBV ' / label for field 11 TFORM11 = 'E ' / data format of field: 4-byte REAL TTYPE12 = 'FLUX_G ' / label for field 12 TFORM12 = 'E ' / data format of field: 4-byte REAL TTYPE13 = 'FLUX_R ' / label for field 13 TFORM13 = 'E ' / data format of field: 4-byte REAL TTYPE14 = 'FLUX_Z ' / label for field 14 TFORM14 = 'E ' / data format of field: 4-byte REAL TTYPE15 = 'FLUX_IVAR_G' / label for field 15 TFORM15 = 'E ' / data format of field: 4-byte REAL TTYPE16 = 'FLUX_IVAR_R' / label for field 16 TFORM16 = 'E ' / data format of field: 4-byte REAL TTYPE17 = 'FLUX_IVAR_Z' / label for field 17 TFORM17 = 'E ' / data format of field: 4-byte REAL TTYPE18 = 'MW_TRANSMISSION_G' / label for field 18 TFORM18 = 'E ' / data format of field: 4-byte REAL TTYPE19 = 'MW_TRANSMISSION_R' / label for field 19 TFORM19 = 'E ' / data format of field: 4-byte REAL TTYPE20 = 'MW_TRANSMISSION_Z' / label for field 20 TFORM20 = 'E ' / data format of field: 4-byte REAL TTYPE21 = 'FRACFLUX_G' / label for field 21 TFORM21 = 'E ' / data format of field: 4-byte REAL TTYPE22 = 'FRACFLUX_R' / label for field 22 TFORM22 = 'E ' / data format of field: 4-byte REAL TTYPE23 = 'FRACFLUX_Z' / label for field 23 TFORM23 = 'E ' / data format of field: 4-byte REAL TTYPE24 = 'FRACMASKED_G' / label for field 24 TFORM24 = 'E ' / data format of field: 4-byte REAL TTYPE25 = 'FRACMASKED_R' / label for field 25 TFORM25 = 'E ' / data format of field: 4-byte REAL TTYPE26 = 'FRACMASKED_Z' / label for field 26 TFORM26 = 'E ' / data format of field: 4-byte REAL TTYPE27 = 'FRACIN_G' / label for field 27 TFORM27 = 'E ' / data format of field: 4-byte REAL TTYPE28 = 'FRACIN_R' / label for field 28 TFORM28 = 'E ' / data format of field: 4-byte REAL TTYPE29 = 'FRACIN_Z' / label for field 29 TFORM29 = 'E ' / data format of field: 4-byte REAL TTYPE30 = 'NOBS_G ' / label for field 30 TFORM30 = 'I ' / data format of field: 2-byte INTEGER TTYPE31 = 'NOBS_R ' / label for field 31 TFORM31 = 'I ' / data format of field: 2-byte INTEGER TTYPE32 = 'NOBS_Z ' / label for field 32 TFORM32 = 'I ' / data format of field: 2-byte INTEGER TTYPE33 = 'PSFDEPTH_G' / label for field 33 TFORM33 = 'E ' / data format of field: 4-byte REAL TTYPE34 = 'PSFDEPTH_R' / label for field 34 TFORM34 = 'E ' / data format of field: 4-byte REAL TTYPE35 = 'PSFDEPTH_Z' / label for field 35 TFORM35 = 'E ' / data format of field: 4-byte REAL TTYPE36 = 'GALDEPTH_G' / label for field 36 TFORM36 = 'E ' / data format of field: 4-byte REAL TTYPE37 = 'GALDEPTH_R' / label for field 37 TFORM37 = 'E ' / data format of field: 4-byte REAL TTYPE38 = 'GALDEPTH_Z' / label for field 38 TFORM38 = 'E ' / data format of field: 4-byte REAL TTYPE39 = 'FLUX_W1 ' / label for field 39 TFORM39 = 'E ' / data format of field: 4-byte REAL TTYPE40 = 'FLUX_W2 ' / label for field 40 TFORM40 = 'E ' / data format of field: 4-byte REAL TTYPE41 = 'FLUX_W3 ' / label for field 41 TFORM41 = 'E ' / data format of field: 4-byte REAL TTYPE42 = 'FLUX_W4 ' / label for field 42 TFORM42 = 'E ' / data format of field: 4-byte REAL TTYPE43 = 'FLUX_IVAR_W1' / label for field 43 TFORM43 = 'E ' / data format of field: 4-byte REAL TTYPE44 = 'FLUX_IVAR_W2' / label for field 44 TFORM44 = 'E ' / data format of field: 4-byte REAL TTYPE45 = 'FLUX_IVAR_W3' / label for field 45 TFORM45 = 'E ' / data format of field: 4-byte REAL TTYPE46 = 'FLUX_IVAR_W4' / label for field 46 TFORM46 = 'E ' / data format of field: 4-byte REAL TTYPE47 = 'MW_TRANSMISSION_W1' / label for field 47 TFORM47 = 'E ' / data format of field: 4-byte REAL TTYPE48 = 'MW_TRANSMISSION_W2' / label for field 48 TFORM48 = 'E ' / data format of field: 4-byte REAL TTYPE49 = 'MW_TRANSMISSION_W3' / label for field 49 TFORM49 = 'E ' / data format of field: 4-byte REAL TTYPE50 = 'MW_TRANSMISSION_W4' / label for field 50 TFORM50 = 'E ' / data format of field: 4-byte REAL TTYPE51 = 'ALLMASK_G' / label for field 51 TFORM51 = 'I ' / data format of field: 2-byte INTEGER TTYPE52 = 'ALLMASK_R' / label for field 52 TFORM52 = 'I ' / data format of field: 2-byte INTEGER TTYPE53 = 'ALLMASK_Z' / label for field 53 TFORM53 = 'I ' / data format of field: 2-byte INTEGER TTYPE54 = 'FRACDEV ' / label for field 54 TFORM54 = 'E ' / data format of field: 4-byte REAL TTYPE55 = 'FRACDEV_IVAR' / label for field 55 TFORM55 = 'E ' / data format of field: 4-byte REAL TTYPE56 = 'SHAPEDEV_R' / label for field 56 TFORM56 = 'E ' / data format of field: 4-byte REAL TTYPE57 = 'SHAPEDEV_E1' / label for field 57 TFORM57 = 'E ' / data format of field: 4-byte REAL TTYPE58 = 'SHAPEDEV_E2' / label for field 58 TFORM58 = 'E ' / data format of field: 4-byte REAL TTYPE59 = 'SHAPEDEV_R_IVAR' / label for field 59 TFORM59 = 'E ' / data format of field: 4-byte REAL TTYPE60 = 'SHAPEDEV_E1_IVAR' / label for field 60 TFORM60 = 'E ' / data format of field: 4-byte REAL TTYPE61 = 'SHAPEDEV_E2_IVAR' / label for field 61 TFORM61 = 'E ' / data format of field: 4-byte REAL TTYPE62 = 'SHAPEEXP_R' / label for field 62 TFORM62 = 'E ' / data format of field: 4-byte REAL TTYPE63 = 'SHAPEEXP_E1' / label for field 63 TFORM63 = 'E ' / data format of field: 4-byte REAL TTYPE64 = 'SHAPEEXP_E2' / label for field 64 TFORM64 = 'E ' / data format of field: 4-byte REAL TTYPE65 = 'SHAPEEXP_R_IVAR' / label for field 65 TFORM65 = 'E ' / data format of field: 4-byte REAL TTYPE66 = 'SHAPEEXP_E1_IVAR' / label for field 66 TFORM66 = 'E ' / data format of field: 4-byte REAL TTYPE67 = 'SHAPEEXP_E2_IVAR' / label for field 67 TFORM67 = 'E ' / data format of field: 4-byte REAL TTYPE68 = 'FIBERFLUX_G' / label for field 68 TFORM68 = 'E ' / data format of field: 4-byte REAL TTYPE69 = 'FIBERFLUX_R' / label for field 69 TFORM69 = 'E ' / data format of field: 4-byte REAL TTYPE70 = 'FIBERFLUX_Z' / label for field 70 TFORM70 = 'E ' / data format of field: 4-byte REAL TTYPE71 = 'FIBERTOTFLUX_G' / label for field 71 TFORM71 = 'E ' / data format of field: 4-byte REAL TTYPE72 = 'FIBERTOTFLUX_R' / label for field 72 TFORM72 = 'E ' / data format of field: 4-byte REAL TTYPE73 = 'FIBERTOTFLUX_Z' / label for field 73 TFORM73 = 'E ' / data format of field: 4-byte REAL TTYPE74 = 'WISEMASK_W1' / label for field 74 TFORM74 = 'B ' / data format of field: BYTE TTYPE75 = 'WISEMASK_W2' / label for field 75 TFORM75 = 'B ' / data format of field: BYTE TTYPE76 = 'MASKBITS' / label for field 76 TFORM76 = 'I ' / data format of field: 2-byte INTEGER TTYPE77 = 'REF_ID ' / label for field 77 TFORM77 = 'K ' / data format of field: 8-byte INTEGER TTYPE78 = 'REF_CAT ' / label for field 78 TFORM78 = '2A ' / data format of field: ASCII Character TTYPE79 = 'GAIA_PHOT_G_MEAN_MAG' / label for field 79 TFORM79 = 'E ' / data format of field: 4-byte REAL TTYPE80 = 'GAIA_PHOT_G_MEAN_FLUX_OVER_ERROR' / label for field 80 TFORM80 = 'E ' / data format of field: 4-byte REAL TTYPE81 = 'GAIA_PHOT_BP_MEAN_MAG' / label for field 81 TFORM81 = 'E ' / data format of field: 4-byte REAL TTYPE82 = 'GAIA_PHOT_BP_MEAN_FLUX_OVER_ERROR' / label for field 82 TFORM82 = 'E ' / data format of field: 4-byte REAL TTYPE83 = 'GAIA_PHOT_RP_MEAN_MAG' / label for field 83 TFORM83 = 'E ' / data format of field: 4-byte REAL TTYPE84 = 'GAIA_PHOT_RP_MEAN_FLUX_OVER_ERROR' / label for field 84 TFORM84 = 'E ' / data format of field: 4-byte REAL TTYPE85 = 'GAIA_PHOT_BP_RP_EXCESS_FACTOR' / label for field 85 TFORM85 = 'E ' / data format of field: 4-byte REAL TTYPE86 = 'GAIA_ASTROMETRIC_EXCESS_NOISE' / label for field 86 TFORM86 = 'E ' / data format of field: 4-byte REAL TTYPE87 = 'GAIA_DUPLICATED_SOURCE' / label for field 87 TFORM87 = 'L ' / data format of field: 1-byte LOGICAL TTYPE88 = 'GAIA_ASTROMETRIC_SIGMA5D_MAX' / label for field 88 TFORM88 = 'E ' / data format of field: 4-byte REAL TTYPE89 = 'GAIA_ASTROMETRIC_PARAMS_SOLVED' / label for field 89 TFORM89 = 'B ' / data format of field TZERO89 = -128 / offset for signed bytes TSCAL89 = 1 / data are not scaled TTYPE90 = 'PARALLAX' / label for field 90 TFORM90 = 'E ' / data format of field: 4-byte REAL TTYPE91 = 'PARALLAX_IVAR' / label for field 91 TFORM91 = 'E ' / data format of field: 4-byte REAL TTYPE92 = 'PMRA ' / label for field 92 TFORM92 = 'E ' / data format of field: 4-byte REAL TTYPE93 = 'PMRA_IVAR' / label for field 93 TFORM93 = 'E ' / data format of field: 4-byte REAL TTYPE94 = 'PMDEC ' / label for field 94 TFORM94 = 'E ' / data format of field: 4-byte REAL TTYPE95 = 'PMDEC_IVAR' / label for field 95 TFORM95 = 'E ' / data format of field: 4-byte REAL TTYPE96 = 'PHOTSYS ' / label for field 96 TFORM96 = '1A ' / data format of field: ASCII Character ABITN0 = 'BADPIX ' ABITN1 = 'SATUR ' ABITN11 = 'OUTLIER ' ABITN2 = 'INTERP ' ABITN4 = 'CR ' ABITN6 = 'BLEED ' ABITN7 = 'TRANS ' ABITN8 = 'EDGE ' ABITN9 = 'EDGE2 ' APRAD0 = 0.5 APRAD1 = 0.75 APRAD2 = 1. APRAD3 = 1.5 APRAD4 = 2. APRAD5 = 3.5 APRAD6 = 5. APRAD7 = 7. ARRAY_ID= 'none ' BAND0 = 'g ' BAND1 = 'r ' BAND2 = 'z ' BANDS = 'grz ' BBITN0 = 'BRIGHT ' BBITN1 = 'MEDIUM ' BBITN2 = 'CLUSTER ' BBITN3 = 'GALAXY ' BRICK_G = T BRICK_R = T BRICK_Z = T CAMS_G = 'decam ' CAMS_R = 'decam ' CAMS_Z = 'decam ' COMMENT BRIGHTBLOB bit values: DECMAX = 0. DECMIN = -5. DEPNAM00= 'desiconda' DEPNAM01= 'astropy ' DEPNAM02= 'matplotlib' DEPNAM03= 'mkl ' DEPNAM04= 'numpy ' DEPNAM05= 'python ' DEPNAM06= 'scipy ' DEPNAM07= 'astrometry-path' DEPNAM08= 'astrometry' DEPNAM09= 'tractor-path' DEPNAM10= 'tractor ' DEPNAM11= 'fitsio-path' DEPNAM12= 'fitsio ' DEPNAM13= 'TYCHO2_KD' DEPNAM14= 'GAIA_CAT' DEPNAM15= 'LARGEGALAXIES' DEPNAM16= 'WISE_PSF' DEPNAM17= 'unwise ' DEPNAM18= 'unwise_tr' DEPNAM19= 'unwise_modelsky' DEPVER00= '20190311-1.2.7-img' DEPVER01= '2.0.9 ' DEPVER02= '2.1.2 ' DEPVER03= 2019.1 DEPVER04= '1.16.2 ' DEPVER05= '3.6.6 ' DEPVER06= '1.2.1 ' DEPVER07= '/usr/local/lib/python/astrometry' DEPVER08= '0.78-2-g23579ec6' DEPVER09= '/usr/local/lib/python/tractor' DEPVER10= 'dr8.1 ' DEPVER11= '/usr/local/lib/python3.5/dist-packages/fitsio' DEPVER12= '0.9.12 ' DEPVER13= '/global/cscratch1/sd/landriau/dr8/alldata/tycho2' DEPVER14= '/global/cscratch1/sd/landriau/dr8/alldata/chunks-gaia-dr2-astrom-2' DEPVER15= '/global/cscratch1/sd/landriau/dr8/alldata/largegalaxies_v2.0' DEPVER16= '/src/unwise_psf/etc' DEPVER17= '/global/cscratch1/sd/landriau/dr8/alldata/unwise_coadds_4:/global/c&'DEPVER18= '/global/cscratch1/sd/landriau/dr8/alldata/unwise_timeresolved_coadds'DEPVER19= '/global/cscratch1/sd/landriau/dr8/calib/wise/modelsky' DRVERSIO= 8000 HOSTNAME= 'cori ' LONGSTRN= 'OGIP 1.0' LSDIR = '/global/cscratch1/sd/landriau/dr8' LSDR = 'DR8 ' MASKB0 = 'badpix ' MASKB1 = 'satur ' MASKB11 = 'outlier ' MASKB2 = 'interp ' MASKB4 = 'cr ' MASKB6 = 'bleed ' MASKB7 = 'trans ' MASKB8 = 'edge ' MASKB9 = 'edge2 ' MBITN0 = 'NPRIMARY' MBITN1 = 'BRIGHT ' MBITN10 = 'BAILOUT ' MBITN11 = 'MEDIUM ' MBITN12 = 'GALAXY ' MBITN13 = 'CLUSTER ' MBITN2 = 'SATUR_G ' MBITN3 = 'SATUR_R ' MBITN4 = 'SATUR_Z ' MBITN5 = 'ALLMASK_G' MBITN6 = 'ALLMASK_R' MBITN7 = 'ALLMASK_Z' MBITN8 = 'WISEM1 ' MBITN9 = 'WISEM2 ' NBANDS = 3 OBSTYPE = 'object ' PROCTYPE= 'tile ' PRODTYPE= 'catalog ' RAMAX = 200. RAMIN = 190. SURVEY = 'DECaLS+BASS+MzLS' SURVEYID= 'DECaLS BASS MzLS' UNWISD1 = '/global/cscratch1/sd/landriau/dr8/alldata/unwise_coadds_4' UNWISD2 = '/global/cscratch1/sd/landriau/dr8/alldata/unwise_coadds_1' UNWISSKY= '/global/cscratch1/sd/landriau/dr8/calib/wise/modelsky' UNWISTD = '/global/cscratch1/sd/landriau/dr8/alldata/unwise_timeresolved_coadds'WBITN0 = 'BRIGHT ' WBITN1 = 'SPIKE ' WBITN2 = 'GHOST ' WBITN3 = 'LATENT ' WBITN4 = 'LATENT2 ' WBITN5 = 'HALO ' WBITN6 = 'SATUR ' WBITN7 = 'SPIKE2 ' WISEAB1 = 2.699 WISEAB2 = 3.339 WISEAB3 = 5.174 WISEAB4 = 6.62 END @ z1906p000;DEV @gG×, k?eI͠KMLL)+L]<%G0GGn=< J;?ne?tO?y8B= < <> > >>$o>#r=piDaC訹C*D`CuBkGFH?Gę;a:296~4?~B?\????CPKO>OyGzMdMd8A B09BA %B0uB<ĖL2FS@=1913m005 -DEV @g#h݋)KKdN`o`LT+Mr= d>>4>DCBi RD03C"B -6HkIDHѠI@OIL9=2;7K5+?~y?WD??e??B!=C?vLڲOOm>C(CxDDC(CxDD vTL2FS@1961m007 FPSF @h#V" -YW8O+8Zv,J<Gv[G?.GnU:|Ie9 8e?q7?v8?z{?02?w?j?q??M]nIMIHb-G:;;Kt:|:7M/?'m?z??FG=IG_GLG=IG_GL3'~ G2@]EP@累D T@z?Cؠ?F=yj@~C䳢@MBCZS@d1931m012 DEV @h"^n0nKU7LNMN-=\]GpH.gH=@<;#>??D_CD]C4}dH?*G]H$/Gf8'), ('DEC', '>f8'), ('IN_RADIUS', '>f4'), - ('E1', '>f4'), ('E2', '>f4'), ('TYPE', 'S4')]) + # ADM pick a faint maglim (as unit tests deal with few objects). + cls.maglim = 20. + # ADM also pick a reasonable epoch at which to make the mask. + cls.maskepoch = 2025.5 + + # ADM an example mask, made from all of the test HEALPixels. + cls.allmx = brightmask.make_bright_star_mask( + numproc=1, nside=cls.nside, pixels=cls.pixnum, + maglim=cls.maglim, maskepoch=cls.maskepoch) + + # ADM read in some targets. + targdir = resource_filename(testdir, 't') + fn = os.path.join(targdir, 'sweep-320m005-330p000.fits') + ts = fitsio.read(fn) + # ADM targets are really sweeps objects, so add target fields. + zs = np.zeros(len(ts)) + targs = rfn.append_fields(ts, ["DESI_TARGET", "TARGETID"], [zs, zs], + usemask=False, dtypes='>i8') + cls.targs = rfn.append_fields(targs, "BRICK_OBJID", zs, usemask=False, + dtypes='>i4') + cls.targs["BRICK_OBJID"] = cls.targs["OBJID"] + + # ADM invent a mask with various testing properties. + cls.mask = np.zeros(3, dtype=brightmask.maskdatamodel.dtype) cls.mask["DEC"] = [0, 70, 35] cls.mask["IN_RADIUS"] = [1, 20, 10] cls.mask["E1"] = [0., 0., -0.3] cls.mask["E2"] = [0., 0., 0.5] - cls.mask["TYPE"] = ['REX', b'PSF ', 'EXP '] + cls.mask["TYPE"] = ['PSF', b'PSF ', 'PSF '] @classmethod def tearDownClass(cls): - # ADM remove any existing test files in this directory - if os.path.exists(cls.testmaskfile): - os.remove(cls.testmaskfile) - if os.path.exists(cls.testtargfile): - os.remove(cls.testtargfile) - if os.path.exists(cls.testbsfile): - os.remove(cls.testbsfile) + # ADM remove the temporary output directory. + if os.path.exists(cls.maskdir): + shutil.rmtree(cls.maskdir) + # ADM reset the environment variables. if cls.gaiadir_orig is not None: os.environ["GAIA_DIR"] = cls.gaiadir_orig @@ -99,77 +95,153 @@ def tearDownClass(cls): def test_make_bright_star_mask(self): """Test the construction of a bright star mask. """ - mask = make_bright_star_mask_in_hp(cls.nside, cls.pixnum[0], maglim=20.) + # ADM test making the mask in an individual pixel. + mx = brightmask.make_bright_star_mask_in_hp( + self.nside, self.pixnum[0], + maglim=self.maglim, maskepoch=self.maskepoch) + + # ADM check that running across all pixels contains the subset + # ADM of masks in the single pixel. + self.assertTrue(len(set(mx["REF_ID"]) - set(self.allmx["REF_ID"])) == 0) + self.assertTrue(len(set(self.allmx["REF_ID"]) - set(mx["REF_ID"])) > 0) + + def test_make_bright_star_mask_parallel(self): + """Check running the mask-making code in parallel. + """ + # ADM run on two processors. + two = brightmask.make_bright_star_mask( + numproc=2, nside=self.nside, pixels=self.pixnum, + maglim=self.maglim, maskepoch=self.maskepoch) + + # ADM check that running in parallel recovers the same masks as + # ADM running on one processor. + one = self.allmx[np.argsort(self.allmx["REF_ID"])] + two = two[np.argsort(two["REF_ID"])] + self.assertTrue(np.all(one == two)) + + def test_mask_write(self): + """Test that masks are written to file correctly. + """ + # ADM some meaningless magnitude limits and mask epochs. + ml, me = 62.3, 2062.3 + # ADM a keyword dictionary to write to the output file header. + extra = {'BLAT': 'blat', 'FOO': 'foo'} - self.assertTrue(np.all(mask1["TARGETID"] == mask2["TARGETID"])) + # ADM test writing without HEALPixel-split. + _, mxdir = io.write_masks(self.maskdir, self.allmx, maglim=ml, + maskepoch=me, extra=extra) + + # ADM test writing with HEALPixel-split. + _, mxdir = io.write_masks(self.maskdir, self.allmx, maglim=ml, + maskepoch=me, extra=extra, nside=self.nside) + + # ADM construct the output directory and file name. + mxd = io.find_target_files(self.maskdir, flavor="masks", + maglim=ml, epoch=me) + mxfn = io.find_target_files(self.maskdir, flavor="masks", + maglim=ml, epoch=me, hp=self.pixnum[0]) + + # ADM check the output directory is as expected. + self.assertEqual(mxdir, mxd) + + # ADM check all of the files were made in the correct place. + fns = glob(os.path.join(mxdir, "masks-hp*fits")) + self.assertEqual(len(fns), len(self.pixnum)+1) + + # ADM check the extra kwargs were written to the header. + for key in extra: + hdr = fitsio.read_header(mxfn, "MASKS") + self.assertEqual(hdr[key].rstrip(), extra[key]) def test_mask_targets(self): - """Test that targets in masks are flagged as being in masks + """Test that targets in masks are flagged accordingly. """ - # ADM mask the targets, creating the mask - targs = brightmask.mask_targets(self.masktargs, maglim=[8, 10], numproc=1, - rootdirname=self.bsdatadir, outfilename=self.testmaskfile) - self.assertTrue(np.any(targs["DESI_TARGET"] != 0)) + # ADM create the output mask directory. + _, mxdir = io.write_masks(self.maskdir, self.allmx, maglim=self.maglim, + maskepoch=self.maskepoch, nside=self.nside) + + # ADM make targets with the same coordinates as the masks. + # ADM remembering to select masks that actually have a radius. + ii = self.allmx["IN_RADIUS"] > 0 + targs = self.targs.copy() + targs["RA"] = self.allmx["RA"][ii][:len(targs)] + targs["DEC"] = self.allmx["DEC"][ii][:len(targs)] + + # ADM add mask information to DESI_TARGET. + mxt = brightmask.mask_targets(targs, mxdir, nside=self.nside, + pixlist=self.pixnum) + + # ADM all the targs should have been masked. + nmasked = np.sum(mxt["DESI_TARGET"] & desi_mask["IN_BRIGHT_OBJECT"] != 0) + self.assertEqual(nmasked, len(targs)) + + # ADM and we should have added some safe targets that will be + # ADM "near" bright objects. + is_nbo = mxt["DESI_TARGET"] & desi_mask["NEAR_BRIGHT_OBJECT"] != 0 + self.assertTrue(np.all(is_nbo)) def test_non_mask_targets(self): - """Test that targets that are NOT in masks are flagged as not being in masks + """Test targets that are NOT in masks are flagged accordingly. """ - # ADM write targs to file in order to check input file method - fitsio.write(self.testtargfile, self.unmasktargs, clobber=True) - # ADM create the mask and write it to file - mask = brightmask.make_bright_source_mask('RZ', [8, 10], - rootdirname=self.bsdatadir, outfilename=self.testmaskfile) - # ADM mask the targets, reading in the mask - targs = brightmask.mask_targets(self.testtargfile, inmaskfile=self.testmaskfile) + # ADM create the output mask directory. + _, mxdir = io.write_masks(self.maskdir, self.allmx, maglim=self.maglim, + maskepoch=self.maskepoch, nside=self.nside) + + # ADM update DESI_TARGET for any targets in masks. + mxtargs = brightmask.mask_targets(self.targs, mxdir, nside=self.nside, + pixlist=self.pixnum) - # ADM none of the targets should have been masked - self.assertTrue(np.all((targs["DESI_TARGET"] == 0) | ((targs["DESI_TARGET"] & desi_mask.BAD_SKY) != 0))) + # ADM none of the targets should be in a mask. + self.assertTrue(np.all(mxtargs["DESI_TARGET"] == 0)) def test_safe_locations(self): - """Test that SAFE/BADSKY locations are equidistant from mask centers + """Test SAFE/BADSKY locations are equidistant from mask centers. """ - # ADM append SAFE (BADSKY) locations around the perimeter of the mask - safes = brightmask.get_safe_targets(self.unmasktargs, self.mask) - targs = np.concatenate([self.unmasktargs, safes]) - # ADM restrict to just SAFE (BADSKY) locations + # ADM append SAFE locations around the perimeter of the mask. + safes = brightmask.get_safe_targets(self.targs, self.mask) + targs = np.concatenate([self.targs, safes]) + # ADM restrict to just SAFE locations. skybitset = ((targs["TARGETID"] & targetid_mask.SKY) != 0) safes = targs[np.where(skybitset)] - # ADM for each mask location check that every safe location is equidistant from the mask center + # ADM for each mask location check that every safe location is + # ADM equidistant from the mask center. c = SkyCoord(safes["RA"]*u.deg, safes["DEC"]*u.deg) for i in range(2): cent = SkyCoord(self.mask[i]["RA"]*u.deg, self.mask[i]["DEC"]*u.deg) sep = cent.separation(c) # ADM only things close to mask i w = np.where(sep < np.min(sep)*1.002) - # ADM are these all the same distance to a very high precision? - print("mask information", self.mask[i]) + # ADM are these all the same distance to high precision? self.assertTrue(np.max(sep[w] - sep[w[0]]) < 1e-15*u.deg) def test_targetid(self): - """Test SKY/RELEASE/BRICKID/OBJID are set correctly in TARGETID and DESI_TARGET for SAFE/BADSKY locations + """Test SKY/RELEASE/BRICKID/OBJID are set correctly in TARGETID + and DESI_TARGET for SAFE/BADSKY locations. """ - # ADM append SAFE (BADSKY) locations around the periphery of the mask - safes = brightmask.get_safe_targets(self.unmasktargs, self.mask) - targs = np.concatenate([self.unmasktargs, safes]) + # ADM append SAFE locations around the perimeter of the mask. + safes = brightmask.get_safe_targets(self.targs, self.mask) + targs = np.concatenate([self.targs, safes]) - # ADM first check that the SKY bit and BADSKY bits are appropriately set + # ADM first check the SKY and BADSKY bits are appropriately set. skybitset = ((targs["TARGETID"] & targetid_mask.SKY) != 0) badskybitset = ((targs["DESI_TARGET"] & desi_mask.BAD_SKY) != 0) self.assertTrue(np.all(skybitset == badskybitset)) # ADM now check that the other bits are in the correct locations - # ADM first restrict to just things in BRICK 330368 - w = np.where(targs["BRICKID"] == 330368) - targs = targs[w] + # ADM first restrict to the ~half-dozen targets in BRICK 521233. + bid = 521233 + ii = targs["BRICKID"] == bid + targs = targs[ii] - # ADM check that the TARGETIDs are unique - self.assertEqual(len(set(targs["TARGETID"])), len(targs["TARGETID"])) + # ADM check that the TARGETIDs are unique. + s = set(targs["TARGETID"]) + self.assertEqual(len(s), len(targs["TARGETID"])) - # ADM the targetids as a binary string + # ADM the TARGETIDs as a binary string. bintargids = [np.binary_repr(targid) for targid in targs["TARGETID"]] - # ADM check that the data release is set (in a way unlike the normal bit-setting in brightmask.py) - # ADM note that release should be zero for SAFE LOCATIONS + # ADM check the DR is set (in a way unlike the normal bit-setting + # in brightmask.py). Release should be zero for SAFE locations. rmostbit = targetid_mask.RELEASE.bitnum lmostbit = targetid_mask.RELEASE.bitnum + targetid_mask.RELEASE.nbits drbitset = int(bintargids[0][-lmostbit:-rmostbit], 2) @@ -177,11 +249,12 @@ def test_targetid(self): self.assertEqual(drbitset, drbitshould) self.assertEqual(drbitset, 0) - # ADM finally check that the BRICKIDs are all 330368 + # ADM check that the BRICKIDs are as restricted/requested. rmostbit = targetid_mask.BRICKID.bitnum lmostbit = targetid_mask.BRICKID.bitnum + targetid_mask.BRICKID.nbits - brickidset = np.array([int(bintargid[-lmostbit:-rmostbit], 2) for bintargid in bintargids]) - self.assertTrue(np.all(brickidset == 330368)) + brickidset = np.array( + [int(bintargid[-lmostbit:-rmostbit], 2) for bintargid in bintargids]) + self.assertTrue(np.all(brickidset == bid)) if __name__ == '__main__': From 2da07814fb32df5cc418e0aac0acae335508af4e Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 16 Jun 2020 15:46:24 -0700 Subject: [PATCH 24/25] update changes docs --- doc/changes.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/doc/changes.rst b/doc/changes.rst index 7956d1063..1c875d24b 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -5,6 +5,18 @@ desitarget Change Log 0.40.1 (unreleased) ------------------- +* Update masking to be "all-sky" using Gaia/Tycho/URAT [`PR #625`_]: + * General desitarget functionality to work with Tycho files. + * Deprecate using the sweeps to mask bright objects as this is now + being done using MASKBITS from the imaging catalogs. + * Functionality to allow masks to be built at different epochs, via + careful treatment of Tycho/Gaia/URAT proper motions. + * Bright star masks are now explicitly written to a $MASK_DIR. + * The radius-magnitude relationship is now a single function. + * Refactoring of unit tests to be simpler and have more coverage. + * Skies and supplemental skies are now always masked by default. + * A lack of backward compatibility, which should be OK as the masking + formalism wasn't being extensively used. * Functionality for iterations of SV beyond sv1 [`PR #624`_]. Includes: * A script to create the necessary files for new iterations of SV. * Generalized mask/cuts handling for survey=svX, X being any integer. @@ -21,6 +33,7 @@ desitarget Change Log .. _`PR #623`: https://github.com/desihub/desitarget/pull/623 .. _`PR #624`: https://github.com/desihub/desitarget/pull/624 +.. _`PR #625`: https://github.com/desihub/desitarget/pull/625 0.40.0 (2020-05-26) ------------------- From a4f2c051d0fc4c91889616ce798730dc89627fe8 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Wed, 17 Jun 2020 07:58:51 -0700 Subject: [PATCH 25/25] fix sphinx errors --- py/desitarget/brightmask.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/py/desitarget/brightmask.py b/py/desitarget/brightmask.py index ecc78bee6..d62b2b0a1 100644 --- a/py/desitarget/brightmask.py +++ b/py/desitarget/brightmask.py @@ -391,7 +391,7 @@ def make_bright_star_mask(maglim=12., matchrad=1., numproc=32, If passed, create a mask only in nested HEALPixels in `pixels` at this `nside`. Otherwise, run for the whole sky. If `nside` is passed then `pixels` must be passed too. - pixels :class:`list`, optional, defaults to ``None`` + pixels : :class:`list`, optional, defaults to ``None`` If passed, create a mask only in nested HEALPixels at `nside` for pixel integers in `pixels`. Otherwise, run for the whole sky. If `pixels` is passed then `nside` must be passed too. @@ -418,7 +418,7 @@ def make_bright_star_mask(maglim=12., matchrad=1., numproc=32, Notes ----- - - Runs (all-sky) in ~20 minutes for `numproc`=32 and `maglim`=12. + - Runs (all-sky) in ~20 minutes for `numproc=32` and `maglim=12`. - `IN_RADIUS` (`NEAR_RADIUS`) corresponds to `IN_BRIGHT_OBJECT` (`NEAR_BRIGHT_OBJECT`) in `data/targetmask.yaml`. These radii are set in the function `desitarget.brightmask.radius()`.