Merge pull request #229 from desihub/DR5cleanupADMrebase

General clean-up prior to running DR5 official targets

bin/make_star_mask

@@ -14,11 +14,13 @@ warnings.simplefilter('error')
 import multiprocessing
 nproc = multiprocessing.cpu_count() // 2
 
+from desiutil.log import get_logger
+log = get_logger()
+
 from argparse import ArgumentParser
 ap = ArgumentParser()
 ap.add_argument("src", help="root directory with tractor/sweeps files from which to build mask")
 ap.add_argument("dest", help='Output file name to which to write mask')
-ap.add_argument('-v', "--verbose", action='store_true')
 ap.add_argument("--bands", 
                 help='Bands to use to build the star mask (e.g. GRZ)',
                 default="GRZ")
@@ -35,13 +37,13 @@ infiles = io.list_sweepfiles(ns.src)
 if len(infiles) == 0:
     infiles = io.list_tractorfiles(ns.src)
 if len(infiles) == 0:
-    print('FATAL: no sweep or tractor files found')
+    log.critical('no sweep or tractor files found')
     sys.exit(1)
 
 maglim = [ float(ml) for ml in ns.maglim.split(',') ]
 
 starmask = make_bright_star_mask(ns.bands,maglim,numproc=ns.numproc,
-                                 rootdirname=ns.src,outfilename=ns.dest,verbose=ns.verbose)
+                                 rootdirname=ns.src,outfilename=ns.dest)
 
-print('wrote a file of {} masks to {}'.format(len(starmask), ns.dest))
+log.info('wrote a file of {} masks to {}'.format(len(starmask), ns.dest))
 

bin/select_targets

@@ -18,11 +18,13 @@ import multiprocessing
 nproc = multiprocessing.cpu_count() // 2
 nside = 64 #ADM default HEALPix Nside used throughout desitarget
 
+from desiutil.log import get_logger
+log = get_logger()
+
 from argparse import ArgumentParser
 ap = ArgumentParser()
 ap.add_argument("src", help="Tractor/sweeps file or root directory with tractor/sweeps files")
 ap.add_argument("dest", help="Output target selection file")
-ap.add_argument('-v', "--verbose", action='store_true')
 ap.add_argument('-s', "--skies", action='store_true',help="Generate blank sky positions in addition to targets")
 ap.add_argument('-c', "--check", action='store_true',help="Process tractor/sweeps files to check for corruption, without running full target selection")
 ap.add_argument('-m', "--mask", help="If sent then mask the targets, the name of the mask file should be supplied")
@@ -42,15 +44,15 @@ infiles = io.list_sweepfiles(ns.src)
 if len(infiles) == 0:
     infiles = io.list_tractorfiles(ns.src)
 if len(infiles) == 0:
-    print('FATAL: no sweep or tractor files found')
+    log.critical('no sweep or tractor files found')
     sys.exit(1)
 
 if ns.check:
-    print('Check input files...')
-    nbadfiles = check_input_files(infiles, numproc=ns.numproc, verbose=ns.verbose)
-    print('{} potentially corrupt files'.format(nbadfiles))
+    log.info('Check input files...')
+    nbadfiles = check_input_files(infiles, numproc=ns.numproc)
+    log.info('{} potentially corrupt files'.format(nbadfiles))
 else:
-    targets = select_targets(infiles, numproc=ns.numproc, verbose=ns.verbose,
+    targets = select_targets(infiles, numproc=ns.numproc,
                              qso_selection=ns.qsoselection, sandbox=ns.sandbox, FoMthresh=ns.FoMthresh, Method=ns.Method)
     if ns.mask:
         targets = mask_targets(targets, instarmaskfile=ns.mask, nside=nside)
@@ -68,5 +70,5 @@ else:
     io.write_targets(ns.dest, targets, indir=ns.src,
         qso_selection=ns.qsoselection, sandboxcuts=ns.sandbox, nside=nside)
 
-    print('{} targets written to {}'.format(len(targets), ns.dest))
+    log.info('{} targets written to {}'.format(len(targets), ns.dest))
 

doc/changes.rst

@@ -5,11 +5,16 @@ desitarget Change Log
 0.15.1 (unreleased)
 -------------------
 
+* General clean-up prior to running DR5 targets [`PR #229`_].
+   * Use :mod:`desiutil.log` instead of verbose (everywhere except mocks)
+   * Change ``HEALPix`` references to header keywords instead of dependencies
+   * Include ``SUBPRIORITY`` and shape parameter ``IVARs`` in target outputs
 * Include GMM model data for mocks when installing [`PR #222`_].
 * Initial simplistic code for generating sky positions [`PR #220`_]
 
 .. _`PR #220`: https://github.com/desihub/desitarget/pull/220
 .. _`PR #222`: https://github.com/desihub/desitarget/pull/222
+.. _`PR #229`: https://github.com/desihub/desitarget/pull/229
 
 0.15.0 (2017-09-29)
 -------------------

py/desitarget/QA.py

@@ -455,6 +455,11 @@ def model_map(brickfilename,plot=False):
         target density fluctuations and x would be the DEPTH or EBV value.
 
     """
+
+    #ADM set up the default logger
+    from desiutil.log import get_logger
+    log = get_logger()
+
     flucmap = fluc_map(brickfilename)
 
     #ADM the percentiles to consider for "mean" and "sigma
@@ -472,7 +477,7 @@ def model_map(brickfilename,plot=False):
             for fcol in cols:
                 if re.search("FLUC",fcol):
                     if plot:
-                        print("doing",col,fcol)
+                        log.info("doing",col,fcol)
                     quadparams = fit_quad(flucmap[col],flucmap[fcol],plot=plot)
                     #ADD this to the dictionary
                     coldict = dict({fcol:quadparams},**coldict)
@@ -644,17 +649,21 @@ def mag_histogram(targetfilename,binsize,outfile):
     :class:`Nonetype`
         No return...but prints a raw N(m) to screen for each target type
     """
+
+    #ADM set up the default logger
+    from desiutil.log import get_logger
+    log = get_logger()
 
     #ADM read in target file
-    print('Reading in targets file')
+    log.info('Reading in targets file')
     fx = fitsio.FITS(targetfilename, upper=True)
     targetdata = fx[1].read(columns=['BRICKID','DESI_TARGET','BGS_TARGET','MWS_TARGET','DECAM_FLUX'])
 
     #ADM open output file for writing
     file = open(outfile, "w")
 
     #ADM calculate the magnitudes of interest
-    print('Calculating magnitudes')
+    log.info('Calculating magnitudes')
     gfluxes = targetdata["DECAM_FLUX"][...,1]
     gmags = 22.5-2.5*np.log10(gfluxes*(gfluxes  > 1e-5) + 1e-5*(gfluxes < 1e-5))
     rfluxes = targetdata["DECAM_FLUX"][...,2]
@@ -670,7 +679,7 @@ def mag_histogram(targetfilename,binsize,outfile):
 
     #ADM loop through bits and print histogram of raw target numbers per magnitude
     for i, bitval in enumerate(bitvals):
-        print('Doing',bitnames[i])
+        log.info('Doing',bitnames[i])
         w = np.where(targetdata["DESI_TARGET"] & bitval)
         if len(w[0]):
             ghist,dum = np.histogram(gmags[w],bins=binedges)
@@ -679,7 +688,7 @@ def mag_histogram(targetfilename,binsize,outfile):
             file.write('{}    {}     {}     {}\n'.format(bitnames[i],'g','r','z'))
             for i in range(len(binedges)-1):
                 outs = '{:.1f} {} {} {}\n'.format(0.5*(binedges[i]+binedges[i+1]),ghist[i],rhist[i],zhist[i])
-                print(outs)
+                log.info(outs)
                 file.write(outs)
 
     file.close()
@@ -1253,34 +1262,38 @@ def brick_info(targetfilename,rootdirname='/global/project/projectdirs/cosmo/dat
          numpy structured array of brick information with columns as in construct_QA_file
     """
 
+    #ADM set up the default logger
+    from desiutil.log import get_logger
+    log = get_logger()
+
     start = time()
 
     #ADM read in target file
-    print('Reading in target file...t = {:.1f}s'.format(time()-start))
+    log.info('Reading in target file...t = {:.1f}s'.format(time()-start))
     fx = fitsio.FITS(targetfilename, upper=True)
     targetdata = fx[1].read(columns=['BRICKID','DESI_TARGET','BGS_TARGET','MWS_TARGET'])
 
     #ADM add col
 
-    print('Determining unique bricks...t = {:.1f}s'.format(time()-start))
+    log.info('Determining unique bricks...t = {:.1f}s'.format(time()-start))
     #ADM determine number of unique bricks and their integer IDs
     brickids = np.array(list(set(targetdata['BRICKID'])))
     brickids.sort()
 
-    print('Creating output brick structure...t = {:.1f}s'.format(time()-start))
+    log.info('Creating output brick structure...t = {:.1f}s'.format(time()-start))
     #ADM set up an output structure of size of the number of unique bricks
     nbricks = len(brickids)
     outstruc = construct_QA_file(nbricks)
 
-    print('Adding brick information...t = {:.1f}s'.format(time()-start))
+    log.info('Adding brick information...t = {:.1f}s'.format(time()-start))
     #ADM add brick-specific information based on the brickids
     outstruc = populate_brick_info(outstruc,brickids,rootdirname)
 
-    print('Adding depth information...t = {:.1f}s'.format(time()-start))
+    log.info('Adding depth information...t = {:.1f}s'.format(time()-start))
     #ADM add per-brick depth and area information
     outstruc = populate_depths(outstruc,rootdirname)
 
-    print('Adding target density information...t = {:.1f}s'.format(time()-start))
+    log.info('Adding target density information...t = {:.1f}s'.format(time()-start))
     #ADM bits and names of interest for desitarget
     #ADM -1 as a bit will return all values
     bitnames = ["DENSITY_ALL","DENSITY_LRG","DENSITY_ELG",
@@ -1294,11 +1307,11 @@ def brick_info(targetfilename,rootdirname='/global/project/projectdirs/cosmo/dat
             targsperbrick = np.bincount(targetdata[w]['BRICKID'])
             outstruc[bitnames[i]] = targsperbrick[outstruc['BRICKID']]/outstruc['BRICKAREA']
 
-    print('Writing output file...t = {:.1f}s'.format(time()-start))
+    log.info('Writing output file...t = {:.1f}s'.format(time()-start))
     #ADM everything should be populated, just write it out
     fitsio.write(outfilename, outstruc, extname='BRICKINFO', clobber=True)
 
-    print('Done...t = {:.1f}s'.format(time()-start))
+    log.info('Done...t = {:.1f}s'.format(time()-start))
     return outstruc
 
 
@@ -1612,9 +1625,9 @@ def make_qa_plots(targs, qadir='.', targdens=None, max_bin_area=1.0, weight=True
     """
 
     #ADM set up the default logger from desiutil
-    from desimodel import io, footprint
     from desiutil.log import get_logger, DEBUG
     log = get_logger(DEBUG)
+    from desimodel import io, footprint
 
     start = time()
     log.info('Start making targeting QA density plots...t = {:.1f}s'.format(time()-start))

py/desitarget/brightstar.py

@@ -288,7 +288,7 @@ def sphere_circle_ra_off(theta,centdec,declocs):
     return  np.degrees(offrar)
 
 
-def collect_bright_stars(bands,maglim,numproc=4,rootdirname='/global/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1',outfilename=None,verbose=False):
+def collect_bright_stars(bands,maglim,numproc=4,rootdirname='/global/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1',outfilename=None):
     """Extract a structure from the sweeps containing only bright stars in a given band to a given magnitude limit
 
     Parameters
@@ -308,15 +308,16 @@ def collect_bright_stars(bands,maglim,numproc=4,rootdirname='/global/project/pro
         /global/project/projectdirs/cosmo/data/legacysurvey/dr3/sweeps/dr3.1
     outfilename : :class:`str`, optional, defaults to not writing anything to file
         (FITS) File name to which to write the output structure of bright stars
-    verbose : :class:`bool`, optional
-        Send to write progress to screen
 
     Returns
     -------
     :class:`recarray`
         The structure of bright stars from the sweeps limited in the passed band(s) to the
         passed maglim(s).
     """
+    #ADM set up default logger
+    from desiutil.log import get_logger
+    log = get_logger()
 
     #ADM use io.py to retrieve list of sweeps or tractor files
     infiles = io.list_sweepfiles(rootdirname)
@@ -359,16 +360,15 @@ def _get_bright_stars(filename):
     totfiles = np.ones((),dtype='i8')*len(infiles)
     nfiles = np.ones((), dtype='i8')
     t0 = time()
-    if verbose:
-        print('Collecting bright stars from sweeps...')
+    log.info('Collecting bright stars from sweeps...')
 
     def _update_status(result):
         '''wrapper function for the critical reduction operation,
         that occurs on the main parallel process'''
-        if verbose and nfiles%25 == 0:
+        if nfiles%25 == 0:
             elapsed = time() - t0
             rate = nfiles / elapsed
-            print('{}/{} files; {:.1f} files/sec; {:.1f} total mins elapsed'.format(nfiles, totfiles, rate, elapsed/60.))
+            log.info('{}/{} files; {:.1f} files/sec; {:.1f} total mins elapsed'.format(nfiles, totfiles, rate, elapsed/60.))
         nfiles[...] += 1  #this is an in-place modification
         return result
 
@@ -486,7 +486,7 @@ def model_bright_stars(band,instarfile,rootdirname='/global/project/projectdirs/
     return ldict, bdict
 
 
-def make_bright_star_mask(bands,maglim,numproc=4,rootdirname='/global/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1',infilename=None,outfilename=None,verbose=False):
+def make_bright_star_mask(bands,maglim,numproc=4,rootdirname='/global/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1',infilename=None,outfilename=None):
     """Make a bright star mask from a structure of bright stars drawn from the sweeps
 
     Parameters
@@ -510,8 +510,6 @@ def make_bright_star_mask(bands,maglim,numproc=4,rootdirname='/global/project/pr
         via a call to collect_bright_stars
     outfilename : :class:`str`, optional, defaults to not writing anything to file
         (FITS) File name to which to write the output bright star mask
-    verbose : :class:`bool`, optional
-        Send to write progress to screen
 
     Returns
     -------
@@ -552,7 +550,7 @@ def make_bright_star_mask(bands,maglim,numproc=4,rootdirname='/global/project/pr
     if infilename is not None:
         objs = io.read_tractor(infilename)
     else:
-        objs = collect_bright_stars(bands,maglim,numproc,rootdirname,outfilename,verbose)
+        objs = collect_bright_stars(bands,maglim,numproc,rootdirname,outfilename)
 
     #ADM write the fluxes and bands as arrays instead of named columns
     fluxes = objs[bandnames].view(objs[bandnames].dtype[0]).reshape(objs[bandnames].shape + (-1,))
@@ -902,7 +900,7 @@ def set_target_bits(targs,starmask):
     return desi_target
 
 
-def mask_targets(targs,instarmaskfile=None,nside=None,bands="GRZ",maglim=[10,10,10],numproc=4,rootdirname='/global/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1',outfilename=None,verbose=False,drbricks=None):
+def mask_targets(targs,instarmaskfile=None,nside=None,bands="GRZ",maglim=[10,10,10],numproc=4,rootdirname='/global/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1',outfilename=None,drbricks=None):
     """Add bits for whether objects are in a bright star mask, and SAFE (BADSKY) sky locations, to a list of targets
 
     Parameters
@@ -932,8 +930,6 @@ def mask_targets(targs,instarmaskfile=None,nside=None,bands="GRZ",maglim=[10,10,
     outfilename : :class:`str`, optional, defaults to not writing anything to file
         (FITS) File name to which to write the output bright star mask ONE OF outfilename or
         instarmaskfile MUST BE PASSED
-    verbose : :class:`bool`, optional
-        Send to write progress to screen
     drbricks : :class:`~numpy.ndarray`, optional
         A rec array containing at least the "release", "ra", "dec" and "nobjs" columns from a survey bricks file
         This is typically used for testing only.
@@ -952,6 +948,10 @@ def mask_targets(targs,instarmaskfile=None,nside=None,bands="GRZ",maglim=[10,10,
         - (not including 5-10 minutes to build the star mask from scratch)
     """
 
+    #ADM set up default logger
+    from desiutil.log import get_logger
+    log = get_logger()
+
     t0 = time()
 
     if instarmaskfile is None and outfilename is None:
@@ -966,20 +966,18 @@ def mask_targets(targs,instarmaskfile=None,nside=None,bands="GRZ",maglim=[10,10,
     #ADM check if a file for the bright star mask was passed, if not then create it
     if instarmaskfile is None:
         starmask = make_bright_star_mask(bands,maglim,numproc=numproc,
-                                         rootdirname=rootdirname,outfilename=outfilename,verbose=verbose)
+                                         rootdirname=rootdirname,outfilename=outfilename)
     else:
         starmask = fitsio.read(instarmaskfile)
 
-    if verbose:
-        ntargsin = len(targs)
-        print('Number of targets {}...t={:.1f}s'.format(ntargsin, time()-t0))
-        print('Number of star masks {}...t={:.1f}s'.format(len(starmask), time()-t0))
+    ntargsin = len(targs)
+    log.info('Number of targets {}...t={:.1f}s'.format(ntargsin, time()-t0))
+    log.info('Number of star masks {}...t={:.1f}s'.format(len(starmask), time()-t0))
 
     #ADM generate SAFE locations and add them to the target list
     targs = append_safe_targets(targs,starmask,nside=nside,drbricks=drbricks)
 
-    if verbose:
-        print('Generated {} SAFE (BADSKY) locations...t={:.1f}s'.format(len(targs)-ntargsin, time()-t0))
+    log.info('Generated {} SAFE (BADSKY) locations...t={:.1f}s'.format(len(targs)-ntargsin, time()-t0))
 
     #ADM update the bits depending on whether targets are in a mask
     dt = set_target_bits(targs,starmask)
@@ -992,11 +990,10 @@ def mask_targets(targs,instarmaskfile=None,nside=None,bands="GRZ",maglim=[10,10,
     if len(w[0]) > 0:
         done = done[w]
 
-    if verbose:
-        print("...of these, {} SAFE (BADSKY) locations aren't in masks...t={:.1f}s".format(len(done)-ntargsin, time()-t0))
+    log.info("...of these, {} SAFE (BADSKY) locations aren't in masks...t={:.1f}s"
+             .format(len(done)-ntargsin, time()-t0))
 
-    if verbose:
-        print('Finishing up...t={:.1f}s'.format(time()-t0))
+    log.info('Finishing up...t={:.1f}s'.format(time()-t0))
 
     return done