diff --git a/py/desispec/io/meta.py b/py/desispec/io/meta.py index a7ae1af49..6b22d6a5c 100755 --- a/py/desispec/io/meta.py +++ b/py/desispec/io/meta.py @@ -69,7 +69,7 @@ def get_readonly_filepath(filepath): return filepath def findfile(filetype, night=None, expid=None, camera=None, - tile=None, groupname=None, + tile=None, groupname=None, subgroup=None, healpix=None, nside=64, band=None, spectrograph=None, survey=None, faprogram=None, @@ -88,6 +88,7 @@ def findfile(filetype, night=None, expid=None, camera=None, camera : 'b0' 'r1' .. 'z9' tile : integer tile (pointing) number groupname : spectral grouping name (e.g. "healpix", "cumulative", "pernight") + subgroup : (str) subgrouping name for non-standard groupnames healpix : healpix pixel number nside : healpix nside band : one of 'b','r','z' identifying the camera band @@ -203,16 +204,16 @@ def findfile(filetype, night=None, expid=None, camera=None, # # spectra- tile based # - coadd_tile='{specprod_dir}/tiles/{groupname}/{tile:d}/{night}/coadd-{spectrograph:d}-{tile:d}-{nightprefix}{night}.fits', - rrdetails_tile='{specprod_dir}/tiles/{groupname}/{tile:d}/{night}/rrdetails-{spectrograph:d}-{tile:d}-{nightprefix}{night}.h5', - rrmodel_tile='{specprod_dir}/tiles/{groupname}/{tile:d}/{night}/rrmodel-{spectrograph:d}-{tile:d}-{nightprefix}{night}.fits', - spectra_tile='{specprod_dir}/tiles/{groupname}/{tile:d}/{night}/spectra-{spectrograph:d}-{tile:d}-{nightprefix}{night}.fits.gz', - redrock_tile='{specprod_dir}/tiles/{groupname}/{tile:d}/{night}/redrock-{spectrograph:d}-{tile:d}-{nightprefix}{night}.fits', - qso_mgii_tile='{specprod_dir}/tiles/{groupname}/{tile:d}/{night}/qso_mgii-{spectrograph:d}-{tile:d}-{nightprefix}{night}.fits', - qso_qn_tile='{specprod_dir}/tiles/{groupname}/{tile:d}/{night}/qso_qn-{spectrograph:d}-{tile:d}-{nightprefix}{night}.fits', - emline_tile='{specprod_dir}/tiles/{groupname}/{tile:d}/{night}/emline-{spectrograph:d}-{tile:d}-{nightprefix}{night}.fits', + coadd_tile='{specprod_dir}/tiles/{groupname}/{tile:d}/{subgroup}/coadd-{spectrograph:d}-{tile:d}-{nightprefix}{subgroup}.fits', + rrdetails_tile='{specprod_dir}/tiles/{groupname}/{tile:d}/{subgroup}/rrdetails-{spectrograph:d}-{tile:d}-{nightprefix}{subgroup}.h5', + rrmodel_tile='{specprod_dir}/tiles/{groupname}/{tile:d}/{subgroup}/rrmodel-{spectrograph:d}-{tile:d}-{nightprefix}{subgroup}.fits', + spectra_tile='{specprod_dir}/tiles/{groupname}/{tile:d}/{subgroup}/spectra-{spectrograph:d}-{tile:d}-{nightprefix}{subgroup}.fits.gz', + redrock_tile='{specprod_dir}/tiles/{groupname}/{tile:d}/{subgroup}/redrock-{spectrograph:d}-{tile:d}-{nightprefix}{subgroup}.fits', + qso_mgii_tile='{specprod_dir}/tiles/{groupname}/{tile:d}/{subgroup}/qso_mgii-{spectrograph:d}-{tile:d}-{nightprefix}{subgroup}.fits', + qso_qn_tile='{specprod_dir}/tiles/{groupname}/{tile:d}/{subgroup}/qso_qn-{spectrograph:d}-{tile:d}-{nightprefix}{subgroup}.fits', + emline_tile='{specprod_dir}/tiles/{groupname}/{tile:d}/{subgroup}/emline-{spectrograph:d}-{tile:d}-{nightprefix}{subgroup}.fits', # - # spectra- single exp tile based + # spectra- single exp tile based requires cusom formatting for expid:08d # coadd_single='{specprod_dir}/tiles/perexp/{tile:d}/{expid:08d}/coadd-{spectrograph:d}-{tile:d}-exp{expid:08d}.fits', rrdetails_single='{specprod_dir}/tiles/perexp/{tile:d}/{expid:08d}/rrdetails-{spectrograph:d}-{tile:d}-exp{expid:08d}.h5', @@ -267,12 +268,18 @@ def findfile(filetype, night=None, expid=None, camera=None, ): groupname = 'cumulative' - if str(groupname) == "cumulative": + if groupname == "cumulative": nightprefix = "thru" - elif groupname == 'perexp': + subgroup = str(night) + elif groupname == "pernight": + nightprefix = "" + subgroup = str(night) + elif groupname == "perexp": nightprefix = "exp" - else: + elif groupname == "healpix": nightprefix = "" + else: + nightprefix = str(groupname)+'-' #- backwards compatibility: try interpreting groupname as a healpix number if healpix is None and tile is None and groupname is not None: @@ -297,6 +304,9 @@ def findfile(filetype, night=None, expid=None, camera=None, if isinstance(tile, str): tile = int(tile) if isinstance(spectrograph, str): spectrograph = int(spectrograph) + #- Determine if this is healpix-based or tile-based objects, and update + #- location dict for which flavor of coadd/spectra/redrock/etc is needed, + #- removing the _hp, _single, _tile suffixes from the keys loc_copy = location.copy() if tile is not None: log.debug("Tile-based files selected; healpix-based files and input will be ignored.") @@ -314,6 +324,11 @@ def findfile(filetype, night=None, expid=None, camera=None, if key.endswith('_tile'): root_key = key.removesuffix('_tile') location[root_key] = val + + ## cumulative and pernight use night as subgroup + if groupname in ('cumulative', 'pernight'): + subgroup = night + else: ## If not tile based then use the hp naming scheme ## Do loop to improve scaling with additional file types @@ -372,7 +387,8 @@ def findfile(filetype, night=None, expid=None, camera=None, actual_inputs = { 'specprod_dir':specprod_dir, 'specprod':specprod, 'qaprod_dir':qaprod_dir, 'tiles_dir':tiles_dir, - 'night':night, 'expid':expid, 'tile':tile, 'camera':camera, 'groupname':groupname, + 'night':night, 'expid':expid, 'tile':tile, 'camera':camera, + 'groupname':groupname, 'subgroup':subgroup, 'healpix':healpix, 'nside':nside, 'hpixdir':hpixdir, 'band':band, 'spectrograph':spectrograph, 'nightprefix':nightprefix, 'month':month } @@ -385,9 +401,17 @@ def findfile(filetype, night=None, expid=None, camera=None, if 'rawdata_dir' in required_inputs: actual_inputs['rawdata_dir'] = rawdata_dir + #- If any inputs missing, print all missing inputs, then raise single ValueError + missing_inputs = False for i in required_inputs: if actual_inputs[i] is None: - raise ValueError("Required input '{0}' is not set for type '{1}'!".format(i,filetype)) + log.error("Required input '{0}' is not set for type '{1}'!".format(i,filetype)) + missing_inputs = True + + if missing_inputs: + msg = f"Missing inputs for {location[filetype]}" + log.critical(msg) + raise ValueError(msg) #- normpath to remove extraneous double slashes /a/b//c/d filepath = os.path.normpath(location[filetype].format(**actual_inputs)) diff --git a/py/desispec/scripts/zproc.py b/py/desispec/scripts/zproc.py index 8311ffcd9..f8f07c294 100644 --- a/py/desispec/scripts/zproc.py +++ b/py/desispec/scripts/zproc.py @@ -58,7 +58,7 @@ def parse(options=None): parser = argparse.ArgumentParser(usage="{prog} [options]") parser.add_argument("-g", "--groupname", type=str, - help="Redshift grouping type: cumulative, perexp, pernight, healpix") + help="Redshift grouping type: cumulative, perexp, pernight, healpix, or custom name") #- Options for tile-based redshifts tiles_options = parser.add_argument_group("tile-based options (--groupname perexp, pernight, or cumulative)") @@ -74,6 +74,8 @@ def parse(options=None): tiles_options.add_argument("-c", "--cameras", type=str, help="Subset of cameras to process, either as a camword (e.g. a012)" + "Or a comma separated list (e.g. b0,r0,z0).") + parser.add_argument("--subgroup", type=str, + help="subgroup to use for non-standard groupname values") #- Options for healpix-based redshifts healpix_options = parser.add_argument_group("healpix-based options (--groupname healpix)") @@ -225,6 +227,21 @@ def main(args=None, comm=None): raise ValueError(msg) + ## Unpack arguments for shorter names (tileid might be None, ok) + tileid, groupname, subgroup = args.tileid, args.groupname, args.subgroup + + known_groups = ['cumulative', 'pernight', 'perexp', 'healpix'] + if groupname not in known_groups: + if subgroup is None: + msg = f'Non-standard --groupname={groupname} requires --subgroup' + if rank == 0: + log.critical(msg) + raise ValueError(msg) + else: + msg = f'Non-standard {groupname=} not in {known_groups}; using {subgroup=}' + if rank == 0: + log.warning(msg) + #- redrock non-MPI mode isn't compatible with GPUs, #- so if zproc is running in non-MPI mode, force --no-gpu #- https://github.com/desihub/redrock/issues/223 @@ -276,15 +293,6 @@ def main(args=None, comm=None): else: camword = create_camword(args.cameras) - ## Unpack arguments for shorter names (tileid might be None, ok) - tileid, groupname = args.tileid, args.groupname - - known_groups = ['cumulative', 'pernight', 'perexp', 'healpix'] - if groupname not in known_groups: - msg = 'obstype {} not in {}'.format(groupname, known_groups) - log.error(msg) - raise ValueError(msg) - if args.batch: err = 0 #------------------------------------------------------------------------- @@ -293,6 +301,7 @@ def main(args=None, comm=None): ## create the batch script cmdline = list(sys.argv).copy() scriptfile = create_desi_zproc_batch_script(group=groupname, + subgroup=subgroup, tileid=tileid, cameras=camword, thrunight=args.thrunight, @@ -395,6 +404,9 @@ def main(args=None, comm=None): if rank == 0: log.info('------------------------------') log.info('Groupname {}'.format(groupname)) + if subgroup is not None: + log.info('Subgroup {}'.format(subgroup)) + if args.healpix is not None: log.info(f'Healpixels {args.healpix}') else: @@ -443,10 +455,14 @@ def main(args=None, comm=None): if groupname == 'healpix': findfileopts = dict(groupname=groupname, survey=args.survey, faprogram=args.program) else: - findfileopts = dict(night=thrunight, tile=tileid, groupname=groupname) - if groupname == 'perexp': + findfileopts = dict(tile=tileid, groupname=groupname, subgroup=subgroup) + if groupname in ('cumulative', 'pernight'): + findfileopts['night'] = thrunight + elif groupname == 'perexp': assert len(expids) == 1 findfileopts['expid'] = expids[0] + elif subgroup is not None: + findfileopts['subgroup'] = subgroup timer.stop('preflight') diff --git a/py/desispec/test/test_io.py b/py/desispec/test/test_io.py index 2302964a8..7656e7c62 100644 --- a/py/desispec/test/test_io.py +++ b/py/desispec/test/test_io.py @@ -844,11 +844,12 @@ def test_findfile(self): with self.assertRaises(ValueError) as cm: foo = findfile('stdstars',expid=2,spectrograph=0) the_exception = cm.exception - self.assertEqual(str(the_exception), "Required input 'night' is not set for type 'stdstars'!") + self.assertTrue(str(the_exception), "Missing inputs for") + with self.assertRaises(ValueError) as cm: foo = findfile('spectra', survey='main', groupname=123) the_exception = cm.exception - self.assertEqual(str(the_exception), "Required input 'faprogram' is not set for type 'spectra'!") + self.assertTrue(str(the_exception), "Missing inputs for") #- Some findfile calls require $DESI_SPECTRO_DATA; others do not del os.environ['DESI_SPECTRO_DATA'] @@ -891,13 +892,14 @@ def test_findfile(self): with self.assertRaises(ValueError): a = findfile('cframe', night=20200317, expid=18, camera='Hasselblad') - # Test healpix versus tiles + # Test healpix versus tiles for various groupings a = findfile('spectra', groupname='5286', survey='main', faprogram='BRIGHT') b = os.path.join(os.environ['DESI_SPECTRO_REDUX'], os.environ['SPECPROD'], 'healpix', 'main', 'bright', '52', '5286', 'spectra-main-bright-5286.fits.gz') self.assertEqual(a, b) + a = findfile('spectra', tile=68000, night=20200314, spectrograph=2) b = os.path.join(os.environ['DESI_SPECTRO_REDUX'], os.environ['SPECPROD'], 'tiles', 'cumulative', @@ -905,6 +907,44 @@ def test_findfile(self): 'spectra-2-68000-thru20200314.fits.gz') self.assertEqual(a, b) + a = findfile('coadd', tile=68000, groupname='perexp', expid=1234, + spectrograph=2) + b = os.path.join(os.environ['DESI_SPECTRO_REDUX'], + os.environ['SPECPROD'], 'tiles', 'perexp', + '68000', '00001234', + 'coadd-2-68000-exp00001234.fits') + self.assertEqual(a, b) + + a = findfile('coadd', tile=68000, groupname='1x_depth', subgroup=42, + spectrograph=2) + b = os.path.join(os.environ['DESI_SPECTRO_REDUX'], + os.environ['SPECPROD'], 'tiles', '1x_depth', + '68000', '42', + 'coadd-2-68000-1x_depth-42.fits') + self.assertEqual(a, b) + + a = findfile('redrock', tile=68000, groupname='coffeeboba', subgroup=13, + spectrograph=2) + b = os.path.join(os.environ['DESI_SPECTRO_REDUX'], + os.environ['SPECPROD'], 'tiles', 'coffeeboba', + '68000', '13', + 'redrock-2-68000-coffeeboba-13.fits') + self.assertEqual(a, b) + + #- groupname shouldn't impact non-tile non-healpix files + night = 20201010 + expid = 1234 + tileid = 8888 + refpath = os.path.expandvars(f'$DESI_SPECTRO_DATA/{night}/{expid:08d}/fiberassign-{tileid:06d}.fits.gz') + for groupname in ('healpix', 'pernight', 'cumulative', 'perexp', 'blatfoo'): + testpath = findfile('fiberassign', night=night, expid=expid, tile=tileid, groupname=groupname) + self.assertEqual(testpath, refpath) + + refpath = os.path.expandvars(f'$DESI_SPECTRO_REDUX/$SPECPROD/preproc/{night}/{expid:08d}/tilepix-{tileid}.json') + for groupname in ('healpix', 'pernight', 'cumulative', 'perexp', 'blatfoo'): + testpath = findfile('tilepix', night=night, expid=expid, tile=tileid, groupname=groupname) + self.assertEqual(testpath, refpath) + #- Can't set both tile and healpix with self.assertRaises(ValueError): findfile('redrock', tile=1234, healpix=1234, survey='main', faprogram='dark') diff --git a/py/desispec/workflow/desi_proc_funcs.py b/py/desispec/workflow/desi_proc_funcs.py index 86b960fff..41989633a 100755 --- a/py/desispec/workflow/desi_proc_funcs.py +++ b/py/desispec/workflow/desi_proc_funcs.py @@ -455,7 +455,7 @@ def determine_resources(ncameras, jobdesc, nexps=1, forced_runtime=None, queue=N elif jobdesc == 'NIGHTLYBIAS': ncores, runtime = 15, 5 nodes = 2 - elif jobdesc in ['PEREXP', 'PERNIGHT', 'CUMULATIVE']: + elif jobdesc in ['PEREXP', 'PERNIGHT', 'CUMULATIVE', 'CUSTOMZTILE']: if system_name.startswith('perlmutter'): nodes, runtime = 1, 50 #- timefactor will bring time back down else: diff --git a/py/desispec/workflow/redshifts.py b/py/desispec/workflow/redshifts.py index 0bae1f731..8ea42c528 100644 --- a/py/desispec/workflow/redshifts.py +++ b/py/desispec/workflow/redshifts.py @@ -23,15 +23,18 @@ -def get_ztile_relpath(tileid,group,night=None,expid=None): +def get_ztile_relpath(tileid, group, night=None, expid=None, subgroup=None): """ Determine the relative output directory of the tile redshift batch script for spectra+coadd+redshifts for a tile Args: tileid (int): Tile ID group (str): cumulative, pernight, perexp, or a custom name + + Options: night (int): Night expid (int): Exposure ID + subgroup (str): subgroup name for non-standard group values Returns: outdir (str): the relative path of output directory of the batch script from the specprod/run/scripts @@ -46,40 +49,49 @@ def get_ztile_relpath(tileid,group,night=None,expid=None): outdir = f'tiles/{group}/{tileid}/{expid:08d}' elif group == 'pernight-v0': outdir = f'tiles/{tileid}/{night}' + elif subgroup is not None: + outdir = f'tiles/{group}/{tileid}/{subgroup}' + log.warning(f'Non-standard tile group={group}; writing outputs to {outdir}/*') else: outdir = f'tiles/{group}/{tileid}' log.warning(f'Non-standard tile group={group}; writing outputs to {outdir}/*') return outdir -def get_ztile_script_pathname(tileid,group,night=None,expid=None): +def get_ztile_script_pathname(tileid, group, night=None, expid=None, subgroup=None): """ Generate the pathname of the tile redshift batch script for spectra+coadd+redshifts for a tile Args: tileid (int): Tile ID group (str): cumulative, pernight, perexp, or a custom name + + Options: night (int): Night expid (int): Exposure ID + subgroup (str): Custom group string to use instead of night or expid Returns: (str): the pathname of the tile redshift batch script """ reduxdir = desispec.io.specprod_root() - outdir = get_ztile_relpath(tileid,group,night=night,expid=expid) + outdir = get_ztile_relpath(tileid,group,night=night,expid=expid,subgroup=subgroup) scriptdir = f'{reduxdir}/run/scripts/{outdir}' - suffix = get_ztile_script_suffix(tileid,group,night=night,expid=expid) + suffix = get_ztile_script_suffix(tileid,group,night=night,expid=expid,subgroup=subgroup) batchscript = f'ztile-{suffix}.slurm' return os.path.join(scriptdir, batchscript) -def get_ztile_script_suffix(tileid,group,night=None,expid=None): +def get_ztile_script_suffix(tileid, group, night=None, expid=None, subgroup=None): """ Generate the suffix of the tile redshift batch script for spectra+coadd+redshifts for a tile Args: tileid (int): Tile ID group (str): cumulative, pernight, perexp, or a custom name + + Options: night (int): Night expid (int): Exposure ID + subgroup (str): Custom group string to use instead of night or expid Returns: suffix (str): the suffix of the batch script @@ -93,6 +105,9 @@ def get_ztile_script_suffix(tileid,group,night=None,expid=None): suffix = f'{tileid}-exp{expid:08d}' elif group == 'pernight-v0': suffix = f'{tileid}-{night}' + elif subgroup is not None: + suffix = f'{tileid}-{group}-{subgroup}' + log.warning(f'Non-standard tile group={group}; writing outputs to {suffix}.*') else: suffix = f'{tileid}-{group}' log.warning(f'Non-standard tile group={group}; writing outputs to {suffix}.*') @@ -126,6 +141,7 @@ def get_zpix_redshift_script_pathname(healpix, survey, program): def create_desi_zproc_batch_script(group, tileid=None, cameras=None, thrunight=None, nights=None, expids=None, + subgroup=None, healpix=None, survey=None, program=None, queue='regular', batch_opts=None, runtime=None, timingfile=None, batchdir=None, @@ -144,6 +160,7 @@ def create_desi_zproc_batch_script(group, thrunight (int), optional: For group=cumulative, include exposures through this night nights (list of int), optional: The nights the data was acquired. expids (list of int), optional: The exposure id(s) for the data. + subgroup (str): subgroup name for non-standard group values healpix (list of int), optional: healpixels to process (group='healpix') queue (str), optional: Queue to be used. batch_opts (str), optional: Other options to give to the slurm batch scheduler (written into the script). @@ -197,7 +214,7 @@ def create_desi_zproc_batch_script(group, scriptpath = get_zpix_redshift_script_pathname(healpix, survey, program) else: scriptpath = get_ztile_script_pathname(tileid, group=group, - night=night, expid=expid) + night=night, expid=expid, subgroup=subgroup) if cameras is None: cameras = decode_camword('a0123456789') @@ -219,9 +236,15 @@ def create_desi_zproc_batch_script(group, scriptfile = os.path.join(batchdir, jobname + '.slurm') + ## Derive job description name from group + jobdesc = group + if jobdesc not in ('perexp', 'pernight', 'cumulative'): + jobdesc = 'customztile' + log.warning(f'Unrecognized {group=}, using {jobdesc=}') + ## If system name isn't specified, guess it if system_name is None: - system_name = batch.default_system(jobdesc=group, no_gpu=no_gpu) + system_name = batch.default_system(jobdesc=jobdesc, no_gpu=no_gpu) batch_config = batch.get_config(system_name) threads_per_core = batch_config['threads_per_core'] @@ -287,7 +310,7 @@ def create_desi_zproc_batch_script(group, cmd += ' --mpi' ncores, nodes, runtime = determine_resources( - ncameras, group.upper(), queue=queue, nexps=nexps, + ncameras, jobdesc=jobdesc, queue=queue, nexps=nexps, forced_runtime=runtime, system_name=system_name) runtime_hh = int(runtime // 60)