From bddbeb4c70acdd855653558d7cedda19da979d2a Mon Sep 17 00:00:00 2001 From: michaelJwilson Date: Wed, 15 Apr 2020 15:55:53 -0700 Subject: [PATCH 1/8] Realistic imaging props. --- py/desitarget/mock/mockmaker.py | 64 ++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/py/desitarget/mock/mockmaker.py b/py/desitarget/mock/mockmaker.py index 51b72972a..50756f49e 100644 --- a/py/desitarget/mock/mockmaker.py +++ b/py/desitarget/mock/mockmaker.py @@ -340,7 +340,7 @@ def mw_dust_extinction(self, Rv=3.1): extinction = Rv * ext_odonnell(self.wave, Rv=Rv) return extinction - def imaging_depth(self, data): + def simple_imaging_depth(self, data): """Add the imaging depth to the data dictionary. Note: In future, this should be a much more sophisticated model based on the @@ -392,6 +392,68 @@ def imaging_depth(self, data): wisedepth_ivar = 1 / (5 * 10**(-0.4 * (wisedepth_mag - 22.5)))**2 # 5-sigma, 1/nanomaggies**2 data['PSFDEPTH_{}'.format(band)] = wisedepth_ivar + def imaging_depth(self, data, release=8): + import desitarget.randoms as randoms + + from desitarget.targets import resolve + from astropy.table import Table + + + log.info('Setting realistic imaging depths (including MASKBITS).') + + bricks = self.Bricks + + # Return brick name of brick covering (ra, dec). + bricknames = bricks.brickname(data['RA'], data['DEC']) + + ubricknames = np.unique(bricknames) + + drdir = '/global/project/projectdirs/cosmo/data/legacysurvey/dr{}'.format(release) + + # determine if we must traverse two sets of brick directories, i.e. north/, south/. + drdirs = randoms._pre_or_post_dr8(drdir) + + # + keep = ['MASKBITS', 'PHOTSYS'] + bands = ['G', 'R', 'Z', 'W1', 'W2'] + bandkeep = ['NOBS', 'PSFDEPTH', 'GALDEPTH'] + + toremove = np.zeros_like(data['RA'], dtype=bool) + + # + data['BRICKNAME'] = bricknames + + for ubrickname in ubricknames: + indx = data['BRICKNAME'] == ubrickname + + rtn = randoms.dr8_quantities_at_positions_in_a_brick(data['RA'][indx], data['DEC'][indx], ubrickname, drdir) + + if rtn: + for key in list(rtn.keys()): + rtn[key.upper()] = rtn.pop(key) + + rtn = Table(rtn) + rtn['RA'] = data['RA'][indx] + rtn['DEC'] = data['DEC'][indx] + + rtn = resolve(rtn) + + for key in keep: + data[key][indx] = rtn[key] + + for band in bands: + for bk in bandkeep: + key = bk + '_' + band + data[key][indx] = rtn[key] + + else: + toremove[indx] = True + + log.info('Removing {} of {} targets not in reduced DESI imaging.'.format(np.count_nonzero(toremove), len(data['RA']))) + + for key in data: + data[key] = data[key][~toremove] + def scatter_photometry(self, data, truth, targets, indx=None, seed=None, qaplot=False): """Add noise to the input (noiseless) photometry based on the depth (as well as From cdaa0394aee512c5bc3017d065d17e10064f9e7c Mon Sep 17 00:00:00 2001 From: michaelJwilson Date: Thu, 16 Apr 2020 08:56:48 -0700 Subject: [PATCH 2/8] Realistic imaging updates. --- .../data/select-mock-targets-no-contam.yaml | 63 ++++++ py/desitarget/mock/mockmaker.py | 206 ++++++++++++------ 2 files changed, 201 insertions(+), 68 deletions(-) create mode 100644 py/desitarget/mock/data/select-mock-targets-no-contam.yaml diff --git a/py/desitarget/mock/data/select-mock-targets-no-contam.yaml b/py/desitarget/mock/data/select-mock-targets-no-contam.yaml new file mode 100644 index 000000000..66d928db2 --- /dev/null +++ b/py/desitarget/mock/data/select-mock-targets-no-contam.yaml @@ -0,0 +1,63 @@ +# Parameter file for desitarget/bin/select_mock_targets, including contaminants. +targets: + BGS: { + target_type: BGS, + mockfile: '{DESI_ROOT}/mocks/bgs/MXXL/full_sky/v0.0.4/BGS_r20.6.hdf5', + format: durham_mxxl_hdf5, + magcut: 20.3, + } + ELG: { + target_type: ELG, + mockfile: '{DESI_ROOT}/mocks/DarkSky/v1.0.1/elg_0_inpt.fits', + format: gaussianfield, + density: 2200, + } + LRG: { + target_type: LRG, + mockfile: '{DESI_ROOT}/mocks/DarkSky/v1.0.1/lrg_0_inpt.fits', + format: gaussianfield, + density: 480, + } + QSO: { + target_type: QSO, + mockfile: '{DESI_ROOT}/mocks/DarkSky/v1.0.1/qso_0_inpt.fits', + format: gaussianfield, + zmax_qso: 1.8, + use_simqso: True, + density: 120, + } + LYA: { + target_type: QSO, + mockfile: '{DESI_ROOT}/mocks/lya_forest/london/v9.0/v9.0.0/master.fits', + format: CoLoRe, + nside_lya: 16, + zmin_lya: 1.8, + density: 50, + use_simqso: True, + sqmodel: 'default', #Other options are 'lya_simqso_model_develop' and 'lya_simqso_model' (same as quickquasars) + balprob: 0.0, + add_dla: False, + add_metals: False, #If adding metals use 'all', this is to keep the same argument as in quickquasars. + add_lyb: False + } + MWS_MAIN: { + target_type: STAR, + mockfile: '{DESI_ROOT}/mocks/mws/galaxia/alpha/v0.0.6/healpix', + nside_galaxia: 8, + format: galaxia, + } + MWS_NEARBY: { + target_type: STAR, + mockfile: '{DESI_ROOT}/mocks/mws/100pc/v0.0.4/mock_100pc.fits', + format: mws_100pc, + } + WD: { + target_type: WD, + mockfile: '{DESI_ROOT}/mocks/mws/wd/v1.0.0/mock_wd.fits', + format: mws_wd, + } + SKY: { + target_type: SKY, + mockfile: '{DESI_ROOT}/mocks/uniformsky/0.2/uniformsky-2048-0.2.fits', + format: uniformsky, + } \ No newline at end of file diff --git a/py/desitarget/mock/mockmaker.py b/py/desitarget/mock/mockmaker.py index 50756f49e..09c310484 100644 --- a/py/desitarget/mock/mockmaker.py +++ b/py/desitarget/mock/mockmaker.py @@ -132,7 +132,7 @@ def empty_targets_table(nobj=1): targets.add_column(Column(name='FIBERTOTFLUX_G', length=nobj, dtype='>f4')) targets.add_column(Column(name='FIBERTOTFLUX_R', length=nobj, dtype='>f4')) targets.add_column(Column(name='FIBERTOTFLUX_Z', length=nobj, dtype='>f4')) - + # Gaia columns targets.add_column(Column(name='REF_CAT', length=nobj, dtype='S2')) targets.add_column(Column(name='REF_ID', data=np.repeat(-1, nobj).astype('int64'))) # default is -1 @@ -340,6 +340,37 @@ def mw_dust_extinction(self, Rv=3.1): extinction = Rv * ext_odonnell(self.wave, Rv=Rv) return extinction + def set_wise_depth(self, data): + import astropy.units as u + from astropy.coordinates import SkyCoord + + # compute the WISE depth, which is largely a function of ecliptic latitude + coord = SkyCoord(data['RA']*u.deg, data['DEC']*u.deg) + ecoord = coord.transform_to('barycentrictrueecliptic') + beta = ecoord.lat.value + + if np.count_nonzero(beta > 89) > 0: # don't explode at the pole! + beta[beta > 89] = 89 + + if np.count_nonzero(beta < -89) > 0: # don't explode at the pole! + beta[beta < -89] = -89 + + beta = np.radians(ecoord.lat.value) # [radians] + + sig_syst = [0.5, 2.0] # systematic uncertainty due to low-level + # background structure e.g. striping + neff = [15.7832, 18.5233] # effective number of pixels in PSF + vega2ab = [2.699, 3.339] + sig_stat_beta0 = [3.5127802, 9.1581879] # random uncertainty [AB nanomaggies] + + for ii, band in enumerate(('W1', 'W2')): + sig_stat = sig_stat_beta0[ii] / np.sqrt( 1.0 / np.cos(beta) ) + sig = np.sqrt( sig_stat**2 + sig_syst[ii]**2 ) + + wisedepth_mag = 22.5 - 2.5 * np.log10( sig * np.sqrt(neff[ii]) ) + vega2ab[ii] # 1-sigma, AB mag + wisedepth_ivar = 1 / (5 * 10**(-0.4 * (wisedepth_mag - 22.5)))**2 # 5-sigma, 1/nanomaggies**2 + data['PSFDEPTH_{}'.format(band)] = wisedepth_ivar + def simple_imaging_depth(self, data): """Add the imaging depth to the data dictionary. @@ -368,92 +399,113 @@ def simple_imaging_depth(self, data): data['PSFDEPTH_{}'.format(band)] = np.repeat(psfdepth_ivar[ii], nobj) data['GALDEPTH_{}'.format(band)] = np.repeat(galdepth_ivar[ii], nobj) - # compute the WISE depth, which is largely a function of ecliptic latitude - coord = SkyCoord(data['RA']*u.deg, data['DEC']*u.deg) - ecoord = coord.transform_to('barycentrictrueecliptic') - beta = ecoord.lat.value - if np.count_nonzero(beta > 89) > 0: # don't explode at the pole! - beta[beta > 89] = 89 - if np.count_nonzero(beta < -89) > 0: # don't explode at the pole! - beta[beta < -89] = -89 - beta = np.radians(ecoord.lat.value) # [radians] - - sig_syst = [0.5, 2.0] # systematic uncertainty due to low-level - # background structure e.g. striping - neff = [15.7832, 18.5233] # effective number of pixels in PSF - vega2ab = [2.699, 3.339] - sig_stat_beta0 = [3.5127802, 9.1581879] # random uncertainty [AB nanomaggies] - - for ii, band in enumerate(('W1', 'W2')): - sig_stat = sig_stat_beta0[ii] / np.sqrt( 1.0 / np.cos(beta) ) - sig = np.sqrt( sig_stat**2 + sig_syst[ii]**2 ) - - wisedepth_mag = 22.5 - 2.5 * np.log10( sig * np.sqrt(neff[ii]) ) + vega2ab[ii] # 1-sigma, AB mag - wisedepth_ivar = 1 / (5 * 10**(-0.4 * (wisedepth_mag - 22.5)))**2 # 5-sigma, 1/nanomaggies**2 - data['PSFDEPTH_{}'.format(band)] = wisedepth_ivar + self.set_wise_depth(data) - def imaging_depth(self, data, release=8): - import desitarget.randoms as randoms + def imaging_depth(self, data, release=8, aprad=0.0, simple=True): + import pandas as pd + import desitarget.randoms as randoms from desitarget.targets import resolve from astropy.table import Table + from desitarget.io import basetsdatamodel + + if not simple: + log.info('Setting realistic imaging depths (including MASKBITS).') + + bricks = self.Bricks + # Return brick name of brick covering (ra, dec). + bricknames = bricks.brickname(data['RA'], data['DEC']) - log.info('Setting realistic imaging depths (including MASKBITS).') + ubricknames = np.unique(bricknames) - bricks = self.Bricks + drdir = '/global/project/projectdirs/cosmo/data/legacysurvey/dr{}'.format(release) - # Return brick name of brick covering (ra, dec). - bricknames = bricks.brickname(data['RA'], data['DEC']) + # determine if we must traverse two sets of brick directories, i.e. north/, south/. + drdirs = randoms._pre_or_post_dr8(drdir) - ubricknames = np.unique(bricknames) + # + keep = ['MASKBITS', 'PHOTSYS'] + + bands = ['G', 'R', 'Z'] + bandkeep = ['NOBS', 'PSFDEPTH', 'GALDEPTH'] - drdir = '/global/project/projectdirs/cosmo/data/legacysurvey/dr{}'.format(release) + _ = empty_targets_table(nobj=1) + dtypes = dict(zip(_.columns, [_[x].dtype for x in _.columns])) - # determine if we must traverse two sets of brick directories, i.e. north/, south/. - drdirs = randoms._pre_or_post_dr8(drdir) + # + data['BRICKNAME'] = bricknames + + for key in keep: + data[key] = np.zeros(len(data['RA']), dtype=dtypes[key]) - # - keep = ['MASKBITS', 'PHOTSYS'] - bands = ['G', 'R', 'Z', 'W1', 'W2'] - bandkeep = ['NOBS', 'PSFDEPTH', 'GALDEPTH'] + for band in bands: + for bk in bandkeep: + key = bk + '_' + band + data[key] = np.zeros(len(data['RA']), dtype=dtypes[key]) - toremove = np.zeros_like(data['RA'], dtype=bool) + data['PSFDEPTH_W1'] = np.zeros(len(data['RA']), dtype=dtypes['PSFDEPTH_G']) + data['PSFDEPTH_W2'] = np.zeros(len(data['RA']), dtype=dtypes['PSFDEPTH_G']) - # - data['BRICKNAME'] = bricknames + toremove = np.zeros_like(data['RA'], dtype=bool) - for ubrickname in ubricknames: - indx = data['BRICKNAME'] == ubrickname + for ubrickname in ubricknames: + indx = np.atleast_1d(data['BRICKNAME'] == ubrickname) - rtn = randoms.dr8_quantities_at_positions_in_a_brick(data['RA'][indx], data['DEC'][indx], ubrickname, drdir) + # Defaults to no sky background ap. fluxes. + rtn = randoms.dr8_quantities_at_positions_in_a_brick(data['RA'][indx], data['DEC'][indx], ubrickname, drdir, aprad=aprad) - if rtn: - for key in list(rtn.keys()): - rtn[key.upper()] = rtn.pop(key) + if rtn: + for key in list(rtn.keys()): + rtn[key.upper()] = rtn.pop(key) - rtn = Table(rtn) - rtn['RA'] = data['RA'][indx] - rtn['DEC'] = data['DEC'][indx] + rtn = Table(rtn) - rtn = resolve(rtn) + if len(rtn['PHOTSYS']) == len(data['RA'][indx]): + rtn['RA'] = data['RA'][indx] + rtn['DEC'] = data['DEC'][indx] - for key in keep: - data[key][indx] = rtn[key] + else: + rtn['RA'] = np.concatenate((data['RA'][indx], data['RA'][indx])) + rtn['DEC'] = np.concatenate((data['DEC'][indx], data['DEC'][indx])) + + rtn = resolve(rtn) - for band in bands: - for bk in bandkeep: - key = bk + '_' + band + for key in keep: data[key][indx] = rtn[key] - else: - toremove[indx] = True + for band in bands: + for bk in bandkeep: + key = bk + '_' + band + data[key][indx] = rtn[key] - log.info('Removing {} of {} targets not in reduced DESI imaging.'.format(np.count_nonzero(toremove), len(data['RA']))) - - for key in data: - data[key] = data[key][~toremove] + data['PSFDEPTH_W1'][indx] = rtn['PSFDEPTH_W1'] + data['PSFDEPTH_W2'][indx] = rtn['PSFDEPTH_W2'] + + else: + # Empty dict: missing brick. Remove these targets. + toremove[np.where(indx)[0]] = True + + log.info('Removing {} targets not in reduced DESI imaging (of {}).'.format(np.count_nonzero(toremove), len(data['RA']))) + for key in list(data.keys()): + try: + if len(data[key]) == len(data['RA']): + data[key] = data[key][~toremove] + + except(TypeError): + pass + + # Overwrite with simple depths for now. + self.simple_imaging_depth(data) + + else: + log.info('Setting simple imaging depths.') + + self.simple_imaging_depth(data) + + # print(pd.DataFrame(data)) + def scatter_photometry(self, data, truth, targets, indx=None, seed=None, qaplot=False): """Add noise to the input (noiseless) photometry based on the depth (as well as @@ -507,11 +559,11 @@ def scatter_photometry(self, data, truth, targets, indx=None, # WISE sources are all point sources for band in ('W1', 'W2'): - fluxkey = 'FLUX_{}'.format(band) - ivarkey = 'FLUX_IVAR_{}'.format(band) + fluxkey = 'FLUX_{}'.format(band) + ivarkey = 'FLUX_IVAR_{}'.format(band) depthkey = 'PSFDEPTH_{}'.format(band) - sigma = 1 / np.sqrt(data[depthkey][indx]) / 5 # nanomaggies, 1-sigma + sigma = 1 / np.sqrt(data[depthkey][indx]) / 5 # nanomaggies, 1-sigma targets[fluxkey][:] = truth[fluxkey] + rand.normal(scale=sigma) targets[ivarkey][:] = 1 / sigma**2 @@ -1075,11 +1127,18 @@ def populate_targets_truth(self, flux, data, meta, objmeta, indx=None, # Assign RELEASE, PHOTSYS, [RA,DEC]_IVAR, and DCHISQ targets['RELEASE'][:] = 9999 - isouth = self.is_south(targets['DEC']) + if 'PHOTSYS' in list(data.keys()): + isouth = targets['PHOTSYS'] == 'S' + + else: + isouth = self.is_south(targets['DEC']) + south = np.where( isouth )[0] north = np.where( ~isouth )[0] + if len(south) > 0: targets['PHOTSYS'][south] = 'S' + if len(north) > 0: targets['PHOTSYS'][north] = 'N' @@ -1091,12 +1150,23 @@ def populate_targets_truth(self, flux, data, meta, objmeta, indx=None, key = 'MW_TRANSMISSION_{}'.format(band) targets[key][:] = data[key][indx] + _keys = [] + for band in ('G', 'R', 'Z'): for prefix in ('PSF', 'GAL'): key = '{}DEPTH_{}'.format(prefix, band) - targets[key][:] = data[key][indx] + _keys.append(key) + nobskey = 'NOBS_{}'.format(band) - targets[nobskey][:] = 2 # assume constant! + _keys.append(nobskey) + + for key in _keys: + if key not in list(data.keys()): + if key.split('_')[0] == 'NOBS': + targets[key][:] = 2 # assume constant! + + else: + targets[key][:] = data[key][indx] # Add spectral / template type and subtype. for value, key in zip( (truespectype, templatetype, templatesubtype), From 14ac98e78a94c095171b44b2ef109e00b5072009 Mon Sep 17 00:00:00 2001 From: michaelJwilson Date: Mon, 27 Apr 2020 10:56:41 -0700 Subject: [PATCH 3/8] one percent updates. --- bin/mpi_select_mock_targets | 4 ++-- bin/select_mock_targets | 8 ++++---- py/desitarget/mock/io.py | 9 +++++++-- py/desitarget/mock/mockmaker.py | 30 +++++++++++++----------------- 4 files changed, 26 insertions(+), 25 deletions(-) diff --git a/bin/mpi_select_mock_targets b/bin/mpi_select_mock_targets index 73ce37ad2..d5a76003a 100755 --- a/bin/mpi_select_mock_targets +++ b/bin/mpi_select_mock_targets @@ -92,13 +92,13 @@ if rank == 0: keep = list() for i, pixnum in enumerate(pixels): truthspecfile = mockio.findfile('truth', args.nside, pixnum, - basedir=args.output_dir) + basedir=args.output_dir, obscon='dark') if not os.path.exists(truthspecfile): keep.append(i) log.info('{}/{} pixels remaining to do'.format(len(keep), len(pixels))) pixels = pixels[keep] - + #- pre-create output directories for pixnum in pixels: outdir = os.path.dirname(mockio.findfile('blat', args.nside, pixnum, diff --git a/bin/select_mock_targets b/bin/select_mock_targets index 30e35c550..5e3a82ffb 100755 --- a/bin/select_mock_targets +++ b/bin/select_mock_targets @@ -69,19 +69,19 @@ if args.healpixels is None: healpixels = desimodel.footprint.tiles2pix(args.nside, tiles) else: healpixels = np.array(args.healpixels) - + if args.overwrite: log.info('Processing {} pixel(s).'.format(len(healpixels))) else: keep = list() for i, pixnum in enumerate(healpixels): - truthfile = mockio.findfile('truth', args.nside, pixnum, basedir=args.output_dir) + truthfile = mockio.findfile('truth', args.nside, pixnum, basedir=args.output_dir, obscon='dark') if not os.path.exists(truthfile): keep.append(i) - log.info('{}/{} pixels remaining to do'.format(len(keep), len(healpixels))) + log.info('{}/{} pixels remaining to do. {}'.format(len(keep), len(healpixels), healpixels[keep])) healpixels = healpixels[keep] - + if args.join: from desitarget.mock.build import join_targets_truth join_targets_truth(mockdir=args.output_dir, outdir=args.output_dir, overwrite=args.overwrite) diff --git a/py/desitarget/mock/io.py b/py/desitarget/mock/io.py index 6bf1817b3..c22e8acc0 100644 --- a/py/desitarget/mock/io.py +++ b/py/desitarget/mock/io.py @@ -42,10 +42,15 @@ def findfile(filetype, nside, pixnum, basedir='.', ext='fits', obscon=None): obscon: (str) e.g. 'dark', 'bright' to add extra dir grouping ''' path = get_healpix_dir(nside, pixnum, basedir=basedir) + if obscon is not None: path = os.path.join(path, obscon.lower()) - filename = '{filetype}-{nside}-{pixnum}.{ext}'.format( - filetype=filetype, nside=nside, pixnum=pixnum, ext=ext) + filename = '{filetype}-{obscon}-{nside}-{pixnum}.{ext}'.format( + filetype=filetype, obscon=obscon.lower(), nside=nside, pixnum=pixnum, ext=ext) + + else: + filename = '{filetype}-{nside}-{pixnum}.{ext}'.format( + filetype=filetype, nside=nside, pixnum=pixnum, ext=ext) return os.path.join(path, filename) diff --git a/py/desitarget/mock/mockmaker.py b/py/desitarget/mock/mockmaker.py index 09c310484..860bfd8ee 100644 --- a/py/desitarget/mock/mockmaker.py +++ b/py/desitarget/mock/mockmaker.py @@ -401,7 +401,7 @@ def simple_imaging_depth(self, data): self.set_wise_depth(data) - def imaging_depth(self, data, release=8, aprad=0.0, simple=True): + def imaging_depth(self, data, release=8, aprad=0.0, simple=False): import pandas as pd import desitarget.randoms as randoms @@ -487,15 +487,11 @@ def imaging_depth(self, data, release=8, aprad=0.0, simple=True): toremove[np.where(indx)[0]] = True log.info('Removing {} targets not in reduced DESI imaging (of {}).'.format(np.count_nonzero(toremove), len(data['RA']))) - + for key in list(data.keys()): - try: - if len(data[key]) == len(data['RA']): - data[key] = data[key][~toremove] - - except(TypeError): - pass - + if isinstance(data[key], np.ndarray): + data[key] = data[key][~toremove] + # Overwrite with simple depths for now. self.simple_imaging_depth(data) @@ -504,8 +500,6 @@ def imaging_depth(self, data, release=8, aprad=0.0, simple=True): self.simple_imaging_depth(data) - # print(pd.DataFrame(data)) - def scatter_photometry(self, data, truth, targets, indx=None, seed=None, qaplot=False): """Add noise to the input (noiseless) photometry based on the depth (as well as @@ -1575,16 +1569,18 @@ def readmock(self, mockfile=None, healpixels=None, nside=None, # fig.savefig('/global/homes/i/ioannis/rmag-vs-oiiflux.png') if target_name.upper() == 'ELG' and gmmout is not None: - rmagpivot = 22 - rand = np.random.RandomState(seed) + rmagpivot = 22 + rand = np.random.RandomState(seed) - oiicoeff_median = np.array([ -5.31384197e-04, -4.24876618e-01, 1.17861064e+00]) + oiicoeff_median = np.array([ -5.31384197e-04, -4.24876618e-01, 1.17861064e+00]) oiicoeff_scatter = np.array([-0.00188197, -0.00077902, 0.30842156]) - oiiflux = np.polyval(oiicoeff_median, gmmout['MAG'] - rmagpivot) - oiiflux_sigma = np.polyval(oiicoeff_scatter, gmmout['MAG'] - rmagpivot) + oiiflux = np.polyval(oiicoeff_median, gmmout['MAG'] - rmagpivot) + oiiflux_sigma = np.polyval(oiicoeff_scatter, gmmout['MAG'] - rmagpivot) + for ii, sigma in enumerate(oiiflux_sigma): if sigma < 0: # edge case - sigma = np.polyval(oiicoeff_scatter, 0) + sigma = np.polyval(oiicoeff_scatter, 0) + oiiflux[ii] += rand.normal(loc=0, scale=sigma) gmmout.update({'OIIFLUX': 1e-17 * oiiflux}) From 0e695f457c85214cf70a9c21b349fe46f557d064 Mon Sep 17 00:00:00 2001 From: michaelJwilson Date: Mon, 27 Apr 2020 11:02:18 -0700 Subject: [PATCH 4/8] Removed remaining pandas dependency. --- py/desitarget/mock/mockmaker.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/py/desitarget/mock/mockmaker.py b/py/desitarget/mock/mockmaker.py index 860bfd8ee..7482540d8 100644 --- a/py/desitarget/mock/mockmaker.py +++ b/py/desitarget/mock/mockmaker.py @@ -402,7 +402,6 @@ def simple_imaging_depth(self, data): self.set_wise_depth(data) def imaging_depth(self, data, release=8, aprad=0.0, simple=False): - import pandas as pd import desitarget.randoms as randoms from desitarget.targets import resolve @@ -476,8 +475,8 @@ def imaging_depth(self, data, release=8, aprad=0.0, simple=False): for band in bands: for bk in bandkeep: - key = bk + '_' + band - data[key][indx] = rtn[key] + key = bk + '_' + band + data[key][indx] = rtn[key] data['PSFDEPTH_W1'][indx] = rtn['PSFDEPTH_W1'] data['PSFDEPTH_W2'][indx] = rtn['PSFDEPTH_W2'] From 8f703436ab109fe9a46f2ddd62274602c2d01c81 Mon Sep 17 00:00:00 2001 From: michaelJwilson Date: Mon, 27 Apr 2020 18:42:40 -0700 Subject: [PATCH 5/8] Added legacy_dir to param file to flag location of dir. & enable MASKBITS machinery. --- bin/select_mock_targets | 12 +++ py/desitarget/mock/build.py | 21 ++++-- .../data/select-mock-targets-no-contam.yaml | 4 +- py/desitarget/mock/mockmaker.py | 73 ++++++++++--------- 4 files changed, 65 insertions(+), 45 deletions(-) diff --git a/bin/select_mock_targets b/bin/select_mock_targets index 5e3a82ffb..ae1260bc6 100755 --- a/bin/select_mock_targets +++ b/bin/select_mock_targets @@ -96,6 +96,18 @@ log.info('Reading configuration file {}'.format(args.config)) with open(args.config, 'r') as pfile: params = yaml.safe_load(pfile) +# Set up MASKBITS lookup. Logic check for the necessary files, if not there wget the per brick files for +# all overlapping pixels. +if 'legacy_dir' in params.keys(): + # E.g. global/project/projectdirs/cosmo/data/legacysurvey/dr8/ + # wget https://portal.nersc.gov/project/cosmo/data/legacysurvey/dr8/north/coadd/338/3380p295/ + log.info('Assuming maskbits & depths based on LEGACYDIR={}'.format(params['legacy_dir'])) + +else: + params['legacy_dir'] = None + + log.info('Assuming simple homogeneous depths & no masking.') + log.info('Calling targets_truth with survey={} at {}'.format(args.survey, time.asctime())) targets_truth(params, healpixels=healpixels, nside=args.nside, seed=args.seed, output_dir=args.output_dir, nproc=args.nproc, verbose=args.verbose, diff --git a/py/desitarget/mock/build.py b/py/desitarget/mock/build.py index 07939f815..631df90a1 100644 --- a/py/desitarget/mock/build.py +++ b/py/desitarget/mock/build.py @@ -141,7 +141,7 @@ def read_mock(params, log=None, target_name='', seed=None, healpixels=None, magcut = params.get('magcut') nside_galaxia = params.get('nside_galaxia') calib_only = params.get('calib_only', False) - + # QSO/Lya parameters nside_lya = params.get('nside_lya') zmin_lya = params.get('zmin_lya') @@ -152,7 +152,7 @@ def read_mock(params, log=None, target_name='', seed=None, healpixels=None, add_dla = params.get('add_dla', False) add_metals = params.get('add_metals', False) add_lyb = params.get('add_lyb', False) - + if 'density' in params.keys(): mock_density = True else: @@ -160,14 +160,17 @@ def read_mock(params, log=None, target_name='', seed=None, healpixels=None, log.info('Target: {}, type: {}, format: {}, mockfile: {}'.format( target_name, target_type, mockformat, mockfile)) - + if MakeMock is None: MakeMock = getattr(mockmaker, '{}Maker'.format(target_name))( seed=seed, nside_chunk=nside_chunk, calib_only=calib_only, use_simqso=use_simqso, sqmodel=sqmodel, balprob=balprob, add_dla=add_dla, - add_metals=add_metals, add_lyb=add_lyb) + add_metals=add_metals, add_lyb=add_lyb, legacy_dir=legacy_dir) + else: MakeMock.seed = seed # updated seed + + print(target_name, MakeMock.legacy_dir) data = MakeMock.read(mockfile=mockfile, mockformat=mockformat, healpixels=healpixels, nside=nside, @@ -176,7 +179,7 @@ def read_mock(params, log=None, target_name='', seed=None, healpixels=None, nside_galaxia=nside_galaxia, mock_density=mock_density) if not bool(data): return data, MakeMock - + # Add the information we need to incorporate density fluctuations. if 'density' in params.keys(): if 'MOCK_DENSITY' not in data.keys(): @@ -854,11 +857,13 @@ def targets_truth(params, healpixels=None, nside=None, output_dir='.', add_dla = params['targets'][target_name].get('add_dla', False) add_metals = params['targets'][target_name].get('add_metals', False) add_lyb = params['targets'][target_name].get('add_lyb', False) + legacy_dir = params['legacy_dir'] + AllMakeMock.append(getattr(mockmaker, '{}Maker'.format(target_name))( seed=seed, nside_chunk=nside_chunk, calib_only=calib_only, use_simqso=use_simqso, sqmodel=sqmodel, balprob=balprob, add_dla=add_dla, add_metals=add_metals, add_lyb=add_lyb, no_spectra=no_spectra, - survey=survey)) + survey=survey, legacy_dir=legacy_dir)) # Are we adding contaminants? If so, cache the relevant classes here. if 'contaminants' in params.keys(): @@ -870,7 +875,7 @@ def targets_truth(params, healpixels=None, nside=None, output_dir='.', star_name, _ = list(params['contaminants']['stars'].items())[0] ContamStarsMock = getattr(mockmaker, '{}Maker'.format(star_name))( seed=seed, nside_chunk=nside_chunk, no_spectra=no_spectra, - survey=survey) + survey=survey, legacy_dir=legacy_dir) else: ContamStarsMock = None @@ -882,7 +887,7 @@ def targets_truth(params, healpixels=None, nside=None, output_dir='.', galaxies_name, _ = list(params['contaminants']['galaxies'].items())[0] ContamGalaxiesMock = getattr(mockmaker, '{}Maker'.format(galaxies_name))( seed=seed, nside_chunk=nside_chunk, no_spectra=no_spectra, - target_name='CONTAM_GALAXY', survey=survey) + target_name='CONTAM_GALAXY', survey=survey, legacy_dir=legacy_dir) else: ContamGalaxiesMock = None diff --git a/py/desitarget/mock/data/select-mock-targets-no-contam.yaml b/py/desitarget/mock/data/select-mock-targets-no-contam.yaml index 66d928db2..89fab5b1c 100644 --- a/py/desitarget/mock/data/select-mock-targets-no-contam.yaml +++ b/py/desitarget/mock/data/select-mock-targets-no-contam.yaml @@ -1,4 +1,6 @@ # Parameter file for desitarget/bin/select_mock_targets, including contaminants. +legacy_dir: /global/project/projectdirs/cosmo/data/legacysurvey/dr8/ + targets: BGS: { target_type: BGS, @@ -60,4 +62,4 @@ targets: target_type: SKY, mockfile: '{DESI_ROOT}/mocks/uniformsky/0.2/uniformsky-2048-0.2.fits', format: uniformsky, - } \ No newline at end of file + } diff --git a/py/desitarget/mock/mockmaker.py b/py/desitarget/mock/mockmaker.py index 7482540d8..6fe096ed5 100644 --- a/py/desitarget/mock/mockmaker.py +++ b/py/desitarget/mock/mockmaker.py @@ -265,7 +265,7 @@ class SelectTargets(object): """ GMM_LRG, GMM_ELG, GMM_BGS, GMM_QSO, FFA = None, None, None, None, None - def __init__(self, bricksize=0.25, survey='main', **kwargs): + def __init__(self, bricksize=0.25, survey='main', legacy_dir=None, **kwargs): from astropy.io import fits from speclite import filters @@ -274,6 +274,8 @@ def __init__(self, bricksize=0.25, survey='main', **kwargs): from specsim.fastfiberacceptance import FastFiberAcceptance self.survey = survey + self.legacy_dir = legacy_dir + if survey == 'main': from desitarget.targetmask import desi_mask, bgs_mask, mws_mask elif survey == 'sv1': @@ -401,27 +403,25 @@ def simple_imaging_depth(self, data): self.set_wise_depth(data) - def imaging_depth(self, data, release=8, aprad=0.0, simple=False): + def imaging_depth(self, data, legacy_dir=None, aprad=0.0): import desitarget.randoms as randoms from desitarget.targets import resolve from astropy.table import Table from desitarget.io import basetsdatamodel - if not simple: - log.info('Setting realistic imaging depths (including MASKBITS).') - + if legacy_dir is not None: + log.info('Populating image depths & maskbits.') + bricks = self.Bricks # Return brick name of brick covering (ra, dec). bricknames = bricks.brickname(data['RA'], data['DEC']) ubricknames = np.unique(bricknames) - - drdir = '/global/project/projectdirs/cosmo/data/legacysurvey/dr{}'.format(release) # determine if we must traverse two sets of brick directories, i.e. north/, south/. - drdirs = randoms._pre_or_post_dr8(drdir) + # drdirs = randoms._pre_or_post_dr8(legacy_dir) # keep = ['MASKBITS', 'PHOTSYS'] @@ -452,7 +452,7 @@ def imaging_depth(self, data, release=8, aprad=0.0, simple=False): indx = np.atleast_1d(data['BRICKNAME'] == ubrickname) # Defaults to no sky background ap. fluxes. - rtn = randoms.dr8_quantities_at_positions_in_a_brick(data['RA'][indx], data['DEC'][indx], ubrickname, drdir, aprad=aprad) + rtn = randoms.dr8_quantities_at_positions_in_a_brick(data['RA'][indx], data['DEC'][indx], ubrickname, legacy_dir, aprad=aprad) if rtn: for key in list(rtn.keys()): @@ -2452,7 +2452,7 @@ def __init__(self, **kwargs): def readmock(self, mockfile=None, healpixels=None, nside=None, target_name='BGS', magcut=None, only_coords=False, - mock_density=False, seed=None): + mock_density=False, seed=None, legacy_dir=None): """Read the catalog. Parameters @@ -2516,7 +2516,7 @@ def readmock(self, mockfile=None, healpixels=None, nside=None, if nside is None: log.warning('Nside must be a scalar input.') raise ValueError - + # Read the data, generate mockid, and then restrict to the input # healpixel. def _read_mockfile(mockfile, nside, pixmap): @@ -2631,7 +2631,7 @@ def _read_mockfile(mockfile, nside, pixmap): # Add MW transmission and the imaging depth. self.mw_transmission(out) - self.imaging_depth(out) + self.imaging_depth(out, legacy_dir=legacy_dir) # Optionally compute the mean mock density. if mock_density: @@ -3101,11 +3101,11 @@ class QSOMaker(SelectTargets): """ wave, template_maker, GMM_QSO = None, None, None - def __init__(self, seed=None, use_simqso=True, survey='main', **kwargs): + def __init__(self, seed=None, use_simqso=True, survey='main', legacy_dir=None, **kwargs): from desisim.templates import SIMQSO, QSO from desiutil.sklearn import GaussianMixtureModel - super(QSOMaker, self).__init__(survey=survey) + super(QSOMaker, self).__init__(survey=survey, legacy_dir=legacy_dir) self.seed = seed self.objtype = 'QSO' @@ -3317,11 +3317,11 @@ class LYAMaker(SelectTargets): def __init__(self, seed=None, use_simqso=True,sqmodel='default',\ balprob=0.0,add_dla=False,add_metals=False,add_lyb=False,\ - survey='main', **kwargs): + survey='main', legacy_dir=None, **kwargs): from desisim.templates import SIMQSO, QSO from desiutil.sklearn import GaussianMixtureModel - super(LYAMaker, self).__init__(survey=survey) + super(LYAMaker, self).__init__(survey=survey, legacy_dir=legacy_dir) self.seed = seed self.objtype = 'LYA' @@ -3664,11 +3664,11 @@ class LRGMaker(SelectTargets): wave, template_maker = None, None GMM_LRG, KDTree_north, KDTree_south = None, None, None - def __init__(self, seed=None, nside_chunk=128, survey='main', **kwargs): + def __init__(self, seed=None, nside_chunk=128, survey='main', legacy_dir=None, **kwargs): from desisim.templates import LRG from desiutil.sklearn import GaussianMixtureModel - super(LRGMaker, self).__init__(survey=survey) + super(LRGMaker, self).__init__(survey=survey, legacy_dir=legacy_dir) self.seed = seed self.nside_chunk = nside_chunk @@ -3892,11 +3892,11 @@ class ELGMaker(SelectTargets): wave, template_maker = None, None GMM_ELG, KDTree_north, KDTree_south = None, None, None - def __init__(self, seed=None, nside_chunk=128, survey='main', **kwargs): + def __init__(self, seed=None, nside_chunk=128, survey='main', legacy_dir=None, **kwargs): from desisim.templates import ELG from desiutil.sklearn import GaussianMixtureModel - super(ELGMaker, self).__init__(survey=survey) + super(ELGMaker, self).__init__(survey=survey, legacy_dir=legacy_dir) self.seed = seed self.nside_chunk = nside_chunk @@ -4113,8 +4113,8 @@ class BGSMaker(SelectTargets): """ wave, template_maker, GMM_BGS, KDTree = None, None, None, None - def __init__(self, seed=None, nside_chunk=128, survey='main', **kwargs): - super(BGSMaker, self).__init__(survey=survey) + def __init__(self, seed=None, nside_chunk=128, survey='main', legacy_dir=None, **kwargs): + super(BGSMaker, self).__init__(survey=survey, legacy_dir=legacy_dir) self.seed = seed self.nside_chunk = nside_chunk @@ -4197,7 +4197,8 @@ def read(self, mockfile=None, mockformat='durham_mxxl_hdf5', healpixels=None, data = MockReader.readmock(mockfile, target_name=self.objtype, healpixels=healpixels, nside=nside, magcut=magcut, only_coords=only_coords, - mock_density=mock_density, seed=self.seed) + mock_density=mock_density, seed=self.seed, + legacy_dir=self.legacy_dir) return data @@ -4336,15 +4337,15 @@ class STARMaker(SelectTargets): star_maggies_g_north, star_maggies_r_north = None, None star_maggies_g_south, star_maggies_r_south = None, None - def __init__(self, seed=None, no_spectra=False, survey='main', **kwargs): + def __init__(self, seed=None, no_spectra=False, survey='main', legacy_dir=None, **kwargs): from speclite import filters from desisim.templates import STAR - super(STARMaker, self).__init__(survey=survey) + super(STARMaker, self).__init__(survey=survey, legacy_dir=legacy_dir) self.seed = seed self.objtype = 'STAR' - + if self.wave is None: STARMaker.wave = _default_wave() self.extinction = self.mw_dust_extinction() @@ -4497,9 +4498,9 @@ class MWS_MAINMaker(STARMaker): """ def __init__(self, seed=None, calib_only=False, no_spectra=False, - survey='main', **kwargs): + survey='main', legacy_dir=None, **kwargs): super(MWS_MAINMaker, self).__init__(seed=seed, no_spectra=no_spectra, - survey=survey) + survey=survey, legacy_dir=legacy_dir) self.seed = seed self.calib_only = calib_only @@ -4711,9 +4712,9 @@ class MWS_NEARBYMaker(STARMaker): (main survey) and `sv1` (first iteration of SV). Defaults to `main`. """ - def __init__(self, seed=None, no_spectra=False, survey='main', **kwargs): + def __init__(self, seed=None, no_spectra=False, survey='main', legacy_dir=None, **kwargs): super(MWS_NEARBYMaker, self).__init__(seed=seed, no_spectra=no_spectra, - survey=survey) + survey=survey, legacy_dir=legacy_dir) def read(self, mockfile=None, mockformat='mws_100pc', healpixels=None, nside=None, mock_density=False, **kwargs): @@ -4898,11 +4899,11 @@ class WDMaker(SelectTargets): wd_maggies_db_south, wd_maggies_db_south = None, None def __init__(self, seed=None, calib_only=False, no_spectra=False, - survey='main', **kwargs): + survey='main', legacy_dir=None, **kwargs): from speclite import filters from desisim.templates import WD - super(WDMaker, self).__init__(survey=survey) + super(WDMaker, self).__init__(survey=survey, legacy_dir=legacy_dir) self.seed = seed self.objtype = 'WD' @@ -5223,8 +5224,8 @@ class SKYMaker(SelectTargets): """ wave = None - def __init__(self, seed=None, survey='main', **kwargs): - super(SKYMaker, self).__init__(survey=survey) + def __init__(self, seed=None, survey='main', legacy_dir=None, **kwargs): + super(SKYMaker, self).__init__(survey=survey, legacy_dir=legacy_dir) self.seed = seed self.objtype = 'SKY' @@ -5372,8 +5373,8 @@ class BuzzardMaker(SelectTargets): wave, template_maker = None, None def __init__(self, seed=None, nside_chunk=128, no_spectra=False, - survey='main', **kwargs): - super(BuzzardMaker, self).__init__(survey=survey) + survey='main', legacy_dir=None, **kwargs): + super(BuzzardMaker, self).__init__(survey=survey, legacy_dir=legacy_dir) self.seed = seed self.objtype = 'BGS' From c609190491891034a0a6dcf1ec68ce5c5af888f4 Mon Sep 17 00:00:00 2001 From: michaelJwilson Date: Tue, 28 Apr 2020 10:14:08 -0700 Subject: [PATCH 6/8] Add legacy_dir propagation for other tracers. --- .../data/select-mock-targets-no-contam.yaml | 2 +- py/desitarget/mock/log.txt | 155 ++++++++++++++++++ py/desitarget/mock/log_simple.txt | 147 +++++++++++++++++ py/desitarget/mock/mockmaker.py | 60 ++++--- py/desitarget/mock/test.sh | 5 + py/desitarget/randoms.py | 8 +- 6 files changed, 350 insertions(+), 27 deletions(-) create mode 100644 py/desitarget/mock/log.txt create mode 100644 py/desitarget/mock/log_simple.txt create mode 100644 py/desitarget/mock/test.sh diff --git a/py/desitarget/mock/data/select-mock-targets-no-contam.yaml b/py/desitarget/mock/data/select-mock-targets-no-contam.yaml index 89fab5b1c..e8da1202a 100644 --- a/py/desitarget/mock/data/select-mock-targets-no-contam.yaml +++ b/py/desitarget/mock/data/select-mock-targets-no-contam.yaml @@ -1,5 +1,5 @@ # Parameter file for desitarget/bin/select_mock_targets, including contaminants. -legacy_dir: /global/project/projectdirs/cosmo/data/legacysurvey/dr8/ +# legacy_dir: /global/project/projectdirs/cosmo/data/legacysurvey/dr8/ targets: BGS: { diff --git a/py/desitarget/mock/log.txt b/py/desitarget/mock/log.txt new file mode 100644 index 000000000..404159643 --- /dev/null +++ b/py/desitarget/mock/log.txt @@ -0,0 +1,155 @@ +20:35:22 +INFO:select_mock_targets:44:: Starting select_mock_targets at Wed Apr 15 20:35:25 2020 +INFO:select_mock_targets:74:: Processing 1 pixel(s). +INFO:select_mock_targets:95:: Reading configuration file data/select-mock-targets-no-contam.yaml +INFO:select_mock_targets:99:: Calling targets_truth with survey=main at Wed Apr 15 20:35:25 2020 +WARNING:build.py:87:initialize_targets_truth: Output directory /global/cscratch1/sd/mjwilson/trash is not empty. +INFO:build.py:91:initialize_targets_truth: Writing to output directory /global/cscratch1/sd/mjwilson/trash +INFO:build.py:95:initialize_targets_truth: Processing 1 healpixel(s) (nside = 64, 0.839 deg2/pixel) spanning 0.839 deg2. +INFO:build.py:846:targets_truth: Initializing and caching all MockMaker classes. +INFO:io.py:959:read_basis_templates: Reading /global/cfs/cdirs/desi/spectro/templates/basis_templates/v3.2/bgs_templates_v2.3.fits +INFO:io.py:959:read_basis_templates: Reading /global/cfs/cdirs/desi/spectro/templates/basis_templates/v3.2/elg_templates_v2.2.fits +INFO:io.py:959:read_basis_templates: Reading /global/cfs/cdirs/desi/spectro/templates/basis_templates/v3.2/lrg_templates_v2.2.fits +WARNING:templates.py:2350:__init__: Using default SIMQSO model +INFO:io.py:959:read_basis_templates: Reading /global/cfs/cdirs/desi/spectro/templates/basis_templates/v3.2/star_templates_v3.1.fits +INFO:mockmaker.py:4366:__init__: Caching stellar template photometry. +WARNING:templates.py:2350:__init__: Using default SIMQSO model +INFO:io.py:959:read_basis_templates: Reading /global/cfs/cdirs/desi/spectro/templates/basis_templates/v3.2/wd_templates_v2.2.fits +INFO:io.py:959:read_basis_templates: Reading /global/cfs/cdirs/desi/spectro/templates/basis_templates/v3.2/wd_templates_v2.2.fits +INFO:mockmaker.py:4933:__init__: Caching WD template photometry. +INFO:build.py:891:targets_truth: Working on healpixel 0 +INFO:build.py:904:targets_truth: Working on target class BGS on healpixel 0 +INFO:build.py:162:read_mock: Target: BGS, type: BGS, format: durham_mxxl_hdf5, mockfile: {DESI_ROOT}/mocks/bgs/MXXL/full_sky/v0.0.4/BGS_r20.6.hdf5 +INFO:mockmaker.py:2545:_read_mockfile: Assigning healpix pixels with nside = 64 +INFO:mockmaker.py:2580:readmock: Trimmed to 2653 BGSs in 1 healpixel(s). +INFO:mockmaker.py:2608:readmock: Trimmed to 1950 BGSs with r < 20.3. +INFO:mockmaker.py:2621:readmock: Sampling from BGS Gaussian mixture model. +INFO:mockmaker.py:413:imaging_depth: Setting realistic imaging depths (including MASKBITS). +INFO:mockmaker.py:489:imaging_depth: Removing 0 targets not in reduced DESI imaging (of 1950). +INFO:build.py:400:density_fluctuations: Dividing each nside=64 healpixel (0.84 deg2) into 4 nside=128 healpixel(s) (0.21 deg2). +INFO:build.py:510:get_spectra: Goal: Generate spectra for 1950 BGS targets (2323.38 / deg2). +INFO:build.py:586:get_spectra: Done: Generated spectra for 1336 BGS targets (1591.81 / deg2). +INFO:build.py:588:get_spectra: Total time for BGSs = 0.638 minutes (1459.554 cpu minutes/deg2). +INFO:build.py:904:targets_truth: Working on target class ELG on healpixel 0 +INFO:build.py:162:read_mock: Target: ELG, type: ELG, format: gaussianfield, mockfile: {DESI_ROOT}/mocks/DarkSky/v1.0.1/elg_0_inpt.fits +INFO:mockmaker.py:230:_get_radec: Reading /global/cfs/cdirs/desi/mocks/DarkSky/v1.0.1/elg_0_inpt.fits +INFO:mockmaker.py:235:_get_radec: Assigning healpix pixels with nside = 64. +INFO:mockmaker.py:1480:readmock: Trimmed to 4324 ELGs in 1 healpixel(s) +INFO:mockmaker.py:1520:readmock: Sampling from ELG Gaussian mixture model. +INFO:mockmaker.py:413:imaging_depth: Setting realistic imaging depths (including MASKBITS). +INFO:mockmaker.py:489:imaging_depth: Removing 0 targets not in reduced DESI imaging (of 4324). +INFO:build.py:216:read_mock: Computed median mock density for ELGs of 4863.61 targets/deg2. +INFO:build.py:218:read_mock: Target density = 2200.00 targets/deg2 (downsampling factor = 0.452). +INFO:build.py:400:density_fluctuations: Dividing each nside=64 healpixel (0.84 deg2) into 4 nside=128 healpixel(s) (0.21 deg2). +INFO:build.py:510:get_spectra: Goal: Generate spectra for 1955 ELG targets (2329.34 / deg2). +INFO:build.py:586:get_spectra: Done: Generated spectra for 1955 ELG targets (2329.34 / deg2). +INFO:build.py:588:get_spectra: Total time for ELGs = 0.865 minutes (1979.481 cpu minutes/deg2). +INFO:build.py:904:targets_truth: Working on target class LRG on healpixel 0 +INFO:build.py:162:read_mock: Target: LRG, type: LRG, format: gaussianfield, mockfile: {DESI_ROOT}/mocks/DarkSky/v1.0.1/lrg_0_inpt.fits +INFO:mockmaker.py:230:_get_radec: Reading /global/cfs/cdirs/desi/mocks/DarkSky/v1.0.1/lrg_0_inpt.fits +INFO:mockmaker.py:235:_get_radec: Assigning healpix pixels with nside = 64. +INFO:mockmaker.py:1480:readmock: Trimmed to 575 LRGs in 1 healpixel(s) +INFO:mockmaker.py:1520:readmock: Sampling from LRG Gaussian mixture model. +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +INFO:mockmaker.py:413:imaging_depth: Setting realistic imaging depths (including MASKBITS). +INFO:mockmaker.py:489:imaging_depth: Removing 0 targets not in reduced DESI imaging (of 575). +INFO:build.py:216:read_mock: Computed median mock density for LRGs of 770.89 targets/deg2. +INFO:build.py:218:read_mock: Target density = 480.00 targets/deg2 (downsampling factor = 0.623). +INFO:build.py:400:density_fluctuations: Dividing each nside=64 healpixel (0.84 deg2) into 4 nside=128 healpixel(s) (0.21 deg2). +INFO:build.py:510:get_spectra: Goal: Generate spectra for 358 LRG targets (426.55 / deg2). +INFO:build.py:586:get_spectra: Done: Generated spectra for 358 LRG targets (426.55 / deg2). +INFO:build.py:588:get_spectra: Total time for LRGs = 0.695 minutes (1590.212 cpu minutes/deg2). +INFO:build.py:904:targets_truth: Working on target class LYA on healpixel 0 +INFO:build.py:162:read_mock: Target: LYA, type: QSO, format: CoLoRe, mockfile: {DESI_ROOT}/mocks/lya_forest/london/v9.0/v9.0.0/master.fits +INFO:mockmaker.py:2309:readmock: Reading /global/cfs/cdirs/desi/mocks/lya_forest/london/v9.0/v9.0.0/master.fits +INFO:mockmaker.py:2329:readmock: Assigning healpix pixels with nside = 64 +INFO:mockmaker.py:2345:readmock: Trimmed to 167 LYAs in 1 healpixel(s) +INFO:mockmaker.py:2361:readmock: Trimmed to 167 LYAs with z>=1.800 +INFO:mockmaker.py:2424:readmock: Sampling from LYA Gaussian mixture model. +INFO:mockmaker.py:413:imaging_depth: Setting realistic imaging depths (including MASKBITS). +INFO:mockmaker.py:489:imaging_depth: Removing 0 targets not in reduced DESI imaging (of 167). +INFO:build.py:203:read_mock: Density adjustment factor for target type LYA: 1.495. +INFO:build.py:216:read_mock: Computed median mock density for LYAs of 181.10 targets/deg2. +INFO:build.py:218:read_mock: Target density = 50.00 targets/deg2 (downsampling factor = 0.413). +INFO:build.py:400:density_fluctuations: Dividing each nside=64 healpixel (0.84 deg2) into 4 nside=128 healpixel(s) (0.21 deg2). +INFO:build.py:510:get_spectra: Goal: Generate spectra for 69 LYA targets (82.21 / deg2). +INFO:build.py:586:get_spectra: Done: Generated spectra for 69 LYA targets (82.21 / deg2). +INFO:build.py:588:get_spectra: Total time for LYAs = 0.567 minutes (1297.358 cpu minutes/deg2). +INFO:build.py:904:targets_truth: Working on target class MWS_MAIN on healpixel 0 +INFO:build.py:162:read_mock: Target: MWS_MAIN, type: STAR, format: galaxia, mockfile: {DESI_ROOT}/mocks/mws/galaxia/alpha/v0.0.6/healpix +INFO:mockmaker.py:2053:readmock: Reading /global/cfs/cdirs/desi/mocks/mws/galaxia/alpha/v0.0.6/healpix/8/0/0/mock_allsky_galaxia_desi-8-0.fits +INFO:mockmaker.py:2074:readmock: Trimmed to 1660 MWS_MAINs in 1 healpixel(s) +INFO:mockmaker.py:413:imaging_depth: Setting realistic imaging depths (including MASKBITS). +INFO:mockmaker.py:489:imaging_depth: Removing 0 targets not in reduced DESI imaging (of 1660). +INFO:build.py:400:density_fluctuations: Dividing each nside=64 healpixel (0.84 deg2) into 4 nside=128 healpixel(s) (0.21 deg2). +INFO:build.py:510:get_spectra: Goal: Generate spectra for 1660 MWS_MAIN targets (1977.85 / deg2). +INFO:build.py:586:get_spectra: Done: Generated spectra for 1336 MWS_MAIN targets (1591.81 / deg2). +INFO:build.py:588:get_spectra: Total time for MWS_MAINs = 0.606 minutes (1386.798 cpu minutes/deg2). +INFO:build.py:904:targets_truth: Working on target class MWS_NEARBY on healpixel 0 +INFO:build.py:162:read_mock: Target: MWS_NEARBY, type: STAR, format: mws_100pc, mockfile: {DESI_ROOT}/mocks/mws/100pc/v0.0.4/mock_100pc.fits +INFO:mockmaker.py:230:_get_radec: Reading /global/cfs/cdirs/desi/mocks/mws/100pc/v0.0.4/mock_100pc.fits +INFO:mockmaker.py:235:_get_radec: Assigning healpix pixels with nside = 64. +INFO:mockmaker.py:3027:readmock: Trimmed to 4 MWS_NEARBYs in 1 healpixel(s). +INFO:mockmaker.py:413:imaging_depth: Setting realistic imaging depths (including MASKBITS). +INFO:mockmaker.py:489:imaging_depth: Removing 0 targets not in reduced DESI imaging (of 4). +INFO:build.py:400:density_fluctuations: Dividing each nside=64 healpixel (0.84 deg2) into 4 nside=128 healpixel(s) (0.21 deg2). +INFO:build.py:510:get_spectra: Goal: Generate spectra for 4 MWS_NEARBY targets (4.77 / deg2). +INFO:build.py:586:get_spectra: Done: Generated spectra for 4 MWS_NEARBY targets (4.77 / deg2). +INFO:build.py:588:get_spectra: Total time for MWS_NEARBYs = 0.600 minutes (1373.207 cpu minutes/deg2). +INFO:build.py:904:targets_truth: Working on target class QSO on healpixel 0 +INFO:build.py:162:read_mock: Target: QSO, type: QSO, format: gaussianfield, mockfile: {DESI_ROOT}/mocks/DarkSky/v1.0.1/qso_0_inpt.fits +INFO:mockmaker.py:230:_get_radec: Reading /global/cfs/cdirs/desi/mocks/DarkSky/v1.0.1/qso_0_inpt.fits +INFO:mockmaker.py:235:_get_radec: Assigning healpix pixels with nside = 64. +INFO:mockmaker.py:1480:readmock: Trimmed to 272 QSOs in 1 healpixel(s) +INFO:mockmaker.py:1500:readmock: Trimmed to 220 objects with z<1.800 +INFO:mockmaker.py:1520:readmock: Sampling from QSO Gaussian mixture model. +INFO:mockmaker.py:413:imaging_depth: Setting realistic imaging depths (including MASKBITS). +INFO:mockmaker.py:489:imaging_depth: Removing 0 targets not in reduced DESI imaging (of 220). +INFO:build.py:203:read_mock: Density adjustment factor for target type QSO: 0.793. +INFO:build.py:216:read_mock: Computed median mock density for QSOs of 266.89 targets/deg2. +INFO:build.py:218:read_mock: Target density = 120.00 targets/deg2 (downsampling factor = 0.356). +INFO:build.py:400:density_fluctuations: Dividing each nside=64 healpixel (0.84 deg2) into 4 nside=128 healpixel(s) (0.21 deg2). +INFO:build.py:510:get_spectra: Goal: Generate spectra for 78 QSO targets (92.94 / deg2). +INFO:build.py:586:get_spectra: Done: Generated spectra for 78 QSO targets (92.94 / deg2). +INFO:build.py:588:get_spectra: Total time for QSOs = 0.587 minutes (1342.481 cpu minutes/deg2). +INFO:build.py:904:targets_truth: Working on target class SKY on healpixel 0 +INFO:build.py:162:read_mock: Target: SKY, type: SKY, format: uniformsky, mockfile: {DESI_ROOT}/mocks/uniformsky/0.2/uniformsky-2048-0.2.fits +INFO:mockmaker.py:230:_get_radec: Reading /global/cfs/cdirs/desi/mocks/uniformsky/0.2/uniformsky-2048-0.2.fits +INFO:mockmaker.py:235:_get_radec: Assigning healpix pixels with nside = 64. +INFO:mockmaker.py:1902:readmock: Trimmed to 1024 SKYs in 1 healpixel(s). +INFO:mockmaker.py:413:imaging_depth: Setting realistic imaging depths (including MASKBITS). +INFO:mockmaker.py:489:imaging_depth: Removing 0 targets not in reduced DESI imaging (of 1024). +INFO:build.py:400:density_fluctuations: Dividing each nside=64 healpixel (0.84 deg2) into 4 nside=128 healpixel(s) (0.21 deg2). +INFO:build.py:510:get_spectra: Goal: Generate spectra for 1024 SKY targets (1220.07 / deg2). +INFO:build.py:586:get_spectra: Done: Generated spectra for 1024 SKY targets (1220.07 / deg2). +INFO:build.py:588:get_spectra: Total time for SKYs = 0.689 minutes (1575.107 cpu minutes/deg2). +INFO:build.py:904:targets_truth: Working on target class WD on healpixel 0 +INFO:build.py:162:read_mock: Target: WD, type: WD, format: mws_wd, mockfile: {DESI_ROOT}/mocks/mws/wd/v1.0.0/mock_wd.fits +INFO:mockmaker.py:230:_get_radec: Reading /global/cfs/cdirs/desi/mocks/mws/wd/v1.0.0/mock_wd.fits +INFO:mockmaker.py:235:_get_radec: Assigning healpix pixels with nside = 64. +INFO:mockmaker.py:2868:readmock: Trimmed to 2 WDs in 1 healpixel(s). +INFO:mockmaker.py:413:imaging_depth: Setting realistic imaging depths (including MASKBITS). +INFO:mockmaker.py:489:imaging_depth: Removing 0 targets not in reduced DESI imaging (of 2). +INFO:build.py:400:density_fluctuations: Dividing each nside=64 healpixel (0.84 deg2) into 4 nside=128 healpixel(s) (0.21 deg2). +INFO:build.py:510:get_spectra: Goal: Generate spectra for 2 WD targets (2.38 / deg2). +INFO:build.py:586:get_spectra: Done: Generated spectra for 1 WD targets (1.19 / deg2). +INFO:build.py:588:get_spectra: Total time for WDs = 0.708 minutes (1619.876 cpu minutes/deg2). +INFO:build.py:1037:finish_catalog: Summary: ntargets = 5137 (6120.62 targets/deg2), nsky = 1024 (1220.07 targets/deg2) in pixel 0. +INFO:build.py:982:targets_truth: 2677 targets written to /global/cscratch1/sd/mjwilson/trash/0/0/bright/targets-bright-64-0.fits +INFO:build.py:982:targets_truth: 4055 targets written to /global/cscratch1/sd/mjwilson/trash/0/0/dark/targets-dark-64-0.fits +INFO:build.py:987:targets_truth: Writing 1024 SKY targets to /global/cscratch1/sd/mjwilson/trash/0/0/sky-64-0.fits +INFO:select_mock_targets:103:: All done at Wed Apr 15 20:56:34 2020 +20:56:36 diff --git a/py/desitarget/mock/log_simple.txt b/py/desitarget/mock/log_simple.txt new file mode 100644 index 000000000..89eea2e82 --- /dev/null +++ b/py/desitarget/mock/log_simple.txt @@ -0,0 +1,147 @@ +21:04:38 +INFO:select_mock_targets:44:: Starting select_mock_targets at Wed Apr 15 21:04:44 2020 +INFO:select_mock_targets:74:: Processing 1 pixel(s). +INFO:select_mock_targets:95:: Reading configuration file data/select-mock-targets-no-contam.yaml +INFO:select_mock_targets:99:: Calling targets_truth with survey=main at Wed Apr 15 21:04:44 2020 +WARNING:build.py:87:initialize_targets_truth: Output directory /global/cscratch1/sd/mjwilson/trash is not empty. +INFO:build.py:91:initialize_targets_truth: Writing to output directory /global/cscratch1/sd/mjwilson/trash +INFO:build.py:95:initialize_targets_truth: Processing 1 healpixel(s) (nside = 64, 0.839 deg2/pixel) spanning 0.839 deg2. +INFO:build.py:846:targets_truth: Initializing and caching all MockMaker classes. +INFO:io.py:959:read_basis_templates: Reading /global/cfs/cdirs/desi/spectro/templates/basis_templates/v3.2/bgs_templates_v2.3.fits +INFO:io.py:959:read_basis_templates: Reading /global/cfs/cdirs/desi/spectro/templates/basis_templates/v3.2/elg_templates_v2.2.fits +INFO:io.py:959:read_basis_templates: Reading /global/cfs/cdirs/desi/spectro/templates/basis_templates/v3.2/lrg_templates_v2.2.fits +WARNING:templates.py:2350:__init__: Using default SIMQSO model +INFO:io.py:959:read_basis_templates: Reading /global/cfs/cdirs/desi/spectro/templates/basis_templates/v3.2/star_templates_v3.1.fits +INFO:mockmaker.py:4366:__init__: Caching stellar template photometry. +WARNING:templates.py:2350:__init__: Using default SIMQSO model +INFO:io.py:959:read_basis_templates: Reading /global/cfs/cdirs/desi/spectro/templates/basis_templates/v3.2/wd_templates_v2.2.fits +INFO:io.py:959:read_basis_templates: Reading /global/cfs/cdirs/desi/spectro/templates/basis_templates/v3.2/wd_templates_v2.2.fits +INFO:mockmaker.py:4933:__init__: Caching WD template photometry. +INFO:build.py:891:targets_truth: Working on healpixel 0 +INFO:build.py:904:targets_truth: Working on target class BGS on healpixel 0 +INFO:build.py:162:read_mock: Target: BGS, type: BGS, format: durham_mxxl_hdf5, mockfile: {DESI_ROOT}/mocks/bgs/MXXL/full_sky/v0.0.4/BGS_r20.6.hdf5 +INFO:mockmaker.py:2545:_read_mockfile: Assigning healpix pixels with nside = 64 +INFO:mockmaker.py:2580:readmock: Trimmed to 2653 BGSs in 1 healpixel(s). +INFO:mockmaker.py:2608:readmock: Trimmed to 1950 BGSs with r < 20.3. +INFO:mockmaker.py:2621:readmock: Sampling from BGS Gaussian mixture model. +INFO:mockmaker.py:503:imaging_depth: Setting simple imaging depths. +INFO:build.py:400:density_fluctuations: Dividing each nside=64 healpixel (0.84 deg2) into 4 nside=128 healpixel(s) (0.21 deg2). +INFO:build.py:510:get_spectra: Goal: Generate spectra for 1950 BGS targets (2323.38 / deg2). +INFO:build.py:586:get_spectra: Done: Generated spectra for 1365 BGS targets (1626.37 / deg2). +INFO:build.py:588:get_spectra: Total time for BGSs = 0.507 minutes (1160.167 cpu minutes/deg2). +INFO:build.py:904:targets_truth: Working on target class ELG on healpixel 0 +INFO:build.py:162:read_mock: Target: ELG, type: ELG, format: gaussianfield, mockfile: {DESI_ROOT}/mocks/DarkSky/v1.0.1/elg_0_inpt.fits +INFO:mockmaker.py:230:_get_radec: Reading /global/cfs/cdirs/desi/mocks/DarkSky/v1.0.1/elg_0_inpt.fits +INFO:mockmaker.py:235:_get_radec: Assigning healpix pixels with nside = 64. +INFO:mockmaker.py:1480:readmock: Trimmed to 4324 ELGs in 1 healpixel(s) +INFO:mockmaker.py:1520:readmock: Sampling from ELG Gaussian mixture model. +INFO:mockmaker.py:503:imaging_depth: Setting simple imaging depths. +INFO:build.py:216:read_mock: Computed median mock density for ELGs of 4863.61 targets/deg2. +INFO:build.py:218:read_mock: Target density = 2200.00 targets/deg2 (downsampling factor = 0.452). +INFO:build.py:400:density_fluctuations: Dividing each nside=64 healpixel (0.84 deg2) into 4 nside=128 healpixel(s) (0.21 deg2). +INFO:build.py:510:get_spectra: Goal: Generate spectra for 1955 ELG targets (2329.34 / deg2). +INFO:build.py:586:get_spectra: Done: Generated spectra for 1955 ELG targets (2329.34 / deg2). +INFO:build.py:588:get_spectra: Total time for ELGs = 0.818 minutes (1870.664 cpu minutes/deg2). +INFO:build.py:904:targets_truth: Working on target class LRG on healpixel 0 +INFO:build.py:162:read_mock: Target: LRG, type: LRG, format: gaussianfield, mockfile: {DESI_ROOT}/mocks/DarkSky/v1.0.1/lrg_0_inpt.fits +INFO:mockmaker.py:230:_get_radec: Reading /global/cfs/cdirs/desi/mocks/DarkSky/v1.0.1/lrg_0_inpt.fits +INFO:mockmaker.py:235:_get_radec: Assigning healpix pixels with nside = 64. +INFO:mockmaker.py:1480:readmock: Trimmed to 575 LRGs in 1 healpixel(s) +INFO:mockmaker.py:1520:readmock: Sampling from LRG Gaussian mixture model. +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +WARNING:cuts.py:292:isLRG_colors: Setting zfiberflux to zflux!!! +INFO:mockmaker.py:503:imaging_depth: Setting simple imaging depths. +INFO:build.py:216:read_mock: Computed median mock density for LRGs of 770.89 targets/deg2. +INFO:build.py:218:read_mock: Target density = 480.00 targets/deg2 (downsampling factor = 0.623). +INFO:build.py:400:density_fluctuations: Dividing each nside=64 healpixel (0.84 deg2) into 4 nside=128 healpixel(s) (0.21 deg2). +INFO:build.py:510:get_spectra: Goal: Generate spectra for 358 LRG targets (426.55 / deg2). +INFO:build.py:586:get_spectra: Done: Generated spectra for 358 LRG targets (426.55 / deg2). +INFO:build.py:588:get_spectra: Total time for LRGs = 0.645 minutes (1474.886 cpu minutes/deg2). +INFO:build.py:904:targets_truth: Working on target class LYA on healpixel 0 +INFO:build.py:162:read_mock: Target: LYA, type: QSO, format: CoLoRe, mockfile: {DESI_ROOT}/mocks/lya_forest/london/v9.0/v9.0.0/master.fits +INFO:mockmaker.py:2309:readmock: Reading /global/cfs/cdirs/desi/mocks/lya_forest/london/v9.0/v9.0.0/master.fits +INFO:mockmaker.py:2329:readmock: Assigning healpix pixels with nside = 64 +INFO:mockmaker.py:2345:readmock: Trimmed to 167 LYAs in 1 healpixel(s) +INFO:mockmaker.py:2361:readmock: Trimmed to 167 LYAs with z>=1.800 +INFO:mockmaker.py:2424:readmock: Sampling from LYA Gaussian mixture model. +INFO:mockmaker.py:503:imaging_depth: Setting simple imaging depths. +INFO:build.py:203:read_mock: Density adjustment factor for target type LYA: 1.495. +INFO:build.py:216:read_mock: Computed median mock density for LYAs of 181.10 targets/deg2. +INFO:build.py:218:read_mock: Target density = 50.00 targets/deg2 (downsampling factor = 0.413). +INFO:build.py:400:density_fluctuations: Dividing each nside=64 healpixel (0.84 deg2) into 4 nside=128 healpixel(s) (0.21 deg2). +INFO:build.py:510:get_spectra: Goal: Generate spectra for 69 LYA targets (82.21 / deg2). +INFO:build.py:586:get_spectra: Done: Generated spectra for 69 LYA targets (82.21 / deg2). +INFO:build.py:588:get_spectra: Total time for LYAs = 0.602 minutes (1377.425 cpu minutes/deg2). +INFO:build.py:904:targets_truth: Working on target class MWS_MAIN on healpixel 0 +INFO:build.py:162:read_mock: Target: MWS_MAIN, type: STAR, format: galaxia, mockfile: {DESI_ROOT}/mocks/mws/galaxia/alpha/v0.0.6/healpix +INFO:mockmaker.py:2053:readmock: Reading /global/cfs/cdirs/desi/mocks/mws/galaxia/alpha/v0.0.6/healpix/8/0/0/mock_allsky_galaxia_desi-8-0.fits +INFO:mockmaker.py:2074:readmock: Trimmed to 1660 MWS_MAINs in 1 healpixel(s) +INFO:mockmaker.py:503:imaging_depth: Setting simple imaging depths. +INFO:build.py:400:density_fluctuations: Dividing each nside=64 healpixel (0.84 deg2) into 4 nside=128 healpixel(s) (0.21 deg2). +INFO:build.py:510:get_spectra: Goal: Generate spectra for 1660 MWS_MAIN targets (1977.85 / deg2). +INFO:build.py:586:get_spectra: Done: Generated spectra for 1338 MWS_MAIN targets (1594.20 / deg2). +INFO:build.py:588:get_spectra: Total time for MWS_MAINs = 0.549 minutes (1255.991 cpu minutes/deg2). +INFO:build.py:904:targets_truth: Working on target class MWS_NEARBY on healpixel 0 +INFO:build.py:162:read_mock: Target: MWS_NEARBY, type: STAR, format: mws_100pc, mockfile: {DESI_ROOT}/mocks/mws/100pc/v0.0.4/mock_100pc.fits +INFO:mockmaker.py:230:_get_radec: Reading /global/cfs/cdirs/desi/mocks/mws/100pc/v0.0.4/mock_100pc.fits +INFO:mockmaker.py:235:_get_radec: Assigning healpix pixels with nside = 64. +INFO:mockmaker.py:3027:readmock: Trimmed to 4 MWS_NEARBYs in 1 healpixel(s). +INFO:mockmaker.py:503:imaging_depth: Setting simple imaging depths. +INFO:build.py:400:density_fluctuations: Dividing each nside=64 healpixel (0.84 deg2) into 4 nside=128 healpixel(s) (0.21 deg2). +INFO:build.py:510:get_spectra: Goal: Generate spectra for 4 MWS_NEARBY targets (4.77 / deg2). +INFO:build.py:586:get_spectra: Done: Generated spectra for 4 MWS_NEARBY targets (4.77 / deg2). +INFO:build.py:588:get_spectra: Total time for MWS_NEARBYs = 0.568 minutes (1298.248 cpu minutes/deg2). +INFO:build.py:904:targets_truth: Working on target class QSO on healpixel 0 +INFO:build.py:162:read_mock: Target: QSO, type: QSO, format: gaussianfield, mockfile: {DESI_ROOT}/mocks/DarkSky/v1.0.1/qso_0_inpt.fits +INFO:mockmaker.py:230:_get_radec: Reading /global/cfs/cdirs/desi/mocks/DarkSky/v1.0.1/qso_0_inpt.fits +INFO:mockmaker.py:235:_get_radec: Assigning healpix pixels with nside = 64. +INFO:mockmaker.py:1480:readmock: Trimmed to 272 QSOs in 1 healpixel(s) +INFO:mockmaker.py:1500:readmock: Trimmed to 220 objects with z<1.800 +INFO:mockmaker.py:1520:readmock: Sampling from QSO Gaussian mixture model. +INFO:mockmaker.py:503:imaging_depth: Setting simple imaging depths. +INFO:build.py:203:read_mock: Density adjustment factor for target type QSO: 0.793. +INFO:build.py:216:read_mock: Computed median mock density for QSOs of 266.89 targets/deg2. +INFO:build.py:218:read_mock: Target density = 120.00 targets/deg2 (downsampling factor = 0.356). +INFO:build.py:400:density_fluctuations: Dividing each nside=64 healpixel (0.84 deg2) into 4 nside=128 healpixel(s) (0.21 deg2). +INFO:build.py:510:get_spectra: Goal: Generate spectra for 78 QSO targets (92.94 / deg2). +INFO:build.py:586:get_spectra: Done: Generated spectra for 78 QSO targets (92.94 / deg2). +INFO:build.py:588:get_spectra: Total time for QSOs = 0.686 minutes (1570.189 cpu minutes/deg2). +INFO:build.py:904:targets_truth: Working on target class SKY on healpixel 0 +INFO:build.py:162:read_mock: Target: SKY, type: SKY, format: uniformsky, mockfile: {DESI_ROOT}/mocks/uniformsky/0.2/uniformsky-2048-0.2.fits +INFO:mockmaker.py:230:_get_radec: Reading /global/cfs/cdirs/desi/mocks/uniformsky/0.2/uniformsky-2048-0.2.fits +INFO:mockmaker.py:235:_get_radec: Assigning healpix pixels with nside = 64. +INFO:mockmaker.py:1902:readmock: Trimmed to 1024 SKYs in 1 healpixel(s). +INFO:mockmaker.py:503:imaging_depth: Setting simple imaging depths. +INFO:build.py:400:density_fluctuations: Dividing each nside=64 healpixel (0.84 deg2) into 4 nside=128 healpixel(s) (0.21 deg2). +INFO:build.py:510:get_spectra: Goal: Generate spectra for 1024 SKY targets (1220.07 / deg2). +INFO:build.py:586:get_spectra: Done: Generated spectra for 1024 SKY targets (1220.07 / deg2). +INFO:build.py:588:get_spectra: Total time for SKYs = 0.633 minutes (1448.551 cpu minutes/deg2). +INFO:build.py:904:targets_truth: Working on target class WD on healpixel 0 +INFO:build.py:162:read_mock: Target: WD, type: WD, format: mws_wd, mockfile: {DESI_ROOT}/mocks/mws/wd/v1.0.0/mock_wd.fits +INFO:mockmaker.py:230:_get_radec: Reading /global/cfs/cdirs/desi/mocks/mws/wd/v1.0.0/mock_wd.fits +INFO:mockmaker.py:235:_get_radec: Assigning healpix pixels with nside = 64. +INFO:mockmaker.py:2868:readmock: Trimmed to 2 WDs in 1 healpixel(s). +INFO:mockmaker.py:503:imaging_depth: Setting simple imaging depths. +INFO:build.py:400:density_fluctuations: Dividing each nside=64 healpixel (0.84 deg2) into 4 nside=128 healpixel(s) (0.21 deg2). +INFO:build.py:510:get_spectra: Goal: Generate spectra for 2 WD targets (2.38 / deg2). +INFO:build.py:586:get_spectra: Done: Generated spectra for 1 WD targets (1.19 / deg2). +INFO:build.py:588:get_spectra: Total time for WDs = 0.627 minutes (1435.281 cpu minutes/deg2). +INFO:build.py:1037:finish_catalog: Summary: ntargets = 5168 (6157.56 targets/deg2), nsky = 1024 (1220.07 targets/deg2) in pixel 0. +INFO:build.py:982:targets_truth: 2708 targets written to /global/cscratch1/sd/mjwilson/trash/0/0/bright/targets-bright-64-0.fits +INFO:build.py:982:targets_truth: 4064 targets written to /global/cscratch1/sd/mjwilson/trash/0/0/dark/targets-dark-64-0.fits +INFO:build.py:987:targets_truth: Writing 1024 SKY targets to /global/cscratch1/sd/mjwilson/trash/0/0/sky-64-0.fits +INFO:select_mock_targets:103:: All done at Wed Apr 15 21:14:31 2020 +21:14:35 diff --git a/py/desitarget/mock/mockmaker.py b/py/desitarget/mock/mockmaker.py index 6fe096ed5..ac1707837 100644 --- a/py/desitarget/mock/mockmaker.py +++ b/py/desitarget/mock/mockmaker.py @@ -1379,7 +1379,7 @@ def __init__(self, **kwargs): def readmock(self, mockfile=None, healpixels=None, nside=None, zmax_qso=None, target_name='', mock_density=False, - only_coords=False, seed=None): + only_coords=False, seed=None, legacy_dir=None): """Read the catalog. Parameters @@ -1596,7 +1596,7 @@ def readmock(self, mockfile=None, healpixels=None, nside=None, # Add MW transmission and the imaging depth. self.mw_transmission(out) - self.imaging_depth(out) + self.imaging_depth(out, legacy_dir=legacy_dir) # Optionally compute the mean mock density. if mock_density: @@ -1612,7 +1612,8 @@ def __init__(self, **kwargs): super(ReadBuzzard, self).__init__(**kwargs) def readmock(self, mockfile=None, healpixels=[], nside=[], nside_buzzard=8, - target_name='', magcut=None, only_coords=False, seed=None): + target_name='', magcut=None, only_coords=False, seed=None, + legacy_dir=None): """Read the catalog. Parameters @@ -1793,7 +1794,7 @@ def readmock(self, mockfile=None, healpixels=[], nside=[], nside_buzzard=8, # Add MW transmission and the imaging depth. self.mw_transmission(out) - self.imaging_depth(out) + self.imaging_depth(out, legacy_dir=legacy_dir) ## Optionally compute the mean mock density. #if mock_density: @@ -1809,7 +1810,8 @@ def __init__(self, **kwargs): super(ReadUniformSky, self).__init__(**kwargs) def readmock(self, mockfile=None, healpixels=None, nside=None, - target_name='', mock_density=False, only_coords=False): + target_name='', mock_density=False, only_coords=False, + legacy_dir=None): """Read the catalog. Parameters @@ -1919,7 +1921,7 @@ def readmock(self, mockfile=None, healpixels=None, nside=None, # Add MW transmission and the imaging depth. self.mw_transmission(out) - self.imaging_depth(out) + self.imaging_depth(out, legacy_dir=legacy_dir) # Optionally compute the mean mock density. if mock_density: @@ -1936,7 +1938,7 @@ def __init__(self, **kwargs): def readmock(self, mockfile=None, healpixels=[], nside=[], nside_galaxia=8, target_name='MWS_MAIN', magcut=None, faintstar_mockfile=None, - faintstar_magcut=None, seed=None): + faintstar_magcut=None, seed=None, legacy_dir=None): """Read the catalog. Parameters @@ -2182,7 +2184,7 @@ def readmock(self, mockfile=None, healpixels=[], nside=[], nside_galaxia=8, # Add MW transmission and the imaging depth. self.mw_transmission(out) - self.imaging_depth(out) + self.imaging_depth(out, legacy_dir=legacy_dir) # Optionally include faint stars. if faintstar_mockfile is not None: @@ -2212,7 +2214,8 @@ def __init__(self, **kwargs): def readmock(self, mockfile=None, healpixels=None, nside=None, target_name='LYA', nside_lya=16, zmin_lya=None, - mock_density=False, sqmodel='default',only_coords=False, seed=None): + mock_density=False, sqmodel='default',only_coords=False, seed=None, + legacy_dir=None): """Read the catalog. Parameters @@ -2435,7 +2438,7 @@ def readmock(self, mockfile=None, healpixels=None, nside=None, # Add MW transmission and the imaging depth. self.mw_transmission(out) - self.imaging_depth(out) + self.imaging_depth(out, legacy_dir=legacy_dir) # Optionally compute the mean mock density. if mock_density: @@ -2648,7 +2651,8 @@ def __init__(self, **kwargs): super(ReadGAMA, self).__init__(**kwargs) def readmock(self, mockfile=None, healpixels=None, nside=None, - target_name='', magcut=None, only_coords=False): + target_name='', magcut=None, only_coords=False, + legacy_dir=None): """Read the catalog. Parameters @@ -2764,7 +2768,7 @@ def readmock(self, mockfile=None, healpixels=None, nside=None, # Add MW transmission and the imaging depth. self.mw_transmission(out) - self.imaging_depth(out) + self.imaging_depth(out, legacy_dir=legacy_dir) return out @@ -2776,7 +2780,7 @@ def __init__(self, **kwargs): super(ReadMWS_WD, self).__init__(**kwargs) def readmock(self, mockfile=None, healpixels=None, nside=None, - target_name='WD', mock_density=False): + target_name='WD', mock_density=False, legacy_dir=None): """Read the catalog. Parameters @@ -2920,7 +2924,7 @@ def readmock(self, mockfile=None, healpixels=None, nside=None, # Add MW transmission and the imaging depth. self.mw_transmission(out) - self.imaging_depth(out) + self.imaging_depth(out, legacy_dir=legacy_dir) # Optionally compute the mean mock density. if mock_density: @@ -2936,7 +2940,8 @@ def __init__(self, **kwargs): super(ReadMWS_NEARBY, self).__init__(**kwargs) def readmock(self, mockfile=None, healpixels=None, nside=None, - target_name='MWS_NEARBY', mock_density=False): + target_name='MWS_NEARBY', mock_density=False, + legacy_dir=None): """Read the catalog. Parameters @@ -3076,7 +3081,7 @@ def readmock(self, mockfile=None, healpixels=None, nside=None, # Add MW transmission and the imaging depth. self.mw_transmission(out) - self.imaging_depth(out) + self.imaging_depth(out, legacy_dir=legacy_dir) # Optionally compute the mean mock density. if mock_density: @@ -3174,7 +3179,8 @@ def read(self, mockfile=None, mockformat='gaussianfield', healpixels=None, data = MockReader.readmock(mockfile, target_name=self.objtype, healpixels=healpixels, nside=nside, only_coords=only_coords, seed=self.seed, - zmax_qso=zmax_qso, mock_density=mock_density) + zmax_qso=zmax_qso, mock_density=mock_density, + legacy_dir=self.legacy_dir) return data @@ -3398,7 +3404,8 @@ def read(self, mockfile=None, mockformat='CoLoRe', healpixels=None, nside=None, healpixels=healpixels, nside=nside, nside_lya=nside_lya, zmin_lya=zmin_lya, mock_density=mock_density, - only_coords=only_coords, seed=self.seed) + only_coords=only_coords, seed=self.seed, + legacy_dir=self.legacy_dir) return data def make_spectra(self, data=None, indx=None, seed=None,no_spectra=False,add_dlas=None,add_metals=None,add_lyb=None): @@ -3761,7 +3768,8 @@ def read(self, mockfile=None, mockformat='gaussianfield', healpixels=None, data = MockReader.readmock(mockfile, target_name=self.objtype, healpixels=healpixels, nside=nside, only_coords=only_coords, - mock_density=mock_density, seed=self.seed) + mock_density=mock_density, seed=self.seed, + legacy_dir=self.legacy_dir) return data @@ -3984,7 +3992,8 @@ def read(self, mockfile=None, mockformat='gaussianfield', healpixels=None, data = MockReader.readmock(mockfile, target_name=self.objtype, healpixels=healpixels, nside=nside, only_coords=only_coords, - mock_density=mock_density, seed=self.seed) + mock_density=mock_density, seed=self.seed, + legacy_dir=self.legacy_dir) return data @@ -4563,7 +4572,7 @@ def read(self, mockfile=None, mockformat='galaxia', healpixels=None, nside_galaxia=nside_galaxia, magcut=magcut, faintstar_mockfile=faintstar_mockfile, faintstar_magcut=faintstar_magcut, - seed=self.seed) + seed=self.seed, legacy_dir=self.legacy_dir) return data @@ -4758,7 +4767,7 @@ def read(self, mockfile=None, mockformat='mws_100pc', healpixels=None, data = MockReader.readmock(mockfile, target_name='MWS_NEARBY', healpixels=healpixels, nside=nside, - mock_density=mock_density) + mock_density=mock_density, legacy_dir=self.legacy_dir) return data @@ -5031,7 +5040,7 @@ def read(self, mockfile=None, mockformat='mws_wd', healpixels=None, data = MockReader.readmock(mockfile, target_name=self.objtype, healpixels=healpixels, nside=nside, - mock_density=mock_density) + mock_density=mock_density, legacy_dir=self.legacy_dir) return data @@ -5283,7 +5292,8 @@ def read(self, mockfile=None, mockformat='uniformsky', healpixels=None, data = MockReader.readmock(mockfile, target_name=self.objtype, healpixels=healpixels, nside=nside, only_coords=only_coords, - mock_density=mock_density) + mock_density=mock_density, + legacy_dir=self.legacy_dir) return data @@ -5555,7 +5565,7 @@ def read(self, mockfile=None, mockformat='buzzard', healpixels=None, healpixels=healpixels, nside=nside, nside_buzzard=nside_buzzard, only_coords=only_coords, - magcut=magcut) + magcut=magcut, legacy_dir=self.legacy_dir) return data diff --git a/py/desitarget/mock/test.sh b/py/desitarget/mock/test.sh new file mode 100644 index 000000000..0fbecfd4d --- /dev/null +++ b/py/desitarget/mock/test.sh @@ -0,0 +1,5 @@ +date +"%T" + +select_mock_targets --output_dir /global/cscratch1/sd/mjwilson/trash --nside 64 --config data/select-mock-targets-no-contam.yaml --healpixels 0 --no-spectra --overwrite + +date +"%T" diff --git a/py/desitarget/randoms.py b/py/desitarget/randoms.py index c7cf379bc..32db37a39 100644 --- a/py/desitarget/randoms.py +++ b/py/desitarget/randoms.py @@ -334,7 +334,13 @@ def quantities_at_positions_in_a_brick(ras, decs, brickname, drdir, iswcs = True # ADM get the quantity of interest at each location and # ADM store in a dictionary with the filter and quantity. - if qout == 'apflux': + if (qout == 'apflux'): + if aprad == 0.0: + qdict['apflux_ivar_'+filt] = np.zeros(npts, dtype=qform) + qdict[qout+'_'+filt] = np.zeros(npts, dtype=qform) + + continue + # ADM special treatment to photometer sky. # ADM Read in the ivar image. fnivar = fileform.format(brickname, 'invvar', filt, extn) From f347fa2522dc449638d625ece82b473bf9d2bd2e Mon Sep 17 00:00:00 2001 From: michaelJwilson Date: Tue, 28 Apr 2020 16:44:28 -0700 Subject: [PATCH 7/8] Update to better handle legacy lookup & get_legacy to facilitate local laptop tests. --- bin/select_mock_targets | 8 +++ .../data/select-mock-targets-no-contam.yaml | 1 + py/desitarget/mock/get_legacy.py | 52 +++++++++++++++++++ py/desitarget/mock/mockmaker.py | 35 +++++++++---- 4 files changed, 85 insertions(+), 11 deletions(-) create mode 100644 py/desitarget/mock/get_legacy.py diff --git a/bin/select_mock_targets b/bin/select_mock_targets index ae1260bc6..3330616d0 100755 --- a/bin/select_mock_targets +++ b/bin/select_mock_targets @@ -103,6 +103,14 @@ if 'legacy_dir' in params.keys(): # wget https://portal.nersc.gov/project/cosmo/data/legacysurvey/dr8/north/coadd/338/3380p295/ log.info('Assuming maskbits & depths based on LEGACYDIR={}'.format(params['legacy_dir'])) + # Check the survey bricks file exists. We'll use this to ensure that the local files are compelte for + # the healpixels being requested. + legacy_dir = params['legacy_dir'] + allbricks = legacy_dir + '/survey-bricks.fits.gz' + + if not os.path.isfile(allbricks): + raise RuntimeError('survey-bricks.fits.gz is required under legacy_dir.') + else: params['legacy_dir'] = None diff --git a/py/desitarget/mock/data/select-mock-targets-no-contam.yaml b/py/desitarget/mock/data/select-mock-targets-no-contam.yaml index e8da1202a..24e9892f0 100644 --- a/py/desitarget/mock/data/select-mock-targets-no-contam.yaml +++ b/py/desitarget/mock/data/select-mock-targets-no-contam.yaml @@ -1,5 +1,6 @@ # Parameter file for desitarget/bin/select_mock_targets, including contaminants. # legacy_dir: /global/project/projectdirs/cosmo/data/legacysurvey/dr8/ +legacy_dir: /global/homes/m/mjwilson/desi/survey-validation/svdc-spring2020g-onepercent/legacydir targets: BGS: { diff --git a/py/desitarget/mock/get_legacy.py b/py/desitarget/mock/get_legacy.py new file mode 100644 index 000000000..66ccf5c8d --- /dev/null +++ b/py/desitarget/mock/get_legacy.py @@ -0,0 +1,52 @@ +import os + +from desitarget.geomask import hp_in_box +from desitarget.randoms import dr_extension, _pre_or_post_dr8 +from desiutil import brick + + +nside = 8 +bricks = brick.Bricks(bricksize=0.25) +bricktable = bricks.to_table() +lookupdict = {bt["BRICKNAME"]: hp_in_box(nside, [bt["RA1"], bt["RA2"], bt["DEC1"], bt["DEC2"]]) for bt in bricktable} + +pixnum = 6195 +bricknames = [key for key in lookupdict if pixnum in lookupdict[key]] + +print(bricknames) + +# bricknames = ['0880p587', '0880p590', '0881p582', '0882p592'] + +# drdir = '/global/homes/m/mjwilson/desi/survey-validation/svdc-spring2020g-onepercent/legacydir/' +drdir = '/global/project/projectdirs/cosmo/data/legacysurvey/dr8/' + +drdirs = _pre_or_post_dr8(drdir) + +for drdir in drdirs: + # ADM determine whether the coadd files have extension .gz or .fz based on the DR directory. + extn, extn_nb = dr_extension(drdir) + + filt = ['g', 'r', 'z'] + qnames = ['nexp', 'depth', 'galdepth', 'psfsize', 'image'] + + for brickname in bricknames: + brickname = brickname.replace('m', 'p') + + rootdir = os.path.join(drdir, 'coadd', brickname[:3], brickname) + fileform = os.path.join(rootdir, 'legacysurvey-{}-{}-{}.fits.{}') + + # ADM loop through the filters and store the number of observations + # ADM etc. at the RA and Dec positions of the passed points. + for f in ['g', 'r', 'z']: + # ADM the input file labels, and output column names and output + # ADM formats for each of the quantities of interest. + for q in qnames: + fn = fileform.format(brickname, q, f, extn) + + cmd = 'cp {} legacydir/'.format(fn) + + # os.system(cmd) + + print(cmd) + + break diff --git a/py/desitarget/mock/mockmaker.py b/py/desitarget/mock/mockmaker.py index ac1707837..9674f15f8 100644 --- a/py/desitarget/mock/mockmaker.py +++ b/py/desitarget/mock/mockmaker.py @@ -171,6 +171,10 @@ def empty_targets_table(nobj=1): #targets.add_column(Column(name='NUMOBS_INIT', length=nobj, dtype='i8')) #targets.add_column(Column(name='HPXPIXEL', length=nobj, dtype='i8')) + # Default MASKBITS for all mock targets to non-zero. If populating with realistic depths + # these will later be overwritten. + targets['MASKBITS'] = 2e10 + return targets def empty_truth_table(nobj=1, templatetype='', use_simqso=True): @@ -420,9 +424,11 @@ def imaging_depth(self, data, legacy_dir=None, aprad=0.0): ubricknames = np.unique(bricknames) - # determine if we must traverse two sets of brick directories, i.e. north/, south/. - # drdirs = randoms._pre_or_post_dr8(legacy_dir) - + # All bricks potentially available in this release. If not present, + # code has exited before targets_truth call. + standard = fitsio.read(legacy_dir + '/survey-bricks.fits.gz') + standard = np.char.decode(standard['BRICKNAME'], 'UTF-8') + # keep = ['MASKBITS', 'PHOTSYS'] @@ -446,7 +452,7 @@ def imaging_depth(self, data, legacy_dir=None, aprad=0.0): data['PSFDEPTH_W1'] = np.zeros(len(data['RA']), dtype=dtypes['PSFDEPTH_G']) data['PSFDEPTH_W2'] = np.zeros(len(data['RA']), dtype=dtypes['PSFDEPTH_G']) - toremove = np.zeros_like(data['RA'], dtype=bool) + # toflag = np.zeros_like(data['RA'], dtype=bool) for ubrickname in ubricknames: indx = np.atleast_1d(data['BRICKNAME'] == ubrickname) @@ -482,14 +488,21 @@ def imaging_depth(self, data, legacy_dir=None, aprad=0.0): data['PSFDEPTH_W2'][indx] = rtn['PSFDEPTH_W2'] else: - # Empty dict: missing brick. Remove these targets. - toremove[np.where(indx)[0]] = True + # dr8_quantities_at_positions_in_a_brick suggest this a brick is missing. This is either because + # the brick was not successully reduced, or is not present locally but should be. Check the latter. + if np.isin(ubrickname, standard): + raise RuntimeError('{} should be under {}, but was not found.'.format(ubrickname, legacy_dir)) - log.info('Removing {} targets not in reduced DESI imaging (of {}).'.format(np.count_nonzero(toremove), len(data['RA']))) - - for key in list(data.keys()): - if isinstance(data[key], np.ndarray): - data[key] = data[key][~toremove] + else: + # DEPRECATED: Empty dict = missing brick. Potentially remove these targets. + # toflag[np.where(indx)[0]] = True + pass + + # DEPRECATED: Remove targets not in a stack, as for randoms. + # log.info('Removing {} targets not in reduced DESI imaging (of {}).'.format(np.count_nonzero(toremove), len(data['RA']))) + # for key in list(data.keys()): + # if isinstance(data[key], np.ndarray): + # data[key] = data[key][~toremove] # Overwrite with simple depths for now. self.simple_imaging_depth(data) From 9001caeaedf68bbf98f7f1c07ae358349ffabaeb Mon Sep 17 00:00:00 2001 From: michaelJwilson Date: Tue, 28 Apr 2020 16:46:51 -0700 Subject: [PATCH 8/8] Fix 2e10 2^10 bug. --- py/desitarget/mock/mockmaker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/desitarget/mock/mockmaker.py b/py/desitarget/mock/mockmaker.py index 9674f15f8..759711f96 100644 --- a/py/desitarget/mock/mockmaker.py +++ b/py/desitarget/mock/mockmaker.py @@ -173,7 +173,7 @@ def empty_targets_table(nobj=1): # Default MASKBITS for all mock targets to non-zero. If populating with realistic depths # these will later be overwritten. - targets['MASKBITS'] = 2e10 + targets['MASKBITS'] = 2**10 return targets