Maskbits #608
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -69,19 +69,19 @@ if args.healpixels is None: | |
| healpixels = desimodel.footprint.tiles2pix(args.nside, tiles) | ||
| else: | ||
| healpixels = np.array(args.healpixels) | ||
| if args.overwrite: | ||
| log.info('Processing {} pixel(s).'.format(len(healpixels))) | ||
| else: | ||
| keep = list() | ||
| for i, pixnum in enumerate(healpixels): | ||
| truthfile = mockio.findfile('truth', args.nside, pixnum, basedir=args.output_dir, obscon='dark') | ||
| if not os.path.exists(truthfile): | ||
| keep.append(i) | ||
|
|
||
| log.info('{}/{} pixels remaining to do. {}'.format(len(keep), len(healpixels), healpixels[keep])) | ||
| healpixels = healpixels[keep] | ||
|
There was a problem hiding this comment. I don't understand the intent of the third number. There was a problem hiding this comment. This was a lazy way for me know the healpixels to actually be processed, as opposed to the ones I requested, given some had already been generated and were on disk. I was playing games like submitting the second half of that list to a separate job to run in tandem. Obviously, there are more sophisticated approaches and it doesn't need to be kept. |
||
| if args.join: | ||
| from desitarget.mock.build import join_targets_truth | ||
| join_targets_truth(mockdir=args.output_dir, outdir=args.output_dir, overwrite=args.overwrite) | ||
|
|
@@ -96,6 +96,26 @@ log.info('Reading configuration file {}'.format(args.config)) | |
| with open(args.config, 'r') as pfile: | ||
| params = yaml.safe_load(pfile) | ||
|
|
||
| # Set up MASKBITS lookup. Logic check for the necessary files, if not there wget the per brick files for | ||
| # all overlapping pixels. | ||
| if 'legacy_dir' in params.keys(): | ||
| # E.g. global/project/projectdirs/cosmo/data/legacysurvey/dr8/ | ||
| # wget https://portal.nersc.gov/project/cosmo/data/legacysurvey/dr8/north/coadd/338/3380p295/ | ||
| log.info('Assuming maskbits & depths based on LEGACYDIR={}'.format(params['legacy_dir'])) | ||
|
|
||
| # Check the survey bricks file exists. We'll use this to ensure that the local files are compelte for | ||
| # the healpixels being requested. | ||
| legacy_dir = params['legacy_dir'] | ||
| allbricks = legacy_dir + '/survey-bricks.fits.gz' | ||
|
|
||
| if not os.path.isfile(allbricks): | ||
| raise RuntimeError('survey-bricks.fits.gz is required under legacy_dir.') | ||
|
|
||
| else: | ||
| params['legacy_dir'] = None | ||
|
|
||
| log.info('Assuming simple homogeneous depths & no masking.') | ||
|
|
||
|
There was a problem hiding this comment. I don't think we can realistically expect people to copy over entire brick-sized There was a problem hiding this comment. Yeh, it'd have been better had we been able to coordinate more at the start. Both I and @geordie666 didn't see this as particularly useful, but we understood that to be the thing that was compatible with "being able to run on a laptop". I.e. we tried to support also running/development the pre-burner on a laptop - by requiring only the minimal number of files given a list of heal pixels. Maybe you're willing to give up on the pre-burner on a laptop, this wasn't clear to me before. I certainly follow the logic of trying to separate the two steps and this probably wouldn't be hard. My one "concern", which I think you're remembering from the telecon, is that I think you have in mind populating PSF depths etc. for something like the mocks that currently exist, e.g. ~20, including dark sky, etc, to get something desi-like. There is an additional use case of just the mask for ~1000s of (approximate) mocks in the clustering analysis, for which it's less clear that we want redundant information saved to disk in this case. I'd suggest it makes sense to develop select_mock_targets in such a way that it's a one stop shop for both exercises. I.e. in minimal mode, it turns a (ra, dec, z) mock into something accepted by downstream pipeline code, but which has flags etc. that let you dial up the sophistication of effects included. There's a discussion planned on this at the cosmosim group telecon on Thursday. |
||
| log.info('Calling targets_truth with survey={} at {}'.format(args.survey, time.asctime())) | ||
| targets_truth(params, healpixels=healpixels, nside=args.nside, seed=args.seed, | ||
| output_dir=args.output_dir, nproc=args.nproc, verbose=args.verbose, | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| # Parameter file for desitarget/bin/select_mock_targets, including contaminants. | ||
|
There was a problem hiding this comment. The contaminants can just be commented out of the configuration file. That said, maybe it'd be worth adding a There was a problem hiding this comment. I always find myself doing this so such a flag, or yaml, would be useful in my opinion. |
||
| # legacy_dir: /global/project/projectdirs/cosmo/data/legacysurvey/dr8/ | ||
| legacy_dir: /global/homes/m/mjwilson/desi/survey-validation/svdc-spring2020g-onepercent/legacydir | ||
|
|
||
| targets: | ||
| BGS: { | ||
| target_type: BGS, | ||
| mockfile: '{DESI_ROOT}/mocks/bgs/MXXL/full_sky/v0.0.4/BGS_r20.6.hdf5', | ||
| format: durham_mxxl_hdf5, | ||
| magcut: 20.3, | ||
| } | ||
| ELG: { | ||
| target_type: ELG, | ||
| mockfile: '{DESI_ROOT}/mocks/DarkSky/v1.0.1/elg_0_inpt.fits', | ||
| format: gaussianfield, | ||
| density: 2200, | ||
| } | ||
| LRG: { | ||
| target_type: LRG, | ||
| mockfile: '{DESI_ROOT}/mocks/DarkSky/v1.0.1/lrg_0_inpt.fits', | ||
| format: gaussianfield, | ||
| density: 480, | ||
| } | ||
| QSO: { | ||
| target_type: QSO, | ||
| mockfile: '{DESI_ROOT}/mocks/DarkSky/v1.0.1/qso_0_inpt.fits', | ||
| format: gaussianfield, | ||
| zmax_qso: 1.8, | ||
| use_simqso: True, | ||
| density: 120, | ||
| } | ||
| LYA: { | ||
| target_type: QSO, | ||
| mockfile: '{DESI_ROOT}/mocks/lya_forest/london/v9.0/v9.0.0/master.fits', | ||
| format: CoLoRe, | ||
| nside_lya: 16, | ||
| zmin_lya: 1.8, | ||
| density: 50, | ||
| use_simqso: True, | ||
| sqmodel: 'default', #Other options are 'lya_simqso_model_develop' and 'lya_simqso_model' (same as quickquasars) | ||
| balprob: 0.0, | ||
| add_dla: False, | ||
| add_metals: False, #If adding metals use 'all', this is to keep the same argument as in quickquasars. | ||
| add_lyb: False | ||
| } | ||
| MWS_MAIN: { | ||
| target_type: STAR, | ||
| mockfile: '{DESI_ROOT}/mocks/mws/galaxia/alpha/v0.0.6/healpix', | ||
| nside_galaxia: 8, | ||
| format: galaxia, | ||
| } | ||
| MWS_NEARBY: { | ||
| target_type: STAR, | ||
| mockfile: '{DESI_ROOT}/mocks/mws/100pc/v0.0.4/mock_100pc.fits', | ||
| format: mws_100pc, | ||
| } | ||
| WD: { | ||
| target_type: WD, | ||
| mockfile: '{DESI_ROOT}/mocks/mws/wd/v1.0.0/mock_wd.fits', | ||
| format: mws_wd, | ||
| } | ||
| SKY: { | ||
| target_type: SKY, | ||
| mockfile: '{DESI_ROOT}/mocks/uniformsky/0.2/uniformsky-2048-0.2.fits', | ||
| format: uniformsky, | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| import os | ||
|
|
||
| from desitarget.geomask import hp_in_box | ||
| from desitarget.randoms import dr_extension, _pre_or_post_dr8 | ||
| from desiutil import brick | ||
|
|
||
|
|
||
| nside = 8 | ||
| bricks = brick.Bricks(bricksize=0.25) | ||
| bricktable = bricks.to_table() | ||
| lookupdict = {bt["BRICKNAME"]: hp_in_box(nside, [bt["RA1"], bt["RA2"], bt["DEC1"], bt["DEC2"]]) for bt in bricktable} | ||
|
|
||
| pixnum = 6195 | ||
| bricknames = [key for key in lookupdict if pixnum in lookupdict[key]] | ||
|
|
||
| print(bricknames) | ||
|
|
||
| # bricknames = ['0880p587', '0880p590', '0881p582', '0882p592'] | ||
|
|
||
| # drdir = '/global/homes/m/mjwilson/desi/survey-validation/svdc-spring2020g-onepercent/legacydir/' | ||
| drdir = '/global/project/projectdirs/cosmo/data/legacysurvey/dr8/' | ||
|
|
||
| drdirs = _pre_or_post_dr8(drdir) | ||
|
|
||
| for drdir in drdirs: | ||
| # ADM determine whether the coadd files have extension .gz or .fz based on the DR directory. | ||
| extn, extn_nb = dr_extension(drdir) | ||
|
|
||
| filt = ['g', 'r', 'z'] | ||
| qnames = ['nexp', 'depth', 'galdepth', 'psfsize', 'image'] | ||
|
|
||
| for brickname in bricknames: | ||
| brickname = brickname.replace('m', 'p') | ||
|
|
||
| rootdir = os.path.join(drdir, 'coadd', brickname[:3], brickname) | ||
| fileform = os.path.join(rootdir, 'legacysurvey-{}-{}-{}.fits.{}') | ||
|
|
||
| # ADM loop through the filters and store the number of observations | ||
| # ADM etc. at the RA and Dec positions of the passed points. | ||
| for f in ['g', 'r', 'z']: | ||
| # ADM the input file labels, and output column names and output | ||
| # ADM formats for each of the quantities of interest. | ||
| for q in qnames: | ||
| fn = fileform.format(brickname, q, f, extn) | ||
|
|
||
| cmd = 'cp {} legacydir/'.format(fn) | ||
|
|
||
| # os.system(cmd) | ||
|
|
||
| print(cmd) | ||
|
|
||
| break |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We should definitely make sure that
mock.io.findfilematches how the "real" target catalogs are organized, but I thought we already had adark/brightorganization implemented.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
There is, but I ran into errors due to a bug on this. I'd have to refresh my memory by comparing to master.