Skip to content

Commit

Permalink
update boss paths for v6_2+ (#65)
Browse files Browse the repository at this point in the history 
* update boss paths for v6_2+

* Fixed bug in sptypefolder special function

* Update path.py

* Update path.py

* Update python/sdss_access/path/path.py

* Update path.py

* Update test_sdss5.py

* Update path.py

* Update path.py

* Update conftest.py

* Update test_sdss5.py

* Update test_sdss5.py

* Update test_sdss5.py

* restore tilegrp

---------

Co-authored-by: Brian Cherinka <[email protected]>
  • Loading branch information
@Sean-Morrison @havok2063
Sean-Morrison and havok2063 authored Nov 7, 2024
1 parent cf47a46 commit 7896f78
Show file tree
Hide file tree
Showing 3 changed files with 233 additions and 14 deletions.
226 changes: 216 additions & 10 deletions python/sdss_access/path/path.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -325,20 +325,32 @@ def extract(self, name, example):
# handle special functions; perform a drop in replacement
if re.match('@spectrodir[|]', template):
template = re.sub('@spectrodir[|]', os.environ['BOSS_SPECTRO_REDUX'], template)
elif re.search('@platedir[|]', template):
if re.search('@platedir[|]', template):
template = re.sub('@platedir[|]', r'(.*)/{plateid:0>6}', template)
elif re.search('@definitiondir[|]', template):
if re.search('@definitiondir[|]', template):
template = re.sub('@definitiondir[|]', '{designid:0>6}', template)
elif re.search('@apgprefix[|]', template):
if re.search('@apgprefix[|]', template):
template = re.sub('@apgprefix[|]', '{prefix}', template)
elif re.search('@healpixgrp[|]', template):
if re.search('@healpixgrp[|]', template):
template = re.sub('@healpixgrp[|]', '{healpixgrp}', template)
elif re.search('@configgrp[|]', template):
if re.search('@configgrp[|]', template):
template = re.sub('@configgrp[|]', '{configgrp}', template)
elif re.search('@isplate[|]', template):
if re.search('@isplate[|]', template):
template = re.sub('@isplate[|]', '{isplate}', template)
elif re.search('@pad_fieldid[|]', template):
if re.search('@pad_fieldid[|]', template):
template = re.sub('@pad_fieldid[|]', '{fieldid}', template)
if re.search('@spcoaddfolder[|]',template):
template = re.sub('@spcoaddfolder[|]', '{spcoaddfolder}', template)
if re.search('@sptypefolder[|]',template):
template = re.sub('@sptypefolder[|]', '{sptypefolder}', template)
if re.search('@epochflag[|]', template):
template = re.sub('@epochflag[|]', '{epochflag}', template)
if re.search('@spcoaddobs[|]',template):
template = re.sub('@spcoaddobs[|]', '{obs}', template)
if re.search('@spcoaddgrp[|]',template):
template = re.sub('@spcoaddgrp[|]', '{spcoaddgrp}', template)
if re.search('@fieldgrp[|]',template):
template = re.sub('@fieldgrp[|]', '{fieldgrp}', template)
if re.search('@plateid6[|]', template):
template = re.sub('@plateid6[|]', '{plateid:0>6}', template)
if re.search('@component_default[|]', template):
Expand All @@ -347,6 +359,8 @@ def extract(self, name, example):
template = re.sub('@cat_id_groups[|]', '{cat_id_groups}', template)
if re.search('@sdss_id_groups[|]', template):
template = re.sub('@sdss_id_groups[|]', '{sdss_id_groups}', template)
if re.search('@tilegrp[|]', template):
template = re.sub('@tilegrp[|]', '{tilegrp}', template)

# check if template has any brackets
haskwargs = re.search('[{}]', template)
Expand Down Expand Up @@ -386,12 +400,28 @@ def extract(self, name, example):
drval = re.match('^DR[1-9][0-9]', value).group(0)
otherval = value.split(drval)[-1]
pdict = {keys[0]: drval, keys[1]: otherval}
elif keys == ['fieldid','isplate']:
# for {fieldid}{isplate} as isplate is calculated automatically
if value.endswith('p') and value[:-1].isdigit():
value = value[:-1]
pdict = {keys[0]:value}
elif keys == ['run2d','epochflag']:
# for {run2d}{epochflag} as epochflag is calculated automatically
pdict = {keys[0]:value.replace('-epoch','')}
elif keys == ['coadd', 'obs']:
value = value.split('_')
if len(value) == 1:
value.append('')
pdict = {keys[0]: value[0], keys[1]: value[1]}
elif keys[0] in ['rc', 'br', 'filter', 'camrow']:
# for {camrow}{camcol}, {filter}{camcol}, {br}{id}, etc
pdict = {keys[0]: value[0], keys[1]: value[1:]}
else:
raise ValueError('This case has not yet been accounted for.')
path_dict.update(pdict)
elif keys[0] in ['sptypefolder','fieldgrp','spcoaddfolder','spcoaddgrp']:
# supress the keys since they are automatically calculated
continue
else:
path_dict[keys[0]] = value
return path_dict
Expand Down Expand Up @@ -678,7 +708,7 @@ def full(self, filetype, **kwargs):
if not skip_tag_check:
template = re.sub(r'tags/(v?[0-9._]+)', r'\1', template, count=1)

return self._check_compression(template)
return self._check_compression(os.path.normpath(template))

@staticmethod
def check_modules(template, permanent=None):
Expand Down Expand Up @@ -1531,8 +1561,184 @@ def pad_fieldid(self, filetype, **kwargs):
return str(fieldid)
if fieldid.isnumeric():
return str(fieldid).zfill(6)
else:
return fieldid
return fieldid

def spcoaddfolder(self, filetype, **kwargs):
''' Returns the reorganized subfolder structure for the BOSS idlspec2d run2d version
Parameters
---------
filetype : str
File type parameter
run2d : str
BOSS idlspec2d run2d version
coadd : str
Name of the custom coadd schema
Returns
-------
sptypefolder : str
'''

run2d = kwargs.get('run2d', None)
coaddname = kwargs.get('coadd', None)

if (not run2d):
return ''
if (('v5' in run2d) or (str(run2d) in ['26','103','104']) or
('v6_0' in run2d) or ('v6_1' in run2d)):
return ''
if filetype.lower() in ['spall_coadd','spall-lite_coadd','spallline_coadd']:
return 'summary'
if filetype.lower() in ['speclite_coadd','specfull_coadd',
'spallfield_coadd','spalllinefield_coadd']:
return coaddname
return 'fields'


def spcoaddgrp(self, filetype, **kwargs):
''' Returns the coadd group (field group analog) subfolder structure for the BOSS idlspec2d run2d version
Parameters
---------
filetype : str
File type parameter
run2d : str
BOSS idlspec2d run2d version
coadd : str
Name of the custom coadd schema
Returns
-------
spcoaddgrp : str
'''
run2d = kwargs.get('run2d', None)
coaddname = kwargs.get('coadd', None)

if (not run2d):
return ''
if (('v5' in run2d) or (str(run2d) in ['26','103','104']) or
('v6_0' in run2d) or ('v6_1' in run2d)):
return ''
return coaddname

def sptypefolder(self, filetype, **kwargs):
''' Returns the reorganized subfolder structure for the BOSS idlspec2d run2d version
Parameters
---------
filetype : str
File type parameter
run2d : str
BOSS idlspec2d run2d version
Returns
-------
sptypefolder : str
'''

run2d = kwargs.get('run2d', None)

if (not run2d) or ('v5' in run2d) or (str(run2d) in ['26','103','104']):
return ''
if ('v6_0' in run2d) or ('v6_1' in run2d):
if filetype.lower() in ['speclite_epoch','specfull_epoch',
'spallfield_epoch','spalllinefield_epoch']:
return 'epoch/spectra'
if 'epoch' in filetype.lower():
return 'epoch'
return ''
if filetype.lower() in ['fieldlist_epoch','spall_epoch',
'spall-lite_epoch','spallline_epoch']:
return 'summary/epoch'
if filetype.lower() in ['speclite_epoch','specfull_epoch',
'spallfield_epoch','spalllinefield_epoch']:
return 'spectra/epoch'
if filetype.lower() in ['conflist','fieldlist','spall',
'spall-lite','spallline']:
return 'summary/daily'
if filetype.lower() in ['speclite','specfull',
'spallfield','spalllinefield']:
return 'daily'
return 'fields'

def spcoaddobs(self, filetype, **kwargs):
''' Returns the formatted observatory flag for custom coadds for the BOSS idlspec2d
Parameters
----------
filetype : str
File type parameter
run2d : str
BOSS idlspec2d run2d version
obs : str
Observatory of observations: LCO, APO, ''(for merged)
Returns
-------
obs : str
'''

obs = kwargs.get('obs', None)
run2d = kwargs.get('run2d', None)

if not obs or obs == '':
return ''
if (('v5' in run2d) or (str(run2d) in ['26','103','104']) or
(('v6_0' in run2d) or ('v6_1' in run2d)) and obs.lower()) == 'apo':
return ''
if obs == '*':
return obs
return '_{}'.format(obs.lower())

def epochflag(self, filetype, **kwargs):
''' Returns the flag for epoch coadds for the BOSS idlspec2d
Parameters
----------
filetype : str
File type parameter
run2d : str
BOSS idlspec2d run2d version
Returns
-------
epochflag : str
'''

run2d = kwargs.get('run2d', None)
if (('v5' in run2d) or (str(run2d) in ['26','103','104']) or
('v6_0' in run2d) or ('v6_1' in run2d)):
return ''
return '-epoch'

def fieldgrp(self, filetype, **kwargs):
''' Returns the fieldid group for the BOSS idlspec2d run2d version
Parameters
---------
filetype : str
File type parameter
run2d : str
BOSS idlspec2d run2d version
fieldid : str or int
Field ID number. Will be converted to str internally.
Returns
-------
fieldgrp : str
'''

fieldid = kwargs.get('fieldid', None)
run2d = kwargs.get('run2d', None)

if (not fieldid):
return ''

if (('v5' in run2d) or (str(run2d) in ['26','103','104']) or
('v6_0' in run2d) or ('v6_1' in run2d)):
return ''
fieldid = str(fieldid)
if fieldid.isnumeric():
return '{:0>3d}XXX'.format(int(fieldid) // 1000)
return fieldid

def tilegrp(self, filetype, **kwargs):
''' Returns LVM tile id group subdirectory
Expand Down
2 changes: 1 addition & 1 deletion tests/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ def expdata(datapath):
# full source file location
source = os.path.join(base, sas_module, location)
# full final file location
destination = os.path.join(os.getenv('SAS_BASE_DIR'), sas_module, location)
destination = os.path.normpath(os.path.join(os.getenv('SAS_BASE_DIR'), sas_module, location))
# combined dict
result = {'name': datapath['name'], 'params': datapath['params'], 'base': base,
'sas_module': sas_module, 'location': location, 'source': source,
Expand Down
19 changes: 16 additions & 3 deletions tests/path/test_sdss5.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,15 +59,28 @@ def test_apogee_paths(self, path, name, special, keys, exp):
'v6_0_4/1234p/spFrame-b1-00005432.fits.gz'),
('spField', '@pad_fieldid', {'run2d': 'v6_1_1', 'mjd': '59630', 'fieldid': '*'},
'v6_1_1/*/spField-*-59630.fits'),
('spCOADDS','@spcoaddfolder',{'run2d':'v6_2_0','survey':'BHM','coadd':'allepoch','phase':'SDSSV'},
'v6_2_0/fields/SDSSV_BHM_COADDS.par'),
('spField','@sptypefolder',{'run2d':'v6_2_0','mjd': '59630', 'fieldid': '123456'},
'v6_2_0/fields/123XXX/123456/spField-123456-59630.fits'),
('spField','@fieldgrp',{'run2d':'v6_2_0','mjd': '59630', 'fieldid': '123456'},
'v6_2_0/fields/123XXX/123456/spField-123456-59630.fits'),
('spFullsky','@spcoaddobs',{'run2d':'v6_2_0','mjd': '59630','coadd':'allepoch','obs':'apo'},
'v6_2_0/fields/allepoch/allepoch_apo/spFullsky-allepoch_apo-59630.fits'),
('spAll_epoch','@epochflag',{'run2d':'v6_2_0'},
'v6_2_0/summary/epoch/spAll-v6_2_0-epoch.fits'),
('spAll_coadd','@spcoaddgrp',{'run2d':'v6_2_0','coadd':'allepoch'},
'v6_2_0/summary/allepoch/spAll-v6_2_0-allepoch.fits'),
('lvm_frame', '@tilegrp', {'drpver': 'master', 'mjd': 60235,
'tileid': 1055360, 'kind': 'CFrame', 'expnum': 6817},
'1055XX/1055360/60235/lvmCFrame-00006817.fits')],
ids=['configgrp', 'apgprefix-apo', 'apgprefix-lco', 'apgprefix-ins',
'isplate-v6_0_4','pad_fieldid-5','pad_fieldid-6', 'frame-pad', 'frame-nopadp',
'pad_fieldid-*', 'lvm-tileid'])
'isplate-v6_0_4','pad_fieldid-5','pad_fieldid-6', 'frame-pad',
'frame-nopadp', 'pad_fieldid-*','spcoaddfolder', 'sptypefolder',
'fieldgrp','spcoaddobs','epochflag','spcoaddgrp','lvm-tileid'])
def test_special_function(self, path, name, special, keys, exp):
assert special in path.templates[name]
full = path.full(name, **keys)
full = os.path.normpath(path.full(name, **keys))
assert exp in full

@pytest.mark.parametrize('name, keys', [('specLite', ['fieldid', 'catalogid', 'run2d', 'mjd']),
Expand Down

0 comments on commit 7896f78

Please sign in to comment.