Skip to content

Commit

Permalink
Add NaN safety checks.
Browse files Browse the repository at this point in the history
  • Loading branch information
@weaverba137
weaverba137 committed Aug 22, 2024
1 parent ac91a5d commit bbdc2ae
Show file tree
Hide file tree
Showing 2 changed files with 65 additions and 36 deletions.
68 changes: 40 additions & 28 deletions bin/desi_tsnr_afterburner
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1085,44 +1085,56 @@ def main():

gfa_table = None

if args.gfa_proc_dir is not None :
try :
if args.gfa_proc_dir is not None:
try:
gfa_table = read_gfa_data(args.gfa_proc_dir)
except PermissionError as e :
except PermissionError as e:
log.error(e)
log.error("could not read some files in {}".format(args.gfa_proc_dir))
args.gfa_proc_dir = None

if args.gfa_proc_dir is not None and gfa_table is not None :
e2i = {e:i for i,e in enumerate(gfa_table["EXPID"])}
jj=[]
ii=[]
for j,e in enumerate(tsnr2_expid_table["EXPID"]) :
if e in e2i :
if args.gfa_proc_dir is not None and gfa_table is not None:
e2i = {e: i for i, e in enumerate(gfa_table["EXPID"])}
jj = []
ii = []
for j, e in enumerate(tsnr2_expid_table["EXPID"]):
if e in e2i:
jj.append(j)
ii.append(e2i[e])

cols=["TRANSPARENCY","FWHM_ASEC","FIBER_FRACFLUX","FIBER_FRACFLUX_ELG","FIBER_FRACFLUX_BGS","FIBERFAC","FIBERFAC_ELG","FIBERFAC_BGS","SKY_MAG_AB","AIRMASS",]
for col in cols :
if col in gfa_table.dtype.names :
if col == "FWHM_ASEC" :
cols = ("TRANSPARENCY", "FWHM_ASEC", "FIBER_FRACFLUX", "FIBER_FRACFLUX_ELG",
"FIBER_FRACFLUX_BGS", "FIBERFAC", "FIBERFAC_ELG", "FIBERFAC_BGS",
"SKY_MAG_AB", "AIRMASS")
for col in cols:
if col in gfa_table.dtype.names:
if col == "FWHM_ASEC":
col2 = "SEEING_GFA"
else :
col2 = col+"_GFA"
tsnr2_expid_table[col2]=np.zeros(len(tsnr2_expid_table),dtype=float)
tsnr2_expid_table[col2][jj] = gfa_table[col][ii]

if args.gfa_proc_dir is not None and (args.skymags is not None or args.compute_skymags) :
else:
col2 = col + "_GFA"
tsnr2_expid_table[col2] = np.zeros(len(tsnr2_expid_table), dtype=float)
if np.isfinite(gfa_table[col][ii]).all():
tsnr2_expid_table[col2][jj] = gfa_table[col][ii]
else:
log.warning("gfa_table column '%s' contains NaN or other non-finite values. Invalid values will be replaced with zero (0).", col)
good_values = gfa_table[col][ii].copy()
good_values[~np.isfinite(good_values)] = 0
tsnr2_expid_table[col2][jj] = good_values

if args.gfa_proc_dir is not None and (args.skymags is not None or args.compute_skymags):
#
# Assuming NaN are replaced with zero above, compute_efftime() will return zero for the affected rows.
# That is, there are no divide-by-zero or other errors induced by replacing NaN with zero.
#
efftime_dark, efftime_bright, efftime_backup = compute_efftime(tsnr2_expid_table[jj])
for col in ["EFFTIME_DARK_GFA","EFFTIME_BRIGHT_GFA","EFFTIME_BACKUP_GFA"] :
tsnr2_expid_table[col]=np.zeros(len(tsnr2_expid_table),dtype=float)
tsnr2_expid_table["EFFTIME_DARK_GFA"][jj]=efftime_dark
tsnr2_expid_table["EFFTIME_BRIGHT_GFA"][jj]=efftime_bright
tsnr2_expid_table["EFFTIME_BACKUP_GFA"][jj]=efftime_backup
goaltype=tsnr2_expid_table["GOALTYPE"]
tsnr2_expid_table["EFFTIME_GFA"] = tsnr2_expid_table["EFFTIME_DARK_GFA"] # default
tsnr2_expid_table["EFFTIME_GFA"][goaltype=="bright"]=tsnr2_expid_table["EFFTIME_BRIGHT_GFA"][goaltype=="bright"]
tsnr2_expid_table["EFFTIME_GFA"][goaltype=="backup"]=tsnr2_expid_table["EFFTIME_BACKUP_GFA"][goaltype=="backup"]
for col in ("EFFTIME_DARK_GFA", "EFFTIME_BRIGHT_GFA", "EFFTIME_BACKUP_GFA"):
tsnr2_expid_table[col] = np.zeros(len(tsnr2_expid_table), dtype=float)
tsnr2_expid_table["EFFTIME_DARK_GFA"][jj] = efftime_dark
tsnr2_expid_table["EFFTIME_BRIGHT_GFA"][jj] = efftime_bright
tsnr2_expid_table["EFFTIME_BACKUP_GFA"][jj] = efftime_backup
goaltype = tsnr2_expid_table["GOALTYPE"]
tsnr2_expid_table["EFFTIME_GFA"] = tsnr2_expid_table["EFFTIME_DARK_GFA"] # default
tsnr2_expid_table["EFFTIME_GFA"][goaltype == "bright"] = tsnr2_expid_table["EFFTIME_BRIGHT_GFA"][goaltype == "bright"]
tsnr2_expid_table["EFFTIME_GFA"][goaltype == "backup"] = tsnr2_expid_table["EFFTIME_BACKUP_GFA"][goaltype == "backup"]

# end of loop on nights

Expand Down
33 changes: 25 additions & 8 deletions py/desispec/tilecompleteness.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,14 +157,31 @@ def compute_tile_completeness_table(exposure_table,specprod_dir,auxiliary_table_
# test default
res["GOALTIME"][res["GOALTIME"]==0] = default_goaltime

for i,tile in enumerate(tiles) :
jj=(exposure_table["TILEID"]==tile)
res["NEXP"][i]=np.sum(jj)
for k in ["EXPTIME","LRG_EFFTIME_DARK","ELG_EFFTIME_DARK","BGS_EFFTIME_BRIGHT","LYA_EFFTIME_DARK","EFFTIME_SPEC","EFFTIME_ETC","EFFTIME_GFA"] :
if k in exposure_table.dtype.names :
res[k][i] = np.sum(exposure_table[k][jj])
if k == "EFFTIME_ETC" or k == "EFFTIME_GFA" :
if np.any((exposure_table[k][jj]==0)&(exposure_table["EFFTIME_SPEC"][jj]>0)) : res[k][i]=0 # because we are missing data
for i, tile in enumerate(tiles):
jj = (exposure_table["TILEID"] == tile)
res["NEXP"][i] = np.sum(jj)
for k in ("EXPTIME", "LRG_EFFTIME_DARK", "ELG_EFFTIME_DARK", "BGS_EFFTIME_BRIGHT",
"LYA_EFFTIME_DARK", "EFFTIME_SPEC", "EFFTIME_ETC", "EFFTIME_GFA"):
if k in exposure_table.dtype.names:
#
# If exposure_table comes from a Table.read(), it may contain masked values.
# If exposure_table comes from desispec.io.read_table(), it may contain NaN,
# in particular for a small number of exposures in the GFA summary file.
# In some cases the NaN values will already have been replaced by zero, but
# we can and should be careful.
#
if not np.isfinite(exposure_table[k][jj]).all():
log.warning("exposure_table column '%s' contains NaN or other non-finite values, a sum will be performed over valid values.", k)
good_values = exposure_table[k][jj].copy()
good_values[~np.isfinite(good_values)] = 0
res[k][i] = np.sum(good_values)
else:
if hasattr(exposure_table[k], 'mask'):
log.warning("exposure_table column '%s' contains masked values, a sum will be performed over unmasked values.", k)
res[k][i] = np.sum(exposure_table[k][jj])
if k == "EFFTIME_ETC" or k == "EFFTIME_GFA":
if np.any((exposure_table[k][jj] == 0) & (exposure_table["EFFTIME_SPEC"][jj] > 0)):
res[k][i] = 0 # because we are missing data

# copy the following from the exposure table if it exists and not already set as sv1

Expand Down

0 comments on commit bbdc2ae

Please sign in to comment.