From bbdc2ae9cacecb7d0f314b69fe48bf6ca15aa7ae Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 22 Aug 2024 10:30:46 -0700 Subject: [PATCH] Add NaN safety checks. --- bin/desi_tsnr_afterburner | 68 +++++++++++++++++++-------------- py/desispec/tilecompleteness.py | 33 ++++++++++++---- 2 files changed, 65 insertions(+), 36 deletions(-) diff --git a/bin/desi_tsnr_afterburner b/bin/desi_tsnr_afterburner index 7faa77015..a32d55b3d 100755 --- a/bin/desi_tsnr_afterburner +++ b/bin/desi_tsnr_afterburner @@ -1085,44 +1085,56 @@ def main(): gfa_table = None - if args.gfa_proc_dir is not None : - try : + if args.gfa_proc_dir is not None: + try: gfa_table = read_gfa_data(args.gfa_proc_dir) - except PermissionError as e : + except PermissionError as e: log.error(e) log.error("could not read some files in {}".format(args.gfa_proc_dir)) args.gfa_proc_dir = None - if args.gfa_proc_dir is not None and gfa_table is not None : - e2i = {e:i for i,e in enumerate(gfa_table["EXPID"])} - jj=[] - ii=[] - for j,e in enumerate(tsnr2_expid_table["EXPID"]) : - if e in e2i : + if args.gfa_proc_dir is not None and gfa_table is not None: + e2i = {e: i for i, e in enumerate(gfa_table["EXPID"])} + jj = [] + ii = [] + for j, e in enumerate(tsnr2_expid_table["EXPID"]): + if e in e2i: jj.append(j) ii.append(e2i[e]) - cols=["TRANSPARENCY","FWHM_ASEC","FIBER_FRACFLUX","FIBER_FRACFLUX_ELG","FIBER_FRACFLUX_BGS","FIBERFAC","FIBERFAC_ELG","FIBERFAC_BGS","SKY_MAG_AB","AIRMASS",] - for col in cols : - if col in gfa_table.dtype.names : - if col == "FWHM_ASEC" : + cols = ("TRANSPARENCY", "FWHM_ASEC", "FIBER_FRACFLUX", "FIBER_FRACFLUX_ELG", + "FIBER_FRACFLUX_BGS", "FIBERFAC", "FIBERFAC_ELG", "FIBERFAC_BGS", + "SKY_MAG_AB", "AIRMASS") + for col in cols: + if col in gfa_table.dtype.names: + if col == "FWHM_ASEC": col2 = "SEEING_GFA" - else : - col2 = col+"_GFA" - tsnr2_expid_table[col2]=np.zeros(len(tsnr2_expid_table),dtype=float) - tsnr2_expid_table[col2][jj] = gfa_table[col][ii] - - if args.gfa_proc_dir is not None and (args.skymags is not None or args.compute_skymags) : + else: + col2 = col + "_GFA" + tsnr2_expid_table[col2] = np.zeros(len(tsnr2_expid_table), dtype=float) + if np.isfinite(gfa_table[col][ii]).all(): + tsnr2_expid_table[col2][jj] = gfa_table[col][ii] + else: + log.warning("gfa_table column '%s' contains NaN or other non-finite values. Invalid values will be replaced with zero (0).", col) + good_values = gfa_table[col][ii].copy() + good_values[~np.isfinite(good_values)] = 0 + tsnr2_expid_table[col2][jj] = good_values + + if args.gfa_proc_dir is not None and (args.skymags is not None or args.compute_skymags): + # + # Assuming NaN are replaced with zero above, compute_efftime() will return zero for the affected rows. + # That is, there are no divide-by-zero or other errors induced by replacing NaN with zero. + # efftime_dark, efftime_bright, efftime_backup = compute_efftime(tsnr2_expid_table[jj]) - for col in ["EFFTIME_DARK_GFA","EFFTIME_BRIGHT_GFA","EFFTIME_BACKUP_GFA"] : - tsnr2_expid_table[col]=np.zeros(len(tsnr2_expid_table),dtype=float) - tsnr2_expid_table["EFFTIME_DARK_GFA"][jj]=efftime_dark - tsnr2_expid_table["EFFTIME_BRIGHT_GFA"][jj]=efftime_bright - tsnr2_expid_table["EFFTIME_BACKUP_GFA"][jj]=efftime_backup - goaltype=tsnr2_expid_table["GOALTYPE"] - tsnr2_expid_table["EFFTIME_GFA"] = tsnr2_expid_table["EFFTIME_DARK_GFA"] # default - tsnr2_expid_table["EFFTIME_GFA"][goaltype=="bright"]=tsnr2_expid_table["EFFTIME_BRIGHT_GFA"][goaltype=="bright"] - tsnr2_expid_table["EFFTIME_GFA"][goaltype=="backup"]=tsnr2_expid_table["EFFTIME_BACKUP_GFA"][goaltype=="backup"] + for col in ("EFFTIME_DARK_GFA", "EFFTIME_BRIGHT_GFA", "EFFTIME_BACKUP_GFA"): + tsnr2_expid_table[col] = np.zeros(len(tsnr2_expid_table), dtype=float) + tsnr2_expid_table["EFFTIME_DARK_GFA"][jj] = efftime_dark + tsnr2_expid_table["EFFTIME_BRIGHT_GFA"][jj] = efftime_bright + tsnr2_expid_table["EFFTIME_BACKUP_GFA"][jj] = efftime_backup + goaltype = tsnr2_expid_table["GOALTYPE"] + tsnr2_expid_table["EFFTIME_GFA"] = tsnr2_expid_table["EFFTIME_DARK_GFA"] # default + tsnr2_expid_table["EFFTIME_GFA"][goaltype == "bright"] = tsnr2_expid_table["EFFTIME_BRIGHT_GFA"][goaltype == "bright"] + tsnr2_expid_table["EFFTIME_GFA"][goaltype == "backup"] = tsnr2_expid_table["EFFTIME_BACKUP_GFA"][goaltype == "backup"] # end of loop on nights diff --git a/py/desispec/tilecompleteness.py b/py/desispec/tilecompleteness.py index 9372d9abc..8aeaa42d0 100644 --- a/py/desispec/tilecompleteness.py +++ b/py/desispec/tilecompleteness.py @@ -157,14 +157,31 @@ def compute_tile_completeness_table(exposure_table,specprod_dir,auxiliary_table_ # test default res["GOALTIME"][res["GOALTIME"]==0] = default_goaltime - for i,tile in enumerate(tiles) : - jj=(exposure_table["TILEID"]==tile) - res["NEXP"][i]=np.sum(jj) - for k in ["EXPTIME","LRG_EFFTIME_DARK","ELG_EFFTIME_DARK","BGS_EFFTIME_BRIGHT","LYA_EFFTIME_DARK","EFFTIME_SPEC","EFFTIME_ETC","EFFTIME_GFA"] : - if k in exposure_table.dtype.names : - res[k][i] = np.sum(exposure_table[k][jj]) - if k == "EFFTIME_ETC" or k == "EFFTIME_GFA" : - if np.any((exposure_table[k][jj]==0)&(exposure_table["EFFTIME_SPEC"][jj]>0)) : res[k][i]=0 # because we are missing data + for i, tile in enumerate(tiles): + jj = (exposure_table["TILEID"] == tile) + res["NEXP"][i] = np.sum(jj) + for k in ("EXPTIME", "LRG_EFFTIME_DARK", "ELG_EFFTIME_DARK", "BGS_EFFTIME_BRIGHT", + "LYA_EFFTIME_DARK", "EFFTIME_SPEC", "EFFTIME_ETC", "EFFTIME_GFA"): + if k in exposure_table.dtype.names: + # + # If exposure_table comes from a Table.read(), it may contain masked values. + # If exposure_table comes from desispec.io.read_table(), it may contain NaN, + # in particular for a small number of exposures in the GFA summary file. + # In some cases the NaN values will already have been replaced by zero, but + # we can and should be careful. + # + if not np.isfinite(exposure_table[k][jj]).all(): + log.warning("exposure_table column '%s' contains NaN or other non-finite values, a sum will be performed over valid values.", k) + good_values = exposure_table[k][jj].copy() + good_values[~np.isfinite(good_values)] = 0 + res[k][i] = np.sum(good_values) + else: + if hasattr(exposure_table[k], 'mask'): + log.warning("exposure_table column '%s' contains masked values, a sum will be performed over unmasked values.", k) + res[k][i] = np.sum(exposure_table[k][jj]) + if k == "EFFTIME_ETC" or k == "EFFTIME_GFA": + if np.any((exposure_table[k][jj] == 0) & (exposure_table["EFFTIME_SPEC"][jj] > 0)): + res[k][i] = 0 # because we are missing data # copy the following from the exposure table if it exists and not already set as sv1