From a5cac286e4f2e1fa7a9dd977db85d85bcdd409c8 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Fri, 5 Nov 2021 13:39:07 +0100 Subject: [PATCH 001/244] in prep for telluric pca, make qso pca more explicit --- pypeit/core/telluric.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 7b4856d2ae..070841e066 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -37,7 +37,7 @@ # TODO These codes should probably be in a separate qso_pca module. Also pickle functionality needs to be removed. # The prior is not used (that was the reason for pickling), so the components could be stored in fits format. # npca is not actually required here. -def init_pca(filename,wave_grid,redshift, npca): +def init_qso_pca(filename,wave_grid,redshift, npca): """ This routine reads in the pickle file created by coarse_pca.create_coarse_pca. The relevant pieces are the wavelengths (wave_pca_c), the PCA components (pca_comp_c), and the Gaussian mixture @@ -76,7 +76,7 @@ def init_pca(filename,wave_grid,redshift, npca): pca_dict = {'npca': npca, 'components': pca_comp_new, 'coeffs': coeffs_c, 'z_fid': redshift, 'dloglam': dloglam} return pca_dict -def pca_eval(theta,pca_dict): +def qso_pca_eval(theta,qso_pca_dict): """ Function for evaluating the quasar PCA @@ -85,7 +85,7 @@ def pca_eval(theta,pca_dict): Parameter vector, where theta_pca[0] is redshift, theta_pca[1] is the normalization, and theta_pca[2:npca+1] are the PCA coefficients, where npca is the PCA dimensionality pca_dict (dict): - Dictionary continaing the PCA information generated by init_pca + Dictionary continaing the PCA information generated by init_qso_pca Returns: @@ -854,7 +854,7 @@ def init_qso_model(obj_params, iord, wave, flux, ivar, mask, tellmodel): """ - pca_dict = init_pca(obj_params['pca_file'], wave, obj_params['z_qso'], obj_params['npca']) + pca_dict = init_qso_pca(obj_params['pca_file'], wave, obj_params['z_qso'], obj_params['npca']) pca_mean = np.exp(pca_dict['components'][0, :]) tell_mask = tellmodel > obj_params['tell_norm_thresh'] # Create a reference model and bogus noise @@ -893,21 +893,21 @@ def eval_qso_model(theta, obj_dict): Returns ------- - pca_model : array with same shape as the PCA vectors (tored in the obj_dict['pca_dict']) + qso_pca_model : array with same shape as the PCA vectors (tored in the obj_dict['pca_dict']) PCA vectors were already interpolated onto the telluric model grid by init_qso_model - gpm : : array with same shape as the pca_model + gpm : : array with same shape as the qso_pca_model Good pixel mask indicating where the model is valid """ - pca_model = pca_eval(theta, obj_dict['pca_dict']) + qso_pca_model = qso_pca_eval(theta, obj_dict['pca_dict']) # TODO Is the prior evaluation slowing things down?? # TODO Disablingthe prior for now as I think it slows things down for no big gain #ln_pca_pri = qso_pca.pca_lnprior(theta_PCA, arg_dict['pca_dict']) #ln_pca_pri = 0.0 #flux_model, tell_model, spec_model, modelmask - return pca_model, (pca_model > 0.0) + return qso_pca_model, (qso_pca_model > 0.0) ############## @@ -1490,7 +1490,7 @@ def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile, # Apply the telluric correction telluric = TelObj.model['TELLURIC'][0,:] - pca_model = TelObj.model['OBJ_MODEL'][0,:] + qso_pca_model = TelObj.model['OBJ_MODEL'][0,:] # Plot the telluric corrected and rescaled orders flux_corr = flux/(telluric + (telluric == 0.0)) @@ -1510,11 +1510,11 @@ def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile, alpha=0.7, zorder=5) plt.plot(wave, sig_corr, drawstyle='steps-mid', color='r', label='noise', alpha=0.3, zorder=1) - plt.plot(wave, pca_model, color='cornflowerblue', linewidth=1.0, label='PCA model', + plt.plot(wave, qso_pca_model, color='cornflowerblue', linewidth=1.0, label='PCA model', zorder=7, alpha=0.7) - plt.plot(wave, pca_model.max()*0.9*telluric, color='magenta', drawstyle='steps-mid', + plt.plot(wave, qso_pca_model.max()*0.9*telluric, color='magenta', drawstyle='steps-mid', label='telluric', alpha=0.4) - plt.ylim(-0.1*pca_model.max(), 1.5*pca_model.max()) + plt.ylim(-0.1*qso_pca_model.max(), 1.5*qso_pca_model.max()) plt.legend() plt.xlabel('Wavelength') plt.ylabel('Flux') @@ -1522,7 +1522,7 @@ def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile, # save the telluric corrected spectrum save_coadd1d_tofits(outfile, wave, flux_corr, ivar_corr, mask_corr, - spectrograph=header['PYP_SPEC'], telluric=telluric, obj_model=pca_model, + spectrograph=header['PYP_SPEC'], telluric=telluric, obj_model=qso_pca_model, header=header, ex_value='OPT', overwrite=True) return TelObj From e58ab6fcd58ced4bb72af7f6f2195e55fca8678d Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Fri, 5 Nov 2021 13:43:28 +0100 Subject: [PATCH 002/244] adjusted qso pca redshift implementation to smooth out derivative --- pypeit/core/telluric.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 070841e066..2e3cb9a044 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -64,16 +64,16 @@ def init_qso_pca(filename,wave_grid,redshift, npca): pca_table = table.Table.read(filename) wave_pca_c = pca_table['WAVE_PCA'][0].flatten() pca_comp_c = pca_table['PCA_COMP'][0][0,:,:] - coeffs_c =pca_table['PCA_COEFFS'][0][0,:,:] - #wave_pca_c, cont_all_c, pca_comp_c, coeffs_c, mean_pca, covar_pca, diff_pca, mix_fit, chi2, dof = pickle.load(open(filename,'rb')) + coeffs_c = pca_table['PCA_COEFFS'][0][0,:,:] num_comp = pca_comp_c.shape[0] # number of PCA components # Interpolate PCA components onto wave_grid - pca_interp = scipy.interpolate.interp1d(wave_pca_c*(1.0 + redshift),pca_comp_c, bounds_error=False, fill_value=0.0, axis=1) + pca_interp = scipy.interpolate.interp1d(wave_pca_c, pca_comp_c, bounds_error=False, fill_value=0.0, axis=1) pca_comp_new = pca_interp(wave_grid) # Generate a mixture model for the coefficients prior, what should ngauss be? #prior = mixture.GaussianMixture(n_components = npca-1).fit(coeffs_c[:, 1:npca]) # Construct the PCA dict - pca_dict = {'npca': npca, 'components': pca_comp_new, 'coeffs': coeffs_c, 'z_fid': redshift, 'dloglam': dloglam} + pca_dict = {'npca': npca, 'components': pca_comp_new, 'coeffs': coeffs_c, 'z_fid': redshift, 'dloglam': dloglam, + 'interp': pca_interp, 'wave_grid': wave_grid} return pca_dict def qso_pca_eval(theta,qso_pca_dict): @@ -98,8 +98,7 @@ def qso_pca_eval(theta,qso_pca_dict): z_qso = theta[0] norm = theta[1] A = theta[2:] - dshift = int(np.round(np.log10((1.0 + z_qso)/(1.0 + z_fid))/dloglam)) - C_now = np.roll(C[:npca,:], dshift, axis=1) + C_now = pca_dict['interp'](pca_dict['wave_grid']/(1.0+z_qso))[:npca,:] return norm*np.exp(np.dot(np.append(1.0,A),C_now)) # TODO The prior is not currently used, but this is left in here anyway. From 2150bb3442814a2d0876c2471396da03c5bcb9fa Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Fri, 5 Nov 2021 14:07:10 +0100 Subject: [PATCH 003/244] reworking for telluric pca --- pypeit/core/telluric.py | 150 ++++++++++++++++------------------------ 1 file changed, 61 insertions(+), 89 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 2e3cb9a044..17fb5b0419 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -130,99 +130,53 @@ def qso_pca_eval(theta,qso_pca_dict): # return gaussian_mixture_model.score_samples(A.reshape(1,-1)) -def read_telluric_grid(filename, wave_min=None, wave_max=None, pad_frac=0.10): +def read_telluric_pca(filename, wave_min=None, wave_max=None, pad_frac=0.10): """ - Reads in the telluric grid from a file. - - Optionally, this method also trims the grid to be in within ``wave_min`` + Reads in the telluric PCA components from a file. + + Optionally, this method also trims wavelength to be in within ``wave_min`` and ``wave_max`` and pads the data (see ``pad_frac``). - + .. todo:: List and describe the contents of the dictionary in the return description. - + Args: filename (:obj:`str`): - Telluric grid filename + Telluric PCA filename wave_min (:obj:`float`): - Minimum wavelength at which the grid is desired + Minimum wavelength at which the grid is desired wave_max (:obj:`float`): - Maximum wavelength at which the grid is desired. + Maximum wavelength at which the grid is desired. pad_frac (:obj:`float`): - Percentage padding to be added to the grid boundaries if - ``wave_min`` or ``wave_max`` are input; ignored otherwise. The - resulting grid will extend from ``(1.0 - pad_frac)*wave_min`` to - ``(1.0 + pad_frac)*wave_max``. - + Percentage padding to be added to the grid boundaries if + ``wave_min`` or ``wave_max`` are input; ignored otherwise. The + resulting grid will extend from ``(1.0 - pad_frac)*wave_min`` to + ``(1.0 + pad_frac)*wave_max``. + Returns: - :obj:`dict`: Dictionary containing the telluric grid. + :obj:`dict`: Dictionary containing the telluric PCA components. """ hdul = io.fits_open(filename) - wave_grid_full = 10.0*hdul[1].data - model_grid_full = hdul[0].data - nspec_full = wave_grid_full.size - + wave_grid_full = hdul[1].data + pca_comp_full = hdul[0].data + ncomp = hdul[0].header['NCOMP'] + bounds = hdul[2].data + ind_lower = np.argmin(np.abs(wave_grid_full - (1.0 - pad_frac)*wave_min)) \ if wave_min is not None else 0 ind_upper = np.argmin(np.abs(wave_grid_full - (1.0 + pad_frac)*wave_max)) \ if wave_max is not None else nspec_full wave_grid = wave_grid_full[ind_lower:ind_upper] - model_grid = model_grid_full[...,ind_lower:ind_upper] - - pg = hdul[0].header['PRES0']+hdul[0].header['DPRES']*np.arange(0,hdul[0].header['NPRES']) - tg = hdul[0].header['TEMP0']+hdul[0].header['DTEMP']*np.arange(0,hdul[0].header['NTEMP']) - hg = hdul[0].header['HUM0']+hdul[0].header['DHUM']*np.arange(0,hdul[0].header['NHUM']) - if hdul[0].header['NAM'] > 1: - ag = hdul[0].header['AM0']+hdul[0].header['DAM']*np.arange(0,hdul[0].header['NAM']) - else: - ag = hdul[0].header['AM0']+1*np.arange(0,1) + pca_comp_grid = pca_comp_full[:,ind_lower:ind_upper] dwave, dloglam, resln_guess, pix_per_sigma = wvutils.get_sampling(wave_grid) tell_pad_pix = int(np.ceil(10.0 * pix_per_sigma)) return dict(wave_grid=wave_grid, dloglam=dloglam, resln_guess=resln_guess, - pix_per_sigma=pix_per_sigma, tell_pad_pix=tell_pad_pix, pressure_grid=pg, - temp_grid=tg, h2o_grid=hg, airmass_grid=ag, tell_grid=model_grid) - - -def interp_telluric_grid(theta, tell_dict): - """ - Interpolate the telluric model grid to the specified location in - parameter space. - - The interpolation is only performed over the 4D parameter space specified - by pressure, temperature, humidity, and airmass. This routine performs - nearest-gridpoint interpolation to evaluate the telluric model at an - arbitrary location in this 4-d space. The telluric grid is assumed to be - uniformly sampled in this parameter space. - - Args: - theta (`numpy.ndarray`_): - A 4-element vector with the telluric model parameters: pressure, - temperature, humidity, and airmass. - tell_dict (dict): - Dictionary containing the telluric grid. See - :func:`read_telluric_grid`. - - Returns: - `numpy.ndarray`_: Telluric model evaluated at the provided 4D - position in parameter space. The telluric model is provided over all - available wavelengths in ``tell_dict``. - """ - if len(theta) != 4: - msgs.error('Input parameter vector must have 4 and only 4 values.') - pg = tell_dict['pressure_grid'] - tg = tell_dict['temp_grid'] - hg = tell_dict['h2o_grid'] - ag = tell_dict['airmass_grid'] - p, t, h, a = theta - pi = int(np.round((p-pg[0])/(pg[1]-pg[0]))) if len(pg) > 1 else 0 - ti = int(np.round((t-tg[0])/(tg[1]-tg[0]))) if len(tg) > 1 else 0 - hi = int(np.round((h-hg[0])/(hg[1]-hg[0]))) if len(hg) > 1 else 0 - ai = int(np.round((a-ag[0])/(ag[1]-ag[0]))) if len(ag) > 1 else 0 - return tell_dict['tell_grid'][pi,ti,hi,ai] - + pix_per_sigma=pix_per_sigma, tell_pad_pix=tell_pad_pix, ncomp_tell_pca=ncomp, + tell_pca=pca_comp_grid, bounds_tell_pca=bounds) def conv_telluric(tell_model, dloglam, res): """ @@ -332,29 +286,42 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): theta_tell in model atmosphere parameter space. """ ntheta = len(theta_tell) - if ntheta not in [5, 7]: - msgs.error('Input model atmosphere parameters must have length 5 or 7.') - - tellmodel_hires = interp_telluric_grid(theta_tell[:4], tell_dict) - + # Infer number of used components from the number of parameters + # TODO: make this work even without shift and stretch + ncomp_use = ntheta-3 + if comp_use > tell_dict['ncomp_tell_pca']: + msgs.error('Asked for more PCA components than exist in PCA file.') + + tellmodel_hires = np.dot(np.append(1,theta_tell[:ncomp_use]),tell_dict['tell_pca'][:ncomp_use]) + + # PCA model can technically give some unphysical values, + # so trim to stay between 0 and 1 + clip_hi = tellmodel_hires > 1.0 + tellmodel_hires[clip_hi] = 1.0 + clip_lo = tellmodel_hires < 0.0 + tellmodel_hires[clip_lo] = 0.0 + # Set the wavelength range if not provided ind_lower = 0 if ind_lower is None else ind_lower ind_upper = tell_dict['wave_grid'].size - 1 if ind_upper is None else ind_upper - + # Deal with padding for the convolutions ind_lower_pad = np.fmax(ind_lower - tell_dict['tell_pad_pix'], 0) ind_upper_pad = np.fmin(ind_upper + tell_dict['tell_pad_pix'], tell_dict['wave_grid'].size - 1) ## FW: There is an extreme case with ind_upper == ind_upper_pad, the previous -0 won't work ind_lower_final = ind_lower_pad if ind_lower_pad == ind_lower else ind_lower - ind_lower_pad ind_upper_final = ind_upper_pad if ind_upper_pad == ind_upper else ind_upper - ind_upper_pad + # FD: currently assumes shift + stretch is on tellmodel_conv = conv_telluric(tellmodel_hires[ind_lower_pad:ind_upper_pad+1], - tell_dict['dloglam'], theta_tell[4]) - if ntheta == 5: - return tellmodel_conv[ind_lower_final:ind_upper_final] + tell_dict['dloglam'], theta_tell[-3]) + + #if ntheta == 5: + # return tellmodel_conv[ind_lower_final:ind_upper_final] + # FD: currently assumes shift + stretch is on tellmodel_out = shift_telluric(tellmodel_conv, np.log10(tell_dict['wave_grid'][ind_lower_pad:ind_upper_pad+1]), - tell_dict['dloglam'], theta_tell[5], theta_tell[6]) + tell_dict['dloglam'], theta_tell[-2], theta_tell[-1]) return tellmodel_out[ind_lower_final:ind_upper_final] @@ -387,9 +354,13 @@ def tellfit_chi2(theta, flux, thismask, arg_dict): obj_model_func = arg_dict['obj_model_func'] flux_ivar = arg_dict['ivar'] - - theta_obj = theta[:-7] - theta_tell = theta[-7:] + ncomp_use = arg_dict['ncomp_tell_use'] + + # TODO: make this work without shift and stretch turned on + nfit = arg_dict['ncomp_tell_use']+3 + + theta_obj = theta[:-nfit] + theta_tell = theta[-nfit:] tell_model = eval_telluric(theta_tell, arg_dict['tell_dict'], ind_lower=arg_dict['ind_lower'], ind_upper=arg_dict['ind_upper']) obj_model, model_gpm = obj_model_func(theta_obj, arg_dict['obj_dict']) @@ -422,9 +393,8 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): theta_obj = theta[:-7] theta_tell = theta[-7:] - The telluric model theta_tell is currently hard wired to be six dimensional:: - - pressure, temperature, humidity, airmass, resln, shift = theta_tell + The telluric model theta_tell includes a user-specified number + of PCA coefficients, spectral resolution, shift, and stretch. The object model theta_obj can have an arbitrary size and is provided as an argument to obj_model_func @@ -490,6 +460,8 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): nparams = len(bounds) # Number of parameters in the model popsize = arg_dict['popsize'] # Note this does nothing if the init is done from a previous iteration or optimum nsamples = arg_dict['popsize']*nparams + # FD: Currently assumes shift and stretch are turned on. + ntell = arg_dict['ncomp_tell_use']+3 # Decide how to initialize if init_from_last is not None: # Use a Gaussian ball about the optimum from a previous iteration @@ -503,9 +475,9 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): init_obj = np.array([[np.clip(param + ballsize*(bounds_obj[i][1] - bounds_obj[i][0]) * rng.standard_normal(1)[0], bounds_obj[i][0], bounds_obj[i][1]) for i, param in enumerate(arg_dict['obj_dict']['init_obj_opt_theta'])] for jsamp in range(nsamples)]) - tell_lhs = utils.lhs(7, samples=nsamples) + tell_lhs = utils.lhs(ntell, samples=nsamples) init_tell = np.array([[bounds[-idim][0] + tell_lhs[isamp, idim] * (bounds[-idim][1] - bounds[-idim][0]) - for idim in range(7)] for isamp in range(nsamples)]) + for idim in range(ntell)] for isamp in range(nsamples)]) init = np.hstack((init_obj, init_tell)) else: # If this is the first iteration and no object model optimum is presented, use a latin hypercube which is the default @@ -515,8 +487,8 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): init = init, updating='immediate', popsize=popsize, recombination=arg_dict['recombination'], maxiter=arg_dict['diff_evol_maxiter'], polish=arg_dict['polish'], disp=arg_dict['disp']) - theta_obj = result.x[:-7] - theta_tell = result.x[-7:] + theta_obj = result.x[:-ntell] + theta_tell = result.x[-ntell:] tell_model = eval_telluric(theta_tell, arg_dict['tell_dict'], ind_lower=arg_dict['ind_lower'], ind_upper=arg_dict['ind_upper']) obj_model, modelmask = obj_model_func(theta_obj, arg_dict['obj_dict']) From 3481727056fe553abddff0aa4a5531cdab9d5483 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Fri, 5 Nov 2021 14:31:57 +0100 Subject: [PATCH 004/244] reworking for telluric pca --- pypeit/core/telluric.py | 99 ++++++++++++++++++++--------------------- 1 file changed, 49 insertions(+), 50 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 17fb5b0419..898ee5b7a9 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -354,10 +354,10 @@ def tellfit_chi2(theta, flux, thismask, arg_dict): obj_model_func = arg_dict['obj_model_func'] flux_ivar = arg_dict['ivar'] - ncomp_use = arg_dict['ncomp_tell_use'] + ncomp_use = arg_dict['ntell'] # TODO: make this work without shift and stretch turned on - nfit = arg_dict['ncomp_tell_use']+3 + nfit = arg_dict['ntell']+3 theta_obj = theta[:-nfit] theta_tell = theta[-nfit:] @@ -461,7 +461,7 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): popsize = arg_dict['popsize'] # Note this does nothing if the init is done from a previous iteration or optimum nsamples = arg_dict['popsize']*nparams # FD: Currently assumes shift and stretch are turned on. - ntell = arg_dict['ncomp_tell_use']+3 + ntheta_tell = arg_dict['ntell']+3 # Decide how to initialize if init_from_last is not None: # Use a Gaussian ball about the optimum from a previous iteration @@ -487,8 +487,8 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): init = init, updating='immediate', popsize=popsize, recombination=arg_dict['recombination'], maxiter=arg_dict['diff_evol_maxiter'], polish=arg_dict['polish'], disp=arg_dict['disp']) - theta_obj = result.x[:-ntell] - theta_tell = result.x[-ntell:] + theta_obj = result.x[:-ntheta_tell] + theta_tell = result.x[-ntheta_tell:] tell_model = eval_telluric(theta_tell, arg_dict['tell_dict'], ind_lower=arg_dict['ind_lower'], ind_upper=arg_dict['ind_upper']) obj_model, modelmask = obj_model_func(theta_obj, arg_dict['obj_dict']) @@ -826,7 +826,7 @@ def init_qso_model(obj_params, iord, wave, flux, ivar, mask, tellmodel): """ pca_dict = init_qso_pca(obj_params['pca_file'], wave, obj_params['z_qso'], obj_params['npca']) - pca_mean = np.exp(pca_dict['components'][0, :]) + pca_mean = np.exp(pca_dict['interp'](pca_dict['wave_grid']*(1+pca_dict['z_fid']))[0, :]) tell_mask = tellmodel > obj_params['tell_norm_thresh'] # Create a reference model and bogus noise flux_ref = pca_mean * tellmodel @@ -1163,7 +1163,7 @@ def mask_star_lines(wave_star, mask_width=10.0): return mask_star def sensfunc_telluric(wave, counts, counts_ivar, counts_mask, exptime, airmass, std_dict, - telgridfile, ech_orders=None, polyorder=8, mask_abs_lines=True, + telgridfile, ech_orders=None, polyorder=8, mask_abs_lines=True, ntell=4, delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), sn_clip=30.0, ballsize=5e-4, only_orders=None, maxiter=3, lower=3.0, upper=3.0, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, @@ -1207,6 +1207,8 @@ def sensfunc_telluric(wave, counts, counts_ivar, counts_mask, exptime, airmass, Polynomial order for the sensitivity function fit. mask_abs_lines : :obj:`bool`, optional, default=True Mask proiminent stellar absorption lines? + ntell : :obj:`int`, optional, default = 4 + Number of telluric PCA components to use delta_coeff_bounds : :obj:`tuple`, optional, default = (-20.0, 20.0) Parameters setting the polynomial coefficient bounds for sensfunc optimization. @@ -1295,6 +1297,7 @@ def sensfunc_telluric(wave, counts, counts_ivar, counts_mask, exptime, airmass, # Since we are fitting a sensitivity function, first compute counts per second per angstrom. TelObj = Telluric(wave, counts, counts_ivar, mask_tot, telgridfile, obj_params, + ntell=ntell, init_sensfunc_model, eval_sensfunc_model, ech_orders=ech_orders, sn_clip=sn_clip, maxiter=maxiter, lower=lower, upper=upper, tol=tol, popsize=popsize, recombination=recombination, polish=polish, disp=disp, @@ -1336,7 +1339,7 @@ def create_bal_mask(wave, bal_wv_min_max): def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile, npca=8, - pca_lower=1220.0, pca_upper=3100.0, bal_wv_min_max=None, delta_zqso=0.1, + pca_lower=1220.0, pca_upper=3100.0, bal_wv_min_max=None, delta_zqso=0.1, ntell=4, bounds_norm=(0.1, 3.0), tell_norm_thresh=0.9, sn_clip=30.0, only_orders=None, maxiter=3, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, debug_init=False, debug=False, show=False): @@ -1372,6 +1375,8 @@ def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile, delta_zqso : :obj:`float`, optional During the fit, the QSO redshift is allowed to vary within ``+/-delta_zqso``. + ntell : :obj:`int`, optional, default = 4 + Number of telluric PCA components to use bounds_norm : :obj:`tuple`, optional A two-tuple with the lower and upper bounds on the fractional adjustment of the flux in the QSO model spectrum. For example, a value of ``(0.1, @@ -1455,7 +1460,7 @@ def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile, # parameters lowered for testing TelObj = Telluric(wave, flux, ivar, mask_tot, telgridfile, obj_params, init_qso_model, eval_qso_model, sn_clip=sn_clip, maxiter=maxiter, tol=tol, popsize=popsize, - recombination=recombination, polish=polish, disp=disp, debug=debug) + ntell=ntell, recombination=recombination, polish=polish, disp=disp, debug=debug) TelObj.run(only_orders=only_orders) TelObj.to_file(telloutfile, overwrite=True) @@ -1500,7 +1505,7 @@ def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile, def star_telluric(spec1dfile, telgridfile, telloutfile, outfile, star_type=None, star_mag=None, star_ra=None, star_dec=None, func='legendre', model='exp', polyorder=5, - mask_abs_lines=True, delta_coeff_bounds=(-20.0, 20.0), + mask_abs_lines=True, ntell=4, delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), only_orders=None, sn_clip=30.0, maxiter=3, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, debug_init=False, debug=False, show=False): @@ -1553,7 +1558,7 @@ def star_telluric(spec1dfile, telgridfile, telloutfile, outfile, star_type=None, # parameters lowered for testing TelObj = Telluric(wave, flux, ivar, mask_tot, telgridfile, obj_params, init_star_model, - eval_star_model, sn_clip=sn_clip, tol=tol, popsize=popsize, + eval_star_model, ntell=ntell, sn_clip=sn_clip, tol=tol, popsize=popsize, recombination=recombination, polish=polish, disp=disp, debug=debug) TelObj.run(only_orders=only_orders) TelObj.to_file(telloutfile, overwrite=True) @@ -1598,7 +1603,7 @@ def star_telluric(spec1dfile, telgridfile, telloutfile, outfile, star_type=None, return TelObj def poly_telluric(spec1dfile, telgridfile, telloutfile, outfile, z_obj=0.0, func='legendre', - model='exp', polyorder=3, fit_wv_min_max=None, mask_lyman_a=True, + model='exp', polyorder=3, fit_wv_min_max=None, mask_lyman_a=True, ntell=4, delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), only_orders=None, sn_clip=30.0, maxiter=3, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, debug_init=False, debug=False, @@ -1648,7 +1653,7 @@ def poly_telluric(spec1dfile, telgridfile, telloutfile, outfile, z_obj=0.0, func # parameters lowered for testing TelObj = Telluric(wave, flux, ivar, mask_tot, telgridfile, obj_params, init_poly_model, eval_poly_model, sn_clip=sn_clip, maxiter=maxiter, tol=tol, popsize=popsize, - recombination=recombination, polish=polish, disp=disp, debug=debug) + ntell=ntell, recombination=recombination, polish=polish, disp=disp, debug=debug) TelObj.run(only_orders=only_orders) TelObj.to_file(telloutfile, overwrite=True) @@ -1948,6 +1953,8 @@ class Telluric(datamodel.DataContainer): datamodel = {'telgrid': dict(otype=str, descr='File containing grid of HITRAN atmosphere models'), + 'ntell': dict(otype=int, + descr='Number of telluric PCA components used') 'std_src': dict(otype=str, descr='Name of the standard source'), 'std_name': dict(otype=str, descr='Type of standard source'), 'std_cal': dict(otype=str, @@ -1983,7 +1990,7 @@ class Telluric(datamodel.DataContainer): """DataContainer datamodel.""" @staticmethod - def empty_model_table(norders, nspec, n_obj_par=0): + def empty_model_table(norders, nspec, ntell=4, n_obj_par=0): """ Construct an empty `astropy.table.Table`_ for the telluric model results. @@ -2008,16 +2015,10 @@ def empty_model_table(norders, nspec, n_obj_par=0): table.Column(name='OBJ_MODEL', dtype=float, length=norders, shape=(nspec,), description='Best-fitting object model spectrum'), # TODO: Why do we need both TELL_THETA and all the individual parameters... - table.Column(name='TELL_THETA', dtype=float, length=norders, shape=(7,), + table.Column(name='TELL_THETA', dtype=float, length=norders, shape=(ntell+3,), description='Best-fitting telluric model parameters'), - table.Column(name='TELL_PRESS', dtype=float, length=norders, - description='Best-fitting telluric model pressure'), - table.Column(name='TELL_TEMP', dtype=float, length=norders, - description='Best-fitting telluric model temperature'), - table.Column(name='TELL_H2O', dtype=float, length=norders, - description='Best-fitting telluric model humidity'), - table.Column(name='TELL_AIRMASS', dtype=float, length=norders, - description='Best-fitting telluric model airmass'), + table.Column(name='TELL_COEFF', dtype=float, length=norders, shape=(ntell,), + description='Best-fitting telluric PCA coefficients'), table.Column(name='TELL_RESLN', dtype=float, length=norders, description='Best-fitting telluric model spectral resolution'), table.Column(name='TELL_SHIFT', dtype=float, length=norders, @@ -2046,7 +2047,7 @@ def empty_model_table(norders, nspec, n_obj_par=0): description='Maximum wavelength included in the fit')]) def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_model, - eval_obj_model, ech_orders=None, sn_clip=30.0, airmass_guess=1.5, + eval_obj_model, ech_orders=None, sn_clip=30.0, ntell=4, resln_guess=None, resln_frac_bounds=(0.5, 1.5), pix_shift_bounds=(-5.0, 5.0), pix_stretch_bounds=(0.9,1.1), maxiter=2, sticky=True, lower=3.0, upper=3.0, seed=777, ballsize = 5e-4, tol=1e-3, diff_evol_maxiter=1000, popsize=30, @@ -2067,7 +2068,7 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode self.telgrid = telgridfile self.obj_params = obj_params self.init_obj_model = init_obj_model - self.airmass_guess = airmass_guess + self.ntell = ntell self.eval_obj_model = eval_obj_model self.ech_orders = ech_orders self.sn_clip = sn_clip @@ -2206,6 +2207,7 @@ def _init_internals(self): self.ngrid = None self.resln_guess = None + self.ntell = None self.tell_guess = None self.bounds_tell = None @@ -2355,13 +2357,10 @@ def assign_output(self, iord): kind='linear', bounds_error=False, fill_value=0.0)(wave_in_gd) self.model['TELL_THETA'][iord] = self.theta_tell_list[iord] - self.model['TELL_PRESS'][iord] = self.theta_tell_list[iord][0] - self.model['TELL_TEMP'][iord] = self.theta_tell_list[iord][1] - self.model['TELL_H2O'][iord] = self.theta_tell_list[iord][2] - self.model['TELL_AIRMASS'][iord] = self.theta_tell_list[iord][3] - self.model['TELL_RESLN'][iord] = self.theta_tell_list[iord][4] - self.model['TELL_SHIFT'][iord] = self.theta_tell_list[iord][5] - self.model['TELL_STRETCH'][iord] = self.theta_tell_list[iord][6] + self.model['TELL_COEFF'][iord] = self.theta_tell_list[iord][:self.ntell] + self.model['TELL_RESLN'][iord] = self.theta_tell_list[iord][self.ntell] + self.model['TELL_SHIFT'][iord] = self.theta_tell_list[iord][self.ntell+1] + self.model['TELL_STRETCH'][iord] = self.theta_tell_list[iord][self.ntell+2] ntheta_iord = len(self.theta_obj_list[iord]) self.model['OBJ_THETA'][iord][:ntheta_iord] = self.theta_obj_list[iord] self.model['CHI2'][iord] = self.result_list[iord].fun @@ -2436,14 +2435,14 @@ def get_tell_guess(self): function. Returns: - :obj:`tuple`: The guess pressure, temperature, humidity, - airmass, resolution, shift, and stretch parameters. The first - three are the median of the parameters covered by the telluric - model grid. + :obj:`tuple`: The guess telluric PCA coefficients, + resolution, shift, and stretch parameters. """ - return np.median(self.tell_dict['pressure_grid']), np.median(self.tell_dict['temp_grid']), \ - np.median(self.tell_dict['h2o_grid']), self.airmass_guess, self.resln_guess, \ - 0.0, 1.0 + guess = list(np.zeros(ntell)) + guess.append(self.resln_guess) + guess.append(0.0) + guess.append(1.0) + return tuple(guess) def get_bounds_tell(self): """ @@ -2456,13 +2455,15 @@ def get_bounds_tell(self): stretch parameters. """ # Set the bounds for the optimization - return [(self.tell_dict['pressure_grid'].min(), self.tell_dict['pressure_grid'].max()), - (self.tell_dict['temp_grid'].min(), self.tell_dict['temp_grid'].max()), - (self.tell_dict['h2o_grid'].min(), self.tell_dict['h2o_grid'].max()), - (self.tell_dict['airmass_grid'].min(), self.tell_dict['airmass_grid'].max()), - (self.resln_guess * self.resln_frac_bounds[0], - self.resln_guess * self.resln_frac_bounds[1]), - self.pix_shift_bounds, self.pix_stretch_bounds] + bounds = [] + for ii in range(self.ntell): + bounds.append((self.tell_dict['bounds_tell_pca'][0][ii+1], + self.tell_dict['bounds_tell_pca'][1][ii+1])) + bounds.append((self.resln_guess * self.resln_frac_bounds[0], + self.resln_guess * self.resln_frac_bounds[1])) + bounds.append(self.pix_shift_bounds) + bounds.append(self.pix_stretch_bounds) + return bounds def sort_telluric(self): """ @@ -2484,10 +2485,8 @@ def sort_telluric(self): # Do a quick loop over all the orders to sort them in order of strongest # to weakest telluric absorption for iord in range(self.norders): - tm_grid = self.tell_dict['tell_grid'][...,self.ind_lower[iord]:self.ind_upper[iord]+1] - tell_model_mid = tm_grid[tm_grid.shape[0]//2, tm_grid.shape[1]//2, - tm_grid.shape[2]//2, tm_grid.shape[3]//2, :] - tell_med[iord] = np.mean(tell_model_mid) + tell_model_mean = self.tell_dict['tell_pca'][0,self.ind_lower[iord]:self.ind_upper[iord]+1] + tell_med[iord] = np.mean(tell_model_mean) # Perform fits in order of telluric strength return tell_med.argsort() From 8927febb433457fb861ddd30b95a68d9851f1fef Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Fri, 5 Nov 2021 14:43:13 +0100 Subject: [PATCH 005/244] minor fixes to get things running --- pypeit/core/telluric.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 898ee5b7a9..263706b34b 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -289,10 +289,10 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): # Infer number of used components from the number of parameters # TODO: make this work even without shift and stretch ncomp_use = ntheta-3 - if comp_use > tell_dict['ncomp_tell_pca']: + if ncomp_use > tell_dict['ncomp_tell_pca']: msgs.error('Asked for more PCA components than exist in PCA file.') - tellmodel_hires = np.dot(np.append(1,theta_tell[:ncomp_use]),tell_dict['tell_pca'][:ncomp_use]) + tellmodel_hires = np.dot(np.append(1,theta_tell[:ncomp_use]),tell_dict['tell_pca'][:ncomp_use+1]) # PCA model can technically give some unphysical values, # so trim to stay between 0 and 1 @@ -1297,8 +1297,7 @@ def sensfunc_telluric(wave, counts, counts_ivar, counts_mask, exptime, airmass, # Since we are fitting a sensitivity function, first compute counts per second per angstrom. TelObj = Telluric(wave, counts, counts_ivar, mask_tot, telgridfile, obj_params, - ntell=ntell, - init_sensfunc_model, eval_sensfunc_model, ech_orders=ech_orders, + init_sensfunc_model, eval_sensfunc_model, ntell=ntell, ech_orders=ech_orders, sn_clip=sn_clip, maxiter=maxiter, lower=lower, upper=upper, tol=tol, popsize=popsize, recombination=recombination, polish=polish, disp=disp, sensfunc=True, debug=debug) @@ -1954,7 +1953,7 @@ class Telluric(datamodel.DataContainer): datamodel = {'telgrid': dict(otype=str, descr='File containing grid of HITRAN atmosphere models'), 'ntell': dict(otype=int, - descr='Number of telluric PCA components used') + descr='Number of telluric PCA components used'), 'std_src': dict(otype=str, descr='Name of the standard source'), 'std_name': dict(otype=str, descr='Type of standard source'), 'std_cal': dict(otype=str, @@ -2100,7 +2099,7 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode # 3) Read the telluric grid and initalize associated parameters wv_gpm = self.wave_in_arr > 1.0 - self.tell_dict = read_telluric_grid(self.telgrid, wave_min=self.wave_in_arr[wv_gpm].min(), + self.tell_dict = read_telluric_pca(self.telgrid, wave_min=self.wave_in_arr[wv_gpm].min(), wave_max=self.wave_in_arr[wv_gpm].max()) self.wave_grid = self.tell_dict['wave_grid'] self.ngrid = self.wave_grid.size @@ -2159,7 +2158,7 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode self.bounds_list[iord] = bounds_iord arg_dict_iord = dict(ivar=self.ivar_arr[self.ind_lower[iord]:self.ind_upper[iord]+1,iord], tell_dict=self.tell_dict, ind_lower=self.ind_lower[iord], - ind_upper=self.ind_upper[iord], + ind_upper=self.ind_upper[iord], ntell=self.ntell, obj_model_func=self.eval_obj_model, obj_dict=obj_dict, ballsize=self.ballsize, bounds=bounds_iord, rng=self.rng, diff_evol_maxiter=self.diff_evol_maxiter, tol=self.tol, @@ -2438,7 +2437,7 @@ def get_tell_guess(self): :obj:`tuple`: The guess telluric PCA coefficients, resolution, shift, and stretch parameters. """ - guess = list(np.zeros(ntell)) + guess = list(np.zeros(self.ntell)) guess.append(self.resln_guess) guess.append(0.0) guess.append(1.0) From b3a42415e5ef6f91b359ee0d7bf58adc61fbfe9d Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Fri, 5 Nov 2021 15:01:15 +0100 Subject: [PATCH 006/244] adding par for ntell --- pypeit/par/pypeitpar.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index 718da98567..ca53f15aad 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -1750,7 +1750,7 @@ class TelluricPar(ParSet): """ def __init__(self, telgridfile=None, sn_clip=None, resln_guess=None, resln_frac_bounds=None, pix_shift_bounds=None, - delta_coeff_bounds=None, minmax_coeff_bounds=None, maxiter=None, + delta_coeff_bounds=None, minmax_coeff_bounds=None, maxiter=None, ntell=None, sticky=None, lower=None, upper=None, seed=None, tol=None, popsize=None, recombination=None, polish=None, disp=None, objmodel=None, redshift=None, delta_redshift=None, pca_file=None, npca=None, bal_wv_min_max=None, bounds_norm=None, tell_norm_thresh=None, only_orders=None, pca_lower=None, @@ -1773,6 +1773,10 @@ def __init__(self, telgridfile=None, sn_clip=None, resln_guess=None, resln_frac_ descr['telgridfile'] = 'File containing the telluric grid for the observatory in question. These grids are ' \ 'generated from HITRAN models for each observatory using nominal site parameters. They ' \ 'must be downloaded from the GoogleDrive and stored in PypeIt/pypeit/data/telluric/' + + defaults['ntell'] = 4 + dtypes['ntell'] = int + descr['ntell'] = 'Number of telluric PCA components to use in the fitting' defaults['sn_clip'] = 30.0 dtypes['sn_clip'] = [int, float] From de16a0c7f65b346407b612692a0daa6c6be30123 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Fri, 5 Nov 2021 15:13:47 +0100 Subject: [PATCH 007/244] making ntell par actually go through --- pypeit/core/telluric.py | 3 +-- pypeit/scripts/tellfit.py | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 263706b34b..0303da0302 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -2206,7 +2206,6 @@ def _init_internals(self): self.ngrid = None self.resln_guess = None - self.ntell = None self.tell_guess = None self.bounds_tell = None @@ -2317,7 +2316,7 @@ def init_output(self): """ self.model = self.empty_model_table(self.norders, self.nspec_in, - n_obj_par=self.max_ntheta_obj) + ntell=self.ntell, n_obj_par=self.max_ntheta_obj) if 'output_meta_keys' in self.obj_params: for key in self.obj_params['output_meta_keys']: if key.lower() in self.datamodel.keys(): diff --git a/pypeit/scripts/tellfit.py b/pypeit/scripts/tellfit.py index b9f1870c85..b7ca406041 100644 --- a/pypeit/scripts/tellfit.py +++ b/pypeit/scripts/tellfit.py @@ -142,7 +142,7 @@ def main(args): TelQSO = telluric.qso_telluric(args.spec1dfile, par['telluric']['telgridfile'], par['telluric']['pca_file'], par['telluric']['redshift'], modelfile, outfile, - npca=par['telluric']['npca'], + npca=par['telluric']['npca'], ntell=par['telluric']['ntell'], pca_lower=par['telluric']['pca_lower'], pca_upper=par['telluric']['pca_upper'], bounds_norm=par['telluric']['bounds_norm'], @@ -164,6 +164,7 @@ def main(args): polyorder=par['telluric']['polyorder'], only_orders=par['telluric']['only_orders'], mask_abs_lines=par['telluric']['mask_abs_lines'], + ntell=par['telluric']['ntell'], delta_coeff_bounds=par['telluric']['delta_coeff_bounds'], minmax_coeff_bounds=par['telluric']['minmax_coeff_bounds'], maxiter=par['telluric']['maxiter'], @@ -176,6 +177,7 @@ def main(args): func=par['telluric']['func'], model=par['telluric']['model'], polyorder=par['telluric']['polyorder'], + ntell=par['telluric']['ntell'], fit_wv_min_max=par['telluric']['fit_wv_min_max'], mask_lyman_a=par['telluric']['mask_lyman_a'], delta_coeff_bounds=par['telluric']['delta_coeff_bounds'], From 6f48d746778728920652db61729ac5ecfa294312 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Fri, 12 Nov 2021 13:12:00 +0100 Subject: [PATCH 008/244] minor fix to initialization --- pypeit/core/telluric.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 0303da0302..6b2e29a794 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -475,9 +475,9 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): init_obj = np.array([[np.clip(param + ballsize*(bounds_obj[i][1] - bounds_obj[i][0]) * rng.standard_normal(1)[0], bounds_obj[i][0], bounds_obj[i][1]) for i, param in enumerate(arg_dict['obj_dict']['init_obj_opt_theta'])] for jsamp in range(nsamples)]) - tell_lhs = utils.lhs(ntell, samples=nsamples) + tell_lhs = utils.lhs(ntheta_tell, samples=nsamples) init_tell = np.array([[bounds[-idim][0] + tell_lhs[isamp, idim] * (bounds[-idim][1] - bounds[-idim][0]) - for idim in range(ntell)] for isamp in range(nsamples)]) + for idim in range(ntheta_tell)] for isamp in range(nsamples)]) init = np.hstack((init_obj, init_tell)) else: # If this is the first iteration and no object model optimum is presented, use a latin hypercube which is the default @@ -2265,8 +2265,8 @@ def run(self, only_orders=None): inmask=self.mask_arr[self.ind_lower[iord]:self.ind_upper[iord]+1,iord], maxiter=self.maxiter, lower=self.lower, upper=self.upper, sticky=self.sticky) - self.theta_obj_list[iord] = self.result_list[iord].x[:-7] - self.theta_tell_list[iord] = self.result_list[iord].x[-7:] + self.theta_obj_list[iord] = self.result_list[iord].x[:-(self.ntell+3)] + self.theta_tell_list[iord] = self.result_list[iord].x[-(self.ntell+3):] self.obj_model_list[iord], modelmask \ = self.eval_obj_model(self.theta_obj_list[iord], self.obj_dict_list[iord]) self.tellmodel_list[iord] = eval_telluric(self.theta_tell_list[iord], self.tell_dict, From 422efcd82dae1ddbb1139c391153f241620ebcb8 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Fri, 12 Nov 2021 13:36:37 +0100 Subject: [PATCH 009/244] par fix --- pypeit/par/pypeitpar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index ca53f15aad..bbe6884c6c 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -2011,7 +2011,7 @@ def __init__(self, telgridfile=None, sn_clip=None, resln_guess=None, resln_frac_ @classmethod def from_dict(cls, cfg): k = np.array([*cfg.keys()]) - parkeys = ['telgridfile', 'sn_clip', 'resln_guess', 'resln_frac_bounds', + parkeys = ['telgridfile', 'sn_clip', 'resln_guess', 'resln_frac_bounds', 'ntell', 'pix_shift_bounds', 'delta_coeff_bounds', 'minmax_coeff_bounds', 'maxiter', 'sticky', 'lower', 'upper', 'seed', 'tol', 'popsize', 'recombination', 'polish', 'disp', 'objmodel','redshift', 'delta_redshift', From 1b598ae860a8899ea5ab092bc75295b1a21cd63b Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Sat, 13 Nov 2021 11:00:58 +0100 Subject: [PATCH 010/244] finished renaming pca_dict to be more specific --- pypeit/core/telluric.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 6b2e29a794..faf6d9b669 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -91,14 +91,14 @@ def qso_pca_eval(theta,qso_pca_dict): """ - C = pca_dict['components'] - z_fid = pca_dict['z_fid'] - dloglam = pca_dict['dloglam'] - npca = pca_dict['npca'] # Size of the PCA currently being used, original PCA in the dict could be larger + C = qso_pca_dict['components'] + z_fid = qso_pca_dict['z_fid'] + dloglam = qso_pca_dict['dloglam'] + npca = qso_pca_dict['npca'] # Size of the PCA currently being used, original PCA in the dict could be larger z_qso = theta[0] norm = theta[1] A = theta[2:] - C_now = pca_dict['interp'](pca_dict['wave_grid']/(1.0+z_qso))[:npca,:] + C_now = qso_pca_dict['interp'](qso_pca_dict['wave_grid']/(1.0+z_qso))[:npca,:] return norm*np.exp(np.dot(np.append(1.0,A),C_now)) # TODO The prior is not currently used, but this is left in here anyway. From 4f11a6d28243ccc9284d080ad724779926743ecf Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Sat, 13 Nov 2021 11:02:55 +0100 Subject: [PATCH 011/244] finished renaming pca_dict to be more specific --- pypeit/core/telluric.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index faf6d9b669..2a0c8299fb 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -72,9 +72,9 @@ def init_qso_pca(filename,wave_grid,redshift, npca): # Generate a mixture model for the coefficients prior, what should ngauss be? #prior = mixture.GaussianMixture(n_components = npca-1).fit(coeffs_c[:, 1:npca]) # Construct the PCA dict - pca_dict = {'npca': npca, 'components': pca_comp_new, 'coeffs': coeffs_c, 'z_fid': redshift, 'dloglam': dloglam, + qso_pca_dict = {'npca': npca, 'components': pca_comp_new, 'coeffs': coeffs_c, 'z_fid': redshift, 'dloglam': dloglam, 'interp': pca_interp, 'wave_grid': wave_grid} - return pca_dict + return qso_pca_dict def qso_pca_eval(theta,qso_pca_dict): """ @@ -84,7 +84,7 @@ def qso_pca_eval(theta,qso_pca_dict): theta (`numpy.ndarray`_): Parameter vector, where theta_pca[0] is redshift, theta_pca[1] is the normalization, and theta_pca[2:npca+1] are the PCA coefficients, where npca is the PCA dimensionality - pca_dict (dict): + qso_pca_dict (dict): Dictionary continaing the PCA information generated by init_qso_pca Returns: From 3e983060aceb9e84c7731a037691eff634d93dd7 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Tue, 16 Nov 2021 11:22:44 +0100 Subject: [PATCH 012/244] reverting because interp was just way too slow... --- pypeit/core/telluric.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 2a0c8299fb..0cbfd41fc4 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -67,7 +67,7 @@ def init_qso_pca(filename,wave_grid,redshift, npca): coeffs_c = pca_table['PCA_COEFFS'][0][0,:,:] num_comp = pca_comp_c.shape[0] # number of PCA components # Interpolate PCA components onto wave_grid - pca_interp = scipy.interpolate.interp1d(wave_pca_c, pca_comp_c, bounds_error=False, fill_value=0.0, axis=1) + pca_interp = scipy.interpolate.interp1d(wave_pca_c*(1.0+redshift), pca_comp_c, bounds_error=False, fill_value=0.0, axis=1) pca_comp_new = pca_interp(wave_grid) # Generate a mixture model for the coefficients prior, what should ngauss be? #prior = mixture.GaussianMixture(n_components = npca-1).fit(coeffs_c[:, 1:npca]) @@ -98,7 +98,9 @@ def qso_pca_eval(theta,qso_pca_dict): z_qso = theta[0] norm = theta[1] A = theta[2:] - C_now = qso_pca_dict['interp'](qso_pca_dict['wave_grid']/(1.0+z_qso))[:npca,:] + #C_now = qso_pca_dict['interp'](qso_pca_dict['wave_grid']/(1.0+z_qso))[:npca,:] + dshift = int(np.round(np.log10((1.0 + z_qso)/(1.0 + z_fid))/dloglam)) + C_now = np.roll(C[:npca,:], dshift, axis=1) return norm*np.exp(np.dot(np.append(1.0,A),C_now)) # TODO The prior is not currently used, but this is left in here anyway. From b78d74e05a1d1cb0927775ea76cb0bf6482be8dc Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Wed, 17 Nov 2021 13:42:03 +0100 Subject: [PATCH 013/244] use ntell when computing sensfunc --- pypeit/sensfunc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pypeit/sensfunc.py b/pypeit/sensfunc.py index db7266014a..b30afe6cca 100644 --- a/pypeit/sensfunc.py +++ b/pypeit/sensfunc.py @@ -732,6 +732,7 @@ def compute_zeropoint(self): ech_orders=self.meta_spec['ECH_ORDERS'], sn_clip=self.par['IR']['sn_clip'], mask_abs_lines=self.par['mask_abs_lines'], + ntell=self.par['IR']['ntell'], maxiter=self.par['IR']['maxiter'], lower=self.par['IR']['lower'], upper=self.par['IR']['upper'], From 010ff22b97faf2f707979078f0c311e2caba30fe Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Thu, 17 Mar 2022 14:26:17 +0100 Subject: [PATCH 014/244] cleanup --- pypeit/core/telluric.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index c54d37749b..17f1469974 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -166,7 +166,7 @@ def read_telluric_pca(filename, wave_min=None, wave_max=None, pad_frac=0.10): if not os.path.isfile(filename): msgs.error(f"File {filename} is not on your disk. You likely need to download the Telluric files. See https://pypeit.readthedocs.io/en/release/installing.html#atmospheric-model-grids") - hdul = io.fits_open(filename) + hdul = fits.open(filename) wave_grid_full = hdul[1].data pca_comp_full = hdul[0].data ncomp = hdul[0].header['NCOMP'] From 9e0251318d57bf282efbf6dc868471270089b1ba Mon Sep 17 00:00:00 2001 From: profxj Date: Sun, 28 Aug 2022 06:32:11 -0700 Subject: [PATCH 015/244] citation --- CITATION.cff | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 CITATION.cff diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000000..8389c61de5 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,9 @@ +cff-version: 1.2.0 +message: "If you use this software, please cite it as below." +authors: + - family-names: Prochaska, Hennawi, + given-names: J. Xavier, Joseph +title: "PypeIt" +version: 1.10.0 +doi: 10.5281/zenodo.3743493 +date-released: 2022-08-28 \ No newline at end of file From 7576029181b3470acf292c5d0be91759326e9112 Mon Sep 17 00:00:00 2001 From: profxj Date: Sun, 28 Aug 2022 06:33:50 -0700 Subject: [PATCH 016/244] nvm --- CITATION.cff | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index 8389c61de5..4cca8f5b19 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -1,8 +1,8 @@ cff-version: 1.2.0 message: "If you use this software, please cite it as below." authors: - - family-names: Prochaska, Hennawi, - given-names: J. Xavier, Joseph + - family-names: Prochaska, Hennawi, Westfall, Cooke, Wang + given-names: J. Xavier, Joseph, Kyle, Ryan title: "PypeIt" version: 1.10.0 doi: 10.5281/zenodo.3743493 From 6b64d5980f76b163907ed41bbfdaaa2c76046478 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Fri, 14 Jul 2023 15:05:04 +0200 Subject: [PATCH 017/244] os not imported anymore --- pypeit/core/telluric.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index aabd640637..2a2fde8c6f 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -167,9 +167,9 @@ def read_telluric_pca(filename, wave_min=None, wave_max=None, pad_frac=0.10): Returns: :obj:`dict`: Dictionary containing the telluric PCA components. """ - # Check for file - if not os.path.isfile(filename): - msgs.error(f"File {filename} is not on your disk. You likely need to download the Telluric files. See https://pypeit.readthedocs.io/en/release/installing.html#atmospheric-model-grids") +# # Check for file +# if not os.path.isfile(filename): +# msgs.error(f"File {filename} is not on your disk. You likely need to download the Telluric files. See https://pypeit.readthedocs.io/en/release/installing.html#atmospheric-model-grids") hdul = fits.open(filename) wave_grid_full = hdul[1].data From 23136a41064d3d340e4334c7699981ebe43c5f0d Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Sun, 16 Jul 2023 22:21:44 +0200 Subject: [PATCH 018/244] minor things --- pypeit/core/telluric.py | 11 ++++++----- pypeit/sensfunc.py | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index f0a9e74bd1..6d5ec6fde4 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -315,12 +315,13 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): tellmodel_hires = np.exp(-np.sinh(np.dot(np.append(1,theta_tell[:ncomp_use]),tell_dict['tell_pca'][:ncomp_use+1]))) - # PCA model can technically give some unphysical values, + # PCA model could technically give some unphysical values, # so trim to stay between 0 and 1 - clip_hi = tellmodel_hires > 1.0 - tellmodel_hires[clip_hi] = 1.0 - clip_lo = tellmodel_hires < 0.0 - tellmodel_hires[clip_lo] = 0.0 + # This is NOT required for the asinh grid... + # clip_hi = tellmodel_hires > 1.0 + # tellmodel_hires[clip_hi] = 1.0 + # clip_lo = tellmodel_hires < 0.0 + # tellmodel_hires[clip_lo] = 0.0 # Set the wavelength range if not provided ind_lower = 0 if ind_lower is None else ind_lower diff --git a/pypeit/sensfunc.py b/pypeit/sensfunc.py index 29bfbc12f7..ea1816d30a 100644 --- a/pypeit/sensfunc.py +++ b/pypeit/sensfunc.py @@ -218,7 +218,7 @@ def __init__(self, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False): self.spectrograph.dispname = header['DISPNAME'] # Get the algorithm parameters - #self.par = self.spectrograph.default_pypeit_par()['sensfunc'] if par is None else par + self.par = self.spectrograph.default_pypeit_par()['sensfunc'] if par is None else par # TODO Should we allow the user to pass this in? self.par_fluxcalib = self.spectrograph.default_pypeit_par()['fluxcalib'] if par_fluxcalib is None else par_fluxcalib # TODO: Check the type of the parameter object? From 02f61eea51797d196d8be1813910109f3f4050dd Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Mon, 17 Jul 2023 11:56:26 +0200 Subject: [PATCH 019/244] back to asinh grid --- pypeit/core/telluric.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 6d5ec6fde4..23c79e480d 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -313,15 +313,17 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): if ncomp_use > tell_dict['ncomp_tell_pca']: msgs.error('Asked for more PCA components than exist in PCA file.') + #tellmodel_hires = np.dot(np.append(1,theta_tell[:ncomp_use]),tell_dict['tell_pca'][:ncomp_use+1]) + # Asinh model grid, might work slightly better but may be slower? tellmodel_hires = np.exp(-np.sinh(np.dot(np.append(1,theta_tell[:ncomp_use]),tell_dict['tell_pca'][:ncomp_use+1]))) # PCA model could technically give some unphysical values, # so trim to stay between 0 and 1 # This is NOT required for the asinh grid... - # clip_hi = tellmodel_hires > 1.0 - # tellmodel_hires[clip_hi] = 1.0 - # clip_lo = tellmodel_hires < 0.0 - # tellmodel_hires[clip_lo] = 0.0 +# clip_hi = tellmodel_hires > 1.0 +# tellmodel_hires[clip_hi] = 1.0 +# clip_lo = tellmodel_hires < 0.0 +# tellmodel_hires[clip_lo] = 0.0 # Set the wavelength range if not provided ind_lower = 0 if ind_lower is None else ind_lower From ec8f65a7698cc44a7d5d65c6f95568b98b563bcc Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Thu, 20 Jul 2023 21:05:02 +0200 Subject: [PATCH 020/244] documentation updates --- pypeit/core/telluric.py | 81 ++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 46 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 23c79e480d..634b511639 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -263,34 +263,29 @@ def shift_telluric(tell_model, loglam, dloglam, shift, stretch): def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): """ - Evaluate the telluric model at an arbitrary location in parameter space. - - The full atmosphere model parameter space is either 5 or 7 dimensional, - which is the size of the ``theta_tell`` input parameter vector. + Evaluate the telluric PCA model. - The parameters provided by ``theta_tell`` must be: pressure, temperature, - humidity, airmass, spectral resolution, shift, and stretch. The latter two - can be omitted if a shift and stretch of the telluric model are not - included. + The parameters provided by ``theta_tell`` must be: ntell PCA coefficients, + spectral resolution, shift, and stretch. The latter two can be omitted if + a shift and stretch of the telluric model are not included. This routine performs the following steps: - 1. nearest grid point interpolation of the telluric model onto a - new location in the 4-d space (pressure, temperature, - humidity, airmass) + 1. summation of telluric PCA components multiplied by coefficients + + 2. transformation of telluric PCA model from arsinh(tau) to transmission - 2. convolution of the atmosphere model to the resolution set by + 3. convolution of the atmosphere model to the resolution set by the spectral resolution. - 3. (Optional) shift and stretch the telluric model. + 4. (Optional) shift and stretch the telluric model. Args: theta_tell (`numpy.ndarray`_): - Vector with the telluric model parameters. Must be 5 or 7 - elements long. See method description. + Vector with the telluric PCA coefficients tell_dict (:obj:`dict`): - Dictionary containing the telluric grid data. See - :func:`read_telluric_grid`. + Dictionary containing the telluric PCA data. See + :func:`read_telluric_pca`. ind_lower (:obj:`int`, optional): The index of the first pixel to include in the model. Selecting a wavelength region for the modeling makes things faster because we @@ -313,18 +308,6 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): if ncomp_use > tell_dict['ncomp_tell_pca']: msgs.error('Asked for more PCA components than exist in PCA file.') - #tellmodel_hires = np.dot(np.append(1,theta_tell[:ncomp_use]),tell_dict['tell_pca'][:ncomp_use+1]) - # Asinh model grid, might work slightly better but may be slower? - tellmodel_hires = np.exp(-np.sinh(np.dot(np.append(1,theta_tell[:ncomp_use]),tell_dict['tell_pca'][:ncomp_use+1]))) - - # PCA model could technically give some unphysical values, - # so trim to stay between 0 and 1 - # This is NOT required for the asinh grid... -# clip_hi = tellmodel_hires > 1.0 -# tellmodel_hires[clip_hi] = 1.0 -# clip_lo = tellmodel_hires < 0.0 -# tellmodel_hires[clip_lo] = 0.0 - # Set the wavelength range if not provided ind_lower = 0 if ind_lower is None else ind_lower ind_upper = tell_dict['wave_grid'].size - 1 if ind_upper is None else ind_upper @@ -335,14 +318,20 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): ## FW: There is an extreme case with ind_upper == ind_upper_pad, the previous -0 won't work ind_lower_final = ind_lower_pad if ind_lower_pad == ind_lower else ind_lower - ind_lower_pad ind_upper_final = ind_upper_pad if ind_upper_pad == ind_upper else ind_upper - ind_upper_pad + + # Evaluate PCA model after truncating the wavelength range + tellmodel_hires = np.dot(np.append(1,theta_tell[:ncomp_use][ind_lower_pad:ind_upper_pad+1]), + tell_dict['tell_pca'][:ncomp_use+1]) + # Transform Arsinh PCA model, works slightly better than transmission components but may be slightly slower? + tellmodel_hires = np.exp(-np.sinh(tellmodel_hires)) + # FD: currently assumes shift + stretch is on - tellmodel_conv = conv_telluric(tellmodel_hires[ind_lower_pad:ind_upper_pad+1], - tell_dict['dloglam'], theta_tell[-3]) + tellmodel_conv = conv_telluric(tellmodel_hires,tell_dict['dloglam'], theta_tell[-3]) tellmodel_out = shift_telluric(tellmodel_conv, np.log10(tell_dict['wave_grid'][ind_lower_pad:ind_upper_pad+1]), tell_dict['dloglam'], theta_tell[-2], theta_tell[-1]) - return tellmodel_out[ind_lower_final:ind_upper_final] + return tellmodel_out[tell_dict['tell_pad_pix']:-tell_dict['tell_pad_pix']+1] ############################ @@ -410,8 +399,8 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): This is actually two concatenated paramter vectors, one for the object and one for the telluric, i.e:: - theta_obj = theta[:-7] - theta_tell = theta[-7:] + theta_obj = theta[:-(ntell+3)] + theta_tell = theta[-(ntell+3):] The telluric model theta_tell includes a user-specified number of PCA coefficients, spectral resolution, shift, and stretch. @@ -432,8 +421,8 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): - ``arg_dict['flux_ivar']``: Inverse variance for the flux array - ``arg_dict['tell_dict']``: Dictionary containing the - telluric grid and its parameters read in by - read_telluric_grid + telluric PCA model and its parameters read in by + read_telluric_pca - ``arg_dict['ind_lower']``: Lower index into the telluric model wave_grid to trim down the telluric model. @@ -481,7 +470,7 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): popsize = arg_dict['popsize'] # Note this does nothing if the init is done from a previous iteration or optimum nsamples = arg_dict['popsize']*nparams # FD: Currently assumes shift and stretch are turned on. - ntheta_tell = arg_dict['ntell']+3 + ntheta_tell = arg_dict['ntell']+3 # Total number of telluric model parameters # Decide how to initialize if init_from_last is not None: # Use a Gaussian ball about the optimum from a previous iteration @@ -528,7 +517,7 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): # TODO This should be a general reader once we get our act together with the data model. # For echelle: read in all the orders into a (nspec, nporders) array -# FOr longslit: read in the stanard into a (nspec, 1) array +# For longslit: read in the standard into a (nspec, 1) array def unpack_orders(sobjs, ret_flam=False): """ Utility function to unpack the sobjs object and return the @@ -845,8 +834,8 @@ def init_qso_model(obj_params, iord, wave, flux, ivar, mask, tellmodel): tellmodel : array shape (nspec,) This is a telluric model computed on the wave wavelength grid. Initialization usually requires some initial - best guess for the telluric absorption, which is computed from the midpoint of the telluric model grid parameter - space using the resolution of the spectrograph and the airmass of the observations. + best guess for the telluric absorption, which is computed from the mean of the telluric model grid using + the resolution of the spectrograph. (TODO: revisit this for PCA) Returns ------- @@ -862,17 +851,17 @@ def init_qso_model(obj_params, iord, wave, flux, ivar, mask, tellmodel): """ - pca_dict = init_qso_pca(obj_params['pca_file'], wave, obj_params['z_qso'], obj_params['npca']) - pca_mean = np.exp(pca_dict['interp'](pca_dict['wave_grid']*(1+pca_dict['z_fid']))[0, :]) + qso_pca_dict = init_qso_pca(obj_params['pca_file'], wave, obj_params['z_qso'], obj_params['npca']) + qso_pca_mean = np.exp(qso_pca_dict['interp'](qso_pca_dict['wave_grid']*(1+qso_pca_dict['z_fid']))[0, :]) tell_mask = tellmodel > obj_params['tell_norm_thresh'] # Create a reference model and bogus noise - flux_ref = pca_mean * tellmodel - ivar_ref = utils.inverse((pca_mean/100.0) ** 2) + flux_ref = qso_pca_mean * tellmodel + ivar_ref = utils.inverse((qso_pca_mean/100.0) ** 2) flam_norm_inv = coadd.robust_median_ratio(flux, ivar, flux_ref, ivar_ref, mask=mask, mask_ref=tell_mask) flam_norm = 1.0/flam_norm_inv # Set the bounds for the PCA and truncate to the right dimension - coeffs = pca_dict['coeffs'][:,1:obj_params['npca']] + coeffs = qso_pca_dict['coeffs'][:,1:obj_params['npca']] # Compute the min and max arrays of the coefficients which are not the norm, i.e. grab the coeffs that aren't the first one coeff_min = np.amin(coeffs, axis=0) # only coeff_max = np.amax(coeffs, axis=0) @@ -882,7 +871,7 @@ def init_qso_model(obj_params, iord, wave, flux, ivar, mask, tellmodel): bounds_pca = [(i, j) for i, j in zip(coeff_min, coeff_max)] # Coefficients: determined from PCA model bounds_obj = bounds_z + bounds_flam + bounds_pca # Create the obj_dict - obj_dict = dict(npca=obj_params['npca'], pca_dict=pca_dict) + obj_dict = dict(npca=obj_params['npca'], pca_dict=qso_pca_dict) return obj_dict, bounds_obj From 4b6980bbe358193ce6a2d0b5ccd62c19b180708f Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Fri, 21 Jul 2023 11:16:08 +0200 Subject: [PATCH 021/244] fixed wavelength truncation --- pypeit/core/telluric.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 634b511639..611f70084c 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -320,18 +320,20 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): ind_upper_final = ind_upper_pad if ind_upper_pad == ind_upper else ind_upper - ind_upper_pad # Evaluate PCA model after truncating the wavelength range - tellmodel_hires = np.dot(np.append(1,theta_tell[:ncomp_use][ind_lower_pad:ind_upper_pad+1]), - tell_dict['tell_pca'][:ncomp_use+1]) + tellmodel_hires = np.zeros_like(tell_dict['tell_pca'][0]) + tellmodel_hires[ind_lower_pad:ind_upper_pad+1] = np.dot(np.append(1,theta_tell[:ncomp_use]), + tell_dict['tell_pca'][:ncomp_use+1][:,ind_lower_pad:ind_upper_pad+1]) # Transform Arsinh PCA model, works slightly better than transmission components but may be slightly slower? tellmodel_hires = np.exp(-np.sinh(tellmodel_hires)) # FD: currently assumes shift + stretch is on - tellmodel_conv = conv_telluric(tellmodel_hires,tell_dict['dloglam'], theta_tell[-3]) - + tellmodel_conv = conv_telluric(tellmodel_hires[ind_lower_pad:ind_upper_pad+1], + tell_dict['dloglam'], theta_tell[-3]) + embed() tellmodel_out = shift_telluric(tellmodel_conv, np.log10(tell_dict['wave_grid'][ind_lower_pad:ind_upper_pad+1]), tell_dict['dloglam'], theta_tell[-2], theta_tell[-1]) - return tellmodel_out[tell_dict['tell_pad_pix']:-tell_dict['tell_pad_pix']+1] + return tellmodel_out[ind_lower_final:ind_upper_final] ############################ From 3c8457dfa583c54b40ddbaac1a223932e194bbb8 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Fri, 21 Jul 2023 11:16:19 +0200 Subject: [PATCH 022/244] fixed wavelength truncation --- pypeit/core/telluric.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 611f70084c..f73376a15d 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -329,7 +329,7 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): # FD: currently assumes shift + stretch is on tellmodel_conv = conv_telluric(tellmodel_hires[ind_lower_pad:ind_upper_pad+1], tell_dict['dloglam'], theta_tell[-3]) - embed() + tellmodel_out = shift_telluric(tellmodel_conv, np.log10(tell_dict['wave_grid'][ind_lower_pad:ind_upper_pad+1]), tell_dict['dloglam'], theta_tell[-2], theta_tell[-1]) From 2bda22ac47ad28a50a2a23657e67ce1a36ffa06e Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Wed, 26 Jul 2023 11:27:46 +0200 Subject: [PATCH 023/244] playing around with convex hull prior --- pypeit/core/telluric.py | 46 ++++++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index f73376a15d..25d0f378b3 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -113,7 +113,7 @@ def qso_pca_eval(theta,qso_pca_dict): return norm*np.exp(np.dot(np.append(1.0,A),C_now)) # TODO The prior is not currently used, but this is left in here anyway. -#def pca_lnprior(theta,pca_dict): +#def qso_pca_lnprior(theta,qso_pca_dict): # """ # Routine to evaluate the the ln of the prior probability lnPrior, # from the Gaussian mixture model fit to the distriution of PCA @@ -125,9 +125,9 @@ def qso_pca_eval(theta,qso_pca_dict): # ``theta_pca[1]`` is the normalization, and # ``theta_pca[2:npca+1]`` are the PCA coefficients, where npca # is the PCA dimensionality -# pca_dict (dict): +# qso_pca_dict (dict): # Dictionary continaing the PCA information generated by -# ``init_pca`` +# ``init_qso__pca`` # # # Returns: @@ -136,7 +136,7 @@ def qso_pca_eval(theta,qso_pca_dict): # theta_pca[2:npca+1]`` # # """ -# gaussian_mixture_model = pca_dict['prior'] +# gaussian_mixture_model = qso_pca_dict['prior'] # A = theta[2:] # return gaussian_mixture_model.score_samples(A.reshape(1,-1)) @@ -177,6 +177,7 @@ def read_telluric_pca(filename, wave_min=None, wave_max=None, pad_frac=0.10): pca_comp_full = hdul[0].data ncomp = hdul[0].header['NCOMP'] bounds = hdul[2].data + model_coefs = hdul[3].data ind_lower = np.argmin(np.abs(wave_grid_full - (1.0 - pad_frac)*wave_min)) \ if wave_min is not None else 0 @@ -190,7 +191,7 @@ def read_telluric_pca(filename, wave_min=None, wave_max=None, pad_frac=0.10): return dict(wave_grid=wave_grid, dloglam=dloglam, resln_guess=resln_guess, pix_per_sigma=pix_per_sigma, tell_pad_pix=tell_pad_pix, ncomp_tell_pca=ncomp, - tell_pca=pca_comp_grid, bounds_tell_pca=bounds) + tell_pca=pca_comp_grid, bounds_tell_pca=bounds, coefs_tell_pca=model_coefs) def conv_telluric(tell_model, dloglam, res): """ @@ -335,7 +336,21 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): tell_dict['dloglam'], theta_tell[-2], theta_tell[-1]) return tellmodel_out[ind_lower_final:ind_upper_final] - +def in_hull(points, x): + """ + Checks whether the N-D point 'x' can be expressed as a convex combination + of 'points', equivalent to checking whether 'x' lies within the convex hull + defined by the outer subset of 'points' but much faster. + See: https://stackoverflow.com/a/43564754 + """ + n_points = len(points) + n_dim = len(x) + c = np.zeros(n_points) + A = np.r_[points.T,np.ones((1,n_points))] + b = np.r_[x, np.ones(1)] + lp = scipy.optimize.linprog(c, A_eq=A, b_eq=b) + return lp.success + ############################ # Fitting routines # ############################ @@ -362,6 +377,7 @@ def tellfit_chi2(theta, flux, thismask, arg_dict): that is minimized to perform the fit. """ + obj_model_func = arg_dict['obj_model_func'] flux_ivar = arg_dict['ivar'] @@ -372,6 +388,12 @@ def tellfit_chi2(theta, flux, thismask, arg_dict): theta_obj = theta[:-nfit] theta_tell = theta[-nfit:] + + # If the PCA coefficients lie outside the convex hull of the telluric models, + # it is probably unphysical, so return infinity. +# if not in_hull(arg_dict['tell_dict']['coefs_tell_pca'][:,:(nfit-3)],theta_tell[:-3]): +# return np.inf +# tell_model = eval_telluric(theta_tell, arg_dict['tell_dict'], ind_lower=arg_dict['ind_lower'], ind_upper=arg_dict['ind_upper']) obj_model, model_gpm = obj_model_func(theta_obj, arg_dict['obj_dict']) @@ -384,6 +406,10 @@ def tellfit_chi2(theta, flux, thismask, arg_dict): robust_scale = 2.0 huber_vec = scipy.special.huber(robust_scale, chi_vec) loss_function = np.sum(huber_vec * totalmask) + # If the PCA coefficients lie outside the convex hull of the telluric models, + # it should be disfavored, so penalize the loss function severely + #if not in_hull(arg_dict['tell_dict']['coefs_tell_pca'][:,:(nfit-3)],theta_tell[:-3]): + # loss_function += 1e6 return loss_function def tellfit(flux, thismask, arg_dict, init_from_last=None): @@ -498,6 +524,12 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): init = init, updating='immediate', popsize=popsize, recombination=arg_dict['recombination'], maxiter=arg_dict['diff_evol_maxiter'], polish=arg_dict['polish'], disp=arg_dict['disp']) + + # Let the user know if the best-fit telluric coefficients lie outside the model space, + # probably should not trust the model in that case! + if not in_hull(arg_dict['tell_dict']['coefs_tell_pca'][:,:(ntheta_tell-3)],result.x[-ntheta_tell:-3]): + msgs.warn('Best fit telluric model lies in potentially unphysical parameter space. Use with caution!') + theta_obj = result.x[:-ntheta_tell] theta_tell = result.x[-ntheta_tell:] tell_model = eval_telluric(theta_tell, arg_dict['tell_dict'], @@ -2149,7 +2181,7 @@ def empty_model_table(norders, nspec, ntell=4, n_obj_par=0): def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_model, eval_obj_model, log10_blaze_function=None, ech_orders=None, sn_clip=30.0, ntell=6, airmass_guess=1.5, - resln_guess=None, resln_frac_bounds=(0.5, 1.5), pix_shift_bounds=(-5.0, 5.0), + resln_guess=None, resln_frac_bounds=(0.5, 2.0), pix_shift_bounds=(-5.0, 5.0), pix_stretch_bounds=(0.9,1.1), maxiter=2, sticky=True, lower=3.0, upper=3.0, seed=777, ballsize = 5e-4, tol=1e-3, diff_evol_maxiter=1000, popsize=30, recombination=0.7, polish=True, disp=False, sensfunc=False, debug=False): From ced2a44adca2c50ce65c128465d85f9fe07e1e51 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Fri, 28 Jul 2023 14:00:09 +0200 Subject: [PATCH 024/244] merging --- pypeit/sensfunc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pypeit/sensfunc.py b/pypeit/sensfunc.py index e793fd8eae..2009fe3764 100644 --- a/pypeit/sensfunc.py +++ b/pypeit/sensfunc.py @@ -216,7 +216,6 @@ def __init__(self, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False): # spectrograph objects with configuration specific information from # spec1d files. self.spectrograph.dispname = header['DISPNAME'] - self.par_fluxcalib = self.spectrograph.default_pypeit_par()['fluxcalib'] if par_fluxcalib is None else par_fluxcalib # Set the algorithm in the datamodel From 5bf963d9e723b278c3c14841a01fd8922344acad Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Thu, 3 Aug 2023 15:33:15 +0200 Subject: [PATCH 025/244] changing default ntell to 4, works well in testing and should be faster --- pypeit/core/telluric.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 4d2a3165dd..c8ff87d3cd 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -1180,7 +1180,7 @@ def eval_poly_model(theta, obj_dict): def sensfunc_telluric(wave, counts, counts_ivar, counts_mask, exptime, airmass, std_dict, - telgridfile, log10_blaze_function=None, ech_orders=None, polyorder=8, ntell=6, mask_hydrogen_lines=True, + telgridfile, log10_blaze_function=None, ech_orders=None, polyorder=8, ntell=4, mask_hydrogen_lines=True, mask_helium_lines=False, hydrogen_mask_wid=10., resln_guess=None, resln_frac_bounds=(0.5, 1.5), delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), sn_clip=30.0, ballsize=5e-4, only_orders=None, maxiter=3, lower=3.0, @@ -1555,7 +1555,7 @@ def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile, def star_telluric(spec1dfile, telgridfile, telloutfile, outfile, star_type=None, star_mag=None, star_ra=None, star_dec=None, func='legendre', model='exp', polyorder=5, - ntell=6, mask_hydrogen_lines=True, mask_helium_lines=False, hydrogen_mask_wid=10., + ntell=4, mask_hydrogen_lines=True, mask_helium_lines=False, hydrogen_mask_wid=10., delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), only_orders=None, sn_clip=30.0, maxiter=3, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, @@ -2182,7 +2182,7 @@ def empty_model_table(norders, nspec, ntell=4, n_obj_par=0): description='Maximum wavelength included in the fit')]) def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_model, - eval_obj_model, log10_blaze_function=None, ech_orders=None, sn_clip=30.0, ntell=6, airmass_guess=1.5, + eval_obj_model, log10_blaze_function=None, ech_orders=None, sn_clip=30.0, ntell=4, airmass_guess=1.5, resln_guess=None, resln_frac_bounds=(0.5, 2.0), pix_shift_bounds=(-5.0, 5.0), pix_stretch_bounds=(0.9,1.1), maxiter=2, sticky=True, lower=3.0, upper=3.0, seed=777, ballsize = 5e-4, tol=1e-3, diff_evol_maxiter=1000, popsize=30, From 732ce81500a630fd496d604dcf827e7f68028c9f Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Fri, 4 Aug 2023 10:19:49 +0200 Subject: [PATCH 026/244] bringing back the old telluric grid functionality as an alternative --- pypeit/core/telluric.py | 255 +++++++++++++++++++++++++++++++++------- pypeit/par/pypeitpar.py | 9 +- 2 files changed, 222 insertions(+), 42 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index c8ff87d3cd..1b1e40639b 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -192,6 +192,118 @@ def read_telluric_pca(filename, wave_min=None, wave_max=None, pad_frac=0.10): return dict(wave_grid=wave_grid, dloglam=dloglam, resln_guess=resln_guess, pix_per_sigma=pix_per_sigma, tell_pad_pix=tell_pad_pix, ncomp_tell_pca=ncomp, tell_pca=pca_comp_grid, bounds_tell_pca=bounds, coefs_tell_pca=model_coefs) + +def read_telluric_grid(filename, wave_min=None, wave_max=None, pad_frac=0.10): + """ + Reads in the telluric grid from a file. + + Optionally, this method also trims the grid to be in within ``wave_min`` + and ``wave_max`` and pads the data (see ``pad_frac``). + + .. todo:: + List and describe the contents of the dictionary in the return + description. + + Args: + filename (:obj:`str`): + Telluric grid filename + wave_min (:obj:`float`): + Minimum wavelength at which the grid is desired + wave_max (:obj:`float`): + Maximum wavelength at which the grid is desired. + pad_frac (:obj:`float`): + Percentage padding to be added to the grid boundaries if + ``wave_min`` or ``wave_max`` are input; ignored otherwise. The + resulting grid will extend from ``(1.0 - pad_frac)*wave_min`` to + ``(1.0 + pad_frac)*wave_max``. + + Returns: + :obj:`dict`: Dictionary containing the telluric grid + - wave_grid= + - dloglam= + - resln_guess= + - pix_per_sigma= + - tell_pad_pix= + - pressure_grid= + - temp_grid= + - h2o_grid= + - airmass_grid= + - tell_grid= + """ + # load_telluric_grid() takes care of path and existance check + hdul = data.load_telluric_grid(filename) + wave_grid_full = 10.0*hdul[1].data + model_grid_full = hdul[0].data + nspec_full = wave_grid_full.size + + ind_lower = np.argmin(np.abs(wave_grid_full - (1.0 - pad_frac)*wave_min)) \ + if wave_min is not None else 0 + ind_upper = np.argmin(np.abs(wave_grid_full - (1.0 + pad_frac)*wave_max)) \ + if wave_max is not None else nspec_full + wave_grid = wave_grid_full[ind_lower:ind_upper] + model_grid = model_grid_full[...,ind_lower:ind_upper] + + pg = hdul[0].header['PRES0']+hdul[0].header['DPRES']*np.arange(0,hdul[0].header['NPRES']) + tg = hdul[0].header['TEMP0']+hdul[0].header['DTEMP']*np.arange(0,hdul[0].header['NTEMP']) + hg = hdul[0].header['HUM0']+hdul[0].header['DHUM']*np.arange(0,hdul[0].header['NHUM']) + if hdul[0].header['NAM'] > 1: + ag = hdul[0].header['AM0']+hdul[0].header['DAM']*np.arange(0,hdul[0].header['NAM']) + else: + ag = hdul[0].header['AM0']+1*np.arange(0,1) + + dwave, dloglam, resln_guess, pix_per_sigma = wvutils.get_sampling(wave_grid) + tell_pad_pix = int(np.ceil(10.0 * pix_per_sigma)) + + + return dict(wave_grid=wave_grid, + dloglam=dloglam, + resln_guess=resln_guess, + pix_per_sigma=pix_per_sigma, + tell_pad_pix=tell_pad_pix, + pressure_grid=pg, + temp_grid=tg, + h2o_grid=hg, + airmass_grid=ag, + tell_grid=model_grid) + + +def interp_telluric_grid(theta, tell_dict): + """ + Interpolate the telluric model grid to the specified location in + parameter space. + + The interpolation is only performed over the 4D parameter space specified + by pressure, temperature, humidity, and airmass. This routine performs + nearest-gridpoint interpolation to evaluate the telluric model at an + arbitrary location in this 4-d space. The telluric grid is assumed to be + uniformly sampled in this parameter space. + + Args: + theta (`numpy.ndarray`_): + A 4-element vector with the telluric model parameters: pressure, + temperature, humidity, and airmass. + tell_dict (dict): + Dictionary containing the telluric grid. See + :func:`read_telluric_grid`. + + Returns: + `numpy.ndarray`_: Telluric model evaluated at the provided 4D + position in parameter space. The telluric model is provided over all + available wavelengths in ``tell_dict``. + """ + if len(theta) != 4: + msgs.error('Input parameter vector must have 4 and only 4 values.') + pg = tell_dict['pressure_grid'] + tg = tell_dict['temp_grid'] + hg = tell_dict['h2o_grid'] + ag = tell_dict['airmass_grid'] + p, t, h, a = theta + pi = int(np.round((p-pg[0])/(pg[1]-pg[0]))) if len(pg) > 1 else 0 + ti = int(np.round((t-tg[0])/(tg[1]-tg[0]))) if len(tg) > 1 else 0 + hi = int(np.round((h-hg[0])/(hg[1]-hg[0]))) if len(hg) > 1 else 0 + ai = int(np.round((a-ag[0])/(ag[1]-ag[0]))) if len(ag) > 1 else 0 + return tell_dict['tell_grid'][pi,ti,hi,ai] + def conv_telluric(tell_model, dloglam, res): """ @@ -200,8 +312,8 @@ def conv_telluric(tell_model, dloglam, res): Args: tell_model (`numpy.ndarray`_): Input telluric model at the native resolution of the telluric model grid. The shape of this input is in - general different from the size of the telluric grid (read in by read_telluric_grid above) because it is - trimmed to relevant wavelenghts using ind_lower, ind_upper. See eval_telluric below. + general different from the size of the raw telluric model (read in by read_telluric_* above) because it is + trimmed to relevant wavelengths using ind_lower, ind_upper. See eval_telluric_* below. dloglam (float): Wavelength spacing of the telluric grid expressed as a dlog10(lambda), i.e. stored in the tell_dict as tell_dict['dloglam'] @@ -262,7 +374,7 @@ def shift_telluric(tell_model, loglam, dloglam, shift, stretch): return tell_model_shift -def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): +def eval_telluric_pca(theta_tell, tell_dict, ind_lower=None, ind_upper=None): """ Evaluate the telluric PCA model. @@ -336,21 +448,77 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): tell_dict['dloglam'], theta_tell[-2], theta_tell[-1]) return tellmodel_out[ind_lower_final:ind_upper_final] -def in_hull(points, x): +def eval_telluric_grid(theta_tell, tell_dict, ind_lower=None, ind_upper=None): """ - Checks whether the N-D point 'x' can be expressed as a convex combination - of 'points', equivalent to checking whether 'x' lies within the convex hull - defined by the outer subset of 'points' but much faster. - See: https://stackoverflow.com/a/43564754 - """ - n_points = len(points) - n_dim = len(x) - c = np.zeros(n_points) - A = np.r_[points.T,np.ones((1,n_points))] - b = np.r_[x, np.ones(1)] - lp = scipy.optimize.linprog(c, A_eq=A, b_eq=b) - return lp.success + Evaluate the telluric model at an arbitrary location in parameter space. + The full atmosphere model parameter space is either 5 or 7 dimensional, + which is the size of the ``theta_tell`` input parameter vector. + + The parameters provided by ``theta_tell`` must be: pressure, temperature, + humidity, airmass, spectral resolution, shift, and stretch. The latter two + can be omitted if a shift and stretch of the telluric model are not + included. + + This routine performs the following steps: + + 1. nearest grid point interpolation of the telluric model onto a + new location in the 4-d space (pressure, temperature, + humidity, airmass) + + 2. convolution of the atmosphere model to the resolution set by + the spectral resolution. + + 3. (Optional) shift and stretch the telluric model. + + Args: + theta_tell (`numpy.ndarray`_): + Vector with the telluric model parameters. Must be 5 or 7 + elements long. See method description. + tell_dict (:obj:`dict`): + Dictionary containing the telluric grid data. See + :func:`read_telluric_grid`. + ind_lower (:obj:`int`, optional): + The index of the first pixel to include in the model. Selecting a + wavelength region for the modeling makes things faster because we + only need to convolve the portion that is needed for the current + model fit. + ind_upper: + The index (inclusive) of the last pixel to include in the model. + Selecting a wavelength region for the modeling makes things + faster because we only need to convolve the portion that is + needed for the current model fit. + + Returns: + `numpy.ndarray`_: Telluric model evaluated at the desired location + theta_tell in model atmosphere parameter space. + """ + ntheta = len(theta_tell) + if ntheta not in [5, 7]: + msgs.error('Input model atmosphere parameters must have length 5 or 7.') + + tellmodel_hires = interp_telluric_grid(theta_tell[:4], tell_dict) + + # Set the wavelength range if not provided + ind_lower = 0 if ind_lower is None else ind_lower + ind_upper = tell_dict['wave_grid'].size - 1 if ind_upper is None else ind_upper + + # Deal with padding for the convolutions + ind_lower_pad = np.fmax(ind_lower - tell_dict['tell_pad_pix'], 0) + ind_upper_pad = np.fmin(ind_upper + tell_dict['tell_pad_pix'], tell_dict['wave_grid'].size - 1) + ## FW: There is an extreme case with ind_upper == ind_upper_pad, the previous -0 won't work + ind_lower_final = ind_lower_pad if ind_lower_pad == ind_lower else ind_lower - ind_lower_pad + ind_upper_final = ind_upper_pad if ind_upper_pad == ind_upper else ind_upper - ind_upper_pad + tellmodel_conv = conv_telluric(tellmodel_hires[ind_lower_pad:ind_upper_pad+1], + tell_dict['dloglam'], theta_tell[4]) + if ntheta == 5: + return tellmodel_conv[ind_lower_final:ind_upper_final] + + tellmodel_out = shift_telluric(tellmodel_conv, + np.log10(tell_dict['wave_grid'][ind_lower_pad:ind_upper_pad+1]), + tell_dict['dloglam'], theta_tell[5], theta_tell[6]) + return tellmodel_out[ind_lower_final:ind_upper_final] + ############################ # Fitting routines # ############################ @@ -380,22 +548,18 @@ def tellfit_chi2(theta, flux, thismask, arg_dict): obj_model_func = arg_dict['obj_model_func'] + tell_model_func = arg_dict['tell_model_func'] flux_ivar = arg_dict['ivar'] - ncomp_use = arg_dict['ntell'] # TODO: make this work without shift and stretch turned on + # Number of telluric model parameters, plus shift, stretch, and resolution nfit = arg_dict['ntell']+3 theta_obj = theta[:-nfit] theta_tell = theta[-nfit:] - - # If the PCA coefficients lie outside the convex hull of the telluric models, - # it is probably unphysical, so return infinity. -# if not in_hull(arg_dict['tell_dict']['coefs_tell_pca'][:,:(nfit-3)],theta_tell[:-3]): -# return np.inf -# - tell_model = eval_telluric(theta_tell, arg_dict['tell_dict'], - ind_lower=arg_dict['ind_lower'], ind_upper=arg_dict['ind_upper']) + + tell_model = tell_model_func(theta_tell, arg_dict['tell_dict'], + ind_lower=arg_dict['ind_lower'], ind_upper=arg_dict['ind_upper']) obj_model, model_gpm = obj_model_func(theta_obj, arg_dict['obj_dict']) totalmask = thismask & model_gpm @@ -457,6 +621,8 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): - ``arg_dict['ind_upper']``: Upper index into the telluric model wave_grid to trim down the telluric model. + - ``arg_dict['tell_model_func']``: Function for + evaluating the telluric model, i.e. PCA or grid - ``arg_dict['obj_model_func']``: User provided function for evaluating the object model - ``arg_dict['obj_dict']``: Dictionary containing the @@ -489,6 +655,7 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): # Unpack arguments obj_model_func = arg_dict['obj_model_func'] # Evaluation function + tell_model_func = arg_dict['tell_model_func'] # Telluric model, PCA or Grid flux_ivar = arg_dict['ivar'] # Inverse variance of flux or counts bounds = arg_dict['bounds'] # bounds for differential evolution optimization rng = arg_dict['rng'] # Seed for differential evolution optimizaton @@ -524,16 +691,11 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): init = init, updating='immediate', popsize=popsize, recombination=arg_dict['recombination'], maxiter=arg_dict['diff_evol_maxiter'], polish=arg_dict['polish'], disp=arg_dict['disp']) - - # Let the user know if the best-fit telluric coefficients lie outside the model space, - # probably should not trust the model in that case! - if not in_hull(arg_dict['tell_dict']['coefs_tell_pca'][:,:(ntheta_tell-3)],result.x[-ntheta_tell:-3]): - msgs.warn('Best fit telluric model lies in potentially unphysical parameter space. Use with caution!') - + theta_obj = result.x[:-ntheta_tell] theta_tell = result.x[-ntheta_tell:] - tell_model = eval_telluric(theta_tell, arg_dict['tell_dict'], - ind_lower=arg_dict['ind_lower'], ind_upper=arg_dict['ind_upper']) + tell_model = tell_model_func(theta_tell, arg_dict['tell_dict'], + ind_lower=arg_dict['ind_lower'], ind_upper=arg_dict['ind_upper']) obj_model, modelmask = obj_model_func(theta_obj, arg_dict['obj_dict']) totalmask = thismask & modelmask chi_vec = totalmask*(flux - tell_model*obj_model)*np.sqrt(flux_ivar) @@ -2025,7 +2187,10 @@ class Telluric(datamodel.DataContainer): """Datamodel version.""" datamodel = {'telgrid': dict(otype=str, - descr='File containing grid of HITRAN atmosphere models'), + descr='File containing PCA components or grid of ' + 'HITRAN atmosphere models'), + 'teltype': dict(otype=str, + descr='Type of telluric model, pca or grid'), 'ntell': dict(otype=int, descr='Number of telluric PCA components used'), 'std_src': dict(otype=str, descr='Name of the standard source'), @@ -2181,7 +2346,7 @@ def empty_model_table(norders, nspec, ntell=4, n_obj_par=0): table.Column(name='WAVE_MAX', dtype=float, length=norders, description='Maximum wavelength included in the fit')]) - def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_model, + def __init__(self, wave, flux, ivar, gpm, telgridfile, teltype, obj_params, init_obj_model, eval_obj_model, log10_blaze_function=None, ech_orders=None, sn_clip=30.0, ntell=4, airmass_guess=1.5, resln_guess=None, resln_frac_bounds=(0.5, 2.0), pix_shift_bounds=(-5.0, 5.0), pix_stretch_bounds=(0.9,1.1), maxiter=2, sticky=True, lower=3.0, upper=3.0, @@ -2201,6 +2366,7 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode # 1) Assign arguments self.telgrid = telgridfile + self.teltype = teltype self.obj_params = obj_params self.init_obj_model = init_obj_model self.ntell = ntell @@ -2241,8 +2407,14 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode wave, flux, ivar, gpm) # 3) Read the telluric grid and initalize associated parameters wv_gpm = self.wave_in_arr > 1.0 - self.tell_dict = read_telluric_pca(self.telgrid, wave_min=self.wave_in_arr[wv_gpm].min(), - wave_max=self.wave_in_arr[wv_gpm].max()) + if teltype == 'pca' or teltype == 'PCA': + self.tell_dict = read_telluric_pca(self.telgrid, wave_min=self.wave_in_arr[wv_gpm].min(), + wave_max=self.wave_in_arr[wv_gpm].max()) + tell_model_func = eval_telluric_pca + elif teltype == 'grid': + self.tell_dict = read_telluric_grid(self.telgrid, wave_min=self.wave_in_arr[wv_gpm].min(), + wave_max=self.wave_in_arr[wv_gpm].max()) + tell_model_func = eval_telluric_grid self.wave_grid = self.tell_dict['wave_grid'] self.ngrid = self.wave_grid.size self.resln_guess = wvutils.get_sampling(self.wave_in_arr)[2] \ @@ -2290,9 +2462,9 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode for counter, iord in enumerate(self.srt_order_tell): msgs.info(f'Initializing object model for order: {iord}, {counter}/{self.norders}' + f' with user supplied function: {self.init_obj_model.__name__}') - tellmodel = eval_telluric(self.tell_guess, self.tell_dict, - ind_lower=self.ind_lower[iord], - ind_upper=self.ind_upper[iord]) + tellmodel = tell_model_func(self.tell_guess, self.tell_dict, + ind_lower=self.ind_lower[iord], + ind_upper=self.ind_upper[iord]) # TODO This is a pretty ugly way to pass in the blaze function. Particularly since now all the other models # (star, qso, poly) are going to have this parameter set in their obj_params dictionary. # Is there something more elegant that can be done with e.g. functools.partial? @@ -2313,7 +2485,8 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode self.bounds_list[iord] = bounds_iord arg_dict_iord = dict(ivar=self.ivar_arr[self.ind_lower[iord]:self.ind_upper[iord]+1,iord], tell_dict=self.tell_dict, ind_lower=self.ind_lower[iord], - ind_upper=self.ind_upper[iord], ntell=self.ntell, + ind_upper=self.ind_upper[iord], + ntell=self.ntell, tell_model_func=tell_model_func, obj_model_func=self.eval_obj_model, obj_dict=obj_dict, ballsize=self.ballsize, bounds=bounds_iord, rng=self.rng, diff_evol_maxiter=self.diff_evol_maxiter, tol=self.tol, diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index fef2c7d7c1..ee27c45960 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -2079,7 +2079,7 @@ class TelluricPar(ParSet): """ def __init__(self, telgridfile=None, sn_clip=None, resln_guess=None, resln_frac_bounds=None, pix_shift_bounds=None, - delta_coeff_bounds=None, minmax_coeff_bounds=None, maxiter=None, ntell=None, + delta_coeff_bounds=None, minmax_coeff_bounds=None, maxiter=None, ntell=None, teltype=None, sticky=None, lower=None, upper=None, seed=None, tol=None, popsize=None, recombination=None, polish=None, disp=None, objmodel=None, redshift=None, delta_redshift=None, pca_file=None, npca=None, bal_wv_min_max=None, bounds_norm=None, tell_norm_thresh=None, only_orders=None, pca_lower=None, @@ -2108,6 +2108,13 @@ def __init__(self, telgridfile=None, sn_clip=None, resln_guess=None, resln_frac_ defaults['ntell'] = 4 dtypes['ntell'] = int descr['ntell'] = 'Number of telluric PCA components to use in the fitting' + + defaults['teltype'] = 'PCA' + dtypes['teltype'] = str + descr['teltype'] = 'Method used to evaluate telluric models, either PCA or grid. The grid option uses a ' \ + 'fixed grid of pre-computed HITRAN+LBLRTM atmospheric transmission models for each ' \ + 'observatory, whereas the PCA option uses principal components of a larger model grid ' \ + 'to compute a pseudo-telluric model with a much lighter telgridfile.' defaults['sn_clip'] = 30.0 dtypes['sn_clip'] = [int, float] From 0846fa28d0dbc1104baec2591fcecc388b223940 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Fri, 4 Aug 2023 10:47:20 +0200 Subject: [PATCH 027/244] making stuff work in the dev suite --- pypeit/core/telluric.py | 5 +++-- pypeit/par/pypeitpar.py | 8 +++++--- pypeit/sensfunc.py | 1 + 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 1b1e40639b..6ccea68b30 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -1342,7 +1342,8 @@ def eval_poly_model(theta, obj_dict): def sensfunc_telluric(wave, counts, counts_ivar, counts_mask, exptime, airmass, std_dict, - telgridfile, log10_blaze_function=None, ech_orders=None, polyorder=8, ntell=4, mask_hydrogen_lines=True, + telgridfile, teltype, log10_blaze_function=None, ech_orders=None, polyorder=8, ntell=4, + mask_hydrogen_lines=True, mask_helium_lines=False, hydrogen_mask_wid=10., resln_guess=None, resln_frac_bounds=(0.5, 1.5), delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), sn_clip=30.0, ballsize=5e-4, only_orders=None, maxiter=3, lower=3.0, @@ -1506,7 +1507,7 @@ def sensfunc_telluric(wave, counts, counts_ivar, counts_mask, exptime, airmass, mask_tot = mask_bad & mask_recomb & mask_tell # Since we are fitting a sensitivity function, first compute counts per second per angstrom. - TelObj = Telluric(wave, counts, counts_ivar, mask_tot, telgridfile, obj_params, + TelObj = Telluric(wave, counts, counts_ivar, mask_tot, telgridfile, teltype, obj_params, init_sensfunc_model, eval_sensfunc_model, log10_blaze_function=log10_blaze_function, ntell=ntell, ech_orders=ech_orders, resln_guess=resln_guess, resln_frac_bounds=resln_frac_bounds, sn_clip=sn_clip, maxiter=maxiter, lower=lower, upper=upper, tol=tol, diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index ee27c45960..ae0ec41397 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -2107,14 +2107,16 @@ def __init__(self, telgridfile=None, sn_clip=None, resln_guess=None, resln_frac_ defaults['ntell'] = 4 dtypes['ntell'] = int - descr['ntell'] = 'Number of telluric PCA components to use in the fitting' + descr['ntell'] = 'Number of fitted telluric model parameters. Must be set to 4 (default) for teltype = grid, ' \ + 'but can be set to any number from 1 to 15 for teltype = PCA, corresponding to the number of ' \ + 'fitted PCA coefficients.' defaults['teltype'] = 'PCA' dtypes['teltype'] = str descr['teltype'] = 'Method used to evaluate telluric models, either PCA or grid. The grid option uses a ' \ 'fixed grid of pre-computed HITRAN+LBLRTM atmospheric transmission models for each ' \ 'observatory, whereas the PCA option uses principal components of a larger model grid ' \ - 'to compute a pseudo-telluric model with a much lighter telgridfile.' + 'to compute an accurate pseudo-telluric model with a much lighter telgridfile.' defaults['sn_clip'] = 30.0 dtypes['sn_clip'] = [int, float] @@ -2343,7 +2345,7 @@ def __init__(self, telgridfile=None, sn_clip=None, resln_guess=None, resln_frac_ @classmethod def from_dict(cls, cfg): k = np.array([*cfg.keys()]) - parkeys = ['telgridfile', 'sn_clip', 'resln_guess', 'resln_frac_bounds', 'ntell', + parkeys = ['telgridfile', 'teltype', 'sn_clip', 'resln_guess', 'resln_frac_bounds', 'ntell', 'pix_shift_bounds', 'delta_coeff_bounds', 'minmax_coeff_bounds', 'maxiter', 'sticky', 'lower', 'upper', 'seed', 'tol', 'popsize', 'recombination', 'polish', 'disp', 'objmodel','redshift', 'delta_redshift', diff --git a/pypeit/sensfunc.py b/pypeit/sensfunc.py index a415daa488..e662a4f33e 100644 --- a/pypeit/sensfunc.py +++ b/pypeit/sensfunc.py @@ -880,6 +880,7 @@ def compute_zeropoint(self): self.counts_mask, self.meta_spec['EXPTIME'], self.meta_spec['AIRMASS'], self.std_dict, self.par['IR']['telgridfile'], + self.par['IR']['teltype'], log10_blaze_function=self.log10_blaze_function, polyorder=self.par['polyorder'], ech_orders=self.meta_spec['ECH_ORDERS'], From dccd3faca29bdda7e6f49aaf7e3e1fee9a48fac5 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Fri, 4 Aug 2023 12:15:21 +0200 Subject: [PATCH 028/244] fussing with sensfunc --- pypeit/core/telluric.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 6ccea68b30..b4f7c72a55 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -2258,6 +2258,7 @@ class Telluric(datamodel.DataContainer): 'norders', 'tell_dict', + 'tell_model_func', 'wave_grid', 'ngrid', @@ -2411,11 +2412,11 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, teltype, obj_params, init if teltype == 'pca' or teltype == 'PCA': self.tell_dict = read_telluric_pca(self.telgrid, wave_min=self.wave_in_arr[wv_gpm].min(), wave_max=self.wave_in_arr[wv_gpm].max()) - tell_model_func = eval_telluric_pca + self.tell_model_func = eval_telluric_pca elif teltype == 'grid': self.tell_dict = read_telluric_grid(self.telgrid, wave_min=self.wave_in_arr[wv_gpm].min(), wave_max=self.wave_in_arr[wv_gpm].max()) - tell_model_func = eval_telluric_grid + self.tell_model_func = eval_telluric_grid self.wave_grid = self.tell_dict['wave_grid'] self.ngrid = self.wave_grid.size self.resln_guess = wvutils.get_sampling(self.wave_in_arr)[2] \ @@ -2463,7 +2464,7 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, teltype, obj_params, init for counter, iord in enumerate(self.srt_order_tell): msgs.info(f'Initializing object model for order: {iord}, {counter}/{self.norders}' + f' with user supplied function: {self.init_obj_model.__name__}') - tellmodel = tell_model_func(self.tell_guess, self.tell_dict, + tellmodel = self.tell_model_func(self.tell_guess, self.tell_dict, ind_lower=self.ind_lower[iord], ind_upper=self.ind_upper[iord]) # TODO This is a pretty ugly way to pass in the blaze function. Particularly since now all the other models @@ -2536,7 +2537,7 @@ def run(self, only_orders=None): self.theta_tell_list[iord] = self.result_list[iord].x[-(self.ntell+3):] self.obj_model_list[iord], modelmask \ = self.eval_obj_model(self.theta_obj_list[iord], self.obj_dict_list[iord]) - self.tellmodel_list[iord] = eval_telluric(self.theta_tell_list[iord], self.tell_dict, + self.tellmodel_list[iord] = self.tell_model_func(self.theta_tell_list[iord], self.tell_dict, ind_lower=self.ind_lower[iord], ind_upper=self.ind_upper[iord]) self.assign_output(iord) From c22310418b564d25b40a6ff75e858f601546980d Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Mon, 7 Aug 2023 22:33:40 +0200 Subject: [PATCH 029/244] fixed sensfunc/tellfit functions, changed spectrograph defaults --- pypeit/core/telluric.py | 32 ++++++++++++------------ pypeit/scripts/tellfit.py | 3 +++ pypeit/spectrographs/gemini_flamingos.py | 2 +- pypeit/spectrographs/gemini_gmos.py | 2 +- pypeit/spectrographs/gemini_gnirs.py | 2 +- pypeit/spectrographs/gtc_osiris.py | 4 +-- pypeit/spectrographs/keck_deimos.py | 2 +- pypeit/spectrographs/keck_hires.py | 2 +- pypeit/spectrographs/keck_lris.py | 2 +- pypeit/spectrographs/keck_mosfire.py | 2 +- pypeit/spectrographs/keck_nires.py | 2 +- pypeit/spectrographs/keck_nirspec.py | 2 +- pypeit/spectrographs/magellan_fire.py | 4 +-- pypeit/spectrographs/mmt_binospec.py | 2 +- pypeit/spectrographs/mmt_mmirs.py | 2 +- pypeit/spectrographs/p200_dbsp.py | 2 +- pypeit/spectrographs/p200_tspec.py | 2 +- pypeit/spectrographs/shane_kast.py | 6 ++--- pypeit/spectrographs/soar_goodman.py | 2 +- pypeit/spectrographs/vlt_fors.py | 4 +-- pypeit/spectrographs/vlt_sinfoni.py | 2 +- pypeit/spectrographs/vlt_xshooter.py | 6 ++--- 22 files changed, 46 insertions(+), 43 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index b4f7c72a55..1ac59e1914 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -168,11 +168,8 @@ def read_telluric_pca(filename, wave_min=None, wave_max=None, pad_frac=0.10): Returns: :obj:`dict`: Dictionary containing the telluric PCA components. """ -# # Check for file -# if not os.path.isfile(filename): -# msgs.error(f"File {filename} is not on your disk. You likely need to download the Telluric files. See https://pypeit.readthedocs.io/en/release/installing.html#atmospheric-model-grids") - - hdul = fits.open(filename) + # load_telluric_grid() takes care of path and existance check + hdul = data.load_telluric_grid(filename) wave_grid_full = hdul[1].data pca_comp_full = hdul[0].data ncomp = hdul[0].header['NCOMP'] @@ -1551,7 +1548,7 @@ def create_bal_mask(wave, bal_wv_min_max): -def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile, npca=8, +def qso_telluric(spec1dfile, telgridfile, teltype, pca_file, z_qso, telloutfile, outfile, npca=8, pca_lower=1220.0, pca_upper=3100.0, bal_wv_min_max=None, delta_zqso=0.1, ntell=4, bounds_norm=(0.1, 3.0), tell_norm_thresh=0.9, sn_clip=30.0, only_orders=None, maxiter=3, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, @@ -1671,7 +1668,7 @@ def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile, if bal_wv_min_max is not None else mask & qsomask # parameters lowered for testing - TelObj = Telluric(wave, flux, ivar, mask_tot, telgridfile, obj_params, init_qso_model, + TelObj = Telluric(wave, flux, ivar, mask_tot, telgridfile, teltype, obj_params, init_qso_model, eval_qso_model, sn_clip=sn_clip, maxiter=maxiter, tol=tol, popsize=popsize, ntell=ntell, recombination=recombination, polish=polish, disp=disp, debug=debug) TelObj.run(only_orders=only_orders) @@ -1716,10 +1713,10 @@ def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile, return TelObj -def star_telluric(spec1dfile, telgridfile, telloutfile, outfile, star_type=None, star_mag=None, - star_ra=None, star_dec=None, func='legendre', model='exp', polyorder=5, - ntell=4, mask_hydrogen_lines=True, mask_helium_lines=False, hydrogen_mask_wid=10., - delta_coeff_bounds=(-20.0, 20.0), +def star_telluric(spec1dfile, telgridfile, teltype, telloutfile, outfile, star_type=None, + star_mag=None, star_ra=None, star_dec=None, func='legendre', model='exp', + polyorder=5, ntell=4, mask_hydrogen_lines=True, mask_helium_lines=False, + hydrogen_mask_wid=10., delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), only_orders=None, sn_clip=30.0, maxiter=3, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, debug_init=False, debug=False, show=False): @@ -1778,7 +1775,7 @@ def star_telluric(spec1dfile, telgridfile, telloutfile, outfile, star_type=None, mask_tot = mask_bad & mask_recomb & mask_tell # parameters lowered for testing - TelObj = Telluric(wave, flux, ivar, mask_tot, telgridfile, obj_params, init_star_model, + TelObj = Telluric(wave, flux, ivar, mask_tot, telgridfile, teltype, obj_params, init_star_model, eval_star_model, ntell=ntell, sn_clip=sn_clip, tol=tol, popsize=popsize, recombination=recombination, polish=polish, disp=disp, debug=debug) TelObj.run(only_orders=only_orders) @@ -1823,7 +1820,7 @@ def star_telluric(spec1dfile, telgridfile, telloutfile, outfile, star_type=None, return TelObj -def poly_telluric(spec1dfile, telgridfile, telloutfile, outfile, z_obj=0.0, func='legendre', +def poly_telluric(spec1dfile, telgridfile, teltype, telloutfile, outfile, z_obj=0.0, func='legendre', model='exp', polyorder=3, fit_wv_min_max=None, mask_lyman_a=True, ntell=4, delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), only_orders=None, sn_clip=30.0, maxiter=3, tol=1e-3, popsize=30, @@ -1876,7 +1873,7 @@ def poly_telluric(spec1dfile, telgridfile, telloutfile, outfile, z_obj=0.0, func mask_tot &= np.logical_not(create_bal_mask(wave, fit_wv_min_max)) # parameters lowered for testing - TelObj = Telluric(wave, flux, ivar, mask_tot, telgridfile, obj_params, init_poly_model, + TelObj = Telluric(wave, flux, ivar, mask_tot, telgridfile, teltype, obj_params, init_poly_model, eval_poly_model, sn_clip=sn_clip, maxiter=maxiter, tol=tol, popsize=popsize, ntell=ntell, recombination=recombination, polish=polish, disp=disp, debug=debug) TelObj.run(only_orders=only_orders) @@ -2350,7 +2347,7 @@ def empty_model_table(norders, nspec, ntell=4, n_obj_par=0): def __init__(self, wave, flux, ivar, gpm, telgridfile, teltype, obj_params, init_obj_model, eval_obj_model, log10_blaze_function=None, ech_orders=None, sn_clip=30.0, ntell=4, airmass_guess=1.5, - resln_guess=None, resln_frac_bounds=(0.5, 2.0), pix_shift_bounds=(-5.0, 5.0), + resln_guess=None, resln_frac_bounds=(0.5, 1.5), pix_shift_bounds=(-5.0, 5.0), pix_stretch_bounds=(0.9,1.1), maxiter=2, sticky=True, lower=3.0, upper=3.0, seed=777, ballsize = 5e-4, tol=1e-3, diff_evol_maxiter=1000, popsize=30, recombination=0.7, polish=True, disp=False, sensfunc=False, debug=False): @@ -2417,6 +2414,9 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, teltype, obj_params, init self.tell_dict = read_telluric_grid(self.telgrid, wave_min=self.wave_in_arr[wv_gpm].min(), wave_max=self.wave_in_arr[wv_gpm].max()) self.tell_model_func = eval_telluric_grid + else: + msgs.error('Invalid teltype -- must be `pca` or `grid`') + self.wave_grid = self.tell_dict['wave_grid'] self.ngrid = self.wave_grid.size self.resln_guess = wvutils.get_sampling(self.wave_in_arr)[2] \ @@ -2488,7 +2488,7 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, teltype, obj_params, init arg_dict_iord = dict(ivar=self.ivar_arr[self.ind_lower[iord]:self.ind_upper[iord]+1,iord], tell_dict=self.tell_dict, ind_lower=self.ind_lower[iord], ind_upper=self.ind_upper[iord], - ntell=self.ntell, tell_model_func=tell_model_func, + ntell=self.ntell, tell_model_func=self.tell_model_func, obj_model_func=self.eval_obj_model, obj_dict=obj_dict, ballsize=self.ballsize, bounds=bounds_iord, rng=self.rng, diff_evol_maxiter=self.diff_evol_maxiter, tol=self.tol, diff --git a/pypeit/scripts/tellfit.py b/pypeit/scripts/tellfit.py index b2eb899100..287922713f 100644 --- a/pypeit/scripts/tellfit.py +++ b/pypeit/scripts/tellfit.py @@ -152,6 +152,7 @@ def main(args): if par['telluric']['objmodel']=='qso': # run telluric.qso_telluric to get the final results TelQSO = telluric.qso_telluric(args.spec1dfile, par['telluric']['telgridfile'], + par['telluric']['teltype'], par['telluric']['pca_file'], par['telluric']['redshift'], modelfile, outfile, npca=par['telluric']['npca'], ntell=par['telluric']['ntell'], @@ -166,6 +167,7 @@ def main(args): debug=args.debug, show=args.plot) elif par['telluric']['objmodel']=='star': TelStar = telluric.star_telluric(args.spec1dfile, par['telluric']['telgridfile'], + par['telluric']['teltype'], modelfile, outfile, star_type=par['telluric']['star_type'], star_mag=par['telluric']['star_mag'], @@ -186,6 +188,7 @@ def main(args): debug=args.debug, show=args.plot) elif par['telluric']['objmodel']=='poly': TelPoly = telluric.poly_telluric(args.spec1dfile, par['telluric']['telgridfile'], + par['telluric']['teltype'], modelfile, outfile, z_obj=par['telluric']['redshift'], func=par['telluric']['func'], diff --git a/pypeit/spectrographs/gemini_flamingos.py b/pypeit/spectrographs/gemini_flamingos.py index bdfc336bdf..e0d962cfd0 100644 --- a/pypeit/spectrographs/gemini_flamingos.py +++ b/pypeit/spectrographs/gemini_flamingos.py @@ -143,7 +143,7 @@ def default_pypeit_par(cls): par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 8 # TODO: replace the telluric grid file for Gemini-S site. - par['sensfunc']['IR']['telgridfile'] = 'TelFit_LasCampanas_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' return par diff --git a/pypeit/spectrographs/gemini_gmos.py b/pypeit/spectrographs/gemini_gmos.py index 20ec46343c..347a80679b 100644 --- a/pypeit/spectrographs/gemini_gmos.py +++ b/pypeit/spectrographs/gemini_gmos.py @@ -873,7 +873,7 @@ def default_pypeit_par(cls): """ par = super().default_pypeit_par() par['sensfunc']['algorithm'] = 'IR' - par['sensfunc']['IR']['telgridfile'] = 'TelFit_LasCampanas_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' # Bound the detector with slit edges if no edges are found. These data are often trimmed # so we implement this here as the default. par['calibrations']['slitedges']['bound_detector'] = True diff --git a/pypeit/spectrographs/gemini_gnirs.py b/pypeit/spectrographs/gemini_gnirs.py index 6c6fae1779..a201686fcd 100644 --- a/pypeit/spectrographs/gemini_gnirs.py +++ b/pypeit/spectrographs/gemini_gnirs.py @@ -288,7 +288,7 @@ def default_pypeit_par(cls): # Sensitivity function parameters par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 6 - par['sensfunc']['IR']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' return par def config_specific_par(self, scifile, inp_par=None): diff --git a/pypeit/spectrographs/gtc_osiris.py b/pypeit/spectrographs/gtc_osiris.py index 926eeb4953..f7a1045d41 100644 --- a/pypeit/spectrographs/gtc_osiris.py +++ b/pypeit/spectrographs/gtc_osiris.py @@ -374,7 +374,7 @@ def config_specific_par(self, scifile, inp_par=None): par['calibrations']['wavelengths']['lamps'] = ['ArI','XeI','NeI'] par['calibrations']['wavelengths']['reid_arxiv'] = 'gtc_osiris_R2500I.fits' par['sensfunc']['algorithm'] = 'IR' - par['sensfunc']['IR']['telgridfile'] = "TelFit_MaunaKea_3100_26100_R20000.fits" + par['sensfunc']['IR']['telgridfile'] = "TellPCA_3000_26000_R10000.fits" else: msgs.warn('gtc_osiris.py: template arc missing for this grism! Trying holy-grail...') par['calibrations']['wavelengths']['method'] = 'holy-grail' @@ -943,7 +943,7 @@ def config_specific_par(self, scifile, inp_par=None): par['calibrations']['wavelengths']['lamps'] = ['ArI','XeI','NeI'] par['calibrations']['wavelengths']['reid_arxiv'] = 'gtc_osiris_R2500I.fits' par['sensfunc']['algorithm'] = 'IR' - par['sensfunc']['IR']['telgridfile'] = "TelFit_MaunaKea_3100_26100_R20000.fits" + par['sensfunc']['IR']['telgridfile'] = "TellPCA_3000_26000_R10000.fits" else: msgs.warn('gtc_osiris.py: template arc missing for this grism! Trying holy-grail...') par['calibrations']['wavelengths']['method'] = 'holy-grail' diff --git a/pypeit/spectrographs/keck_deimos.py b/pypeit/spectrographs/keck_deimos.py index 4be0b57b2e..4309bf966f 100644 --- a/pypeit/spectrographs/keck_deimos.py +++ b/pypeit/spectrographs/keck_deimos.py @@ -301,7 +301,7 @@ def default_pypeit_par(cls): par['scienceframe']['process']['objlim'] = 1.5 # If telluric is triggered - par['sensfunc']['IR']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R15000.fits' return par def config_specific_par(self, scifile, inp_par=None): diff --git a/pypeit/spectrographs/keck_hires.py b/pypeit/spectrographs/keck_hires.py index e2ddfd5a57..7d24d179c8 100644 --- a/pypeit/spectrographs/keck_hires.py +++ b/pypeit/spectrographs/keck_hires.py @@ -151,7 +151,7 @@ def default_pypeit_par(cls): # Sensitivity function parameters par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 5 #[9, 11, 11, 9, 9, 8, 8, 7, 7, 7, 7, 7, 7, 7, 7] - par['sensfunc']['IR']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_10500_R60000.fits' # Coadding par['coadd1d']['wave_method'] = 'log10' diff --git a/pypeit/spectrographs/keck_lris.py b/pypeit/spectrographs/keck_lris.py index 9822bc5f1d..614a41e41e 100644 --- a/pypeit/spectrographs/keck_lris.py +++ b/pypeit/spectrographs/keck_lris.py @@ -82,7 +82,7 @@ def default_pypeit_par(cls): # If telluric is triggered - par['sensfunc']['IR']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' return par diff --git a/pypeit/spectrographs/keck_mosfire.py b/pypeit/spectrographs/keck_mosfire.py index 64e3bee121..d95e01ff35 100644 --- a/pypeit/spectrographs/keck_mosfire.py +++ b/pypeit/spectrographs/keck_mosfire.py @@ -127,7 +127,7 @@ def default_pypeit_par(cls): par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 13 par['sensfunc']['IR']['maxiter'] = 2 - par['sensfunc']['IR']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' return par # NOTE: This function is used by the dev-suite diff --git a/pypeit/spectrographs/keck_nires.py b/pypeit/spectrographs/keck_nires.py index 9ce4f241df..8a54f6e1ac 100644 --- a/pypeit/spectrographs/keck_nires.py +++ b/pypeit/spectrographs/keck_nires.py @@ -140,7 +140,7 @@ def default_pypeit_par(cls): par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 8 par['sensfunc']['IR']['maxiter'] = 2 - par['sensfunc']['IR']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' # Coadding par['coadd1d']['wave_method'] = 'log10' diff --git a/pypeit/spectrographs/keck_nirspec.py b/pypeit/spectrographs/keck_nirspec.py index caf58dbf4c..cc6ab587b1 100644 --- a/pypeit/spectrographs/keck_nirspec.py +++ b/pypeit/spectrographs/keck_nirspec.py @@ -129,7 +129,7 @@ def default_pypeit_par(cls): # Sensitivity function parameters par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 8 - par['sensfunc']['IR']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R25000.fits' return par def init_meta(self): diff --git a/pypeit/spectrographs/magellan_fire.py b/pypeit/spectrographs/magellan_fire.py index 4d30943c43..1c50237c26 100644 --- a/pypeit/spectrographs/magellan_fire.py +++ b/pypeit/spectrographs/magellan_fire.py @@ -201,7 +201,7 @@ def default_pypeit_par(cls): par['sensfunc']['polyorder'] = 5 par['sensfunc']['IR']['maxiter'] = 2 # place holder for telgrid file - par['sensfunc']['IR']['telgridfile'] = 'TelFit_LasCampanas_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R15000.fits' # Coadding. I'm not sure what this should be for PRISM mode? par['coadd1d']['wave_method'] = 'log10' @@ -418,7 +418,7 @@ def default_pypeit_par(cls): par['reduce']['findobj']['find_trim_edge'] = [50,50] par['flexure']['spec_method'] = 'skip' - par['sensfunc']['IR']['telgridfile'] = 'TelFit_LasCampanas_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' # Set the default exposure time ranges for the frame typing par['calibrations']['standardframe']['exprng'] = [None, 60] diff --git a/pypeit/spectrographs/mmt_binospec.py b/pypeit/spectrographs/mmt_binospec.py index f3bc5d1983..669b5d6a29 100644 --- a/pypeit/spectrographs/mmt_binospec.py +++ b/pypeit/spectrographs/mmt_binospec.py @@ -215,7 +215,7 @@ def default_pypeit_par(cls): # Sensitivity function parameters par['sensfunc']['polyorder'] = 7 - par['sensfunc']['IR']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' return par diff --git a/pypeit/spectrographs/mmt_mmirs.py b/pypeit/spectrographs/mmt_mmirs.py index 94705905ee..9d2706e86b 100644 --- a/pypeit/spectrographs/mmt_mmirs.py +++ b/pypeit/spectrographs/mmt_mmirs.py @@ -203,7 +203,7 @@ def default_pypeit_par(cls): par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 8 # ToDo: replace the telluric grid file for MMT site. - par['sensfunc']['IR']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' return par diff --git a/pypeit/spectrographs/p200_dbsp.py b/pypeit/spectrographs/p200_dbsp.py index 84c60e393a..3037f397a9 100644 --- a/pypeit/spectrographs/p200_dbsp.py +++ b/pypeit/spectrographs/p200_dbsp.py @@ -525,7 +525,7 @@ def default_pypeit_par(cls): par['sensfunc']['algorithm'] = 'UVIS' par['sensfunc']['UVIS']['polycorrect'] = False - par['sensfunc']['IR']['telgridfile'] = 'TelFit_Lick_3100_11100_R10000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' return par def config_specific_par(self, scifile, inp_par=None): diff --git a/pypeit/spectrographs/p200_tspec.py b/pypeit/spectrographs/p200_tspec.py index 0100fc42ff..1b5a7d8dcf 100644 --- a/pypeit/spectrographs/p200_tspec.py +++ b/pypeit/spectrographs/p200_tspec.py @@ -215,7 +215,7 @@ def default_pypeit_par(cls): # Sensitivity function parameters par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 8 - par['sensfunc']['IR']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' # Coadding par['coadd1d']['wave_method'] = 'log10' diff --git a/pypeit/spectrographs/shane_kast.py b/pypeit/spectrographs/shane_kast.py index 965b49eb27..8528d334a2 100644 --- a/pypeit/spectrographs/shane_kast.py +++ b/pypeit/spectrographs/shane_kast.py @@ -57,7 +57,7 @@ def default_pypeit_par(cls): par['calibrations']['standardframe']['exprng'] = [1, 61] # par['scienceframe']['exprng'] = [61, None] - par['sensfunc']['IR']['telgridfile'] = 'TelFit_Lick_3100_11100_R10000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' return par def init_meta(self): @@ -460,7 +460,7 @@ def default_pypeit_par(cls): # TODO In case someone wants to use the IR algorithm for shane kast this is the telluric file. Note the IR # algorithm is not the default. - par['sensfunc']['IR']['telgridfile'] = 'TelFit_Lick_3100_11100_R10000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' return par def init_meta(self): @@ -680,7 +680,7 @@ def default_pypeit_par(cls): par['calibrations']['wavelengths']['rms_threshold'] = 0.20 par['calibrations']['wavelengths']['sigdetect'] = 5. par['calibrations']['wavelengths']['use_instr_flag'] = True - par['sensfunc']['IR']['telgridfile'] = 'TelFit_Lick_3100_11100_R10000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' return par diff --git a/pypeit/spectrographs/soar_goodman.py b/pypeit/spectrographs/soar_goodman.py index 607653d06a..bad7ae7b37 100644 --- a/pypeit/spectrographs/soar_goodman.py +++ b/pypeit/spectrographs/soar_goodman.py @@ -330,7 +330,7 @@ def default_pypeit_par(cls): par['scienceframe']['exprng'] = [90, None] #par['sensfunc']['algorithm'] = 'IR' - par['sensfunc']['IR']['telgridfile'] = 'TelFit_LasCampanas_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R15000.fits' # TODO: Temporary fix for failure mode. Remove once Ryan provides a # fix. diff --git a/pypeit/spectrographs/vlt_fors.py b/pypeit/spectrographs/vlt_fors.py index ccadf7b501..62250647aa 100644 --- a/pypeit/spectrographs/vlt_fors.py +++ b/pypeit/spectrographs/vlt_fors.py @@ -69,7 +69,7 @@ def default_pypeit_par(cls): # Sensitivity function parameters par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 5 - par['sensfunc']['IR']['telgridfile'] = 'TelFit_Paranal_VIS_9800_25000_R25000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' @@ -451,4 +451,4 @@ def parse_dither_pattern(self, file_list, ext=None): # dither_id.append(hdr['FRAMEID']) # offset_arcsec[ifile] = hdr['YOFFSET'] - return dither_pattern, dither_id, offset_arcsec \ No newline at end of file + return dither_pattern, dither_id, offset_arcsec diff --git a/pypeit/spectrographs/vlt_sinfoni.py b/pypeit/spectrographs/vlt_sinfoni.py index 569e8affbe..481f0dd1e3 100644 --- a/pypeit/spectrographs/vlt_sinfoni.py +++ b/pypeit/spectrographs/vlt_sinfoni.py @@ -157,7 +157,7 @@ def default_pypeit_par(cls): # Sensitivity function parameters par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 7 - par['sensfunc']['IR']['telgridfile'] = 'TelFit_Paranal_NIR_9800_25000_R25000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' return par diff --git a/pypeit/spectrographs/vlt_xshooter.py b/pypeit/spectrographs/vlt_xshooter.py index f75d11d0b2..fcecb07992 100644 --- a/pypeit/spectrographs/vlt_xshooter.py +++ b/pypeit/spectrographs/vlt_xshooter.py @@ -344,7 +344,7 @@ def default_pypeit_par(cls): # Sensitivity function parameters par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 8 - par['sensfunc']['IR']['telgridfile'] = 'TelFit_Paranal_NIR_9800_25000_R25000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R25000.fits' # Coadding par['coadd1d']['wave_method'] = 'log10' @@ -722,7 +722,7 @@ def default_pypeit_par(cls): # Sensitivity function parameters par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 8 #[9, 11, 11, 9, 9, 8, 8, 7, 7, 7, 7, 7, 7, 7, 7] - par['sensfunc']['IR']['telgridfile'] = 'TelFit_Paranal_VIS_4900_11100_R25000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R25000.fits' # Coadding par['coadd1d']['wave_method'] = 'log10' @@ -1007,7 +1007,7 @@ def default_pypeit_par(cls): # Sensitivity function parameters par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 8 - par['sensfunc']['IR']['telgridfile'] = 'TelFit_LasCampanas_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R25000.fits' # This is a hack until we we have a Paranal file generated that covers the UVB wavelength range. # Coadding From 41f6d970831d98191f66ebc22d334da3431b2f18 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Tue, 8 Aug 2023 09:57:25 +0200 Subject: [PATCH 030/244] docs --- pypeit/core/telluric.py | 91 +++++++++++++++++++++++++++++------------ 1 file changed, 64 insertions(+), 27 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 1ac59e1914..3be089a021 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -167,6 +167,15 @@ def read_telluric_pca(filename, wave_min=None, wave_max=None, pad_frac=0.10): Returns: :obj:`dict`: Dictionary containing the telluric PCA components. + - wave_grid= + - dloglam= + - resln_guess= + - pix_per_sigma= + - tell_pad_pix= + - ncomp_tell_pca= + - tell_pca= + - bounds_tell_pca= + - coefs_tell_pca= """ # load_telluric_grid() takes care of path and existance check hdul = data.load_telluric_grid(filename) @@ -347,9 +356,9 @@ def shift_telluric(tell_model, loglam, dloglam, shift, stretch): Args: tell_model (`numpy.ndarray`_): - Input telluric model. The shape of this input is in general different from the size of the telluric grid - (read in by read_telluric_grid above) because it is trimmed to relevant wavelenghts using ind_lower, ind_upper. - See eval_telluric below. + Input telluric model. The shape of this input is in general different from the size of the telluric models + (read in by read_telluric_* above) because it is trimmed to relevant wavelengths using ind_lower, ind_upper. + See eval_telluric_* below. loglam (`numpy.ndarray`_): The log10 of the wavelength grid on which the tell_model is evaluated. @@ -433,16 +442,20 @@ def eval_telluric_pca(theta_tell, tell_dict, ind_lower=None, ind_upper=None): tellmodel_hires = np.zeros_like(tell_dict['tell_pca'][0]) tellmodel_hires[ind_lower_pad:ind_upper_pad+1] = np.dot(np.append(1,theta_tell[:ncomp_use]), tell_dict['tell_pca'][:ncomp_use+1][:,ind_lower_pad:ind_upper_pad+1]) - # Transform Arsinh PCA model, works slightly better than transmission components but may be slightly slower? + + # PCA model is inverse sinh of the optical depth, convert to transmission here tellmodel_hires = np.exp(-np.sinh(tellmodel_hires)) - # FD: currently assumes shift + stretch is on + # Convolve transmission to given spectral resolution, and resample onto spectrum wavelengths tellmodel_conv = conv_telluric(tellmodel_hires[ind_lower_pad:ind_upper_pad+1], tell_dict['dloglam'], theta_tell[-3]) + # Stretch and shift telluric wavelength grid + # FD: currently assumes shift + stretch is on... tellmodel_out = shift_telluric(tellmodel_conv, np.log10(tell_dict['wave_grid'][ind_lower_pad:ind_upper_pad+1]), tell_dict['dloglam'], theta_tell[-2], theta_tell[-1]) + return tellmodel_out[ind_lower_final:ind_upper_final] def eval_telluric_grid(theta_tell, tell_dict, ind_lower=None, ind_upper=None): @@ -1386,12 +1399,14 @@ def sensfunc_telluric(wave, counts, counts_ivar, counts_mask, exptime, airmass, telgridfile : :obj:`str` File containing grid of HITRAN atmosphere models; see :class:`~pypeit.par.pypeitpar.TelluricPar`. + teltype : :obj:`str` + Method for evaluating telluric models, either `PCA` or `grid`. ech_orders : `numpy.ndarray`_, shape is (norders,), optional If passed, provides the true order numbers for the spectra provided. polyorder : :obj:`int`, optional, default = 8 Polynomial order for the sensitivity function fit. - ntell : :obj:`int`, optional, default = 6 - Number of telluric PCA components to use + ntell : :obj:`int`, optional, default = 4 + Number of telluric model parameters (must be 4 for teltype=`grid`, or <=10 for teltype=`PCA`) mask_hydrogen_lines : :obj:`bool`, optional If True, mask stellar hydrogen absorption lines before fitting sensitivity function. Default = True mask_helium_lines : :obj:`bool`, optional @@ -1563,6 +1578,8 @@ def qso_telluric(spec1dfile, telgridfile, teltype, pca_file, z_qso, telloutfile, the output from a ``PypeIt`` 1D coadd. telgridfile : :obj:`str` File name with the grid of telluric spectra. + teltype : :obj:`str` + Method for evaluating telluric models, either `PCA` or `grid`. pca_file: :obj:`str` File name of the QSO PCA model fits file. z_qso : :obj:`float` @@ -1586,7 +1603,7 @@ def qso_telluric(spec1dfile, telgridfile, teltype, pca_file, z_qso, telloutfile, During the fit, the QSO redshift is allowed to vary within ``+/-delta_zqso``. ntell : :obj:`int`, optional, default = 4 - Number of telluric PCA components to use + Number of telluric model parameters (must be 4 for teltype=`grid`, or <=10 for teltype=`PCA`) bounds_norm : :obj:`tuple`, optional A two-tuple with the lower and upper bounds on the fractional adjustment of the flux in the QSO model spectrum. For example, a value of ``(0.1, @@ -1925,7 +1942,12 @@ class Telluric(datamodel.DataContainer): The model telluric absorption spectra for the atmosphere are generated by HITRAN atmosphere models with a baseline atmospheric profile model of the - observatory in question. The modeling can be applied to both multi-slit + observatory in question. One can interpolate over grids of these absorption + spectra directly, which have been created for each observatory, or instead + (by default) use a PCA decomposition of the telluric models across all + observatories, which is more flexible and much lighter in file size. + + The modeling can be applied to both multi-slit or echelle data. The code performs a differential evolution optimization using `scipy.optimize.differential_evolution`_. Several iterations are performed with rejection of outliers (governed by the maxiter parameter @@ -1956,26 +1978,29 @@ class Telluric(datamodel.DataContainer): object model, the bounds depend on the nature of the object model, which is why ``init_obj_model`` must be provided. - The telluric model is governed by seven parameters --- ``pressure``, - ``temperature``, ``humidity``, ``airmass``, ``resln``, ``shift``, + The telluric model is governed by seven parameters --- for the `grid` + approach: ``pressure``, ``temperature``, ``humidity``, ``airmass``; for + the `PCA` approach 4 PCA coefficients; plus ``resln``, ``shift``, and ``stretch`` --- where ``resln`` is the resolution of the spectrograph and ``shift`` is a shift in pixels. The ``stretch`` is a stretch of the pixel scale. The airmass of the object will be used to initalize the fit (this helps with initalizing the object model), but the models are sufficiently flexible that often the best-fit airmass actually differs from the - airmass of the spectrum. + airmass of the spectrum. In the `PCA` approach, the number of PCA + components used can be adjusted between 1 and 10, with a tradeoff between + speed and accuracy. This code can be run on stacked spectra covering a large range of - airmasses and will still provide good results. The resulting airmass will - be an effective value, and as per above may not have much relation to the - true airmass of the observation. The exception to this rule is extremely - high signal-to-noise ratio data S/N > 30, for which it can be difficult - to obtain a good fit within the noise of the data. In such cases, the - user should split up the data into chunks taken around the same airmass, - fit those individually with this class, and then combine the resulting - telluric corrected spectra. This will, in general, result in better fits, - and will also average down the residuals from the telluric model fit in - the final averaged spectrum. + airmasses and will still provide good results. The resulting airmass + (in the `grid` approach) will be an effective value, and as per above may + not have much relation to the true airmass of the observation. The + exception to this rule is extremely high signal-to-noise ratio data + (S/N > 30), for which it can be difficult to obtain a good fit within the + noise of the data. In such cases, the user should split up the data into + chunks taken around the same airmass, fit those individually with this + class, and then combine the resulting telluric corrected spectra. This + will, in general, result in better fits, and will also average down the + residuals from the telluric model fit in the final averaged spectrum. The datamodel attributes are: @@ -1998,8 +2023,20 @@ class Telluric(datamodel.DataContainer): Good pixel gpm for the object in question. Same shape as ``wave``. telgridfile (:obj:`str`): - File containing grid of HITRAN atmosphere models; see + File containing grid of HITRAN atmosphere models or PCA + decomposition of such models, see :class:`~pypeit.par.pypeitpar.TelluricPar`. + teltype (:obj:`str`, optional): + Method for evaluating telluric models, either `PCA` or `grid`. + The `grid` method directly interpolates a pre-computed grid of + HITRAN atmospheric models with nearest grid-point interpolation, + while the `PCA` method instead fits for the coefficients of a + PCA decomposition of HITRAN atmospheric models, enabling a more + flexible and far more file-size efficient interpolation + of the telluric absorption model space. + ntell (:obj:`int`, optional): + Number of telluric model parameters. Must be 4 for teltype=`grid`, + or <=10 for teltype=`PCA`. obj_params (:obj:`dict`): Dictionary of parameters for initializing the object model. init_obj_model (callable): @@ -2188,7 +2225,7 @@ class Telluric(datamodel.DataContainer): descr='File containing PCA components or grid of ' 'HITRAN atmosphere models'), 'teltype': dict(otype=str, - descr='Type of telluric model, pca or grid'), + descr='Type of telluric model, `PCA` or `grid`'), 'ntell': dict(otype=int, descr='Number of telluric PCA components used'), 'std_src': dict(otype=str, descr='Name of the standard source'), @@ -2316,8 +2353,8 @@ def empty_model_table(norders, nspec, ntell=4, n_obj_par=0): # TODO: Why do we need both TELL_THETA and all the individual parameters... table.Column(name='TELL_THETA', dtype=float, length=norders, shape=(ntell+3,), description='Best-fitting telluric model parameters'), - table.Column(name='TELL_COEFF', dtype=float, length=norders, shape=(ntell,), - description='Best-fitting telluric PCA coefficients'), + table.Column(name='TELL_PARAM', dtype=float, length=norders, shape=(ntell,), + description='Best-fitting telluric atmospheric parameters or PCA coefficients'), table.Column(name='TELL_RESLN', dtype=float, length=norders, description='Best-fitting telluric model spectral resolution'), table.Column(name='TELL_SHIFT', dtype=float, length=norders, @@ -2624,7 +2661,7 @@ def assign_output(self, iord): kind='linear', bounds_error=False, fill_value=0.0)(wave_in_gd) self.model['TELL_THETA'][iord] = self.theta_tell_list[iord] - self.model['TELL_COEFF'][iord] = self.theta_tell_list[iord][:self.ntell] + self.model['TELL_PARAM'][iord] = self.theta_tell_list[iord][:self.ntell] self.model['TELL_RESLN'][iord] = self.theta_tell_list[iord][self.ntell] self.model['TELL_SHIFT'][iord] = self.theta_tell_list[iord][self.ntell+1] self.model['TELL_STRETCH'][iord] = self.theta_tell_list[iord][self.ntell+2] From 768fe9500c177e9071e65ed1074c9d6a37ab76d6 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Tue, 8 Aug 2023 10:08:10 +0200 Subject: [PATCH 031/244] sort fix, more docs --- pypeit/core/telluric.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 3be089a021..28c4930094 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -2739,7 +2739,7 @@ def get_tell_guess(self): function. Returns: - :obj:`tuple`: The guess telluric PCA coefficients, + :obj:`tuple`: The guess telluric model parameters, resolution, shift, and stretch parameters. """ guess = list(np.zeros(self.ntell)) @@ -2754,9 +2754,9 @@ def get_bounds_tell(self): Returns: :obj:`list`: A list of two-tuples, where each two-tuple proives - the minimum and maximum allowed model values for the pressure, - temperature, humidity, airmass, resolution, shift, and - stretch parameters. + the minimum and maximum allowed model values for the PCA coefficients, + (or for the `grid` approach: pressure, temperature, humidity, airmass) + as well as the resolution, shift, and stretch parameters. """ # Set the bounds for the optimization bounds = [] @@ -2789,8 +2789,14 @@ def sort_telluric(self): # Do a quick loop over all the orders to sort them in order of strongest # to weakest telluric absorption for iord in range(self.norders): - tell_model_mean = self.tell_dict['tell_pca'][0,self.ind_lower[iord]:self.ind_upper[iord]+1] - tell_med[iord] = np.mean(tell_model_mean) + if self.teltype == 'grid': + tm_grid = self.tell_dict['tell_grid'][...,self.ind_lower[iord]:self.ind_upper[iord]+1] + tell_model_mid = tm_grid[tm_grid.shape[0]//2, tm_grid.shape[1]//2, + tm_grid.shape[2]//2, tm_grid.shape[3]//2, :] + tell_med[iord] = np.mean(tell_model_mid) + else: + tell_model_mean = self.tell_dict['tell_pca'][0,self.ind_lower[iord]:self.ind_upper[iord]+1] + tell_med[iord] = np.mean(tell_model_mean) # Perform fits in order of telluric strength return tell_med.argsort() From 38397ac46f4ab6e03c465f62a828b330b9ddd3e3 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Tue, 8 Aug 2023 10:15:30 +0200 Subject: [PATCH 032/244] cleanup --- pypeit/core/telluric.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 28c4930094..ac41b73147 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -561,7 +561,7 @@ def tellfit_chi2(theta, flux, thismask, arg_dict): tell_model_func = arg_dict['tell_model_func'] flux_ivar = arg_dict['ivar'] - # TODO: make this work without shift and stretch turned on + # TODO: make this work without shift and stretch turned on? # Number of telluric model parameters, plus shift, stretch, and resolution nfit = arg_dict['ntell']+3 @@ -580,10 +580,6 @@ def tellfit_chi2(theta, flux, thismask, arg_dict): robust_scale = 2.0 huber_vec = scipy.special.huber(robust_scale, chi_vec) loss_function = np.sum(huber_vec * totalmask) - # If the PCA coefficients lie outside the convex hull of the telluric models, - # it should be disfavored, so penalize the loss function severely - #if not in_hull(arg_dict['tell_dict']['coefs_tell_pca'][:,:(nfit-3)],theta_tell[:-3]): - # loss_function += 1e6 return loss_function def tellfit(flux, thismask, arg_dict, init_from_last=None): @@ -623,8 +619,8 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): - ``arg_dict['flux_ivar']``: Inverse variance for the flux array - ``arg_dict['tell_dict']``: Dictionary containing the - telluric PCA model and its parameters read in by - read_telluric_pca + telluric model and its parameters read in by + read_telluric_pca or read_telluric_grid - ``arg_dict['ind_lower']``: Lower index into the telluric model wave_grid to trim down the telluric model. @@ -674,7 +670,7 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): nparams = len(bounds) # Number of parameters in the model popsize = arg_dict['popsize'] # Note this does nothing if the init is done from a previous iteration or optimum nsamples = arg_dict['popsize']*nparams - # FD: Currently assumes shift and stretch are turned on. + # FD: Assumes shift and stretch are turned on. ntheta_tell = arg_dict['ntell']+3 # Total number of telluric model parameters # Decide how to initialize if init_from_last is not None: From b39db4f1b17afcd63435b50f680c5e568d64c309 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Tue, 8 Aug 2023 10:53:58 +0200 Subject: [PATCH 033/244] docs --- pypeit/core/telluric.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index ac41b73147..1fcd1a9edb 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -422,7 +422,7 @@ def eval_telluric_pca(theta_tell, tell_dict, ind_lower=None, ind_upper=None): """ ntheta = len(theta_tell) # Infer number of used components from the number of parameters - # TODO: make this work even without shift and stretch + # TODO: make this work even without shift and stretch? ncomp_use = ntheta-3 if ncomp_use > tell_dict['ncomp_tell_pca']: msgs.error('Asked for more PCA components than exist in PCA file.') @@ -561,7 +561,7 @@ def tellfit_chi2(theta, flux, thismask, arg_dict): tell_model_func = arg_dict['tell_model_func'] flux_ivar = arg_dict['ivar'] - # TODO: make this work without shift and stretch turned on? + # TODO: make this work without shift and stretch? # Number of telluric model parameters, plus shift, stretch, and resolution nfit = arg_dict['ntell']+3 @@ -600,8 +600,10 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): theta_obj = theta[:-(ntell+3)] theta_tell = theta[-(ntell+3):] - The telluric model theta_tell includes a user-specified number - of PCA coefficients, spectral resolution, shift, and stretch. + The telluric model theta_tell includes a either user-specified + number of PCA coefficients (in PCA mode) or ambient pressure, + temperature, humidity, and airmass (in grid mode) as well as + spectral resolution, shift, and stretch. The object model theta_obj can have an arbitrary size and is provided as an argument to obj_model_func @@ -1037,7 +1039,7 @@ def init_qso_model(obj_params, iord, wave, flux, ivar, mask, tellmodel): tellmodel : array shape (nspec,) This is a telluric model computed on the wave wavelength grid. Initialization usually requires some initial best guess for the telluric absorption, which is computed from the mean of the telluric model grid using - the resolution of the spectrograph. (TODO: revisit this for PCA) + the resolution of the spectrograph. Returns ------- From 109c28045dda3f02d07a714fd2f334f0bacbb508 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Tue, 8 Aug 2023 11:02:08 +0200 Subject: [PATCH 034/244] cleanup --- pypeit/spectrographs/soar_goodman.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/spectrographs/soar_goodman.py b/pypeit/spectrographs/soar_goodman.py index bad7ae7b37..41f52d680c 100644 --- a/pypeit/spectrographs/soar_goodman.py +++ b/pypeit/spectrographs/soar_goodman.py @@ -527,7 +527,7 @@ def default_pypeit_par(cls): par['scienceframe']['exprng'] = [90, None] # par['sensfunc']['algorithm'] = 'IR' - par['sensfunc']['IR']['telgridfile'] = 'TelFit_LasCampanas_3100_26100_R20000.fits' + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R15000.fits' return par From c8c11d57ee77b46aee2ecee6a7d78952762ea3a6 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Tue, 8 Aug 2023 11:55:04 +0200 Subject: [PATCH 035/244] fixing changelog --- CHANGES.rst | 7 ------- 1 file changed, 7 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4da7c7986d..39bad9447e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -26,11 +26,6 @@ - Changes to how masking is dealt with in extraction to fix a bug in how masks were being treated for echelle data - Various fixes and changes required to add more support for Keck/HIRES and JWST -<<<<<<< HEAD - -<<<<<<< HEAD -======= -======= - Fix a bug in ``spectrograph.select_detectors``, where a list of ``slitspatnum`` could not be used. - Improvements in 2D coaddition - Fix a bug in `pypeit_setup_coadd2d` for the output file name of the .coadd2d file @@ -38,10 +33,8 @@ - Now ``only_slits`` parameter in `pypeit_coadd_2dspec` includes the detector number (similar to ``slitspatnum``) - Added ``exclude_slits`` parameter in `pypeit_coadd_2dspec` to exclude specific slits - Fix wrong RA & Dec for 2D coadded serendips ->>>>>>> sensfunc_blaze_jwst_lists ->>>>>>> sensfunc_blaze_jwst_lists 1.13.0 (2 June 2023) -------------------- From 25cf6f7bcd1e8818de89533e70970a909cba3a19 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Tue, 8 Aug 2023 11:57:18 +0200 Subject: [PATCH 036/244] changelog --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 39bad9447e..e00707f737 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -33,7 +33,7 @@ - Now ``only_slits`` parameter in `pypeit_coadd_2dspec` includes the detector number (similar to ``slitspatnum``) - Added ``exclude_slits`` parameter in `pypeit_coadd_2dspec` to exclude specific slits - Fix wrong RA & Dec for 2D coadded serendips - +- Introduced PCA method for telluric corrections 1.13.0 (2 June 2023) -------------------- From 51581f1bbdcc2d6047ab280cf7fbba3a02ee23ee Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Tue, 8 Aug 2023 12:04:55 +0200 Subject: [PATCH 037/244] cleanup --- pypeit/core/telluric.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 1fcd1a9edb..2c3fa9f168 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -80,8 +80,7 @@ def init_qso_pca(filename,wave_grid,redshift, npca): # Generate a mixture model for the coefficients prior, what should ngauss be? #prior = mixture.GaussianMixture(n_components = npca-1).fit(coeffs_c[:, 1:npca]) # Construct the PCA dict - qso_pca_dict = {'npca': npca, 'components': pca_comp_new, 'coeffs': coeffs_c, 'z_fid': redshift, 'dloglam': dloglam, - 'interp': pca_interp, 'wave_grid': wave_grid} + qso_pca_dict = {'npca': npca, 'components': pca_comp_new, 'coeffs': coeffs_c, 'z_fid': redshift, 'dloglam': dloglam} return qso_pca_dict def qso_pca_eval(theta,qso_pca_dict): @@ -107,7 +106,6 @@ def qso_pca_eval(theta,qso_pca_dict): z_qso = theta[0] norm = theta[1] A = theta[2:] - #C_now = qso_pca_dict['interp'](qso_pca_dict['wave_grid']/(1.0+z_qso))[:npca,:] dshift = int(np.round(np.log10((1.0 + z_qso)/(1.0 + z_fid))/dloglam)) C_now = np.roll(C[:npca,:], dshift, axis=1) return norm*np.exp(np.dot(np.append(1.0,A),C_now)) From 73782d37dbb9970b49f53c318ebec086592c3aa0 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Tue, 8 Aug 2023 12:16:32 +0200 Subject: [PATCH 038/244] cleanup --- pypeit/core/telluric.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 2c3fa9f168..d93c87b19a 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -1054,7 +1054,7 @@ def init_qso_model(obj_params, iord, wave, flux, ivar, mask, tellmodel): """ qso_pca_dict = init_qso_pca(obj_params['pca_file'], wave, obj_params['z_qso'], obj_params['npca']) - qso_pca_mean = np.exp(qso_pca_dict['interp'](qso_pca_dict['wave_grid']*(1+qso_pca_dict['z_fid']))[0, :]) + qso_pca_mean = np.exp(qso_pca_dict['components'][0, :]) tell_mask = tellmodel > obj_params['tell_norm_thresh'] # Create a reference model and bogus noise flux_ref = qso_pca_mean * tellmodel @@ -1094,10 +1094,10 @@ def eval_qso_model(theta, obj_dict): Returns ------- - qso_pca_model : array with same shape as the PCA vectors (tored in the obj_dict['pca_dict']) + qso_pca_model : array with same shape as the PCA vectors (stored in the obj_dict['pca_dict']) PCA vectors were already interpolated onto the telluric model grid by init_qso_model - gpm : : array with same shape as the qso_pca_model + gpm : `numpy.ndarray`_ : array with same shape as the qso_pca_model Good pixel mask indicating where the model is valid """ @@ -1516,7 +1516,8 @@ def sensfunc_telluric(wave, counts, counts_ivar, counts_mask, exptime, airmass, # Since we are fitting a sensitivity function, first compute counts per second per angstrom. TelObj = Telluric(wave, counts, counts_ivar, mask_tot, telgridfile, teltype, obj_params, - init_sensfunc_model, eval_sensfunc_model, log10_blaze_function=log10_blaze_function, ntell=ntell, ech_orders=ech_orders, + init_sensfunc_model, eval_sensfunc_model, log10_blaze_function=log10_blaze_function, + ntell=ntell, ech_orders=ech_orders, resln_guess=resln_guess, resln_frac_bounds=resln_frac_bounds, sn_clip=sn_clip, maxiter=maxiter, lower=lower, upper=upper, tol=tol, popsize=popsize, recombination=recombination, polish=polish, disp=disp, @@ -2379,8 +2380,8 @@ def empty_model_table(norders, nspec, ntell=4, n_obj_par=0): description='Maximum wavelength included in the fit')]) def __init__(self, wave, flux, ivar, gpm, telgridfile, teltype, obj_params, init_obj_model, - eval_obj_model, log10_blaze_function=None, ech_orders=None, sn_clip=30.0, ntell=4, airmass_guess=1.5, - resln_guess=None, resln_frac_bounds=(0.5, 1.5), pix_shift_bounds=(-5.0, 5.0), + eval_obj_model, log10_blaze_function=None, ech_orders=None, sn_clip=30.0, ntell=4, + airmass_guess=1.5, resln_guess=None, resln_frac_bounds=(0.5, 1.5), pix_shift_bounds=(-5.0, 5.0), pix_stretch_bounds=(0.9,1.1), maxiter=2, sticky=True, lower=3.0, upper=3.0, seed=777, ballsize = 5e-4, tol=1e-3, diff_evol_maxiter=1000, popsize=30, recombination=0.7, polish=True, disp=False, sensfunc=False, debug=False): @@ -2402,6 +2403,7 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, teltype, obj_params, init self.obj_params = obj_params self.init_obj_model = init_obj_model self.ntell = ntell + self.airmass_guess = airmass_guess self.eval_obj_model = eval_obj_model self.ech_orders = ech_orders self.sn_clip = sn_clip From b6a4963265e2b2b8b203cbf35ee7374bbf0a24fc Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Tue, 8 Aug 2023 13:10:32 +0200 Subject: [PATCH 039/244] edit to maximum ntell in docs --- pypeit/par/pypeitpar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index 8b1bc640d7..c270652641 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -2108,7 +2108,7 @@ def __init__(self, telgridfile=None, sn_clip=None, resln_guess=None, resln_frac_ defaults['ntell'] = 4 dtypes['ntell'] = int descr['ntell'] = 'Number of fitted telluric model parameters. Must be set to 4 (default) for teltype = grid, ' \ - 'but can be set to any number from 1 to 15 for teltype = PCA, corresponding to the number of ' \ + 'but can be set to any number from 1 to 10 for teltype = PCA, corresponding to the number of ' \ 'fitted PCA coefficients.' defaults['teltype'] = 'PCA' From cb3fe2b61f64beeb7129ea11b6db119e3cd02827 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Thu, 10 Aug 2023 15:11:07 +0200 Subject: [PATCH 040/244] grid mode was broken in tellfit, added popsize/tol to tellfit pars --- pypeit/core/telluric.py | 24 ++++++++++++++++++++---- pypeit/scripts/tellfit.py | 6 ++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index d93c87b19a..d9549b42f7 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -2740,7 +2740,13 @@ def get_tell_guess(self): :obj:`tuple`: The guess telluric model parameters, resolution, shift, and stretch parameters. """ - guess = list(np.zeros(self.ntell)) + if self.teltype == 'grid': + guess = [np.median(self.tell_dict['pressure_grid'])] + guess.append(np.median(self.tell_dict['temp_grid'])) + guess.append(np.median(self.tell_dict['h2o_grid'])) + guess.append(self.airmass_guess) + else: + guess = list(np.zeros(self.ntell)) guess.append(self.resln_guess) guess.append(0.0) guess.append(1.0) @@ -2758,9 +2764,19 @@ def get_bounds_tell(self): """ # Set the bounds for the optimization bounds = [] - for ii in range(self.ntell): - bounds.append((self.tell_dict['bounds_tell_pca'][0][ii+1], - self.tell_dict['bounds_tell_pca'][1][ii+1])) + if self.teltype == 'grid': + bounds.append((self.tell_dict['pressure_grid'].min(), + self.tell_dict['pressure_grid'].max())) + bounds.append((self.tell_dict['temp_grid'].min(), + self.tell_dict['temp_grid'].max())) + bounds.append((self.tell_dict['h2o_grid'].min(), + self.tell_dict['h2o_grid'].max())) + bounds.append((self.tell_dict['airmass_grid'].min(), + self.tell_dict['airmass_grid'].max())) + else: + for ii in range(self.ntell): + bounds.append((self.tell_dict['bounds_tell_pca'][0][ii+1], + self.tell_dict['bounds_tell_pca'][1][ii+1])) bounds.append((self.resln_guess * self.resln_frac_bounds[0], self.resln_guess * self.resln_frac_bounds[1])) bounds.append(self.pix_shift_bounds) diff --git a/pypeit/scripts/tellfit.py b/pypeit/scripts/tellfit.py index 287922713f..8ec75acea5 100644 --- a/pypeit/scripts/tellfit.py +++ b/pypeit/scripts/tellfit.py @@ -163,6 +163,8 @@ def main(args): only_orders=par['telluric']['only_orders'], bal_wv_min_max=par['telluric']['bal_wv_min_max'], maxiter=par['telluric']['maxiter'], + popsize=par['telluric']['popsize'], + tol=par['telluric']['tol'], debug_init=args.debug, disp=args.debug, debug=args.debug, show=args.plot) elif par['telluric']['objmodel']=='star': @@ -184,6 +186,8 @@ def main(args): delta_coeff_bounds=par['telluric']['delta_coeff_bounds'], minmax_coeff_bounds=par['telluric']['minmax_coeff_bounds'], maxiter=par['telluric']['maxiter'], + popsize=par['telluric']['popsize'], + tol=par['telluric']['tol'], debug_init=args.debug, disp=args.debug, debug=args.debug, show=args.plot) elif par['telluric']['objmodel']=='poly': @@ -201,6 +205,8 @@ def main(args): minmax_coeff_bounds=par['telluric']['minmax_coeff_bounds'], only_orders=par['telluric']['only_orders'], maxiter=par['telluric']['maxiter'], + popsize=par['telluric']['popsize'], + tol=par['telluric']['tol'], debug_init=args.debug, disp=args.debug, debug=args.debug, show=args.plot) else: From 44c9f2a9b636c4968b6469ee761361ac37be6fab Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Tue, 15 Aug 2023 11:18:34 +0200 Subject: [PATCH 041/244] trim negative optical depths --- pypeit/core/telluric.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index d9549b42f7..252becffc0 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -442,7 +442,11 @@ def eval_telluric_pca(theta_tell, tell_dict, ind_lower=None, ind_upper=None): tell_dict['tell_pca'][:ncomp_use+1][:,ind_lower_pad:ind_upper_pad+1]) # PCA model is inverse sinh of the optical depth, convert to transmission here - tellmodel_hires = np.exp(-np.sinh(tellmodel_hires)) + tellmodel_hires = np.sinh(tellmodel_hires) + # It should generally be very rare, but trim negative optical depths here just in case. + clip = tellmodel_hires < 0 + tellmodel_hires[clip] = 0 + tellmodel_hires = np.exp(-tellmodel_hires) # Convolve transmission to given spectral resolution, and resample onto spectrum wavelengths tellmodel_conv = conv_telluric(tellmodel_hires[ind_lower_pad:ind_upper_pad+1], From 59872a681cf36cd6fe9fd087633e8d0f753eeb05 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Tue, 15 Aug 2023 11:40:34 +0200 Subject: [PATCH 042/244] better error handling for ntell --- pypeit/core/telluric.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 252becffc0..1edcdd25ed 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -422,8 +422,6 @@ def eval_telluric_pca(theta_tell, tell_dict, ind_lower=None, ind_upper=None): # Infer number of used components from the number of parameters # TODO: make this work even without shift and stretch? ncomp_use = ntheta-3 - if ncomp_use > tell_dict['ncomp_tell_pca']: - msgs.error('Asked for more PCA components than exist in PCA file.') # Set the wavelength range if not provided ind_lower = 0 if ind_lower is None else ind_lower @@ -2449,7 +2447,13 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, teltype, obj_params, init self.tell_dict = read_telluric_pca(self.telgrid, wave_min=self.wave_in_arr[wv_gpm].min(), wave_max=self.wave_in_arr[wv_gpm].max()) self.tell_model_func = eval_telluric_pca + if self.ntell > self.tell_dict['ncomp_tell_pca']: + msgs.error('Asked for more telluric PCA components ({}) ' \ + 'than exist in the PCA file ({}).'.format(self.ntell, + self.tell_dict['ncomp_tell_pca'])) elif teltype == 'grid': + if self.ntell != 4: + msgs.error('Parameter ntell must be 4 for teltype = grid') self.tell_dict = read_telluric_grid(self.telgrid, wave_min=self.wave_in_arr[wv_gpm].min(), wave_max=self.wave_in_arr[wv_gpm].max()) self.tell_model_func = eval_telluric_grid From d8e10cfbf84e5c6626a1ba5d7754fe18403a0d17 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Tue, 15 Aug 2023 14:37:00 +0200 Subject: [PATCH 043/244] adding pix_shift_bounds functionality to sensfunc --- pypeit/core/telluric.py | 3 ++- pypeit/sensfunc.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 1edcdd25ed..4801a26447 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -1352,7 +1352,8 @@ def eval_poly_model(theta, obj_dict): def sensfunc_telluric(wave, counts, counts_ivar, counts_mask, exptime, airmass, std_dict, telgridfile, teltype, log10_blaze_function=None, ech_orders=None, polyorder=8, ntell=4, mask_hydrogen_lines=True, - mask_helium_lines=False, hydrogen_mask_wid=10., resln_guess=None, resln_frac_bounds=(0.5, 1.5), + mask_helium_lines=False, hydrogen_mask_wid=10., resln_guess=None, + resln_frac_bounds=(0.5, 1.5), pix_shift_bounds=(-5.0, 5.0), delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), sn_clip=30.0, ballsize=5e-4, only_orders=None, maxiter=3, lower=3.0, upper=3.0, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, diff --git a/pypeit/sensfunc.py b/pypeit/sensfunc.py index e662a4f33e..ac828c1044 100644 --- a/pypeit/sensfunc.py +++ b/pypeit/sensfunc.py @@ -886,6 +886,7 @@ def compute_zeropoint(self): ech_orders=self.meta_spec['ECH_ORDERS'], resln_guess=self.par['IR']['resln_guess'], resln_frac_bounds=self.par['IR']['resln_frac_bounds'], + pix_shift_bounds=self.par['IR']['pix_shift_bounds'], sn_clip=self.par['IR']['sn_clip'], ntell=self.par['IR']['ntell'], mask_hydrogen_lines=self.par['mask_hydrogen_lines'], From 1523dd21247edf2b9c3dd2638095bda32709da80 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Wed, 16 Aug 2023 10:12:34 +0200 Subject: [PATCH 044/244] increase pixel shift bounds for HIRES to accommodate helio corr --- pypeit/core/telluric.py | 5 ++--- pypeit/spectrographs/keck_hires.py | 4 ++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 4801a26447..ecdd7e393a 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -1351,9 +1351,8 @@ def eval_poly_model(theta, obj_dict): def sensfunc_telluric(wave, counts, counts_ivar, counts_mask, exptime, airmass, std_dict, telgridfile, teltype, log10_blaze_function=None, ech_orders=None, polyorder=8, ntell=4, - mask_hydrogen_lines=True, - mask_helium_lines=False, hydrogen_mask_wid=10., resln_guess=None, - resln_frac_bounds=(0.5, 1.5), pix_shift_bounds=(-5.0, 5.0), + mask_hydrogen_lines=True, mask_helium_lines=False, hydrogen_mask_wid=10., + resln_guess=None, resln_frac_bounds=(0.5, 1.5), pix_shift_bounds=(-5.0, 5.0), delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), sn_clip=30.0, ballsize=5e-4, only_orders=None, maxiter=3, lower=3.0, upper=3.0, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, diff --git a/pypeit/spectrographs/keck_hires.py b/pypeit/spectrographs/keck_hires.py index 7d24d179c8..683b11ec3e 100644 --- a/pypeit/spectrographs/keck_hires.py +++ b/pypeit/spectrographs/keck_hires.py @@ -152,6 +152,10 @@ def default_pypeit_par(cls): par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 5 #[9, 11, 11, 9, 9, 8, 8, 7, 7, 7, 7, 7, 7, 7, 7] par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_10500_R60000.fits' + par['sensfunc']['IR']['pix_shift_bounds'] = (-30.0,30.0) + + # Telluric parameters + par['telluric']['pix_shift_bounds'] = (-30.0,30.0) # Coadding par['coadd1d']['wave_method'] = 'log10' From 1ac3bcb55b8a335a1f2a1453dbc9d420989bf765 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Wed, 16 Aug 2023 14:02:22 +0200 Subject: [PATCH 045/244] hires defaults --- pypeit/core/telluric.py | 3 ++- pypeit/spectrographs/keck_hires.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index ecdd7e393a..8c98fe1b92 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -351,6 +351,7 @@ def conv_telluric(tell_model, dloglam, res): def shift_telluric(tell_model, loglam, dloglam, shift, stretch): """ Routine to apply a shift to the telluric model. Note that the shift can be sub-pixel, i.e this routine interpolates. + Note that the shift units are pixels *of the telluric model*. Args: tell_model (`numpy.ndarray`_): @@ -1519,7 +1520,7 @@ def sensfunc_telluric(wave, counts, counts_ivar, counts_mask, exptime, airmass, # Since we are fitting a sensitivity function, first compute counts per second per angstrom. TelObj = Telluric(wave, counts, counts_ivar, mask_tot, telgridfile, teltype, obj_params, init_sensfunc_model, eval_sensfunc_model, log10_blaze_function=log10_blaze_function, - ntell=ntell, ech_orders=ech_orders, + ntell=ntell, ech_orders=ech_orders, pix_shift_bounds=pix_shift_bounds, resln_guess=resln_guess, resln_frac_bounds=resln_frac_bounds, sn_clip=sn_clip, maxiter=maxiter, lower=lower, upper=upper, tol=tol, popsize=popsize, recombination=recombination, polish=polish, disp=disp, diff --git a/pypeit/spectrographs/keck_hires.py b/pypeit/spectrographs/keck_hires.py index 683b11ec3e..3f2f537914 100644 --- a/pypeit/spectrographs/keck_hires.py +++ b/pypeit/spectrographs/keck_hires.py @@ -151,11 +151,11 @@ def default_pypeit_par(cls): # Sensitivity function parameters par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 5 #[9, 11, 11, 9, 9, 8, 8, 7, 7, 7, 7, 7, 7, 7, 7] - par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_10500_R60000.fits' - par['sensfunc']['IR']['pix_shift_bounds'] = (-30.0,30.0) + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_10500_R120000.fits' + par['sensfunc']['IR']['pix_shift_bounds'] = (-40.0,40.0) # Telluric parameters - par['telluric']['pix_shift_bounds'] = (-30.0,30.0) + par['telluric']['pix_shift_bounds'] = (-40.0,40.0) # Coadding par['coadd1d']['wave_method'] = 'log10' From 6f93ac184b0ddb9cbe6beadf4cf9593aec24f245 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Wed, 16 Aug 2023 14:09:07 +0200 Subject: [PATCH 046/244] pix_shift_bounds slightly larger for R25000 model --- pypeit/core/telluric.py | 17 ++++++++++------- pypeit/spectrographs/keck_nirspec.py | 5 +++++ pypeit/spectrographs/vlt_xshooter.py | 13 ++++++++++++- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 8c98fe1b92..09d6329919 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -1567,7 +1567,7 @@ def qso_telluric(spec1dfile, telgridfile, teltype, pca_file, z_qso, telloutfile, pca_lower=1220.0, pca_upper=3100.0, bal_wv_min_max=None, delta_zqso=0.1, ntell=4, bounds_norm=(0.1, 3.0), tell_norm_thresh=0.9, sn_clip=30.0, only_orders=None, maxiter=3, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, - debug_init=False, debug=False, show=False): + pix_shift_bounds=(-5.0,5.0), debug_init=False, debug=False, show=False): """ Fit and correct a QSO spectrum for telluric absorption. @@ -1686,7 +1686,8 @@ def qso_telluric(spec1dfile, telgridfile, teltype, pca_file, z_qso, telloutfile, # parameters lowered for testing TelObj = Telluric(wave, flux, ivar, mask_tot, telgridfile, teltype, obj_params, init_qso_model, - eval_qso_model, sn_clip=sn_clip, maxiter=maxiter, tol=tol, popsize=popsize, + eval_qso_model, pix_shift_bounds=pix_shift_bounds, + sn_clip=sn_clip, maxiter=maxiter, tol=tol, popsize=popsize, ntell=ntell, recombination=recombination, polish=polish, disp=disp, debug=debug) TelObj.run(only_orders=only_orders) TelObj.to_file(telloutfile, overwrite=True) @@ -1736,7 +1737,7 @@ def star_telluric(spec1dfile, telgridfile, teltype, telloutfile, outfile, star_t hydrogen_mask_wid=10., delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), only_orders=None, sn_clip=30.0, maxiter=3, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, - debug_init=False, debug=False, show=False): + pix_shift_bounds=(-5.0,5.0), debug_init=False, debug=False, show=False): """ This needs a doc string. @@ -1793,7 +1794,8 @@ def star_telluric(spec1dfile, telgridfile, teltype, telloutfile, outfile, star_t # parameters lowered for testing TelObj = Telluric(wave, flux, ivar, mask_tot, telgridfile, teltype, obj_params, init_star_model, - eval_star_model, ntell=ntell, sn_clip=sn_clip, tol=tol, popsize=popsize, + eval_star_model, pix_shift_bounds=pix_shift_bounds, + ntell=ntell, sn_clip=sn_clip, tol=tol, popsize=popsize, recombination=recombination, polish=polish, disp=disp, debug=debug) TelObj.run(only_orders=only_orders) TelObj.to_file(telloutfile, overwrite=True) @@ -1841,8 +1843,8 @@ def poly_telluric(spec1dfile, telgridfile, teltype, telloutfile, outfile, z_obj= model='exp', polyorder=3, fit_wv_min_max=None, mask_lyman_a=True, ntell=4, delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), only_orders=None, sn_clip=30.0, maxiter=3, tol=1e-3, popsize=30, - recombination=0.7, polish=True, disp=False, debug_init=False, debug=False, - show=False): + recombination=0.7, polish=True, disp=False, pix_shift_bounds=(-5.0,5.0), + debug_init=False, debug=False, show=False): """ This needs a doc string. @@ -1891,7 +1893,8 @@ def poly_telluric(spec1dfile, telgridfile, teltype, telloutfile, outfile, z_obj= # parameters lowered for testing TelObj = Telluric(wave, flux, ivar, mask_tot, telgridfile, teltype, obj_params, init_poly_model, - eval_poly_model, sn_clip=sn_clip, maxiter=maxiter, tol=tol, popsize=popsize, + eval_poly_model, pix_shift_bounds=pix_shift_bounds, + sn_clip=sn_clip, maxiter=maxiter, tol=tol, popsize=popsize, ntell=ntell, recombination=recombination, polish=polish, disp=disp, debug=debug) TelObj.run(only_orders=only_orders) TelObj.to_file(telloutfile, overwrite=True) diff --git a/pypeit/spectrographs/keck_nirspec.py b/pypeit/spectrographs/keck_nirspec.py index cc6ab587b1..71decc0600 100644 --- a/pypeit/spectrographs/keck_nirspec.py +++ b/pypeit/spectrographs/keck_nirspec.py @@ -130,6 +130,11 @@ def default_pypeit_par(cls): par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 8 par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R25000.fits' + par['sensfunc']['IR']['pix_shift_bounds'] = (-8.0,8.0) + + # Telluric parameters + par['telluric']['pix_shift_bounds'] = (-8.0,8.0) + return par def init_meta(self): diff --git a/pypeit/spectrographs/vlt_xshooter.py b/pypeit/spectrographs/vlt_xshooter.py index 9ad0913a93..184c2d41e5 100644 --- a/pypeit/spectrographs/vlt_xshooter.py +++ b/pypeit/spectrographs/vlt_xshooter.py @@ -345,6 +345,10 @@ def default_pypeit_par(cls): par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 8 par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R25000.fits' + par['sensfunc']['IR']['pix_shift_bounds'] = (-8.0,8.0) + + # Telluric parameters + par['telluric']['pix_shift_bounds'] = (-8.0,8.0) # Coadding par['coadd1d']['wave_method'] = 'log10' @@ -721,6 +725,10 @@ def default_pypeit_par(cls): par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 8 #[9, 11, 11, 9, 9, 8, 8, 7, 7, 7, 7, 7, 7, 7, 7] par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R25000.fits' + par['sensfunc']['IR']['pix_shift_bounds'] = (-8.0,8.0) + + # Telluric parameters + par['telluric']['pix_shift_bounds'] = (-8.0,8.0) # Coadding par['coadd1d']['wave_method'] = 'log10' @@ -1010,7 +1018,10 @@ def default_pypeit_par(cls): par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 8 par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R25000.fits' - # This is a hack until we we have a Paranal file generated that covers the UVB wavelength range. + par['sensfunc']['IR']['pix_shift_bounds'] = (-8.0,8.0) + + # Telluric parameters + par['telluric']['pix_shift_bounds'] = (-8.0,8.0) # Coadding par['coadd1d']['wave_method'] = 'log10' From aefd1392b33e10c53cb780952065a28cd8907024 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Fri, 18 Aug 2023 09:45:04 +0200 Subject: [PATCH 047/244] expanded default resln_frac_bounds to accommodate oversampled spectrographs --- pypeit/core/telluric.py | 4 ++-- pypeit/par/pypeitpar.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 09d6329919..39b933e1dc 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -1353,7 +1353,7 @@ def eval_poly_model(theta, obj_dict): def sensfunc_telluric(wave, counts, counts_ivar, counts_mask, exptime, airmass, std_dict, telgridfile, teltype, log10_blaze_function=None, ech_orders=None, polyorder=8, ntell=4, mask_hydrogen_lines=True, mask_helium_lines=False, hydrogen_mask_wid=10., - resln_guess=None, resln_frac_bounds=(0.5, 1.5), pix_shift_bounds=(-5.0, 5.0), + resln_guess=None, resln_frac_bounds=(0.3, 1.5), pix_shift_bounds=(-5.0, 5.0), delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), sn_clip=30.0, ballsize=5e-4, only_orders=None, maxiter=3, lower=3.0, upper=3.0, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, @@ -2387,7 +2387,7 @@ def empty_model_table(norders, nspec, ntell=4, n_obj_par=0): def __init__(self, wave, flux, ivar, gpm, telgridfile, teltype, obj_params, init_obj_model, eval_obj_model, log10_blaze_function=None, ech_orders=None, sn_clip=30.0, ntell=4, - airmass_guess=1.5, resln_guess=None, resln_frac_bounds=(0.5, 1.5), pix_shift_bounds=(-5.0, 5.0), + airmass_guess=1.5, resln_guess=None, resln_frac_bounds=(0.3, 1.5), pix_shift_bounds=(-5.0, 5.0), pix_stretch_bounds=(0.9,1.1), maxiter=2, sticky=True, lower=3.0, upper=3.0, seed=777, ballsize = 5e-4, tol=1e-3, diff_evol_maxiter=1000, popsize=30, recombination=0.7, polish=True, disp=False, sensfunc=False, debug=False): diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index f833b9a32e..025cb27ae5 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -2137,12 +2137,12 @@ def __init__(self, telgridfile=None, sn_clip=None, resln_guess=None, resln_frac_ pars['resln_frac_bounds'] = tuple_force(pars['resln_frac_bounds']) - defaults['resln_frac_bounds'] = (0.5,1.5) + defaults['resln_frac_bounds'] = (0.3,1.5) dtypes['resln_frac_bounds'] = tuple descr['resln_frac_bounds'] = 'Bounds for the resolution fit optimization which is part of the telluric model. ' \ - 'This range is in units of the resln_guess, so the (0.5, 1.5) would bound the ' \ + 'This range is in units of the resln_guess, so the (0.3, 1.5) would bound the ' \ 'spectral resolution fit to be within the range ' \ - 'bounds_resln = (0.5*resln_guess, 1.5*resln_guess)' + 'bounds_resln = (0.3*resln_guess, 1.5*resln_guess)' pars['pix_shift_bounds'] = tuple_force(pars['pix_shift_bounds']) defaults['pix_shift_bounds'] = (-5.0,5.0) From 4a0c7438ee19fd81439e1c30ba7a6d0c60bccd8f Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Thu, 7 Sep 2023 21:02:41 +0200 Subject: [PATCH 048/244] addressing PR comments --- CHANGES.rst | 2 - pypeit/core/telluric.py | 88 +++++++++++++++++++---------------------- 2 files changed, 40 insertions(+), 50 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 19fa602e34..ce65745152 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -42,8 +42,6 @@ - Refactored ``load_line_lists()`` yet again! - Introduced PCA method for telluric corrections ->>>>>>> develop - 1.13.0 (2 June 2023) -------------------- diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 39b933e1dc..e0389a0c72 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -146,10 +146,6 @@ def read_telluric_pca(filename, wave_min=None, wave_max=None, pad_frac=0.10): Optionally, this method also trims wavelength to be in within ``wave_min`` and ``wave_max`` and pads the data (see ``pad_frac``). - .. todo:: - List and describe the contents of the dictionary in the return - description. - Args: filename (:obj:`str`): Telluric PCA filename @@ -165,15 +161,13 @@ def read_telluric_pca(filename, wave_min=None, wave_max=None, pad_frac=0.10): Returns: :obj:`dict`: Dictionary containing the telluric PCA components. - - wave_grid= - - dloglam= - - resln_guess= - - pix_per_sigma= - - tell_pad_pix= - - ncomp_tell_pca= - - tell_pca= - - bounds_tell_pca= - - coefs_tell_pca= + - wave_grid: Telluric model wavelength grid + - dloglam: Wavelength sampling of telluric model + - tell_pad_pix: Number of pixels to pad at edges for convolution + - ncomp_tell_pca: Number of PCA components + - tell_pca: PCA component vectors + - bounds_tell_pca: Maximum/minimum coefficient + - coefs_tell_pca: Set of model coefficient values (for prior in future) """ # load_telluric_grid() takes care of path and existance check hdul = data.load_telluric_grid(filename) @@ -193,21 +187,18 @@ def read_telluric_pca(filename, wave_min=None, wave_max=None, pad_frac=0.10): dwave, dloglam, resln_guess, pix_per_sigma = wvutils.get_sampling(wave_grid) tell_pad_pix = int(np.ceil(10.0 * pix_per_sigma)) - return dict(wave_grid=wave_grid, dloglam=dloglam, resln_guess=resln_guess, - pix_per_sigma=pix_per_sigma, tell_pad_pix=tell_pad_pix, ncomp_tell_pca=ncomp, - tell_pca=pca_comp_grid, bounds_tell_pca=bounds, coefs_tell_pca=model_coefs) + return dict(wave_grid=wave_grid, dloglam=dloglam, + tell_pad_pix=tell_pad_pix, ncomp_tell_pca=ncomp, + tell_pca=pca_comp_grid, bounds_tell_pca=bounds, coefs_tell_pca=model_coefs) def read_telluric_grid(filename, wave_min=None, wave_max=None, pad_frac=0.10): """ - Reads in the telluric grid from a file. + Reads in the telluric grid from a file. This method is no longer the + preferred approach; see "read_telluric_pca" and other functions. Optionally, this method also trims the grid to be in within ``wave_min`` and ``wave_max`` and pads the data (see ``pad_frac``). - .. todo:: - List and describe the contents of the dictionary in the return - description. - Args: filename (:obj:`str`): Telluric grid filename @@ -223,16 +214,14 @@ def read_telluric_grid(filename, wave_min=None, wave_max=None, pad_frac=0.10): Returns: :obj:`dict`: Dictionary containing the telluric grid - - wave_grid= - - dloglam= - - resln_guess= - - pix_per_sigma= - - tell_pad_pix= - - pressure_grid= - - temp_grid= - - h2o_grid= - - airmass_grid= - - tell_grid= + - wave_grid: Telluric model wavelength grid + - dloglam: Wavelength sampling of telluric model + - tell_pad_pix: Number of pixels to pad at edges for convolution + - pressure_grid: Atmospheric pressure values in telluric grid [mbar] + - temp_grid: Temperature values in telluric grid [degrees C] + - h2o_grid: Humidity values in telluric grid [%] + - airmass_grid: Airmass values in telluric grid + - tell_grid: Grid of telluric models """ # load_telluric_grid() takes care of path and existance check hdul = data.load_telluric_grid(filename) @@ -258,11 +247,8 @@ def read_telluric_grid(filename, wave_min=None, wave_max=None, pad_frac=0.10): dwave, dloglam, resln_guess, pix_per_sigma = wvutils.get_sampling(wave_grid) tell_pad_pix = int(np.ceil(10.0 * pix_per_sigma)) - return dict(wave_grid=wave_grid, dloglam=dloglam, - resln_guess=resln_guess, - pix_per_sigma=pix_per_sigma, tell_pad_pix=tell_pad_pix, pressure_grid=pg, temp_grid=tg, @@ -378,7 +364,6 @@ def shift_telluric(tell_model, loglam, dloglam, shift, stretch): tell_model_shift = np.interp(loglam_shift, loglam, tell_model) return tell_model_shift - def eval_telluric_pca(theta_tell, tell_dict, ind_lower=None, ind_upper=None): """ Evaluate the telluric PCA model. @@ -400,7 +385,8 @@ def eval_telluric_pca(theta_tell, tell_dict, ind_lower=None, ind_upper=None): Args: theta_tell (`numpy.ndarray`_): - Vector with the telluric PCA coefficients + Vector with ntell PCA coefficients, followed by spectral resolution, + shift, and stretch. Final length is ntell+3. tell_dict (:obj:`dict`): Dictionary containing the telluric PCA data. See :func:`read_telluric_pca`. @@ -604,7 +590,12 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): The telluric model theta_tell includes a either user-specified number of PCA coefficients (in PCA mode) or ambient pressure, temperature, humidity, and airmass (in grid mode) as well as - spectral resolution, shift, and stretch. + spectral resolution, shift, and stretch. That is, in PCA mode, + + pca_coeffs = theta_tell[:ntell] + resolution = theta_tell[ntell] + shift = theta_tell[ntell+1] + stretch = theta_tell[ntell+2] The object model theta_obj can have an arbitrary size and is provided as an argument to obj_model_func @@ -1563,8 +1554,9 @@ def create_bal_mask(wave, bal_wv_min_max): -def qso_telluric(spec1dfile, telgridfile, teltype, pca_file, z_qso, telloutfile, outfile, npca=8, - pca_lower=1220.0, pca_upper=3100.0, bal_wv_min_max=None, delta_zqso=0.1, ntell=4, +def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile, npca=8, + pca_lower=1220.0, pca_upper=3100.0, bal_wv_min_max=None, delta_zqso=0.1, + teltype='PCA', ntell=4, bounds_norm=(0.1, 3.0), tell_norm_thresh=0.9, sn_clip=30.0, only_orders=None, maxiter=3, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, pix_shift_bounds=(-5.0,5.0), debug_init=False, debug=False, show=False): @@ -1731,10 +1723,10 @@ def qso_telluric(spec1dfile, telgridfile, teltype, pca_file, z_qso, telloutfile, return TelObj -def star_telluric(spec1dfile, telgridfile, teltype, telloutfile, outfile, star_type=None, +def star_telluric(spec1dfile, telgridfile, telloutfile, outfile, star_type=None, star_mag=None, star_ra=None, star_dec=None, func='legendre', model='exp', - polyorder=5, ntell=4, mask_hydrogen_lines=True, mask_helium_lines=False, - hydrogen_mask_wid=10., delta_coeff_bounds=(-20.0, 20.0), + polyorder=5, teltype='PCA', ntell=4, mask_hydrogen_lines=True, + mask_helium_lines=False, hydrogen_mask_wid=10., delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), only_orders=None, sn_clip=30.0, maxiter=3, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, pix_shift_bounds=(-5.0,5.0), debug_init=False, debug=False, show=False): @@ -1839,9 +1831,9 @@ def star_telluric(spec1dfile, telgridfile, teltype, telloutfile, outfile, star_t return TelObj -def poly_telluric(spec1dfile, telgridfile, teltype, telloutfile, outfile, z_obj=0.0, func='legendre', - model='exp', polyorder=3, fit_wv_min_max=None, mask_lyman_a=True, ntell=4, - delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), +def poly_telluric(spec1dfile, telgridfile, telloutfile, outfile, z_obj=0.0, func='legendre', + model='exp', polyorder=3, fit_wv_min_max=None, mask_lyman_a=True, teltype='PCA', + ntell=4, delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), only_orders=None, sn_clip=30.0, maxiter=3, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, pix_shift_bounds=(-5.0,5.0), debug_init=False, debug=False, show=False): @@ -2385,8 +2377,8 @@ def empty_model_table(norders, nspec, ntell=4, n_obj_par=0): table.Column(name='WAVE_MAX', dtype=float, length=norders, description='Maximum wavelength included in the fit')]) - def __init__(self, wave, flux, ivar, gpm, telgridfile, teltype, obj_params, init_obj_model, - eval_obj_model, log10_blaze_function=None, ech_orders=None, sn_clip=30.0, ntell=4, + def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_model, eval_obj_model, + log10_blaze_function=None, ech_orders=None, sn_clip=30.0, teltype='PCA', ntell=4, airmass_guess=1.5, resln_guess=None, resln_frac_bounds=(0.3, 1.5), pix_shift_bounds=(-5.0, 5.0), pix_stretch_bounds=(0.9,1.1), maxiter=2, sticky=True, lower=3.0, upper=3.0, seed=777, ballsize = 5e-4, tol=1e-3, diff_evol_maxiter=1000, popsize=30, @@ -2447,7 +2439,7 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, teltype, obj_params, init wave, flux, ivar, gpm) # 3) Read the telluric grid and initalize associated parameters wv_gpm = self.wave_in_arr > 1.0 - if teltype == 'pca' or teltype == 'PCA': + if upper(teltype) == 'PCA': self.tell_dict = read_telluric_pca(self.telgrid, wave_min=self.wave_in_arr[wv_gpm].min(), wave_max=self.wave_in_arr[wv_gpm].max()) self.tell_model_func = eval_telluric_pca From ba340c18a80a053a2d664543d16ac01c7aa5b6f9 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Thu, 7 Sep 2023 21:25:54 +0200 Subject: [PATCH 049/244] addressing PR comments --- pypeit/core/telluric.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index e0389a0c72..adccdadb25 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -402,8 +402,8 @@ def eval_telluric_pca(theta_tell, tell_dict, ind_lower=None, ind_upper=None): needed for the current model fit. Returns: - `numpy.ndarray`_: Telluric model evaluated at the desired location - theta_tell in model atmosphere parameter space. + `numpy.ndarray`_: Telluric model evaluated from the PCA coefficients + given by theta_tell, with convolution, shift, and stretch applied. """ ntheta = len(theta_tell) # Infer number of used components from the number of parameters From 7943a6ab4a9783a7ce8b292abcbdf45aaacae21a Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Thu, 7 Sep 2023 21:28:42 +0200 Subject: [PATCH 050/244] addressing PR comments --- pypeit/core/telluric.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index adccdadb25..5e63515692 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -589,13 +589,25 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): The telluric model theta_tell includes a either user-specified number of PCA coefficients (in PCA mode) or ambient pressure, - temperature, humidity, and airmass (in grid mode) as well as - spectral resolution, shift, and stretch. That is, in PCA mode, + temperature, humidity, and airmass (in grid mode) followed by + spectral resolution, shift, and stretch. + + That is, in PCA mode, pca_coeffs = theta_tell[:ntell] - resolution = theta_tell[ntell] - shift = theta_tell[ntell+1] - stretch = theta_tell[ntell+2] + + while in grid mode, + + pressure = theta_tell[0] + temperature = theta_tell[1] + humidity = theta_tell[2] + airmass = theta_tell[3] + + with the last three indices of the array corresponding to + + resolution = theta_tell[-3] + shift = theta_tell[-2] + stretch = theta_tell[-1] The object model theta_obj can have an arbitrary size and is provided as an argument to obj_model_func From 8be493cc8702534c3483aed7cca3c01d117c9669 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Thu, 7 Sep 2023 21:39:22 +0200 Subject: [PATCH 051/244] addressing PR comments --- pypeit/core/telluric.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 5e63515692..9ba68ae33a 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -369,8 +369,7 @@ def eval_telluric_pca(theta_tell, tell_dict, ind_lower=None, ind_upper=None): Evaluate the telluric PCA model. The parameters provided by ``theta_tell`` must be: ntell PCA coefficients, - spectral resolution, shift, and stretch. The latter two can be omitted if - a shift and stretch of the telluric model are not included. + spectral resolution, shift, and stretch. This routine performs the following steps: @@ -381,7 +380,7 @@ def eval_telluric_pca(theta_tell, tell_dict, ind_lower=None, ind_upper=None): 3. convolution of the atmosphere model to the resolution set by the spectral resolution. - 4. (Optional) shift and stretch the telluric model. + 4. shift and stretch the telluric model. Args: theta_tell (`numpy.ndarray`_): From 08105beb6c0e488a08cc6971e4c30cc6d808f83c Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Fri, 8 Sep 2023 14:39:24 +0200 Subject: [PATCH 052/244] unified the telluric evaluators --- pypeit/core/telluric.py | 175 ++++++++++++---------------------------- 1 file changed, 52 insertions(+), 123 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 9ba68ae33a..fc0ad5ae6c 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -189,7 +189,8 @@ def read_telluric_pca(filename, wave_min=None, wave_max=None, pad_frac=0.10): return dict(wave_grid=wave_grid, dloglam=dloglam, tell_pad_pix=tell_pad_pix, ncomp_tell_pca=ncomp, - tell_pca=pca_comp_grid, bounds_tell_pca=bounds, coefs_tell_pca=model_coefs) + tell_pca=pca_comp_grid, bounds_tell_pca=bounds, + coefs_tell_pca=model_coefs, teltype='PCA') def read_telluric_grid(filename, wave_min=None, wave_max=None, pad_frac=0.10): """ @@ -247,14 +248,9 @@ def read_telluric_grid(filename, wave_min=None, wave_max=None, pad_frac=0.10): dwave, dloglam, resln_guess, pix_per_sigma = wvutils.get_sampling(wave_grid) tell_pad_pix = int(np.ceil(10.0 * pix_per_sigma)) - return dict(wave_grid=wave_grid, - dloglam=dloglam, - tell_pad_pix=tell_pad_pix, - pressure_grid=pg, - temp_grid=tg, - h2o_grid=hg, - airmass_grid=ag, - tell_grid=model_grid) + return dict(wave_grid=wave_grid, dloglam=dloglam, tell_pad_pix=tell_pad_pix, + pressure_grid=pg, temp_grid=tg, h2o_grid=hg, airmass_grid=ag, + tell_grid=model_grid, teltype='grid') def interp_telluric_grid(theta, tell_dict): @@ -364,12 +360,14 @@ def shift_telluric(tell_model, loglam, dloglam, shift, stretch): tell_model_shift = np.interp(loglam_shift, loglam, tell_model) return tell_model_shift -def eval_telluric_pca(theta_tell, tell_dict, ind_lower=None, ind_upper=None): +def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None):) """ - Evaluate the telluric PCA model. + Evaluate the telluric model. - The parameters provided by ``theta_tell`` must be: ntell PCA coefficients, - spectral resolution, shift, and stretch. + Parameters from ``theta_tell`` are: (if teltype == 'PCA') the PCA coefficients + or (if teltype == 'grid') the telluric grid parameters pressure, temperature, + humidity, and airmass, in both cases followed by spectral resolution, shift, + and stretch. This routine performs the following steps: @@ -380,15 +378,20 @@ def eval_telluric_pca(theta_tell, tell_dict, ind_lower=None, ind_upper=None): 3. convolution of the atmosphere model to the resolution set by the spectral resolution. - 4. shift and stretch the telluric model. + 4. Shift and stretch the telluric model. Args: theta_tell (`numpy.ndarray`_): - Vector with ntell PCA coefficients, followed by spectral resolution, - shift, and stretch. Final length is ntell+3. + Vector with ntell PCA coefficients (if teltype='PCA') + or pressure, temperature, humidity, and airmass (if teltype='grid'), + followed by spectral resolution, shift, and stretch. + Final length is ntell+3. tell_dict (:obj:`dict`): - Dictionary containing the telluric PCA data. See - :func:`read_telluric_pca`. + Dictionary containing the telluric data. See + :func:`read_telluric_pca` if teltype=='PCA'. + or + :func:`read_telluric_grid` if teltype=='grid'. + teltype (: ind_lower (:obj:`int`, optional): The index of the first pixel to include in the model. Selecting a wavelength region for the modeling makes things faster because we @@ -401,14 +404,14 @@ def eval_telluric_pca(theta_tell, tell_dict, ind_lower=None, ind_upper=None): needed for the current model fit. Returns: - `numpy.ndarray`_: Telluric model evaluated from the PCA coefficients - given by theta_tell, with convolution, shift, and stretch applied. + `numpy.ndarray`_: Telluric model evaluated at the desired location + theta_tell in model atmosphere parameter space. """ ntheta = len(theta_tell) - # Infer number of used components from the number of parameters - # TODO: make this work even without shift and stretch? - ncomp_use = ntheta-3 - + # FD: Currently assumes that shift and stretch are on. + # TODO: Make this work without shift and stretch. + ntell = ntheta-3 + # Set the wavelength range if not provided ind_lower = 0 if ind_lower is None else ind_lower ind_upper = tell_dict['wave_grid'].size - 1 if ind_upper is None else ind_upper @@ -419,100 +422,33 @@ def eval_telluric_pca(theta_tell, tell_dict, ind_lower=None, ind_upper=None): ## FW: There is an extreme case with ind_upper == ind_upper_pad, the previous -0 won't work ind_lower_final = ind_lower_pad if ind_lower_pad == ind_lower else ind_lower - ind_lower_pad ind_upper_final = ind_upper_pad if ind_upper_pad == ind_upper else ind_upper - ind_upper_pad - - # Evaluate PCA model after truncating the wavelength range - tellmodel_hires = np.zeros_like(tell_dict['tell_pca'][0]) - tellmodel_hires[ind_lower_pad:ind_upper_pad+1] = np.dot(np.append(1,theta_tell[:ncomp_use]), - tell_dict['tell_pca'][:ncomp_use+1][:,ind_lower_pad:ind_upper_pad+1]) - - # PCA model is inverse sinh of the optical depth, convert to transmission here - tellmodel_hires = np.sinh(tellmodel_hires) - # It should generally be very rare, but trim negative optical depths here just in case. - clip = tellmodel_hires < 0 - tellmodel_hires[clip] = 0 - tellmodel_hires = np.exp(-tellmodel_hires) - - # Convolve transmission to given spectral resolution, and resample onto spectrum wavelengths + + if tell_dict['teltype'] == 'PCA': + # Evaluate PCA model after truncating the wavelength range + tellmodel_hires = np.zeros_like(tell_dict['tell_pca'][0]) + tellmodel_hires[ind_lower_pad:ind_upper_pad+1] = np.dot(np.append(1,theta_tell[:ncomp_use]), + tell_dict['tell_pca'][:ntell+1][:,ind_lower_pad:ind_upper_pad+1]) + + # PCA model is inverse sinh of the optical depth, convert to transmission here + tellmodel_hires = np.sinh(tellmodel_hires) + # It should generally be very rare, but trim negative optical depths here just in case. + clip = tellmodel_hires < 0 + tellmodel_hires[clip] = 0 + tellmodel_hires = np.exp(-tellmodel_hires) + elif tell_dict['teltype'] == 'grid': + # Interpolate within the telluric grid + tellmodel_hires = interp_telluric_grid(theta_tell[:ntell], tell_dict) + else: + msgs.error("Unsupported teltype, must be 'PCA' or 'grid'") + tellmodel_conv = conv_telluric(tellmodel_hires[ind_lower_pad:ind_upper_pad+1], tell_dict['dloglam'], theta_tell[-3]) - + # Stretch and shift telluric wavelength grid - # FD: currently assumes shift + stretch is on... tellmodel_out = shift_telluric(tellmodel_conv, np.log10(tell_dict['wave_grid'][ind_lower_pad:ind_upper_pad+1]), tell_dict['dloglam'], theta_tell[-2], theta_tell[-1]) - - return tellmodel_out[ind_lower_final:ind_upper_final] - -def eval_telluric_grid(theta_tell, tell_dict, ind_lower=None, ind_upper=None): - """ - Evaluate the telluric model at an arbitrary location in parameter space. - - The full atmosphere model parameter space is either 5 or 7 dimensional, - which is the size of the ``theta_tell`` input parameter vector. - - The parameters provided by ``theta_tell`` must be: pressure, temperature, - humidity, airmass, spectral resolution, shift, and stretch. The latter two - can be omitted if a shift and stretch of the telluric model are not - included. - - This routine performs the following steps: - - 1. nearest grid point interpolation of the telluric model onto a - new location in the 4-d space (pressure, temperature, - humidity, airmass) - - 2. convolution of the atmosphere model to the resolution set by - the spectral resolution. - - 3. (Optional) shift and stretch the telluric model. - Args: - theta_tell (`numpy.ndarray`_): - Vector with the telluric model parameters. Must be 5 or 7 - elements long. See method description. - tell_dict (:obj:`dict`): - Dictionary containing the telluric grid data. See - :func:`read_telluric_grid`. - ind_lower (:obj:`int`, optional): - The index of the first pixel to include in the model. Selecting a - wavelength region for the modeling makes things faster because we - only need to convolve the portion that is needed for the current - model fit. - ind_upper: - The index (inclusive) of the last pixel to include in the model. - Selecting a wavelength region for the modeling makes things - faster because we only need to convolve the portion that is - needed for the current model fit. - - Returns: - `numpy.ndarray`_: Telluric model evaluated at the desired location - theta_tell in model atmosphere parameter space. - """ - ntheta = len(theta_tell) - if ntheta not in [5, 7]: - msgs.error('Input model atmosphere parameters must have length 5 or 7.') - - tellmodel_hires = interp_telluric_grid(theta_tell[:4], tell_dict) - - # Set the wavelength range if not provided - ind_lower = 0 if ind_lower is None else ind_lower - ind_upper = tell_dict['wave_grid'].size - 1 if ind_upper is None else ind_upper - - # Deal with padding for the convolutions - ind_lower_pad = np.fmax(ind_lower - tell_dict['tell_pad_pix'], 0) - ind_upper_pad = np.fmin(ind_upper + tell_dict['tell_pad_pix'], tell_dict['wave_grid'].size - 1) - ## FW: There is an extreme case with ind_upper == ind_upper_pad, the previous -0 won't work - ind_lower_final = ind_lower_pad if ind_lower_pad == ind_lower else ind_lower - ind_lower_pad - ind_upper_final = ind_upper_pad if ind_upper_pad == ind_upper else ind_upper - ind_upper_pad - tellmodel_conv = conv_telluric(tellmodel_hires[ind_lower_pad:ind_upper_pad+1], - tell_dict['dloglam'], theta_tell[4]) - if ntheta == 5: - return tellmodel_conv[ind_lower_final:ind_upper_final] - - tellmodel_out = shift_telluric(tellmodel_conv, - np.log10(tell_dict['wave_grid'][ind_lower_pad:ind_upper_pad+1]), - tell_dict['dloglam'], theta_tell[5], theta_tell[6]) return tellmodel_out[ind_lower_final:ind_upper_final] ############################ @@ -544,7 +480,6 @@ def tellfit_chi2(theta, flux, thismask, arg_dict): obj_model_func = arg_dict['obj_model_func'] - tell_model_func = arg_dict['tell_model_func'] flux_ivar = arg_dict['ivar'] # TODO: make this work without shift and stretch? @@ -554,7 +489,7 @@ def tellfit_chi2(theta, flux, thismask, arg_dict): theta_obj = theta[:-nfit] theta_tell = theta[-nfit:] - tell_model = tell_model_func(theta_tell, arg_dict['tell_dict'], + tell_model = eval_telluric(theta_tell, arg_dict['tell_dict'], ind_lower=arg_dict['ind_lower'], ind_upper=arg_dict['ind_upper']) obj_model, model_gpm = obj_model_func(theta_obj, arg_dict['obj_dict']) @@ -632,8 +567,6 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): - ``arg_dict['ind_upper']``: Upper index into the telluric model wave_grid to trim down the telluric model. - - ``arg_dict['tell_model_func']``: Function for - evaluating the telluric model, i.e. PCA or grid - ``arg_dict['obj_model_func']``: User provided function for evaluating the object model - ``arg_dict['obj_dict']``: Dictionary containing the @@ -666,7 +599,6 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): # Unpack arguments obj_model_func = arg_dict['obj_model_func'] # Evaluation function - tell_model_func = arg_dict['tell_model_func'] # Telluric model, PCA or Grid flux_ivar = arg_dict['ivar'] # Inverse variance of flux or counts bounds = arg_dict['bounds'] # bounds for differential evolution optimization rng = arg_dict['rng'] # Seed for differential evolution optimizaton @@ -705,7 +637,7 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): theta_obj = result.x[:-ntheta_tell] theta_tell = result.x[-ntheta_tell:] - tell_model = tell_model_func(theta_tell, arg_dict['tell_dict'], + tell_model = eval_telluric(theta_tell, arg_dict['tell_dict'], ind_lower=arg_dict['ind_lower'], ind_upper=arg_dict['ind_upper']) obj_model, modelmask = obj_model_func(theta_obj, arg_dict['obj_dict']) totalmask = thismask & modelmask @@ -2298,7 +2230,6 @@ class Telluric(datamodel.DataContainer): 'norders', 'tell_dict', - 'tell_model_func', 'wave_grid', 'ngrid', @@ -2453,7 +2384,6 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode if upper(teltype) == 'PCA': self.tell_dict = read_telluric_pca(self.telgrid, wave_min=self.wave_in_arr[wv_gpm].min(), wave_max=self.wave_in_arr[wv_gpm].max()) - self.tell_model_func = eval_telluric_pca if self.ntell > self.tell_dict['ncomp_tell_pca']: msgs.error('Asked for more telluric PCA components ({}) ' \ 'than exist in the PCA file ({}).'.format(self.ntell, @@ -2463,7 +2393,6 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode msgs.error('Parameter ntell must be 4 for teltype = grid') self.tell_dict = read_telluric_grid(self.telgrid, wave_min=self.wave_in_arr[wv_gpm].min(), wave_max=self.wave_in_arr[wv_gpm].max()) - self.tell_model_func = eval_telluric_grid else: msgs.error('Invalid teltype -- must be `pca` or `grid`') @@ -2514,7 +2443,7 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode for counter, iord in enumerate(self.srt_order_tell): msgs.info(f'Initializing object model for order: {iord}, {counter}/{self.norders}' + f' with user supplied function: {self.init_obj_model.__name__}') - tellmodel = self.tell_model_func(self.tell_guess, self.tell_dict, + tellmodel = eval_telluric(self.tell_guess, self.tell_dict, ind_lower=self.ind_lower[iord], ind_upper=self.ind_upper[iord]) # TODO This is a pretty ugly way to pass in the blaze function. Particularly since now all the other models @@ -2538,7 +2467,7 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode arg_dict_iord = dict(ivar=self.ivar_arr[self.ind_lower[iord]:self.ind_upper[iord]+1,iord], tell_dict=self.tell_dict, ind_lower=self.ind_lower[iord], ind_upper=self.ind_upper[iord], - ntell=self.ntell, tell_model_func=self.tell_model_func, + ntell=self.ntell, obj_model_func=self.eval_obj_model, obj_dict=obj_dict, ballsize=self.ballsize, bounds=bounds_iord, rng=self.rng, diff_evol_maxiter=self.diff_evol_maxiter, tol=self.tol, @@ -2587,7 +2516,7 @@ def run(self, only_orders=None): self.theta_tell_list[iord] = self.result_list[iord].x[-(self.ntell+3):] self.obj_model_list[iord], modelmask \ = self.eval_obj_model(self.theta_obj_list[iord], self.obj_dict_list[iord]) - self.tellmodel_list[iord] = self.tell_model_func(self.theta_tell_list[iord], self.tell_dict, + self.tellmodel_list[iord] = eval_telluric(self.theta_tell_list[iord], self.tell_dict, ind_lower=self.ind_lower[iord], ind_upper=self.ind_upper[iord]) self.assign_output(iord) From 21be2c329b1ca5bc33e4e547b1ca536fa09df830 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Fri, 8 Sep 2023 14:50:44 +0200 Subject: [PATCH 053/244] cleaning up --- pypeit/core/telluric.py | 22 +++++++++++----------- pypeit/scripts/tellfit.py | 10 ++++------ 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index fc0ad5ae6c..b9259361d1 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -195,7 +195,7 @@ def read_telluric_pca(filename, wave_min=None, wave_max=None, pad_frac=0.10): def read_telluric_grid(filename, wave_min=None, wave_max=None, pad_frac=0.10): """ Reads in the telluric grid from a file. This method is no longer the - preferred approach; see "read_telluric_pca" and other functions. + preferred approach; see "read_telluric_pca" for the PCA mode. Optionally, this method also trims the grid to be in within ``wave_min`` and ``wave_max`` and pads the data (see ``pad_frac``). @@ -299,7 +299,7 @@ def conv_telluric(tell_model, dloglam, res): tell_model (`numpy.ndarray`_): Input telluric model at the native resolution of the telluric model grid. The shape of this input is in general different from the size of the raw telluric model (read in by read_telluric_* above) because it is - trimmed to relevant wavelengths using ind_lower, ind_upper. See eval_telluric_* below. + trimmed to relevant wavelengths using ind_lower, ind_upper. See eval_telluric below. dloglam (float): Wavelength spacing of the telluric grid expressed as a dlog10(lambda), i.e. stored in the tell_dict as tell_dict['dloglam'] @@ -360,7 +360,7 @@ def shift_telluric(tell_model, loglam, dloglam, shift, stretch): tell_model_shift = np.interp(loglam_shift, loglam, tell_model) return tell_model_shift -def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None):) +def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): """ Evaluate the telluric model. @@ -426,7 +426,7 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None):) if tell_dict['teltype'] == 'PCA': # Evaluate PCA model after truncating the wavelength range tellmodel_hires = np.zeros_like(tell_dict['tell_pca'][0]) - tellmodel_hires[ind_lower_pad:ind_upper_pad+1] = np.dot(np.append(1,theta_tell[:ncomp_use]), + tellmodel_hires[ind_lower_pad:ind_upper_pad+1] = np.dot(np.append(1,theta_tell[:ntell]), tell_dict['tell_pca'][:ntell+1][:,ind_lower_pad:ind_upper_pad+1]) # PCA model is inverse sinh of the optical depth, convert to transmission here @@ -1620,9 +1620,9 @@ def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile if bal_wv_min_max is not None else mask & qsomask # parameters lowered for testing - TelObj = Telluric(wave, flux, ivar, mask_tot, telgridfile, teltype, obj_params, init_qso_model, + TelObj = Telluric(wave, flux, ivar, mask_tot, telgridfile, obj_params, init_qso_model, eval_qso_model, pix_shift_bounds=pix_shift_bounds, - sn_clip=sn_clip, maxiter=maxiter, tol=tol, popsize=popsize, + sn_clip=sn_clip, maxiter=maxiter, tol=tol, popsize=popsize, teltype=teltype, ntell=ntell, recombination=recombination, polish=polish, disp=disp, debug=debug) TelObj.run(only_orders=only_orders) TelObj.to_file(telloutfile, overwrite=True) @@ -1730,7 +1730,7 @@ def star_telluric(spec1dfile, telgridfile, telloutfile, outfile, star_type=None, # parameters lowered for testing TelObj = Telluric(wave, flux, ivar, mask_tot, telgridfile, teltype, obj_params, init_star_model, eval_star_model, pix_shift_bounds=pix_shift_bounds, - ntell=ntell, sn_clip=sn_clip, tol=tol, popsize=popsize, + teltype=teltype, ntell=ntell, sn_clip=sn_clip, tol=tol, popsize=popsize, recombination=recombination, polish=polish, disp=disp, debug=debug) TelObj.run(only_orders=only_orders) TelObj.to_file(telloutfile, overwrite=True) @@ -1829,7 +1829,7 @@ def poly_telluric(spec1dfile, telgridfile, telloutfile, outfile, z_obj=0.0, func # parameters lowered for testing TelObj = Telluric(wave, flux, ivar, mask_tot, telgridfile, teltype, obj_params, init_poly_model, eval_poly_model, pix_shift_bounds=pix_shift_bounds, - sn_clip=sn_clip, maxiter=maxiter, tol=tol, popsize=popsize, + sn_clip=sn_clip, maxiter=maxiter, tol=tol, popsize=popsize, teltype=teltype, ntell=ntell, recombination=recombination, polish=polish, disp=disp, debug=debug) TelObj.run(only_orders=only_orders) TelObj.to_file(telloutfile, overwrite=True) @@ -2339,7 +2339,7 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode # 1) Assign arguments self.telgrid = telgridfile - self.teltype = teltype + self.teltype = teltype.upper() self.obj_params = obj_params self.init_obj_model = init_obj_model self.ntell = ntell @@ -2381,14 +2381,14 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode wave, flux, ivar, gpm) # 3) Read the telluric grid and initalize associated parameters wv_gpm = self.wave_in_arr > 1.0 - if upper(teltype) == 'PCA': + if self.teltype == 'PCA': self.tell_dict = read_telluric_pca(self.telgrid, wave_min=self.wave_in_arr[wv_gpm].min(), wave_max=self.wave_in_arr[wv_gpm].max()) if self.ntell > self.tell_dict['ncomp_tell_pca']: msgs.error('Asked for more telluric PCA components ({}) ' \ 'than exist in the PCA file ({}).'.format(self.ntell, self.tell_dict['ncomp_tell_pca'])) - elif teltype == 'grid': + elif self.teltype == 'grid': if self.ntell != 4: msgs.error('Parameter ntell must be 4 for teltype = grid') self.tell_dict = read_telluric_grid(self.telgrid, wave_min=self.wave_in_arr[wv_gpm].min(), diff --git a/pypeit/scripts/tellfit.py b/pypeit/scripts/tellfit.py index 8ec75acea5..3dc586b4a8 100644 --- a/pypeit/scripts/tellfit.py +++ b/pypeit/scripts/tellfit.py @@ -152,10 +152,10 @@ def main(args): if par['telluric']['objmodel']=='qso': # run telluric.qso_telluric to get the final results TelQSO = telluric.qso_telluric(args.spec1dfile, par['telluric']['telgridfile'], - par['telluric']['teltype'], par['telluric']['pca_file'], par['telluric']['redshift'], modelfile, outfile, - npca=par['telluric']['npca'], ntell=par['telluric']['ntell'], + npca=par['telluric']['npca'], + teltype=par['telluric']['teltype'], ntell=par['telluric']['ntell'], pca_lower=par['telluric']['pca_lower'], pca_upper=par['telluric']['pca_upper'], bounds_norm=par['telluric']['bounds_norm'], @@ -169,7 +169,6 @@ def main(args): debug=args.debug, show=args.plot) elif par['telluric']['objmodel']=='star': TelStar = telluric.star_telluric(args.spec1dfile, par['telluric']['telgridfile'], - par['telluric']['teltype'], modelfile, outfile, star_type=par['telluric']['star_type'], star_mag=par['telluric']['star_mag'], @@ -179,7 +178,7 @@ def main(args): model=par['telluric']['model'], polyorder=par['telluric']['polyorder'], only_orders=par['telluric']['only_orders'], - ntell=par['telluric']['ntell'], + teltype=par['telluric']['teltype'], ntell=par['telluric']['ntell'], mask_hydrogen_lines=par['sensfunc']['mask_hydrogen_lines'], mask_helium_lines=par['sensfunc']['mask_helium_lines'], hydrogen_mask_wid=par['sensfunc']['hydrogen_mask_wid'], @@ -192,13 +191,12 @@ def main(args): debug=args.debug, show=args.plot) elif par['telluric']['objmodel']=='poly': TelPoly = telluric.poly_telluric(args.spec1dfile, par['telluric']['telgridfile'], - par['telluric']['teltype'], modelfile, outfile, z_obj=par['telluric']['redshift'], func=par['telluric']['func'], model=par['telluric']['model'], polyorder=par['telluric']['polyorder'], - ntell=par['telluric']['ntell'], + teltype=par['telluric']['teltype'], ntell=par['telluric']['ntell'], fit_wv_min_max=par['telluric']['fit_wv_min_max'], mask_lyman_a=par['telluric']['mask_lyman_a'], delta_coeff_bounds=par['telluric']['delta_coeff_bounds'], From 360e990a07c1bf20254d1bb233c59d9a1a73d27c Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Tue, 12 Sep 2023 11:35:32 +0200 Subject: [PATCH 054/244] cleaning up --- pypeit/core/telluric.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index b9259361d1..b88fb3b75f 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -378,7 +378,7 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): 3. convolution of the atmosphere model to the resolution set by the spectral resolution. - 4. Shift and stretch the telluric model. + 4. shift and stretch the telluric model. Args: theta_tell (`numpy.ndarray`_): @@ -391,7 +391,6 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): :func:`read_telluric_pca` if teltype=='PCA'. or :func:`read_telluric_grid` if teltype=='grid'. - teltype (: ind_lower (:obj:`int`, optional): The index of the first pixel to include in the model. Selecting a wavelength region for the modeling makes things faster because we From 3b15eeeb7c33de7705cafaf6fd1c74b66c707a68 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Tue, 12 Sep 2023 11:40:38 +0200 Subject: [PATCH 055/244] eval return shape --- pypeit/core/telluric.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index b88fb3b75f..aa1d5cf52e 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -396,7 +396,7 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): wavelength region for the modeling makes things faster because we only need to convolve the portion that is needed for the current model fit. - ind_upper: + ind_upper (:obj:`int`, optional): The index (inclusive) of the last pixel to include in the model. Selecting a wavelength region for the modeling makes things faster because we only need to convolve the portion that is @@ -404,7 +404,10 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): Returns: `numpy.ndarray`_: Telluric model evaluated at the desired location - theta_tell in model atmosphere parameter space. + theta_tell in model atmosphere parameter space. Shape is given by + the size of `wave_grid' plus `tell_pad_pix' padding from the input + tell_dict. + """ ntheta = len(theta_tell) # FD: Currently assumes that shift and stretch are on. From 2705924809805e7add775bc93c1f7aeedf27f783 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Tue, 12 Sep 2023 11:45:28 +0200 Subject: [PATCH 056/244] added teltype to the dict docs --- pypeit/core/telluric.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index aa1d5cf52e..350cf53e57 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -168,6 +168,7 @@ def read_telluric_pca(filename, wave_min=None, wave_max=None, pad_frac=0.10): - tell_pca: PCA component vectors - bounds_tell_pca: Maximum/minimum coefficient - coefs_tell_pca: Set of model coefficient values (for prior in future) + - teltype: Type of telluric model, i.e. 'PCA' """ # load_telluric_grid() takes care of path and existance check hdul = data.load_telluric_grid(filename) @@ -223,6 +224,7 @@ def read_telluric_grid(filename, wave_min=None, wave_max=None, pad_frac=0.10): - h2o_grid: Humidity values in telluric grid [%] - airmass_grid: Airmass values in telluric grid - tell_grid: Grid of telluric models + - teltype: Type of telluric model, i.e. 'grid' """ # load_telluric_grid() takes care of path and existance check hdul = data.load_telluric_grid(filename) From 0e77c1c6d6b3b910e1bdbaa9f18c9579c9457561 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Wed, 13 Sep 2023 20:45:50 +0200 Subject: [PATCH 057/244] goodbye ntell, hello tell_npca --- pypeit/core/telluric.py | 138 +++++++++++++++++++++++++--------------- 1 file changed, 87 insertions(+), 51 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 350cf53e57..e9ca6c96d1 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -384,10 +384,10 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): Args: theta_tell (`numpy.ndarray`_): - Vector with ntell PCA coefficients (if teltype='PCA') + Vector with tell_npca PCA coefficients (if teltype='PCA') or pressure, temperature, humidity, and airmass (if teltype='grid'), followed by spectral resolution, shift, and stretch. - Final length is ntell+3. + Final length is then tell_npca+3 or 7. tell_dict (:obj:`dict`): Dictionary containing the telluric data. See :func:`read_telluric_pca` if teltype=='PCA'. @@ -412,9 +412,15 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): """ ntheta = len(theta_tell) + teltype = tell_dict['teltype'] # FD: Currently assumes that shift and stretch are on. # TODO: Make this work without shift and stretch. - ntell = ntheta-3 + if teltype == 'PCA': + ntell = ntheta-3 + elif teltype == 'grid': + ntell = 4 + else: + msgs.error("Unsupported teltype, must be 'PCA' or 'grid'") # Set the wavelength range if not provided ind_lower = 0 if ind_lower is None else ind_lower @@ -427,7 +433,7 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): ind_lower_final = ind_lower_pad if ind_lower_pad == ind_lower else ind_lower - ind_lower_pad ind_upper_final = ind_upper_pad if ind_upper_pad == ind_upper else ind_upper - ind_upper_pad - if tell_dict['teltype'] == 'PCA': + if teltype == 'PCA': # Evaluate PCA model after truncating the wavelength range tellmodel_hires = np.zeros_like(tell_dict['tell_pca'][0]) tellmodel_hires[ind_lower_pad:ind_upper_pad+1] = np.dot(np.append(1,theta_tell[:ntell]), @@ -439,7 +445,7 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): clip = tellmodel_hires < 0 tellmodel_hires[clip] = 0 tellmodel_hires = np.exp(-tellmodel_hires) - elif tell_dict['teltype'] == 'grid': + elif teltype == 'grid': # Interpolate within the telluric grid tellmodel_hires = interp_telluric_grid(theta_tell[:ntell], tell_dict) else: @@ -485,10 +491,16 @@ def tellfit_chi2(theta, flux, thismask, arg_dict): obj_model_func = arg_dict['obj_model_func'] flux_ivar = arg_dict['ivar'] + teltype = arg_dict['tell_dict']['teltype'] # TODO: make this work without shift and stretch? # Number of telluric model parameters, plus shift, stretch, and resolution - nfit = arg_dict['ntell']+3 + if teltype == 'PCA': + nfit = arg_dict['tell_npca']+3 + elif teltype == 'grid': + nfit = 4+3 + else: + msgs.error("Unsupported teltype, must be 'PCA' or 'grid'") theta_obj = theta[:-nfit] theta_tell = theta[-nfit:] @@ -520,11 +532,16 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): Parameter vector for the object + telluric model. This is actually two concatenated paramter vectors, one for - the object and one for the telluric, i.e:: - - theta_obj = theta[:-(ntell+3)] - theta_tell = theta[-(ntell+3):] - + the object and one for the telluric, i.e.: + + (in PCA mode) + theta_obj = theta[:-(tell_npca+3)] + theta_tell = theta[-(tell_npca+3):] + + (in grid mode) + theta_obj = theta[:-7] + theta_tell = theta[-7:] + The telluric model theta_tell includes a either user-specified number of PCA coefficients (in PCA mode) or ambient pressure, temperature, humidity, and airmass (in grid mode) followed by @@ -532,7 +549,7 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): That is, in PCA mode, - pca_coeffs = theta_tell[:ntell] + pca_coeffs = theta_tell[:tell_npca] while in grid mode, @@ -611,8 +628,14 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): nparams = len(bounds) # Number of parameters in the model popsize = arg_dict['popsize'] # Note this does nothing if the init is done from a previous iteration or optimum nsamples = arg_dict['popsize']*nparams + teltype = arg_dict['tell_dict']['teltype'] # FD: Assumes shift and stretch are turned on. - ntheta_tell = arg_dict['ntell']+3 # Total number of telluric model parameters + if teltype == 'PCA': + ntheta_tell = arg_dict['tell_npca']+3 # Total number of telluric model parameters in PCA mode + elif teltype == 'grid': + ntheta_tell = 4+3 # Total number of telluric model parameters in grid mode + else: + msgs.error("Unsupported teltype, must be 'PCA' or 'grid'") # Decide how to initialize if init_from_last is not None: # Use a Gaussian ball about the optimum from a previous iteration @@ -1289,7 +1312,8 @@ def eval_poly_model(theta, obj_dict): def sensfunc_telluric(wave, counts, counts_ivar, counts_mask, exptime, airmass, std_dict, - telgridfile, teltype, log10_blaze_function=None, ech_orders=None, polyorder=8, ntell=4, + telgridfile, log10_blaze_function=None, ech_orders=None, polyorder=8, + tell_npca=4, teltype='PCA', mask_hydrogen_lines=True, mask_helium_lines=False, hydrogen_mask_wid=10., resln_guess=None, resln_frac_bounds=(0.3, 1.5), pix_shift_bounds=(-5.0, 5.0), delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), @@ -1336,14 +1360,14 @@ def sensfunc_telluric(wave, counts, counts_ivar, counts_mask, exptime, airmass, telgridfile : :obj:`str` File containing grid of HITRAN atmosphere models; see :class:`~pypeit.par.pypeitpar.TelluricPar`. - teltype : :obj:`str` - Method for evaluating telluric models, either `PCA` or `grid`. ech_orders : `numpy.ndarray`_, shape is (norders,), optional If passed, provides the true order numbers for the spectra provided. polyorder : :obj:`int`, optional, default = 8 Polynomial order for the sensitivity function fit. - ntell : :obj:`int`, optional, default = 4 - Number of telluric model parameters (must be 4 for teltype=`grid`, or <=10 for teltype=`PCA`) + teltype : :obj:`str`, optional, default = 'PCA' + Method for evaluating telluric models, either `PCA` or `grid`. + tell_npca : :obj:`int`, optional, default = 4 + Number of telluric PCA components used, must be <= 10 mask_hydrogen_lines : :obj:`bool`, optional If True, mask stellar hydrogen absorption lines before fitting sensitivity function. Default = True mask_helium_lines : :obj:`bool`, optional @@ -1456,9 +1480,10 @@ def sensfunc_telluric(wave, counts, counts_ivar, counts_mask, exptime, airmass, mask_tot = mask_bad & mask_recomb & mask_tell # Since we are fitting a sensitivity function, first compute counts per second per angstrom. - TelObj = Telluric(wave, counts, counts_ivar, mask_tot, telgridfile, teltype, obj_params, + TelObj = Telluric(wave, counts, counts_ivar, mask_tot, telgridfile, obj_params, init_sensfunc_model, eval_sensfunc_model, log10_blaze_function=log10_blaze_function, - ntell=ntell, ech_orders=ech_orders, pix_shift_bounds=pix_shift_bounds, + teltype=teltype, tell_npca=tell_npca, + ech_orders=ech_orders, pix_shift_bounds=pix_shift_bounds, resln_guess=resln_guess, resln_frac_bounds=resln_frac_bounds, sn_clip=sn_clip, maxiter=maxiter, lower=lower, upper=upper, tol=tol, popsize=popsize, recombination=recombination, polish=polish, disp=disp, @@ -1503,7 +1528,7 @@ def create_bal_mask(wave, bal_wv_min_max): def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile, npca=8, pca_lower=1220.0, pca_upper=3100.0, bal_wv_min_max=None, delta_zqso=0.1, - teltype='PCA', ntell=4, + teltype='PCA', tell_npca=4, bounds_norm=(0.1, 3.0), tell_norm_thresh=0.9, sn_clip=30.0, only_orders=None, maxiter=3, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, pix_shift_bounds=(-5.0,5.0), debug_init=False, debug=False, show=False): @@ -1517,8 +1542,6 @@ def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile the output from a ``PypeIt`` 1D coadd. telgridfile : :obj:`str` File name with the grid of telluric spectra. - teltype : :obj:`str` - Method for evaluating telluric models, either `PCA` or `grid`. pca_file: :obj:`str` File name of the QSO PCA model fits file. z_qso : :obj:`float` @@ -1541,8 +1564,10 @@ def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile delta_zqso : :obj:`float`, optional During the fit, the QSO redshift is allowed to vary within ``+/-delta_zqso``. - ntell : :obj:`int`, optional, default = 4 - Number of telluric model parameters (must be 4 for teltype=`grid`, or <=10 for teltype=`PCA`) + teltype : :obj:`str`, optional, default = 'PCA' + Method for evaluating telluric models, either `PCA` or `grid`. + tell_npca : :obj:`int`, optional, default = 4 + Number of telluric PCA components used, must be <=10 bounds_norm : :obj:`tuple`, optional A two-tuple with the lower and upper bounds on the fractional adjustment of the flux in the QSO model spectrum. For example, a value of ``(0.1, @@ -1627,7 +1652,8 @@ def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile TelObj = Telluric(wave, flux, ivar, mask_tot, telgridfile, obj_params, init_qso_model, eval_qso_model, pix_shift_bounds=pix_shift_bounds, sn_clip=sn_clip, maxiter=maxiter, tol=tol, popsize=popsize, teltype=teltype, - ntell=ntell, recombination=recombination, polish=polish, disp=disp, debug=debug) + tell_npca=tell_npca, recombination=recombination, polish=polish, + disp=disp, debug=debug) TelObj.run(only_orders=only_orders) TelObj.to_file(telloutfile, overwrite=True) @@ -1672,7 +1698,7 @@ def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile def star_telluric(spec1dfile, telgridfile, telloutfile, outfile, star_type=None, star_mag=None, star_ra=None, star_dec=None, func='legendre', model='exp', - polyorder=5, teltype='PCA', ntell=4, mask_hydrogen_lines=True, + polyorder=5, teltype='PCA', tell_npca=4, mask_hydrogen_lines=True, mask_helium_lines=False, hydrogen_mask_wid=10., delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), only_orders=None, sn_clip=30.0, maxiter=3, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, @@ -1732,9 +1758,10 @@ def star_telluric(spec1dfile, telgridfile, telloutfile, outfile, star_type=None, mask_tot = mask_bad & mask_recomb & mask_tell # parameters lowered for testing - TelObj = Telluric(wave, flux, ivar, mask_tot, telgridfile, teltype, obj_params, init_star_model, + TelObj = Telluric(wave, flux, ivar, mask_tot, telgridfile, obj_params, init_star_model, eval_star_model, pix_shift_bounds=pix_shift_bounds, - teltype=teltype, ntell=ntell, sn_clip=sn_clip, tol=tol, popsize=popsize, + teltype=teltype, tell_npca=tell_npca, + sn_clip=sn_clip, tol=tol, popsize=popsize, recombination=recombination, polish=polish, disp=disp, debug=debug) TelObj.run(only_orders=only_orders) TelObj.to_file(telloutfile, overwrite=True) @@ -1780,7 +1807,7 @@ def star_telluric(spec1dfile, telgridfile, telloutfile, outfile, star_type=None, def poly_telluric(spec1dfile, telgridfile, telloutfile, outfile, z_obj=0.0, func='legendre', model='exp', polyorder=3, fit_wv_min_max=None, mask_lyman_a=True, teltype='PCA', - ntell=4, delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), + tell_npca=4, delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), only_orders=None, sn_clip=30.0, maxiter=3, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, pix_shift_bounds=(-5.0,5.0), debug_init=False, debug=False, show=False): @@ -1831,10 +1858,10 @@ def poly_telluric(spec1dfile, telgridfile, telloutfile, outfile, z_obj=0.0, func mask_tot &= np.logical_not(create_bal_mask(wave, fit_wv_min_max)) # parameters lowered for testing - TelObj = Telluric(wave, flux, ivar, mask_tot, telgridfile, teltype, obj_params, init_poly_model, + TelObj = Telluric(wave, flux, ivar, mask_tot, telgridfile, obj_params, init_poly_model, eval_poly_model, pix_shift_bounds=pix_shift_bounds, sn_clip=sn_clip, maxiter=maxiter, tol=tol, popsize=popsize, teltype=teltype, - ntell=ntell, recombination=recombination, polish=polish, disp=disp, debug=debug) + tell_npca=tell_npca, recombination=recombination, polish=polish, disp=disp, debug=debug) TelObj.run(only_orders=only_orders) TelObj.to_file(telloutfile, overwrite=True) @@ -1976,9 +2003,8 @@ class Telluric(datamodel.DataContainer): PCA decomposition of HITRAN atmospheric models, enabling a more flexible and far more file-size efficient interpolation of the telluric absorption model space. - ntell (:obj:`int`, optional): - Number of telluric model parameters. Must be 4 for teltype=`grid`, - or <=10 for teltype=`PCA`. + tell_npca (:obj:`int`, optional): + Number of telluric PCA components used, must be <=10 obj_params (:obj:`dict`): Dictionary of parameters for initializing the object model. init_obj_model (callable): @@ -2168,7 +2194,7 @@ class Telluric(datamodel.DataContainer): 'HITRAN atmosphere models'), 'teltype': dict(otype=str, descr='Type of telluric model, `PCA` or `grid`'), - 'ntell': dict(otype=int, + 'tell_npca': dict(otype=int, descr='Number of telluric PCA components used'), 'std_src': dict(otype=str, descr='Name of the standard source'), 'std_name': dict(otype=str, descr='Type of standard source'), @@ -2267,7 +2293,7 @@ class Telluric(datamodel.DataContainer): ] @staticmethod - def empty_model_table(norders, nspec, ntell=4, n_obj_par=0): + def empty_model_table(norders, nspec, tell_npca=4, n_obj_par=0): """ Construct an empty `astropy.table.Table`_ for the telluric model results. @@ -2277,6 +2303,8 @@ def empty_model_table(norders, nspec, ntell=4, n_obj_par=0): The number of slits/orders on the detector. nspec (:obj:`int`): The number of spectral pixels on the detector. + tell_npca (:obj:`int`): + Number of telluric model parameters n_obj_par (:obj:`int`, optional): The number of parameters used to construct the object model spectrum. @@ -2292,9 +2320,9 @@ def empty_model_table(norders, nspec, ntell=4, n_obj_par=0): table.Column(name='OBJ_MODEL', dtype=float, length=norders, shape=(nspec,), description='Best-fitting object model spectrum'), # TODO: Why do we need both TELL_THETA and all the individual parameters... - table.Column(name='TELL_THETA', dtype=float, length=norders, shape=(ntell+3,), + table.Column(name='TELL_THETA', dtype=float, length=norders, shape=(tell_npca+3,), description='Best-fitting telluric model parameters'), - table.Column(name='TELL_PARAM', dtype=float, length=norders, shape=(ntell,), + table.Column(name='TELL_PARAM', dtype=float, length=norders, shape=(tell_npca,), description='Best-fitting telluric atmospheric parameters or PCA coefficients'), table.Column(name='TELL_RESLN', dtype=float, length=norders, description='Best-fitting telluric model spectral resolution'), @@ -2324,7 +2352,7 @@ def empty_model_table(norders, nspec, ntell=4, n_obj_par=0): description='Maximum wavelength included in the fit')]) def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_model, eval_obj_model, - log10_blaze_function=None, ech_orders=None, sn_clip=30.0, teltype='PCA', ntell=4, + log10_blaze_function=None, ech_orders=None, sn_clip=30.0, teltype='PCA', tell_npca=4, airmass_guess=1.5, resln_guess=None, resln_frac_bounds=(0.3, 1.5), pix_shift_bounds=(-5.0, 5.0), pix_stretch_bounds=(0.9,1.1), maxiter=2, sticky=True, lower=3.0, upper=3.0, seed=777, ballsize = 5e-4, tol=1e-3, diff_evol_maxiter=1000, popsize=30, @@ -2346,7 +2374,7 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode self.teltype = teltype.upper() self.obj_params = obj_params self.init_obj_model = init_obj_model - self.ntell = ntell + self.tell_npca = tell_npca self.airmass_guess = airmass_guess self.eval_obj_model = eval_obj_model self.ech_orders = ech_orders @@ -2388,13 +2416,11 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode if self.teltype == 'PCA': self.tell_dict = read_telluric_pca(self.telgrid, wave_min=self.wave_in_arr[wv_gpm].min(), wave_max=self.wave_in_arr[wv_gpm].max()) - if self.ntell > self.tell_dict['ncomp_tell_pca']: + if self.tell_npca > self.tell_dict['ncomp_tell_pca']: msgs.error('Asked for more telluric PCA components ({}) ' \ - 'than exist in the PCA file ({}).'.format(self.ntell, + 'than exist in the PCA file ({}).'.format(self.tell_npca, self.tell_dict['ncomp_tell_pca'])) elif self.teltype == 'grid': - if self.ntell != 4: - msgs.error('Parameter ntell must be 4 for teltype = grid') self.tell_dict = read_telluric_grid(self.telgrid, wave_min=self.wave_in_arr[wv_gpm].min(), wave_max=self.wave_in_arr[wv_gpm].max()) else: @@ -2471,7 +2497,7 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode arg_dict_iord = dict(ivar=self.ivar_arr[self.ind_lower[iord]:self.ind_upper[iord]+1,iord], tell_dict=self.tell_dict, ind_lower=self.ind_lower[iord], ind_upper=self.ind_upper[iord], - ntell=self.ntell, + tell_npca=self.tell_npca, obj_model_func=self.eval_obj_model, obj_dict=obj_dict, ballsize=self.ballsize, bounds=bounds_iord, rng=self.rng, diff_evol_maxiter=self.diff_evol_maxiter, tol=self.tol, @@ -2516,8 +2542,12 @@ def run(self, only_orders=None): inmask=self.mask_arr[self.ind_lower[iord]:self.ind_upper[iord]+1,iord], maxiter=self.maxiter, lower=self.lower, upper=self.upper, sticky=self.sticky) - self.theta_obj_list[iord] = self.result_list[iord].x[:-(self.ntell+3)] - self.theta_tell_list[iord] = self.result_list[iord].x[-(self.ntell+3):] + if self.teltype == 'PCA': + self.theta_obj_list[iord] = self.result_list[iord].x[:-(self.tell_npca+3)] + self.theta_tell_list[iord] = self.result_list[iord].x[-(self.tell_npca+3):] + elif self.teltype == 'grid': + self.theta_obj_list[iord] = self.result_list[iord].x[:-(4+3)] + self.theta_tell_list[iord] = self.result_list[iord].x[-(4+3):] self.obj_model_list[iord], modelmask \ = self.eval_obj_model(self.theta_obj_list[iord], self.obj_dict_list[iord]) self.tellmodel_list[iord] = eval_telluric(self.theta_tell_list[iord], self.tell_dict, @@ -2568,7 +2598,7 @@ def init_output(self): """ self.model = self.empty_model_table(self.norders, self.nspec_in, - ntell=self.ntell, n_obj_par=self.max_ntheta_obj) + tell_npca=self.tell_npca, n_obj_par=self.max_ntheta_obj) if 'output_meta_keys' in self.obj_params: for key in self.obj_params['output_meta_keys']: if key.lower() in self.datamodel.keys(): @@ -2596,6 +2626,11 @@ def assign_output(self, iord): gdwave = self.wave_in_arr[:,iord] > 1.0 wave_in_gd = self.wave_in_arr[gdwave,iord] wave_grid_now = self.wave_grid[self.ind_lower[iord]:self.ind_upper[iord]+1] + + if self.teltype == 'PCA': + ntell = tell_npca + elif self.teltype == 'grid': + ntell = 4 self.model['WAVE'][iord] = self.wave_in_arr[:,iord] self.model['TELLURIC'][iord][gdwave] \ @@ -2607,6 +2642,7 @@ def assign_output(self, iord): kind='linear', bounds_error=False, fill_value=0.0)(wave_in_gd) self.model['TELL_THETA'][iord] = self.theta_tell_list[iord] + if self.model['TELL_PARAM'][iord] = self.theta_tell_list[iord][:self.ntell] self.model['TELL_RESLN'][iord] = self.theta_tell_list[iord][self.ntell] self.model['TELL_SHIFT'][iord] = self.theta_tell_list[iord][self.ntell+1] @@ -2694,7 +2730,7 @@ def get_tell_guess(self): guess.append(np.median(self.tell_dict['h2o_grid'])) guess.append(self.airmass_guess) else: - guess = list(np.zeros(self.ntell)) + guess = list(np.zeros(self.tell_npca)) guess.append(self.resln_guess) guess.append(0.0) guess.append(1.0) @@ -2722,7 +2758,7 @@ def get_bounds_tell(self): bounds.append((self.tell_dict['airmass_grid'].min(), self.tell_dict['airmass_grid'].max())) else: - for ii in range(self.ntell): + for ii in range(self.tell_npca): bounds.append((self.tell_dict['bounds_tell_pca'][0][ii+1], self.tell_dict['bounds_tell_pca'][1][ii+1])) bounds.append((self.resln_guess * self.resln_frac_bounds[0], From 548d17aacbbb4e78ca66c315591ec72d61298180 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Wed, 13 Sep 2023 20:47:28 +0200 Subject: [PATCH 058/244] cleanup --- pypeit/core/telluric.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index e9ca6c96d1..2c46b1fc5f 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -2642,7 +2642,6 @@ def assign_output(self, iord): kind='linear', bounds_error=False, fill_value=0.0)(wave_in_gd) self.model['TELL_THETA'][iord] = self.theta_tell_list[iord] - if self.model['TELL_PARAM'][iord] = self.theta_tell_list[iord][:self.ntell] self.model['TELL_RESLN'][iord] = self.theta_tell_list[iord][self.ntell] self.model['TELL_SHIFT'][iord] = self.theta_tell_list[iord][self.ntell+1] From 88573153e5ebf9541de67efbcdb260ea949fae7d Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Wed, 13 Sep 2023 22:13:49 +0200 Subject: [PATCH 059/244] cleanup --- pypeit/core/telluric.py | 10 +++++----- pypeit/par/pypeitpar.py | 12 +++++------- pypeit/scripts/tellfit.py | 6 +++--- pypeit/sensfunc.py | 4 ++-- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 2c46b1fc5f..3e73f7397a 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -2628,7 +2628,7 @@ def assign_output(self, iord): wave_grid_now = self.wave_grid[self.ind_lower[iord]:self.ind_upper[iord]+1] if self.teltype == 'PCA': - ntell = tell_npca + ntell = self.tell_npca elif self.teltype == 'grid': ntell = 4 @@ -2642,10 +2642,10 @@ def assign_output(self, iord): kind='linear', bounds_error=False, fill_value=0.0)(wave_in_gd) self.model['TELL_THETA'][iord] = self.theta_tell_list[iord] - self.model['TELL_PARAM'][iord] = self.theta_tell_list[iord][:self.ntell] - self.model['TELL_RESLN'][iord] = self.theta_tell_list[iord][self.ntell] - self.model['TELL_SHIFT'][iord] = self.theta_tell_list[iord][self.ntell+1] - self.model['TELL_STRETCH'][iord] = self.theta_tell_list[iord][self.ntell+2] + self.model['TELL_PARAM'][iord] = self.theta_tell_list[iord][:ntell] + self.model['TELL_RESLN'][iord] = self.theta_tell_list[iord][ntell] + self.model['TELL_SHIFT'][iord] = self.theta_tell_list[iord][ntell+1] + self.model['TELL_STRETCH'][iord] = self.theta_tell_list[iord][ntell+2] ntheta_iord = len(self.theta_obj_list[iord]) self.model['OBJ_THETA'][iord][:ntheta_iord] = self.theta_obj_list[iord] self.model['CHI2'][iord] = self.result_list[iord].fun diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index 083ad9a1e4..46a095ad45 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -2082,7 +2082,7 @@ class TelluricPar(ParSet): """ def __init__(self, telgridfile=None, sn_clip=None, resln_guess=None, resln_frac_bounds=None, pix_shift_bounds=None, - delta_coeff_bounds=None, minmax_coeff_bounds=None, maxiter=None, ntell=None, teltype=None, + delta_coeff_bounds=None, minmax_coeff_bounds=None, maxiter=None, tell_npca=None, teltype=None, sticky=None, lower=None, upper=None, seed=None, tol=None, popsize=None, recombination=None, polish=None, disp=None, objmodel=None, redshift=None, delta_redshift=None, pca_file=None, npca=None, bal_wv_min_max=None, bounds_norm=None, tell_norm_thresh=None, only_orders=None, pca_lower=None, @@ -2108,11 +2108,9 @@ def __init__(self, telgridfile=None, sn_clip=None, resln_guess=None, resln_frac_ 'the pypeit_install_telluric script. NOTE: This parameter no longer includes the full ' \ 'pathname to the Telluric Grid file, but is just the filename of the grid itself.' - defaults['ntell'] = 4 - dtypes['ntell'] = int - descr['ntell'] = 'Number of fitted telluric model parameters. Must be set to 4 (default) for teltype = grid, ' \ - 'but can be set to any number from 1 to 10 for teltype = PCA, corresponding to the number of ' \ - 'fitted PCA coefficients.' + defaults['tell_npca'] = 4 + dtypes['tell_npca'] = int + descr['tell_npca'] = 'Number of telluric PCA components used. Can be set to any number from 1 to 10.' defaults['teltype'] = 'PCA' dtypes['teltype'] = str @@ -2348,7 +2346,7 @@ def __init__(self, telgridfile=None, sn_clip=None, resln_guess=None, resln_frac_ @classmethod def from_dict(cls, cfg): k = np.array([*cfg.keys()]) - parkeys = ['telgridfile', 'teltype', 'sn_clip', 'resln_guess', 'resln_frac_bounds', 'ntell', + parkeys = ['telgridfile', 'teltype', 'sn_clip', 'resln_guess', 'resln_frac_bounds', 'tell_npca', 'pix_shift_bounds', 'delta_coeff_bounds', 'minmax_coeff_bounds', 'maxiter', 'sticky', 'lower', 'upper', 'seed', 'tol', 'popsize', 'recombination', 'polish', 'disp', 'objmodel','redshift', 'delta_redshift', diff --git a/pypeit/scripts/tellfit.py b/pypeit/scripts/tellfit.py index 3dc586b4a8..36101c2b9a 100644 --- a/pypeit/scripts/tellfit.py +++ b/pypeit/scripts/tellfit.py @@ -155,7 +155,7 @@ def main(args): par['telluric']['pca_file'], par['telluric']['redshift'], modelfile, outfile, npca=par['telluric']['npca'], - teltype=par['telluric']['teltype'], ntell=par['telluric']['ntell'], + teltype=par['telluric']['teltype'], tell_npca=par['telluric']['tell_npca'], pca_lower=par['telluric']['pca_lower'], pca_upper=par['telluric']['pca_upper'], bounds_norm=par['telluric']['bounds_norm'], @@ -178,7 +178,7 @@ def main(args): model=par['telluric']['model'], polyorder=par['telluric']['polyorder'], only_orders=par['telluric']['only_orders'], - teltype=par['telluric']['teltype'], ntell=par['telluric']['ntell'], + teltype=par['telluric']['teltype'], tell_npca=par['telluric']['tell_npca'], mask_hydrogen_lines=par['sensfunc']['mask_hydrogen_lines'], mask_helium_lines=par['sensfunc']['mask_helium_lines'], hydrogen_mask_wid=par['sensfunc']['hydrogen_mask_wid'], @@ -196,7 +196,7 @@ def main(args): func=par['telluric']['func'], model=par['telluric']['model'], polyorder=par['telluric']['polyorder'], - teltype=par['telluric']['teltype'], ntell=par['telluric']['ntell'], + teltype=par['telluric']['teltype'], tell_npca=par['telluric']['tell_npca'], fit_wv_min_max=par['telluric']['fit_wv_min_max'], mask_lyman_a=par['telluric']['mask_lyman_a'], delta_coeff_bounds=par['telluric']['delta_coeff_bounds'], diff --git a/pypeit/sensfunc.py b/pypeit/sensfunc.py index ac828c1044..df2a2985b3 100644 --- a/pypeit/sensfunc.py +++ b/pypeit/sensfunc.py @@ -880,7 +880,6 @@ def compute_zeropoint(self): self.counts_mask, self.meta_spec['EXPTIME'], self.meta_spec['AIRMASS'], self.std_dict, self.par['IR']['telgridfile'], - self.par['IR']['teltype'], log10_blaze_function=self.log10_blaze_function, polyorder=self.par['polyorder'], ech_orders=self.meta_spec['ECH_ORDERS'], @@ -888,7 +887,8 @@ def compute_zeropoint(self): resln_frac_bounds=self.par['IR']['resln_frac_bounds'], pix_shift_bounds=self.par['IR']['pix_shift_bounds'], sn_clip=self.par['IR']['sn_clip'], - ntell=self.par['IR']['ntell'], + teltype=self.par['IR']['teltype'], + tell_npca=self.par['IR']['tell_npca'], mask_hydrogen_lines=self.par['mask_hydrogen_lines'], maxiter=self.par['IR']['maxiter'], lower=self.par['IR']['lower'], From ec15801d2ee5273df923c63c77d58af621944b78 Mon Sep 17 00:00:00 2001 From: rcooke Date: Sat, 23 Sep 2023 14:00:31 +0100 Subject: [PATCH 060/244] fixed --- pypeit/coadd3d.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index 97dd9c4855..353afd4d89 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -1818,11 +1818,6 @@ def subpixellate(output_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_w # Transform this to spatial location spatpos_subpix = _astrom_trans[fr].transform(sl, spat_xx, spec_yy) spatpos = _astrom_trans[fr].transform(sl, all_spatpos[this_sl], all_specpos[this_sl]) - # OLD (WRONG) ROUTINE - # ra_coeff = np.polyfit(spatpos, all_ra[this_sl], 1) - # dec_coeff = np.polyfit(spatpos, all_dec[this_sl], 1) - # this_ra = np.polyval(ra_coeff, spatpos_subpix)#ra_spl(spatpos_subpix) - # this_dec = np.polyval(dec_coeff, spatpos_subpix)#dec_spl(spatpos_subpix) ssrt = np.argsort(spatpos) tmp_ra = all_ra[this_sl] tmp_dec = all_dec[this_sl] From f3c5de61d6f5591d90471740777cf4a7e49a2f69 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Tue, 26 Sep 2023 11:18:26 -0700 Subject: [PATCH 061/244] added pix_shift_bounds to tellfit calls --- pypeit/scripts/tellfit.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pypeit/scripts/tellfit.py b/pypeit/scripts/tellfit.py index 36101c2b9a..d0e0626f2f 100644 --- a/pypeit/scripts/tellfit.py +++ b/pypeit/scripts/tellfit.py @@ -162,6 +162,7 @@ def main(args): tell_norm_thresh=par['telluric']['tell_norm_thresh'], only_orders=par['telluric']['only_orders'], bal_wv_min_max=par['telluric']['bal_wv_min_max'], + pix_shift_bounds=par['telluric']['pix_shift_bounds'], maxiter=par['telluric']['maxiter'], popsize=par['telluric']['popsize'], tol=par['telluric']['tol'], @@ -184,6 +185,7 @@ def main(args): hydrogen_mask_wid=par['sensfunc']['hydrogen_mask_wid'], delta_coeff_bounds=par['telluric']['delta_coeff_bounds'], minmax_coeff_bounds=par['telluric']['minmax_coeff_bounds'], + pix_shift_bounds=par['telluric']['pix_shift_bounds'], maxiter=par['telluric']['maxiter'], popsize=par['telluric']['popsize'], tol=par['telluric']['tol'], @@ -202,6 +204,7 @@ def main(args): delta_coeff_bounds=par['telluric']['delta_coeff_bounds'], minmax_coeff_bounds=par['telluric']['minmax_coeff_bounds'], only_orders=par['telluric']['only_orders'], + pix_shift_bounds=par['telluric']['pix_shift_bounds'], maxiter=par['telluric']['maxiter'], popsize=par['telluric']['popsize'], tol=par['telluric']['tol'], From e332f991fca97593f479a4f9300788b97c4fc9ea Mon Sep 17 00:00:00 2001 From: joe Date: Fri, 29 Sep 2023 23:46:09 +0200 Subject: [PATCH 062/244] Fixed bugs in coadding and other changes. --- pypeit/coadd1d.py | 19 ++- pypeit/core/coadd.py | 239 +++++++++++++++++------------ pypeit/core/telluric.py | 3 +- pypeit/core/wavecal/wvutils.py | 1 - pypeit/onespec.py | 22 +-- pypeit/par/pypeitpar.py | 39 ++++- pypeit/specobj.py | 22 ++- pypeit/specutils/pypeit_loaders.py | 5 +- 8 files changed, 221 insertions(+), 129 deletions(-) diff --git a/pypeit/coadd1d.py b/pypeit/coadd1d.py index 34b03974a5..b4386a1103 100644 --- a/pypeit/coadd1d.py +++ b/pypeit/coadd1d.py @@ -16,6 +16,7 @@ from pypeit.spectrographs.util import load_spectrograph from pypeit.onespec import OneSpec +from pypeit import utils from pypeit import sensfunc from pypeit import specobjs from pypeit import msgs @@ -23,6 +24,7 @@ from pypeit.history import History + class CoAdd1D: @classmethod @@ -123,11 +125,12 @@ def save(self, coaddfile, telluric=None, obj_model=None, overwrite=True): Overwrite existing file? """ self.coaddfile = coaddfile - wave_gpm = self.wave_coadd > 1.0 + #wave_gpm = self.wave_grid_mid > 1.0 # Generate the spectrum container object - onespec = OneSpec(wave=self.wave_coadd[wave_gpm], wave_grid_mid=self.wave_grid_mid[wave_gpm], flux=self.flux_coadd[wave_gpm], - PYP_SPEC=self.spectrograph.name, ivar=self.ivar_coadd[wave_gpm], - mask=self.gpm_coadd[wave_gpm].astype(int), + onespec = OneSpec(wave=self.wave_coadd, wave_grid_mid=self.wave_grid_mid, flux=self.flux_coadd, + PYP_SPEC=self.spectrograph.name, ivar=self.ivar_coadd, + sigma = np.sqrt(utils.inverse(self.ivar_coadd)), + mask=self.gpm_coadd.astype(int), ext_mode=self.par['ex_value'], fluxed=self.par['flux_value']) # TODO This is a hack, not sure how to merge the headers at present @@ -139,9 +142,9 @@ def save(self, coaddfile, telluric=None, obj_model=None, overwrite=True): # Add on others if telluric is not None: - onespec.telluric = telluric[wave_gpm] + onespec.telluric = telluric if obj_model is not None: - onespec.obj_model = obj_model[wave_gpm] + onespec.obj_model = obj_model # Write onespec.to_file(coaddfile, history=history, overwrite=overwrite) @@ -267,8 +270,8 @@ def check_exposures(self): # check if there is any bad exposure by comparing the rms_sn with the median rms_sn among all exposures if len(_fluxes) > 2: - # Evaluate the sn_weights. - rms_sn, weights = coadd.sn_weights(_fluxes, _ivars, _gpms, const_weights=True) + # Evaluate the rms_sn + rms_sn, _ = coadd.calc_snr(_fluxes, _ivars, _gpms) # some stats mean, med, sigma = stats.sigma_clipped_stats(rms_sn, sigma_lower=2., sigma_upper=2.) _sigrej = self.par['sigrej_exp'] if self.par['sigrej_exp'] is not None else 10.0 diff --git a/pypeit/core/coadd.py b/pypeit/core/coadd.py index e3158372ab..1315603b17 100644 --- a/pypeit/core/coadd.py +++ b/pypeit/core/coadd.py @@ -180,7 +180,7 @@ def poly_ratio_fitfunc_chi2(theta, gpm, arg_dict): There are two non-standard things implemented here which increase ther robustness. The first is a non-standard error used for the chi, which adds robustness and increases the stability of the optimization. This was taken from the idlutils solve_poly_ratio code. The second thing is that the chi is remapped using the scipy huber loss function to - reduce sensitivity to outliers, ased on the scipy cookbook on robust optimization. + reduce sensitivity to outliers, based on the scipy cookbook on robust optimization. Args: theta (`numpy.ndarray`_): parameter vector for the polymomial fit @@ -304,7 +304,7 @@ def poly_ratio_fitfunc(flux_ref, gpm, arg_dict, init_from_last=None, **kwargs_op except KeyError: debug = False - sigma_corr, maskchi = renormalize_errors(chi, mask=gpm, title = 'poly_ratio_fitfunc', debug=debug) + sigma_corr, maskchi = renormalize_errors(chi, mask=mask_both, title = 'poly_ratio_fitfunc', debug=debug) ivartot = ivartot1/sigma_corr**2 return result, flux_scale, ivartot @@ -749,8 +749,54 @@ def smooth_weights(inarr, gdmsk, sn_smooth_npix): sn_conv = convolution.convolve(sn_med2, gauss_kernel, boundary='extend') return sn_conv -def sn_weights(fluxes, ivars, gpms, sn_smooth_npix=None, const_weights=False, - ivar_weights=False, relative_weights=False, verbose=False): + +def calc_snr(fluxes, ivars, gpms): + + """ + Calculate the rms S/N of each input spectrum. + + Parameters + ---------- + fluxes : list + List of length nexp containing the `numpy.ndarray`_ 1d float spectra. The + shapes in the list can be different. nexp = len(fluxes) + ivars : list + List of length nexp containing the `numpy.ndarray`_ 1d float inverse + variances of the spectra. + gpms : list + List of length nexp containing the `numpy.ndarray`_ 1d float boolean good + pixel masks of the spectra. + verbose : bool, optional + Verbosity of print out. + + Returns + ------- + rms_sn : list + List of length nexp root-mean-square S/N value for each input spectra where nexp=len(fluxes). + sn_val : list + List of length nexp containing the wavelength dependent S/N arrays for each input spectrum, i.e. + each element contains the array flux*sqrt(ivar) + """ + + nexp = len(fluxes) + + # Calculate S/N + sn_val, rms_sn, sn2 = [], [], [] + for iexp in range(nexp): + sn_val_iexp = fluxes[iexp]*np.sqrt(ivars[iexp]) + sn_val.append(sn_val_iexp) + sn_val_ma = np.ma.array(sn_val_iexp, mask=np.logical_not(gpms[iexp])) + sn_sigclip = stats.sigma_clip(sn_val_ma, sigma=3, maxiters=5) + sn2_iexp = sn_sigclip.mean()**2 # S/N^2 value for each spectrum + if sn2_iexp is np.ma.masked: + msgs.error(f'No unmasked value in iexp={iexp+1}/{nexp}. Check inputs.') + else: + sn2.append(sn2_iexp) + rms_sn.append(np.sqrt(sn2_iexp)) # Root Mean S/N**2 value for all spectra + + return rms_sn, sn_val + +def sn_weights(fluxes, ivars, gpms, sn_smooth_npix=None, weight_method='auto', verbose=False): """ Calculate the S/N of each input spectrum and create an array of @@ -769,22 +815,33 @@ def sn_weights(fluxes, ivars, gpms, sn_smooth_npix=None, const_weights=False, pixel masks of the spectra. sn_smooth_npix : float, optional Number of pixels used for determining smoothly varying S/N ratio - weights. This can be set to None if const_weights is True, since then + weights. This must be passed for all weight methods excpet for weight_method='constant' or 'uniform', since then wavelength dependent weights are not used. - const_weights : bool, optional - Use a constant weights for each spectrum? - ivar_weights : bool, optional - Use inverse variance weighted scheme? - relative_weights : bool, optional - Calculate weights by fitting to the ratio of spectra? Note, relative - weighting will only work well when there is at least one spectrum with a - reasonable S/N, and a continuum. RJC note - This argument may only be - better when the object being used has a strong continuum + emission - lines. The reference spectrum is assigned a value of 1 for all - wavelengths, and the weights of all other spectra will be determined - relative to the reference spectrum. This is particularly useful if you - are dealing with highly variable spectra (e.g. emission lines) and - require a precision better than ~1 per cent. + weight_method (str): + The weighting method to be used. Options are 'auto', 'constant', 'relative', or 'ivar'. The defaulti is'auto'. + Behavior is as follows: + 'auto': + Use constant weights if rms_sn < 3.0, otherwise use wavelength dependent. + 'constant': + Constant weights based on rms_sn**2 + 'uniform': + Uniform weighting. + 'wave_dependent': + Wavelength dependent weights will be used irrespective of the rms_sn ratio. This option + will not work well at low S/N ratio although it is useful for objects where only a small + fraction of the spectral coverage has high S/N ratio (like high-z quasars). + 'relative': + Calculate weights by fitting to the ratio of spectra? Note, relative + weighting will only work well when there is at least one spectrum with a + reasonable S/N, and a continuum. RJC note - This argument may only be + better when the object being used has a strong continuum + emission + lines. The reference spectrum is assigned a value of 1 for all + wavelengths, and the weights of all other spectra will be determined + relative to the reference spectrum. This is particularly useful if you + are dealing with highly variable spectra (e.g. emission lines) and + require a precision better than ~1 per cent. + 'ivar': + Use inverse variance weighting. This is not well tested and should probably be deprecated. verbose : bool, optional Verbosity of print out. @@ -798,37 +855,29 @@ def sn_weights(fluxes, ivars, gpms, sn_smooth_npix=None, const_weights=False, vectors) provided in waves, i.e. it is a list of arrays of type `numpy.ndarray`_ with the same shape as those in waves. """ + if weight_method not in ['auto', 'constant', 'wave_dependent', 'relative', 'ivar']: + msgs.error('Unrecognized option for weight_method=%s').format(weight_method) nexp = len(fluxes) # Check that all the input lists have the same length if len(ivars) != nexp or len(gpms) != nexp: msgs.error("Input lists of spectra must have the same length") - # Check sn_smooth_npix is set if const_weights=False - if sn_smooth_npix is None and not const_weights: - msgs.error('sn_smooth_npix cannot be None if const_weights=False') + # Check that sn_smooth_npix if weight_method = constant or uniform + if sn_smooth_npix is None and weight_method not in ['constant', 'uniform']: + msgs.error("sn_smooth_npix cannot be None unless the weight_method='constant' or weight_method='uniform'") - # Give preference to ivar_weights - if ivar_weights and relative_weights: - msgs.warn("Performing inverse variance weights instead of relative weighting") - relative_weights = False - - # Calculate S/N - sn_val, rms_sn, sn2 = [], [], [] - for iexp in range(nexp): - sn_val_iexp = fluxes[iexp]*np.sqrt(ivars[iexp]) - sn_val.append(sn_val_iexp) - sn_val_ma = np.ma.array(sn_val_iexp, mask=np.logical_not(gpms[iexp])) - sn_sigclip = stats.sigma_clip(sn_val_ma, sigma=3, maxiters=5) - sn2_iexp = sn_sigclip.mean()**2 # S/N^2 value for each spectrum - if sn2_iexp is np.ma.masked: - msgs.error(f'No unmasked value in iexp={iexp+1}/{nexp}. Check inputs.') - else: - sn2.append(sn2_iexp) - rms_sn.append(np.sqrt(sn2_iexp)) # Root Mean S/N**2 value for all spectra + rms_sn, sn_val = calc_snr(fluxes, ivars, gpms) + sn2 = np.square(rms_sn) # Check if relative weights input - if relative_weights: + if verbose: + msgs.info('Computing weights with weight_method=%s'.format(weight_method)) + + weights = [] + + weight_method_used = [] if weight_method is 'auto' else nexp*[weight_method] + if weight_method == 'relative': # Relative weights are requested, use the highest S/N spectrum as a reference ref_spec = np.argmax(sn2) if verbose: @@ -836,43 +885,39 @@ def sn_weights(fluxes, ivars, gpms, sn_smooth_npix=None, const_weights=False, "The reference spectrum (ref_spec={0:d}) has a typical S/N = {1:.3f}".format(ref_spec, sn2[ref_spec])) # Adjust the arrays to be relative refscale = utils.inverse(sn_val[ref_spec]) - gpms_rel = [] for iexp in range(nexp): # Compute the relative (S/N)^2 and update the mask sn2[iexp] /= sn2[ref_spec] - gpms_rel.append(gpms[iexp] & ((gpms[ref_spec]) | (sn_val[ref_spec] != 0))) - sn_val[iexp] *= refscale - - sn_gpms = gpms_rel - else: - sn_gpms = gpms - - # TODO: Should ivar weights be deprecated?? - # Initialise weights - weights = [] - if ivar_weights: - if verbose: - msgs.info("Using ivar weights for merging orders") + gpm_rel = gpms[iexp] & ((gpms[ref_spec]) | (sn_val[ref_spec] != 0)) + sn_val_rescaled = sn_val[iexp]*refscale + weights.append(smooth_weights(sn_val_rescaled** 2, gpm_rel, sn_smooth_npix)) + elif weight_method == 'ivar': + # TODO: Should ivar weights be deprecated?? for ivar, mask in zip(ivars, gpms): weights.append(smooth_weights(ivar, mask, sn_smooth_npix)) - else: + elif weight_method == 'constant': for iexp in range(nexp): - # Now - if (rms_sn[iexp] < 3.0) or const_weights: - weight_method = 'constant' + weights.append(np.full(fluxes[iexp].size, np.fmax(sn2[iexp], 1e-2))) # set the minimum to be 1e-2 to avoid zeros + elif weight_method == 'uniform': + for iexp in range(nexp): + weights.append( + np.full(fluxes[iexp].size, 1.0)) + elif weight_method == 'wave_dependent': + for iexp in range(nexp): + weights.append(smooth_weights(sn_val[iexp] ** 2, gpms[iexp], sn_smooth_npix)) + elif weight_method == 'auto': + for iexp in range(nexp): + if rms_sn[iexp] < 3.0: weights.append(np.full(fluxes[iexp].size, np.fmax(sn2[iexp], 1e-2))) # set the minimum to be 1e-2 to avoid zeros + weight_method_used.append('constant') else: - weight_method = 'wavelength dependent' - # JFH THis line is experimental but it deals with cases where the spectrum drops to zero. We thus - # transition to using ivar_weights. This needs more work because the spectra are not rescaled at this point. - # RJC - also note that nothing should be changed to sn_val is relative_weights=True - #sn_val[sn_val[:, iexp] < 1.0, iexp] = ivar_stack[sn_val[:, iexp] < 1.0, iexp] - weights.append(smooth_weights(sn_val[iexp]**2, sn_gpms[iexp], sn_smooth_npix)) - if verbose: - msgs.info('Using {:s} weights for coadding, S/N '.format(weight_method) + - '= {:4.2f}, weight = {:4.2f} for {:}th exposure'.format( - rms_sn[iexp], np.mean(weights[iexp]), iexp)) + weights.append(smooth_weights(sn_val[iexp]**2, gpms[iexp], sn_smooth_npix)) + weight_method_used.append('wavelength dependent') + if verbose: + for iexp in range(nexp): + msgs.info('Using {:s} weights for coadding, S/N '.format(weight_method_used[iexp]) + + '= {:4.2f}, weight = {:4.2f} for {:}th exposure'.format(rms_sn[iexp], np.mean(weights[iexp]), iexp)) # Finish return np.array(rms_sn), weights @@ -1170,7 +1215,7 @@ def scale_spec(wave, flux, ivar, sn, wave_ref, flux_ref, ivar_ref, mask=None, ma msgs.error("Scale method not recognized! Check documentation for available options") # Finish if show: - scale_spec_qa(wave, flux, ivar, wave_ref, flux_ref, ivar_ref, scale, method_used, mask = mask, mask_ref=mask_ref, + scale_spec_qa(wave, flux*mask, ivar*mask, wave_ref, flux_ref*mask_ref, ivar_ref*mask_ref, scale, method_used, mask = mask, mask_ref=mask_ref, title='Scaling Applied to the Data') return flux_scale, ivar_scale, scale, method_used @@ -1851,8 +1896,6 @@ def spec_reject_comb(wave_grid, wave_grid_mid, waves_list, fluxes_list, ivars_li qdone = False while (not qdone) and (iter < maxiter_reject): # Compute the stack - #from IPython import embed - #embed() wave_stack, flux_stack, ivar_stack, gpm_stack, nused = compute_stack( wave_grid, waves_list, fluxes_list, ivars_list, utils.array_to_explist(this_gpms, nspec_list=nspec_list), weights_list) # Interpolate the individual spectra onto the wavelength grid of the stack. Use wave_grid_mid for this @@ -2022,10 +2065,9 @@ def combspec(waves, fluxes, ivars, gpms, sn_smooth_npix, ref_percentile=70.0, maxiter_scale=5, wave_grid_input=None, sigrej_scale=3.0, scale_method='auto', hand_scale=None, sn_min_polyscale=2.0, sn_min_medscale=0.5, - const_weights=False, maxiter_reject=5, sn_clip=30.0, + weight_method='auto', maxiter_reject=5, sn_clip=30.0, lower=3.0, upper=3.0, maxrej=None, qafile=None, title='', debug=False, - debug_scale=False, debug_order_stack=False, - debug_global_stack=False, show_scale=False, show=False, verbose=True): + debug_scale=False, show_scale=False, show=False, verbose=True): ''' Main driver routine for coadding longslit/multi-slit spectra. @@ -2100,8 +2142,9 @@ def combspec(waves, fluxes, ivars, gpms, sn_smooth_npix, maximum SNR for perforing median scaling sn_min_medscale : float, optional, default = 0.5 minimum SNR for perforing median scaling - const_weights : bool, optional - If True, apply constant weight + weight_method (str): + Weight method to use for coadding spectra (see + :func:`~pypeit.core.coadd.sn_weights`) for documentation. Default='auto' sn_clip: float, optional, default=30.0 Errors are capped during rejection so that the S/N is never greater than sn_clip. This prevents overly aggressive rejection in high S/N ratio @@ -2157,6 +2200,12 @@ def combspec(waves, fluxes, ivars, gpms, sn_smooth_npix, spectrum on wave_stack wavelength grid. True=Good. shape=(ngrid,) ''' + + #debug_scale=True + #show_scale=True + #debug=True + #show=True + #from IPython import embed #embed() # We cast to float64 because of a bug in np.histogram @@ -2172,7 +2221,7 @@ def combspec(waves, fluxes, ivars, gpms, sn_smooth_npix, dwave=dwave, dv=dv, dloglam=dloglam, spec_samp_fact=spec_samp_fact) # Evaluate the sn_weights. This is done once at the beginning - rms_sn, weights = sn_weights(_fluxes, _ivars, gpms, sn_smooth_npix=sn_smooth_npix, const_weights=const_weights, verbose=verbose) + rms_sn, weights = sn_weights(_fluxes, _ivars, gpms, sn_smooth_npix=sn_smooth_npix, weight_method=weight_method, verbose=verbose) fluxes_scale, ivars_scale, scales, scale_method_used = scale_spec_stack( wave_grid, wave_grid_mid, _waves, _fluxes, _ivars, gpms, rms_sn, weights, ref_percentile=ref_percentile, maxiter_scale=maxiter_scale, sigrej_scale=sigrej_scale, scale_method=scale_method, hand_scale=hand_scale, @@ -2192,7 +2241,7 @@ def multi_combspec(waves, fluxes, ivars, masks, sn_smooth_npix=None, wave_method='linear', dwave=None, dv=None, dloglam=None, spec_samp_fact=1.0, wave_grid_min=None, wave_grid_max=None, ref_percentile=70.0, maxiter_scale=5, sigrej_scale=3.0, scale_method='auto', hand_scale=None, sn_min_polyscale=2.0, sn_min_medscale=0.5, - const_weights=False, maxiter_reject=5, sn_clip=30.0, lower=3.0, upper=3.0, + weight_method='auto', maxiter_reject=5, sn_clip=30.0, lower=3.0, upper=3.0, maxrej=None, qafile=None, debug=False, debug_scale=False, show_scale=False, show=False): """ @@ -2267,8 +2316,9 @@ def multi_combspec(waves, fluxes, ivars, masks, sn_smooth_npix=None, maximum SNR for perforing median scaling sn_min_medscale : float, optional minimum SNR for perforing median scaling - const_weights : `numpy.ndarray`_, optional - Constant weight factors specified + weight_method (str): + Weight method to use for coadding spectra (see + :func:`~pypeit.core.coadd.sn_weights`) for documentation. Default='auto' maxiter_reject : int, optional maximum number of iterations for stacking and rejection. The code stops iterating either when the output mask does not change betweeen @@ -2333,7 +2383,7 @@ def multi_combspec(waves, fluxes, ivars, masks, sn_smooth_npix=None, spec_samp_fact=spec_samp_fact, wave_grid_min=wave_grid_min, wave_grid_max=wave_grid_max, ref_percentile=ref_percentile, maxiter_scale=maxiter_scale, sigrej_scale=sigrej_scale, scale_method=scale_method, hand_scale=hand_scale, sn_min_medscale=sn_min_medscale, sn_min_polyscale=sn_min_polyscale, sn_smooth_npix=sn_smooth_npix, - const_weights=const_weights, maxiter_reject=maxiter_reject, sn_clip=sn_clip, lower=lower, upper=upper, + weight_method=weight_method, maxiter_reject=maxiter_reject, sn_clip=sn_clip, lower=lower, upper=upper, maxrej=maxrej, qafile=qafile, title='multi_combspec', debug=debug, debug_scale=debug_scale, show_scale=show_scale, show=show) @@ -2348,7 +2398,7 @@ def ech_combspec(waves_arr_setup, fluxes_arr_setup, ivars_arr_setup, gpms_arr_se ref_percentile=70.0, maxiter_scale=5, niter_order_scale=3, sigrej_scale=3.0, scale_method='auto', hand_scale=None, sn_min_polyscale=2.0, sn_min_medscale=0.5, - sn_smooth_npix=None, const_weights=False, maxiter_reject=5, + maxiter_reject=5, sn_clip=30.0, lower=3.0, upper=3.0, maxrej=None, qafile=None, debug_scale=False, debug_order_stack=False, debug_global_stack=False, debug=False, @@ -2436,8 +2486,6 @@ def ech_combspec(waves_arr_setup, fluxes_arr_setup, ivars_arr_setup, gpms_arr_se maximum SNR for perforing median scaling sn_min_medscale : float, optional, default = 0.5 minimum SNR for perforing median scaling - const_weights : bool, optional - If True, apply constant weight maxiter_reject : int, optional, default=5 maximum number of iterations for stacking and rejection. The code stops iterating either when the output mask does not change betweeen @@ -2575,15 +2623,15 @@ def ech_combspec(waves_arr_setup, fluxes_arr_setup, ivars_arr_setup, gpms_arr_se # Decide how much to smooth the spectra by if this number was not passed in nspec_good = [] ngood = [] - if sn_smooth_npix is None: - # Loop over setups - for wave, norder, nexp in zip(waves_arr_setup, norders, nexps): - # This is the effective good number of spectral pixels in the stack - nspec_good.append(np.sum(wave > 1.0)) - ngood.append(norder*nexp) - nspec_eff = np.sum(nspec_good)/np.sum(ngood) - sn_smooth_npix = int(np.round(0.1 * nspec_eff)) - msgs.info('Using a sn_smooth_pix={:d} to decide how to scale and weight your spectra'.format(sn_smooth_npix)) + #if sn_smooth_npix is None: + # # Loop over setups + # for wave, norder, nexp in zip(waves_arr_setup, norders, nexps): + # # This is the effective good number of spectral pixels in the stack + # nspec_good.append(np.sum(wave > 1.0)) + # ngood.append(norder*nexp) + # nspec_eff = np.sum(nspec_good)/np.sum(ngood) + # sn_smooth_npix = int(np.round(0.1 * nspec_eff)) + # msgs.info('Using a sn_smooth_pix={:d} to decide how to scale and weight your spectra'.format(sn_smooth_npix)) # Create the setup lists waves_setup_list = [utils.echarr_to_echlist(wave)[0] for wave in waves_arr_setup] @@ -2607,8 +2655,7 @@ def ech_combspec(waves_arr_setup, fluxes_arr_setup, ivars_arr_setup, gpms_arr_se rms_sn_setup_list = [] colors = [] for isetup in range(nsetups): - rms_sn_vec, _ = sn_weights(fluxes_setup_list[isetup], ivars_setup_list[isetup], gpms_setup_list[isetup], - sn_smooth_npix=sn_smooth_npix, const_weights=const_weights, verbose=verbose) + rms_sn_vec, _ = calc_snr(fluxes_setup_list[isetup], ivars_setup_list[isetup], gpms_setup_list[isetup]) rms_sn = rms_sn_vec.reshape(norders[isetup], nexps[isetup]) mean_sn_ord = np.mean(rms_sn, axis=1) best_orders = np.argsort(mean_sn_ord)[::-1][0:_nbests[isetup]] diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 378e57ce81..a59cc07da1 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -722,7 +722,8 @@ def save_coadd1d_tofits(outfile, wave, flux, ivar, gpm, wave_grid_mid=None, spec wave_gpm = wave > 1.0 spec = onespec.OneSpec(wave=wave[wave_gpm], wave_grid_mid=None if wave_grid_mid is None else wave_grid_mid[wave_gpm], - flux=flux[wave_gpm], PYP_SPEC=spectrograph, ivar=ivar[wave_gpm], mask=gpm[wave_gpm].astype(int), + flux=flux[wave_gpm], PYP_SPEC=spectrograph, ivar=ivar[wave_gpm], sigma=np.sqrt(utils.inverse(ivar[wave_gpm])), + mask=gpm[wave_gpm].astype(int), telluric=None if telluric is None else telluric[wave_gpm], obj_model=None if obj_model is None else obj_model[wave_gpm], ext_mode=ex_value, fluxed=True) diff --git a/pypeit/core/wavecal/wvutils.py b/pypeit/core/wavecal/wvutils.py index 63c15dad94..126d386a8d 100644 --- a/pypeit/core/wavecal/wvutils.py +++ b/pypeit/core/wavecal/wvutils.py @@ -244,7 +244,6 @@ def get_wave_grid(waves=None, gpms=None, wave_method='linear', iref=0, wave_grid dloglam_pix = dloglam else: dloglam_pix = dloglam_data - # Generate wavelength array wave_grid, wave_grid_mid, dsamp = wavegrid(wave_grid_min, wave_grid_max, dloglam_pix, spec_samp_fact=spec_samp_fact, log10=True) diff --git a/pypeit/onespec.py b/pypeit/onespec.py index 61faae4eed..a381174a53 100644 --- a/pypeit/onespec.py +++ b/pypeit/onespec.py @@ -46,7 +46,7 @@ class OneSpec(datamodel.DataContainer): Build from PYP_SPEC """ - version = '1.0.1' + version = '1.0.2' datamodel = {'wave': dict(otype=np.ndarray, atype=np.floating, # TODO: The "weighted by pixel contributions" part @@ -62,6 +62,8 @@ class OneSpec(datamodel.DataContainer): 'see ``fluxed``'), 'ivar': dict(otype=np.ndarray, atype=np.floating, descr='Inverse variance array (matches units of flux)'), + 'sigma': dict(otype=np.ndarray, atype=np.floating, + descr='One sigma noise array, equivalent to 1/sqrt(ivar) (matches units of flux)'), 'mask': dict(otype=np.ndarray, atype=np.integer, descr='Mask array (1=Good,0=Bad)'), 'telluric': dict(otype=np.ndarray, atype=np.floating, descr='Telluric model'), @@ -104,7 +106,7 @@ def from_file(cls, ifile): # return slf - def __init__(self, wave, wave_grid_mid, flux, PYP_SPEC=None, ivar=None, mask=None, telluric=None, + def __init__(self, wave, wave_grid_mid, flux, PYP_SPEC=None, ivar=None, sigma=None, mask=None, telluric=None, obj_model=None, ext_mode=None, fluxed=None): args, _, _, values = inspect.getargvalues(inspect.currentframe()) @@ -118,14 +120,14 @@ def _bundle(self): """ return super()._bundle(ext='SPECTRUM') - @property - def sig(self): - """ Return the 1-sigma array - - Returns: - `numpy.ndarray`_: error array - """ - return np.sqrt(utils.inverse(self.ivar)) + #@property + #def sig(self): + # """ Return the 1-sigma array + # + # Returns: + # `numpy.ndarray`_: error array + # """ + # return np.sqrt(utils.inverse(self.ivar)) def to_file(self, ofile, primary_hdr=None, history=None, **kwargs): """ diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index e3732266cc..0068325d03 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -996,7 +996,8 @@ class Coadd1DPar(ParSet): def __init__(self, ex_value=None, flux_value=None, nmaskedge=None, sn_smooth_npix=None, sigrej_exp=None, wave_method=None, dv=None, dwave=None, dloglam=None, wave_grid_min=None, wave_grid_max=None, spec_samp_fact=None, ref_percentile=None, maxiter_scale=None, - sigrej_scale=None, scale_method=None, sn_min_medscale=None, sn_min_polyscale=None, maxiter_reject=None, + sigrej_scale=None, scale_method=None, sn_min_medscale=None, sn_min_polyscale=None, + weight_method=None, maxiter_reject=None, lower=None, upper=None, maxrej=None, sn_clip=None, nbests=None, coaddfile=None, mag_type=None, filter=None, filter_mag=None, filter_mask=None, chk_version=None): @@ -1098,6 +1099,8 @@ def __init__(self, ex_value=None, flux_value=None, nmaskedge=None, dtypes['sigrej_scale'] = [int, float] descr['sigrej_scale'] = 'Rejection threshold used for rejecting pixels when rescaling spectra with scale_spec.' + + defaults['scale_method'] = 'auto' dtypes['scale_method'] = str descr['scale_method'] = "Method used to rescale the spectra prior to coadding. The options are:" \ @@ -1108,6 +1111,7 @@ def __init__(self, ex_value=None, flux_value=None, nmaskedge=None, "'none' -- Do not rescale. " \ "'hand' -- Pass in hand scaling factors. This option is not well tested." + defaults['sn_min_medscale'] = 0.5 dtypes['sn_min_medscale'] = [int, float] descr['sn_min_medscale'] = "For scale method set to ``auto``, this sets the minimum SNR for which median scaling is attempted." @@ -1116,6 +1120,26 @@ def __init__(self, ex_value=None, flux_value=None, nmaskedge=None, dtypes['sn_min_polyscale'] = [int, float] descr['sn_min_polyscale'] = "For scale method set to ``auto``, this sets the minimum SNR for which polynomial scaling is attempted." + defaults['weight_method'] = 'auto' + dtypes['weight_method'] = str + descr['weight_method'] = "Method used to rescale the spectra prior to coadding. The options are:" \ + " " \ + "'auto' -- Use constant weights if rms_sn < 3.0, otherwise use wavelength dependent." \ + "'constant' -- Constant weights based on rms_sn**2" \ + "'uniform' -- Uniform weighting" \ + "'wave_dependent' -- Wavelength dependent weights will be used irrespective of the rms_" \ + "sn ratio. This option will not work well at low S/N ratio although it is useful for " \ + "objects where only a small fraction of the spectral coverage has high S/N ratio " \ + "(like high-z quasars)." \ + "'relative' -- Apply relative weights implying one reference exposure will receive unit " \ + "weight at all wavelengths and all others receive relatively wavelength dependent "\ + "weights . Note, relative weighting will only work well " \ + "when there is at least one spectrum with a reasonable S/N, and a continuum. " \ + "This option may only be better when the object being used has a strong " \ + "continuum + emission lines. This is particularly useful if you " \ + "are dealing with highly variable spectra (e.g. emission lines) and" \ + "require a precision better than ~1 per cent." \ + "'ivar' -- Use inverse variance weighting. This is not well tested and should probably be deprecated." defaults['maxiter_reject'] = 5 dtypes['maxiter_reject'] = int @@ -1190,7 +1214,7 @@ def from_dict(cls, cfg): parkeys = ['ex_value', 'flux_value', 'nmaskedge', 'sn_smooth_npix', 'sigrej_exp', 'wave_method', 'dv', 'dwave', 'dloglam', 'wave_grid_min', 'wave_grid_max', 'spec_samp_fact', 'ref_percentile', 'maxiter_scale', 'sigrej_scale', 'scale_method', - 'sn_min_medscale', 'sn_min_polyscale', 'maxiter_reject', 'lower', 'upper', + 'sn_min_medscale', 'sn_min_polyscale', 'weight_method', 'maxiter_reject', 'lower', 'upper', 'maxrej', 'sn_clip', 'nbests', 'coaddfile', 'chk_version', 'filter', 'mag_type', 'filter_mag', 'filter_mask'] @@ -1207,7 +1231,15 @@ def validate(self): """ Check the parameters are valid for the provided method. """ - pass + allowed_scale_methods = ['auto', 'poly', 'median', 'none', 'hand'] + if self.data['scale_method'] not in allowed_scale_methods: + raise ValueError("If 'wave_method' is not None it must be one of:\n" + ", ".join(allowed_scale_methods)) + + allowed_weight_methods = ['auto', 'constant', 'uniform', 'wave_dependent', 'relative', 'ivar'] + if self.data['weight_method'] not in allowed_scale_methods: + raise ValueError("If 'weight_method' is not None it must be one of:\n" + ", ".join(allowed_weight_methods)) + + @staticmethod def valid_ex(): @@ -1351,6 +1383,7 @@ def validate(self): + class CubePar(ParSet): """ The parameter set used to hold arguments for functionality relevant diff --git a/pypeit/specobj.py b/pypeit/specobj.py index 530dfef9a1..4644f11d2a 100644 --- a/pypeit/specobj.py +++ b/pypeit/specobj.py @@ -578,8 +578,10 @@ def to_arrays(self, extraction='OPT', fluxed=True): Convert spectrum into np.ndarray arrays Args: - extraction (str): Extraction method to convert + extraction (str): + Extraction method to convert fluxed: + Use the fluxed tags Returns: tuple: wave, flux, ivar, mask arrays @@ -599,7 +601,7 @@ def to_arrays(self, extraction='OPT', fluxed=True): # Return return self[swave], self[sflux], self[sivar], self[smask] - def to_xspec1d(self, masked=False, **kwargs): + def to_xspec1d(self, masked=True, extraction='OPT', fluxed=True): """ Create an `XSpectrum1D `_ using this spectrum. @@ -607,18 +609,22 @@ def to_xspec1d(self, masked=False, **kwargs): Args: masked (:obj:`bool`, optional): If True, only unmasked data are included. - kwargs (:obj:`dict`, optional): - Passed directly to :func:`to_arrays`. + extraction (str): + Extraction method to convert + fluxed: + Use the fluxed tags Returns: `linetools.spectra.xspectrum1d.XSpectrum1D`_: Spectrum object """ - wave, flux, ivar, gpm = self.to_arrays(**kwargs) + wave, flux, ivar, gpm = self.to_arrays(extraction=extraction, fluxed=fluxed) sig = np.sqrt(utils.inverse(ivar)) + wave_gpm = wave > 1.0 + wave, flux, sig, gpm = wave[wave_gpm], flux[wave_gpm], sig[wave_gpm], gpm[wave_gpm] if masked: - wave = wave[gpm] - flux = flux[gpm] - sig = sig[gpm] + flux = flux*gpm + sig = sig*gpm + # Create return xspectrum1d.XSpectrum1D.from_tuple((wave, flux, sig)) diff --git a/pypeit/specutils/pypeit_loaders.py b/pypeit/specutils/pypeit_loaders.py index 3cd8cee4c8..63fa9e8f81 100644 --- a/pypeit/specutils/pypeit_loaders.py +++ b/pypeit/specutils/pypeit_loaders.py @@ -179,9 +179,10 @@ def pypeit_onespec_loader(filename, grid=False, **kwargs): except AttributeError: name = '' + # TODO We should be dealing with masking here return Spectrum1D(flux=astropy.units.Quantity(spec.flux * flux_unit), - uncertainty=None if spec.ivar is None - else astropy.nddata.InverseVariance(spec.ivar / flux_unit**2), + uncertainty=None if spec.sigma is None else astropy.units.Quantity(spec.sigma * flux_unit), + # else astropy.nddata.InverseVariance(spec.ivar / flux_unit**2), meta={'name': name, 'extract': spec.ext_mode, 'fluxed': spec.fluxed, 'grid': grid}, spectral_axis=astropy.units.Quantity(wave * astropy.units.angstrom), From 341ad0512bb461abb7e22799a00f40b3c4b5077e Mon Sep 17 00:00:00 2001 From: joe Date: Fri, 29 Sep 2023 23:54:10 +0200 Subject: [PATCH 063/244] Hopefully made correct modifications to 2d coadding. --- pypeit/coadd1d.py | 6 +++--- pypeit/coadd2d.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pypeit/coadd1d.py b/pypeit/coadd1d.py index b4386a1103..a6eda80977 100644 --- a/pypeit/coadd1d.py +++ b/pypeit/coadd1d.py @@ -323,9 +323,9 @@ def coadd(self): spec_samp_fact=self.par['spec_samp_fact'], ref_percentile=self.par['ref_percentile'], maxiter_scale=self.par['maxiter_scale'], sigrej_scale=self.par['sigrej_scale'], scale_method=self.par['scale_method'], sn_min_medscale=self.par['sn_min_medscale'], - sn_min_polyscale=self.par['sn_min_polyscale'], maxiter_reject=self.par['maxiter_reject'], - lower=self.par['lower'], upper=self.par['upper'], maxrej=self.par['maxrej'], sn_clip=self.par['sn_clip'], - debug=self.debug, show=self.show) + sn_min_polyscale=self.par['sn_min_polyscale'], weight_method = self.par['weight_method'], + maxiter_reject=self.par['maxiter_reject'], lower=self.par['lower'], upper=self.par['upper'], + maxrej=self.par['maxrej'], sn_clip=self.par['sn_clip'], debug=self.debug, show=self.show) diff --git a/pypeit/coadd2d.py b/pypeit/coadd2d.py index 16e19ed909..41389f25b8 100644 --- a/pypeit/coadd2d.py +++ b/pypeit/coadd2d.py @@ -465,7 +465,7 @@ def good_slitindx(self, only_slits=None, exclude_slits=None): # these are the good slit index excluding the slits that are selected by the user return np.delete(good_slitindx, exclude_slitindx) - def optimal_weights(self, slitorderid, objid, const_weights=False): + def optimal_weights(self, slitorderid, objid, weight_method='auto'): """ Determine optimal weights for 2d coadds. This script grabs the information from SpecObjs list for the object with specified slitid and objid and passes to coadd.sn_weights to determine the optimal weights for @@ -523,7 +523,7 @@ def optimal_weights(self, slitorderid, objid, const_weights=False): f'flux not available in slit/order = {slitorderid}') # TODO For now just use the zero as the reference for the wavelengths? Perhaps we should be rebinning the data though? - rms_sn, weights = coadd.sn_weights(fluxes, ivars, gpms, self.sn_smooth_npix, const_weights=const_weights) + rms_sn, weights = coadd.sn_weights(fluxes, ivars, gpms, sn_smooth_npix=self.sn_smooth_npix, weight_method=weight_method) return rms_sn, weights def coadd(self, interp_dspat=True): @@ -1397,7 +1397,7 @@ def compute_weights(self, weights): # adjustment for multislit to case 3) Bright object exists and parset `weights` is equal to 'auto' if (self.objid_bri is not None) and (weights == 'auto'): # compute weights using bright object - _, self.use_weights = self.optimal_weights(self.spatid_bri, self.objid_bri, const_weights=True) + _, self.use_weights = self.optimal_weights(self.spatid_bri, self.objid_bri, weight_method='constant') if self.par['coadd2d']['user_obj'] is not None: msgs.info(f'Weights computed using a unique reference object in slit={self.spatid_bri} provided by the user') else: @@ -1464,7 +1464,7 @@ def get_brightest_obj(self, specobjs_list, spat_ids): #remove_indx.append(iobj) # if there are objects on this slit left, we can proceed with computing rms_sn if len(fluxes) > 0: - rms_sn, weights = coadd.sn_weights(fluxes, ivars, gpms, const_weights=True) + rms_sn, _ = coadd.calc_snr(fluxes, ivars, gpms) imax = np.argmax(rms_sn) slit_snr_max[islit, iexp] = rms_sn[imax] objid_max[islit, iexp] = objid_this[imax] @@ -1748,7 +1748,7 @@ def get_brightest_obj(self, specobjs_list, nslits): f'object {sobjs[ind][0].ECH_OBJID} in order {sobjs[ind][0].ECH_ORDER}.') continue if flux is not None: - rms_sn, weights = coadd.sn_weights([flux], [ivar], [mask], const_weights=True) + rms_sn, _ = coadd.calc_snr([flux], [ivar], [mask]) order_snr[iord, iobj] = rms_sn bpm[iord, iobj] = False From 2175dc0ba3a3b377ce6f4e24e9829bf5bc9d11cc Mon Sep 17 00:00:00 2001 From: joe Date: Fri, 29 Sep 2023 23:58:58 +0200 Subject: [PATCH 064/244] minor syntax. --- pypeit/core/coadd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/core/coadd.py b/pypeit/core/coadd.py index 1315603b17..5bd7443efd 100644 --- a/pypeit/core/coadd.py +++ b/pypeit/core/coadd.py @@ -876,7 +876,7 @@ def sn_weights(fluxes, ivars, gpms, sn_smooth_npix=None, weight_method='auto', v weights = [] - weight_method_used = [] if weight_method is 'auto' else nexp*[weight_method] + weight_method_used = [] if weight_method == 'auto' else nexp*[weight_method] if weight_method == 'relative': # Relative weights are requested, use the highest S/N spectrum as a reference ref_spec = np.argmax(sn2) From 5b0115d42f8320ac483b085b59f6b6456fedaa5e Mon Sep 17 00:00:00 2001 From: joe Date: Tue, 3 Oct 2023 22:22:03 -0700 Subject: [PATCH 065/244] Bug fix in calc_snr and minor modification to coadd QA --- pypeit/coadd2d.py | 2 +- pypeit/core/coadd.py | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pypeit/coadd2d.py b/pypeit/coadd2d.py index 41389f25b8..2d63c9cab5 100644 --- a/pypeit/coadd2d.py +++ b/pypeit/coadd2d.py @@ -1749,7 +1749,7 @@ def get_brightest_obj(self, specobjs_list, nslits): continue if flux is not None: rms_sn, _ = coadd.calc_snr([flux], [ivar], [mask]) - order_snr[iord, iobj] = rms_sn + order_snr[iord, iobj] = rms_sn[0] bpm[iord, iobj] = False # If there are orders that have bpm = True for some objs and not for others, set bpm = True for all objs diff --git a/pypeit/core/coadd.py b/pypeit/core/coadd.py index 5bd7443efd..70be970c70 100644 --- a/pypeit/core/coadd.py +++ b/pypeit/core/coadd.py @@ -771,8 +771,8 @@ def calc_snr(fluxes, ivars, gpms): Returns ------- - rms_sn : list - List of length nexp root-mean-square S/N value for each input spectra where nexp=len(fluxes). + rms_sn (np.ndarray): + Array of shape (nexp,) of root-mean-square S/N value for each input spectra where nexp=len(fluxes). sn_val : list List of length nexp containing the wavelength dependent S/N arrays for each input spectrum, i.e. each element contains the array flux*sqrt(ivar) @@ -794,7 +794,7 @@ def calc_snr(fluxes, ivars, gpms): sn2.append(sn2_iexp) rms_sn.append(np.sqrt(sn2_iexp)) # Root Mean S/N**2 value for all spectra - return rms_sn, sn_val + return np.array(rms_sn), sn_val def sn_weights(fluxes, ivars, gpms, sn_smooth_npix=None, weight_method='auto', verbose=False): @@ -1445,7 +1445,7 @@ def scale_spec_qa(wave, flux, ivar, wave_ref, flux_ref, ivar_ref, ymult, # TODO: Change mask to gpm def coadd_iexp_qa(wave, flux, rejivar, mask, wave_stack, flux_stack, ivar_stack, mask_stack, - outmask, norder=None, title='', qafile=None): + outmask, norder=None, title='', qafile=None, show_telluric=False): """ Routine to creqate QA for showing the individual spectrum @@ -1481,6 +1481,8 @@ def coadd_iexp_qa(wave, flux, rejivar, mask, wave_stack, flux_stack, ivar_stack, Plot title qafile (:obj:`str`, optional): QA file name + show_telluric (:obj:`bool`, optional): + Show the atmospheric absorption if wavelengths > 9000A are covered by the spectrum """ @@ -1509,7 +1511,7 @@ def coadd_iexp_qa(wave, flux, rejivar, mask, wave_stack, flux_stack, ivar_stack, # TODO Use one of our telluric models here instead # Plot transmission - if (np.max(wave[mask]) > 9000.0): + if (np.max(wave[mask]) > 9000.0) and show_telluric: skytrans_file = data.get_skisim_filepath('atm_transmission_secz1.5_1.6mm.dat') skycat = np.genfromtxt(skytrans_file, dtype='float') scale = 0.8 * ymax From 2c6f1ebea236d6ac38a3a0fcd32cdb038ff0d90e Mon Sep 17 00:00:00 2001 From: joe Date: Tue, 3 Oct 2023 22:40:37 -0700 Subject: [PATCH 066/244] Minor QA changes. --- pypeit/core/coadd.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pypeit/core/coadd.py b/pypeit/core/coadd.py index 70be970c70..c9c492e760 100644 --- a/pypeit/core/coadd.py +++ b/pypeit/core/coadd.py @@ -1584,7 +1584,7 @@ def weights_qa(waves, weights, gpms, title='', colors=None): plt.show() def coadd_qa(wave, flux, ivar, nused, gpm=None, tell=None, - title=None, qafile=None): + title=None, qafile=None, show_telluric=False): """ Routine to make QA plot of the final stacked spectrum. It works for both longslit/mulitslit, coadded individual order spectrum of the Echelle data @@ -1609,6 +1609,8 @@ def coadd_qa(wave, flux, ivar, nused, gpm=None, tell=None, plot title qafile : str, optional QA file name + show_telluric : bool, optional + Show a telluric absorptoin model on top of the data if wavelengths cover > 9000A """ #TODO: This routine should take a parset @@ -1639,7 +1641,7 @@ def coadd_qa(wave, flux, ivar, nused, gpm=None, tell=None, ymin, ymax = get_ylim(flux, ivar, gpm) # Plot transmission - if (np.max(wave[gpm])>9000.0) and (tell is None): + if (np.max(wave[gpm])>9000.0) and (tell is None) and show_telluric: skytrans_file = data.get_skisim_filepath('atm_transmission_secz1.5_1.6mm.dat') skycat = np.genfromtxt(skytrans_file,dtype='float') scale = 0.8*ymax From aba421633101f3b19f30cb2a30829f370e2b64c5 Mon Sep 17 00:00:00 2001 From: rcooke Date: Wed, 4 Oct 2023 09:39:03 +0100 Subject: [PATCH 067/244] tmp fix --- pypeit/coadd3d.py | 67 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index b4a7f5887f..a7eeb787fd 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -1127,6 +1127,20 @@ def load(self): this_specpos, this_spatpos = np.where(onslit_gpm) this_spatid = slitid_img_init[onslit_gpm] + ################################## + # Astrometric alignment to HST frames + # TODO :: RJC requests this remains here... it is only used by RJC + _ra_sort, _dec_sort = hst_alignment(ra_sort, dec_sort, wave_sort, flux_sort, ivar_sort, np.max(self._spatscale[ff,:]), self._specscale[ff], + np.ones(ra_sort.size), this_spatpos[wvsrt], this_specpos[wvsrt], + this_spatid[wvsrt], spec2DObj.tilts, slits, alignSplines, darcorr) + ra_del = np.median(_ra_sort-ra_sort) + dec_del = np.median(_dec_sort-dec_sort) + ra_sort = _ra_sort + dec_sort = _dec_sort + spec2DObj.head0['RA'] += ra_del + spec2DObj.head0['DEC'] += dec_del + ################################## + # If individual frames are to be output without aligning them, # there's no need to store information, just make the cubes now numpix = ra_sort.size @@ -1319,7 +1333,7 @@ def coadd(self): self.set_voxel_sampling() # Align the frames - if self.align: + if self.align and False: self.run_align() # Compute the relative weights on the spectra @@ -1824,3 +1838,54 @@ def subpixellate(output_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_w residcube *= nc_inverse return flxcube, varcube, bpmcube, residcube return flxcube, varcube, bpmcube + +def hst_alignment(ra_sort, dec_sort, wave_sort, flux_sort, ivar_sort, dspat, dwave, + wghts, spatpos, specpos, + all_spatid, tilts, slits, astrom_trans, all_dar, + spat_subpixel=10, spec_subpixel=10): + """ + This is currently only used by RJC. This function adds corrections to the RA and Dec pixels + to align the daatcubes to an HST image. + + Process: + * Send away pixel RA, Dec, wave, flux, error. + * ------ + * Compute emission line map + - Need to generate full cube around H I gamma + - Fit to continuum and subtract it off + - Sum all flux above continuum + - Estimate error + * MPFIT HST emission line map to + * ------ + * Return updated pixel RA, Dec + """ + from pypeit import astrometry + ############ + ## STEP 1 ## - Create a datacube around Hgamma + ############ + # Only use a small wavelength range + wv_mask = (wave_sort > 4346.0) & (wave_sort < 4358.0) + # Create a WCS for this subcube + subcube_wcs, voxedge, reference_image = datacube.create_wcs(ra_sort[wv_mask], dec_sort[wv_mask], wave_sort[wv_mask], + dspat, dwave) + # Create the subcube + flxcube, varcube, bpmcube = subpixellate(subcube_wcs, ra_sort[wv_mask], dec_sort[wv_mask], wave_sort[wv_mask], flux_sort[wv_mask], ivar_sort[wv_mask], + wghts[wv_mask], spatpos[wv_mask], specpos[wv_mask], + all_spatid[wv_mask], tilts, slits, astrom_trans, all_dar, + voxedge, all_idx=None, + spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, debug=False) + if False: + hdu = fits.PrimaryHDU(flxcube) + hdu.writeto("tstHg.fits", overwrite=True) + + ############ + ## STEP 2 ## - Create an emission line map of Hgamma + ############ + # Compute an emission line map that is as consistent as possible to an archival HST image + HgMap, HgMapErr = astrometry.fit_cube(flxcube.T, varcube.T, subcube_wcs) + ############ + ## STEP 3 ## - Map the emission line map to an HST image, and vice-versa + ############ + ra_corr, dec_corr = astrometry.map_image(HgMap, HgMapErr, subcube_wcs, ra_sort, dec_sort) + # embed() + return ra_corr, dec_corr \ No newline at end of file From e1ece79417cc602a998bacf97f43df6785c4958e Mon Sep 17 00:00:00 2001 From: joe Date: Wed, 4 Oct 2023 14:15:04 -0700 Subject: [PATCH 068/244] Minor QA changes. --- pypeit/coadd2d.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pypeit/coadd2d.py b/pypeit/coadd2d.py index 2d63c9cab5..4f792fd7a2 100644 --- a/pypeit/coadd2d.py +++ b/pypeit/coadd2d.py @@ -778,8 +778,8 @@ def reduce(self, pseudo_dict, show=False, clear_ginga=True, show_peaks=False, sh # Make changes to parset specific to 2d coadds parcopy = copy.deepcopy(self.par) - parcopy['reduce']['findobj']['trace_npoly'] = 3 # Low order traces since we are rectified - + # Enforce low order traces since we are rectified + parcopy['reduce']['findobj']['trace_npoly'] = int(np.clip(parcopy['reduce']['findobj']['trace_npoly'],None,3)) # Manual extraction. manual_obj = None if self.par['coadd2d']['manual'] is not None and len(self.par['coadd2d']['manual']) > 0: From fd778ed0fbd3d1036506e03e937404b188519527 Mon Sep 17 00:00:00 2001 From: joe Date: Wed, 11 Oct 2023 23:31:06 +0200 Subject: [PATCH 069/244] overwrote datacube.py with Ryan's version --- pypeit/core/datacube.py | 1204 +++++++++++---------------------------- 1 file changed, 325 insertions(+), 879 deletions(-) diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index 56502c1d51..499e14c5fd 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -17,6 +17,9 @@ from pypeit import utils from pypeit.core import coadd, flux_calib +# Use a fast histogram for speed! +from fast_histogram import histogramdd + from IPython import embed @@ -213,8 +216,9 @@ def extract_standard_spec(stdcube, subpixel=20): # Setup the WCS stdwcs = wcs.WCS(stdcube['FLUX'].header) - wcs_wav = stdwcs.wcs_pix2world(np.vstack((np.zeros(numwave), np.zeros(numwave), np.arange(numwave))).T, 0) - wave = wcs_wav[:, 2] * 1.0E10 * units.AA + + wcs_scale = (1.0 * stdwcs.spectral.wcs.cunit[0]).to(units.Angstrom).value # Ensures the WCS is in Angstroms + wave = wcs_scale * stdwcs.spectral.wcs_pix2world(np.arange(numwave), 0)[0] # Generate a whitelight image, and fit a 2D Gaussian to estimate centroid and width wl_img = make_whitelight_fromcube(flxcube) @@ -317,7 +321,7 @@ def make_sensfunc(ss_file, senspar, blaze_wave=None, blaze_spline=None, grating_ blaze_spline_curr = interp1d(blaze_wave_curr, blaze_spec_curr, kind='linear', bounds_error=False, fill_value="extrapolate") # Perform a grating correction - grat_corr = correct_grating_shift(wave.value, blaze_wave_curr, blaze_spline_curr, blaze_wave, blaze_spline) + grat_corr = correct_grating_shift(wave, blaze_wave_curr, blaze_spline_curr, blaze_wave, blaze_spline) # Apply the grating correction to the standard star spectrum Nlam_star /= grat_corr Nlam_ivar_star *= grat_corr ** 2 @@ -330,7 +334,7 @@ def make_sensfunc(ss_file, senspar, blaze_wave=None, blaze_spline=None, grating_ # TODO :: This needs to be addressed... unify flux calibration into the main PypeIt routines. msgs.warn("Datacubes are currently flux-calibrated using the UVIS algorithm... this will be deprecated soon") zeropoint_data, zeropoint_data_gpm, zeropoint_fit, zeropoint_fit_gpm = \ - flux_calib.fit_zeropoint(wave.value, Nlam_star, Nlam_ivar_star, gpm_star, std_dict, + flux_calib.fit_zeropoint(wave, Nlam_star, Nlam_ivar_star, gpm_star, std_dict, mask_hydrogen_lines=senspar['mask_hydrogen_lines'], mask_helium_lines=senspar['mask_helium_lines'], hydrogen_mask_wid=senspar['hydrogen_mask_wid'], @@ -342,8 +346,7 @@ def make_sensfunc(ss_file, senspar, blaze_wave=None, blaze_spline=None, grating_ polyfunc=senspar['UVIS']['polyfunc']) wgd = np.where(zeropoint_fit_gpm) sens = np.power(10.0, -0.4 * (zeropoint_fit[wgd] - flux_calib.ZP_UNIT_CONST)) / np.square(wave[wgd]) - flux_spline = interp1d(wave[wgd], sens, kind='linear', bounds_error=False, fill_value="extrapolate") - return flux_spline + return interp1d(wave[wgd], sens, kind='linear', bounds_error=False, fill_value="extrapolate") def make_good_skymask(slitimg, tilts): @@ -399,18 +402,14 @@ def get_output_filename(fil, par_outfile, combine, idx=1): str: The output filename to use. """ if combine: - if par_outfile == "": - par_outfile = "datacube.fits" - # Check the output files don't exist - outfile = par_outfile if ".fits" in par_outfile else par_outfile + ".fits" - else: - if par_outfile == "": - outfile = fil.replace("spec2d_", "spec3d_") - else: - # Use the output filename as a prefix - outfile = os.path.splitext(par_outfile)[0] + "_{0:03d}.fits".format(idx) - # Return the outfile - return outfile + if par_outfile == '': + par_outfile = 'datacube.fits' + # Check if we needs to append an extension + return par_outfile if '.fits' in par_outfile else f'{par_outfile}.fits' + if par_outfile == '': + return fil.replace('spec2d_', 'spec3d_') + # Finally, if nothing else, use the output filename as a prefix, and a numerical suffic + return os.path.splitext(par_outfile)[0] + f'_{idx:03}.fits' def get_output_whitelight_filename(outfile): @@ -423,10 +422,9 @@ def get_output_whitelight_filename(outfile): The output filename used for the datacube. Returns: - str: The output filename to use for the whitelight image. + A string containing the output filename to use for the whitelight image. """ - out_wl_filename = os.path.splitext(outfile)[0] + "_whitelight.fits" - return out_wl_filename + return os.path.splitext(outfile)[0] + "_whitelight.fits" def get_whitelight_pixels(all_wave, min_wl, max_wl): @@ -515,7 +513,7 @@ def make_whitelight_fromcube(cube, wave=None, wavemin=None, wavemax=None): reduce the wavelength range. Returns: - `numpy.ndarray`_: Whitelight image of the input cube. + A whitelight image of the input cube (of type `numpy.ndarray`_). """ # Make a wavelength cut, if requested cutcube = cube.copy() @@ -591,8 +589,8 @@ def align_user_offsets(all_ra, all_dec, all_idx, ifu_ra, ifu_dec, ra_offset, dec A list of Dec offsets to be applied to the input pixel values (one value per frame). Returns: - `numpy.ndarray`_: A new set of RA values that have been aligned - `numpy.ndarray`_: A new set of Dec values that has been aligned + A tuple containing a new set of RA and Dec values that have been aligned. Both arrays + are of type `numpy.ndarray`_. """ # First, translate all coordinates to the coordinates of the first frame # Note: You do not need cos(dec) here, this just overrides the IFU coordinate centre of each frame @@ -604,11 +602,121 @@ def align_user_offsets(all_ra, all_dec, all_idx, ifu_ra, ifu_dec, ra_offset, dec # Apply the shift all_ra[all_idx == ff] += ref_shift_ra[ff] + ra_offset[ff] / 3600.0 all_dec[all_idx == ff] += ref_shift_dec[ff] + dec_offset[ff] / 3600.0 - msgs.info("Spatial shift of cube #{0:d}:" + msgs.newline() + - "RA, DEC (arcsec) = {1:+0.3f} E, {2:+0.3f} N".format(ff + 1, ra_offset[ff], dec_offset[ff])) + msgs.info("Spatial shift of cube #{0:d}:".format(ff + 1) + msgs.newline() + + "RA, DEC (arcsec) = {0:+0.3f} E, {1:+0.3f} N".format(ra_offset[ff], dec_offset[ff])) return all_ra, all_dec +def set_voxel_sampling(spatscale, specscale, dspat=None, dwv=None): + """ + This function checks if the spatial and spectral scales of all frames are consistent. + If the user has not specified either the spatial or spectral scales, they will be set here. + + Parameters + ---------- + spatscale : `numpy.ndarray`_ + 2D array, shape is (N, 2), listing the native spatial scales of N spec2d frames. + spatscale[:,0] refers to the spatial pixel scale of each frame + spatscale[:,1] refers to the slicer scale of each frame + Each element of the array must be in degrees + specscale : `numpy.ndarray`_ + 1D array listing the native spectral scales of multiple frames. The length of this array should be equal + to the number of frames you are using. Each element of the array must be in Angstrom + dspat: :obj:`float`, optional + Spatial scale to use as the voxel spatial sampling. If None, a new value will be derived based on the inputs + dwv: :obj:`float`, optional + Spectral scale to use as the voxel spectral sampling. If None, a new value will be derived based on the inputs + + Returns + ------- + _dspat : :obj:`float` + Spatial sampling + _dwv : :obj:`float` + Wavelength sampling + """ + # Make sure all frames have consistent pixel scales + ratio = (spatscale[:, 0] - spatscale[0, 0]) / spatscale[0, 0] + if np.any(np.abs(ratio) > 1E-4): + msgs.warn("The pixel scales of all input frames are not the same!") + spatstr = ", ".join(["{0:.6f}".format(ss) for ss in spatscale[:,0]*3600.0]) + msgs.info("Pixel scales of all input frames:" + msgs.newline() + spatstr + "arcseconds") + # Make sure all frames have consistent slicer scales + ratio = (spatscale[:, 1] - spatscale[0, 1]) / spatscale[0, 1] + if np.any(np.abs(ratio) > 1E-4): + msgs.warn("The slicer scales of all input frames are not the same!") + spatstr = ", ".join(["{0:.6f}".format(ss) for ss in spatscale[:,1]*3600.0]) + msgs.info("Slicer scales of all input frames:" + msgs.newline() + spatstr + "arcseconds") + # Make sure all frames have consistent wavelength sampling + ratio = (specscale - specscale[0]) / specscale[0] + if np.any(np.abs(ratio) > 1E-2): + msgs.warn("The wavelength samplings of the input frames are not the same!") + specstr = ", ".join(["{0:.6f}".format(ss) for ss in specscale]) + msgs.info("Wavelength samplings of all input frames:" + msgs.newline() + specstr + "Angstrom") + + # If the user has not specified the spatial scale, then set it appropriately now to the largest spatial scale + _dspat = np.max(spatscale) if dspat is None else dspat + msgs.info("Adopting a square pixel spatial scale of {0:f} arcsec".format(3600.0 * _dspat)) + # If the user has not specified the spectral sampling, then set it now to the largest value + _dwv = np.max(specscale) if dwv is None else dwv + msgs.info("Adopting a wavelength sampling of {0:f} Angstrom".format(_dwv)) + return _dspat, _dwv + + +def wcs_bounds(all_ra, all_dec, all_wave, ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None): + """ + Create a WCS and the expected edges of the voxels, based on user-specified + parameters or the extremities of the data. This is a convenience function + that calls the core function in `pypeit.core.datacube`_. + + Parameters + ---------- + all_ra : `numpy.ndarray`_ + 1D flattened array containing the RA values of each pixel from all + spec2d files + all_dec : `numpy.ndarray`_ + 1D flattened array containing the DEC values of each pixel from all + spec2d files + all_wave : `numpy.ndarray`_ + 1D flattened array containing the wavelength values of each pixel from + all spec2d files + ra_min : :obj:`float`, optional + Minimum RA of the WCS + ra_max : :obj:`float`, optional + Maximum RA of the WCS + dec_min : :obj:`float`, optional + Minimum Dec of the WCS + dec_max : :obj:`float`, optional + Maximum RA of the WCS + wav_min : :obj:`float`, optional + Minimum wavelength of the WCS + wav_max : :obj:`float`, optional + Maximum RA of the WCS + + Returns + ------- + _ra_min : :obj:`float` + Minimum RA of the WCS + _ra_max : :obj:`float` + Maximum RA of the WCS + _dec_min : :obj:`float` + Minimum Dec of the WCS + _dec_max : :obj:`float` + Maximum RA of the WCS + _wav_min : :obj:`float` + Minimum wavelength of the WCS + _wav_max : :obj:`float` + Maximum RA of the WCS + """ + # Setup the cube ranges + _ra_min = ra_min if ra_min is not None else np.min(all_ra) + _ra_max = ra_max if ra_max is not None else np.max(all_ra) + _dec_min = dec_min if dec_min is not None else np.min(all_dec) + _dec_max = dec_max if dec_max is not None else np.max(all_dec) + _wav_min = wave_min if wave_min is not None else np.min(all_wave) + _wav_max = wave_max if wave_max is not None else np.max(all_wave) + return _ra_min, _ra_max, _dec_min, _dec_max, _wav_min, _wav_max + + def create_wcs(all_ra, all_dec, all_wave, dspat, dwave, ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None, reference=None, collapse=False, equinox=2000.0, specname="PYP_SPEC"): @@ -668,13 +776,9 @@ def create_wcs(all_ra, all_dec, all_wave, dspat, dwave, cosdec = np.cos(np.mean(all_dec) * np.pi / 180.0) # Setup the cube ranges - _ra_min = ra_min if ra_min is not None else np.min(all_ra) - _ra_max = ra_max if ra_max is not None else np.max(all_ra) - _dec_min = dec_min if dec_min is not None else np.min(all_dec) - _dec_max = dec_max if dec_max is not None else np.max(all_dec) - _wav_min = wave_min if wave_min is not None else np.min(all_wave) - _wav_max = wave_max if wave_max is not None else np.max(all_wave) - # dwave = self.cubepar['wave_delta'] if self.cubepar['wave_delta'] is not None else dwv + _ra_min, _ra_max, _dec_min, _dec_max, _wav_min, _wav_max = \ + wcs_bounds(all_ra, all_dec, all_wave, ra_min=ra_min, ra_max=ra_max, dec_min=dec_min, dec_max=dec_max, + wave_min=wave_min, wave_max=wave_max) # Number of voxels in each dimension numra = int((_ra_max - _ra_min) * cosdec / dspat) @@ -756,6 +860,120 @@ def generate_WCS(crval, cdelt, equinox=2000.0, name="PYP_SPEC"): return w +def compute_weights_frompix(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, dspat, dwv, mnmx_wv, all_wghts, + all_spatpos, all_specpos, all_spatid, all_tilts, all_slits, all_align, all_dar, + ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None, + sn_smooth_npix=None, relative_weights=False, reference_image=None, whitelight_range=None, + specname="PYPSPEC"): + r""" + Calculate wavelength dependent optimal weights. The weighting is currently + based on a relative :math:`(S/N)^2` at each wavelength. Note, this function + first prepares a whitelight image, and then calls compute_weights() to + determine the appropriate weights of each pixel. + + Args: + all_ra (`numpy.ndarray`_): + 1D flattened array containing the RA values of each pixel from all + spec2d files + all_dec (`numpy.ndarray`_): + 1D flattened array containing the DEC values of each pixel from all + spec2d files + all_wave (`numpy.ndarray`_): + 1D flattened array containing the wavelength values of each pixel + from all spec2d files + all_sci (`numpy.ndarray`_): + 1D flattened array containing the counts of each pixel from all + spec2d files + all_ivar (`numpy.ndarray`_): + 1D flattened array containing the inverse variance of each pixel + from all spec2d files + all_idx (`numpy.ndarray`_): + 1D flattened array containing an integer identifier indicating which + spec2d file each pixel originates from. For example, a 0 would + indicate that a pixel originates from the first spec2d frame listed + in the input file. a 1 would indicate that this pixel originates + from the second spec2d file, and so forth. + dspat (float): + The size of each spaxel on the sky (in degrees) + dwv (float): + The size of each wavelength pixel (in Angstroms) + mnmx_wv (`numpy.ndarray`_): + The minimum and maximum wavelengths of every slit and frame. The shape is (Nframes, Nslits, 2), + The minimum and maximum wavelengths are stored in the [:,:,0] and [:,:,1] indices, respectively. + all_wghts (`numpy.ndarray`_): + 1D flattened array containing the weights of each pixel to be used + in the combination + all_spatpos (`numpy.ndarray`_): + 1D flattened array containing the detector pixel location in the + spatial direction + all_specpos (`numpy.ndarray`_): + 1D flattened array containing the detector pixel location in the + spectral direction + all_spatid (`numpy.ndarray`_): + 1D flattened array containing the spatid of each pixel + tilts (`numpy.ndarray`_, list): + 2D wavelength tilts frame, or a list of tilt frames (see all_idx) + slits (:class:`~pypeit.slittrace.SlitTraceSet`, list): + Information stored about the slits, or a list of SlitTraceSet (see + all_idx) + astrom_trans (:class:`~pypeit.alignframe.AlignmentSplines`, list): + A Class containing the transformation between detector pixel + coordinates and WCS pixel coordinates, or a list of Alignment + Splines (see all_idx) + all_dar (:class:`~pypeit.coadd3d.DARcorrection`, list): + A Class containing the DAR correction information, or a list of DARcorrection + classes. If a list, it must be the same length as astrom_trans. + ra_min (float, optional): + Minimum RA of the WCS (degrees) + ra_max (float, optional): + Maximum RA of the WCS (degrees) + dec_min (float, optional): + Minimum Dec of the WCS (degrees) + dec_max (float, optional): + Maximum Dec of the WCS (degrees) + wave_min (float, optional): + Minimum wavelength of the WCS (degrees) + wave_max (float, optional): + Maximum wavelength of the WCS (degrees) + sn_smooth_npix (float, optional): + Number of pixels used for determining smoothly varying S/N ratio + weights. This is currently not required, since a relative weighting + scheme with a polynomial fit is used to calculate the S/N weights. + relative_weights (bool, optional): + Calculate weights by fitting to the ratio of spectra? + reference_image (`numpy.ndarray`_): + Reference image to use for the determination of the highest S/N spaxel in the image. + specname (str): + Name of the spectrograph + + Returns: + `numpy.ndarray`_ : a 1D array the same size as all_sci, containing + relative wavelength dependent weights of each input pixel. + """ + # Find the wavelength range where all frames overlap + min_wl, max_wl = get_whitelight_range(np.max(mnmx_wv[:, :, 0]), # The max blue wavelength + np.min(mnmx_wv[:, :, 1]), # The min red wavelength + whitelight_range) # The user-specified values (if any) + # Get the good white light pixels + ww, wavediff = get_whitelight_pixels(all_wave, min_wl, max_wl) + + # Generate the WCS + image_wcs, voxedge, reference_image = \ + create_wcs(all_ra, all_dec, all_wave, dspat, wavediff, + ra_min=ra_min, ra_max=ra_max, dec_min=dec_min, dec_max=dec_max, wave_min=wave_min, wave_max=wave_max, + reference=reference_image, collapse=True, equinox=2000.0, + specname=specname) + + # Generate the white light image + # NOTE: hard-coding subpixel=1 in both directions for speed, and combining into a single image + wl_full = generate_image_subpixel(image_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_wghts, + all_spatpos, all_specpos, all_spatid, all_tilts, all_slits, all_align, all_dar, + voxedge, all_idx=all_idx, spec_subpixel=1, spat_subpixel=1, combine=True) + # Compute the weights + return compute_weights(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, wl_full[:, :, 0], dspat, dwv, + sn_smooth_npix=sn_smooth_npix, relative_weights=relative_weights) + + def compute_weights(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, whitelight_img, dspat, dwv, sn_smooth_npix=None, relative_weights=False): r""" @@ -842,7 +1060,8 @@ def compute_weights(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, white mask_stack = (flux_stack != 0.0) & (ivar_stack != 0.0) # Obtain a wavelength of each pixel wcs_res = whitelightWCS.wcs_pix2world(np.vstack((np.zeros(numwav), np.zeros(numwav), np.arange(numwav))).T, 0) - wave_spec = wcs_res[:, 2] * 1.0E10 + wcs_scale = (1.0 * whitelightWCS.wcs.cunit[2]).to_value(units.Angstrom) # Ensures the WCS is in Angstroms + wave_spec = wcs_scale * wcs_res[:, 2] # Compute the smoothing scale to use if sn_smooth_npix is None: sn_smooth_npix = int(np.round(0.1 * wave_spec.size)) @@ -859,9 +1078,9 @@ def compute_weights(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, white return all_wghts -def generate_image_subpixel(image_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_wghts, all_spatpos, all_specpos, - all_spatid, tilts, slits, astrom_trans, bins, all_idx=None, - spec_subpixel=10, spat_subpixel=10, combine=False): +def generate_image_subpixel(image_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_wghts, + all_spatpos, all_specpos, all_spatid, tilts, slits, astrom_trans, all_dar, bins, + all_idx=None, spec_subpixel=10, spat_subpixel=10, combine=False): """ Generate a white light image from the input pixels @@ -903,6 +1122,9 @@ def generate_image_subpixel(image_wcs, all_ra, all_dec, all_wave, all_sci, all_i A Class containing the transformation between detector pixel coordinates and WCS pixel coordinates, or a list of Alignment Splines (see all_idx) + all_dar (:class:`~pypeit.coadd3d.DARcorrection`, list): + A Class containing the DAR correction information, or a list of DARcorrection + classes. If a list, it must be the same length as astrom_trans. bins (tuple): A 3-tuple (x,y,z) containing the histogram bin edges in x,y spatial and z wavelength coordinates @@ -937,9 +1159,9 @@ def generate_image_subpixel(image_wcs, all_ra, all_dec, all_wave, all_sci, all_i numfr = 1 else: numfr = np.unique(_all_idx).size - if len(tilts) != numfr or len(slits) != numfr or len(astrom_trans) != numfr: + if len(tilts) != numfr or len(slits) != numfr or len(astrom_trans) != numfr or len(all_dar) != numfr: msgs.error("The following arguments must be the same length as the expected number of frames to be combined:" - + msgs.newline() + "tilts, slits, astrom_trans") + + msgs.newline() + "tilts, slits, astrom_trans, all_dar") # Prepare the array of white light images to be stored numra = bins[0].size-1 numdec = bins[1].size-1 @@ -952,26 +1174,25 @@ def generate_image_subpixel(image_wcs, all_ra, all_dec, all_wave, all_sci, all_i # Subpixellate img, _, _ = subpixellate(image_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_wghts, all_spatpos, - all_specpos, all_spatid, tilts, slits, astrom_trans, bins, + all_specpos, all_spatid, tilts, slits, astrom_trans, all_dar, bins, spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, all_idx=_all_idx) else: ww = np.where(_all_idx == fr) # Subpixellate img, _, _ = subpixellate(image_wcs, all_ra[ww], all_dec[ww], all_wave[ww], all_sci[ww], all_ivar[ww], all_wghts[ww], all_spatpos[ww], - all_specpos[ww], all_spatid[ww], tilts[fr], slits[fr], astrom_trans[fr], bins, - spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel) + all_specpos[ww], all_spatid[ww], tilts[fr], slits[fr], astrom_trans[fr], + all_dar[fr], bins, spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel) all_wl_imgs[:, :, fr] = img[:, :, 0] # Return the constructed white light images return all_wl_imgs def generate_cube_subpixel(outfile, output_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_wghts, - all_spatpos, all_specpos, all_spatid, tilts, slits, astrom_trans, bins, - all_idx=None, spec_subpixel=10, spat_subpixel=10, overwrite=False, blaze_wave=None, - blaze_spec=None, fluxcal=False, sensfunc=None, whitelight_range=None, - specname="PYP_SPEC", debug=False): - r""" + all_spatpos, all_specpos, all_spatid, tilts, slits, astrom_trans, all_dar, bins, + all_idx=None, spec_subpixel=10, spat_subpixel=10, overwrite=False, + whitelight_range=None, debug=False): + """ Save a datacube using the subpixel algorithm. Refer to the subpixellate() docstring for further details about this algorithm @@ -1015,6 +1236,9 @@ def generate_cube_subpixel(outfile, output_wcs, all_ra, all_dec, all_wave, all_s A Class containing the transformation between detector pixel coordinates and WCS pixel coordinates, or a list of Alignment Splines (see all_idx) + all_dar (:class:`~pypeit.coadd3d.DARcorrection`, list): + A Class containing the DAR correction information, or a list of DARcorrection + classes. If a list, it must be the same length as astrom_trans. bins (tuple): A 3-tuple (x,y,z) containing the histogram bin edges in x,y spatial and z wavelength coordinates @@ -1038,17 +1262,6 @@ def generate_cube_subpixel(outfile, output_wcs, all_ra, all_dec, all_wave, all_s direction. overwrite (bool, optional): If True, the output cube will be overwritten. - blaze_wave (`numpy.ndarray`_, optional): - Wavelength array of the spectral blaze function - blaze_spec (`numpy.ndarray`_, optional): - Spectral blaze function - fluxcal (bool, optional): - Are the data flux calibrated? If True, the units are: :math:`{\rm - erg/s/cm}^2{\rm /Angstrom/arcsec}^2` multiplied by the - PYPEIT_FLUX_SCALE. Otherwise, the units are: :math:`{\rm - counts/s/Angstrom/arcsec}^2`. - sensfunc (`numpy.ndarray`_, None, optional): - Sensitivity function that has been applied to the datacube whitelight_range (None, list, optional): A two element list that specifies the minimum and maximum wavelengths (in Angstroms) to use when constructing the white light @@ -1057,34 +1270,40 @@ def generate_cube_subpixel(outfile, output_wcs, all_ra, all_dec, all_wave, all_s either element of the list is None, then the minimum/maximum wavelength range of that element will be set by the minimum/maximum wavelength of all_wave. - specname (str, optional): - Name of the spectrograph debug (bool, optional): If True, a residuals cube will be output. If the datacube generation is correct, the distribution of pixels in the residual cube with no flux should have mean=0 and std=1. + + Returns: + :obj:`tuple`: Four `numpy.ndarray`_ objects containing + (1) the datacube generated from the subpixellated inputs, + (2) the corresponding error cube (standard deviation), + (3) the corresponding bad pixel mask cube, and + (4) a 1D array containing the wavelength at each spectral coordinate of the datacube. """ # Prepare the header, and add the unit of flux to the header hdr = output_wcs.to_header() - if fluxcal: - hdr['FLUXUNIT'] = (flux_calib.PYPEIT_FLUX_SCALE, "Flux units -- erg/s/cm^2/Angstrom/arcsec^2") - else: - hdr['FLUXUNIT'] = (1, "Flux units -- counts/s/Angstrom/arcsec^2") # Subpixellate subpix = subpixellate(output_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_wghts, all_spatpos, all_specpos, - all_spatid, tilts, slits, astrom_trans, bins, all_idx=all_idx, + all_spatid, tilts, slits, astrom_trans, all_dar, bins, all_idx=all_idx, spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, debug=debug) # Extract the variables that we need if debug: - datacube, varcube, bpmcube, residcube = subpix + flxcube, varcube, bpmcube, residcube = subpix # Save a residuals cube outfile_resid = outfile.replace(".fits", "_resid.fits") msgs.info("Saving residuals datacube as: {0:s}".format(outfile_resid)) hdu = fits.PrimaryHDU(residcube.T, header=hdr) hdu.writeto(outfile_resid, overwrite=overwrite) else: - datacube, varcube, bpmcube = subpix + flxcube, varcube, bpmcube = subpix + + # Get wavelength of each pixel + nspec = flxcube.shape[2] + wcs_scale = (1.0*output_wcs.spectral.wcs.cunit[0]).to(units.Angstrom).value # Ensures the WCS is in Angstroms + wave = wcs_scale * output_wcs.spectral.wcs_pix2world(np.arange(nspec), 0)[0] # Check if the user requested a white light image if whitelight_range is not None: @@ -1099,23 +1318,16 @@ def generate_cube_subpixel(outfile, output_wcs, all_ra, all_dec, all_wave, all_s whitelight_range[0], whitelight_range[1])) # Get the output filename for the white light image out_whitelight = get_output_whitelight_filename(outfile) - nspec = datacube.shape[2] - # Get wavelength of each pixel, and note that the WCS gives this in m, so convert to Angstroms (x 1E10) - wave = 1.0E10 * output_wcs.spectral.wcs_pix2world(np.arange(nspec), 0)[0] - whitelight_img = make_whitelight_fromcube(datacube, wave=wave, wavemin=whitelight_range[0], wavemax=whitelight_range[1]) + whitelight_img = make_whitelight_fromcube(flxcube, wave=wave, wavemin=whitelight_range[0], wavemax=whitelight_range[1]) msgs.info("Saving white light image as: {0:s}".format(out_whitelight)) img_hdu = fits.PrimaryHDU(whitelight_img.T, header=whitelight_wcs.to_header()) img_hdu.writeto(out_whitelight, overwrite=overwrite) - - # Write out the datacube - msgs.info("Saving datacube as: {0:s}".format(outfile)) - final_cube = DataCube(datacube.T, np.sqrt(varcube.T), bpmcube.T, specname, blaze_wave, blaze_spec, - sensfunc=sensfunc, fluxed=fluxcal) - final_cube.to_file(outfile, hdr=hdr, overwrite=overwrite) + # TODO :: Avoid transposing these large cubes + return flxcube.T, np.sqrt(varcube.T), bpmcube.T, wave def subpixellate(output_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_wghts, all_spatpos, all_specpos, - all_spatid, tilts, slits, astrom_trans, bins, all_idx=None, + all_spatid, tilts, slits, astrom_trans, all_dar, bins, all_idx=None, spec_subpixel=10, spat_subpixel=10, debug=False): r""" Subpixellate the input data into a datacube. This algorithm splits each @@ -1168,6 +1380,9 @@ def subpixellate(output_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_w A Class containing the transformation between detector pixel coordinates and WCS pixel coordinates, or a list of Alignment Splines (see all_idx) + all_dar (:class:`~pypeit.coadd3d.DARcorrection`, list): + A Class containing the DAR correction information, or a list of DARcorrection + classes. If a list, it must be the same length as astrom_trans. bins (tuple): A 3-tuple (x,y,z) containing the histogram bin edges in x,y spatial and z wavelength coordinates @@ -1201,12 +1416,12 @@ def subpixellate(output_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_w residual cube. The latter is only returned if debug is True. """ # Check for combinations of lists or not - if type(tilts) is list and type(slits) is list and type(astrom_trans) is list: + if all([isinstance(l, list) for l in [tilts, slits, astrom_trans, all_dar]]): # Several frames are being combined. Check the lists have the same length numframes = len(tilts) - if len(slits) != numframes or len(astrom_trans) != numframes: + if len(slits) != numframes or len(astrom_trans) != numframes or len(all_dar) != numframes: msgs.error("The following lists must have the same length:" + msgs.newline() + - "tilts, slits, astrom_trans") + "tilts, slits, astrom_trans, all_dar") # Check all_idx has been set if all_idx is None: if numframes != 1: @@ -1218,20 +1433,19 @@ def subpixellate(output_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_w if tmp != numframes: msgs.warn("Indices in argument 'all_idx' does not match the number of frames expected.") # Store in the following variables - _tilts, _slits, _astrom_trans = tilts, slits, astrom_trans - elif type(tilts) is not list and type(slits) is not list and \ - type(astrom_trans) is not list: + _tilts, _slits, _astrom_trans, _all_dar = tilts, slits, astrom_trans, all_dar + elif all([not isinstance(l, list) for l in [tilts, slits, astrom_trans, all_dar]]): # Just a single frame - store as lists for this code - _tilts, _slits, _astrom_trans = [tilts], [slits], [astrom_trans], + _tilts, _slits, _astrom_trans, _all_dar = [tilts], [slits], [astrom_trans], [all_dar] all_idx = np.zeros(all_sci.size) numframes = 1 else: msgs.error("The following input arguments should all be of type 'list', or all not be type 'list':" + - msgs.newline() + "tilts, slits, astrom_trans") + msgs.newline() + "tilts, slits, astrom_trans, all_dar") # Prepare the output arrays outshape = (bins[0].size-1, bins[1].size-1, bins[2].size-1) binrng = [[bins[0][0], bins[0][-1]], [bins[1][0], bins[1][-1]], [bins[2][0], bins[2][-1]]] - datacube, varcube, normcube = np.zeros(outshape), np.zeros(outshape), np.zeros(outshape) + flxcube, varcube, normcube = np.zeros(outshape), np.zeros(outshape), np.zeros(outshape) if debug: residcube = np.zeros(outshape) # Divide each pixel into subpixels @@ -1261,809 +1475,41 @@ def subpixellate(output_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_w wspl = all_wave[this_sl] asrt = np.argsort(yspl) wave_spl = interp1d(yspl[asrt], wspl[asrt], kind='linear', bounds_error=False, fill_value='extrapolate') + # Calculate the wavelength at each subpixel + this_wave = wave_spl(tiltpos) + # Calculate the DAR correction at each sub pixel + ra_corr, dec_corr = _all_dar[fr].correction(this_wave) # This routine needs the wavelengths to be expressed in Angstroms # Calculate spatial and spectral positions of the subpixels spat_xx = np.add.outer(wpix[1], spat_x.flatten()).flatten() spec_yy = np.add.outer(wpix[0], spec_y.flatten()).flatten() # Transform this to spatial location spatpos_subpix = _astrom_trans[fr].transform(sl, spat_xx, spec_yy) spatpos = _astrom_trans[fr].transform(sl, all_spatpos[this_sl], all_specpos[this_sl]) - ra_coeff = np.polyfit(spatpos, all_ra[this_sl], 1) - dec_coeff = np.polyfit(spatpos, all_dec[this_sl], 1) - this_ra = np.polyval(ra_coeff, spatpos_subpix)#ra_spl(spatpos_subpix) - this_dec = np.polyval(dec_coeff, spatpos_subpix)#dec_spl(spatpos_subpix) - # ssrt = np.argsort(spatpos) - # ra_spl = interp1d(spatpos[ssrt], all_ra[this_sl][ssrt], kind='linear', bounds_error=False, fill_value='extrapolate') - # dec_spl = interp1d(spatpos[ssrt], all_dec[this_sl][ssrt], kind='linear', bounds_error=False, fill_value='extrapolate') - # this_ra = ra_spl(spatpos_subpix) - # this_dec = dec_spl(spatpos_subpix) - this_wave = wave_spl(tiltpos) + # Interpolate the RA/Dec over the subpixel spatial positions + ssrt = np.argsort(spatpos) + tmp_ra = all_ra[this_sl] + tmp_dec = all_dec[this_sl] + ra_spl = interp1d(spatpos[ssrt], tmp_ra[ssrt], kind='linear', bounds_error=False, fill_value='extrapolate') + dec_spl = interp1d(spatpos[ssrt], tmp_dec[ssrt], kind='linear', bounds_error=False, fill_value='extrapolate') + this_ra = ra_spl(spatpos_subpix) + this_dec = dec_spl(spatpos_subpix) + # Now apply the DAR correction + this_ra += ra_corr + this_dec += dec_corr # Convert world coordinates to voxel coordinates, then histogram vox_coord = output_wcs.wcs_world2pix(np.vstack((this_ra, this_dec, this_wave * 1.0E-10)).T, 0) - if histogramdd is not None: - # use the "fast histogram" algorithm, that assumes regular bin spacing - datacube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(all_sci[this_sl] * all_wght_subpix[this_sl], num_subpixels)) - varcube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(all_var[this_sl] * all_wght_subpix[this_sl]**2, num_subpixels)) - normcube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(all_wght_subpix[this_sl], num_subpixels)) - if debug: - residcube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(all_sci[this_sl] * np.sqrt(all_ivar[this_sl]), num_subpixels)) - else: - datacube += np.histogramdd(vox_coord, bins=outshape, weights=np.repeat(all_sci[this_sl] * all_wght_subpix[this_sl], num_subpixels))[0] - varcube += np.histogramdd(vox_coord, bins=outshape, weights=np.repeat(all_var[this_sl] * all_wght_subpix[this_sl]**2, num_subpixels))[0] - normcube += np.histogramdd(vox_coord, bins=outshape, weights=np.repeat(all_wght_subpix[this_sl], num_subpixels))[0] - if debug: - residcube += np.histogramdd(vox_coord, bins=outshape, weights=np.repeat(all_sci[this_sl] * np.sqrt(all_ivar[this_sl]), num_subpixels))[0] + # Use the "fast histogram" algorithm, that assumes regular bin spacing + flxcube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(all_sci[this_sl] * all_wght_subpix[this_sl], num_subpixels)) + varcube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(all_var[this_sl] * all_wght_subpix[this_sl]**2, num_subpixels)) + normcube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(all_wght_subpix[this_sl], num_subpixels)) + if debug: + residcube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(all_sci[this_sl] * np.sqrt(all_ivar[this_sl]), num_subpixels)) # Normalise the datacube and variance cube nc_inverse = utils.inverse(normcube) - datacube *= nc_inverse + flxcube *= nc_inverse varcube *= nc_inverse**2 bpmcube = (normcube == 0).astype(np.uint8) if debug: residcube *= nc_inverse - return datacube, varcube, bpmcube, residcube - return datacube, varcube, bpmcube - - -def get_output_filename(fil, par_outfile, combine, idx=1): - """ - Get the output filename of a datacube, given the input - - Args: - fil (str): - The spec2d filename. - par_outfile (str): - The user-specified output filename (see cubepar['output_filename']) - combine (bool): - Should the input frames be combined into a single datacube? - idx (int, optional): - Index of filename to be saved. Required if combine=False. - - Returns: - str: The output filename to use. - """ - if combine: - if par_outfile == "": - par_outfile = "datacube.fits" - # Check the output files don't exist - outfile = par_outfile if ".fits" in par_outfile else par_outfile + ".fits" - else: - if par_outfile == "": - outfile = fil.replace("spec2d_", "spec3d_") - else: - # Use the output filename as a prefix - outfile = os.path.splitext(par_outfile)[0] + "_{0:03d}.fits".format(idx) - # Return the outfile - return outfile - - -def get_output_whitelight_filename(outfile): - """ - Given the output filename of a datacube, create an appropriate whitelight - fits file name - - Args: - outfile (str): - The output filename used for the datacube. - - Returns: - str: The output filename to use for the whitelight image. - """ - out_wl_filename = os.path.splitext(outfile)[0] + "_whitelight.fits" - return out_wl_filename - - -def coadd_cube(files, opts, spectrograph=None, parset=None, overwrite=False): - """ Main routine to coadd spec2D files into a 3D datacube - - Args: - files (:obj:`list`): - List of all spec2D files - opts (:obj:`dict`): - coadd2d options associated with each spec2d file - spectrograph (:obj:`str`, :class:`~pypeit.spectrographs.spectrograph.Spectrograph`, optional): - The name or instance of the spectrograph used to obtain the data. - If None, this is pulled from the file header. - parset (:class:`~pypeit.par.pypeitpar.PypeItPar`, optional): - An instance of the parameter set. If None, assumes that detector 1 - is the one reduced and uses the default reduction parameters for the - spectrograph (see - :func:`~pypeit.spectrographs.spectrograph.Spectrograph.default_pypeit_par` - for the relevant spectrograph class). - overwrite (:obj:`bool`, optional): - Overwrite the output file, if it exists? - """ - if spectrograph is None: - with fits.open(files[0]) as hdu: - spectrograph = hdu[0].header['PYP_SPEC'] - - if isinstance(spectrograph, str): - spec = load_spectrograph(spectrograph) - specname = spectrograph - else: - # Assume it's a Spectrograph instance - spec = spectrograph - specname = spectrograph.name - - # Grab the parset, if not provided - if parset is None: - # TODO :: Use config_specific_par instead? - parset = spec.default_pypeit_par() - cubepar = parset['reduce']['cube'] - flatpar = parset['calibrations']['flatfield'] - senspar = parset['sensfunc'] - - # prep - numfiles = len(files) - method = cubepar['method'].lower() - combine = cubepar['combine'] - align = cubepar['align'] - # If there is only one frame being "combined" AND there's no reference image, then don't compute the translation. - if numfiles == 1 and cubepar["reference_image"] is None: - if not align: - msgs.warn("Parameter 'align' should be False when there is only one frame and no reference image") - msgs.info("Setting 'align' to False") - align = False - if opts['ra_offset'] is not None: - if not align: - msgs.warn("When 'ra_offset' and 'dec_offset' are set, 'align' must be True.") - msgs.info("Setting 'align' to True") - align = True - # TODO :: The default behaviour (combine=False, align=False) produces a datacube that uses the instrument WCS - # It should be possible (and perhaps desirable) to do a spatial alignment (i.e. align=True), apply this to the - # RA,Dec values of each pixel, and then use the instrument WCS to save the output (or, just adjust the crval). - # At the moment, if the user wishes to spatially align the frames, a different WCS is generated. - if histogramdd is None: - msgs.warn("Generating a datacube is faster if you install fast-histogram:"+msgs.newline()+ - "https://pypi.org/project/fast-histogram/") - if method != 'ngp': - msgs.warn("Forcing NGP algorithm, because fast-histogram is not installed") - method = 'ngp' - - # Determine what method is requested - spec_subpixel, spat_subpixel = 1, 1 - if method == "subpixel": - msgs.info("Adopting the subpixel algorithm to generate the datacube.") - spec_subpixel, spat_subpixel = cubepar['spec_subpixel'], cubepar['spat_subpixel'] - elif method == "ngp": - msgs.info("Adopting the nearest grid point (NGP) algorithm to generate the datacube.") - else: - msgs.error(f"The following datacube method is not allowed: {method}") - - # Get the detector number and string representation - det = 1 if parset['rdx']['detnum'] is None else parset['rdx']['detnum'] - detname = spec.get_det_name(det) - - # Check if the output file exists - if combine: - outfile = get_output_filename("", cubepar['output_filename'], combine) - out_whitelight = get_output_whitelight_filename(outfile) - if os.path.exists(outfile) and not overwrite: - msgs.error("Output filename already exists:"+msgs.newline()+outfile) - if os.path.exists(out_whitelight) and cubepar['save_whitelight'] and not overwrite: - msgs.error("Output filename already exists:"+msgs.newline()+out_whitelight) - else: - # Finally, if there's just one file, check if the output filename is given - if numfiles == 1 and cubepar['output_filename'] != "": - outfile = get_output_filename("", cubepar['output_filename'], True, -1) - out_whitelight = get_output_whitelight_filename(outfile) - if os.path.exists(outfile) and not overwrite: - msgs.error("Output filename already exists:" + msgs.newline() + outfile) - if os.path.exists(out_whitelight) and cubepar['save_whitelight'] and not overwrite: - msgs.error("Output filename already exists:" + msgs.newline() + out_whitelight) - else: - for ff in range(numfiles): - outfile = get_output_filename(files[ff], cubepar['output_filename'], combine, ff+1) - out_whitelight = get_output_whitelight_filename(outfile) - if os.path.exists(outfile) and not overwrite: - msgs.error("Output filename already exists:" + msgs.newline() + outfile) - if os.path.exists(out_whitelight) and cubepar['save_whitelight'] and not overwrite: - msgs.error("Output filename already exists:" + msgs.newline() + out_whitelight) - - # Check the reference cube and image exist, if requested - fluxcal = False - blaze_wave, blaze_spec = None, None - blaze_spline, flux_spline = None, None - if cubepar['standard_cube'] is not None: - fluxcal = True - ss_file = cubepar['standard_cube'] - if not os.path.exists(ss_file): - msgs.error("Standard cube does not exist:" + msgs.newline() + ss_file) - msgs.info(f"Loading standard star cube: {ss_file:s}") - # Load the standard star cube and retrieve its RA + DEC - stdcube = fits.open(ss_file) - star_ra, star_dec = stdcube[1].header['CRVAL1'], stdcube[1].header['CRVAL2'] - - # Extract a spectrum of the standard star - wave, Nlam_star, Nlam_ivar_star, gpm_star = extract_standard_spec(stdcube) - - # Extract the information about the blaze - if cubepar['grating_corr']: - blaze_wave_curr, blaze_spec_curr = stdcube['BLAZE_WAVE'].data, stdcube['BLAZE_SPEC'].data - blaze_spline_curr = interp1d(blaze_wave_curr, blaze_spec_curr, - kind='linear', bounds_error=False, fill_value="extrapolate") - # The first standard star cube is used as the reference blaze spline - if blaze_spline is None: - blaze_wave, blaze_spec = stdcube['BLAZE_WAVE'].data, stdcube['BLAZE_SPEC'].data - blaze_spline = interp1d(blaze_wave, blaze_spec, - kind='linear', bounds_error=False, fill_value="extrapolate") - # Perform a grating correction - grat_corr = correct_grating_shift(wave.value, blaze_wave_curr, blaze_spline_curr, blaze_wave, blaze_spline) - # Apply the grating correction to the standard star spectrum - Nlam_star /= grat_corr - Nlam_ivar_star *= grat_corr**2 - - # Read in some information above the standard star - std_dict = flux_calib.get_standard_spectrum(star_type=senspar['star_type'], - star_mag=senspar['star_mag'], - ra=star_ra, dec=star_dec) - # Calculate the sensitivity curve - # TODO :: This needs to be addressed... unify flux calibration into the main PypeIt routines. - msgs.warn("Datacubes are currently flux-calibrated using the UVIS algorithm... this will be deprecated soon") - zeropoint_data, zeropoint_data_gpm, zeropoint_fit, zeropoint_fit_gpm =\ - flux_calib.fit_zeropoint(wave.value, Nlam_star, Nlam_ivar_star, gpm_star, std_dict, - mask_hydrogen_lines=senspar['mask_hydrogen_lines'], - mask_helium_lines=senspar['mask_helium_lines'], - hydrogen_mask_wid=senspar['hydrogen_mask_wid'], - nresln=senspar['UVIS']['nresln'], resolution=senspar['UVIS']['resolution'], - trans_thresh=senspar['UVIS']['trans_thresh'], polyorder=senspar['polyorder'], - polycorrect=senspar['UVIS']['polycorrect'], polyfunc=senspar['UVIS']['polyfunc']) - wgd = np.where(zeropoint_fit_gpm) - sens = np.power(10.0, -0.4 * (zeropoint_fit[wgd] - flux_calib.ZP_UNIT_CONST)) / np.square(wave[wgd]) - flux_spline = interp1d(wave[wgd], sens, kind='linear', bounds_error=False, fill_value="extrapolate") - - # If a reference image has been set, check that it exists - if cubepar['reference_image'] is not None: - if not os.path.exists(cubepar['reference_image']): - msgs.error("Reference image does not exist:" + msgs.newline() + cubepar['reference_image']) - - # Initialise arrays for storage - ifu_ra, ifu_dec = np.array([]), np.array([]) # The RA and Dec at the centre of the IFU, as stored in the header - all_ra, all_dec, all_wave = np.array([]), np.array([]), np.array([]) - all_sci, all_ivar, all_idx, all_wghts = np.array([]), np.array([]), np.array([]), np.array([]) - all_spatpos, all_specpos, all_spatid = np.array([], dtype=int), np.array([], dtype=int), np.array([], dtype=int) - all_tilts, all_slits, all_align = [], [], [] - all_wcs = [] - dspat = None if cubepar['spatial_delta'] is None else cubepar['spatial_delta']/3600.0 # binning size on the sky (/3600 to convert to degrees) - dwv = cubepar['wave_delta'] # binning size in wavelength direction (in Angstroms) - wave_ref = None - mnmx_wv = None # Will be used to store the minimum and maximum wavelengths of every slit and frame. - weights = np.ones(numfiles) # Weights to use when combining cubes - flat_splines = dict() # A dictionary containing the splines of the flatfield - # Load the default scaleimg frame for the scale correction - scalecorr_default = "none" - relScaleImgDef = np.array([1]) - if cubepar['scale_corr'] is not None: - if cubepar['scale_corr'] == "image": - msgs.info("The default relative spectral illumination correction will use the science image") - scalecorr_default = "image" - else: - msgs.info("Loading default scale image for relative spectral illumination correction:" + - msgs.newline() + cubepar['scale_corr']) - try: - spec2DObj = spec2dobj.Spec2DObj.from_file(cubepar['scale_corr'], detname) - relScaleImgDef = spec2DObj.scaleimg - scalecorr_default = cubepar['scale_corr'] - except: - msgs.warn("Could not load scaleimg from spec2d file:" + msgs.newline() + - cubepar['scale_corr'] + msgs.newline() + - "scale correction will not be performed unless you have specified the correct" + msgs.newline() + - "scale_corr file in the spec2d block") - cubepar['scale_corr'] = None - scalecorr_default = "none" - - # Load the default sky frame to be used for sky subtraction - skysub_default = "image" - skyImgDef, skySclDef = None, None # This is the default behaviour (i.e. to use the "image" for the sky subtraction) - if cubepar['skysub_frame'] in [None, 'none', '', 'None']: - skysub_default = "none" - skyImgDef = np.array([0.0]) # Do not perform sky subtraction - skySclDef = np.array([0.0]) # Do not perform sky subtraction - elif cubepar['skysub_frame'].lower() == "image": - msgs.info("The sky model in the spec2d science frames will be used for sky subtraction" +msgs.newline() + - "(unless specific skysub frames have been specified)") - skysub_default = "image" - else: - msgs.info("Loading default image for sky subtraction:" + - msgs.newline() + cubepar['skysub_frame']) - try: - spec2DObj = spec2dobj.Spec2DObj.from_file(cubepar['skysub_frame'], detname) - skysub_exptime = fits.open(cubepar['skysub_frame'])[0].header['EXPTIME'] - except: - msgs.error("Could not load skysub image from spec2d file:" + msgs.newline() + cubepar['skysub_frame']) - skysub_default = cubepar['skysub_frame'] - skyImgDef = spec2DObj.sciimg/skysub_exptime # Sky counts/second - skySclDef = spec2DObj.scaleimg - - # Load all spec2d files and prepare the data for making a datacube - for ff, fil in enumerate(files): - # Load it up - msgs.info("Loading PypeIt spec2d frame:" + msgs.newline() + fil) - spec2DObj = spec2dobj.Spec2DObj.from_file(fil, detname) - detector = spec2DObj.detector - spat_flexure = None #spec2DObj.sci_spat_flexure - - # Load the header - hdr = spec2DObj.head0 - ifu_ra = np.append(ifu_ra, spec.get_meta_value([hdr], 'ra')) - ifu_dec = np.append(ifu_dec, spec.get_meta_value([hdr], 'dec')) - - # Get the exposure time - exptime = hdr['EXPTIME'] - - # Setup for PypeIt imports - msgs.reset(verbosity=2) - - # TODO :: Consider loading all calibrations into a single variable. - - # Initialise the slit edges - msgs.info("Constructing slit image") - slits = spec2DObj.slits - slitid_img_init = slits.slit_img(pad=0, initial=True, flexure=spat_flexure) - slits_left, slits_right, _ = slits.select_edges(initial=True, flexure=spat_flexure) - - # The order of operations below proceeds as follows: - # (1) Get science image - # (2) Subtract sky (note, if a joint fit has been performed, the relative scale correction is applied in the reduction!) - # (3) Apply relative scale correction to both science and ivar - - # Set the default behaviour if a global skysub frame has been specified - this_skysub = skysub_default - if skysub_default == "image": - skyImg = spec2DObj.skymodel - skyScl = spec2DObj.scaleimg - else: - skyImg = skyImgDef.copy() * exptime - skyScl = skySclDef.copy() - # See if there's any changes from the default behaviour - if opts['skysub_frame'][ff] is not None: - if opts['skysub_frame'][ff].lower() == 'default': - if skysub_default == "image": - skyImg = spec2DObj.skymodel - skyScl = spec2DObj.scaleimg - this_skysub = "image" # Use the current spec2d for sky subtraction - else: - skyImg = skyImgDef.copy() * exptime - skyScl = skySclDef.copy() - this_skysub = skysub_default # Use the global value for sky subtraction - elif opts['skysub_frame'][ff].lower() == 'image': - skyImg = spec2DObj.skymodel - skyScl = spec2DObj.scaleimg - this_skysub = "image" # Use the current spec2d for sky subtraction - elif opts['skysub_frame'][ff].lower() == 'none': - skyImg = np.array([0.0]) - skyScl = np.array([1.0]) - this_skysub = "none" # Don't do sky subtraction - else: - # Load a user specified frame for sky subtraction - msgs.info("Loading skysub frame:" + msgs.newline() + opts['skysub_frame'][ff]) - try: - spec2DObj_sky = spec2dobj.Spec2DObj.from_file(opts['skysub_frame'][ff], detname) - skysub_exptime = fits.open(opts['skysub_frame'][ff])[0].header['EXPTIME'] - except: - msgs.error("Could not load skysub image from spec2d file:" + msgs.newline() + opts['skysub_frame'][ff]) - # TODO :: Consider allowing the actual frame (instead of the skymodel) to be used as the skysub image - make sure the BPM is carried over. - # :: Allow sky data fitting (i.e. scale the flux of a skysub frame to the science frame data) - skyImg = spec2DObj_sky.sciimg * exptime / skysub_exptime # Sky counts - # skyImg = spec2DObj_sky.skymodel * exptime / skysub_exptime # Sky counts - skyScl = spec2DObj_sky.scaleimg - this_skysub = opts['skysub_frame'][ff] # User specified spec2d for sky subtraction - if this_skysub == "none": - msgs.info("Sky subtraction will not be performed.") - else: - msgs.info("Using the following frame for sky subtraction:"+msgs.newline()+this_skysub) - - # Load the relative scale image, if something other than the default has been provided - this_scalecorr = scalecorr_default - relScaleImg = relScaleImgDef.copy() - if opts['scale_corr'][ff] is not None: - if opts['scale_corr'][ff].lower() == 'default': - if scalecorr_default == "image": - relScaleImg = spec2DObj.scaleimg - this_scalecorr = "image" # Use the current spec2d for the relative spectral illumination scaling - else: - this_scalecorr = scalecorr_default # Use the default value for the scale correction - elif opts['scale_corr'][ff].lower() == 'image': - relScaleImg = spec2DObj.scaleimg - this_scalecorr = "image" # Use the current spec2d for the relative spectral illumination scaling - elif opts['scale_corr'][ff].lower() == 'none': - relScaleImg = np.array([1]) - this_scalecorr = "none" # Don't do relative spectral illumination scaling - else: - # Load a user specified frame for sky subtraction - msgs.info("Loading the following frame for the relative spectral illumination correction:" + - msgs.newline() + opts['scale_corr'][ff]) - try: - spec2DObj_scl = spec2dobj.Spec2DObj.from_file(opts['scale_corr'][ff], detname) - except: - msgs.error("Could not load skysub image from spec2d file:" + msgs.newline() + opts['skysub_frame'][ff]) - relScaleImg = spec2DObj_scl.scaleimg - this_scalecorr = opts['scale_corr'][ff] - if this_scalecorr == "none": - msgs.info("Relative spectral illumination correction will not be performed.") - else: - msgs.info("Using the following frame for the relative spectral illumination correction:" + - msgs.newline()+this_scalecorr) - - # Prepare the relative scaling factors - relSclSky = skyScl/spec2DObj.scaleimg # This factor ensures the sky has the same relative scaling as the science frame - relScale = spec2DObj.scaleimg/relScaleImg # This factor is applied to the sky subtracted science frame - - # Extract the relevant information from the spec2d file - sciImg = (spec2DObj.sciimg - skyImg*relSclSky)*relScale # Subtract sky and apply relative illumination - ivar = spec2DObj.ivarraw / relScale**2 - waveimg = spec2DObj.waveimg - bpmmask = spec2DObj.bpmmask - - # TODO :: Really need to write some detailed information in the docs about all of the various corrections that can optionally be applied - - # TODO :: Include a flexure correction from the sky frame? Note, you cannot use the waveimg from a sky frame, - # since the heliocentric correction may have been applied to the sky frame. Need to recalculate waveimg using - # the slitshifts from a skyimage, and then apply the vel_corr from the science image. - - wnonzero = (waveimg != 0.0) - if not np.any(wnonzero): - msgs.error("The wavelength image contains only zeros - You need to check the data reduction.") - wave0 = waveimg[wnonzero].min() - # Calculate the delta wave in every pixel on the slit - waveimp = np.roll(waveimg, 1, axis=0) - waveimn = np.roll(waveimg, -1, axis=0) - dwaveimg = np.zeros_like(waveimg) - # All good pixels - wnz = np.where((waveimg!=0) & (waveimp!=0)) - dwaveimg[wnz] = np.abs(waveimg[wnz]-waveimp[wnz]) - # All bad pixels - wnz = np.where((waveimg!=0) & (waveimp==0)) - dwaveimg[wnz] = np.abs(waveimg[wnz]-waveimn[wnz]) - # All endpoint pixels - dwaveimg[0, :] = np.abs(waveimg[0, :] - waveimn[0, :]) - dwaveimg[-1, :] = np.abs(waveimg[-1, :] - waveimp[-1, :]) - dwv = np.median(dwaveimg[dwaveimg != 0.0]) if cubepar['wave_delta'] is None else cubepar['wave_delta'] - - msgs.info("Using wavelength solution: wave0={0:.3f}, dispersion={1:.3f} Angstrom/pixel".format(wave0, dwv)) - - # Obtain the minimum and maximum wavelength of all slits - if mnmx_wv is None: - mnmx_wv = np.zeros((len(files), slits.nslits, 2)) - for slit_idx, slit_spat in enumerate(slits.spat_id): - onslit_init = (slitid_img_init == slit_spat) - mnmx_wv[ff, slit_idx, 0] = np.min(waveimg[onslit_init]) - mnmx_wv[ff, slit_idx, 1] = np.max(waveimg[onslit_init]) - - # Remove edges of the spectrum where the sky model is bad - sky_is_good = make_good_skymask(slitid_img_init, spec2DObj.tilts) - - # Construct a good pixel mask - # TODO: This should use the mask function to figure out which elements are masked. - onslit_gpm = (slitid_img_init > 0) & (bpmmask.mask == 0) & sky_is_good - - # Grab the WCS of this frame - frame_wcs = spec.get_wcs(spec2DObj.head0, slits, detector.platescale, wave0, dwv) - all_wcs.append(copy.deepcopy(frame_wcs)) - - # Find the largest spatial scale of all images being combined - # TODO :: probably need to put this in the DetectorContainer - pxscl = detector.platescale * parse.parse_binning(detector.binning)[1] / 3600.0 # This should be degrees/pixel - slscl = spec.get_meta_value([spec2DObj.head0], 'slitwid') - if dspat is None: - dspat = max(pxscl, slscl) - if pxscl > dspat: - msgs.warn("Spatial scale requested ({0:f} arcsec) is less than the pixel scale ({1:f} arcsec)".format(3600.0*dspat, 3600.0*pxscl)) - if slscl > dspat: - msgs.warn("Spatial scale requested ({0:f} arcsec) is less than the slicer scale ({1:f} arcsec)".format(3600.0*dspat, 3600.0*slscl)) - - # Loading the alignments frame for these data - alignments = None - if cubepar['astrometric']: - key = alignframe.Alignments.calib_type.upper() - if key in spec2DObj.calibs: - alignfile = os.path.join(spec2DObj.calibs['DIR'], spec2DObj.calibs[key]) - if os.path.exists(alignfile) and cubepar['astrometric']: - msgs.info("Loading alignments") - alignments = alignframe.Alignments.from_file(alignfile) - else: - msgs.warn(f'Processed alignment frame not recorded or not found!') - msgs.info("Using slit edges for astrometric transform") - else: - msgs.info("Using slit edges for astrometric transform") - # If nothing better was provided, use the slit edges - if alignments is None: - left, right, _ = slits.select_edges(initial=True, flexure=spat_flexure) - locations = [0.0, 1.0] - traces = np.append(left[:,None,:], right[:,None,:], axis=1) - else: - locations = parset['calibrations']['alignment']['locations'] - traces = alignments.traces - # Generate an RA/DEC image - msgs.info("Generating RA/DEC image") - alignSplines = alignframe.AlignmentSplines(traces, locations, spec2DObj.tilts) - raimg, decimg, minmax = slits.get_radec_image(frame_wcs, alignSplines, spec2DObj.tilts, - initial=True, flexure=spat_flexure) - # Perform the DAR correction - if wave_ref is None: - wave_ref = 0.5*(np.min(waveimg[onslit_gpm]) + np.max(waveimg[onslit_gpm])) - # Get DAR parameters - raval = spec.get_meta_value([spec2DObj.head0], 'ra') - decval = spec.get_meta_value([spec2DObj.head0], 'dec') - obstime = spec.get_meta_value([spec2DObj.head0], 'obstime') - pressure = spec.get_meta_value([spec2DObj.head0], 'pressure') - temperature = spec.get_meta_value([spec2DObj.head0], 'temperature') - rel_humidity = spec.get_meta_value([spec2DObj.head0], 'humidity') - coord = SkyCoord(raval, decval, unit=(units.deg, units.deg)) - location = spec.location # TODO :: spec.location should probably end up in the TelescopePar (spec.telescope.location) - if pressure == 0.0: - msgs.warn("Pressure is set to zero - DAR correction will not be performed") - else: - msgs.info("DAR correction parameters:"+msgs.newline() + - " Pressure = {0:f} bar".format(pressure) + msgs.newline() + - " Temperature = {0:f} deg C".format(temperature) + msgs.newline() + - " Humidity = {0:f}".format(rel_humidity)) - ra_corr, dec_corr = correct_dar(waveimg[onslit_gpm], coord, obstime, location, - pressure * units.bar, temperature * units.deg_C, rel_humidity, wave_ref=wave_ref) - raimg[onslit_gpm] += ra_corr*np.cos(np.mean(decimg[onslit_gpm]) * np.pi / 180.0) - decimg[onslit_gpm] += dec_corr - - # Get copies of arrays to be saved - wave_ext = waveimg[onslit_gpm].copy() - flux_ext = sciImg[onslit_gpm].copy() - ivar_ext = ivar[onslit_gpm].copy() - dwav_ext = dwaveimg[onslit_gpm].copy() - - # Correct for sensitivity as a function of grating angle - # (this assumes the spectrum of the flatfield lamp has the same shape for all setups) - key = flatfield.FlatImages.calib_type.upper() - if key not in spec2DObj.calibs: - msgs.error('Processed flat calibration file not recorded by spec2d file!') - flatfile = os.path.join(spec2DObj.calibs['DIR'], spec2DObj.calibs[key]) - if cubepar['grating_corr'] and flatfile not in flat_splines.keys(): - msgs.info("Calculating relative sensitivity for grating correction") - # Check if the Flat file exists - if not os.path.exists(flatfile): - msgs.error("Grating correction requested, but the following file does not exist:" + - msgs.newline() + flatfile) - # Load the Flat file - flatimages = flatfield.FlatImages.from_file(flatfile) - total_illum = flatimages.fit2illumflat(slits, finecorr=False, frametype='illum', initial=True, spat_flexure=spat_flexure) * \ - flatimages.fit2illumflat(slits, finecorr=True, frametype='illum', initial=True, spat_flexure=spat_flexure) - flatframe = flatimages.pixelflat_raw / total_illum - if flatimages.pixelflat_spec_illum is None: - # Calculate the relative scale - scale_model = flatfield.illum_profile_spectral(flatframe, waveimg, slits, - slit_illum_ref_idx=flatpar['slit_illum_ref_idx'], model=None, - skymask=None, trim=flatpar['slit_trim'], flexure=spat_flexure, - smooth_npix=flatpar['slit_illum_smooth_npix']) - else: - msgs.info("Using relative spectral illumination from FlatImages") - scale_model = flatimages.pixelflat_spec_illum - # Apply the relative scale and generate a 1D "spectrum" - onslit = waveimg != 0 - wavebins = np.linspace(np.min(waveimg[onslit]), np.max(waveimg[onslit]), slits.nspec) - hist, edge = np.histogram(waveimg[onslit], bins=wavebins, weights=flatframe[onslit]/scale_model[onslit]) - cntr, edge = np.histogram(waveimg[onslit], bins=wavebins) - cntr = cntr.astype(float) - norm = (cntr != 0) / (cntr + (cntr == 0)) - spec_spl = hist * norm - wave_spl = 0.5 * (wavebins[1:] + wavebins[:-1]) - flat_splines[flatfile] = interp1d(wave_spl, spec_spl, kind='linear', - bounds_error=False, fill_value="extrapolate") - flat_splines[flatfile+"_wave"] = wave_spl.copy() - # Check if a reference blaze spline exists (either from a standard star if fluxing or from a previous - # exposure in this for loop) - if blaze_spline is None: - blaze_wave, blaze_spec = wave_spl, spec_spl - blaze_spline = interp1d(wave_spl, spec_spl, kind='linear', - bounds_error=False, fill_value="extrapolate") - - # Perform extinction correction - msgs.info("Applying extinction correction") - longitude = spec.telescope['longitude'] - latitude = spec.telescope['latitude'] - airmass = spec2DObj.head0[spec.meta['airmass']['card']] - extinct = flux_calib.load_extinction_data(longitude, latitude, senspar['UVIS']['extinct_file']) - # extinction_correction requires the wavelength is sorted - wvsrt = np.argsort(wave_ext) - ext_corr = flux_calib.extinction_correction(wave_ext[wvsrt] * units.AA, airmass, extinct) - # Grating correction - grat_corr = 1.0 - if cubepar['grating_corr']: - grat_corr = correct_grating_shift(wave_ext[wvsrt], flat_splines[flatfile + "_wave"], flat_splines[flatfile], - blaze_wave, blaze_spline) - # Sensitivity function - sens_func = 1.0 - if fluxcal: - msgs.info("Calculating the sensitivity function") - sens_func = flux_spline(wave_ext[wvsrt]) - # Convert the flux_sav to counts/s, correct for the relative sensitivity of different setups - ext_corr *= sens_func / (exptime * grat_corr) - # Correct for extinction - flux_sav = flux_ext[wvsrt] * ext_corr - ivar_sav = ivar_ext[wvsrt] / ext_corr ** 2 - - # Convert units to Counts/s/Ang/arcsec2 - # Slicer sampling * spatial pixel sampling - sl_deg = np.sqrt(frame_wcs.wcs.cd[0, 0] ** 2 + frame_wcs.wcs.cd[1, 0] ** 2) - px_deg = np.sqrt(frame_wcs.wcs.cd[1, 1] ** 2 + frame_wcs.wcs.cd[0, 1] ** 2) - scl_units = dwav_ext[wvsrt] * (3600.0 * sl_deg) * (3600.0 * px_deg) - flux_sav /= scl_units - ivar_sav *= scl_units ** 2 - - # sort back to the original ordering - resrt = np.argsort(wvsrt) - numpix = raimg[onslit_gpm].size - - # Calculate the weights relative to the zeroth cube - weights[ff] = 1.0#exptime #np.median(flux_sav[resrt]*np.sqrt(ivar_sav[resrt]))**2 - - # Get the slit image and then unset pixels in the slit image that are bad - this_specpos, this_spatpos = np.where(onslit_gpm) - this_spatid = slitid_img_init[onslit_gpm] - - # If individual frames are to be output without aligning them, - # there's no need to store information, just make the cubes now - if not combine and not align: - # Get the output filename - if numfiles == 1 and cubepar['output_filename'] != "": - outfile = get_output_filename("", cubepar['output_filename'], True, -1) - else: - outfile = get_output_filename(fil, cubepar['output_filename'], combine, ff+1) - # Get the coordinate bounds - slitlength = int(np.round(np.median(slits.get_slitlengths(initial=True, median=True)))) - numwav = int((np.max(waveimg) - wave0) / dwv) - bins = spec.get_datacube_bins(slitlength, minmax, numwav) - # Generate the output WCS for the datacube - crval_wv = cubepar['wave_min'] if cubepar['wave_min'] is not None else 1.0E10 * frame_wcs.wcs.crval[2] - cd_wv = cubepar['wave_delta'] if cubepar['wave_delta'] is not None else 1.0E10 * frame_wcs.wcs.cd[2, 2] - output_wcs = spec.get_wcs(spec2DObj.head0, slits, detector.platescale, crval_wv, cd_wv) - # Set the wavelength range of the white light image. - wl_wvrng = None - if cubepar['save_whitelight']: - wl_wvrng = get_whitelight_range(np.max(mnmx_wv[ff, :, 0]), - np.min(mnmx_wv[ff, :, 1]), - cubepar['whitelight_range']) - # Make the datacube - if method in ['subpixel', 'ngp']: - # Generate the datacube - generate_cube_subpixel(outfile, output_wcs, raimg[onslit_gpm], decimg[onslit_gpm], wave_ext, - flux_sav[resrt], ivar_sav[resrt], np.ones(numpix), - this_spatpos, this_specpos, this_spatid, - spec2DObj.tilts, slits, alignSplines, bins, - all_idx=None, overwrite=overwrite, blaze_wave=blaze_wave, blaze_spec=blaze_spec, - fluxcal=fluxcal, specname=specname, whitelight_range=wl_wvrng, - spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel) - continue - - # Store the information if we are combining multiple frames - all_ra = np.append(all_ra, raimg[onslit_gpm].copy()) - all_dec = np.append(all_dec, decimg[onslit_gpm].copy()) - all_wave = np.append(all_wave, wave_ext.copy()) - all_sci = np.append(all_sci, flux_sav[resrt].copy()) - all_ivar = np.append(all_ivar, ivar_sav[resrt].copy()) - all_idx = np.append(all_idx, ff*np.ones(numpix)) - all_wghts = np.append(all_wghts, weights[ff]*np.ones(numpix)/weights[0]) - all_spatpos = np.append(all_spatpos, this_spatpos) - all_specpos = np.append(all_specpos, this_specpos) - all_spatid = np.append(all_spatid, this_spatid) - all_tilts.append(spec2DObj.tilts) - all_slits.append(slits) - all_align.append(alignSplines) - - # No need to continue if we are not combining nor aligning frames - if not combine and not align: - return - - # Grab cos(dec) for convenience - cosdec = np.cos(np.mean(all_dec) * np.pi / 180.0) - - # Register spatial offsets between all frames - if align: - if opts['ra_offset'] is not None: - # First, translate all coordinates to the coordinates of the first frame - # Note :: Don't need cosdec here, this just overrides the IFU coordinate centre of each frame - ref_shift_ra = ifu_ra[0] - ifu_ra - ref_shift_dec = ifu_dec[0] - ifu_dec - for ff in range(numfiles): - # Apply the shift - all_ra[all_idx == ff] += ref_shift_ra[ff] + opts['ra_offset'][ff]/3600.0 - all_dec[all_idx == ff] += ref_shift_dec[ff] + opts['dec_offset'][ff]/3600.0 - msgs.info("Spatial shift of cube #{0:d}: RA, DEC (arcsec) = {1:+0.3f} E, {2:+0.3f} N".format(ff + 1, opts['ra_offset'][ff], opts['dec_offset'][ff])) - else: - # Find the wavelength range where all frames overlap - min_wl, max_wl = get_whitelight_range(np.max(mnmx_wv[:, :, 0]), # The max blue wavelength - np.min(mnmx_wv[:, :, 1]), # The min red wavelength - cubepar['whitelight_range']) # The user-specified values (if any) - # Get the good whitelight pixels - ww, wavediff = get_whitelight_pixels(all_wave, min_wl, max_wl) - # Iterate over white light image generation and spatial shifting - numiter = 2 - for dd in range(numiter): - msgs.info(f"Iterating on spatial translation - ITERATION #{dd+1}/{numiter}") - # Setup the WCS to use for all white light images - ref_idx = None # Don't use an index - This is the default behaviour when a reference image is supplied - image_wcs, voxedge, reference_image = create_wcs(cubepar, all_ra[ww], all_dec[ww], all_wave[ww], - dspat, wavediff, collapse=True) - if voxedge[2].size != 2: - msgs.error("Spectral range for WCS is incorrect for white light image") - - wl_imgs = generate_image_subpixel(image_wcs, all_ra[ww], all_dec[ww], all_wave[ww], - all_sci[ww], all_ivar[ww], all_wghts[ww], - all_spatpos[ww], all_specpos[ww], all_spatid[ww], - all_tilts, all_slits, all_align, voxedge, all_idx=all_idx[ww], - spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel) - if reference_image is None: - # ref_idx will be the index of the cube with the highest S/N - ref_idx = np.argmax(weights) - reference_image = wl_imgs[:, :, ref_idx].copy() - msgs.info("Calculating spatial translation of each cube relative to cube #{0:d})".format(ref_idx+1)) - else: - msgs.info("Calculating the spatial translation of each cube relative to user-defined 'reference_image'") - - # Calculate the image offsets relative to the reference image - for ff in range(numfiles): - # Calculate the shift - ra_shift, dec_shift = calculate_image_phase(reference_image.copy(), wl_imgs[:, :, ff], maskval=0.0) - # Convert pixel shift to degrees shift - ra_shift *= dspat/cosdec - dec_shift *= dspat - msgs.info("Spatial shift of cube #{0:d}: RA, DEC (arcsec) = {1:+0.3f} E, {2:+0.3f} N".format(ff+1, ra_shift*3600.0, dec_shift*3600.0)) - # Apply the shift - all_ra[all_idx == ff] += ra_shift - all_dec[all_idx == ff] += dec_shift - - # Calculate the relative spectral weights of all pixels - if numfiles == 1: - # No need to calculate weights if there's just one frame - all_wghts = np.ones_like(all_sci) - else: - # Find the wavelength range where all frames overlap - min_wl, max_wl = get_whitelight_range(np.max(mnmx_wv[:, :, 0]), # The max blue wavelength - np.min(mnmx_wv[:, :, 1]), # The min red wavelength - cubepar['whitelight_range']) # The user-specified values (if any) - # Get the good white light pixels - ww, wavediff = get_whitelight_pixels(all_wave, min_wl, max_wl) - # Get a suitable WCS - image_wcs, voxedge, reference_image = create_wcs(cubepar, all_ra, all_dec, all_wave, dspat, wavediff, collapse=True) - # Generate the white light image (note: hard-coding subpixel=1 in both directions, and combining into a single image) - wl_full = generate_image_subpixel(image_wcs, all_ra, all_dec, all_wave, - all_sci, all_ivar, all_wghts, - all_spatpos, all_specpos, all_spatid, - all_tilts, all_slits, all_align, voxedge, all_idx=all_idx, - spec_subpixel=1, spat_subpixel=1, combine=True) - # Compute the weights - all_wghts = compute_weights(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, wl_full[:, :, 0], - dspat, dwv, relative_weights=cubepar['relative_weights']) - - # Generate the WCS, and the voxel edges - cube_wcs, vox_edges, _ = create_wcs(cubepar, all_ra, all_dec, all_wave, dspat, dwv) - - sensfunc = None - if flux_spline is not None: - # Get wavelength of each pixel, and note that the WCS gives this in m, so convert to Angstroms (x 1E10) - numwav = vox_edges[2].size-1 - senswave = cube_wcs.spectral.wcs_pix2world(np.arange(numwav), 0)[0] * 1.0E10 - sensfunc = flux_spline(senswave) - - # Generate a datacube - outfile = get_output_filename("", cubepar['output_filename'], True, -1) - if method in ['subpixel', 'ngp']: - # Generate the datacube - wl_wvrng = None - if cubepar['save_whitelight']: - wl_wvrng = get_whitelight_range(np.max(mnmx_wv[:, :, 0]), - np.min(mnmx_wv[:, :, 1]), - cubepar['whitelight_range']) - if combine: - generate_cube_subpixel(outfile, cube_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, - np.ones(all_wghts.size), # all_wghts, - all_spatpos, all_specpos, all_spatid, all_tilts, all_slits, all_align, vox_edges, - all_idx=all_idx, overwrite=overwrite, blaze_wave=blaze_wave, blaze_spec=blaze_spec, - fluxcal=fluxcal, sensfunc=sensfunc, specname=specname, whitelight_range=wl_wvrng, - spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel) - else: - for ff in range(numfiles): - outfile = get_output_filename("", cubepar['output_filename'], False, ff) - ww = np.where(all_idx == ff) - generate_cube_subpixel(outfile, cube_wcs, all_ra[ww], all_dec[ww], all_wave[ww], all_sci[ww], all_ivar[ww], np.ones(all_wghts[ww].size), - all_spatpos[ww], all_specpos[ww], all_spatid[ww], all_tilts[ff], all_slits[ff], all_align[ff], vox_edges, - all_idx=all_idx[ww], overwrite=overwrite, blaze_wave=blaze_wave, blaze_spec=blaze_spec, - fluxcal=fluxcal, sensfunc=sensfunc, specname=specname, whitelight_range=wl_wvrng, - spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel) - - + return flxcube, varcube, bpmcube, residcube + return flxcube, varcube, bpmcube From 7dbc09ff12aae44ff6b6518b8864504ea5b79614 Mon Sep 17 00:00:00 2001 From: joe Date: Wed, 15 Nov 2023 16:10:19 +0100 Subject: [PATCH 070/244] Trying to get echelle coadding weorking on multiple sertups. --- pypeit/coadd1d.py | 12 +++++++++--- pypeit/core/coadd.py | 4 +++- pypeit/sensfunc.py | 9 ++++++++- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/pypeit/coadd1d.py b/pypeit/coadd1d.py index a6eda80977..fddadbf6ea 100644 --- a/pypeit/coadd1d.py +++ b/pypeit/coadd1d.py @@ -200,7 +200,7 @@ def load(self): msgs.error("Error in spec1d file for exposure {:d}: " "More than one object was identified with the OBJID={:s} in file={:s}".format( iexp, self.objids[iexp], self.spec1dfiles[iexp])) - wave_iexp, flux_iexp, ivar_iexp, gpm_iexp, trace_spec, trace_spat, meta_spec, header = \ + wave_iexp, flux_iexp, ivar_iexp, gpm_iexp, _, _, _, header = \ sobjs[indx].unpack_object(ret_flam=self.par['flux_value'], extract_type=self.par['ex_value']) waves.append(wave_iexp) fluxes.append(flux_iexp) @@ -397,7 +397,6 @@ def coadd(self): = coadd.ech_combspec(self.waves, self.fluxes, self.ivars, self.gpms, self.weights_sens, setup_ids=self.unique_setups, nbests=self.par['nbests'], - sn_smooth_npix=self.par['sn_smooth_npix'], wave_method=self.par['wave_method'], dv=self.par['dv'], dwave=self.par['dwave'], dloglam=self.par['dloglam'], wave_grid_min=self.par['wave_grid_min'], @@ -441,8 +440,13 @@ def load_ech_arrays(self, spec1dfiles, objids, sensfuncfiles): indx = sobjs.name_indices(objids[iexp]) if not np.any(indx): msgs.error("No matching objects for {:s}. Odds are you input the wrong OBJID".format(objids[iexp])) - wave_iexp, flux_iexp, ivar_iexp, gpm_iexp, trace_spec, trace_spat, meta_spec, header = \ + wave_iexp, flux_iexp, ivar_iexp, gpm_iexp, _, _, _, header = \ sobjs[indx].unpack_object(ret_flam=self.par['flux_value'], extract_type=self.par['ex_value']) + # This np.atleast2d hack deals with the situation where we are wave_iexp is actually Multislit data, i.e. we are treating + # it like an echelle spectrograph with a single order. This usage case arises when we want to use the + # echelle coadding code to combine echelle and multislit data + if wave_iexp.ndim == 1: + wave_iexp, flux_iexp, ivar_iexp, gpm_iexp = np.atleast_2d(wave_iexp).T, np.atleast_2d(flux_iexp).T, np.atleast_2d(ivar_iexp).T, np.atleast_2d(gpm_iexp).T weights_sens_iexp = sensfunc.SensFunc.sensfunc_weights(sensfuncfiles[iexp], wave_iexp, debug=self.debug) # Allocate arrays on first iteration # TODO :: We should refactor to use a list of numpy arrays, instead of a 2D numpy array. @@ -460,6 +464,8 @@ def load_ech_arrays(self, spec1dfiles, objids, sensfuncfiles): # Store the information waves[...,iexp], fluxes[...,iexp], ivars[..., iexp], gpms[...,iexp], weights_sens[...,iexp] \ = wave_iexp, flux_iexp, ivar_iexp, gpm_iexp, weights_sens_iexp + + return waves, fluxes, ivars, gpms, weights_sens, header_out diff --git a/pypeit/core/coadd.py b/pypeit/core/coadd.py index c9c492e760..98ed7e886c 100644 --- a/pypeit/core/coadd.py +++ b/pypeit/core/coadd.py @@ -749,7 +749,7 @@ def smooth_weights(inarr, gdmsk, sn_smooth_npix): sn_conv = convolution.convolve(sn_med2, gauss_kernel, boundary='extend') return sn_conv - +# TODO add a wave_min, wave_max option here to deal with objects like high-z QSOs etc. which only have flux in a given wavelength range. def calc_snr(fluxes, ivars, gpms): """ @@ -2659,6 +2659,7 @@ def ech_combspec(waves_arr_setup, fluxes_arr_setup, ivars_arr_setup, gpms_arr_se rms_sn_setup_list = [] colors = [] for isetup in range(nsetups): + ## Need to feed have optional wave_min, wave_max here to deal with sources like high-z quasars? rms_sn_vec, _ = calc_snr(fluxes_setup_list[isetup], ivars_setup_list[isetup], gpms_setup_list[isetup]) rms_sn = rms_sn_vec.reshape(norders[isetup], nexps[isetup]) mean_sn_ord = np.mean(rms_sn, axis=1) @@ -2671,6 +2672,7 @@ def ech_combspec(waves_arr_setup, fluxes_arr_setup, ivars_arr_setup, gpms_arr_se colors.append([setup_colors[isetup]]*norders[isetup]*nexps[isetup]) + # Create the waves_setup_list weights_setup_list = [utils.echarr_to_echlist(weight)[0] for weight in weights] diff --git a/pypeit/sensfunc.py b/pypeit/sensfunc.py index 33937efba5..ff4cb116f7 100644 --- a/pypeit/sensfunc.py +++ b/pypeit/sensfunc.py @@ -807,7 +807,7 @@ def sensfunc_weights(cls, sensfile, waves, debug=False, extrap_sens=True): Returns: `numpy.ndarray`_: sensfunc weights evaluated on the input waves - wavelength grid + wavelength grid, shape = same as waves """ sens = cls.from_file(sensfile) # wave, zeropoint, meta_table, out_table, header_sens = sensfunc.SensFunc.load(sensfile) @@ -819,6 +819,10 @@ def sensfunc_weights(cls, sensfile, waves, debug=False, extrap_sens=True): elif waves.ndim == 3: nspec, norder, nexp = waves.shape waves_stack = waves + elif waves.ndim == 1: + nspec = waves.size + norder, nexp = 1, 1 + waves_stack = np.reshape(waves, (nspec, 1, 1)) else: msgs.error('Unrecognized dimensionality for waves') @@ -839,8 +843,11 @@ def sensfunc_weights(cls, sensfile, waves, debug=False, extrap_sens=True): coadd.weights_qa(utils.echarr_to_echlist(waves_stack)[0], utils.echarr_to_echlist(weights_stack)[0], utils.echarr_to_echlist(waves_stack > 1.0)[0], title='sensfunc_weights') + # Reshape to be the same size/dimension as the input spectrum if waves.ndim == 2: weights_stack = np.reshape(weights_stack, (nspec, norder)) + elif waves.ndim == 1: + weights_stack = np.reshape(weights_stack, (nspec)) return weights_stack From 5c900c3199aa21a0358abcdcea4c3ce011c5cdf7 Mon Sep 17 00:00:00 2001 From: rcooke Date: Mon, 20 Nov 2023 10:31:37 +0000 Subject: [PATCH 071/244] spline eval --- pypeit/coadd3d.py | 3 +- pypeit/core/flat.py | 111 ++++++++++++++++++++++++++++++++++++-- pypeit/flatfield.py | 5 +- pypeit/images/rawimage.py | 3 +- 4 files changed, 113 insertions(+), 9 deletions(-) diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index 58b74265d3..a597152f5a 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -771,8 +771,7 @@ def add_grating_corr(self, flatfile, waveimg, slits, spat_flexure=None): # Calculate the relative scale scale_model = flatfield.illum_profile_spectral(flatframe, waveimg, slits, slit_illum_ref_idx=self.flatpar['slit_illum_ref_idx'], - model=None, - skymask=None, trim=self.flatpar['slit_trim'], + model=None, trim=self.flatpar['slit_trim'], flexure=spat_flexure, smooth_npix=self.flatpar['slit_illum_smooth_npix']) else: diff --git a/pypeit/core/flat.py b/pypeit/core/flat.py index 3d8b640fc5..6c4945c539 100644 --- a/pypeit/core/flat.py +++ b/pypeit/core/flat.py @@ -285,7 +285,7 @@ def construct_illum_profile(norm_spec, spat_coo, slitwidth, spat_gpm=None, spat_ def illum_profile_spectral_poly(rawimg, waveimg, slitmask, slitmask_trim, model, slit_illum_ref_idx=0, - gpmask=None, thismask=None, nbins=20): + gpmask=None, thismask=None, nbins=20, debug=False): """ Use a polynomial fit to control points along the spectral direction to determine the relative spectral illumination of all slits. Currently, this routine is only used for image slicer IFUs. @@ -300,19 +300,21 @@ def illum_profile_spectral_poly(rawimg, waveimg, slitmask, slitmask_trim, model, A 2D int mask, the same shape as rawimg, indicating which pixels are on a slit. A -1 value indicates not on a slit, while any pixels on a slit should have the value of the slit spatial ID number. - slitmask_trim : + slitmask_trim : `numpy.ndarray`_ Same as slitmask, but the slit edges are trimmed. model : `numpy.ndarray`_ A model of the rawimg data. - slit_illum_ref_idx : int + slit_illum_ref_idx : :obj:`int` Index of slit that is used as the reference. gpmask : `numpy.ndarray`_, optional Boolean good pixel mask (True = Good) thismask : `numpy.ndarray`_, optional A boolean mask (True = good) that indicates all pixels where the scaleImg should be constructed. If None, the slitmask that is generated by this routine will be used. - nbins : int + nbins : :obj:`int` Number of bins in the spectral direction to sample the relative spectral sensitivity + debug : :obj:`bool` + If True, some plots will be output to test if the fitting is working correctly. Returns ------- @@ -347,6 +349,11 @@ def illum_profile_spectral_poly(rawimg, waveimg, slitmask, slitmask_trim, model, wgd = np.where(scale_err > 0) coeff = np.polyfit(wavcen[wgd], scale_bin[wgd], w=1/scale_err[wgd], deg=2) scaleImg[this_slit] *= np.polyval(coeff, waveimg[this_slit]) + if debug: + mod = np.polyval(coeff, wavcen[wgd]) + plt.errorbar(wavcen[wgd], scale_bin[wgd], yerr=scale_err[wgd], fmt='o') + plt.plot(wavcen[wgd], mod, 'r-') + plt.show() if sl == slit_illum_ref_idx: scaleImg[_thismask] *= utils.inverse(np.polyval(coeff, waveimg[_thismask])) minv, maxv = np.min(scaleImg[_thismask]), np.max(scaleImg[_thismask]) @@ -354,6 +361,102 @@ def illum_profile_spectral_poly(rawimg, waveimg, slitmask, slitmask_trim, model, return scaleImg +def spline_map(rawimg, rawivar, waveimg, slitmask, slitmask_trim, modelimg, + slit_illum_ref_idx=0, gpmask=None, thismask=None, nbins=20, debug=False): + """ + Use a spline fit to control points along the spectral direction to construct a map between modelimg and + rawimg. Currently, this routine is only used for image slicer IFUs. + + Parameters + ---------- + rawimg : `numpy.ndarray`_ + Image data that will be used to estimate the spectral relative sensitivity + rawivar : `numpy.ndarray`_ + Inverse variance image of rawimg + waveimg : `numpy.ndarray`_ + Wavelength image + slitmask : `numpy.ndarray`_ + A 2D int mask, the same shape as rawimg, indicating which pixels are on a slit. A -1 value + indicates not on a slit, while any pixels on a slit should have the value of the slit spatial ID + number. + slitmask_trim : `numpy.ndarray`_ + Same as slitmask, but the slit edges are trimmed. + modelimg : `numpy.ndarray`_ + A model of the rawimg data. + slit_illum_ref_idx : :obj:`int` + Index of slit that is used as the reference. + gpmask : `numpy.ndarray`_, optional + Boolean good pixel mask (True = Good) + thismask : `numpy.ndarray`_, optional + A boolean mask (True = good) that indicates all pixels where the scaleImg should be constructed. + If None, the slitmask that is generated by this routine will be used. + nbins : :obj:`int` + Number of bins in the spectral direction to sample the relative spectral sensitivity + debug : :obj:`bool` + If True, some plots will be output to test if the fitting is working correctly. + + Returns + ------- + scale_model: `numpy.ndarray`_ + An image containing the appropriate scaling + """ + # TODO :: REMOVE THIS... IT IS USED AS A GUIDE TO DEVELOP THE METHOD!!! + # This problem needs to be recast into a smoothing problem, that can take advantage of the following: + # https://docs.scipy.org/doc/scipy/reference/generated/scipy.interpolate.UnivariateSpline.html#scipy.interpolate.UnivariateSpline + # where: + # y = science/model + # w = model/science_error + # resid = w*(y-f(x)) + # Then, iterate over this. The reason to iterate is that there should be a mapping between science and science_error. Once we have an estimate of f(x), we can do the following: + # spl = spline(science, science_error) + # new_science_error = spl(model*f(x)) + # Now iterate a few times until new_science_error (and the fit) is converged. + + # Some variables to consider putting as function arguments + numiter = 4 + + # Start by calculating a ratio of the raming and the modelimg + nspec = rawimg.shape[0] + _ratio = rawimg * utils.inverse(modelimg) + _ratio_ivar = rawivar * modelimg**2 + _fit_wghts = modelimg * np.sqrt(rawivar) + + # Generate the mask + _thismask = thismask if (thismask is not None) else (slitmask > 0) + gpm = gpmask if (gpmask is not None) else np.ones_like(rawimg, dtype=bool) + # Extract the list of spatial IDs from the slitmask + slitmask_spatid = np.unique(slitmask) + slitmask_spatid = np.sort(slitmask_spatid[slitmask_spatid > 0]) + + # Create a spline between the raw data and the error + embed() + flxsrt = np.argsort(np.ravel(rawimg)) + spl = scipy.interpolate.interp1d(np.ravel(rawimg)[flxsrt], np.ravel(rawivar)[flxsrt], kind='linear', + bounds_error=False, fill_value=0.0, assume_sorted=True) + modelmap = np.zeros_like(rawimg) + for sl, spatid in enumerate(slitmask_spatid): + print("sl") + # Prepare the masks, edges, and fitting variables + this_slit = (slitmask == spatid) + this_slit_trim = (slitmask_trim == spatid) + this_slit_mask = gpm & this_slit_trim + this_wave = waveimg[this_slit_mask] + this_wghts = _fit_wghts[this_slit_mask] + asrt = np.argsort(this_wave) + for ii in range(numiter): + # Generate the map between model and data + splmap = scipy.interpolate.UnivariateSpline(this_wave[asrt], _ratio[this_slit_mask][asrt], w=this_wghts[asrt], + bbox=[None, None], k=3, s=nspec, ext=0, check_finite=False) + # Construct the mapping, and use this to make a model of the rawdata + this_modmap = splmap(this_wave[this_slit_mask]) + this_modflx = modelimg[this_slit_mask] * this_modmap + # Update the fit weights + this_wghts = modelimg[this_slit_mask] * np.sqrt(spl(this_modflx)) + # Produce the final model for this slit + modelmap[this_slit] = splmap(this_wave[this_slit]) + return modelmap + + # TODO: See pypeit/deprecated/flat.py for the previous version. We need # to continue to vet this algorithm to make sure there are no # unforeseen corner cases that cause errors. diff --git a/pypeit/flatfield.py b/pypeit/flatfield.py index ea13b2c3c3..4f0154b929 100644 --- a/pypeit/flatfield.py +++ b/pypeit/flatfield.py @@ -1807,7 +1807,7 @@ def show_flats(image_list, wcs_match=True, slits=None, waveimg=None): def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_npix=None, - model=None, gpmask=None, skymask=None, trim=3, flexure=None): + model=None, gpmask=None, skymask=None, trim=3, flexure=None, maxiter=5): """ TODO :: This could possibly be moved to core.flat @@ -1837,6 +1837,8 @@ def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_ when deriving the spectral illumination flexure : float, None Spatial flexure + maxiter : :obj:`int` + Maximum number of iterations to perform Returns ------- @@ -1879,7 +1881,6 @@ def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_ sn_smooth_npix = wave_ref.size // smooth_npix if (smooth_npix is not None) else wave_ref.size // 10 # Iterate until convergence - maxiter = 5 lo_prev, hi_prev = 1.0E-32, 1.0E32 for rr in range(maxiter): # Reset the relative scaling for this iteration diff --git a/pypeit/images/rawimage.py b/pypeit/images/rawimage.py index d4f029a044..139bf38608 100644 --- a/pypeit/images/rawimage.py +++ b/pypeit/images/rawimage.py @@ -1193,6 +1193,7 @@ def subtract_scattlight(self, msscattlight, slits, debug=False): f" {tmp[10]}, # Relative kernel scale (>1 means the kernel is more Gaussian, >0 but <1 makes the profile more lorentzian)\n" + \ f" {tmp[11]}, {tmp[12]}, # Polynomial terms (coefficients of \"spat\" and \"spat*spec\")\n" + \ f" {tmp[13]}, {tmp[14]}, {tmp[15]}]) # Polynomial terms (coefficients of spec**index)\n" + # f" {tmp[13]}, {tmp[14]}]) # Polynomial terms (coefficients of spec**index)\n" print(strprint) pad = msscattlight.pad // spatbin offslitmask = slits.slit_img(pad=pad, initial=True, flexure=None) == -1 @@ -1206,7 +1207,7 @@ def subtract_scattlight(self, msscattlight, slits, debug=False): plt.subplot(223) plt.imshow(scatt_img*offslitmask, vmin=-2*vmax, vmax=2*vmax) plt.subplot(224) - plt.imshow((_frame - scatt_img)*offslitmask, vmin=-2*vmax, vmax=2*vmax) + plt.imshow((_frame - scatt_img)*offslitmask, vmin=-vmax/5, vmax=vmax/5) # plt.imshow((_frame - scatt_img)*offslitmask, vmin=-vmax/5, vmax=vmax/5) plt.show() elif self.par["scattlight"]["method"] == "archive": From 9d600e7233357145246ad3395e04db323a097ff9 Mon Sep 17 00:00:00 2001 From: rcooke Date: Wed, 29 Nov 2023 16:43:34 +0000 Subject: [PATCH 072/244] comment update --- pypeit/spectrographs/keck_kcwi.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pypeit/spectrographs/keck_kcwi.py b/pypeit/spectrographs/keck_kcwi.py index b3157e055d..cc48e0ddb9 100644 --- a/pypeit/spectrographs/keck_kcwi.py +++ b/pypeit/spectrographs/keck_kcwi.py @@ -286,6 +286,7 @@ def default_pypeit_par(cls): # LACosmics parameters par['scienceframe']['process']['sigclip'] = 4.0 par['scienceframe']['process']['objlim'] = 1.5 + # Illumination corrections par['scienceframe']['process']['use_illumflat'] = True # illumflat is applied when building the relative scale image in reduce.py, so should be applied to scienceframe too. par['scienceframe']['process']['use_specillum'] = True # apply relative spectral illumination par['scienceframe']['process']['spat_flexure_correct'] = False # don't correct for spatial flexure - varying spatial illumination profile could throw this correction off. Also, there's no way to do astrometric correction if we can't correct for spatial flexure of the contbars frames From 8368aeae42363d9991d11c85c2abbb8a4c15d470 Mon Sep 17 00:00:00 2001 From: rcooke Date: Wed, 29 Nov 2023 16:44:16 +0000 Subject: [PATCH 073/244] specillum fix + docstring --- pypeit/flatfield.py | 92 ++++++++++++++++++++++++++------------------- 1 file changed, 53 insertions(+), 39 deletions(-) diff --git a/pypeit/flatfield.py b/pypeit/flatfield.py index 4f0154b929..43c5c54191 100644 --- a/pypeit/flatfield.py +++ b/pypeit/flatfield.py @@ -1572,7 +1572,7 @@ def spectral_illumination(self, gpm=None, debug=False): msgs.info('Performing a joint fit to the flat-field response') # Grab some parameters trim = self.flatpar['slit_trim'] - rawflat = self.rawflatimg.image.copy() / self.msillumflat.copy() + rawflat = self.rawflatimg.image / (self.msillumflat * self.mspixelflat) # Grab the GPM and the slit images if gpm is None: # TODO: Should this be *any* flag, or just BPM? @@ -1770,19 +1770,18 @@ def detector_structure_qa(det_resp, det_resp_model, outfile=None, title="Detecto def show_flats(image_list, wcs_match=True, slits=None, waveimg=None): """ - Interface to ginga to show a set of flat images + Show the flat-field images - Args: - pixelflat (`numpy.ndarray`_): - illumflat (`numpy.ndarray`_ or None): - procflat (`numpy.ndarray`_): - flat_model (`numpy.ndarray`_): - spec_illum (`numpy.ndarray`_ or None): - wcs_match (bool, optional): - slits (:class:`~pypeit.slittrace.SlitTraceSet`, optional): - waveimg (`numpy.ndarray`_ or None): - - Returns: + Parameters + ---------- + image_list : list + List of tuples containing the image data, image name and the cut levels + wcs_match : bool, optional + Match the WCS of the images + slits : :class:`pypeit.slittrace.SlitTraceSet`, optional + Slit traces to be overplotted on the images + waveimg : `numpy.ndarray`_, optional + Wavelength image to be overplotted on the images """ display.connect_to_ginga(raise_err=True, allow_new=True) @@ -1883,34 +1882,49 @@ def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_ # Iterate until convergence lo_prev, hi_prev = 1.0E-32, 1.0E32 for rr in range(maxiter): - # Reset the relative scaling for this iteration - relscl_model = np.ones_like(rawimg) - # Build the relative illumination, by successively finding the slits closest in wavelength to the reference - for ss in range(slits.spat_id.size): - # Calculate the region of overlap - onslit_b = (slitid_img_trim == slits.spat_id[wvsrt[ss]]) - onslit_b_init = (slitid_img_init == slits.spat_id[wvsrt[ss]]) - onslit_b_olap = onslit_b & gpm & (waveimg >= mnmx_wv[wvsrt[ss], 0]) & (waveimg <= mnmx_wv[wvsrt[ss], 1]) & skymask_now - hist, edge = np.histogram(waveimg[onslit_b_olap], bins=wavebins, weights=modelimg_copy[onslit_b_olap]) - cntr, edge = np.histogram(waveimg[onslit_b_olap], bins=wavebins) - cntr = cntr.astype(float) - cntr *= spec_ref - norm = utils.inverse(cntr) - arr = hist * norm - # Calculate a smooth version of the relative response - relscale = coadd.smooth_weights(arr, (arr != 0), sn_smooth_npix) - rescale_model = interpolate.interp1d(wave_ref, relscale, kind='linear', bounds_error=False, + # Perform two iterations: + # (ii=0) dynamically build reference spectrum + # (ii=1) used fixed reference spectrum to calculate the relative illumination. + for ii in range(2): + # Reset the relative scaling for this iteration + relscl_model = np.ones_like(rawimg) + # Build the relative illumination, by successively finding the slits closest in wavelength to the reference + for ss in range(1,slits.spat_id.size): + # Calculate the region of overlap + onslit_b = (slitid_img_trim == slits.spat_id[wvsrt[ss]]) + onslit_b_init = (slitid_img_init == slits.spat_id[wvsrt[ss]]) + onslit_b_olap = onslit_b & gpm & (waveimg >= mnmx_wv[wvsrt[ss], 0]) & (waveimg <= mnmx_wv[wvsrt[ss], 1]) & skymask_now + hist, edge = np.histogram(waveimg[onslit_b_olap], bins=wavebins, weights=modelimg_copy[onslit_b_olap]) + cntr, edge = np.histogram(waveimg[onslit_b_olap], bins=wavebins) + cntr = cntr.astype(float) + # Note, if ii=1 (i.e. the reference spectrum is fixed), we want to make sure that the + # spec_ref will give a result of 1 for all wavelengths. Apply this correction first. + if (ii == 1) and (slits.spat_id[wvsrt[ss]] == slit_illum_ref_idx): + # This must be the first element of the loop by construction, but throw an error just in case + if ss != 0: + msgs.error("An error has occurred in the relative spectral illumination. Please contact the developers.") + tmp_cntr = cntr * spec_ref + tmp_arr = hist * utils.inverse(tmp_cntr) + # Update the reference spectrum + spec_ref /= coadd.smooth_weights(tmp_arr, (tmp_arr != 0), sn_smooth_npix) + # Normalise by the reference spectrum + cntr *= spec_ref + norm = utils.inverse(cntr) + arr = hist * norm + # Calculate a smooth version of the relative response + relscale = coadd.smooth_weights(arr, (arr != 0), sn_smooth_npix) + # Store the result + relscl_model[onslit_b_init] = interpolate.interp1d(wave_ref, relscale, kind='linear', bounds_error=False, fill_value="extrapolate")(waveimg[onslit_b_init]) - # Store the result - relscl_model[onslit_b_init] = rescale_model.copy() - # Build a new reference spectrum to increase wavelength coverage of the reference spectrum (and improve S/N) - onslit_ref_trim = onslit_ref_trim | (onslit_b & gpm & skymask_now) - hist, edge = np.histogram(waveimg[onslit_ref_trim], bins=wavebins, weights=modelimg_copy[onslit_ref_trim]/relscl_model[onslit_ref_trim]) - cntr, edge = np.histogram(waveimg[onslit_ref_trim], bins=wavebins) - cntr = cntr.astype(float) - norm = utils.inverse(cntr) - spec_ref = hist * norm + # Build a new reference spectrum to increase wavelength coverage of the reference spectrum (and improve S/N) + if ii == 0: + onslit_ref_trim = onslit_ref_trim | (onslit_b & gpm & skymask_now) + hist, edge = np.histogram(waveimg[onslit_ref_trim], bins=wavebins, weights=modelimg_copy[onslit_ref_trim]/relscl_model[onslit_ref_trim]) + cntr, edge = np.histogram(waveimg[onslit_ref_trim], bins=wavebins) + cntr = cntr.astype(float) + norm = utils.inverse(cntr) + spec_ref = hist * norm minv, maxv = np.min(relscl_model), np.max(relscl_model) if 1/minv + maxv > lo_prev+hi_prev: # Adding noise, so break From e94260ff2df10e8c429060bef4d47cdc595f4dcb Mon Sep 17 00:00:00 2001 From: rcooke Date: Thu, 30 Nov 2023 12:15:50 +0000 Subject: [PATCH 074/244] rm slit check for FiberIFU --- pypeit/find_objects.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pypeit/find_objects.py b/pypeit/find_objects.py index b93704438d..be82b2f3d1 100644 --- a/pypeit/find_objects.py +++ b/pypeit/find_objects.py @@ -969,12 +969,10 @@ def find_objects_pypeline(self, image, ivar, std_trace=None, """ See MultiSlitReduce for SlicerIFU reductions """ - if self.par['reduce']['cube']['slit_spec']: - return super().find_objects_pypeline(image, ivar, std_trace=std_trace, - show_peaks=show_peaks, show_fits=show_fits, show_trace=show_trace, - show=show, save_objfindQA=save_objfindQA, neg=neg, - debug=debug, manual_extract_dict=manual_extract_dict) - return None, None, None + return super().find_objects_pypeline(image, ivar, std_trace=std_trace, + show_peaks=show_peaks, show_fits=show_fits, show_trace=show_trace, + show=show, save_objfindQA=save_objfindQA, neg=neg, + debug=debug, manual_extract_dict=manual_extract_dict) def apply_relative_scale(self, scaleImg): """Apply a relative scale to the science frame (and correct the varframe, too) From c385acf024ebe1ae65b586ecb735badb8ee0dd17 Mon Sep 17 00:00:00 2001 From: rcooke Date: Thu, 30 Nov 2023 12:17:52 +0000 Subject: [PATCH 075/244] rm findobj for SlicerIFU --- pypeit/find_objects.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/pypeit/find_objects.py b/pypeit/find_objects.py index be82b2f3d1..7d157e3cf4 100644 --- a/pypeit/find_objects.py +++ b/pypeit/find_objects.py @@ -962,18 +962,6 @@ def __init__(self, sciImg, slits, spectrograph, par, objtype, **kwargs): super().__init__(sciImg, slits, spectrograph, par, objtype, **kwargs) self.initialize_slits(slits, initial=True) - def find_objects_pypeline(self, image, ivar, std_trace=None, - show_peaks=False, show_fits=False, show_trace=False, - show=False, save_objfindQA=False, neg=False, debug=False, - manual_extract_dict=None): - """ - See MultiSlitReduce for SlicerIFU reductions - """ - return super().find_objects_pypeline(image, ivar, std_trace=std_trace, - show_peaks=show_peaks, show_fits=show_fits, show_trace=show_trace, - show=show, save_objfindQA=save_objfindQA, neg=neg, - debug=debug, manual_extract_dict=manual_extract_dict) - def apply_relative_scale(self, scaleImg): """Apply a relative scale to the science frame (and correct the varframe, too) From 1d81ab171df225ed8f5d90fdac33a5f4938ef3e8 Mon Sep 17 00:00:00 2001 From: rcooke Date: Thu, 30 Nov 2023 12:27:29 +0000 Subject: [PATCH 076/244] add debug --- pypeit/images/rawimage.py | 58 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/pypeit/images/rawimage.py b/pypeit/images/rawimage.py index 78b42d3ee2..e0903e9324 100644 --- a/pypeit/images/rawimage.py +++ b/pypeit/images/rawimage.py @@ -831,12 +831,68 @@ def flatfield(self, flatimages, slits=None, force=False, debug=False): viewer, ch = display.show_image(self.image[0], chname='orig_image') display.show_slits(viewer, ch, left, right) # , slits.id) - # Apply the relative spectral illumination + # Retrieve the relative spectral illumination profile spec_illum = flatimages.pixelflat_spec_illum if self.par['use_specillum'] else 1. # Apply flat-field correction # NOTE: Using flat.flatfield to effectively multiply image*img_scale is # a bit overkill... + if debug: + iput = self.image[0].copy() + total_flat = flatimages.pixelflat_norm * illum_flat * spec_illum + self.img_scale = np.expand_dims(utils.inverse(total_flat), 0) + oput, flat_bpm = flat.flatfield(self.image[0], total_flat) + from matplotlib import pyplot as plt + from pypeit import edgetrace, wavetilts, wavecalib + from scipy.signal import savgol_filter + + edges_file = "Calibrations/Edges_A_0_DET01.fits.gz" + tilts_file = "Calibrations/Tilts_A_0_DET01.fits" + wvcalib_file = "Calibrations/WaveCalib_A_0_DET01.fits" + flex = None + slits = edgetrace.EdgeTraceSet.from_file(edges_file).get_slits() + wvtilts = wavetilts.WaveTilts.from_file(tilts_file) + wv_calib = wavecalib.WaveCalib.from_file(wvcalib_file) + + slitmask = slits.slit_img(initial=True, flexure=flex) + tilts = wvtilts.fit2tiltimg(slitmask, flexure=flex) + waveimg = wv_calib.build_waveimg(tilts, slits, spat_flexure=flex) + + # extract a spectrum down the centre of each slit + left, right, _ = slits.select_edges(initial=True) + cen = np.round(0.5 * (left + right)).astype(int) + lcen = np.round(0.75*left + 0.25*right).astype(int) + rcen = np.round(0.25*left + 0.75*right).astype(int) + + colors = plt.cm.jet(np.linspace(0, 1, slits.nslits)) + navg = 21 + for ss in range(slits.nslits): + print("slit {0:d}".format(ss+1)) + inarr = np.zeros((iput.shape[0], navg)) + outarra = np.zeros((iput.shape[0], navg)) + outarrb = np.zeros((iput.shape[0], navg)) + outarrc = np.zeros((iput.shape[0], navg)) + for ll in range(navg): + ww = (np.arange(iput.shape[0]), cen[:, ss] + ll - navg // 2) + inarr[:, ll] = iput[ww] + outarra[:, ll] = oput[ww] + ww = (np.arange(iput.shape[0]), lcen[:, ss] + ll - navg // 2) + outarrb[:, ll] = oput[ww] + ww = (np.arange(iput.shape[0]), rcen[:, ss] + ll - navg // 2) + outarrc[:, ll] = oput[ww] + wa = (np.arange(iput.shape[0]), cen[:, ss]) + wb = (np.arange(iput.shape[0]), lcen[:, ss]) + wc = (np.arange(iput.shape[0]), rcen[:, ss]) + plt.subplot(221) + plt.plot(waveimg[wa], np.median(inarr, axis=1), color=colors[ss]) + plt.subplot(222) + plt.plot(waveimg[wa], savgol_filter(np.median(outarra, axis=1), 25, 3), color=colors[ss]) + plt.subplot(223) + plt.plot(waveimg[wb], savgol_filter(np.median(outarrb, axis=1), 25, 3), color=colors[ss]) + plt.subplot(224) + plt.plot(waveimg[wc], savgol_filter(np.median(outarrc, axis=1), 25, 3), color=colors[ss]) + # plt.plot(waveimg[ww], np.median(outarr, axis=1), color=colors[ss]) + plt.show() total_flat = flatimages.pixelflat_norm * illum_flat * spec_illum self.img_scale = np.expand_dims(utils.inverse(total_flat), 0) self.image[0], flat_bpm = flat.flatfield(self.image[0], total_flat) From 1c680ff42f11a95418f355d9436c664bd5bf70cd Mon Sep 17 00:00:00 2001 From: rcooke Date: Thu, 30 Nov 2023 20:47:34 +0000 Subject: [PATCH 077/244] add polynomial version of relative illumination --- pypeit/flatfield.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/pypeit/flatfield.py b/pypeit/flatfield.py index 43c5c54191..a0a3a4cf32 100644 --- a/pypeit/flatfield.py +++ b/pypeit/flatfield.py @@ -1805,7 +1805,7 @@ def show_flats(image_list, wcs_match=True, slits=None, waveimg=None): clear = False -def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_npix=None, +def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_npix=None, polydeg=None, model=None, gpmask=None, skymask=None, trim=3, flexure=None, maxiter=5): """ TODO :: This could possibly be moved to core.flat @@ -1825,6 +1825,9 @@ def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_ Index of slit that is used as the reference. smooth_npix : int, optional smoothing used for determining smoothly varying relative weights by sn_weights + polydeg : int, optional + Degree of polynomial to be used for determining relative spectral sensitivity. If None, + coadd.smooth_weights will be used, with the smoothing length set to smooth_npix. model : `numpy.ndarray`_, None A model of the rawimg data. If None, rawimg will be used. gpmask : `numpy.ndarray`_, None @@ -1845,6 +1848,10 @@ def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_ An image containing the appropriate scaling """ msgs.info("Performing relative spectral sensitivity correction (reference slit = {0:d})".format(slit_illum_ref_idx)) + if polydeg is not None: + msgs.info("Using polynomial of degree {0:d} for relative spectral sensitivity".format(polydeg)) + else: + msgs.info("Using 'smooth_weights' algorithm for relative spectral sensitivity") # Setup some helpful parameters skymask_now = skymask if (skymask is not None) else np.ones_like(rawimg, dtype=bool) gpm = gpmask if (gpmask is not None) else np.ones_like(rawimg, dtype=bool) @@ -1889,7 +1896,7 @@ def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_ # Reset the relative scaling for this iteration relscl_model = np.ones_like(rawimg) # Build the relative illumination, by successively finding the slits closest in wavelength to the reference - for ss in range(1,slits.spat_id.size): + for ss in range(1, slits.spat_id.size): # Calculate the region of overlap onslit_b = (slitid_img_trim == slits.spat_id[wvsrt[ss]]) onslit_b_init = (slitid_img_init == slits.spat_id[wvsrt[ss]]) @@ -1905,14 +1912,28 @@ def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_ msgs.error("An error has occurred in the relative spectral illumination. Please contact the developers.") tmp_cntr = cntr * spec_ref tmp_arr = hist * utils.inverse(tmp_cntr) + # Calculate a smooth version of the relative response + if polydeg is not None: + gd = (tmp_arr != 0) + wave_norm = (wave_ref-wave_ref[0]) / (wave_ref[1]-wave_ref[0]) + coeff = np.polyfit(wave_norm[gd], tmp_arr[gd], polydeg) + ref_relscale = np.polyval(coeff, wave_norm) + else: + ref_relscale = coadd.smooth_weights(tmp_arr, (tmp_arr != 0), sn_smooth_npix) # Update the reference spectrum - spec_ref /= coadd.smooth_weights(tmp_arr, (tmp_arr != 0), sn_smooth_npix) + spec_ref /= ref_relscale # Normalise by the reference spectrum cntr *= spec_ref norm = utils.inverse(cntr) arr = hist * norm # Calculate a smooth version of the relative response - relscale = coadd.smooth_weights(arr, (arr != 0), sn_smooth_npix) + if polydeg is not None: + gd = (arr != 0) + wave_norm = (wave_ref - wave_ref[0]) / (wave_ref[1] - wave_ref[0]) + coeff = np.polyfit(wave_norm[gd], arr[gd], polydeg) + relscale = np.polyval(coeff, wave_norm) + else: + relscale = coadd.smooth_weights(arr, (arr != 0), sn_smooth_npix) # Store the result relscl_model[onslit_b_init] = interpolate.interp1d(wave_ref, relscale, kind='linear', bounds_error=False, fill_value="extrapolate")(waveimg[onslit_b_init]) From badfeab293c59adb357ed129cbb80f25ab032f68 Mon Sep 17 00:00:00 2001 From: rcooke Date: Thu, 30 Nov 2023 20:47:47 +0000 Subject: [PATCH 078/244] add polymap --- pypeit/core/flat.py | 129 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 117 insertions(+), 12 deletions(-) diff --git a/pypeit/core/flat.py b/pypeit/core/flat.py index 6c4945c539..c88e94ad7e 100644 --- a/pypeit/core/flat.py +++ b/pypeit/core/flat.py @@ -361,12 +361,128 @@ def illum_profile_spectral_poly(rawimg, waveimg, slitmask, slitmask_trim, model, return scaleImg +def poly_map(rawimg, rawivar, waveimg, slitmask, slitmask_trim, modelimg, deg=3, + slit_illum_ref_idx=0, gpmask=None, thismask=None, debug=False): + """ + Use a polynomial fit to control points along the spectral direction to construct a map between modelimg and + rawimg. Currently, this routine is only used for image slicer IFUs. + + This problem needs to be recast into a chi-squared problem, that can take advantage of polynomial fitting. + Refer to the following for variable assignments: + https://numpy.org/doc/stable/reference/generated/numpy.polyfit.html + where: + y = science/model + w = model/science_error + resid = w*(y-f(x)) + Then, iterate over this. The reason to iterate is that there should be a mapping between science and + science_error. Once we have an estimate of f(x), we can do the following: + spl = spline(science, science_error) + new_science_error = spl(model*f(x)) + Now iterate a few times until new_science_error (and the fit) is converged. + + Parameters + ---------- + rawimg : `numpy.ndarray`_ + Image data that will be used to estimate the spectral relative sensitivity + rawivar : `numpy.ndarray`_ + Inverse variance image of rawimg + waveimg : `numpy.ndarray`_ + Wavelength image + slitmask : `numpy.ndarray`_ + A 2D int mask, the same shape as rawimg, indicating which pixels are on a slit. A -1 value + indicates not on a slit, while any pixels on a slit should have the value of the slit spatial ID + number. + slitmask_trim : `numpy.ndarray`_ + Same as slitmask, but the slit edges are trimmed. + model : `numpy.ndarray`_ + A model of the rawimg data. + slit_illum_ref_idx : :obj:`int` + Index of slit that is used as the reference. + gpmask : `numpy.ndarray`_, optional + Boolean good pixel mask (True = Good) + thismask : `numpy.ndarray`_, optional + A boolean mask (True = good) that indicates all pixels where the scaleImg should be constructed. + If None, the slitmask that is generated by this routine will be used. + debug : :obj:`bool` + If True, some plots will be output to test if the fitting is working correctly. + + Returns + ------- + modelmap : `numpy.ndarray`_ + A 2D image with the same shape as rawimg, that contains the modelimg mapped to the rawimg + relscale : `numpy.ndarray`_ + A 2D image with the same shape as rawimg, that contains the relative spectral sensitivity + """ + # Some variables to consider putting as function arguments + numiter = 4 + + # Start by calculating a ratio of the raming and the modelimg + nspec = rawimg.shape[0] + _ratio = rawimg * utils.inverse(modelimg) + _ratio_ivar = rawivar * modelimg**2 + _fit_wghts = modelimg * np.sqrt(rawivar) + + # Generate the mask + _thismask = thismask if (thismask is not None) else (slitmask > 0) + gpm = gpmask if (gpmask is not None) else np.ones_like(rawimg, dtype=bool) + # Extract the list of spatial IDs from the slitmask + slitmask_spatid = np.unique(slitmask) + slitmask_spatid = np.sort(slitmask_spatid[slitmask_spatid > 0]) + + # Create a spline between the raw data and the error + flxsrt = np.argsort(np.ravel(rawimg)) + spl = scipy.interpolate.interp1d(np.ravel(rawimg)[flxsrt], np.ravel(rawivar)[flxsrt], kind='linear', + bounds_error=False, fill_value=0.0, assume_sorted=True) + modelmap = np.ones_like(rawimg) + relscale = np.ones_like(rawimg) + msgs.info("Generating a polynomial map between the model and the raw data") + for sl, spatid in enumerate(slitmask_spatid): + # Prepare the masks, edges, and fitting variables + this_slit = (slitmask == spatid) + this_slit_trim = (slitmask_trim == spatid) + this_slit_mask = gpm & this_slit_trim + this_wave = waveimg[this_slit_mask] + wmin, wmax = np.min(this_wave), np.max(this_wave) + this_wghts = _fit_wghts[this_slit_mask] + for ii in range(numiter): + # Generate the map between model and data + coeff = np.polyfit((this_wave-wmin)/(wmax-wmin), _ratio[this_slit_mask], deg, w=this_wghts) + # Construct the mapping, and use this to make a model of the rawdata + this_modmap = np.polyval(coeff, (this_wave-wmin)/(wmax-wmin)) + this_modflx = modelimg[this_slit_mask] * this_modmap + # Update the fit weights + this_wghts = modelimg[this_slit_mask] * np.sqrt(spl(this_modflx)) + # Produce the final model for this slit + modelmap[this_slit] *= np.polyval(coeff, (waveimg[this_slit]-wmin)/(wmax-wmin)) + relscale[this_slit] *= np.polyval(coeff, (waveimg[this_slit]-wmin)/(wmax-wmin)) + # Check if this is the reference slit, and if so, set the scale relative to the reference slit. + if sl == slit_illum_ref_idx: + for slidx, spatid in enumerate(slitmask_spatid): + # Get the slit pixels + norm_slit = (slitmask == spatid) + relscale[norm_slit] /= np.polyval(coeff, (waveimg[norm_slit]-wmin)/(wmax-wmin)) + # Return the modelmap and the relative scale + return modelmap, relscale + + def spline_map(rawimg, rawivar, waveimg, slitmask, slitmask_trim, modelimg, slit_illum_ref_idx=0, gpmask=None, thismask=None, nbins=20, debug=False): """ Use a spline fit to control points along the spectral direction to construct a map between modelimg and rawimg. Currently, this routine is only used for image slicer IFUs. + This problem needs to be recast into a smoothing problem, that can take advantage of the following: + https://docs.scipy.org/doc/scipy/reference/generated/scipy.interpolate.UnivariateSpline.html#scipy.interpolate.UnivariateSpline + where: + y = science/model + w = model/science_error + resid = w*(y-f(x)) + Then, iterate over this. The reason to iterate is that there should be a mapping between science and + science_error. Once we have an estimate of f(x), we can do the following: + spl = spline(science, science_error) + new_science_error = spl(model*f(x)) + Now iterate a few times until new_science_error (and the fit) is converged. + Parameters ---------- rawimg : `numpy.ndarray`_ @@ -400,18 +516,6 @@ def spline_map(rawimg, rawivar, waveimg, slitmask, slitmask_trim, modelimg, scale_model: `numpy.ndarray`_ An image containing the appropriate scaling """ - # TODO :: REMOVE THIS... IT IS USED AS A GUIDE TO DEVELOP THE METHOD!!! - # This problem needs to be recast into a smoothing problem, that can take advantage of the following: - # https://docs.scipy.org/doc/scipy/reference/generated/scipy.interpolate.UnivariateSpline.html#scipy.interpolate.UnivariateSpline - # where: - # y = science/model - # w = model/science_error - # resid = w*(y-f(x)) - # Then, iterate over this. The reason to iterate is that there should be a mapping between science and science_error. Once we have an estimate of f(x), we can do the following: - # spl = spline(science, science_error) - # new_science_error = spl(model*f(x)) - # Now iterate a few times until new_science_error (and the fit) is converged. - # Some variables to consider putting as function arguments numiter = 4 @@ -429,6 +533,7 @@ def spline_map(rawimg, rawivar, waveimg, slitmask, slitmask_trim, modelimg, slitmask_spatid = np.sort(slitmask_spatid[slitmask_spatid > 0]) # Create a spline between the raw data and the error + # TODO :: This is slow and inefficient. embed() flxsrt = np.argsort(np.ravel(rawimg)) spl = scipy.interpolate.interp1d(np.ravel(rawimg)[flxsrt], np.ravel(rawivar)[flxsrt], kind='linear', From b0fa92d1f379209111296ba8c460169f26532fa9 Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 1 Dec 2023 11:38:44 +0000 Subject: [PATCH 079/244] core call --- pypeit/coadd3d.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index 30ccb3f245..58ecf053e6 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -1332,6 +1332,7 @@ def run(self): dspat=self._dspat, dwv=self._dwv) # Align the frames + # TODO :: NOTE THAT THIS IS 'and False' because the HST alignment is done when reading in the data. if self.align and False: self.all_ra, self.all_dec = self.run_align() @@ -1408,6 +1409,7 @@ def run(self): sensfunc=sensfunc, fluxed=self.fluxcal) final_cube.to_file(outfile, hdr=hdr, overwrite=self.overwrite) + def hst_alignment(ra_sort, dec_sort, wave_sort, flux_sort, ivar_sort, dspat, dwave, wghts, spatpos, specpos, all_spatid, tilts, slits, astrom_trans, all_dar, @@ -1438,7 +1440,7 @@ def hst_alignment(ra_sort, dec_sort, wave_sort, flux_sort, ivar_sort, dspat, dwa subcube_wcs, voxedge, reference_image = datacube.create_wcs(ra_sort[wv_mask], dec_sort[wv_mask], wave_sort[wv_mask], dspat, dwave) # Create the subcube - flxcube, varcube, bpmcube = subpixellate(subcube_wcs, ra_sort[wv_mask], dec_sort[wv_mask], wave_sort[wv_mask], flux_sort[wv_mask], ivar_sort[wv_mask], + flxcube, varcube, bpmcube = datacube.subpixellate(subcube_wcs, ra_sort[wv_mask], dec_sort[wv_mask], wave_sort[wv_mask], flux_sort[wv_mask], ivar_sort[wv_mask], wghts[wv_mask], spatpos[wv_mask], specpos[wv_mask], all_spatid[wv_mask], tilts, slits, astrom_trans, all_dar, voxedge, all_idx=None, From 4448fe546ae919c605ef627a8798c822c8dbf4ae Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 8 Dec 2023 13:11:42 +0000 Subject: [PATCH 080/244] add alignment debugging --- pypeit/coadd3d.py | 118 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 88 insertions(+), 30 deletions(-) diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index 58ecf053e6..45aebcf204 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -1125,9 +1125,21 @@ def load(self): ################################## # Astrometric alignment to HST frames # TODO :: RJC requests this remains here... it is only used by RJC + # Get the coordinate bounds + slitlength = int(np.round(np.median(slits.get_slitlengths(initial=True, median=True)))) + numwav = int((np.max(waveimg) - wave0) / dwv) + raw_bins = self.spec.get_datacube_bins(slitlength, minmax, numwav) + # Generate the output WCS for the datacube + tmp_crval_wv = (self.all_wcs[ff].wcs.crval[2] * self.all_wcs[ff].wcs.cunit[2]).to(units.Angstrom).value + tmp_cd_wv = (self.all_wcs[ff].wcs.cd[2, 2] * self.all_wcs[ff].wcs.cunit[2]).to(units.Angstrom).value + crval_wv = self.cubepar['wave_min'] if self.cubepar['wave_min'] is not None else tmp_crval_wv + cd_wv = self.cubepar['wave_delta'] if self.cubepar['wave_delta'] is not None else tmp_cd_wv + raw_wcs = self.spec.get_wcs(spec2DObj.head0, slits, detector.platescale, crval_wv, cd_wv) + # Now do the alignment _ra_sort, _dec_sort = hst_alignment(ra_sort, dec_sort, wave_sort, flux_sort, ivar_sort, np.max(self._spatscale[ff,:]), self._specscale[ff], - np.ones(ra_sort.size), this_spatpos[wvsrt], this_specpos[wvsrt], - this_spatid[wvsrt], spec2DObj.tilts, slits, alignSplines, darcorr) + np.ones(ra_sort.size), this_spatpos[wvsrt], this_specpos[wvsrt], + this_spatid[wvsrt], spec2DObj.tilts, slits, alignSplines, darcorr, + raw_wcs=raw_wcs, raw_bins=raw_bins) ra_del = np.median(_ra_sort-ra_sort) dec_del = np.median(_dec_sort-dec_sort) ra_sort = _ra_sort @@ -1413,7 +1425,7 @@ def run(self): def hst_alignment(ra_sort, dec_sort, wave_sort, flux_sort, ivar_sort, dspat, dwave, wghts, spatpos, specpos, all_spatid, tilts, slits, astrom_trans, all_dar, - spat_subpixel=10, spec_subpixel=10): + spat_subpixel=10, spec_subpixel=10, raw_wcs=None, raw_bins=None): """ This is currently only used by RJC. This function adds corrections to the RA and Dec pixels to align the daatcubes to an HST image. @@ -1431,32 +1443,78 @@ def hst_alignment(ra_sort, dec_sort, wave_sort, flux_sort, ivar_sort, dspat, dwa * Return updated pixel RA, Dec """ from pypeit import astrometry - ############ - ## STEP 1 ## - Create a datacube around Hgamma - ############ - # Only use a small wavelength range - wv_mask = (wave_sort > 4346.0) & (wave_sort < 4358.0) - # Create a WCS for this subcube - subcube_wcs, voxedge, reference_image = datacube.create_wcs(ra_sort[wv_mask], dec_sort[wv_mask], wave_sort[wv_mask], - dspat, dwave) - # Create the subcube - flxcube, varcube, bpmcube = datacube.subpixellate(subcube_wcs, ra_sort[wv_mask], dec_sort[wv_mask], wave_sort[wv_mask], flux_sort[wv_mask], ivar_sort[wv_mask], - wghts[wv_mask], spatpos[wv_mask], specpos[wv_mask], - all_spatid[wv_mask], tilts, slits, astrom_trans, all_dar, - voxedge, all_idx=None, - spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, debug=False) - if False: - hdu = fits.PrimaryHDU(flxcube) - hdu.writeto("tstHg.fits", overwrite=True) - - ############ - ## STEP 2 ## - Create an emission line map of Hgamma - ############ - # Compute an emission line map that is as consistent as possible to an archival HST image - HgMap, HgMapErr = astrometry.fit_cube(flxcube.T, varcube.T, subcube_wcs) - ############ - ## STEP 3 ## - Map the emission line map to an HST image, and vice-versa - ############ - ra_corr, dec_corr = astrometry.map_image(HgMap, HgMapErr, subcube_wcs, ra_sort, dec_sort) + niter = 3 + for ii in range(niter): + ############ + ## STEP 1 ## - Create a datacube around Hgamma + ############ + # Only use a small wavelength range + msgs.warn("TEMPORARY HACK: turning off subpixel spectrum until DAR correction is done properly") + if False: + wv_mask = (wave_sort > 4346.0) & (wave_sort < 4358.0) + # Create a WCS for this subcube + subcube_wcs, voxedge, reference_image = datacube.create_wcs(ra_sort[wv_mask], dec_sort[wv_mask], wave_sort[wv_mask], + dspat, dwave) + # Create the subcube + flxcube, varcube, bpmcube = datacube.subpixellate(subcube_wcs, ra_sort[wv_mask], dec_sort[wv_mask], wave_sort[wv_mask], flux_sort[wv_mask], ivar_sort[wv_mask], + wghts[wv_mask], spatpos[wv_mask], specpos[wv_mask], + all_spatid[wv_mask], tilts, slits, astrom_trans, all_dar, + voxedge, all_idx=None, + spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, debug=False) + if False: + hdu = fits.PrimaryHDU(flxcube) + hdu.writeto("tstHg.fits", overwrite=True) + + ############ + ## STEP 2 ## - Create an emission line map of Hgamma + ############ + print("TEMPORARY HACK: turning off HST alignment until DAR correction is done properly") + # Compute an emission line map that is as consistent as possible to an archival HST image + #HgMap, HgMapErr = astrometry.fit_cube(flxcube.T, varcube.T, subcube_wcs) + + ############ + ## STEP 3A ## - Generate another datacube using the raw_wcs and make an image of the emission line map + # Need to generate a datacube near both Hgamma and Hdelta. + ############ + #Is DAR correction being performed here? + # FIRST DO Hdelta + wv_mask = (wave_sort > 4107.0) & (wave_sort < 4119.0) + flxcube, sigcube, bpmcube, wave = \ + datacube.generate_cube_subpixel("tmpfile.fits", raw_wcs, ra_sort[wv_mask], dec_sort[wv_mask], wave_sort[wv_mask], + flux_sort[wv_mask], ivar_sort[wv_mask], wghts[wv_mask], spatpos[wv_mask], specpos[wv_mask], + all_spatid[wv_mask], tilts, slits, astrom_trans, all_dar, + raw_bins, all_idx=None, overwrite=False, + spec_subpixel=5, spat_subpixel=5) + HdMap_raw, HdMapErr_raw = astrometry.fit_cube(flxcube, sigcube**2, raw_wcs, line="HIdelta") + # THEN DO Hgamma + wv_mask = (wave_sort > 4346.0) & (wave_sort < 4358.0) + flxcube, sigcube, bpmcube, wave = \ + datacube.generate_cube_subpixel("tmpfile.fits", raw_wcs, ra_sort[wv_mask], dec_sort[wv_mask], wave_sort[wv_mask], + flux_sort[wv_mask], ivar_sort[wv_mask], wghts[wv_mask], spatpos[wv_mask], specpos[wv_mask], + all_spatid[wv_mask], tilts, slits, astrom_trans, all_dar, + raw_bins, all_idx=None, overwrite=False, + spec_subpixel=5, spat_subpixel=5) + HgMap_raw, HgMapErr_raw = astrometry.fit_cube(flxcube, sigcube**2, raw_wcs, line="HIgamma") + # Plot the emission line map + from matplotlib import pyplot as plt + plt.subplot(131) + plt.imshow(HdMap_raw, vmin=0, vmax=200, aspect=0.5) + plt.subplot(132) + plt.imshow(HgMap_raw, vmin=0, vmax=300, aspect=0.5) + plt.subplot(133) + plt.imshow(HdMap_raw*utils.inverse(HgMap_raw), vmin=0.5, vmax=0.7, aspect=0.5) + plt.show() + embed() + ############ + ## STEP 3B ## - Map the emission line map to an HST image, and vice-versa + ############ + ra_corr, dec_corr, flx_corr = astrometry.map_image(HgMap, HgMapErr, subcube_wcs, ra_sort, dec_sort, raw_wcs, HgMap_raw, HgMapErr_raw, HdMap_raw, HdMapErr_raw) + # Loop over the slits and apply the corrections + slit_illum_ref_idx = 14 + for ss, spatid in enumerate(slits.spat_id): + idx = np.where(all_spatid == spatid)[0] + # Calculate the correction as a function of wavelength + + # embed() return ra_corr, dec_corr \ No newline at end of file From 39d57032a44434d146dfa5acd222bf498c082876 Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 8 Dec 2023 13:29:34 +0000 Subject: [PATCH 081/244] add slice_subpixel --- pypeit/par/pypeitpar.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index ac9c8251d4..409f719c98 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -1489,7 +1489,7 @@ def __init__(self, slit_spec=None, relative_weights=None, align=None, combine=No standard_cube=None, reference_image=None, save_whitelight=None, whitelight_range=None, method=None, ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None, spatial_delta=None, wave_delta=None, astrometric=None, grating_corr=None, scale_corr=None, - skysub_frame=None, spec_subpixel=None, spat_subpixel=None): + skysub_frame=None, spec_subpixel=None, spat_subpixel=None, slice_subpixel=None): # Grab the parameter names and values from the function # arguments @@ -1597,7 +1597,7 @@ def __init__(self, slit_spec=None, relative_weights=None, align=None, combine=No 'each detector pixel in the spectral direction. The total number of subpixels ' \ 'in each pixel is given by spec_subpixel x spat_subpixel. The default option ' \ 'is to divide each spec2d pixel into 25 subpixels during datacube creation. ' \ - 'See also, spat_subpixel.' + 'See also, spat_subpixel and slice_subpixel.' defaults['spat_subpixel'] = 5 dtypes['spat_subpixel'] = int @@ -1605,7 +1605,14 @@ def __init__(self, slit_spec=None, relative_weights=None, align=None, combine=No 'each detector pixel in the spatial direction. The total number of subpixels ' \ 'in each pixel is given by spec_subpixel x spat_subpixel. The default option ' \ 'is to divide each spec2d pixel into 25 subpixels during datacube creation. ' \ - 'See also, spec_subpixel.' + 'See also, spec_subpixel and slice_subpixel.' + + # TODO :: Change this default value to 5 + defaults['slice_subpixel'] = 1 + dtypes['slice_subpixel'] = int + descr['slice_subpixel'] = 'When method=subpixel, slice_subpixel sets the subpixellation scale of ' \ + 'each IFU slice. The default option is to divide each slice into 5 sub-slices ' \ + 'during datacube creation. See also, spec_subpixel and spat_subpixel.' defaults['ra_min'] = None dtypes['ra_min'] = float @@ -1693,7 +1700,7 @@ def from_dict(cls, cfg): # Basic keywords parkeys = ['slit_spec', 'output_filename', 'standard_cube', 'reference_image', 'save_whitelight', - 'method', 'spec_subpixel', 'spat_subpixel', 'ra_min', 'ra_max', 'dec_min', 'dec_max', + 'method', 'spec_subpixel', 'spat_subpixel', 'slice_subpixel', 'ra_min', 'ra_max', 'dec_min', 'dec_max', 'wave_min', 'wave_max', 'spatial_delta', 'wave_delta', 'relative_weights', 'align', 'combine', 'astrometric', 'grating_corr', 'scale_corr', 'skysub_frame', 'whitelight_range'] From a66b9bc0797be89d6519c623d85069a05d7d3bb3 Mon Sep 17 00:00:00 2001 From: KostasValeckas Date: Fri, 8 Dec 2023 16:58:06 +0100 Subject: [PATCH 082/244] ready for PR --- pypeit/edgetrace.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pypeit/edgetrace.py b/pypeit/edgetrace.py index 684a13d5d8..fae953194b 100644 --- a/pypeit/edgetrace.py +++ b/pypeit/edgetrace.py @@ -5507,8 +5507,15 @@ def match_order(self): for u in uniq[cnts > 1]: # Find the unmasked and multiply-matched indices indx = (slit_indx.data == u) & np.logical_not(np.ma.getmaskarray(slit_indx)) + + # we need a masked version of min_sep array with only relevant indices + # as the below masking of slit_indx needs to be in correct shape + min_sep_mask = np.ones_like(min_sep, dtype = bool) + min_sep_mask[indx] = False + min_sep_masked = np.ma.masked_array(min_sep, min_sep_mask) + # Keep the one with the smallest separation and mask the rest - slit_indx[np.setdiff1d(np.where(indx), [np.argmin(min_sep[indx])])] = np.ma.masked + slit_indx[np.setdiff1d(np.where(indx), [np.argmin(min_sep_masked)])] = np.ma.masked # Flag orders separated by more than the provided threshold if self.par['order_match'] is not None: From 4c0d9fcd52533cc11bd244f470ed1460e632f522 Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 8 Dec 2023 20:57:59 +0000 Subject: [PATCH 083/244] add slice_subpixel --- pypeit/slittrace.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/pypeit/slittrace.py b/pypeit/slittrace.py index 5554ac753b..df38f304f5 100644 --- a/pypeit/slittrace.py +++ b/pypeit/slittrace.py @@ -416,7 +416,7 @@ def get_slitlengths(self, initial=False, median=False): slitlen = np.median(slitlen, axis=1) return slitlen - def get_radec_image(self, wcs, alignSplines, tilts, initial=True, flexure=None): + def get_radec_image(self, wcs, alignSplines, tilts, slice_offset=None, initial=True, flexure=None): """Generate an RA and DEC image for every pixel in the frame NOTE: This function is currently only used for SlicerIFU reductions. @@ -429,6 +429,11 @@ def get_radec_image(self, wcs, alignSplines, tilts, initial=True, flexure=None): transform between detector pixel coordinates and WCS pixel coordinates. tilts : `numpy.ndarray`_ Spectral tilts. + slice_offset : float, optional + Offset to apply to the slice positions. A value of 0.0 means that the + slice positions are the centre of the slits. A value of +/-0.5 means that + the slice positions are at the edges of the slits. If None, the slice_offset + is set to 0.0. initial : bool Select the initial slit edges? flexure : float, optional @@ -447,7 +452,13 @@ def get_radec_image(self, wcs, alignSplines, tilts, initial=True, flexure=None): reference (usually the centre of the slit) and the edges of the slits. Shape is (nslits, 2). """ - msgs.info("Generating an RA/DEC image") + substring = '' if slice_offset is None else f' with slice_offset={slice_offset}' + msgs.info("Generating an RA/DEC image"+substring) + # Check the input + if slice_offset is None: + slice_offset = 0.0 + if slice_offset < -0.5 or slice_offset > 0.5: + msgs.error(f"Slice offset must be between -0.5 and 0.5. slice_offset={slice_offset}") # Initialise the output raimg = np.zeros((self.nspec, self.nspat)) decimg = np.zeros((self.nspec, self.nspat)) @@ -464,7 +475,7 @@ def get_radec_image(self, wcs, alignSplines, tilts, initial=True, flexure=None): minmax[slit_idx, 0] = np.min(evalpos) minmax[slit_idx, 1] = np.max(evalpos) # Calculate the WCS from the pixel positions - slitID = np.ones(evalpos.size) * slit_idx - wcs.wcs.crpix[0] + slitID = np.ones(evalpos.size) * slit_idx + slice_offset - wcs.wcs.crpix[0] world_ra, world_dec, _ = wcs.wcs_pix2world(slitID, evalpos, tilts[onslit_init]*(self.nspec-1), 0) # Set the RA first and DEC next raimg[onslit] = world_ra.copy() From e159b28f6eb38953887896bc229b7a54ce989aca Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 8 Dec 2023 21:43:54 +0000 Subject: [PATCH 084/244] add slice_subpixel --- pypeit/core/datacube.py | 333 +++++++++++++++++++--------------------- 1 file changed, 162 insertions(+), 171 deletions(-) diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index 475db4f93a..1fc5de4440 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -1190,10 +1190,11 @@ def generate_image_subpixel(image_wcs, all_ra, all_dec, all_wave, all_sci, all_i return all_wl_imgs -def generate_cube_subpixel(outfile, output_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_wghts, - all_spatpos, all_specpos, all_spatid, tilts, slits, astrom_trans, all_dar, bins, - all_idx=None, spec_subpixel=10, spat_subpixel=10, overwrite=False, - whitelight_range=None, debug=False): +def generate_cube_subpixel(outfile, output_wcs, bins, + sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, + all_wcs, tilts, slits, astrom_trans, all_dar, + spec_subpixel=5, spat_subpixel=5, slice_subpixel=5, + overwrite=False, whitelight_range=None, debug=False): """ Save a datacube using the subpixel algorithm. Refer to the subpixellate() docstring for further details about this algorithm @@ -1203,37 +1204,30 @@ def generate_cube_subpixel(outfile, output_wcs, all_ra, all_dec, all_wave, all_s Filename to be used to save the datacube output_wcs (`astropy.wcs.WCS`_): Output world coordinate system. - all_ra (`numpy.ndarray`_): - 1D flattened array containing the right ascension of each pixel - (units = degrees) - all_dec (`numpy.ndarray`_): - 1D flattened array containing the declination of each pixel (units = - degrees) - all_wave (`numpy.ndarray`_): - 1D flattened array containing the wavelength of each pixel (units = - Angstroms) - all_sci (`numpy.ndarray`_): - 1D flattened array containing the counts of each pixel from all - spec2d files - all_ivar (`numpy.ndarray`_): - 1D flattened array containing the inverse variance of each pixel - from all spec2d files - all_wghts (`numpy.ndarray`_): - 1D flattened array containing the weights of each pixel to be used - in the combination - all_spatpos (`numpy.ndarray`_): - 1D flattened array containing the detector pixel location in the - spatial direction - all_specpos (`numpy.ndarray`_): - 1D flattened array containing the detector pixel location in the - spectral direction - all_spatid (`numpy.ndarray`_): - 1D flattened array containing the spatid of each pixel - tilts (`numpy.ndarray`_, list): - 2D wavelength tilts frame, or a list of tilt frames (see all_idx) - slits (:class:`~pypeit.slittrace.SlitTraceSet`, list): - Information stored about the slits, or a list of SlitTraceSet (see - all_idx) + bins (tuple): + A 3-tuple (x,y,z) containing the histogram bin edges in x,y spatial + and z wavelength coordinates + sciImg (`numpy.ndarray`_, list): + A list of 2D array containing the counts of each pixel + ivarImg (`numpy.ndarray`_, list): + A list of 2D array containing the inverse variance of each pixel + waveImg (`numpy.ndarray`_, list): + A list of 2D array containing the wavelength of each pixel + slitid_img_gpm (`numpy.ndarray`_, list): + A list of 2D array containing the slitmask of each pixel. + A zero value indicates that a pixel is either not on a slit or it is a bad pixel. + All other values are the slit spatial ID number. + wghtImg (`numpy.ndarray`_, list): + A list of 2D array containing the weights of each pixel to be used in the + combination + all_wcs (`astropy.wcs.WCS`_, list): + A list of `astropy.wcs.WCS`_ objects, one for each spec2d file + tilts (list): + A list of `numpy.ndarray`_ objects, one for each spec2d file, + containing the tilts of each pixel + slits (:class:`pypeit.slittrace.SlitTraceSet`, list): + A list of `pypeit.slittrace.SlitTraceSet`_ objects, one for each + spec2d file, containing the properties of the slit for each spec2d file astrom_trans (:class:`~pypeit.alignframe.AlignmentSplines`, list): A Class containing the transformation between detector pixel coordinates and WCS pixel coordinates, or a list of Alignment @@ -1241,15 +1235,6 @@ def generate_cube_subpixel(outfile, output_wcs, all_ra, all_dec, all_wave, all_s all_dar (:class:`~pypeit.coadd3d.DARcorrection`, list): A Class containing the DAR correction information, or a list of DARcorrection classes. If a list, it must be the same length as astrom_trans. - bins (tuple): - A 3-tuple (x,y,z) containing the histogram bin edges in x,y spatial - and z wavelength coordinates - all_idx (`numpy.ndarray`_, optional): - If tilts, slits, and astrom_trans are lists, this should contain a - 1D flattened array, of the same length as all_sci, containing the - index the tilts, slits, and astrom_trans lists that corresponds to - each pixel. Note that, in this case all of these lists need to be - the same length. spec_subpixel (int, optional): What is the subpixellation factor in the spectral direction. Higher values give more reliable results, but note that the time required @@ -1262,6 +1247,11 @@ def generate_cube_subpixel(outfile, output_wcs, all_ra, all_dec, all_wave, all_s goes as (``spec_subpixel * spat_subpixel``). The default value is 5, which divides each detector pixel into 5 subpixels in the spatial direction. + slice_subpixel (int, optional): + What is the subpixellation factor in the slice direction. Higher + values give more reliable results, but note that the time required + goes as (``slice_subpixel``). The default value is 5, which divides + each IFU slice into 5 subslices in the slice direction. overwrite (bool, optional): If True, the output cube will be overwritten. whitelight_range (None, list, optional): @@ -1288,9 +1278,10 @@ def generate_cube_subpixel(outfile, output_wcs, all_ra, all_dec, all_wave, all_s hdr = output_wcs.to_header() # Subpixellate - subpix = subpixellate(output_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_wghts, all_spatpos, all_specpos, - all_spatid, tilts, slits, astrom_trans, all_dar, bins, all_idx=all_idx, - spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, debug=debug) + subpix = subpixellate(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, + all_wcs, tilts, slits, astrom_trans, all_dar, + spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, slice_subpixel=slice_subpixel, + debug=debug) # Extract the variables that we need if debug: flxcube, varcube, bpmcube, residcube = subpix @@ -1313,9 +1304,9 @@ def generate_cube_subpixel(outfile, output_wcs, all_ra, all_dec, all_wave, all_s whitelight_wcs = output_wcs.celestial # Determine the wavelength range of the whitelight image if whitelight_range[0] is None: - whitelight_range[0] = np.min(all_wave) + whitelight_range[0] = wave[0] if whitelight_range[1] is None: - whitelight_range[1] = np.max(all_wave) + whitelight_range[1] = wave[-1] msgs.info("White light image covers the wavelength range {0:.2f} A - {1:.2f} A".format( whitelight_range[0], whitelight_range[1])) # Get the output filename for the white light image @@ -1328,56 +1319,50 @@ def generate_cube_subpixel(outfile, output_wcs, all_ra, all_dec, all_wave, all_s return flxcube.T, np.sqrt(varcube.T), bpmcube.T, wave -def subpixellate(output_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_wghts, all_spatpos, all_specpos, - all_spatid, tilts, slits, astrom_trans, all_dar, bins, all_idx=None, - spec_subpixel=10, spat_subpixel=10, debug=False): +def subpixellate(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, + all_wcs, tilts, slits, astrom_trans, all_dar, + spec_subpixel=5, spat_subpixel=5, slice_subpixel=5, debug=False): r""" Subpixellate the input data into a datacube. This algorithm splits each - detector pixel into multiple subpixels, and then assigns each subpixel to a - voxel. For example, if ``spec_subpixel = spat_subpixel = 10``, then each - detector pixel is divided into :math:`10^2=100` subpixels. Alternatively, - when spec_subpixel = spat_subpixel = 1, this corresponds to the nearest grid + detector pixel into multiple subpixels and each IFU slice into multiple subslices. + Then, the algorithm assigns each subdivided detector pixel to a + voxel. For example, if ``spec_subpixel = spat_subpixel = slice_subpixel = 5``, then each + detector pixel is divided into :math:`5^3=125` subpixels. Alternatively, + when spec_subpixel = spat_subpixel = slice_subpixel = 1, this corresponds to the nearest grid point (NGP) algorithm. - Important Note: If spec_subpixel > 1 or spat_subpixel > 1, the errors will - be correlated, and the covariance is not being tracked, so the errors will - not be (quite) right. There is a tradeoff one has to make between sampling - and better looking cubes, versus no sampling and better behaved errors. + Important Note: If spec_subpixel > 1 or spat_subpixel > 1 or slice_subpixel > 1, + the errors will be correlated, and the covariance is not being tracked, so the + errors will not be (quite) right. There is a tradeoff one has to make between + sampling and better looking cubes, versus no sampling and better behaved errors. Args: output_wcs (`astropy.wcs.WCS`_): Output world coordinate system. - all_ra (`numpy.ndarray`_): - 1D flattened array containing the right ascension of each pixel - (units = degrees) - all_dec (`numpy.ndarray`_): - 1D flattened array containing the declination of each pixel (units = - degrees) - all_wave (`numpy.ndarray`_): - 1D flattened array containing the wavelength of each pixel (units = - Angstroms) - all_sci (`numpy.ndarray`_): - 1D flattened array containing the counts of each pixel from all - spec2d files - all_ivar (`numpy.ndarray`_): - 1D flattened array containing the inverse variance of each pixel - from all spec2d files - all_wghts (`numpy.ndarray`_): - 1D flattened array containing the weights of each pixel to be used - in the combination - all_spatpos (`numpy.ndarray`_): - 1D flattened array containing the detector pixel location in the - spatial direction - all_specpos (`numpy.ndarray`_): - 1D flattened array containing the detector pixel location in the - spectral direction - all_spatid (`numpy.ndarray`_): - 1D flattened array containing the spatid of each pixel - tilts (`numpy.ndarray`_, list): - 2D wavelength tilts frame, or a list of tilt frames (see all_idx) - slits (:class:`~pypeit.slittrace.SlitTraceSet`, list): - Information stored about the slits, or a list of SlitTraceSet (see - all_idx) + bins (tuple): + A 3-tuple (x,y,z) containing the histogram bin edges in x,y spatial + and z wavelength coordinates + sciImg (`numpy.ndarray`_, list): + A list of 2D array containing the counts of each pixel + ivarImg (`numpy.ndarray`_, list): + A list of 2D array containing the inverse variance of each pixel + waveImg (`numpy.ndarray`_, list): + A list of 2D array containing the wavelength of each pixel + slitid_img_gpm (`numpy.ndarray`_, list): + A list of 2D array containing the slitmask of each pixel. + A zero value indicates that a pixel is either not on a slit or it is a bad pixel. + All other values are the slit spatial ID number. + wghtImg (`numpy.ndarray`_, list): + A list of 2D array containing the weights of each pixel to be used in the + combination + all_wcs (`astropy.wcs.WCS`_, list): + A list of `astropy.wcs.WCS`_ objects, one for each spec2d file + tilts (list): + A list of `numpy.ndarray`_ objects, one for each spec2d file, + containing the tilts of each pixel + slits (:class:`pypeit.slittrace.SlitTraceSet`, list): + A list of `pypeit.slittrace.SlitTraceSet`_ objects, one for each + spec2d file, containing the properties of the slit for each spec2d file astrom_trans (:class:`~pypeit.alignframe.AlignmentSplines`, list): A Class containing the transformation between detector pixel coordinates and WCS pixel coordinates, or a list of Alignment @@ -1385,28 +1370,24 @@ def subpixellate(output_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_w all_dar (:class:`~pypeit.coadd3d.DARcorrection`, list): A Class containing the DAR correction information, or a list of DARcorrection classes. If a list, it must be the same length as astrom_trans. - bins (tuple): - A 3-tuple (x,y,z) containing the histogram bin edges in x,y spatial - and z wavelength coordinates - all_idx (`numpy.ndarray`_, optional): - If tilts, slits, and astrom_trans are lists, this should contain a - 1D flattened array, of the same length as all_sci, containing the - index the tilts, slits, and astrom_trans lists that corresponds to - each pixel. Note that, in this case all of these lists need to be - the same length. - spec_subpixel (:obj:`int`, optional): + spec_subpixel (int, optional): What is the subpixellation factor in the spectral direction. Higher values give more reliable results, but note that the time required goes as (``spec_subpixel * spat_subpixel``). The default value is 5, which divides each detector pixel into 5 subpixels in the spectral direction. - spat_subpixel (:obj:`int`, optional): + spat_subpixel (int, optional): What is the subpixellation factor in the spatial direction. Higher values give more reliable results, but note that the time required goes as (``spec_subpixel * spat_subpixel``). The default value is 5, which divides each detector pixel into 5 subpixels in the spatial direction. - debug (bool): + slice_subpixel (int, optional): + What is the subpixellation factor in the slice direction. Higher + values give more reliable results, but note that the time required + goes as (``slice_subpixel``). The default value is 5, which divides + each IFU slice into 5 subslices in the slice direction. + debug (bool, optional): If True, a residuals cube will be output. If the datacube generation is correct, the distribution of pixels in the residual cube with no flux should have mean=0 and std=1. @@ -1418,32 +1399,22 @@ def subpixellate(output_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_w residual cube. The latter is only returned if debug is True. """ # Check for combinations of lists or not - if all([isinstance(l, list) for l in [tilts, slits, astrom_trans, all_dar]]): + list_inputs = [sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, all_wcs, tilts, slits, astrom_trans, all_dar] + if all([isinstance(l, list) for l in list_inputs]): # Several frames are being combined. Check the lists have the same length - numframes = len(tilts) - if len(slits) != numframes or len(astrom_trans) != numframes or len(all_dar) != numframes: - msgs.error("The following lists must have the same length:" + msgs.newline() + - "tilts, slits, astrom_trans, all_dar") - # Check all_idx has been set - if all_idx is None: - if numframes != 1: - msgs.error("Missing required argument for combining frames: all_idx") - else: - all_idx = np.zeros(all_sci.size) - else: - tmp = np.unique(all_idx).size - if tmp != numframes: - msgs.warn("Indices in argument 'all_idx' does not match the number of frames expected.") + numframes = len(sciImg) + if not all([len(l) == numframes for l in list_inputs]): + msgs.error("All input lists must have the same length when combining multiple frames") # Store in the following variables - _tilts, _slits, _astrom_trans, _all_dar = tilts, slits, astrom_trans, all_dar - elif all([not isinstance(l, list) for l in [tilts, slits, astrom_trans, all_dar]]): + _sciImg, _ivarImg, _waveImg, _gpmImg, _wghtImg, _all_wcs, _tilts, _slits, _astrom_trans, _all_dar = \ + sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, all_wcs, tilts, slits, astrom_trans, all_dar + elif all([not isinstance(l, list) for l in list_inputs]): # Just a single frame - store as lists for this code - _tilts, _slits, _astrom_trans, _all_dar = [tilts], [slits], [astrom_trans], [all_dar] - all_idx = np.zeros(all_sci.size) + _sciImg, _ivarImg, _waveImg, _gpmImg, _wghtImg, _all_wcs, _tilts, _slits, _astrom_trans, _all_dar = \ + [sciImg], [ivarImg], [waveImg], [slitid_img_gpm], [wghtImg], [all_wcs], [tilts], [slits], [astrom_trans], [all_dar] numframes = 1 else: - msgs.error("The following input arguments should all be of type 'list', or all not be type 'list':" + - msgs.newline() + "tilts, slits, astrom_trans, all_dar") + msgs.error("The input arguments should all be of type 'list', or all not be of type 'list':") # Prepare the output arrays outshape = (bins[0].size-1, bins[1].size-1, bins[2].size-1) binrng = [[bins[0][0], bins[0][-1]], [bins[1][0], bins[1][-1]], [bins[2][0], bins[2][-1]]] @@ -1453,59 +1424,79 @@ def subpixellate(output_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_w # Divide each pixel into subpixels spec_offs = np.arange(0.5/spec_subpixel, 1, 1/spec_subpixel) - 0.5 # -0.5 is to offset from the centre of each pixel. spat_offs = np.arange(0.5/spat_subpixel, 1, 1/spat_subpixel) - 0.5 # -0.5 is to offset from the centre of each pixel. + slice_offs = np.arange(0.5/slice_subpixel, 1, 1/slice_subpixel) - 0.5 # -0.5 is to offset from the centre of each slice. spat_x, spec_y = np.meshgrid(spat_offs, spec_offs) - num_subpixels = spec_subpixel * spat_subpixel - area = 1 / num_subpixels - all_wght_subpix = all_wghts * area - all_var = utils.inverse(all_ivar) + num_subpixels = spec_subpixel * spat_subpixel # Number of subpixels per detector pixel + area = 1 / (num_subpixels * slice_subpixel) # Area of each subpixel # Loop through all exposures for fr in range(numframes): + onslit_gpm = _gpmImg[fr] + this_onslit_gpm = onslit_gpm > 0 + this_specpos, this_spatpos = np.where(this_onslit_gpm) + this_spatid = onslit_gpm[this_onslit_gpm] + # Extract tilts and slits for convenience this_tilts = _tilts[fr] this_slits = _slits[fr] - # Loop through all slits - for sl, spatid in enumerate(this_slits.spat_id): - if numframes == 1: - msgs.info(f"Resampling slit {sl+1}/{this_slits.nslits}") - else: - msgs.info(f"Resampling slit {sl+1}/{this_slits.nslits} of frame {fr+1}/{numframes}") - this_sl = np.where((all_spatid == spatid) & (all_idx == fr)) - wpix = (all_specpos[this_sl], all_spatpos[this_sl]) - # Generate a spline between spectral pixel position and wavelength - yspl = this_tilts[wpix]*(this_slits.nspec - 1) - tiltpos = np.add.outer(yspl, spec_y).flatten() - wspl = all_wave[this_sl] - asrt = np.argsort(yspl) - wave_spl = interp1d(yspl[asrt], wspl[asrt], kind='linear', bounds_error=False, fill_value='extrapolate') - # Calculate the wavelength at each subpixel - this_wave = wave_spl(tiltpos) - # Calculate the DAR correction at each sub pixel - ra_corr, dec_corr = _all_dar[fr].correction(this_wave) # This routine needs the wavelengths to be expressed in Angstroms - # Calculate spatial and spectral positions of the subpixels - spat_xx = np.add.outer(wpix[1], spat_x.flatten()).flatten() - spec_yy = np.add.outer(wpix[0], spec_y.flatten()).flatten() - # Transform this to spatial location - spatpos_subpix = _astrom_trans[fr].transform(sl, spat_xx, spec_yy) - spatpos = _astrom_trans[fr].transform(sl, all_spatpos[this_sl], all_specpos[this_sl]) - # Interpolate the RA/Dec over the subpixel spatial positions - ssrt = np.argsort(spatpos) - tmp_ra = all_ra[this_sl] - tmp_dec = all_dec[this_sl] - ra_spl = interp1d(spatpos[ssrt], tmp_ra[ssrt], kind='linear', bounds_error=False, fill_value='extrapolate') - dec_spl = interp1d(spatpos[ssrt], tmp_dec[ssrt], kind='linear', bounds_error=False, fill_value='extrapolate') - this_ra = ra_spl(spatpos_subpix) - this_dec = dec_spl(spatpos_subpix) - # Now apply the DAR correction - this_ra += ra_corr - this_dec += dec_corr - # Convert world coordinates to voxel coordinates, then histogram - vox_coord = output_wcs.wcs_world2pix(np.vstack((this_ra, this_dec, this_wave * 1.0E-10)).T, 0) - # Use the "fast histogram" algorithm, that assumes regular bin spacing - flxcube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(all_sci[this_sl] * all_wght_subpix[this_sl], num_subpixels)) - varcube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(all_var[this_sl] * all_wght_subpix[this_sl]**2, num_subpixels)) - normcube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(all_wght_subpix[this_sl], num_subpixels)) - if debug: - residcube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(all_sci[this_sl] * np.sqrt(all_ivar[this_sl]), num_subpixels)) + this_wcs = _all_wcs[fr] + this_astrom_trans = _astrom_trans[fr] + this_wght_subpix = _wghtImg[fr][this_onslit_gpm] * area + this_sci = _sciImg[fr][this_onslit_gpm] + this_var = utils.inverse(_ivarImg[fr][this_onslit_gpm]) + this_wav = _waveImg[fr][this_onslit_gpm] + # Loop over the subslices + for ss in range(slice_subpixel): + if slice_subpixel > 1: + # Only print this if there are multiple subslices + msgs.info(f"Subslice resampling {ss+1}/{slice_subpixel}") + # Generate an RA/Dec image for this subslice + raimg, decimg, minmax = this_slits.get_radec_image(this_wcs, this_astrom_trans, this_tilts, slice_offset=slice_offs[ss]) + this_ra = raimg[this_onslit_gpm] + this_dec = decimg[this_onslit_gpm] + # Loop through all slits + for sl, spatid in enumerate(this_slits.spat_id): + if numframes == 1: + msgs.info(f"Resampling slit {sl+1}/{this_slits.nslits}") + else: + msgs.info(f"Resampling slit {sl+1}/{this_slits.nslits} of frame {fr+1}/{numframes}") + this_sl = np.where(this_spatid == spatid) + wpix = (this_specpos[this_sl], this_spatpos[this_sl]) + # Generate a spline between spectral pixel position and wavelength + yspl = this_tilts[wpix]*(this_slits.nspec - 1) + tiltpos = np.add.outer(yspl, spec_y).flatten() + wspl = this_wav[this_sl] + asrt = np.argsort(yspl) + wave_spl = interp1d(yspl[asrt], wspl[asrt], kind='linear', bounds_error=False, fill_value='extrapolate') + # Calculate the wavelength at each subpixel + this_wave_subpix = wave_spl(tiltpos) + # Calculate the DAR correction at each sub pixel + ra_corr, dec_corr = _all_dar[fr].correction(this_wave_subpix) # This routine needs the wavelengths to be expressed in Angstroms + # Calculate spatial and spectral positions of the subpixels + spat_xx = np.add.outer(wpix[1], spat_x.flatten()).flatten() + spec_yy = np.add.outer(wpix[0], spec_y.flatten()).flatten() + # Transform this to spatial location + spatpos_subpix = _astrom_trans[fr].transform(sl, spat_xx, spec_yy) + spatpos = _astrom_trans[fr].transform(sl, wpix[1], wpix[0]) + # Interpolate the RA/Dec over the subpixel spatial positions + ssrt = np.argsort(spatpos) + tmp_ra = this_ra[this_sl] + tmp_dec = this_dec[this_sl] + ra_spl = interp1d(spatpos[ssrt], tmp_ra[ssrt], kind='linear', bounds_error=False, fill_value='extrapolate') + dec_spl = interp1d(spatpos[ssrt], tmp_dec[ssrt], kind='linear', bounds_error=False, fill_value='extrapolate') + # Evaluate the RA/Dec at the subpixel spatial positions + this_ra_int = ra_spl(spatpos_subpix) + this_dec_int = dec_spl(spatpos_subpix) + # Now apply the DAR correction + this_ra_int += ra_corr + this_dec_int += dec_corr + # Convert world coordinates to voxel coordinates, then histogram + vox_coord = output_wcs.wcs_world2pix(np.vstack((this_ra_int, this_dec_int, this_wave_subpix * 1.0E-10)).T, 0) + # Use the "fast histogram" algorithm, that assumes regular bin spacing + flxcube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(this_sci[this_sl] * this_wght_subpix[this_sl], num_subpixels)) + varcube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(this_var[this_sl] * this_wght_subpix[this_sl]**2, num_subpixels)) + normcube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(this_wght_subpix[this_sl], num_subpixels)) + if debug: + residcube += histogramdd(vox_coord, bins=outshape, range=binrng, weights=np.repeat(this_sci[this_sl] * np.sqrt(utils.inverse(this_var[this_sl])), num_subpixels)) # Normalise the datacube and variance cube nc_inverse = utils.inverse(normcube) flxcube *= nc_inverse From ce585ea38d5decfb616c3929c42b3b56e955979d Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 8 Dec 2023 21:44:06 +0000 Subject: [PATCH 085/244] add slice_subpixel --- pypeit/coadd3d.py | 150 +++++++++++++++++++--------------------------- 1 file changed, 63 insertions(+), 87 deletions(-) diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index 45aebcf204..93336ea0a8 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -397,16 +397,16 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs self.overwrite = overwrite # Do some quick checks on the input options if skysub_frame is not None: - if len(skysub_frame) != len(spec2dfiles): + if len(skysub_frame) != self.numfiles: msgs.error("The skysub_frame list should be identical length to the spec2dfiles list") if scale_corr is not None: - if len(scale_corr) != len(spec2dfiles): + if len(scale_corr) != self.numfiles: msgs.error("The scale_corr list should be identical length to the spec2dfiles list") if ra_offsets is not None: - if len(ra_offsets) != len(spec2dfiles): + if len(ra_offsets) != self.numfiles: msgs.error("The ra_offsets list should be identical length to the spec2dfiles list") if dec_offsets is not None: - if len(dec_offsets) != len(spec2dfiles): + if len(dec_offsets) != self.numfiles: msgs.error("The dec_offsets list should be identical length to the spec2dfiles list") # Set the frame-specific options self.skysub_frame = skysub_frame @@ -429,9 +429,8 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs # Initialise arrays for storage self.ifu_ra, self.ifu_dec = np.array([]), np.array([]) # The RA and Dec at the centre of the IFU, as stored in the header - self.all_ra, self.all_dec, self.all_wave = np.array([]), np.array([]), np.array([]) - self.all_sci, self.all_ivar, self.all_idx, self.all_wghts = np.array([]), np.array([]), np.array([]), np.array([]) - self.all_spatpos, self.all_specpos, self.all_spatid = np.array([], dtype=int), np.array([], dtype=int), np.array([], dtype=int) + + self.all_sci, self.all_ivar, self.all_wave, self.all_slitid, self.all_wghts = [], [], [], [], [] self.all_tilts, self.all_slits, self.all_align = [], [], [] self.all_wcs, self.all_dar = [], [] self.weights = np.ones(self.numfiles) # Weights to use when combining cubes @@ -460,10 +459,13 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs # At the moment, if the user wishes to spatially align the frames, a different WCS is generated. # Determine what method is requested - self.spec_subpixel, self.spat_subpixel = 1, 1 + self.spec_subpixel, self.spat_subpixel, self.slice_subpixel = 1, 1, 1 if self.method == "subpixel": - msgs.info("Adopting the subpixel algorithm to generate the datacube.") - self.spec_subpixel, self.spat_subpixel = self.cubepar['spec_subpixel'], self.cubepar['spat_subpixel'] + self.spec_subpixel, self.spat_subpixel, self.slice_subpixel = self.cubepar['spec_subpixel'], self.cubepar['spat_subpixel'], self.cubepar['slice_subpixel'] + msgs.info("Adopting the subpixel algorithm to generate the datacube, with subpixellation scales:" + msgs.newline() + + f" Spectral: {self.spec_subpixel}" + msgs.newline() + + f" Spatial: {self.spat_subpixel}" + msgs.newline() + + f" Slices: {self.slice_subpixel}") elif self.method == "ngp": msgs.info("Adopting the nearest grid point (NGP) algorithm to generate the datacube.") else: @@ -919,8 +921,6 @@ def load(self): As well as the primary arrays that store the pixel information for multiple spec2d frames, including: - * self.all_ra - * self.all_dec * self.all_wave * self.all_sci * self.all_ivar @@ -1039,33 +1039,27 @@ def load(self): onslit_gpm = (slitid_img_init > 0) & (bpmmask.mask == 0) & sky_is_good # Grab the WCS of this frame - self.all_wcs.append(self.spec.get_wcs(spec2DObj.head0, slits, detector.platescale, wave0, dwv)) + crval_wv = self.cubepar['wave_min'] if self.cubepar['wave_min'] is not None else wave0 + cd_wv = self.cubepar['wave_delta'] if self.cubepar['wave_delta'] is not None else dwv + self.all_wcs.append(self.spec.get_wcs(spec2DObj.head0, slits, detector.platescale, crval_wv, cd_wv)) # Generate the alignment splines, and then retrieve images of the RA and Dec of every pixel, # and the number of spatial pixels in each slit alignSplines = self.get_alignments(spec2DObj, slits, spat_flexure=spat_flexure) - raimg, decimg, minmax = slits.get_radec_image(self.all_wcs[ff], alignSplines, spec2DObj.tilts, - initial=True, flexure=spat_flexure) - # Get copies of arrays to be saved - ra_ext = raimg[onslit_gpm] - dec_ext = decimg[onslit_gpm] + # Extract wavelength and delta wavelength arrays from the images wave_ext = waveimg[onslit_gpm] - flux_ext = sciImg[onslit_gpm] - ivar_ext = ivar[onslit_gpm] dwav_ext = dwaveimg[onslit_gpm] - # From here on out, work in sorted wavelengths + # For now, work in sorted wavelengths wvsrt = np.argsort(wave_ext) wave_sort = wave_ext[wvsrt] dwav_sort = dwav_ext[wvsrt] - ra_sort = ra_ext[wvsrt] - dec_sort = dec_ext[wvsrt] # Here's an array to get back to the original ordering resrt = np.argsort(wvsrt) # Compute the DAR correction - cosdec = np.cos(np.mean(dec_sort) * np.pi / 180.0) + cosdec = np.cos(self.ifu_dec[ff] * np.pi / 180.0) airmass = self.spec.get_meta_value([spec2DObj.head0], 'airmass') # unitless parangle = self.spec.get_meta_value([spec2DObj.head0], 'parangle') pressure = self.spec.get_meta_value([spec2DObj.head0], 'pressure') # units are pascals @@ -1073,7 +1067,7 @@ def load(self): humidity = self.spec.get_meta_value([spec2DObj.head0], 'humidity') # Expressed as a percentage (not a fraction!) darcorr = DARcorrection(airmass, parangle, pressure, temperature, humidity, cosdec) - # Perform extinction correction + # Compute the extinction correction msgs.info("Applying extinction correction") longitude = self.spec.telescope['longitude'] latitude = self.spec.telescope['latitude'] @@ -1104,23 +1098,22 @@ def load(self): # Convert the flux_sav to counts/s, correct for the relative sensitivity of different setups extcorr_sort *= sensfunc_sort / (exptime * gratcorr_sort) # Correct for extinction - flux_sort = flux_ext[wvsrt] * extcorr_sort - ivar_sort = ivar_ext[wvsrt] / extcorr_sort ** 2 + sciImg[onslit_gpm] *= extcorr_sort[resrt] + ivar[onslit_gpm] /= extcorr_sort[resrt] ** 2 # Convert units to Counts/s/Ang/arcsec2 # Slicer sampling * spatial pixel sampling sl_deg = np.sqrt(self.all_wcs[ff].wcs.cd[0, 0] ** 2 + self.all_wcs[ff].wcs.cd[1, 0] ** 2) px_deg = np.sqrt(self.all_wcs[ff].wcs.cd[1, 1] ** 2 + self.all_wcs[ff].wcs.cd[0, 1] ** 2) scl_units = dwav_sort * (3600.0 * sl_deg) * (3600.0 * px_deg) - flux_sort /= scl_units - ivar_sort *= scl_units ** 2 + sciImg[onslit_gpm] /= scl_units[resrt] + ivar[onslit_gpm] *= scl_units[resrt] ** 2 # Calculate the weights relative to the zeroth cube self.weights[ff] = 1.0 # exptime #np.median(flux_sav[resrt]*np.sqrt(ivar_sav[resrt]))**2 # Get the slit image and then unset pixels in the slit image that are bad - this_specpos, this_spatpos = np.where(onslit_gpm) - this_spatid = slitid_img_init[onslit_gpm] + slitid_img_gpm = slitid_img_init * onslit_gpm.astype(int) ################################## # Astrometric alignment to HST frames @@ -1128,24 +1121,20 @@ def load(self): # Get the coordinate bounds slitlength = int(np.round(np.median(slits.get_slitlengths(initial=True, median=True)))) numwav = int((np.max(waveimg) - wave0) / dwv) + _, _, minmax = slits.get_radec_image(self.all_wcs[ff], alignSplines, spec2DObj.tilts, initial=True, flexure=spat_flexure) raw_bins = self.spec.get_datacube_bins(slitlength, minmax, numwav) - # Generate the output WCS for the datacube - tmp_crval_wv = (self.all_wcs[ff].wcs.crval[2] * self.all_wcs[ff].wcs.cunit[2]).to(units.Angstrom).value - tmp_cd_wv = (self.all_wcs[ff].wcs.cd[2, 2] * self.all_wcs[ff].wcs.cunit[2]).to(units.Angstrom).value - crval_wv = self.cubepar['wave_min'] if self.cubepar['wave_min'] is not None else tmp_crval_wv - cd_wv = self.cubepar['wave_delta'] if self.cubepar['wave_delta'] is not None else tmp_cd_wv - raw_wcs = self.spec.get_wcs(spec2DObj.head0, slits, detector.platescale, crval_wv, cd_wv) + wghts = np.ones(sciImg.shape) # Now do the alignment - _ra_sort, _dec_sort = hst_alignment(ra_sort, dec_sort, wave_sort, flux_sort, ivar_sort, np.max(self._spatscale[ff,:]), self._specscale[ff], - np.ones(ra_sort.size), this_spatpos[wvsrt], this_specpos[wvsrt], - this_spatid[wvsrt], spec2DObj.tilts, slits, alignSplines, darcorr, - raw_wcs=raw_wcs, raw_bins=raw_bins) - ra_del = np.median(_ra_sort-ra_sort) - dec_del = np.median(_dec_sort-dec_sort) + _ra_sort, _dec_sort = hst_alignment(sciImg, ivar, waveimg, slitid_img_gpm, wghts, + #np.max(self._spatscale[ff,:]), self._specscale[ff], + self.all_wcs[ff], spec2DObj.tilts, slits, alignSplines, darcorr, + raw_bins=raw_bins) + self.ra_offsets[ff] = np.median(_ra_sort-ra_sort) + self.dec_offsets[ff] = np.median(_dec_sort-dec_sort) ra_sort = _ra_sort dec_sort = _dec_sort - spec2DObj.head0['RA'] += ra_del - spec2DObj.head0['DEC'] += dec_del + self.ifu_ra[ff] += self.ra_offsets[ff] + self.ifu_dec[ff] += self.dec_offsets[ff] ################################## # If individual frames are to be output without aligning them, @@ -1161,30 +1150,25 @@ def load(self): slitlength = int(np.round(np.median(slits.get_slitlengths(initial=True, median=True)))) numwav = int((np.max(waveimg) - wave0) / dwv) bins = self.spec.get_datacube_bins(slitlength, minmax, numwav) - # Generate the output WCS for the datacube - tmp_crval_wv = (self.all_wcs[ff].wcs.crval[2] * self.all_wcs[ff].wcs.cunit[2]).to(units.Angstrom).value - tmp_cd_wv = (self.all_wcs[ff].wcs.cd[2,2] * self.all_wcs[ff].wcs.cunit[2]).to(units.Angstrom).value - crval_wv = self.cubepar['wave_min'] if self.cubepar['wave_min'] is not None else tmp_crval_wv - cd_wv = self.cubepar['wave_delta'] if self.cubepar['wave_delta'] is not None else tmp_cd_wv - output_wcs = self.spec.get_wcs(spec2DObj.head0, slits, detector.platescale, crval_wv, cd_wv) # Set the wavelength range of the white light image. wl_wvrng = None if self.cubepar['save_whitelight']: wl_wvrng = datacube.get_whitelight_range(np.max(self.mnmx_wv[ff, :, 0]), - np.min(self.mnmx_wv[ff, :, 1]), - self.cubepar['whitelight_range']) + np.min(self.mnmx_wv[ff, :, 1]), + self.cubepar['whitelight_range']) # Make the datacube if self.method in ['subpixel', 'ngp']: # Generate the datacube flxcube, sigcube, bpmcube, wave = \ - datacube.generate_cube_subpixel(outfile, output_wcs, ra_sort[resrt], dec_sort[resrt], wave_sort[resrt], - flux_sort[resrt], ivar_sort[resrt], np.ones(numpix), - this_spatpos, this_specpos, this_spatid, - spec2DObj.tilts, slits, alignSplines, darcorr, bins, all_idx=None, + datacube.generate_cube_subpixel(outfile, self.all_wcs[ff], bins, + sciImg, ivar, waveimg, slitid_img_gpm, wghts, + self.all_wcs[ff], spec2DObj.tilts, slits, alignSplines, darcorr, overwrite=self.overwrite, whitelight_range=wl_wvrng, - spec_subpixel=self.spec_subpixel, spat_subpixel=self.spat_subpixel) + spec_subpixel=self.spec_subpixel, + spat_subpixel=self.spat_subpixel, + slice_subpixel=self.slice_subpixel) # Prepare the header - hdr = output_wcs.to_header() + hdr = self.all_wcs[ff].to_header() if self.fluxcal: hdr['FLUXUNIT'] = (flux_calib.PYPEIT_FLUX_SCALE, "Flux units -- erg/s/cm^2/Angstrom/arcsec^2") else: @@ -1198,16 +1182,11 @@ def load(self): continue # Store the information if we are combining multiple frames - self.all_ra = np.append(self.all_ra, ra_sort[resrt]) - self.all_dec = np.append(self.all_dec, dec_sort[resrt]) - self.all_wave = np.append(self.all_wave, wave_sort[resrt]) - self.all_sci = np.append(self.all_sci, flux_sort[resrt]) - self.all_ivar = np.append(self.all_ivar, ivar_sort[resrt].copy()) - self.all_idx = np.append(self.all_idx, ff * np.ones(numpix)) - self.all_wghts = np.append(self.all_wghts, self.weights[ff] * np.ones(numpix) / self.weights[0]) - self.all_spatpos = np.append(self.all_spatpos, this_spatpos) - self.all_specpos = np.append(self.all_specpos, this_specpos) - self.all_spatid = np.append(self.all_spatid, this_spatid) + self.all_sci.append(sciImg.copy()) + self.all_ivar.append(ivar.copy()) + self.all_wave.append(waveimg.copy()) + self.all_slitid.append(slitid_img_gpm.copy()) + self.all_wghts.append(wghts.copy()) self.all_tilts.append(spec2DObj.tilts) self.all_slits.append(slits) self.all_align.append(alignSplines) @@ -1422,10 +1401,9 @@ def run(self): final_cube.to_file(outfile, hdr=hdr, overwrite=self.overwrite) -def hst_alignment(ra_sort, dec_sort, wave_sort, flux_sort, ivar_sort, dspat, dwave, - wghts, spatpos, specpos, - all_spatid, tilts, slits, astrom_trans, all_dar, - spat_subpixel=10, spec_subpixel=10, raw_wcs=None, raw_bins=None): +def hst_alignment(sciImg, ivar, waveimg, slitid_img_gpm, wghts, + all_wcs, tilts, slits, astrom_trans, all_dar, + spat_subpixel=10, spec_subpixel=10, slice_subpixel=1, raw_bins=None): """ This is currently only used by RJC. This function adds corrections to the RA and Dec pixels to align the daatcubes to an HST image. @@ -1477,24 +1455,21 @@ def hst_alignment(ra_sort, dec_sort, wave_sort, flux_sort, ivar_sort, dspat, dwa # Need to generate a datacube near both Hgamma and Hdelta. ############ #Is DAR correction being performed here? + slice_subpixel = 10 # FIRST DO Hdelta - wv_mask = (wave_sort > 4107.0) & (wave_sort < 4119.0) + inmask = slitid_img_gpm * ((waveimg > 4107.0) & (waveimg < 4119.0)).astype(int) flxcube, sigcube, bpmcube, wave = \ - datacube.generate_cube_subpixel("tmpfile.fits", raw_wcs, ra_sort[wv_mask], dec_sort[wv_mask], wave_sort[wv_mask], - flux_sort[wv_mask], ivar_sort[wv_mask], wghts[wv_mask], spatpos[wv_mask], specpos[wv_mask], - all_spatid[wv_mask], tilts, slits, astrom_trans, all_dar, - raw_bins, all_idx=None, overwrite=False, - spec_subpixel=5, spat_subpixel=5) - HdMap_raw, HdMapErr_raw = astrometry.fit_cube(flxcube, sigcube**2, raw_wcs, line="HIdelta") + datacube.generate_cube_subpixel("tmpfile.fits", all_wcs, raw_bins, sciImg, ivar, waveimg, inmask, wghts, + all_wcs, tilts, slits, astrom_trans, all_dar, overwrite=False, + spec_subpixel=5, spat_subpixel=5, slice_subpixel=slice_subpixel) + HdMap_raw, HdMapErr_raw = astrometry.fit_cube(flxcube, sigcube**2, all_wcs, line="HIdelta") # THEN DO Hgamma - wv_mask = (wave_sort > 4346.0) & (wave_sort < 4358.0) + inmask = slitid_img_gpm * ((waveimg > 4346.0) & (waveimg < 4358.0)).astype(int) flxcube, sigcube, bpmcube, wave = \ - datacube.generate_cube_subpixel("tmpfile.fits", raw_wcs, ra_sort[wv_mask], dec_sort[wv_mask], wave_sort[wv_mask], - flux_sort[wv_mask], ivar_sort[wv_mask], wghts[wv_mask], spatpos[wv_mask], specpos[wv_mask], - all_spatid[wv_mask], tilts, slits, astrom_trans, all_dar, - raw_bins, all_idx=None, overwrite=False, - spec_subpixel=5, spat_subpixel=5) - HgMap_raw, HgMapErr_raw = astrometry.fit_cube(flxcube, sigcube**2, raw_wcs, line="HIgamma") + datacube.generate_cube_subpixel("tmpfile.fits", all_wcs, raw_bins, sciImg, ivar, waveimg, inmask, wghts, + all_wcs, tilts, slits, astrom_trans, all_dar, overwrite=False, + spec_subpixel=5, spat_subpixel=5, slice_subpixel=slice_subpixel) + HgMap_raw, HgMapErr_raw = astrometry.fit_cube(flxcube, sigcube**2, all_wcs, line="HIgamma") # Plot the emission line map from matplotlib import pyplot as plt plt.subplot(131) @@ -1505,6 +1480,7 @@ def hst_alignment(ra_sort, dec_sort, wave_sort, flux_sort, ivar_sort, dspat, dwa plt.imshow(HdMap_raw*utils.inverse(HgMap_raw), vmin=0.5, vmax=0.7, aspect=0.5) plt.show() embed() + # np.save("Hd-Hg_preSlicerDAR.npy", HdMap_raw*utils.inverse(HgMap_raw)) ############ ## STEP 3B ## - Map the emission line map to an HST image, and vice-versa ############ From 519ae25b23d02b04eea07bae40de43a7753913ee Mon Sep 17 00:00:00 2001 From: rcooke Date: Sat, 9 Dec 2023 15:01:15 +0000 Subject: [PATCH 086/244] convert arcsec to degrees for offsets --- pypeit/inputfiles.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pypeit/inputfiles.py b/pypeit/inputfiles.py index 83619c6d8d..fb54c23a61 100644 --- a/pypeit/inputfiles.py +++ b/pypeit/inputfiles.py @@ -822,17 +822,21 @@ def options(self): if off_ra is None: opts['ra_offset'] = None elif len(off_ra) == 1 and len(self.filenames) > 1: - opts['ra_offset'] = off_ra*len(self.filenames) + # Convert from arcsec to degrees + opts['ra_offset'] = [off_ra[0]/3600.0 for ii in range(len(self.filenames))] elif len(off_ra) != 0: - opts['ra_offset'] = off_ra + # Convert from arcsec to degrees + opts['ra_offset'] = [ora/3600.0 for ora in off_ra] # Get the DEC offset of each file off_dec = self.path_and_files('dec_offset', skip_blank=False, check_exists=False) if off_dec is None: opts['dec_offset'] = None elif len(off_dec) == 1 and len(self.filenames) > 1: - opts['dec_offset'] = off_dec*len(self.filenames) + # Convert from arcsec to degrees + opts['dec_offset'] = [off_dec[0]/3600.0 for ii in range(len(self.filenames))] elif len(off_dec) != 0: - opts['dec_offset'] = off_dec + # Convert from arcsec to degrees + opts['dec_offset'] = [odec/3600.0 for odec in off_dec] # Check that both have been set if (off_ra is not None and off_dec is None) or (off_ra is None and off_dec is not None): msgs.error("You must specify both or neither of the following arguments: ra_offset, dec_offset") From c1eed0ac2c844d695967d2fecb8e71ce9b0004ad Mon Sep 17 00:00:00 2001 From: rcooke Date: Sat, 9 Dec 2023 20:13:07 +0000 Subject: [PATCH 087/244] convert arcsec to degrees for offsets --- pypeit/inputfiles.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pypeit/inputfiles.py b/pypeit/inputfiles.py index fb54c23a61..bbf1347e08 100644 --- a/pypeit/inputfiles.py +++ b/pypeit/inputfiles.py @@ -823,7 +823,7 @@ def options(self): opts['ra_offset'] = None elif len(off_ra) == 1 and len(self.filenames) > 1: # Convert from arcsec to degrees - opts['ra_offset'] = [off_ra[0]/3600.0 for ii in range(len(self.filenames))] + opts['ra_offset'] = [off_ra[0]/3600.0 for _ in range(len(self.filenames))] elif len(off_ra) != 0: # Convert from arcsec to degrees opts['ra_offset'] = [ora/3600.0 for ora in off_ra] @@ -833,7 +833,7 @@ def options(self): opts['dec_offset'] = None elif len(off_dec) == 1 and len(self.filenames) > 1: # Convert from arcsec to degrees - opts['dec_offset'] = [off_dec[0]/3600.0 for ii in range(len(self.filenames))] + opts['dec_offset'] = [off_dec[0]/3600.0 for _ in range(len(self.filenames))] elif len(off_dec) != 0: # Convert from arcsec to degrees opts['dec_offset'] = [odec/3600.0 for odec in off_dec] From 4c1487764925749bcca37563e7a071bcb0d36473 Mon Sep 17 00:00:00 2001 From: rcooke Date: Sat, 9 Dec 2023 20:25:29 +0000 Subject: [PATCH 088/244] include offsets --- pypeit/core/datacube.py | 119 +++++++++++++++++++++++++++------------- 1 file changed, 82 insertions(+), 37 deletions(-) diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index 1fc5de4440..f8a7e0b52a 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -428,31 +428,70 @@ def get_output_whitelight_filename(outfile): return os.path.splitext(outfile)[0] + "_whitelight.fits" -def get_whitelight_pixels(all_wave, min_wl, max_wl): +def get_whitelight_pixels(all_wave, all_slitid, min_wl, max_wl): """ Determine which pixels are included within the specified wavelength range Args: - all_wave (`numpy.ndarray`_): - The wavelength of each individual pixel + all_wave (`numpy.ndarray`_, list): + List of `numpy.ndarray`_ wavelength images. The length of the list is the number of spec2d frames. + Each element of the list contains a wavelength image that provides the wavelength at each pixel on + the detector, with shape is (nspec, nspat). + all_slitid (`numpy.ndarray`_, list): + List of `numpy.ndarray`_ slitid images. The length of the list is the number of spec2d frames. + Each element of the list contains a slitid image that provides the slit number at each pixel on + the detector, with shape (nspec, nspat). min_wl (float): Minimum wavelength to consider max_wl (float): Maximum wavelength to consider Returns: - :obj:`tuple`: A `numpy.ndarray`_ object with the indices of all_wave - that contain pixels within the requested wavelength range, and a float - with the wavelength range (i.e. maximum wavelength - minimum wavelength) + :obj:`tuple`: The first element of the tuple is a list of `numpy.ndarray`_ slitid images, + shape is (nspec, nspat), where a zero value corresponds to an excluded pixel + (either outside the desired wavelength range, a bad pixel, a pixel not on the slit). + All other pixels have a value equal to the slit number. The second element of the tuple + is the wavelength difference between the maximum and minimum wavelength in the desired + wavelength range. """ - wavediff = np.max(all_wave) - np.min(all_wave) + # Check if lists or numpy arrays are input + list_inputs = [all_wave, all_slitid] + if all([isinstance(l, list) for l in list_inputs]): + numframes = len(all_wave) + if not all([len(l) == numframes for l in list_inputs]): + msgs.error("All input lists must have the same length") + # Store in the following variables + _all_wave, _all_slitid = all_wave, all_slitid + elif all([not isinstance(l, list) for l in list_inputs]): + _all_wave, _all_slitid = [all_wave], [all_slitid] + numframes = 1 + else: + msgs.error("The input lists must either all be lists (of the same length) or all be numpy arrays") + if max_wl < min_wl: + msgs.error("The maximum wavelength must be greater than the minimum wavelength") + # Initialise the output + out_slitid = [np.zeros(_all_slitid[0].shape, dtype=int) for _ in range(numframes)] + # Loop over all frames and find the pixels that are within the wavelength range if min_wl < max_wl: - ww = np.where((all_wave > min_wl) & (all_wave < max_wl)) - wavediff = max_wl - min_wl + # Loop over files and determine which pixels are within the wavelength range + for ff in range(numframes): + ww = np.where((_all_wave[ff] > min_wl) & (_all_wave[ff] < max_wl)) + out_slitid[ff][ww] = _all_slitid[ff][ww] else: - msgs.warn("Datacubes do not completely overlap in wavelength. Offsets may be unreliable...") - ww = (np.arange(all_wave.size),) - return ww, wavediff + msgs.warn("Datacubes do not completely overlap in wavelength.") + out_slitid = _all_slitid + min_wl, max_wl = None, None + for ff in range(numframes): + this_wave = _all_wave[ff][_all_slitid[ff] > 0] + tmp_min = np.min(this_wave) + tmp_max = np.max(this_wave) + if min_wl is None or tmp_min < min_wl: + min_wl = tmp_min + if max_wl is None or tmp_max > max_wl: + max_wl = tmp_max + # Determine the wavelength range + wavediff = max_wl - min_wl + return out_slitid, wavediff def get_whitelight_range(wavemin, wavemax, wl_range): @@ -562,7 +601,7 @@ def load_imageWCS(filename, ext=0): return image, imgwcs -def align_user_offsets(all_ra, all_dec, all_idx, ifu_ra, ifu_dec, ra_offset, dec_offset): +def align_user_offsets(ifu_ra, ifu_dec, ra_offset, dec_offset): """ Align the RA and DEC of all input frames, and then manually shift the cubes based on user-provided offsets. @@ -570,28 +609,21 @@ def align_user_offsets(all_ra, all_dec, all_idx, ifu_ra, ifu_dec, ra_offset, dec ra_offset should include the cos(dec) factor. Args: - all_ra (`numpy.ndarray`_): - A 1D array containing the RA values of each detector pixel of every frame. - all_dec (`numpy.ndarray`_): - A 1D array containing the Dec values of each detector pixel of every frame. - Same size as all_ra. - all_idx (`numpy.ndarray`_): - A 1D array containing an ID value for each detector frame (0-indexed). - Same size as all_ra. ifu_ra (`numpy.ndarray`_): A list of RA values of the IFU (one value per frame) ifu_dec (`numpy.ndarray`_): A list of Dec values of the IFU (one value per frame) ra_offset (`numpy.ndarray`_): A list of RA offsets to be applied to the input pixel values (one value per frame). - Note, the ra_offset MUST contain the cos(dec) factor. This is the number of arcseconds + Note, the ra_offset MUST contain the cos(dec) factor. This is the number of degrees on the sky that represents the telescope offset. dec_offset (`numpy.ndarray`_): A list of Dec offsets to be applied to the input pixel values (one value per frame). + This is the number of degrees on the sky that represents the telescope offset. Returns: - A tuple containing a new set of RA and Dec values that have been aligned. Both arrays - are of type `numpy.ndarray`_. + A tuple containing a new set of RA and Dec offsets for each frame. + Both arrays are of type `numpy.ndarray`_, and are in units of degrees. """ # First, translate all coordinates to the coordinates of the first frame # Note: You do not need cos(dec) here, this just overrides the IFU coordinate centre of each frame @@ -599,13 +631,15 @@ def align_user_offsets(all_ra, all_dec, all_idx, ifu_ra, ifu_dec, ra_offset, dec ref_shift_ra = ifu_ra[0] - ifu_ra ref_shift_dec = ifu_dec[0] - ifu_dec numfiles = ra_offset.size + out_ra_offsets = np.zeros(numfiles) + out_dec_offsets = np.zeros(numfiles) for ff in range(numfiles): # Apply the shift - all_ra[all_idx == ff] += ref_shift_ra[ff] + ra_offset[ff] / 3600.0 - all_dec[all_idx == ff] += ref_shift_dec[ff] + dec_offset[ff] / 3600.0 + out_ra_offsets[ff] = ref_shift_ra[ff] + ra_offset[ff] + out_dec_offsets[ff] = ref_shift_dec[ff] + dec_offset[ff] msgs.info("Spatial shift of cube #{0:d}:".format(ff + 1) + msgs.newline() + "RA, DEC (arcsec) = {0:+0.3f} E, {1:+0.3f} N".format(ra_offset[ff], dec_offset[ff])) - return all_ra, all_dec + return out_ra_offsets, out_dec_offsets def set_voxel_sampling(spatscale, specscale, dspat=None, dwv=None): @@ -1193,6 +1227,7 @@ def generate_image_subpixel(image_wcs, all_ra, all_dec, all_wave, all_sci, all_i def generate_cube_subpixel(outfile, output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, all_wcs, tilts, slits, astrom_trans, all_dar, + ra_offset, dec_offset, spec_subpixel=5, spat_subpixel=5, slice_subpixel=5, overwrite=False, whitelight_range=None, debug=False): """ @@ -1235,6 +1270,10 @@ def generate_cube_subpixel(outfile, output_wcs, bins, all_dar (:class:`~pypeit.coadd3d.DARcorrection`, list): A Class containing the DAR correction information, or a list of DARcorrection classes. If a list, it must be the same length as astrom_trans. + ra_offset (float, list): + A float or list of floats containing the RA offset of each spec2d file + dec_offset (float, list): + A float or list of floats containing the DEC offset of each spec2d file spec_subpixel (int, optional): What is the subpixellation factor in the spectral direction. Higher values give more reliable results, but note that the time required @@ -1279,7 +1318,7 @@ def generate_cube_subpixel(outfile, output_wcs, bins, # Subpixellate subpix = subpixellate(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, - all_wcs, tilts, slits, astrom_trans, all_dar, + all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, slice_subpixel=slice_subpixel, debug=debug) # Extract the variables that we need @@ -1320,7 +1359,7 @@ def generate_cube_subpixel(outfile, output_wcs, bins, def subpixellate(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, - all_wcs, tilts, slits, astrom_trans, all_dar, + all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, spec_subpixel=5, spat_subpixel=5, slice_subpixel=5, debug=False): r""" Subpixellate the input data into a datacube. This algorithm splits each @@ -1370,6 +1409,12 @@ def subpixellate(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wgh all_dar (:class:`~pypeit.coadd3d.DARcorrection`, list): A Class containing the DAR correction information, or a list of DARcorrection classes. If a list, it must be the same length as astrom_trans. + ra_offset (float, list): + A float or list of floats containing the RA offset of each spec2d file + relative to the first spec2d file + dec_offset (float, list): + A float or list of floats containing the DEC offset of each spec2d file + relative to the first spec2d file spec_subpixel (int, optional): What is the subpixellation factor in the spectral direction. Higher values give more reliable results, but note that the time required @@ -1399,19 +1444,19 @@ def subpixellate(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wgh residual cube. The latter is only returned if debug is True. """ # Check for combinations of lists or not - list_inputs = [sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, all_wcs, tilts, slits, astrom_trans, all_dar] + list_inputs = [sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset] if all([isinstance(l, list) for l in list_inputs]): # Several frames are being combined. Check the lists have the same length numframes = len(sciImg) if not all([len(l) == numframes for l in list_inputs]): msgs.error("All input lists must have the same length when combining multiple frames") # Store in the following variables - _sciImg, _ivarImg, _waveImg, _gpmImg, _wghtImg, _all_wcs, _tilts, _slits, _astrom_trans, _all_dar = \ - sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, all_wcs, tilts, slits, astrom_trans, all_dar + _sciImg, _ivarImg, _waveImg, _gpmImg, _wghtImg, _all_wcs, _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset = \ + sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset elif all([not isinstance(l, list) for l in list_inputs]): # Just a single frame - store as lists for this code - _sciImg, _ivarImg, _waveImg, _gpmImg, _wghtImg, _all_wcs, _tilts, _slits, _astrom_trans, _all_dar = \ - [sciImg], [ivarImg], [waveImg], [slitid_img_gpm], [wghtImg], [all_wcs], [tilts], [slits], [astrom_trans], [all_dar] + _sciImg, _ivarImg, _waveImg, _gpmImg, _wghtImg, _all_wcs, _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset = \ + [sciImg], [ivarImg], [waveImg], [slitid_img_gpm], [wghtImg], [all_wcs], [tilts], [slits], [astrom_trans], [all_dar], [ra_offset], [dec_offset] numframes = 1 else: msgs.error("The input arguments should all be of type 'list', or all not be of type 'list':") @@ -1486,9 +1531,9 @@ def subpixellate(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wgh # Evaluate the RA/Dec at the subpixel spatial positions this_ra_int = ra_spl(spatpos_subpix) this_dec_int = dec_spl(spatpos_subpix) - # Now apply the DAR correction - this_ra_int += ra_corr - this_dec_int += dec_corr + # Now apply the DAR correction and any user-supplied offsets + this_ra_int += ra_corr + _ra_offset[fr] + this_dec_int += dec_corr + _dec_offset[fr] # Convert world coordinates to voxel coordinates, then histogram vox_coord = output_wcs.wcs_world2pix(np.vstack((this_ra_int, this_dec_int, this_wave_subpix * 1.0E-10)).T, 0) # Use the "fast histogram" algorithm, that assumes regular bin spacing From 8bde539642fae009c6b748fd55fdb79e2e5eff18 Mon Sep 17 00:00:00 2001 From: rcooke Date: Sat, 9 Dec 2023 20:43:48 +0000 Subject: [PATCH 089/244] refactor image subpixel --- pypeit/core/datacube.py | 127 ++++++++++++++++++++-------------------- 1 file changed, 63 insertions(+), 64 deletions(-) diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index f8a7e0b52a..a2a7c8134d 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -1114,41 +1114,36 @@ def compute_weights(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, white return all_wghts -def generate_image_subpixel(image_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_wghts, - all_spatpos, all_specpos, all_spatid, tilts, slits, astrom_trans, all_dar, bins, - all_idx=None, spec_subpixel=10, spat_subpixel=10, combine=False): +def generate_image_subpixel(image_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, + all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, + spec_subpixel=5, spat_subpixel=5, slice_subpixel=5, combine=False): """ Generate a white light image from the input pixels Args: image_wcs (`astropy.wcs.WCS`_): World coordinate system to use for the white light images. - all_ra (`numpy.ndarray`_): - 1D flattened array containing the right ascension of each pixel - (units = degrees) - all_dec (`numpy.ndarray`_): - 1D flattened array containing the declination of each pixel (units = - degrees) - all_wave (`numpy.ndarray`_): - 1D flattened array containing the wavelength of each pixel (units = - Angstroms) - all_sci (`numpy.ndarray`_): - 1D flattened array containing the counts of each pixel from all - spec2d files - all_ivar (`numpy.ndarray`_): - 1D flattened array containing the inverse variance of each pixel - from all spec2d files - all_wghts (`numpy.ndarray`_): - 1D flattened array containing the weights of each pixel to be used - in the combination - all_spatpos (`numpy.ndarray`_): - 1D flattened array containing the detector pixel location in the - spatial direction - all_specpos (`numpy.ndarray`_): - 1D flattened array containing the detector pixel location in the - spectral direction - all_spatid (`numpy.ndarray`_): - 1D flattened array containing the spatid of each pixel + bins (tuple): + A 3-tuple (x,y,z) containing the histogram bin edges in x,y spatial + and z wavelength coordinates + sciImg (`numpy.ndarray`_, list): + A list of 2D science images, or a single 2D image containing the + science data. + ivarImg (`numpy.ndarray`_, list): + A list of 2D inverse variance images, or a single 2D image + containing the inverse variance data. + waveImg (`numpy.ndarray`_, list): + A list of 2D wavelength images, or a single 2D image containing the + wavelength data. + slitid_img_gpm (`numpy.ndarray`_, list): + A list of 2D slit ID images, or a single 2D image containing the + slit ID data. + wghtImg (`numpy.ndarray`_, list): + A list of 2D weight images, or a single 2D image containing the + weight data. + all_wcs (`astropy.wcs.WCS`_, list): + A list of WCS objects, or a single WCS object containing the WCS + information of each image. tilts (`numpy.ndarray`_, list): 2D wavelength tilts frame, or a list of tilt frames (see all_idx) slits (:class:`~pypeit.slittrace.SlitTraceSet`, list): @@ -1161,27 +1156,28 @@ def generate_image_subpixel(image_wcs, all_ra, all_dec, all_wave, all_sci, all_i all_dar (:class:`~pypeit.coadd3d.DARcorrection`, list): A Class containing the DAR correction information, or a list of DARcorrection classes. If a list, it must be the same length as astrom_trans. - bins (tuple): - A 3-tuple (x,y,z) containing the histogram bin edges in x,y spatial - and z wavelength coordinates - all_idx (`numpy.ndarray`_, optional): - If tilts, slits, and astrom_trans are lists, this should contain a - 1D flattened array, of the same length as all_sci, containing the - index the tilts, slits, and astrom_trans lists that corresponds to - each pixel. Note that, in this case all of these lists need to be - the same length. + ra_offset (:obj:`float`, list): + The RA offset to apply to each image, or a list of RA offsets. + dec_offset (:obj:`float`, list): + The DEC offset to apply to each image, or a list of DEC offsets. spec_subpixel (:obj:`int`, optional): What is the subpixellation factor in the spectral direction. Higher values give more reliable results, but note that the time required - goes as (``spec_subpixel * spat_subpixel``). The default value is 5, - which divides each detector pixel into 5 subpixels in the spectral - direction. + goes as (``spec_subpixel * spat_subpixel * slice_subpixel``). The + default value is 5, which divides each detector pixel into 5 subpixels + in the spectral direction. spat_subpixel (:obj:`int`, optional): What is the subpixellation factor in the spatial direction. Higher values give more reliable results, but note that the time required - goes as (``spec_subpixel * spat_subpixel``). The default value is 5, - which divides each detector pixel into 5 subpixels in the spatial - direction. + goes as (``spec_subpixel * spat_subpixel * slice_subpixel``). The + default value is 5, which divides each detector pixel into 5 subpixels + in the spatial direction. + slice_subpixel (:obj:`int`, optional): + What is the subpixellation factor in the slice direction. Higher + values give more reliable results, but note that the time required + goes as (``spec_subpixel * spat_subpixel * slice_subpixel``). The + default value is 5, which divides each IFU slice into 5 subpixels + in the slice direction. combine (:obj:`bool`, optional): If True, all of the input frames will be combined into a single output. Otherwise, individual images will be generated. @@ -1190,35 +1186,38 @@ def generate_image_subpixel(image_wcs, all_ra, all_dec, all_wave, all_sci, all_i `numpy.ndarray`_: The white light images for all frames """ # Perform some checks on the input -- note, more complete checks are performed in subpixellate() - _all_idx = np.zeros(all_sci.size) if all_idx is None else all_idx - if combine: - numfr = 1 - else: - numfr = np.unique(_all_idx).size - if len(tilts) != numfr or len(slits) != numfr or len(astrom_trans) != numfr or len(all_dar) != numfr: - msgs.error("The following arguments must be the same length as the expected number of frames to be combined:" - + msgs.newline() + "tilts, slits, astrom_trans, all_dar") + if not combine: + # If we are not combining, then all of the inputs must be lists + list_inputs = [sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset] + if all([isinstance(l, list) for l in list_inputs]): + # If all of the inputs are lists, then they must all have the same length + numframes = len(sciImg) + if not all([len(l) == numframes for l in list_inputs]): + msgs.error("All input lists must have the same length if they are not being combined") + elif all([not isinstance(l, list) for l in list_inputs]): + # Just a single frame - store as lists for this code + _sciImg, _ivarImg, _waveImg, _gpmImg, _wghtImg, _all_wcs, _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset = \ + [sciImg], [ivarImg], [waveImg], [slitid_img_gpm], [wghtImg], [all_wcs], [tilts], [slits], [astrom_trans], [all_dar], [ra_offset], [dec_offset] + numframes = 1 + # Prepare the array of white light images to be stored numra = bins[0].size-1 numdec = bins[1].size-1 - all_wl_imgs = np.zeros((numra, numdec, numfr)) + all_wl_imgs = np.zeros((numra, numdec, numframes)) # Loop through all frames and generate white light images - for fr in range(numfr): - msgs.info(f"Creating image {fr+1}/{numfr}") + for fr in range(numframes): + msgs.info(f"Creating image {fr+1}/{numframes}") if combine: # Subpixellate - img, _, _ = subpixellate(image_wcs, all_ra, all_dec, all_wave, - all_sci, all_ivar, all_wghts, all_spatpos, - all_specpos, all_spatid, tilts, slits, astrom_trans, all_dar, bins, - spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, all_idx=_all_idx) + img, _, _ = subpixellate(image_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, + all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, + spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, slice_subpixel=slice_subpixel) else: - ww = np.where(_all_idx == fr) # Subpixellate - img, _, _ = subpixellate(image_wcs, all_ra[ww], all_dec[ww], all_wave[ww], - all_sci[ww], all_ivar[ww], all_wghts[ww], all_spatpos[ww], - all_specpos[ww], all_spatid[ww], tilts[fr], slits[fr], astrom_trans[fr], - all_dar[fr], bins, spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel) + img, _, _ = subpixellate(image_wcs, bins, sciImg[fr], ivarImg[fr], waveImg[fr], slitid_img_gpm[fr], wghtImg[fr], + all_wcs[fr], tilts[fr], slits[fr], astrom_trans[fr], all_dar[fr], ra_offset[fr], dec_offset[fr], + spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, slice_subpixel=slice_subpixel) all_wl_imgs[:, :, fr] = img[:, :, 0] # Return the constructed white light images return all_wl_imgs From 1475e6dc9a0222a9dbc1e0539bd0cc479f5cf92b Mon Sep 17 00:00:00 2001 From: rcooke Date: Sat, 9 Dec 2023 20:53:55 +0000 Subject: [PATCH 090/244] refactor white light pixels --- pypeit/core/datacube.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index a2a7c8134d..b52e1eaddb 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -447,7 +447,8 @@ def get_whitelight_pixels(all_wave, all_slitid, min_wl, max_wl): Maximum wavelength to consider Returns: - :obj:`tuple`: The first element of the tuple is a list of `numpy.ndarray`_ slitid images, + :obj:`tuple`: The first element of the tuple is a list of `numpy.ndarray`_ slitid images + (or a single `numpy.ndarray`_ slitid image if only one spec2d frame is provided), shape is (nspec, nspat), where a zero value corresponds to an excluded pixel (either outside the desired wavelength range, a bad pixel, a pixel not on the slit). All other pixels have a value equal to the slit number. The second element of the tuple @@ -491,7 +492,9 @@ def get_whitelight_pixels(all_wave, all_slitid, min_wl, max_wl): max_wl = tmp_max # Determine the wavelength range wavediff = max_wl - min_wl - return out_slitid, wavediff + # Need to return a single slitid image if only one frame, otherwise return a list of slitid images. + # Also return the wavelength difference + return out_slitid[0] if numframes == 1 else out_slitid, wavediff def get_whitelight_range(wavemin, wavemax, wl_range): From 209bf853d95567abc5d6cdc14a007a14740473e4 Mon Sep 17 00:00:00 2001 From: rcooke Date: Sat, 9 Dec 2023 21:49:19 +0000 Subject: [PATCH 091/244] refactor wcs bounds --- pypeit/core/datacube.py | 189 ++++++++++++++++++++++++---------------- 1 file changed, 112 insertions(+), 77 deletions(-) diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index b52e1eaddb..5696019866 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -700,24 +700,56 @@ def set_voxel_sampling(spatscale, specscale, dspat=None, dwv=None): return _dspat, _dwv -def wcs_bounds(all_ra, all_dec, all_wave, ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None): +def check_inputs(list_inputs): + """ + This function checks the inputs to several of the cube building routines, and makes sure they are all consistent. + Often, this is to make check if all inputs are lists of the same length, or if all inputs are 2D `numpy.ndarray`_. + The goal of the routine is to return a consistent set of lists of the input. + + Parameters + ---------- + list_inputs : :obj:`list` + A list of inputs to check. + + Returns + ------- + list_inputs : :obj:`list` + A list of inputs that have been checked for consistency. + """ + if all([isinstance(l, list) for l in list_inputs]): + # Several frames are being combined. Check the lists have the same length + numframes = len(list_inputs[0]) + if not all([len(l) == numframes for l in list_inputs]): + msgs.error("All input lists must have the same length") + # The inputs are good, return as is + return tuple(list_inputs) + elif all([not isinstance(l, list) for l in list_inputs]): + # Just a single frame - store as single element lists + ret_list = () + for l in list_inputs: + ret_list += ([l],) + return ret_list + else: + msgs.error("The input arguments should all be of type 'list', or all not be of type 'list':") + + +def wcs_bounds(raImg, decImg, waveImg, slitid_img_gpm, ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None): """ Calculate the bounds of the WCS and the expected edges of the voxels, based - on user-specified parameters or the extremities of the data. This is a - convenience function that calls the core function in - :mod:`~pypeit.core.datacube`. + on user-specified parameters or the extremities of the data. Parameters ---------- - all_ra : `numpy.ndarray`_ - 1D flattened array containing the RA values of each pixel from all - spec2d files - all_dec : `numpy.ndarray`_ - 1D flattened array containing the DEC values of each pixel from all - spec2d files - all_wave : `numpy.ndarray`_ - 1D flattened array containing the wavelength values of each pixel from - all spec2d files + raImg : (`numpy.ndarray`_, list): + A list of 2D array containing the RA of each pixel, with shape (nspec, nspat) + decImg : (`numpy.ndarray`_, list): + A list of 2D array containing the Dec of each pixel, with shape (nspec, nspat) + waveImg (`numpy.ndarray`_, list): + A list of 2D array containing the wavelength of each pixel, with shape (nspec, nspat) + slitid_img_gpm : (`numpy.ndarray`_, list): + A list of 2D array containing the spat ID of each pixel, with shape (nspec, nspat). + A value of 0 indicates that the pixel is not on a slit. All other values indicate the + slit spatial ID. ra_min : :obj:`float`, optional Minimum RA of the WCS ra_max : :obj:`float`, optional @@ -740,23 +772,49 @@ def wcs_bounds(all_ra, all_dec, all_wave, ra_min=None, ra_max=None, dec_min=None _dec_min : :obj:`float` Minimum Dec of the WCS _dec_max : :obj:`float` - Maximum RA of the WCS - _wav_min : :obj:`float` + Maximum Dec of the WCS + _wave_min : :obj:`float` Minimum wavelength of the WCS - _wav_max : :obj:`float` - Maximum RA of the WCS + _wave_max : :obj:`float` + Maximum wavelength of the WCS """ - # Setup the cube ranges - _ra_min = ra_min if ra_min is not None else np.min(all_ra) - _ra_max = ra_max if ra_max is not None else np.max(all_ra) - _dec_min = dec_min if dec_min is not None else np.min(all_dec) - _dec_max = dec_max if dec_max is not None else np.max(all_dec) - _wav_min = wave_min if wave_min is not None else np.min(all_wave) - _wav_max = wave_max if wave_max is not None else np.max(all_wave) - return _ra_min, _ra_max, _dec_min, _dec_max, _wav_min, _wav_max - - -def create_wcs(all_ra, all_dec, all_wave, dspat, dwave, + # Check the inputs + _raImg, _decImg, _waveImg, _slitid_img_gpm = check_inputs([raImg, decImg, waveImg, slitid_img_gpm]) + numframes = len(raImg) + + # Loop over the frames and get the bounds - start by setting the default values + _ra_min, _ra_max = ra_min, ra_max + _dec_min, _dec_max = dec_min, dec_max + _wave_min, _wave_max = wave_min, wave_max + for fr in range(numframes): + # Only do calculations if the min/max inputs are not specified + # Get the RA, Dec, and wavelength of the pixels on the slit + if ra_min is None or ra_max is None: + this_ra = _raImg[fr][_slitid_img_gpm[fr] > 0] + tmp_min, tmp_max = np.min(this_ra), np.max(this_ra) + if fr == 0 or tmp_min < _ra_min: + _ra_min = tmp_min + if fr == 0 or tmp_max > _ra_max: + _ra_max = tmp_max + if dec_min is None or dec_max is None: + this_dec = _decImg[fr][_slitid_img_gpm[fr] > 0] + tmp_min, tmp_max = np.min(this_dec), np.max(this_dec) + if fr == 0 or tmp_min < _dec_min: + _dec_min = tmp_min + if fr == 0 or tmp_max > _dec_max: + _dec_max = tmp_max + if wave_min is None or wave_max is None: + this_wave = _waveImg[fr][_slitid_img_gpm[fr] > 0] + tmp_min, tmp_max = np.min(this_wave), np.max(this_wave) + if fr == 0 or tmp_min < _wave_min: + _wave_min = tmp_min + if fr == 0 or tmp_max > _wave_max: + _wave_max = tmp_max + # Return the bounds + return _ra_min, _ra_max, _dec_min, _dec_max, _wave_min, _wave_max + + +def create_wcs(raImg, decImg, waveImg, slitid_img_gpm, dspat, dwave, ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None, reference=None, collapse=False, equinox=2000.0, specname="PYP_SPEC"): """ @@ -765,15 +823,16 @@ def create_wcs(all_ra, all_dec, all_wave, dspat, dwave, Parameters ---------- - all_ra : `numpy.ndarray`_ - 1D flattened array containing the RA values of each pixel from all - spec2d files - all_dec : `numpy.ndarray`_ - 1D flattened array containing the DEC values of each pixel from all - spec2d files - all_wave : `numpy.ndarray`_ - 1D flattened array containing the wavelength values of each pixel from - all spec2d files + raImg : (`numpy.ndarray`_, list): + A list of 2D array containing the RA of each pixel, with shape (nspec, nspat) + decImg : (`numpy.ndarray`_, list): + A list of 2D array containing the Dec of each pixel, with shape (nspec, nspat) + waveImg (`numpy.ndarray`_, list): + A list of 2D array containing the wavelength of each pixel, with shape (nspec, nspat) + slitid_img_gpm : (`numpy.ndarray`_, list): + A list of 2D array containing the spat ID of each pixel, with shape (nspec, nspat). + A value of 0 indicates that the pixel is not on a slit. All other values indicate the + slit spatial ID. dspat : float Spatial size of each square voxel (in arcsec). The default is to use the values in cubepar. @@ -811,14 +870,14 @@ def create_wcs(all_ra, all_dec, all_wave, dspat, dwave, reference_image : `numpy.ndarray`_ The reference image to be used for the cross-correlation. Can be None. """ - # Grab cos(dec) for convenience - cosdec = np.cos(np.mean(all_dec) * np.pi / 180.0) - # Setup the cube ranges _ra_min, _ra_max, _dec_min, _dec_max, _wav_min, _wav_max = \ - wcs_bounds(all_ra, all_dec, all_wave, ra_min=ra_min, ra_max=ra_max, dec_min=dec_min, dec_max=dec_max, + wcs_bounds(raImg, decImg, waveImg, slitid_img_gpm, ra_min=ra_min, ra_max=ra_max, dec_min=dec_min, dec_max=dec_max, wave_min=wave_min, wave_max=wave_max) + # Grab cos(dec) for convenience. Use the average of the min and max dec + cosdec = np.cos(0.5*(_dec_min+_dec_max) * np.pi / 180.0) + # Number of voxels in each dimension numra = int((_ra_max - _ra_min) * cosdec / dspat) numdec = int((_dec_max - _dec_min) / dspat) @@ -826,8 +885,6 @@ def create_wcs(all_ra, all_dec, all_wave, dspat, dwave, # If a white light WCS is being generated, make sure there's only 1 wavelength bin if collapse: - _wav_min = np.min(all_wave) - _wav_max = np.max(all_wave) dwave = _wav_max - _wav_min numwav = 1 @@ -1189,19 +1246,9 @@ def generate_image_subpixel(image_wcs, bins, sciImg, ivarImg, waveImg, slitid_im `numpy.ndarray`_: The white light images for all frames """ # Perform some checks on the input -- note, more complete checks are performed in subpixellate() - if not combine: - # If we are not combining, then all of the inputs must be lists - list_inputs = [sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset] - if all([isinstance(l, list) for l in list_inputs]): - # If all of the inputs are lists, then they must all have the same length - numframes = len(sciImg) - if not all([len(l) == numframes for l in list_inputs]): - msgs.error("All input lists must have the same length if they are not being combined") - elif all([not isinstance(l, list) for l in list_inputs]): - # Just a single frame - store as lists for this code - _sciImg, _ivarImg, _waveImg, _gpmImg, _wghtImg, _all_wcs, _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset = \ - [sciImg], [ivarImg], [waveImg], [slitid_img_gpm], [wghtImg], [all_wcs], [tilts], [slits], [astrom_trans], [all_dar], [ra_offset], [dec_offset] - numframes = 1 + _sciImg, _ivarImg, _waveImg, _slitid_img_gpm, _wghtImg, _all_wcs, _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset = \ + check_inputs([sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset]) + numframes = len(_sciImg) # Prepare the array of white light images to be stored numra = bins[0].size-1 @@ -1213,13 +1260,13 @@ def generate_image_subpixel(image_wcs, bins, sciImg, ivarImg, waveImg, slitid_im msgs.info(f"Creating image {fr+1}/{numframes}") if combine: # Subpixellate - img, _, _ = subpixellate(image_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, - all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, + img, _, _ = subpixellate(image_wcs, bins, _sciImg, _ivarImg, _waveImg, _slitid_img_gpm, _wghtImg, + _all_wcs, _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset, spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, slice_subpixel=slice_subpixel) else: # Subpixellate - img, _, _ = subpixellate(image_wcs, bins, sciImg[fr], ivarImg[fr], waveImg[fr], slitid_img_gpm[fr], wghtImg[fr], - all_wcs[fr], tilts[fr], slits[fr], astrom_trans[fr], all_dar[fr], ra_offset[fr], dec_offset[fr], + img, _, _ = subpixellate(image_wcs, bins, _sciImg[fr], _ivarImg[fr], _waveImg[fr], _slitid_img_gpm[fr], _wghtImg[fr], + _all_wcs[fr], _tilts[fr], _slits[fr], _astrom_trans[fr], _all_dar[fr], _ra_offset[fr], _dec_offset[fr], spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, slice_subpixel=slice_subpixel) all_wl_imgs[:, :, fr] = img[:, :, 0] # Return the constructed white light images @@ -1445,23 +1492,11 @@ def subpixellate(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wgh variance cube, (3) the corresponding bad pixel mask cube, and (4) the residual cube. The latter is only returned if debug is True. """ - # Check for combinations of lists or not - list_inputs = [sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset] - if all([isinstance(l, list) for l in list_inputs]): - # Several frames are being combined. Check the lists have the same length - numframes = len(sciImg) - if not all([len(l) == numframes for l in list_inputs]): - msgs.error("All input lists must have the same length when combining multiple frames") - # Store in the following variables - _sciImg, _ivarImg, _waveImg, _gpmImg, _wghtImg, _all_wcs, _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset = \ - sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset - elif all([not isinstance(l, list) for l in list_inputs]): - # Just a single frame - store as lists for this code - _sciImg, _ivarImg, _waveImg, _gpmImg, _wghtImg, _all_wcs, _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset = \ - [sciImg], [ivarImg], [waveImg], [slitid_img_gpm], [wghtImg], [all_wcs], [tilts], [slits], [astrom_trans], [all_dar], [ra_offset], [dec_offset] - numframes = 1 - else: - msgs.error("The input arguments should all be of type 'list', or all not be of type 'list':") + # Check the inputs for combinations of lists or not + _sciImg, _ivarImg, _waveImg, _gpmImg, _wghtImg, _all_wcs, _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset = \ + check_inputs([sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset]) + numframes = len(_sciImg) + # Prepare the output arrays outshape = (bins[0].size-1, bins[1].size-1, bins[2].size-1) binrng = [[bins[0][0], bins[0][-1]], [bins[1][0], bins[1][-1]], [bins[2][0], bins[2][-1]]] From c901e56c79eb4ac9a04884ceb4fd1a5628bd9e96 Mon Sep 17 00:00:00 2001 From: rcooke Date: Sat, 9 Dec 2023 21:51:19 +0000 Subject: [PATCH 092/244] refactor wcs bounds --- pypeit/core/datacube.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index 5696019866..e4325bd683 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -757,11 +757,11 @@ def wcs_bounds(raImg, decImg, waveImg, slitid_img_gpm, ra_min=None, ra_max=None, dec_min : :obj:`float`, optional Minimum Dec of the WCS dec_max : :obj:`float`, optional - Maximum RA of the WCS - wav_min : :obj:`float`, optional + Maximum Dec of the WCS + wave_min : :obj:`float`, optional Minimum wavelength of the WCS - wav_max : :obj:`float`, optional - Maximum RA of the WCS + wave_max : :obj:`float`, optional + Maximum wavelength of the WCS Returns ------- @@ -871,7 +871,7 @@ def create_wcs(raImg, decImg, waveImg, slitid_img_gpm, dspat, dwave, The reference image to be used for the cross-correlation. Can be None. """ # Setup the cube ranges - _ra_min, _ra_max, _dec_min, _dec_max, _wav_min, _wav_max = \ + _ra_min, _ra_max, _dec_min, _dec_max, _wave_min, _wave_max = \ wcs_bounds(raImg, decImg, waveImg, slitid_img_gpm, ra_min=ra_min, ra_max=ra_max, dec_min=dec_min, dec_max=dec_max, wave_min=wave_min, wave_max=wave_max) @@ -881,15 +881,15 @@ def create_wcs(raImg, decImg, waveImg, slitid_img_gpm, dspat, dwave, # Number of voxels in each dimension numra = int((_ra_max - _ra_min) * cosdec / dspat) numdec = int((_dec_max - _dec_min) / dspat) - numwav = int(np.round((_wav_max - _wav_min) / dwave)) + numwav = int(np.round((_wave_max - _wave_min) / dwave)) # If a white light WCS is being generated, make sure there's only 1 wavelength bin if collapse: - dwave = _wav_max - _wav_min + dwave = _wave_max - _wave_min numwav = 1 # Generate a master WCS to register all frames - coord_min = [_ra_min, _dec_min, _wav_min] + coord_min = [_ra_min, _dec_min, _wave_min] coord_dlt = [dspat, dspat, dwave] # If a reference image is being used and a white light image is requested (collapse=True) update the celestial parts @@ -907,7 +907,7 @@ def create_wcs(raImg, decImg, waveImg, slitid_img_gpm, dspat, dwave, msgs.newline() + "Parameters of the WCS:" + msgs.newline() + "RA min = {0:f}".format(coord_min[0]) + msgs.newline() + "DEC min = {0:f}".format(coord_min[1]) + - msgs.newline() + "WAVE min, max = {0:f}, {1:f}".format(_wav_min, _wav_max) + + msgs.newline() + "WAVE min, max = {0:f}, {1:f}".format(_wave_min, _wave_max) + msgs.newline() + "Spaxel size = {0:f} arcsec".format(3600.0 * dspat) + msgs.newline() + "Wavelength step = {0:f} A".format(dwave) + msgs.newline() + "-" * 40) From aa7071730b743ee69c4c21ed1ddae766242c5b3b Mon Sep 17 00:00:00 2001 From: rcooke Date: Sat, 9 Dec 2023 21:54:01 +0000 Subject: [PATCH 093/244] refactor align, wcs bounds, and creation of wcs --- pypeit/coadd3d.py | 148 +++++++++++++++++++++++++++------------------- 1 file changed, 86 insertions(+), 62 deletions(-) diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index 93336ea0a8..91af6bcdf4 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -373,10 +373,10 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs same length as spec2dfiles. ra_offsets (:obj:`list`, optional): If not None, this should be a list of relative RA offsets of each frame. It should be the - same length as spec2dfiles. + same length as spec2dfiles. The units should be degrees. dec_offsets (:obj:`list`, optional): If not None, this should be a list of relative Dec offsets of each frame. It should be the - same length as spec2dfiles. + same length as spec2dfiles. The units should be degrees. spectrograph (:obj:`str`, :class:`~pypeit.spectrographs.spectrograph.Spectrograph`, optional): The name or instance of the spectrograph used to obtain the data. If None, this is pulled from the file header. @@ -408,11 +408,27 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs if dec_offsets is not None: if len(dec_offsets) != self.numfiles: msgs.error("The dec_offsets list should be identical length to the spec2dfiles list") - # Set the frame-specific options + # Make sure both ra_offsets and dec_offsets are either both None or both lists + if ra_offsets is None and dec_offsets is not None: + msgs.error("If you provide dec_offsets, you must also provide ra_offsets") + if ra_offsets is not None and dec_offsets is None: + msgs.error("If you provide ra_offsets, you must also provide dec_offsets") + # Set the frame specific options self.skysub_frame = skysub_frame self.scale_corr = scale_corr self.ra_offsets = np.array(ra_offsets) if isinstance(ra_offsets, list) else ra_offsets self.dec_offsets = np.array(dec_offsets) if isinstance(dec_offsets, list) else dec_offsets + # If no ra_offsets or dec_offsets have been provided, initialise the lists + self.user_alignment = True + if self.ra_offsets is None and self.dec_offsets is None: + msgs.info("No RA or Dec offsets have been provided.") + if self.align: + msgs.info("An automatic alignment will be performed using WCS information from the headers.") + # User offsets are not provided, so turn off the user_alignment + self.user_alignment = False + # Initialise the lists of ra_offsets and dec_offsets + self.ra_offsets = np.zeros(self.numfiles) + self.dec_offsets = np.zeros(self.numfiles) # Check on Spectrograph input if spectrograph is None: @@ -432,7 +448,7 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs self.all_sci, self.all_ivar, self.all_wave, self.all_slitid, self.all_wghts = [], [], [], [], [] self.all_tilts, self.all_slits, self.all_align = [], [], [] - self.all_wcs, self.all_dar = [], [] + self.all_wcs, self.all_ra, self.all_dec, self.all_dar = [], [], [], [] self.weights = np.ones(self.numfiles) # Weights to use when combining cubes self._dspat = None if self.cubepar['spatial_delta'] is None else self.cubepar['spatial_delta'] / 3600.0 # binning size on the sky (/3600 to convert to degrees) @@ -901,6 +917,7 @@ def get_alignments(self, spec2DObj, slits, spat_flexure=None): def load(self): """ + TODO :: Update all of the docstrings! This is the main function that loads in the data, and performs several frame-specific corrections. If the user does not wish to align or combine the individual datacubes, then this routine will also produce a spec3d file, which is a DataCube representation of a PypeIt spec2d frame for SlicerIFU data. @@ -1038,15 +1055,18 @@ def load(self): # TODO: This should use the mask function to figure out which elements are masked. onslit_gpm = (slitid_img_init > 0) & (bpmmask.mask == 0) & sky_is_good - # Grab the WCS of this frame - crval_wv = self.cubepar['wave_min'] if self.cubepar['wave_min'] is not None else wave0 - cd_wv = self.cubepar['wave_delta'] if self.cubepar['wave_delta'] is not None else dwv - self.all_wcs.append(self.spec.get_wcs(spec2DObj.head0, slits, detector.platescale, crval_wv, cd_wv)) - # Generate the alignment splines, and then retrieve images of the RA and Dec of every pixel, # and the number of spatial pixels in each slit alignSplines = self.get_alignments(spec2DObj, slits, spat_flexure=spat_flexure) + # Grab the WCS of this frame, and generate the RA and Dec images + # NOTE :: These RA and Dec images are only used to setup the WCS of the datacube. The actual RA and Dec + # of each pixel in the datacube is calculated in the datacube.subpixellate() method. + crval_wv = self.cubepar['wave_min'] if self.cubepar['wave_min'] is not None else wave0 + cd_wv = self.cubepar['wave_delta'] if self.cubepar['wave_delta'] is not None else dwv + self.all_wcs.append(self.spec.get_wcs(spec2DObj.head0, slits, detector.platescale, crval_wv, cd_wv)) + ra_img, dec_img, minmax = slits.get_radec_image(self.all_wcs[ff], alignSplines, spec2DObj.tilts, initial=True, flexure=spat_flexure) + # Extract wavelength and delta wavelength arrays from the images wave_ext = waveimg[onslit_gpm] dwav_ext = dwaveimg[onslit_gpm] @@ -1069,9 +1089,9 @@ def load(self): # Compute the extinction correction msgs.info("Applying extinction correction") - longitude = self.spec.telescope['longitude'] - latitude = self.spec.telescope['latitude'] - extinct = flux_calib.load_extinction_data(longitude, latitude, self.senspar['UVIS']['extinct_file']) + extinct = flux_calib.load_extinction_data(self.spec.telescope['longitude'], + self.spec.telescope['latitude'], + self.senspar['UVIS']['extinct_file']) # extinction_correction requires the wavelength is sorted extcorr_sort = flux_calib.extinction_correction(wave_sort * units.AA, airmass, extinct) @@ -1121,25 +1141,19 @@ def load(self): # Get the coordinate bounds slitlength = int(np.round(np.median(slits.get_slitlengths(initial=True, median=True)))) numwav = int((np.max(waveimg) - wave0) / dwv) - _, _, minmax = slits.get_radec_image(self.all_wcs[ff], alignSplines, spec2DObj.tilts, initial=True, flexure=spat_flexure) raw_bins = self.spec.get_datacube_bins(slitlength, minmax, numwav) wghts = np.ones(sciImg.shape) # Now do the alignment - _ra_sort, _dec_sort = hst_alignment(sciImg, ivar, waveimg, slitid_img_gpm, wghts, - #np.max(self._spatscale[ff,:]), self._specscale[ff], - self.all_wcs[ff], spec2DObj.tilts, slits, alignSplines, darcorr, - raw_bins=raw_bins) - self.ra_offsets[ff] = np.median(_ra_sort-ra_sort) - self.dec_offsets[ff] = np.median(_dec_sort-dec_sort) - ra_sort = _ra_sort - dec_sort = _dec_sort - self.ifu_ra[ff] += self.ra_offsets[ff] - self.ifu_dec[ff] += self.dec_offsets[ff] + self.ra_offsets[ff], self.dec_offsets[ff] = hst_alignment(sciImg, ivar, waveimg, slitid_img_gpm, wghts, + self.all_wcs[ff], spec2DObj.tilts, slits, alignSplines, darcorr, + raw_bins=raw_bins) + # TODO :: Does this need to be included anywhere? + # self.ifu_ra[ff] += self.ra_offsets[ff] + # self.ifu_dec[ff] += self.dec_offsets[ff] ################################## # If individual frames are to be output without aligning them, # there's no need to store information, just make the cubes now - numpix = ra_sort.size if not self.combine and not self.align: # Get the output filename if self.numfiles == 1 and self.cubepar['output_filename'] != "": @@ -1163,6 +1177,7 @@ def load(self): datacube.generate_cube_subpixel(outfile, self.all_wcs[ff], bins, sciImg, ivar, waveimg, slitid_img_gpm, wghts, self.all_wcs[ff], spec2DObj.tilts, slits, alignSplines, darcorr, + self.ra_offsets[ff], self.dec_offsets[ff], overwrite=self.overwrite, whitelight_range=wl_wvrng, spec_subpixel=self.spec_subpixel, spat_subpixel=self.spat_subpixel, @@ -1185,6 +1200,8 @@ def load(self): self.all_sci.append(sciImg.copy()) self.all_ivar.append(ivar.copy()) self.all_wave.append(waveimg.copy()) + self.all_ra.append(ra_img.copy()) + self.all_dec.append(dec_img.copy()) self.all_slitid.append(slitid_img_gpm.copy()) self.all_wghts.append(wghts.copy()) self.all_tilts.append(spec2DObj.tilts) @@ -1201,29 +1218,28 @@ def run_align(self): `numpy.ndarray`_: A new set of Dec values that has been aligned """ # Grab cos(dec) for convenience - cosdec = np.cos(np.mean(self.all_dec) * np.pi / 180.0) + cosdec = np.cos(np.mean(self.ifu_dec[0]) * np.pi / 180.0) + # Initialize the RA and Dec offset arrays + ra_offsets, dec_offsets = np.zeros(self.numfiles), np.zeros(self.numfiles) # Register spatial offsets between all frames - if self.ra_offsets is not None: - # Calculate the offsets - new_ra, new_dec = datacube.align_user_offsets(self.all_ra, self.all_dec, self.all_idx, - self.ifu_ra, self.ifu_dec, - self.ra_offsets, self.dec_offsets) + if self.user_alignment: + # The user has specified offsets - update these values accounting for the difference in header RA/DEC + ra_offsets, dec_offsets = datacube.align_user_offsets(self.ifu_ra, self.ifu_dec, + self.ra_offsets, self.dec_offsets) else: - new_ra, new_dec = self.all_ra.copy(), self.all_dec.copy() # Find the wavelength range where all frames overlap min_wl, max_wl = datacube.get_whitelight_range(np.max(self.mnmx_wv[:, :, 0]), # The max blue wavelength np.min(self.mnmx_wv[:, :, 1]), # The min red wavelength self.cubepar['whitelight_range']) # The user-specified values (if any) - # Get the good whitelight pixels - ww, wavediff = datacube.get_whitelight_pixels(self.all_wave, min_wl, max_wl) + # Get the good white light pixels + slitid_img_gpm, wavediff = datacube.get_whitelight_pixels(self.all_wave, self.all_slitid, min_wl, max_wl) # Iterate over white light image generation and spatial shifting numiter = 2 for dd in range(numiter): msgs.info(f"Iterating on spatial translation - ITERATION #{dd+1}/{numiter}") - ref_idx = None # Don't use an index - This is the default behaviour when a reference image is supplied # Generate the WCS image_wcs, voxedge, reference_image = \ - datacube.create_wcs(new_ra[ww], new_dec[ww], self.all_wave[ww], self._dspat, wavediff, + datacube.create_wcs(self.all_ra, self.all_dec, self.all_wave, slitid_img_gpm, self._dspat, wavediff, ra_min=self.cubepar['ra_min'], ra_max=self.cubepar['ra_max'], dec_min=self.cubepar['dec_min'], dec_max=self.cubepar['dec_max'], wave_min=self.cubepar['wave_min'], wave_max=self.cubepar['wave_max'], @@ -1232,12 +1248,13 @@ def run_align(self): if voxedge[2].size != 2: msgs.error("Spectral range for WCS is incorrect for white light image") - wl_imgs = datacube.generate_image_subpixel(image_wcs, new_ra[ww], new_dec[ww], self.all_wave[ww], - self.all_sci[ww], self.all_ivar[ww], self.all_wghts[ww], - self.all_spatpos[ww], self.all_specpos[ww], self.all_spatid[ww], + wl_imgs = datacube.generate_image_subpixel(image_wcs, voxedge, self.all_sci, self.all_ivar, self.all_wave, + slitid_img_gpm, self.all_wghts, self.all_wcs, self.all_tilts, self.all_slits, self.all_align, self.all_dar, - voxedge, all_idx=self.all_idx[ww], - spec_subpixel=self.spec_subpixel, spat_subpixel=self.spat_subpixel) + ra_offsets, dec_offsets, + spec_subpixel=self.spec_subpixel, + spat_subpixel=self.spat_subpixel, + slice_subpixel=self.slice_subpixel) if reference_image is None: # ref_idx will be the index of the cube with the highest S/N ref_idx = np.argmax(self.weights) @@ -1255,10 +1272,10 @@ def run_align(self): dec_shift *= self._dspat msgs.info("Spatial shift of cube #{0:d}:".format(ff + 1) + msgs.newline() + "RA, DEC (arcsec) = {0:+0.3f} E, {1:+0.3f} N".format(ra_shift*3600.0, dec_shift*3600.0)) - # Apply the shift - new_ra[self.all_idx == ff] += ra_shift - new_dec[self.all_idx == ff] += dec_shift - return new_ra, new_dec + # Store the shift in the RA and DEC offsets in degrees + ra_offsets[ff] += ra_shift + dec_offsets[ff] += dec_shift + return ra_offsets, dec_offsets def compute_weights(self): """ @@ -1325,14 +1342,15 @@ def run(self): # Align the frames # TODO :: NOTE THAT THIS IS 'and False' because the HST alignment is done when reading in the data. if self.align and False: - self.all_ra, self.all_dec = self.run_align() + self.ra_offsets, self.dec_offsets = self.run_align() # Compute the relative weights on the spectra + # TODO :: update this with slice refactor self.all_wghts = self.compute_weights() # Generate the WCS, and the voxel edges cube_wcs, vox_edges, _ = \ - datacube.create_wcs(self.all_ra, self.all_dec, self.all_wave, self._dspat, self._dwv, + datacube.create_wcs(self.all_ra, self.all_dec, self.all_wave, self.all_slitid, self._dspat, self._dwv, ra_min=self.cubepar['ra_min'], ra_max=self.cubepar['ra_max'], dec_min=self.cubepar['dec_min'], dec_max=self.cubepar['dec_max'], wave_min=self.cubepar['wave_min'], wave_max=self.cubepar['wave_max'], @@ -1359,12 +1377,15 @@ def run(self): outfile = datacube.get_output_filename("", self.cubepar['output_filename'], True, -1) # Generate the datacube flxcube, sigcube, bpmcube, wave = \ - datacube.generate_cube_subpixel(outfile, cube_wcs, self.all_ra, self.all_dec, self.all_wave, - self.all_sci, self.all_ivar, np.ones(self.all_wghts.size), - self.all_spatpos, self.all_specpos, self.all_spatid, - self.all_tilts, self.all_slits, self.all_align, self.all_dar, vox_edges, - all_idx=self.all_idx, overwrite=self.overwrite, whitelight_range=wl_wvrng, - spec_subpixel=self.spec_subpixel, spat_subpixel=self.spat_subpixel) + datacube.generate_cube_subpixel(outfile, cube_wcs, vox_edges, + self.all_sci, self.all_ivar, self.all_wave, + self.all_slitid, self.all_wghts, self.all_wcs, + self.all_tilts, self.all_slits, self.all_align, self.all_dar, + self.ra_offsets, self.dec_offsets, + overwrite=self.overwrite, whitelight_range=wl_wvrng, + spec_subpixel=self.spec_subpixel, + spat_subpixel=self.spat_subpixel, + slice_subpixel=self.slice_subpixel) # Prepare the header hdr = cube_wcs.to_header() if self.fluxcal: @@ -1379,15 +1400,17 @@ def run(self): else: for ff in range(self.numfiles): outfile = datacube.get_output_filename("", self.cubepar['output_filename'], False, ff) - ww = np.where(self.all_idx == ff) # Generate the datacube flxcube, sigcube, bpmcube, wave = \ - datacube.generate_cube_subpixel(outfile, cube_wcs, self.all_ra[ww], self.all_dec[ww], self.all_wave[ww], - self.all_sci[ww], self.all_ivar[ww], np.ones(ww[0].size), - self.all_spatpos[ww], self.all_specpos[ww], self.all_spatid[ww], - self.all_tilts[ff], self.all_slits[ff], self.all_align[ff], self.all_dar[ff], vox_edges, - all_idx=self.all_idx[ww], overwrite=self.overwrite, whitelight_range=wl_wvrng, - spec_subpixel=self.spec_subpixel, spat_subpixel=self.spat_subpixel) + datacube.generate_cube_subpixel(outfile, cube_wcs, vox_edges, + self.all_sci[ff], self.all_ivar[ff], self.all_wave[ff], + self.all_slitid[ff], self.all_wghts[ff], self.all_wcs[ff], + self.all_tilts[ff], self.all_slits[ff], self.all_align[ff], self.all_dar[ff], + self.ra_offsets[ff], self.dec_offsets[ff], + overwrite=self.overwrite, whitelight_range=wl_wvrng, + spec_subpixel=self.spec_subpixel, + spat_subpixel=self.spat_subpixel, + slice_subpixel=self.slice_subpixel) # Prepare the header hdr = cube_wcs.to_header() if self.fluxcal: @@ -1420,6 +1443,7 @@ def hst_alignment(sciImg, ivar, waveimg, slitid_img_gpm, wghts, * ------ * Return updated pixel RA, Dec """ + ra_offset, dec_offset = 0.0, 0.0 from pypeit import astrometry niter = 3 for ii in range(niter): @@ -1460,15 +1484,15 @@ def hst_alignment(sciImg, ivar, waveimg, slitid_img_gpm, wghts, inmask = slitid_img_gpm * ((waveimg > 4107.0) & (waveimg < 4119.0)).astype(int) flxcube, sigcube, bpmcube, wave = \ datacube.generate_cube_subpixel("tmpfile.fits", all_wcs, raw_bins, sciImg, ivar, waveimg, inmask, wghts, - all_wcs, tilts, slits, astrom_trans, all_dar, overwrite=False, + all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, overwrite=False, spec_subpixel=5, spat_subpixel=5, slice_subpixel=slice_subpixel) HdMap_raw, HdMapErr_raw = astrometry.fit_cube(flxcube, sigcube**2, all_wcs, line="HIdelta") # THEN DO Hgamma inmask = slitid_img_gpm * ((waveimg > 4346.0) & (waveimg < 4358.0)).astype(int) flxcube, sigcube, bpmcube, wave = \ datacube.generate_cube_subpixel("tmpfile.fits", all_wcs, raw_bins, sciImg, ivar, waveimg, inmask, wghts, - all_wcs, tilts, slits, astrom_trans, all_dar, overwrite=False, - spec_subpixel=5, spat_subpixel=5, slice_subpixel=slice_subpixel) + all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, + overwrite=False, spec_subpixel=5, spat_subpixel=5, slice_subpixel=slice_subpixel) HgMap_raw, HgMapErr_raw = astrometry.fit_cube(flxcube, sigcube**2, all_wcs, line="HIgamma") # Plot the emission line map from matplotlib import pyplot as plt From fb04a1bcd6f3a3c77d7de421b389d0f2ba7b0061 Mon Sep 17 00:00:00 2001 From: rcooke Date: Sun, 10 Dec 2023 14:04:22 +0000 Subject: [PATCH 094/244] refactor compute_weights --- pypeit/coadd3d.py | 46 +++++----- pypeit/core/datacube.py | 185 ++++++++++++++++++---------------------- 2 files changed, 107 insertions(+), 124 deletions(-) diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index 91af6bcdf4..e9a7da30c3 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -1174,11 +1174,10 @@ def load(self): if self.method in ['subpixel', 'ngp']: # Generate the datacube flxcube, sigcube, bpmcube, wave = \ - datacube.generate_cube_subpixel(outfile, self.all_wcs[ff], bins, - sciImg, ivar, waveimg, slitid_img_gpm, wghts, + datacube.generate_cube_subpixel(self.all_wcs[ff], bins, sciImg, ivar, waveimg, slitid_img_gpm, wghts, self.all_wcs[ff], spec2DObj.tilts, slits, alignSplines, darcorr, self.ra_offsets[ff], self.dec_offsets[ff], - overwrite=self.overwrite, whitelight_range=wl_wvrng, + overwrite=self.overwrite, whitelight_range=wl_wvrng, outfile=outfile, spec_subpixel=self.spec_subpixel, spat_subpixel=self.spat_subpixel, slice_subpixel=self.slice_subpixel) @@ -1284,19 +1283,22 @@ def compute_weights(self): Returns: `numpy.ndarray`_: The individual pixel weights for each detector pixel, and every frame. """ + # If there is only one file, then all pixels have the same weight + if self.numfiles == 1: + return np.ones_like(self.all_sci) + # Calculate the relative spectral weights of all pixels - return np.ones_like(self.all_sci) if self.numfiles == 1 else \ - datacube.compute_weights_frompix(self.all_ra, self.all_dec, self.all_wave, self.all_sci, self.all_ivar, - self.all_idx, self._dspat, self._dwv, self.mnmx_wv, self.all_wghts, - self.all_spatpos, self.all_specpos, self.all_spatid, - self.all_tilts, self.all_slits, self.all_align, self.all_dar, - ra_min=self.cubepar['ra_min'], ra_max=self.cubepar['ra_max'], - dec_min=self.cubepar['dec_min'], dec_max=self.cubepar['dec_max'], - wave_min=self.cubepar['wave_min'], wave_max=self.cubepar['wave_max'], - relative_weights=self.cubepar['relative_weights'], - whitelight_range=self.cubepar['whitelight_range'], - reference_image=self.cubepar['reference_image'], - specname=self.specname) + return datacube.compute_weights_frompix(self.all_ra, self.all_dec, self.all_wave, self.all_sci, self.all_ivar, + self.all_slitid, self._dspat, self._dwv, self.mnmx_wv, self.all_wghts, + self.all_wcs, self.all_tilts, self.all_slits, self.all_align, self.all_dar, + self.ra_offsets, self.dec_offsets, + ra_min=self.cubepar['ra_min'], ra_max=self.cubepar['ra_max'], + dec_min=self.cubepar['dec_min'], dec_max=self.cubepar['dec_max'], + wave_min=self.cubepar['wave_min'], wave_max=self.cubepar['wave_max'], + relative_weights=self.cubepar['relative_weights'], + whitelight_range=self.cubepar['whitelight_range'], + reference_image=self.cubepar['reference_image'], + specname=self.specname) def run(self): """ @@ -1345,7 +1347,6 @@ def run(self): self.ra_offsets, self.dec_offsets = self.run_align() # Compute the relative weights on the spectra - # TODO :: update this with slice refactor self.all_wghts = self.compute_weights() # Generate the WCS, and the voxel edges @@ -1377,12 +1378,11 @@ def run(self): outfile = datacube.get_output_filename("", self.cubepar['output_filename'], True, -1) # Generate the datacube flxcube, sigcube, bpmcube, wave = \ - datacube.generate_cube_subpixel(outfile, cube_wcs, vox_edges, - self.all_sci, self.all_ivar, self.all_wave, + datacube.generate_cube_subpixel(cube_wcs, vox_edges, self.all_sci, self.all_ivar, self.all_wave, self.all_slitid, self.all_wghts, self.all_wcs, self.all_tilts, self.all_slits, self.all_align, self.all_dar, self.ra_offsets, self.dec_offsets, - overwrite=self.overwrite, whitelight_range=wl_wvrng, + outfile=outfile, overwrite=self.overwrite, whitelight_range=wl_wvrng, spec_subpixel=self.spec_subpixel, spat_subpixel=self.spat_subpixel, slice_subpixel=self.slice_subpixel) @@ -1402,13 +1402,13 @@ def run(self): outfile = datacube.get_output_filename("", self.cubepar['output_filename'], False, ff) # Generate the datacube flxcube, sigcube, bpmcube, wave = \ - datacube.generate_cube_subpixel(outfile, cube_wcs, vox_edges, + datacube.generate_cube_subpixel(cube_wcs, vox_edges, self.all_sci[ff], self.all_ivar[ff], self.all_wave[ff], self.all_slitid[ff], self.all_wghts[ff], self.all_wcs[ff], self.all_tilts[ff], self.all_slits[ff], self.all_align[ff], self.all_dar[ff], self.ra_offsets[ff], self.dec_offsets[ff], overwrite=self.overwrite, whitelight_range=wl_wvrng, - spec_subpixel=self.spec_subpixel, + outfile=outfile, spec_subpixel=self.spec_subpixel, spat_subpixel=self.spat_subpixel, slice_subpixel=self.slice_subpixel) # Prepare the header @@ -1483,14 +1483,14 @@ def hst_alignment(sciImg, ivar, waveimg, slitid_img_gpm, wghts, # FIRST DO Hdelta inmask = slitid_img_gpm * ((waveimg > 4107.0) & (waveimg < 4119.0)).astype(int) flxcube, sigcube, bpmcube, wave = \ - datacube.generate_cube_subpixel("tmpfile.fits", all_wcs, raw_bins, sciImg, ivar, waveimg, inmask, wghts, + datacube.generate_cube_subpixel(all_wcs, raw_bins, sciImg, ivar, waveimg, inmask, wghts, all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, overwrite=False, spec_subpixel=5, spat_subpixel=5, slice_subpixel=slice_subpixel) HdMap_raw, HdMapErr_raw = astrometry.fit_cube(flxcube, sigcube**2, all_wcs, line="HIdelta") # THEN DO Hgamma inmask = slitid_img_gpm * ((waveimg > 4346.0) & (waveimg < 4358.0)).astype(int) flxcube, sigcube, bpmcube, wave = \ - datacube.generate_cube_subpixel("tmpfile.fits", all_wcs, raw_bins, sciImg, ivar, waveimg, inmask, wghts, + datacube.generate_cube_subpixel(all_wcs, raw_bins, sciImg, ivar, waveimg, inmask, wghts, all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, overwrite=False, spec_subpixel=5, spat_subpixel=5, slice_subpixel=slice_subpixel) HgMap_raw, HgMapErr_raw = astrometry.fit_cube(flxcube, sigcube**2, all_wcs, line="HIgamma") diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index e4325bd683..2aab724b88 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -956,8 +956,8 @@ def generate_WCS(crval, cdelt, equinox=2000.0, name="PYP_SPEC"): return w -def compute_weights_frompix(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, dspat, dwv, mnmx_wv, all_wghts, - all_spatpos, all_specpos, all_spatid, all_tilts, all_slits, all_align, all_dar, +def compute_weights_frompix(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, dspat, dwv, mnmx_wv, wghtsImg, + all_wcs, all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets, ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None, sn_smooth_npix=None, relative_weights=False, reference_image=None, whitelight_range=None, specname="PYPSPEC"): @@ -968,27 +968,18 @@ def compute_weights_frompix(all_ra, all_dec, all_wave, all_sci, all_ivar, all_id determine the appropriate weights of each pixel. Args: - all_ra (`numpy.ndarray`_): - 1D flattened array containing the RA values of each pixel from all - spec2d files - all_dec (`numpy.ndarray`_): - 1D flattened array containing the DEC values of each pixel from all - spec2d files - all_wave (`numpy.ndarray`_): - 1D flattened array containing the wavelength values of each pixel - from all spec2d files - all_sci (`numpy.ndarray`_): - 1D flattened array containing the counts of each pixel from all - spec2d files - all_ivar (`numpy.ndarray`_): - 1D flattened array containing the inverse variance of each pixel - from all spec2d files - all_idx (`numpy.ndarray`_): - 1D flattened array containing an integer identifier indicating which - spec2d file each pixel originates from. For example, a 0 would - indicate that a pixel originates from the first spec2d frame listed - in the input file. a 1 would indicate that this pixel originates - from the second spec2d file, and so forth. + raImg : (`numpy.ndarray`_, list): + A list of 2D array containing the RA of each pixel, with shape (nspec, nspat) + decImg : (`numpy.ndarray`_, list): + A list of 2D array containing the Dec of each pixel, with shape (nspec, nspat) + waveImg (`numpy.ndarray`_, list): + A list of 2D array containing the wavelength of each pixel, with shape (nspec, nspat) + sciImg (`numpy.ndarray`_, list): + A list of 2D array containing the science image of each pixel, with shape (nspec, nspat) + ivarImg (`numpy.ndarray`_, list): + A list of 2D array containing the inverse variance image of each pixel, with shape (nspec, nspat) + slitidImg (`numpy.ndarray`_, list): + A list of 2D array containing the slit ID of each pixel, with shape (nspec, nspat) dspat (float): The size of each spaxel on the sky (in degrees) dwv (float): @@ -996,29 +987,25 @@ def compute_weights_frompix(all_ra, all_dec, all_wave, all_sci, all_ivar, all_id mnmx_wv (`numpy.ndarray`_): The minimum and maximum wavelengths of every slit and frame. The shape is (Nframes, Nslits, 2), The minimum and maximum wavelengths are stored in the [:,:,0] and [:,:,1] indices, respectively. - all_wghts (`numpy.ndarray`_): - 1D flattened array containing the weights of each pixel to be used - in the combination - all_spatpos (`numpy.ndarray`_): - 1D flattened array containing the detector pixel location in the - spatial direction - all_specpos (`numpy.ndarray`_): - 1D flattened array containing the detector pixel location in the - spectral direction - all_spatid (`numpy.ndarray`_): - 1D flattened array containing the spatid of each pixel - tilts (`numpy.ndarray`_, list): - 2D wavelength tilts frame, or a list of tilt frames (see all_idx) - slits (:class:`~pypeit.slittrace.SlitTraceSet`, list): - Information stored about the slits, or a list of SlitTraceSet (see - all_idx) - astrom_trans (:class:`~pypeit.alignframe.AlignmentSplines`, list): + wghtsImg (`numpy.ndarray`_, list): + A list of 2D array containing the weights of each pixel, with shape (nspec, nspat) + all_wcs (`astropy.wcs.WCS`_, list): + A list of WCS objects, one for each frame. + all_tilts (`numpy.ndarray`_, list): + 2D wavelength tilts frame, or a list of tilt frames + all_slits (:class:`~pypeit.slittrace.SlitTraceSet`, list): + Information stored about the slits, or a list of SlitTraceSet objects + all_align (:class:`~pypeit.alignframe.AlignmentSplines`, list): A Class containing the transformation between detector pixel coordinates and WCS pixel coordinates, or a list of Alignment - Splines (see all_idx) + Splines. all_dar (:class:`~pypeit.coadd3d.DARcorrection`, list): A Class containing the DAR correction information, or a list of DARcorrection classes. If a list, it must be the same length as astrom_trans. + ra_offsets (float, list): + RA offsets for each frame in units of degrees + dec_offsets (float, list): + Dec offsets for each frame in units of degrees ra_min (float, optional): Minimum RA of the WCS (degrees) ra_max (float, optional): @@ -1051,53 +1038,39 @@ def compute_weights_frompix(all_ra, all_dec, all_wave, all_sci, all_ivar, all_id np.min(mnmx_wv[:, :, 1]), # The min red wavelength whitelight_range) # The user-specified values (if any) # Get the good white light pixels - ww, wavediff = get_whitelight_pixels(all_wave, min_wl, max_wl) + slitid_img_gpm, wavediff = get_whitelight_pixels(waveImg, slitidImg, min_wl, max_wl) # Generate the WCS image_wcs, voxedge, reference_image = \ - create_wcs(all_ra, all_dec, all_wave, dspat, wavediff, + create_wcs(raImg, decImg, waveImg, slitid_img_gpm, dspat, wavediff, ra_min=ra_min, ra_max=ra_max, dec_min=dec_min, dec_max=dec_max, wave_min=wave_min, wave_max=wave_max, - reference=reference_image, collapse=True, equinox=2000.0, - specname=specname) + reference=reference_image, collapse=True, equinox=2000.0, specname=specname) # Generate the white light image # NOTE: hard-coding subpixel=1 in both directions for speed, and combining into a single image - wl_full = generate_image_subpixel(image_wcs, all_ra, all_dec, all_wave, all_sci, all_ivar, all_wghts, - all_spatpos, all_specpos, all_spatid, all_tilts, all_slits, all_align, all_dar, - voxedge, all_idx=all_idx, spec_subpixel=1, spat_subpixel=1, combine=True) + wl_full = generate_image_subpixel(image_wcs, voxedge, sciImg, ivarImg, waveImg, slitid_img_gpm, wghtsImg, + all_wcs, all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets, + spec_subpixel=1, spat_subpixel=1, slice_subpixel=1, combine=True) + # Compute the weights - return compute_weights(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, wl_full[:, :, 0], dspat, dwv, + return compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitid_img_gpm, + all_wcs, all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets, + wl_full[:, :, 0], dspat, dwv, + ra_min=ra_min, ra_max=ra_max, dec_min=dec_min, dec_max=dec_max, wave_min=wave_min, sn_smooth_npix=sn_smooth_npix, relative_weights=relative_weights) -def compute_weights(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, whitelight_img, dspat, dwv, +def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, + all_wcs, all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets, + whitelight_img, dspat, dwv, + ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None, sn_smooth_npix=None, relative_weights=False): r""" Calculate wavelength dependent optimal weights. The weighting is currently based on a relative :math:`(S/N)^2` at each wavelength Args: - all_ra (`numpy.ndarray`_): - 1D flattened array containing the RA values of each pixel from all - spec2d files - all_dec (`numpy.ndarray`_): - 1D flattened array containing the DEC values of each pixel from all - spec2d files - all_wave (`numpy.ndarray`_): - 1D flattened array containing the wavelength values of each pixel - from all spec2d files - all_sci (`numpy.ndarray`_): - 1D flattened array containing the counts of each pixel from all - spec2d files - all_ivar (`numpy.ndarray`_): - 1D flattened array containing the inverse variance of each pixel - from all spec2d files - all_idx (`numpy.ndarray`_): - 1D flattened array containing an integer identifier indicating which - spec2d file each pixel originates from. For example, a 0 would - indicate that a pixel originates from the first spec2d frame listed - in the input file. a 1 would indicate that this pixel originates - from the second spec2d file, and so forth. + TODO : :docstring + see return statement of docstring as well whitelight_img (`numpy.ndarray`_): A 2D array containing a whitelight image, that was created with the input ``all_`` arrays. @@ -1113,45 +1086,51 @@ def compute_weights(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, white Calculate weights by fitting to the ratio of spectra? Returns: - `numpy.ndarray`_ : a 1D array the same size as all_sci, containing - relative wavelength dependent weights of each input pixel. + TODO :: compelte docstring """ msgs.info("Calculating the optimal weights of each pixel") - # Determine number of files - numfiles = np.unique(all_idx).size + # Check the inputs for combinations of lists or not, and then determine the number of frames + _raImg, _decImg, _waveImg, _sciImg, _ivarImg, _slitid_img_gpm, \ + _all_wcs, _all_tilts, _all_slits, _all_align, _all_dar, _ra_offsets, _dec_offsets = \ + check_inputs([raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, + all_wcs, all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets]) + numframes = len(_sciImg) + + # Check the WCS bounds + _ra_min, _ra_max, _dec_min, _dec_max, _wave_min, _wave_max = \ + wcs_bounds(_raImg, _decImg, _waveImg, _slitid_img_gpm, + ra_min=ra_min, ra_max=ra_max, dec_min=dec_min, dec_max=dec_max, wave_min=wave_min, wave_max=wave_max) # Find the location of the object with the highest S/N in the combined white light image idx_max = np.unravel_index(np.argmax(whitelight_img), whitelight_img.shape) msgs.info("Highest S/N object located at spaxel (x, y) = {0:d}, {1:d}".format(idx_max[0], idx_max[1])) # Generate a 2D WCS to register all frames - coord_min = [np.min(all_ra), np.min(all_dec), np.min(all_wave)] + coord_min = [_ra_min, _dec_min, _wave_min] coord_dlt = [dspat, dspat, dwv] whitelightWCS = generate_WCS(coord_min, coord_dlt) # Make the bin edges to be at +/- 1 pixels around the maximum (i.e. summing 9 pixels total) - numwav = int((np.max(all_wave) - np.min(all_wave)) / dwv) + numwav = int((_wave_max - _wave_min) / dwv) xbins = np.array([idx_max[0]-1, idx_max[0]+2]) - 0.5 ybins = np.array([idx_max[1]-1, idx_max[1]+2]) - 0.5 spec_bins = np.arange(1 + numwav) - 0.5 bins = (xbins, ybins, spec_bins) # Extract the spectrum of the highest S/N object - flux_stack = np.zeros((numwav, numfiles)) - ivar_stack = np.zeros((numwav, numfiles)) - for ff in range(numfiles): - msgs.info("Extracting spectrum of highest S/N detection from frame {0:d}/{1:d}".format(ff + 1, numfiles)) - ww = (all_idx == ff) - # Extract the spectrum - pix_coord = whitelightWCS.wcs_world2pix(np.vstack((all_ra[ww], all_dec[ww], all_wave[ww] * 1.0E-10)).T, 0) - spec, edges = np.histogramdd(pix_coord, bins=bins, weights=all_sci[ww]) - var, edges = np.histogramdd(pix_coord, bins=bins, weights=1/all_ivar[ww]) - norm, edges = np.histogramdd(pix_coord, bins=bins) - normspec = (norm > 0) / (norm + (norm == 0)) - var_spec = var[0, 0, :] - ivar_spec = (var_spec > 0) / (var_spec + (var_spec == 0)) + flux_stack = np.zeros((numwav, numframes)) + ivar_stack = np.zeros((numwav, numframes)) + for ff in range(numframes): + msgs.info("Extracting spectrum of highest S/N detection from frame {0:d}/{1:d}".format(ff + 1, numframes)) + flxcube, sigcube, bpmcube, wave = \ + generate_cube_subpixel(whitelightWCS, bins, _sciImg[ff], _ivarImg[ff], _waveImg[ff], + _slitid_img_gpm[ff], np.ones(_sciImg[ff].shape), _all_wcs[ff], + _all_tilts[ff], _all_slits[ff], _all_align[ff], _all_dar[ff], + _ra_offsets[ff], _dec_offsets[ff], + spec_subpixel=1, spat_subpixel=1, slice_subpixel=1) # Calculate the S/N in a given spectral bin - flux_stack[:, ff] = spec[0, 0, :] * np.sqrt(normspec) # Note: sqrt(nrmspec), is because we want the S/N in a _single_ pixel (i.e. not spectral bin) - ivar_stack[:, ff] = ivar_spec + # TODO :: RJC removed the sqrt here - not sure why it needs to be detector pixel instead of spectral bin... + flux_stack[:, ff] = flxcube[0, 0, :]# * np.sqrt(normspec) # Note: sqrt(nrmspec), is because we want the S/N in a _single_ pixel (i.e. not spectral bin) + ivar_stack[:, ff] = utils.inverse(sigcube[0, 0, :])**2 mask_stack = (flux_stack != 0.0) & (ivar_stack != 0.0) # Obtain a wavelength of each pixel @@ -1165,11 +1144,11 @@ def compute_weights(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, white sn_smooth_npix=sn_smooth_npix, relative_weights=relative_weights) # Because we pass back a weights array, we need to interpolate to assign each detector pixel a weight - all_wghts = np.ones(all_idx.size) - for ff in range(numfiles): - ww = (all_idx == ff) + all_wghts = np.ones(slitidImg.size) + for ff in range(numframes): + ww = (slitidImg == ff) all_wghts[ww] = interp1d(wave_spec, weights[ff], kind='cubic', - bounds_error=False, fill_value="extrapolate")(all_wave[ww]) + bounds_error=False, fill_value="extrapolate")(waveImg[ww]) msgs.info("Optimal weighting complete") return all_wghts @@ -1273,19 +1252,16 @@ def generate_image_subpixel(image_wcs, bins, sciImg, ivarImg, waveImg, slitid_im return all_wl_imgs -def generate_cube_subpixel(outfile, output_wcs, bins, - sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, +def generate_cube_subpixel(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, spec_subpixel=5, spat_subpixel=5, slice_subpixel=5, - overwrite=False, whitelight_range=None, debug=False): + overwrite=False, outfile=None, whitelight_range=None, debug=False): """ Save a datacube using the subpixel algorithm. Refer to the subpixellate() docstring for further details about this algorithm Args: - outfile (str): - Filename to be used to save the datacube output_wcs (`astropy.wcs.WCS`_): Output world coordinate system. bins (tuple): @@ -1342,6 +1318,8 @@ def generate_cube_subpixel(outfile, output_wcs, bins, each IFU slice into 5 subslices in the slice direction. overwrite (bool, optional): If True, the output cube will be overwritten. + outfile (str, optional): + Filename to be used to save the datacube whitelight_range (None, list, optional): A two element list that specifies the minimum and maximum wavelengths (in Angstroms) to use when constructing the white light @@ -1362,6 +1340,11 @@ def generate_cube_subpixel(outfile, output_wcs, bins, (3) the corresponding bad pixel mask cube, and (4) a 1D array containing the wavelength at each spectral coordinate of the datacube. """ + # Check the inputs + if whitelight_range is not None or debug: + if outfile is None: + msgs.error("Must provide an outfile name if either whitelight_range or debug are set") + # Prepare the header, and add the unit of flux to the header hdr = output_wcs.to_header() From 067310d3aaecbb744db6fcc23f3e58c81f6599cd Mon Sep 17 00:00:00 2001 From: rcooke Date: Sun, 10 Dec 2023 14:12:40 +0000 Subject: [PATCH 095/244] refactor compute_weights return --- pypeit/core/datacube.py | 47 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index 2aab724b88..b33e946d49 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -1070,10 +1070,38 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, based on a relative :math:`(S/N)^2` at each wavelength Args: - TODO : :docstring + see return statement of docstring as well + raImg : (`numpy.ndarray`_, list): + A list of 2D array containing the RA of each pixel, with shape (nspec, nspat) + decImg : (`numpy.ndarray`_, list): + A list of 2D array containing the Dec of each pixel, with shape (nspec, nspat) + waveImg (`numpy.ndarray`_, list): + A list of 2D array containing the wavelength of each pixel, with shape (nspec, nspat) + sciImg (`numpy.ndarray`_, list): + A list of 2D array containing the science image of each pixel, with shape (nspec, nspat) + ivarImg (`numpy.ndarray`_, list): + A list of 2D array containing the inverse variance image of each pixel, with shape (nspec, nspat) + slitidImg (`numpy.ndarray`_, list): + A list of 2D array containing the slit ID of each pixel, with shape (nspec, nspat) + all_wcs (`astropy.wcs.WCS`_, list): + A list of WCS objects, one for each frame. + all_tilts (`numpy.ndarray`_, list): + 2D wavelength tilts frame, or a list of tilt frames + all_slits (:class:`~pypeit.slittrace.SlitTraceSet`, list): + Information stored about the slits, or a list of SlitTraceSet objects + all_align (:class:`~pypeit.alignframe.AlignmentSplines`, list): + A Class containing the transformation between detector pixel + coordinates and WCS pixel coordinates, or a list of Alignment + Splines. + all_dar (:class:`~pypeit.coadd3d.DARcorrection`, list): + A Class containing the DAR correction information, or a list of DARcorrection + classes. If a list, it must be the same length as astrom_trans. + ra_offsets (float, list): + RA offsets for each frame in units of degrees + dec_offsets (float, list): + Dec offsets for each frame in units of degrees whitelight_img (`numpy.ndarray`_): - A 2D array containing a whitelight image, that was created with the - input ``all_`` arrays. + A 2D array containing a white light image, that was created with the + input ``all`` arrays. dspat (float): The size of each spaxel on the sky (in degrees) dwv (float): @@ -1096,6 +1124,11 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, all_wcs, all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets]) numframes = len(_sciImg) + # If there's only one frame, use uniform weighting + if numframes == 1: + msgs.warn("Only one frame provided. Using uniform weighting.") + return np.ones_like(sciImg) + # Check the WCS bounds _ra_min, _ra_max, _dec_min, _dec_max, _wave_min, _wave_max = \ wcs_bounds(_raImg, _decImg, _waveImg, _slitid_img_gpm, @@ -1128,7 +1161,9 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, _ra_offsets[ff], _dec_offsets[ff], spec_subpixel=1, spat_subpixel=1, slice_subpixel=1) # Calculate the S/N in a given spectral bin - # TODO :: RJC removed the sqrt here - not sure why it needs to be detector pixel instead of spectral bin... + # TODO :: Think more about this... RJC removed the sqrt here - not sure why it needs to be detector pixel instead of spectral bin... + # I think it's because we want to compare the S/N in a single pixel for a fair comparison, and not the S/N for a spectral bin. + # For example, there may be a masked pixel in the spectral bin, which would give a lower S/N than a single pixel. flux_stack[:, ff] = flxcube[0, 0, :]# * np.sqrt(normspec) # Note: sqrt(nrmspec), is because we want the S/N in a _single_ pixel (i.e. not spectral bin) ivar_stack[:, ff] = utils.inverse(sigcube[0, 0, :])**2 @@ -1144,12 +1179,14 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, sn_smooth_npix=sn_smooth_npix, relative_weights=relative_weights) # Because we pass back a weights array, we need to interpolate to assign each detector pixel a weight - all_wghts = np.ones(slitidImg.size) + all_wghts = [np.ones(_sciImg[0].shape) for _ in range(numframes)] + # TODO :: UP TO HERE! for ff in range(numframes): ww = (slitidImg == ff) all_wghts[ww] = interp1d(wave_spec, weights[ff], kind='cubic', bounds_error=False, fill_value="extrapolate")(waveImg[ww]) msgs.info("Optimal weighting complete") + # TODO :: This really should be a list of weights, not a single array. return all_wghts From e3d2494b700e760b45cc0aa22b9795e976760e38 Mon Sep 17 00:00:00 2001 From: rcooke Date: Sun, 10 Dec 2023 20:26:38 +0000 Subject: [PATCH 096/244] refactor optimal weights --- pypeit/core/datacube.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index b33e946d49..b643d58dc7 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -1114,7 +1114,8 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, Calculate weights by fitting to the ratio of spectra? Returns: - TODO :: compelte docstring + list: Either a 2D `numpy.ndarray`_ or a list of 2D `numpy.ndarray`_ arrays containing the optimal + weights of each pixel for all frames, with shape (nspec, nspat). """ msgs.info("Calculating the optimal weights of each pixel") # Check the inputs for combinations of lists or not, and then determine the number of frames @@ -1180,13 +1181,11 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, # Because we pass back a weights array, we need to interpolate to assign each detector pixel a weight all_wghts = [np.ones(_sciImg[0].shape) for _ in range(numframes)] - # TODO :: UP TO HERE! for ff in range(numframes): - ww = (slitidImg == ff) - all_wghts[ww] = interp1d(wave_spec, weights[ff], kind='cubic', - bounds_error=False, fill_value="extrapolate")(waveImg[ww]) + ww = (slitidImg[ff] > 0) + all_wghts[ff][ww] = interp1d(wave_spec, weights[ff], kind='cubic', + bounds_error=False, fill_value="extrapolate")(waveImg[ff][ww]) msgs.info("Optimal weighting complete") - # TODO :: This really should be a list of weights, not a single array. return all_wghts From 888536b5660edd970edd8fdd641bf492074f203a Mon Sep 17 00:00:00 2001 From: rcooke Date: Mon, 11 Dec 2023 11:17:56 +0000 Subject: [PATCH 097/244] print offsets --- pypeit/core/datacube.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index b643d58dc7..63e55c6865 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -633,15 +633,15 @@ def align_user_offsets(ifu_ra, ifu_dec, ra_offset, dec_offset): # The cos(dec) factor should be input by the user, and should be included in the self.opts['ra_offset'] ref_shift_ra = ifu_ra[0] - ifu_ra ref_shift_dec = ifu_dec[0] - ifu_dec - numfiles = ra_offset.size - out_ra_offsets = np.zeros(numfiles) - out_dec_offsets = np.zeros(numfiles) + numfiles = len(ra_offset) + out_ra_offsets = [0.0 for _ in range(numfiles)] + out_dec_offsets = [0.0 for _ in range(numfiles)] for ff in range(numfiles): # Apply the shift out_ra_offsets[ff] = ref_shift_ra[ff] + ra_offset[ff] out_dec_offsets[ff] = ref_shift_dec[ff] + dec_offset[ff] msgs.info("Spatial shift of cube #{0:d}:".format(ff + 1) + msgs.newline() + - "RA, DEC (arcsec) = {0:+0.3f} E, {1:+0.3f} N".format(ra_offset[ff], dec_offset[ff])) + "RA, DEC (arcsec) = {0:+0.3f} E, {1:+0.3f} N".format(ra_offset[ff]*3600.0, dec_offset[ff]*3600.0)) return out_ra_offsets, out_dec_offsets @@ -1186,6 +1186,7 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, all_wghts[ff][ww] = interp1d(wave_spec, weights[ff], kind='cubic', bounds_error=False, fill_value="extrapolate")(waveImg[ff][ww]) msgs.info("Optimal weighting complete") + embed() return all_wghts From 3593975d76697f9c76a241f6002efdf63066857e Mon Sep 17 00:00:00 2001 From: rcooke Date: Mon, 11 Dec 2023 12:01:39 +0000 Subject: [PATCH 098/244] rm all_wcs --- pypeit/coadd3d.py | 48 ++++++++++++++++++------------------- pypeit/core/datacube.py | 53 ++++++++++++++++------------------------- 2 files changed, 44 insertions(+), 57 deletions(-) diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index e9a7da30c3..4c3d4bd182 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -416,8 +416,8 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs # Set the frame specific options self.skysub_frame = skysub_frame self.scale_corr = scale_corr - self.ra_offsets = np.array(ra_offsets) if isinstance(ra_offsets, list) else ra_offsets - self.dec_offsets = np.array(dec_offsets) if isinstance(dec_offsets, list) else dec_offsets + self.ra_offsets = list(ra_offsets) if isinstance(ra_offsets, np.ndarray) else ra_offsets + self.dec_offsets = list(dec_offsets) if isinstance(dec_offsets, np.ndarray) else dec_offsets # If no ra_offsets or dec_offsets have been provided, initialise the lists self.user_alignment = True if self.ra_offsets is None and self.dec_offsets is None: @@ -427,8 +427,8 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs # User offsets are not provided, so turn off the user_alignment self.user_alignment = False # Initialise the lists of ra_offsets and dec_offsets - self.ra_offsets = np.zeros(self.numfiles) - self.dec_offsets = np.zeros(self.numfiles) + self.ra_offsets = [0.0 for _ in range(self.numfiles)] + self.dec_offsets = [0.0 for _ in range(self.numfiles)] # Check on Spectrograph input if spectrograph is None: @@ -1131,6 +1131,7 @@ def load(self): # Calculate the weights relative to the zeroth cube self.weights[ff] = 1.0 # exptime #np.median(flux_sav[resrt]*np.sqrt(ivar_sav[resrt]))**2 + wghts = self.weights[ff] * np.ones(sciImg.shape) # Get the slit image and then unset pixels in the slit image that are bad slitid_img_gpm = slitid_img_init * onslit_gpm.astype(int) @@ -1138,18 +1139,15 @@ def load(self): ################################## # Astrometric alignment to HST frames # TODO :: RJC requests this remains here... it is only used by RJC - # Get the coordinate bounds - slitlength = int(np.round(np.median(slits.get_slitlengths(initial=True, median=True)))) - numwav = int((np.max(waveimg) - wave0) / dwv) - raw_bins = self.spec.get_datacube_bins(slitlength, minmax, numwav) - wghts = np.ones(sciImg.shape) - # Now do the alignment - self.ra_offsets[ff], self.dec_offsets[ff] = hst_alignment(sciImg, ivar, waveimg, slitid_img_gpm, wghts, - self.all_wcs[ff], spec2DObj.tilts, slits, alignSplines, darcorr, - raw_bins=raw_bins) - # TODO :: Does this need to be included anywhere? - # self.ifu_ra[ff] += self.ra_offsets[ff] - # self.ifu_dec[ff] += self.dec_offsets[ff] + if False: + # Get the coordinate bounds + slitlength = int(np.round(np.median(slits.get_slitlengths(initial=True, median=True)))) + numwav = int((np.max(waveimg) - wave0) / dwv) + raw_bins = self.spec.get_datacube_bins(slitlength, minmax, numwav) + # Now do the alignment + self.ra_offsets[ff], self.dec_offsets[ff] = hst_alignment(sciImg, ivar, waveimg, slitid_img_gpm, wghts, + self.all_wcs[ff], spec2DObj.tilts, slits, alignSplines, darcorr, + raw_bins=raw_bins) ################################## # If individual frames are to be output without aligning them, @@ -1175,7 +1173,7 @@ def load(self): # Generate the datacube flxcube, sigcube, bpmcube, wave = \ datacube.generate_cube_subpixel(self.all_wcs[ff], bins, sciImg, ivar, waveimg, slitid_img_gpm, wghts, - self.all_wcs[ff], spec2DObj.tilts, slits, alignSplines, darcorr, + spec2DObj.tilts, slits, alignSplines, darcorr, self.ra_offsets[ff], self.dec_offsets[ff], overwrite=self.overwrite, whitelight_range=wl_wvrng, outfile=outfile, spec_subpixel=self.spec_subpixel, @@ -1219,12 +1217,12 @@ def run_align(self): # Grab cos(dec) for convenience cosdec = np.cos(np.mean(self.ifu_dec[0]) * np.pi / 180.0) # Initialize the RA and Dec offset arrays - ra_offsets, dec_offsets = np.zeros(self.numfiles), np.zeros(self.numfiles) + ra_offsets, dec_offsets = [0 for _ in range(self.numfiles)], [0 for _ in range(self.numfiles)] # Register spatial offsets between all frames if self.user_alignment: # The user has specified offsets - update these values accounting for the difference in header RA/DEC ra_offsets, dec_offsets = datacube.align_user_offsets(self.ifu_ra, self.ifu_dec, - self.ra_offsets, self.dec_offsets) + self.ra_offsets, self.dec_offsets) else: # Find the wavelength range where all frames overlap min_wl, max_wl = datacube.get_whitelight_range(np.max(self.mnmx_wv[:, :, 0]), # The max blue wavelength @@ -1248,7 +1246,7 @@ def run_align(self): msgs.error("Spectral range for WCS is incorrect for white light image") wl_imgs = datacube.generate_image_subpixel(image_wcs, voxedge, self.all_sci, self.all_ivar, self.all_wave, - slitid_img_gpm, self.all_wghts, self.all_wcs, + slitid_img_gpm, self.all_wghts, self.all_tilts, self.all_slits, self.all_align, self.all_dar, ra_offsets, dec_offsets, spec_subpixel=self.spec_subpixel, @@ -1290,7 +1288,7 @@ def compute_weights(self): # Calculate the relative spectral weights of all pixels return datacube.compute_weights_frompix(self.all_ra, self.all_dec, self.all_wave, self.all_sci, self.all_ivar, self.all_slitid, self._dspat, self._dwv, self.mnmx_wv, self.all_wghts, - self.all_wcs, self.all_tilts, self.all_slits, self.all_align, self.all_dar, + self.all_tilts, self.all_slits, self.all_align, self.all_dar, self.ra_offsets, self.dec_offsets, ra_min=self.cubepar['ra_min'], ra_max=self.cubepar['ra_max'], dec_min=self.cubepar['dec_min'], dec_max=self.cubepar['dec_max'], @@ -1379,7 +1377,7 @@ def run(self): # Generate the datacube flxcube, sigcube, bpmcube, wave = \ datacube.generate_cube_subpixel(cube_wcs, vox_edges, self.all_sci, self.all_ivar, self.all_wave, - self.all_slitid, self.all_wghts, self.all_wcs, + self.all_slitid, self.all_wghts, self.all_tilts, self.all_slits, self.all_align, self.all_dar, self.ra_offsets, self.dec_offsets, outfile=outfile, overwrite=self.overwrite, whitelight_range=wl_wvrng, @@ -1404,7 +1402,7 @@ def run(self): flxcube, sigcube, bpmcube, wave = \ datacube.generate_cube_subpixel(cube_wcs, vox_edges, self.all_sci[ff], self.all_ivar[ff], self.all_wave[ff], - self.all_slitid[ff], self.all_wghts[ff], self.all_wcs[ff], + self.all_slitid[ff], self.all_wghts[ff], self.all_tilts[ff], self.all_slits[ff], self.all_align[ff], self.all_dar[ff], self.ra_offsets[ff], self.dec_offsets[ff], overwrite=self.overwrite, whitelight_range=wl_wvrng, @@ -1484,14 +1482,14 @@ def hst_alignment(sciImg, ivar, waveimg, slitid_img_gpm, wghts, inmask = slitid_img_gpm * ((waveimg > 4107.0) & (waveimg < 4119.0)).astype(int) flxcube, sigcube, bpmcube, wave = \ datacube.generate_cube_subpixel(all_wcs, raw_bins, sciImg, ivar, waveimg, inmask, wghts, - all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, overwrite=False, + tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, overwrite=False, spec_subpixel=5, spat_subpixel=5, slice_subpixel=slice_subpixel) HdMap_raw, HdMapErr_raw = astrometry.fit_cube(flxcube, sigcube**2, all_wcs, line="HIdelta") # THEN DO Hgamma inmask = slitid_img_gpm * ((waveimg > 4346.0) & (waveimg < 4358.0)).astype(int) flxcube, sigcube, bpmcube, wave = \ datacube.generate_cube_subpixel(all_wcs, raw_bins, sciImg, ivar, waveimg, inmask, wghts, - all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, + tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, overwrite=False, spec_subpixel=5, spat_subpixel=5, slice_subpixel=slice_subpixel) HgMap_raw, HgMapErr_raw = astrometry.fit_cube(flxcube, sigcube**2, all_wcs, line="HIgamma") # Plot the emission line map diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index 63e55c6865..9a596b2518 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -957,7 +957,7 @@ def generate_WCS(crval, cdelt, equinox=2000.0, name="PYP_SPEC"): def compute_weights_frompix(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, dspat, dwv, mnmx_wv, wghtsImg, - all_wcs, all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets, + all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets, ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None, sn_smooth_npix=None, relative_weights=False, reference_image=None, whitelight_range=None, specname="PYPSPEC"): @@ -989,8 +989,6 @@ def compute_weights_frompix(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, The minimum and maximum wavelengths are stored in the [:,:,0] and [:,:,1] indices, respectively. wghtsImg (`numpy.ndarray`_, list): A list of 2D array containing the weights of each pixel, with shape (nspec, nspat) - all_wcs (`astropy.wcs.WCS`_, list): - A list of WCS objects, one for each frame. all_tilts (`numpy.ndarray`_, list): 2D wavelength tilts frame, or a list of tilt frames all_slits (:class:`~pypeit.slittrace.SlitTraceSet`, list): @@ -1049,19 +1047,19 @@ def compute_weights_frompix(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, # Generate the white light image # NOTE: hard-coding subpixel=1 in both directions for speed, and combining into a single image wl_full = generate_image_subpixel(image_wcs, voxedge, sciImg, ivarImg, waveImg, slitid_img_gpm, wghtsImg, - all_wcs, all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets, + all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets, spec_subpixel=1, spat_subpixel=1, slice_subpixel=1, combine=True) # Compute the weights - return compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitid_img_gpm, - all_wcs, all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets, + return compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, + all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets, wl_full[:, :, 0], dspat, dwv, ra_min=ra_min, ra_max=ra_max, dec_min=dec_min, dec_max=dec_max, wave_min=wave_min, sn_smooth_npix=sn_smooth_npix, relative_weights=relative_weights) def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, - all_wcs, all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets, + all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets, whitelight_img, dspat, dwv, ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None, sn_smooth_npix=None, relative_weights=False): @@ -1082,8 +1080,6 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, A list of 2D array containing the inverse variance image of each pixel, with shape (nspec, nspat) slitidImg (`numpy.ndarray`_, list): A list of 2D array containing the slit ID of each pixel, with shape (nspec, nspat) - all_wcs (`astropy.wcs.WCS`_, list): - A list of WCS objects, one for each frame. all_tilts (`numpy.ndarray`_, list): 2D wavelength tilts frame, or a list of tilt frames all_slits (:class:`~pypeit.slittrace.SlitTraceSet`, list): @@ -1119,10 +1115,10 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, """ msgs.info("Calculating the optimal weights of each pixel") # Check the inputs for combinations of lists or not, and then determine the number of frames - _raImg, _decImg, _waveImg, _sciImg, _ivarImg, _slitid_img_gpm, \ - _all_wcs, _all_tilts, _all_slits, _all_align, _all_dar, _ra_offsets, _dec_offsets = \ + _raImg, _decImg, _waveImg, _sciImg, _ivarImg, _slitidImg, \ + _all_tilts, _all_slits, _all_align, _all_dar, _ra_offsets, _dec_offsets = \ check_inputs([raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, - all_wcs, all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets]) + all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets]) numframes = len(_sciImg) # If there's only one frame, use uniform weighting @@ -1132,7 +1128,7 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, # Check the WCS bounds _ra_min, _ra_max, _dec_min, _dec_max, _wave_min, _wave_max = \ - wcs_bounds(_raImg, _decImg, _waveImg, _slitid_img_gpm, + wcs_bounds(_raImg, _decImg, _waveImg, _slitidImg, ra_min=ra_min, ra_max=ra_max, dec_min=dec_min, dec_max=dec_max, wave_min=wave_min, wave_max=wave_max) # Find the location of the object with the highest S/N in the combined white light image @@ -1157,7 +1153,7 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, msgs.info("Extracting spectrum of highest S/N detection from frame {0:d}/{1:d}".format(ff + 1, numframes)) flxcube, sigcube, bpmcube, wave = \ generate_cube_subpixel(whitelightWCS, bins, _sciImg[ff], _ivarImg[ff], _waveImg[ff], - _slitid_img_gpm[ff], np.ones(_sciImg[ff].shape), _all_wcs[ff], + _slitidImg[ff], np.ones(_sciImg[ff].shape), _all_tilts[ff], _all_slits[ff], _all_align[ff], _all_dar[ff], _ra_offsets[ff], _dec_offsets[ff], spec_subpixel=1, spat_subpixel=1, slice_subpixel=1) @@ -1168,6 +1164,7 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, flux_stack[:, ff] = flxcube[0, 0, :]# * np.sqrt(normspec) # Note: sqrt(nrmspec), is because we want the S/N in a _single_ pixel (i.e. not spectral bin) ivar_stack[:, ff] = utils.inverse(sigcube[0, 0, :])**2 + embed() mask_stack = (flux_stack != 0.0) & (ivar_stack != 0.0) # Obtain a wavelength of each pixel wcs_res = whitelightWCS.wcs_pix2world(np.vstack((np.zeros(numwav), np.zeros(numwav), np.arange(numwav))).T, 0) @@ -1186,12 +1183,11 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, all_wghts[ff][ww] = interp1d(wave_spec, weights[ff], kind='cubic', bounds_error=False, fill_value="extrapolate")(waveImg[ff][ww]) msgs.info("Optimal weighting complete") - embed() return all_wghts def generate_image_subpixel(image_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, - all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, + tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, spec_subpixel=5, spat_subpixel=5, slice_subpixel=5, combine=False): """ Generate a white light image from the input pixels @@ -1217,9 +1213,6 @@ def generate_image_subpixel(image_wcs, bins, sciImg, ivarImg, waveImg, slitid_im wghtImg (`numpy.ndarray`_, list): A list of 2D weight images, or a single 2D image containing the weight data. - all_wcs (`astropy.wcs.WCS`_, list): - A list of WCS objects, or a single WCS object containing the WCS - information of each image. tilts (`numpy.ndarray`_, list): 2D wavelength tilts frame, or a list of tilt frames (see all_idx) slits (:class:`~pypeit.slittrace.SlitTraceSet`, list): @@ -1262,8 +1255,8 @@ def generate_image_subpixel(image_wcs, bins, sciImg, ivarImg, waveImg, slitid_im `numpy.ndarray`_: The white light images for all frames """ # Perform some checks on the input -- note, more complete checks are performed in subpixellate() - _sciImg, _ivarImg, _waveImg, _slitid_img_gpm, _wghtImg, _all_wcs, _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset = \ - check_inputs([sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset]) + _sciImg, _ivarImg, _waveImg, _slitid_img_gpm, _wghtImg, _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset = \ + check_inputs([sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset]) numframes = len(_sciImg) # Prepare the array of white light images to be stored @@ -1277,12 +1270,12 @@ def generate_image_subpixel(image_wcs, bins, sciImg, ivarImg, waveImg, slitid_im if combine: # Subpixellate img, _, _ = subpixellate(image_wcs, bins, _sciImg, _ivarImg, _waveImg, _slitid_img_gpm, _wghtImg, - _all_wcs, _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset, + _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset, spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, slice_subpixel=slice_subpixel) else: # Subpixellate img, _, _ = subpixellate(image_wcs, bins, _sciImg[fr], _ivarImg[fr], _waveImg[fr], _slitid_img_gpm[fr], _wghtImg[fr], - _all_wcs[fr], _tilts[fr], _slits[fr], _astrom_trans[fr], _all_dar[fr], _ra_offset[fr], _dec_offset[fr], + _tilts[fr], _slits[fr], _astrom_trans[fr], _all_dar[fr], _ra_offset[fr], _dec_offset[fr], spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, slice_subpixel=slice_subpixel) all_wl_imgs[:, :, fr] = img[:, :, 0] # Return the constructed white light images @@ -1290,7 +1283,7 @@ def generate_image_subpixel(image_wcs, bins, sciImg, ivarImg, waveImg, slitid_im def generate_cube_subpixel(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, - all_wcs, tilts, slits, astrom_trans, all_dar, + tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, spec_subpixel=5, spat_subpixel=5, slice_subpixel=5, overwrite=False, outfile=None, whitelight_range=None, debug=False): @@ -1317,8 +1310,6 @@ def generate_cube_subpixel(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_im wghtImg (`numpy.ndarray`_, list): A list of 2D array containing the weights of each pixel to be used in the combination - all_wcs (`astropy.wcs.WCS`_, list): - A list of `astropy.wcs.WCS`_ objects, one for each spec2d file tilts (list): A list of `numpy.ndarray`_ objects, one for each spec2d file, containing the tilts of each pixel @@ -1387,7 +1378,7 @@ def generate_cube_subpixel(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_im # Subpixellate subpix = subpixellate(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, - all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, + tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, slice_subpixel=slice_subpixel, debug=debug) # Extract the variables that we need @@ -1428,7 +1419,7 @@ def generate_cube_subpixel(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_im def subpixellate(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, - all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, + tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, spec_subpixel=5, spat_subpixel=5, slice_subpixel=5, debug=False): r""" Subpixellate the input data into a datacube. This algorithm splits each @@ -1463,8 +1454,6 @@ def subpixellate(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wgh wghtImg (`numpy.ndarray`_, list): A list of 2D array containing the weights of each pixel to be used in the combination - all_wcs (`astropy.wcs.WCS`_, list): - A list of `astropy.wcs.WCS`_ objects, one for each spec2d file tilts (list): A list of `numpy.ndarray`_ objects, one for each spec2d file, containing the tilts of each pixel @@ -1513,8 +1502,8 @@ def subpixellate(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wgh residual cube. The latter is only returned if debug is True. """ # Check the inputs for combinations of lists or not - _sciImg, _ivarImg, _waveImg, _gpmImg, _wghtImg, _all_wcs, _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset = \ - check_inputs([sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset]) + _sciImg, _ivarImg, _waveImg, _gpmImg, _wghtImg, _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset = \ + check_inputs([sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset]) numframes = len(_sciImg) # Prepare the output arrays From ba57966f9b36ebf3a8be80faf1fdabbeec2515b2 Mon Sep 17 00:00:00 2001 From: rcooke Date: Mon, 11 Dec 2023 12:01:56 +0000 Subject: [PATCH 099/244] fmt subslice --- pypeit/slittrace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/slittrace.py b/pypeit/slittrace.py index df38f304f5..74c0f1e414 100644 --- a/pypeit/slittrace.py +++ b/pypeit/slittrace.py @@ -452,7 +452,7 @@ def get_radec_image(self, wcs, alignSplines, tilts, slice_offset=None, initial=T reference (usually the centre of the slit) and the edges of the slits. Shape is (nslits, 2). """ - substring = '' if slice_offset is None else f' with slice_offset={slice_offset}' + substring = '' if slice_offset is None else f' with slice_offset={slice_offset:.3f}' msgs.info("Generating an RA/DEC image"+substring) # Check the input if slice_offset is None: From 95691022f6046e844ebc68f6b8fa47c5d1b83795 Mon Sep 17 00:00:00 2001 From: rcooke Date: Mon, 11 Dec 2023 12:07:59 +0000 Subject: [PATCH 100/244] wavelength in Angstroms --- pypeit/core/datacube.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index 9a596b2518..5aa7365b9d 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -1139,6 +1139,7 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, coord_min = [_ra_min, _dec_min, _wave_min] coord_dlt = [dspat, dspat, dwv] whitelightWCS = generate_WCS(coord_min, coord_dlt) + wcs_scale = (1.0 * whitelightWCS.spectral.wcs.cunit[0]).to(units.Angstrom).value # Ensures the WCS is in Angstroms # Make the bin edges to be at +/- 1 pixels around the maximum (i.e. summing 9 pixels total) numwav = int((_wave_max - _wave_min) / dwv) xbins = np.array([idx_max[0]-1, idx_max[0]+2]) - 0.5 @@ -1152,7 +1153,7 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, for ff in range(numframes): msgs.info("Extracting spectrum of highest S/N detection from frame {0:d}/{1:d}".format(ff + 1, numframes)) flxcube, sigcube, bpmcube, wave = \ - generate_cube_subpixel(whitelightWCS, bins, _sciImg[ff], _ivarImg[ff], _waveImg[ff], + generate_cube_subpixel(whitelightWCS, bins, _sciImg[ff], _ivarImg[ff], _waveImg[ff]/wcs_scale, _slitidImg[ff], np.ones(_sciImg[ff].shape), _all_tilts[ff], _all_slits[ff], _all_align[ff], _all_dar[ff], _ra_offsets[ff], _dec_offsets[ff], From 503f229e1bf5eefe183afcd1275e49f6b2274a92 Mon Sep 17 00:00:00 2001 From: rcooke Date: Mon, 11 Dec 2023 12:15:31 +0000 Subject: [PATCH 101/244] Revert "rm all_wcs" This reverts commit 3593975d --- pypeit/coadd3d.py | 48 +++++++++++++++++++------------------- pypeit/core/datacube.py | 51 +++++++++++++++++++++++++---------------- 2 files changed, 56 insertions(+), 43 deletions(-) diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index 4c3d4bd182..e9a7da30c3 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -416,8 +416,8 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs # Set the frame specific options self.skysub_frame = skysub_frame self.scale_corr = scale_corr - self.ra_offsets = list(ra_offsets) if isinstance(ra_offsets, np.ndarray) else ra_offsets - self.dec_offsets = list(dec_offsets) if isinstance(dec_offsets, np.ndarray) else dec_offsets + self.ra_offsets = np.array(ra_offsets) if isinstance(ra_offsets, list) else ra_offsets + self.dec_offsets = np.array(dec_offsets) if isinstance(dec_offsets, list) else dec_offsets # If no ra_offsets or dec_offsets have been provided, initialise the lists self.user_alignment = True if self.ra_offsets is None and self.dec_offsets is None: @@ -427,8 +427,8 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs # User offsets are not provided, so turn off the user_alignment self.user_alignment = False # Initialise the lists of ra_offsets and dec_offsets - self.ra_offsets = [0.0 for _ in range(self.numfiles)] - self.dec_offsets = [0.0 for _ in range(self.numfiles)] + self.ra_offsets = np.zeros(self.numfiles) + self.dec_offsets = np.zeros(self.numfiles) # Check on Spectrograph input if spectrograph is None: @@ -1131,7 +1131,6 @@ def load(self): # Calculate the weights relative to the zeroth cube self.weights[ff] = 1.0 # exptime #np.median(flux_sav[resrt]*np.sqrt(ivar_sav[resrt]))**2 - wghts = self.weights[ff] * np.ones(sciImg.shape) # Get the slit image and then unset pixels in the slit image that are bad slitid_img_gpm = slitid_img_init * onslit_gpm.astype(int) @@ -1139,15 +1138,18 @@ def load(self): ################################## # Astrometric alignment to HST frames # TODO :: RJC requests this remains here... it is only used by RJC - if False: - # Get the coordinate bounds - slitlength = int(np.round(np.median(slits.get_slitlengths(initial=True, median=True)))) - numwav = int((np.max(waveimg) - wave0) / dwv) - raw_bins = self.spec.get_datacube_bins(slitlength, minmax, numwav) - # Now do the alignment - self.ra_offsets[ff], self.dec_offsets[ff] = hst_alignment(sciImg, ivar, waveimg, slitid_img_gpm, wghts, - self.all_wcs[ff], spec2DObj.tilts, slits, alignSplines, darcorr, - raw_bins=raw_bins) + # Get the coordinate bounds + slitlength = int(np.round(np.median(slits.get_slitlengths(initial=True, median=True)))) + numwav = int((np.max(waveimg) - wave0) / dwv) + raw_bins = self.spec.get_datacube_bins(slitlength, minmax, numwav) + wghts = np.ones(sciImg.shape) + # Now do the alignment + self.ra_offsets[ff], self.dec_offsets[ff] = hst_alignment(sciImg, ivar, waveimg, slitid_img_gpm, wghts, + self.all_wcs[ff], spec2DObj.tilts, slits, alignSplines, darcorr, + raw_bins=raw_bins) + # TODO :: Does this need to be included anywhere? + # self.ifu_ra[ff] += self.ra_offsets[ff] + # self.ifu_dec[ff] += self.dec_offsets[ff] ################################## # If individual frames are to be output without aligning them, @@ -1173,7 +1175,7 @@ def load(self): # Generate the datacube flxcube, sigcube, bpmcube, wave = \ datacube.generate_cube_subpixel(self.all_wcs[ff], bins, sciImg, ivar, waveimg, slitid_img_gpm, wghts, - spec2DObj.tilts, slits, alignSplines, darcorr, + self.all_wcs[ff], spec2DObj.tilts, slits, alignSplines, darcorr, self.ra_offsets[ff], self.dec_offsets[ff], overwrite=self.overwrite, whitelight_range=wl_wvrng, outfile=outfile, spec_subpixel=self.spec_subpixel, @@ -1217,12 +1219,12 @@ def run_align(self): # Grab cos(dec) for convenience cosdec = np.cos(np.mean(self.ifu_dec[0]) * np.pi / 180.0) # Initialize the RA and Dec offset arrays - ra_offsets, dec_offsets = [0 for _ in range(self.numfiles)], [0 for _ in range(self.numfiles)] + ra_offsets, dec_offsets = np.zeros(self.numfiles), np.zeros(self.numfiles) # Register spatial offsets between all frames if self.user_alignment: # The user has specified offsets - update these values accounting for the difference in header RA/DEC ra_offsets, dec_offsets = datacube.align_user_offsets(self.ifu_ra, self.ifu_dec, - self.ra_offsets, self.dec_offsets) + self.ra_offsets, self.dec_offsets) else: # Find the wavelength range where all frames overlap min_wl, max_wl = datacube.get_whitelight_range(np.max(self.mnmx_wv[:, :, 0]), # The max blue wavelength @@ -1246,7 +1248,7 @@ def run_align(self): msgs.error("Spectral range for WCS is incorrect for white light image") wl_imgs = datacube.generate_image_subpixel(image_wcs, voxedge, self.all_sci, self.all_ivar, self.all_wave, - slitid_img_gpm, self.all_wghts, + slitid_img_gpm, self.all_wghts, self.all_wcs, self.all_tilts, self.all_slits, self.all_align, self.all_dar, ra_offsets, dec_offsets, spec_subpixel=self.spec_subpixel, @@ -1288,7 +1290,7 @@ def compute_weights(self): # Calculate the relative spectral weights of all pixels return datacube.compute_weights_frompix(self.all_ra, self.all_dec, self.all_wave, self.all_sci, self.all_ivar, self.all_slitid, self._dspat, self._dwv, self.mnmx_wv, self.all_wghts, - self.all_tilts, self.all_slits, self.all_align, self.all_dar, + self.all_wcs, self.all_tilts, self.all_slits, self.all_align, self.all_dar, self.ra_offsets, self.dec_offsets, ra_min=self.cubepar['ra_min'], ra_max=self.cubepar['ra_max'], dec_min=self.cubepar['dec_min'], dec_max=self.cubepar['dec_max'], @@ -1377,7 +1379,7 @@ def run(self): # Generate the datacube flxcube, sigcube, bpmcube, wave = \ datacube.generate_cube_subpixel(cube_wcs, vox_edges, self.all_sci, self.all_ivar, self.all_wave, - self.all_slitid, self.all_wghts, + self.all_slitid, self.all_wghts, self.all_wcs, self.all_tilts, self.all_slits, self.all_align, self.all_dar, self.ra_offsets, self.dec_offsets, outfile=outfile, overwrite=self.overwrite, whitelight_range=wl_wvrng, @@ -1402,7 +1404,7 @@ def run(self): flxcube, sigcube, bpmcube, wave = \ datacube.generate_cube_subpixel(cube_wcs, vox_edges, self.all_sci[ff], self.all_ivar[ff], self.all_wave[ff], - self.all_slitid[ff], self.all_wghts[ff], + self.all_slitid[ff], self.all_wghts[ff], self.all_wcs[ff], self.all_tilts[ff], self.all_slits[ff], self.all_align[ff], self.all_dar[ff], self.ra_offsets[ff], self.dec_offsets[ff], overwrite=self.overwrite, whitelight_range=wl_wvrng, @@ -1482,14 +1484,14 @@ def hst_alignment(sciImg, ivar, waveimg, slitid_img_gpm, wghts, inmask = slitid_img_gpm * ((waveimg > 4107.0) & (waveimg < 4119.0)).astype(int) flxcube, sigcube, bpmcube, wave = \ datacube.generate_cube_subpixel(all_wcs, raw_bins, sciImg, ivar, waveimg, inmask, wghts, - tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, overwrite=False, + all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, overwrite=False, spec_subpixel=5, spat_subpixel=5, slice_subpixel=slice_subpixel) HdMap_raw, HdMapErr_raw = astrometry.fit_cube(flxcube, sigcube**2, all_wcs, line="HIdelta") # THEN DO Hgamma inmask = slitid_img_gpm * ((waveimg > 4346.0) & (waveimg < 4358.0)).astype(int) flxcube, sigcube, bpmcube, wave = \ datacube.generate_cube_subpixel(all_wcs, raw_bins, sciImg, ivar, waveimg, inmask, wghts, - tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, + all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, overwrite=False, spec_subpixel=5, spat_subpixel=5, slice_subpixel=slice_subpixel) HgMap_raw, HgMapErr_raw = astrometry.fit_cube(flxcube, sigcube**2, all_wcs, line="HIgamma") # Plot the emission line map diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index 5aa7365b9d..da88d3988c 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -957,7 +957,7 @@ def generate_WCS(crval, cdelt, equinox=2000.0, name="PYP_SPEC"): def compute_weights_frompix(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, dspat, dwv, mnmx_wv, wghtsImg, - all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets, + all_wcs, all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets, ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None, sn_smooth_npix=None, relative_weights=False, reference_image=None, whitelight_range=None, specname="PYPSPEC"): @@ -989,6 +989,8 @@ def compute_weights_frompix(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, The minimum and maximum wavelengths are stored in the [:,:,0] and [:,:,1] indices, respectively. wghtsImg (`numpy.ndarray`_, list): A list of 2D array containing the weights of each pixel, with shape (nspec, nspat) + all_wcs (`astropy.wcs.WCS`_, list): + A list of WCS objects, one for each frame. all_tilts (`numpy.ndarray`_, list): 2D wavelength tilts frame, or a list of tilt frames all_slits (:class:`~pypeit.slittrace.SlitTraceSet`, list): @@ -1047,19 +1049,19 @@ def compute_weights_frompix(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, # Generate the white light image # NOTE: hard-coding subpixel=1 in both directions for speed, and combining into a single image wl_full = generate_image_subpixel(image_wcs, voxedge, sciImg, ivarImg, waveImg, slitid_img_gpm, wghtsImg, - all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets, + all_wcs, all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets, spec_subpixel=1, spat_subpixel=1, slice_subpixel=1, combine=True) # Compute the weights - return compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, - all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets, + return compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitid_img_gpm, + all_wcs, all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets, wl_full[:, :, 0], dspat, dwv, ra_min=ra_min, ra_max=ra_max, dec_min=dec_min, dec_max=dec_max, wave_min=wave_min, sn_smooth_npix=sn_smooth_npix, relative_weights=relative_weights) def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, - all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets, + all_wcs, all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets, whitelight_img, dspat, dwv, ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None, sn_smooth_npix=None, relative_weights=False): @@ -1080,6 +1082,8 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, A list of 2D array containing the inverse variance image of each pixel, with shape (nspec, nspat) slitidImg (`numpy.ndarray`_, list): A list of 2D array containing the slit ID of each pixel, with shape (nspec, nspat) + all_wcs (`astropy.wcs.WCS`_, list): + A list of WCS objects, one for each frame. all_tilts (`numpy.ndarray`_, list): 2D wavelength tilts frame, or a list of tilt frames all_slits (:class:`~pypeit.slittrace.SlitTraceSet`, list): @@ -1116,9 +1120,9 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, msgs.info("Calculating the optimal weights of each pixel") # Check the inputs for combinations of lists or not, and then determine the number of frames _raImg, _decImg, _waveImg, _sciImg, _ivarImg, _slitidImg, \ - _all_tilts, _all_slits, _all_align, _all_dar, _ra_offsets, _dec_offsets = \ + _all_wcs, _all_tilts, _all_slits, _all_align, _all_dar, _ra_offsets, _dec_offsets = \ check_inputs([raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, - all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets]) + all_wcs, all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets]) numframes = len(_sciImg) # If there's only one frame, use uniform weighting @@ -1153,8 +1157,8 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, for ff in range(numframes): msgs.info("Extracting spectrum of highest S/N detection from frame {0:d}/{1:d}".format(ff + 1, numframes)) flxcube, sigcube, bpmcube, wave = \ - generate_cube_subpixel(whitelightWCS, bins, _sciImg[ff], _ivarImg[ff], _waveImg[ff]/wcs_scale, - _slitidImg[ff], np.ones(_sciImg[ff].shape), + generate_cube_subpixel(whitelightWCS, bins, _sciImg[ff], _ivarImg[ff], _waveImg[ff], + _slitidImg[ff], np.ones(_sciImg[ff].shape), _all_wcs[ff], _all_tilts[ff], _all_slits[ff], _all_align[ff], _all_dar[ff], _ra_offsets[ff], _dec_offsets[ff], spec_subpixel=1, spat_subpixel=1, slice_subpixel=1) @@ -1165,7 +1169,6 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, flux_stack[:, ff] = flxcube[0, 0, :]# * np.sqrt(normspec) # Note: sqrt(nrmspec), is because we want the S/N in a _single_ pixel (i.e. not spectral bin) ivar_stack[:, ff] = utils.inverse(sigcube[0, 0, :])**2 - embed() mask_stack = (flux_stack != 0.0) & (ivar_stack != 0.0) # Obtain a wavelength of each pixel wcs_res = whitelightWCS.wcs_pix2world(np.vstack((np.zeros(numwav), np.zeros(numwav), np.arange(numwav))).T, 0) @@ -1184,11 +1187,12 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, all_wghts[ff][ww] = interp1d(wave_spec, weights[ff], kind='cubic', bounds_error=False, fill_value="extrapolate")(waveImg[ff][ww]) msgs.info("Optimal weighting complete") + embed() return all_wghts def generate_image_subpixel(image_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, - tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, + all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, spec_subpixel=5, spat_subpixel=5, slice_subpixel=5, combine=False): """ Generate a white light image from the input pixels @@ -1214,6 +1218,9 @@ def generate_image_subpixel(image_wcs, bins, sciImg, ivarImg, waveImg, slitid_im wghtImg (`numpy.ndarray`_, list): A list of 2D weight images, or a single 2D image containing the weight data. + all_wcs (`astropy.wcs.WCS`_, list): + A list of WCS objects, or a single WCS object containing the WCS + information of each image. tilts (`numpy.ndarray`_, list): 2D wavelength tilts frame, or a list of tilt frames (see all_idx) slits (:class:`~pypeit.slittrace.SlitTraceSet`, list): @@ -1256,8 +1263,8 @@ def generate_image_subpixel(image_wcs, bins, sciImg, ivarImg, waveImg, slitid_im `numpy.ndarray`_: The white light images for all frames """ # Perform some checks on the input -- note, more complete checks are performed in subpixellate() - _sciImg, _ivarImg, _waveImg, _slitid_img_gpm, _wghtImg, _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset = \ - check_inputs([sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset]) + _sciImg, _ivarImg, _waveImg, _slitid_img_gpm, _wghtImg, _all_wcs, _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset = \ + check_inputs([sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset]) numframes = len(_sciImg) # Prepare the array of white light images to be stored @@ -1271,12 +1278,12 @@ def generate_image_subpixel(image_wcs, bins, sciImg, ivarImg, waveImg, slitid_im if combine: # Subpixellate img, _, _ = subpixellate(image_wcs, bins, _sciImg, _ivarImg, _waveImg, _slitid_img_gpm, _wghtImg, - _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset, + _all_wcs, _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset, spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, slice_subpixel=slice_subpixel) else: # Subpixellate img, _, _ = subpixellate(image_wcs, bins, _sciImg[fr], _ivarImg[fr], _waveImg[fr], _slitid_img_gpm[fr], _wghtImg[fr], - _tilts[fr], _slits[fr], _astrom_trans[fr], _all_dar[fr], _ra_offset[fr], _dec_offset[fr], + _all_wcs[fr], _tilts[fr], _slits[fr], _astrom_trans[fr], _all_dar[fr], _ra_offset[fr], _dec_offset[fr], spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, slice_subpixel=slice_subpixel) all_wl_imgs[:, :, fr] = img[:, :, 0] # Return the constructed white light images @@ -1284,7 +1291,7 @@ def generate_image_subpixel(image_wcs, bins, sciImg, ivarImg, waveImg, slitid_im def generate_cube_subpixel(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, - tilts, slits, astrom_trans, all_dar, + all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, spec_subpixel=5, spat_subpixel=5, slice_subpixel=5, overwrite=False, outfile=None, whitelight_range=None, debug=False): @@ -1311,6 +1318,8 @@ def generate_cube_subpixel(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_im wghtImg (`numpy.ndarray`_, list): A list of 2D array containing the weights of each pixel to be used in the combination + all_wcs (`astropy.wcs.WCS`_, list): + A list of `astropy.wcs.WCS`_ objects, one for each spec2d file tilts (list): A list of `numpy.ndarray`_ objects, one for each spec2d file, containing the tilts of each pixel @@ -1379,7 +1388,7 @@ def generate_cube_subpixel(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_im # Subpixellate subpix = subpixellate(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, - tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, + all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, slice_subpixel=slice_subpixel, debug=debug) # Extract the variables that we need @@ -1420,7 +1429,7 @@ def generate_cube_subpixel(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_im def subpixellate(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, - tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, + all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, spec_subpixel=5, spat_subpixel=5, slice_subpixel=5, debug=False): r""" Subpixellate the input data into a datacube. This algorithm splits each @@ -1455,6 +1464,8 @@ def subpixellate(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wgh wghtImg (`numpy.ndarray`_, list): A list of 2D array containing the weights of each pixel to be used in the combination + all_wcs (`astropy.wcs.WCS`_, list): + A list of `astropy.wcs.WCS`_ objects, one for each spec2d file tilts (list): A list of `numpy.ndarray`_ objects, one for each spec2d file, containing the tilts of each pixel @@ -1503,8 +1514,8 @@ def subpixellate(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wgh residual cube. The latter is only returned if debug is True. """ # Check the inputs for combinations of lists or not - _sciImg, _ivarImg, _waveImg, _gpmImg, _wghtImg, _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset = \ - check_inputs([sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset]) + _sciImg, _ivarImg, _waveImg, _gpmImg, _wghtImg, _all_wcs, _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset = \ + check_inputs([sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset]) numframes = len(_sciImg) # Prepare the output arrays From 56c229c4a3e2ee593bf41fb33fbbc4c55505688a Mon Sep 17 00:00:00 2001 From: rcooke Date: Mon, 11 Dec 2023 13:26:09 +0000 Subject: [PATCH 102/244] fixed optimal weights --- pypeit/coadd3d.py | 29 +++++++++++++++-------------- pypeit/core/datacube.py | 8 ++++---- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index e9a7da30c3..bfcea4b555 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -416,8 +416,8 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs # Set the frame specific options self.skysub_frame = skysub_frame self.scale_corr = scale_corr - self.ra_offsets = np.array(ra_offsets) if isinstance(ra_offsets, list) else ra_offsets - self.dec_offsets = np.array(dec_offsets) if isinstance(dec_offsets, list) else dec_offsets + self.ra_offsets = list(ra_offsets) if isinstance(ra_offsets, np.ndarray) else ra_offsets + self.dec_offsets = list(dec_offsets) if isinstance(dec_offsets, np.ndarray) else dec_offsets # If no ra_offsets or dec_offsets have been provided, initialise the lists self.user_alignment = True if self.ra_offsets is None and self.dec_offsets is None: @@ -427,8 +427,8 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs # User offsets are not provided, so turn off the user_alignment self.user_alignment = False # Initialise the lists of ra_offsets and dec_offsets - self.ra_offsets = np.zeros(self.numfiles) - self.dec_offsets = np.zeros(self.numfiles) + self.ra_offsets = [0.0 for _ in range(self.numfiles)] + self.dec_offsets = [0.0 for _ in range(self.numfiles)] # Check on Spectrograph input if spectrograph is None: @@ -1131,6 +1131,7 @@ def load(self): # Calculate the weights relative to the zeroth cube self.weights[ff] = 1.0 # exptime #np.median(flux_sav[resrt]*np.sqrt(ivar_sav[resrt]))**2 + wghts = self.weights[ff] * np.ones(sciImg.shape) # Get the slit image and then unset pixels in the slit image that are bad slitid_img_gpm = slitid_img_init * onslit_gpm.astype(int) @@ -1139,14 +1140,14 @@ def load(self): # Astrometric alignment to HST frames # TODO :: RJC requests this remains here... it is only used by RJC # Get the coordinate bounds - slitlength = int(np.round(np.median(slits.get_slitlengths(initial=True, median=True)))) - numwav = int((np.max(waveimg) - wave0) / dwv) - raw_bins = self.spec.get_datacube_bins(slitlength, minmax, numwav) - wghts = np.ones(sciImg.shape) - # Now do the alignment - self.ra_offsets[ff], self.dec_offsets[ff] = hst_alignment(sciImg, ivar, waveimg, slitid_img_gpm, wghts, - self.all_wcs[ff], spec2DObj.tilts, slits, alignSplines, darcorr, - raw_bins=raw_bins) + if False: + slitlength = int(np.round(np.median(slits.get_slitlengths(initial=True, median=True)))) + numwav = int((np.max(waveimg) - wave0) / dwv) + raw_bins = self.spec.get_datacube_bins(slitlength, minmax, numwav) + # Now do the alignment + self.ra_offsets[ff], self.dec_offsets[ff] = hst_alignment(sciImg, ivar, waveimg, slitid_img_gpm, wghts, + self.all_wcs[ff], spec2DObj.tilts, slits, alignSplines, darcorr, + raw_bins=raw_bins) # TODO :: Does this need to be included anywhere? # self.ifu_ra[ff] += self.ra_offsets[ff] # self.ifu_dec[ff] += self.dec_offsets[ff] @@ -1219,12 +1220,12 @@ def run_align(self): # Grab cos(dec) for convenience cosdec = np.cos(np.mean(self.ifu_dec[0]) * np.pi / 180.0) # Initialize the RA and Dec offset arrays - ra_offsets, dec_offsets = np.zeros(self.numfiles), np.zeros(self.numfiles) + ra_offsets, dec_offsets = [0 for _ in range(self.numfiles)], [0 for _ in range(self.numfiles)] # Register spatial offsets between all frames if self.user_alignment: # The user has specified offsets - update these values accounting for the difference in header RA/DEC ra_offsets, dec_offsets = datacube.align_user_offsets(self.ifu_ra, self.ifu_dec, - self.ra_offsets, self.dec_offsets) + self.ra_offsets, self.dec_offsets) else: # Find the wavelength range where all frames overlap min_wl, max_wl = datacube.get_whitelight_range(np.max(self.mnmx_wv[:, :, 0]), # The max blue wavelength diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index da88d3988c..a21b6de5e9 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -1053,7 +1053,7 @@ def compute_weights_frompix(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, spec_subpixel=1, spat_subpixel=1, slice_subpixel=1, combine=True) # Compute the weights - return compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitid_img_gpm, + return compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, all_wcs, all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets, wl_full[:, :, 0], dspat, dwv, ra_min=ra_min, ra_max=ra_max, dec_min=dec_min, dec_max=dec_max, wave_min=wave_min, @@ -1166,9 +1166,10 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, # TODO :: Think more about this... RJC removed the sqrt here - not sure why it needs to be detector pixel instead of spectral bin... # I think it's because we want to compare the S/N in a single pixel for a fair comparison, and not the S/N for a spectral bin. # For example, there may be a masked pixel in the spectral bin, which would give a lower S/N than a single pixel. - flux_stack[:, ff] = flxcube[0, 0, :]# * np.sqrt(normspec) # Note: sqrt(nrmspec), is because we want the S/N in a _single_ pixel (i.e. not spectral bin) - ivar_stack[:, ff] = utils.inverse(sigcube[0, 0, :])**2 + flux_stack[:, ff] = flxcube[:, 0, 0]# * np.sqrt(normspec) # Note: sqrt(nrmspec), is because we want the S/N in a _single_ pixel (i.e. not spectral bin) + ivar_stack[:, ff] = utils.inverse(sigcube[:, 0, 0])**2 + # Mask out any pixels that are zero in the flux or ivar stack mask_stack = (flux_stack != 0.0) & (ivar_stack != 0.0) # Obtain a wavelength of each pixel wcs_res = whitelightWCS.wcs_pix2world(np.vstack((np.zeros(numwav), np.zeros(numwav), np.arange(numwav))).T, 0) @@ -1187,7 +1188,6 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, all_wghts[ff][ww] = interp1d(wave_spec, weights[ff], kind='cubic', bounds_error=False, fill_value="extrapolate")(waveImg[ff][ww]) msgs.info("Optimal weighting complete") - embed() return all_wghts From 23df8f084796134ae82c4996df6155ca3312de7a Mon Sep 17 00:00:00 2001 From: rcooke Date: Mon, 11 Dec 2023 13:34:07 +0000 Subject: [PATCH 103/244] update default value --- pypeit/par/pypeitpar.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index 409f719c98..85b73af301 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -1607,8 +1607,7 @@ def __init__(self, slit_spec=None, relative_weights=None, align=None, combine=No 'is to divide each spec2d pixel into 25 subpixels during datacube creation. ' \ 'See also, spec_subpixel and slice_subpixel.' - # TODO :: Change this default value to 5 - defaults['slice_subpixel'] = 1 + defaults['slice_subpixel'] = 5 dtypes['slice_subpixel'] = int descr['slice_subpixel'] = 'When method=subpixel, slice_subpixel sets the subpixellation scale of ' \ 'each IFU slice. The default option is to divide each slice into 5 sub-slices ' \ From 88bf2336af1b0c1fd8a1a24b279ea314b3a9b856 Mon Sep 17 00:00:00 2001 From: rcooke Date: Mon, 11 Dec 2023 20:54:31 +0000 Subject: [PATCH 104/244] final fixes --- pypeit/coadd3d.py | 112 ++++++++++++++++++++-------------------- pypeit/core/datacube.py | 45 +++++++++++----- 2 files changed, 89 insertions(+), 68 deletions(-) diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index bfcea4b555..e6724a0dcf 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -1140,14 +1140,14 @@ def load(self): # Astrometric alignment to HST frames # TODO :: RJC requests this remains here... it is only used by RJC # Get the coordinate bounds - if False: + if True: slitlength = int(np.round(np.median(slits.get_slitlengths(initial=True, median=True)))) numwav = int((np.max(waveimg) - wave0) / dwv) raw_bins = self.spec.get_datacube_bins(slitlength, minmax, numwav) # Now do the alignment - self.ra_offsets[ff], self.dec_offsets[ff] = hst_alignment(sciImg, ivar, waveimg, slitid_img_gpm, wghts, + self.ra_offsets[ff], self.dec_offsets[ff] = hst_alignment(sciImg, ivar, waveimg, slitid_img_gpm, wghts, ra_img, dec_img, self.all_wcs[ff], spec2DObj.tilts, slits, alignSplines, darcorr, - raw_bins=raw_bins) + self._dspat, self._dwv, raw_bins=raw_bins) # TODO :: Does this need to be included anywhere? # self.ifu_ra[ff] += self.ra_offsets[ff] # self.ifu_dec[ff] += self.dec_offsets[ff] @@ -1240,6 +1240,7 @@ def run_align(self): # Generate the WCS image_wcs, voxedge, reference_image = \ datacube.create_wcs(self.all_ra, self.all_dec, self.all_wave, slitid_img_gpm, self._dspat, wavediff, + ra_offsets=ra_offsets, dec_offsets=dec_offsets, ra_min=self.cubepar['ra_min'], ra_max=self.cubepar['ra_max'], dec_min=self.cubepar['dec_min'], dec_max=self.cubepar['dec_max'], wave_min=self.cubepar['wave_min'], wave_max=self.cubepar['wave_max'], @@ -1353,6 +1354,7 @@ def run(self): # Generate the WCS, and the voxel edges cube_wcs, vox_edges, _ = \ datacube.create_wcs(self.all_ra, self.all_dec, self.all_wave, self.all_slitid, self._dspat, self._dwv, + ra_offsets=self.ra_offsets, dec_offsets=self.dec_offsets, ra_min=self.cubepar['ra_min'], ra_max=self.cubepar['ra_max'], dec_min=self.cubepar['dec_min'], dec_max=self.cubepar['dec_max'], wave_min=self.cubepar['wave_min'], wave_max=self.cubepar['wave_max'], @@ -1425,8 +1427,8 @@ def run(self): final_cube.to_file(outfile, hdr=hdr, overwrite=self.overwrite) -def hst_alignment(sciImg, ivar, waveimg, slitid_img_gpm, wghts, - all_wcs, tilts, slits, astrom_trans, all_dar, +def hst_alignment(sciImg, ivar, waveimg, slitid_img_gpm, wghts, raImg, decImg, + all_wcs, tilts, slits, astrom_trans, all_dar, dspat, dwave, spat_subpixel=10, spec_subpixel=10, slice_subpixel=1, raw_bins=None): """ This is currently only used by RJC. This function adds corrections to the RA and Dec pixels @@ -1444,26 +1446,24 @@ def hst_alignment(sciImg, ivar, waveimg, slitid_img_gpm, wghts, * ------ * Return updated pixel RA, Dec """ - ra_offset, dec_offset = 0.0, 0.0 + ra_offset, dec_offset = 0.0, 0.0 # These must be zero when calculating the offsets from pypeit import astrometry - niter = 3 + niter = 1 for ii in range(niter): ############ ## STEP 1 ## - Create a datacube around Hgamma ############ # Only use a small wavelength range - msgs.warn("TEMPORARY HACK: turning off subpixel spectrum until DAR correction is done properly") - if False: - wv_mask = (wave_sort > 4346.0) & (wave_sort < 4358.0) - # Create a WCS for this subcube - subcube_wcs, voxedge, reference_image = datacube.create_wcs(ra_sort[wv_mask], dec_sort[wv_mask], wave_sort[wv_mask], - dspat, dwave) - # Create the subcube - flxcube, varcube, bpmcube = datacube.subpixellate(subcube_wcs, ra_sort[wv_mask], dec_sort[wv_mask], wave_sort[wv_mask], flux_sort[wv_mask], ivar_sort[wv_mask], - wghts[wv_mask], spatpos[wv_mask], specpos[wv_mask], - all_spatid[wv_mask], tilts, slits, astrom_trans, all_dar, - voxedge, all_idx=None, - spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, debug=False) + inmask = slitid_img_gpm * ((waveimg > 4346.0) & (waveimg < 4358.0)).astype(int) + # Create a WCS for this subcube + subcube_wcs, voxedge, reference_image = datacube.create_wcs(raImg, decImg, waveimg, inmask, dspat, dwave) + # Create the subcube + flxcube, sigcube, bpmcube, wave = \ + datacube.generate_cube_subpixel(subcube_wcs, voxedge, sciImg, ivar, waveimg, inmask, wghts, + all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, + overwrite=False, spec_subpixel=5, spat_subpixel=5, + slice_subpixel=slice_subpixel) + if False: hdu = fits.PrimaryHDU(flxcube) hdu.writeto("tstHg.fits", overwrite=True) @@ -1471,51 +1471,53 @@ def hst_alignment(sciImg, ivar, waveimg, slitid_img_gpm, wghts, ############ ## STEP 2 ## - Create an emission line map of Hgamma ############ - print("TEMPORARY HACK: turning off HST alignment until DAR correction is done properly") # Compute an emission line map that is as consistent as possible to an archival HST image - #HgMap, HgMapErr = astrometry.fit_cube(flxcube.T, varcube.T, subcube_wcs) + HgMap, HgMapErr = astrometry.fit_cube(flxcube, sigcube ** 2, subcube_wcs, line="HIgamma") ############ ## STEP 3A ## - Generate another datacube using the raw_wcs and make an image of the emission line map # Need to generate a datacube near both Hgamma and Hdelta. ############ - #Is DAR correction being performed here? - slice_subpixel = 10 - # FIRST DO Hdelta - inmask = slitid_img_gpm * ((waveimg > 4107.0) & (waveimg < 4119.0)).astype(int) - flxcube, sigcube, bpmcube, wave = \ - datacube.generate_cube_subpixel(all_wcs, raw_bins, sciImg, ivar, waveimg, inmask, wghts, - all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, overwrite=False, - spec_subpixel=5, spat_subpixel=5, slice_subpixel=slice_subpixel) - HdMap_raw, HdMapErr_raw = astrometry.fit_cube(flxcube, sigcube**2, all_wcs, line="HIdelta") - # THEN DO Hgamma - inmask = slitid_img_gpm * ((waveimg > 4346.0) & (waveimg < 4358.0)).astype(int) - flxcube, sigcube, bpmcube, wave = \ - datacube.generate_cube_subpixel(all_wcs, raw_bins, sciImg, ivar, waveimg, inmask, wghts, - all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, - overwrite=False, spec_subpixel=5, spat_subpixel=5, slice_subpixel=slice_subpixel) - HgMap_raw, HgMapErr_raw = astrometry.fit_cube(flxcube, sigcube**2, all_wcs, line="HIgamma") - # Plot the emission line map - from matplotlib import pyplot as plt - plt.subplot(131) - plt.imshow(HdMap_raw, vmin=0, vmax=200, aspect=0.5) - plt.subplot(132) - plt.imshow(HgMap_raw, vmin=0, vmax=300, aspect=0.5) - plt.subplot(133) - plt.imshow(HdMap_raw*utils.inverse(HgMap_raw), vmin=0.5, vmax=0.7, aspect=0.5) - plt.show() - embed() - # np.save("Hd-Hg_preSlicerDAR.npy", HdMap_raw*utils.inverse(HgMap_raw)) + if False: + #Is DAR correction being performed here? + slice_subpixel = 10 + # FIRST DO Hdelta + inmask = slitid_img_gpm * ((waveimg > 4107.0) & (waveimg < 4119.0)).astype(int) + flxcube, sigcube, bpmcube, wave = \ + datacube.generate_cube_subpixel(all_wcs, raw_bins, sciImg, ivar, waveimg, inmask, wghts, + all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, overwrite=False, + spec_subpixel=5, spat_subpixel=5, slice_subpixel=slice_subpixel) + HdMap_raw, HdMapErr_raw = astrometry.fit_cube(flxcube, sigcube**2, all_wcs, line="HIdelta") + # THEN DO Hgamma + inmask = slitid_img_gpm * ((waveimg > 4346.0) & (waveimg < 4358.0)).astype(int) + flxcube, sigcube, bpmcube, wave = \ + datacube.generate_cube_subpixel(all_wcs, raw_bins, sciImg, ivar, waveimg, inmask, wghts, + all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, + overwrite=False, spec_subpixel=5, spat_subpixel=5, slice_subpixel=slice_subpixel) + HgMap_raw, HgMapErr_raw = astrometry.fit_cube(flxcube, sigcube**2, all_wcs, line="HIgamma") + # Plot the emission line map + from matplotlib import pyplot as plt + plt.subplot(131) + plt.imshow(HdMap_raw, vmin=0, vmax=200, aspect=0.5) + plt.subplot(132) + plt.imshow(HgMap_raw, vmin=0, vmax=300, aspect=0.5) + plt.subplot(133) + plt.imshow(HdMap_raw*utils.inverse(HgMap_raw), vmin=0.5, vmax=0.7, aspect=0.5) + plt.show() + embed() + # np.save("Hd-Hg_preSlicerDAR.npy", HdMap_raw*utils.inverse(HgMap_raw)) + ############ ## STEP 3B ## - Map the emission line map to an HST image, and vice-versa ############ - ra_corr, dec_corr, flx_corr = astrometry.map_image(HgMap, HgMapErr, subcube_wcs, ra_sort, dec_sort, raw_wcs, HgMap_raw, HgMapErr_raw, HdMap_raw, HdMapErr_raw) - # Loop over the slits and apply the corrections - slit_illum_ref_idx = 14 - for ss, spatid in enumerate(slits.spat_id): - idx = np.where(all_spatid == spatid)[0] - # Calculate the correction as a function of wavelength - + ra_corr, dec_corr = astrometry.map_image(HgMap, HgMapErr, subcube_wcs, raImg, decImg, slitid_img_gpm)#, ra_sort, dec_sort, raw_wcs) + if False: + # ra_corr, dec_corr, flx_corr = astrometry.map_image(HgMap, HgMapErr, subcube_wcs, ra_sort, dec_sort, raw_wcs, HgMap_raw, HgMapErr_raw, HdMap_raw, HdMapErr_raw) + # Loop over the slits and apply the corrections + slit_illum_ref_idx = 14 + for ss, spatid in enumerate(slits.spat_id): + idx = np.where(all_spatid == spatid)[0] + # Calculate the correction as a function of wavelength # embed() return ra_corr, dec_corr \ No newline at end of file diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index a21b6de5e9..26bc5da3c2 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -733,7 +733,8 @@ def check_inputs(list_inputs): msgs.error("The input arguments should all be of type 'list', or all not be of type 'list':") -def wcs_bounds(raImg, decImg, waveImg, slitid_img_gpm, ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None): +def wcs_bounds(raImg, decImg, waveImg, slitid_img_gpm, ra_offsets=None, dec_offsets=None, + ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None): """ Calculate the bounds of the WCS and the expected edges of the voxels, based on user-specified parameters or the extremities of the data. @@ -750,6 +751,10 @@ def wcs_bounds(raImg, decImg, waveImg, slitid_img_gpm, ra_min=None, ra_max=None, A list of 2D array containing the spat ID of each pixel, with shape (nspec, nspat). A value of 0 indicates that the pixel is not on a slit. All other values indicate the slit spatial ID. + ra_offsets : list, optional + A list of the RA offsets for each frame + dec_offsets : list, optional + A list of the Dec offsets for each frame ra_min : :obj:`float`, optional Minimum RA of the WCS ra_max : :obj:`float`, optional @@ -778,9 +783,18 @@ def wcs_bounds(raImg, decImg, waveImg, slitid_img_gpm, ra_min=None, ra_max=None, _wave_max : :obj:`float` Maximum wavelength of the WCS """ + # Check if the ra_offsets and dec_offsets are specified + if ra_offsets is None or dec_offsets is None: + if isinstance(raImg, list): + ra_offsets = [0.0]*len(raImg) + dec_offsets = [0.0]*len(raImg) + else: + ra_offsets = 0.0 + dec_offsets = 0.0 # Check the inputs - _raImg, _decImg, _waveImg, _slitid_img_gpm = check_inputs([raImg, decImg, waveImg, slitid_img_gpm]) - numframes = len(raImg) + _raImg, _decImg, _waveImg, _slitid_img_gpm, _ra_offsets, _dec_offsets = \ + check_inputs([raImg, decImg, waveImg, slitid_img_gpm, ra_offsets, dec_offsets]) + numframes = len(_raImg) # Loop over the frames and get the bounds - start by setting the default values _ra_min, _ra_max = ra_min, ra_max @@ -791,14 +805,14 @@ def wcs_bounds(raImg, decImg, waveImg, slitid_img_gpm, ra_min=None, ra_max=None, # Get the RA, Dec, and wavelength of the pixels on the slit if ra_min is None or ra_max is None: this_ra = _raImg[fr][_slitid_img_gpm[fr] > 0] - tmp_min, tmp_max = np.min(this_ra), np.max(this_ra) + tmp_min, tmp_max = np.min(this_ra)+_ra_offsets[fr], np.max(this_ra)+_ra_offsets[fr] if fr == 0 or tmp_min < _ra_min: _ra_min = tmp_min if fr == 0 or tmp_max > _ra_max: _ra_max = tmp_max if dec_min is None or dec_max is None: this_dec = _decImg[fr][_slitid_img_gpm[fr] > 0] - tmp_min, tmp_max = np.min(this_dec), np.max(this_dec) + tmp_min, tmp_max = np.min(this_dec)+_dec_offsets[fr], np.max(this_dec)+_dec_offsets[fr] if fr == 0 or tmp_min < _dec_min: _dec_min = tmp_min if fr == 0 or tmp_max > _dec_max: @@ -815,6 +829,7 @@ def wcs_bounds(raImg, decImg, waveImg, slitid_img_gpm, ra_min=None, ra_max=None, def create_wcs(raImg, decImg, waveImg, slitid_img_gpm, dspat, dwave, + ra_offsets=None, dec_offsets=None, ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None, reference=None, collapse=False, equinox=2000.0, specname="PYP_SPEC"): """ @@ -838,6 +853,10 @@ def create_wcs(raImg, decImg, waveImg, slitid_img_gpm, dspat, dwave, values in cubepar. dwave : float Linear wavelength step of each voxel (in Angstroms) + ra_offsets : list, optional + List of RA offsets for each frame (degrees) + dec_offsets : list, optional + List of Dec offsets for each frame (degrees) ra_min : float, optional Minimum RA of the WCS (degrees) ra_max : float, optional @@ -872,8 +891,9 @@ def create_wcs(raImg, decImg, waveImg, slitid_img_gpm, dspat, dwave, """ # Setup the cube ranges _ra_min, _ra_max, _dec_min, _dec_max, _wave_min, _wave_max = \ - wcs_bounds(raImg, decImg, waveImg, slitid_img_gpm, ra_min=ra_min, ra_max=ra_max, dec_min=dec_min, dec_max=dec_max, - wave_min=wave_min, wave_max=wave_max) + wcs_bounds(raImg, decImg, waveImg, slitid_img_gpm, + ra_offsets=ra_offsets, dec_offsets=dec_offsets, + ra_min=ra_min, ra_max=ra_max, dec_min=dec_min, dec_max=dec_max, wave_min=wave_min, wave_max=wave_max) # Grab cos(dec) for convenience. Use the average of the min and max dec cosdec = np.cos(0.5*(_dec_min+_dec_max) * np.pi / 180.0) @@ -1043,6 +1063,7 @@ def compute_weights_frompix(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, # Generate the WCS image_wcs, voxedge, reference_image = \ create_wcs(raImg, decImg, waveImg, slitid_img_gpm, dspat, wavediff, + ra_offsets=ra_offsets, dec_offsets=dec_offsets, ra_min=ra_min, ra_max=ra_max, dec_min=dec_min, dec_max=dec_max, wave_min=wave_min, wave_max=wave_max, reference=reference_image, collapse=True, equinox=2000.0, specname=specname) @@ -1132,7 +1153,7 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, # Check the WCS bounds _ra_min, _ra_max, _dec_min, _dec_max, _wave_min, _wave_max = \ - wcs_bounds(_raImg, _decImg, _waveImg, _slitidImg, + wcs_bounds(_raImg, _decImg, _waveImg, _slitidImg, ra_offsets=_ra_offsets, dec_offsets=_dec_offsets, ra_min=ra_min, ra_max=ra_max, dec_min=dec_min, dec_max=dec_max, wave_min=wave_min, wave_max=wave_max) # Find the location of the object with the highest S/N in the combined white light image @@ -1162,11 +1183,9 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, _all_tilts[ff], _all_slits[ff], _all_align[ff], _all_dar[ff], _ra_offsets[ff], _dec_offsets[ff], spec_subpixel=1, spat_subpixel=1, slice_subpixel=1) - # Calculate the S/N in a given spectral bin - # TODO :: Think more about this... RJC removed the sqrt here - not sure why it needs to be detector pixel instead of spectral bin... - # I think it's because we want to compare the S/N in a single pixel for a fair comparison, and not the S/N for a spectral bin. - # For example, there may be a masked pixel in the spectral bin, which would give a lower S/N than a single pixel. - flux_stack[:, ff] = flxcube[:, 0, 0]# * np.sqrt(normspec) # Note: sqrt(nrmspec), is because we want the S/N in a _single_ pixel (i.e. not spectral bin) + # Store the flux and ivar spectra of the highest S/N object. + # TODO :: This is the flux per spectral pixel, and not per detector pixel. Is this correct? + flux_stack[:, ff] = flxcube[:, 0, 0] ivar_stack[:, ff] = utils.inverse(sigcube[:, 0, 0])**2 # Mask out any pixels that are zero in the flux or ivar stack From 5baa4ba63ddc9ea9d730df8279089faa63305269 Mon Sep 17 00:00:00 2001 From: rcooke Date: Mon, 11 Dec 2023 21:24:01 +0000 Subject: [PATCH 105/244] reorder --- pypeit/coadd3d.py | 18 +++++++++--------- pypeit/core/datacube.py | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index e6724a0dcf..fc051285a9 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -391,10 +391,19 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs """ # TODO :: Consider loading all calibrations into a single variable within the main CoAdd3D parent class. + # Set the variables self.spec2d = spec2dfiles self.numfiles = len(spec2dfiles) self.par = par self.overwrite = overwrite + # Extract some parsets for simplicity + self.cubepar = self.par['reduce']['cube'] + self.flatpar = self.par['calibrations']['flatfield'] + self.senspar = self.par['sensfunc'] + # Extract some commonly used variables + self.method = self.cubepar['method'] + self.combine = self.cubepar['combine'] + self.align = self.cubepar['align'] # Do some quick checks on the input options if skysub_frame is not None: if len(skysub_frame) != self.numfiles: @@ -438,11 +447,6 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs self.spec = load_spectrograph(spectrograph) self.specname = self.spec.name - # Extract some parsets for simplicity - self.cubepar = self.par['reduce']['cube'] - self.flatpar = self.par['calibrations']['flatfield'] - self.senspar = self.par['sensfunc'] - # Initialise arrays for storage self.ifu_ra, self.ifu_dec = np.array([]), np.array([]) # The RA and Dec at the centre of the IFU, as stored in the header @@ -454,10 +458,6 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs self._dspat = None if self.cubepar['spatial_delta'] is None else self.cubepar['spatial_delta'] / 3600.0 # binning size on the sky (/3600 to convert to degrees) self._dwv = self.cubepar['wave_delta'] # linear binning size in wavelength direction (in Angstroms) - # Extract some commonly used variables - self.method = self.cubepar['method'] - self.combine = self.cubepar['combine'] - self.align = self.cubepar['align'] # If there is only one frame being "combined" AND there's no reference image, then don't compute the translation. if self.numfiles == 1 and self.cubepar["reference_image"] is None: if self.align: diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index 26bc5da3c2..afebe7812f 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -1570,7 +1570,7 @@ def subpixellate(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wgh for ss in range(slice_subpixel): if slice_subpixel > 1: # Only print this if there are multiple subslices - msgs.info(f"Subslice resampling {ss+1}/{slice_subpixel}") + msgs.info(f"Resampling subslice {ss+1}/{slice_subpixel}") # Generate an RA/Dec image for this subslice raimg, decimg, minmax = this_slits.get_radec_image(this_wcs, this_astrom_trans, this_tilts, slice_offset=slice_offs[ss]) this_ra = raimg[this_onslit_gpm] From 1e796ba8f92df4eacc422262d5bfb3a0de455cdc Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Mon, 11 Dec 2023 15:28:13 -0800 Subject: [PATCH 106/244] order matching updates --- doc/include/links.rst | 1 + pypeit/core/slitdesign_matching.py | 53 ++++ pypeit/edgetrace.py | 418 ++++------------------------- pypeit/tests/test_slitmatch.py | 65 +++++ 4 files changed, 172 insertions(+), 365 deletions(-) create mode 100644 pypeit/tests/test_slitmatch.py diff --git a/doc/include/links.rst b/doc/include/links.rst index e850dcc513..91e0d4fa7a 100644 --- a/doc/include/links.rst +++ b/doc/include/links.rst @@ -36,6 +36,7 @@ .. _scipy.optimize.curve_fit: https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.curve_fit.html .. _scipy.optimize.leastsq: https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.leastsq.html .. _scipy.optimize.least_squares: http://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.least_squares.html +.. _scipy.optimize.linear_sum_assignment: https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.linear_sum_assignment.html .. _scipy.optimize.OptimizeResult: http://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.OptimizeResult.html .. _scipy.optimize.differential_evolution: https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.differential_evolution.html .. _scipy.interpolate.interp1d: https://docs.scipy.org/doc/scipy/reference/generated/scipy.interpolate.interp1d.html diff --git a/pypeit/core/slitdesign_matching.py b/pypeit/core/slitdesign_matching.py index 3c8e50fc33..933187afab 100644 --- a/pypeit/core/slitdesign_matching.py +++ b/pypeit/core/slitdesign_matching.py @@ -14,6 +14,7 @@ from IPython import embed import numpy as np +from scipy import optimize from matplotlib import pyplot as plt from astropy.stats import sigma_clipped_stats @@ -362,3 +363,55 @@ def plot_matches(edgetrace, ind, x_model, yref, slit_index, nspat=2048, duplicat plt.ylim(0, edgetrace.shape[0]) plt.legend(loc=1) plt.show() + + +def match_positions_1D(measured, nominal, tol=None): + """ + Match a set of measured 1D positions against a nominal expectation with + uniqueness (i.e., more than one measured position cannot be matched to the + same nominal position). + + The function primarily uses `scipy.optimize.linear_sum_assignment`_ to + perform the matching, where the matrix of separations of each measured + position to every nominal position is used as the cost matrix. + + Args: + measured (`numpy.array`_): + Measured positions. Shape is ``(n,)``. + nominal (`numpy.array`_): + Expected positions. Shape is ``(m,)``. + tol (:obj:`float`, optional): + Maximum separation between measured and nominal positions to be + considered a match. If None, no limit is applied. + + Returns: + `numpy.ndarray`_: Indices of the elements in ``measured`` that are + matched to the elements of ``nominal``. Shape is ``(m,)``. If the + tolerance is set or the number of measurements is less than the nominal + set (i.e., ``n < m``), any element of ``nominal`` that does not have an + appropriate match in ``measured`` is given an index of -1. + """ + # Calculate the (n,m) separation matrix + # NOTE: This is the brute force approach. For *lots* of measuremens, this + # can be sped up by using a KDTree to build a sparse matrix with only a + # subset of the distances calculated. + sep = np.absolute(nominal[:,None] - measured[None,:]) + + # Perform the match + n_m, m_m = optimize.linear_sum_assignment(sep) + + # Remove any matches that don't meet the provided tolerance. + if tol is not None: + indx = sep[n_m,m_m] < tol + n_m = n_m[indx] + m_m = m_m[indx] + + # If there aren't any missing matches, just return the match vector + if n_m.size == nominal.size: + return m_m + + # Otherwise, insert -1 placeholders. + _m_m = np.full(nominal.size, -1, dtype=int) + _m_m[n_m] = m_m + return _m_m + diff --git a/pypeit/edgetrace.py b/pypeit/edgetrace.py index fae953194b..1766532a79 100644 --- a/pypeit/edgetrace.py +++ b/pypeit/edgetrace.py @@ -103,7 +103,6 @@ """ import os -import time import inspect from pathlib import Path from collections import OrderedDict @@ -118,21 +117,18 @@ from matplotlib import pyplot as plt from matplotlib import ticker, rc -from astropy.stats import sigma_clipped_stats from astropy import table from pypeit import msgs -from pypeit import io from pypeit import utils from pypeit import sampling from pypeit import slittrace from pypeit.datamodel import DataContainer from pypeit import calibframe -from pypeit.images.mosaic import Mosaic from pypeit.bitmask import BitMask from pypeit.display import display from pypeit.par.pypeitpar import EdgeTracePar -from pypeit.core import parse, pydl, procimg, pca, trace, slitdesign_matching +from pypeit.core import parse, procimg, trace, slitdesign_matching from pypeit.core import fitting from pypeit.images.buildimage import TraceImage from pypeit.tracepca import TracePCA @@ -716,12 +712,12 @@ def rectify(self, flux, bpm=None, extract_width=None, mask_threshold=0.5, side=' """ if self.pcatype is None: msgs.error('Must first run the PCA analysis for the traces; run build_pca.') - pca = (self.left_pca if side == 'left' else self.right_pca) \ + _pca = (self.left_pca if side == 'left' else self.right_pca) \ if self.par['left_right_pca'] else self.pca # Get the traces that cross the reference spatial position at # the first and last pixels of the image - first_last_trace = pca.predict(np.array([0,self.nspat-1])) + first_last_trace = _pca.predict(np.array([0,self.nspat-1])) # Use these two traces to define the spatial pixel coordinates # to sample start = np.ceil(np.amax(np.amin(first_last_trace, axis=1))).astype(int) @@ -730,7 +726,7 @@ def rectify(self, flux, bpm=None, extract_width=None, mask_threshold=0.5, side=' # Rectify the image # TODO: This has its limitations if the PCA is highly non-linear. ocol = np.arange(self.nspat+buffer)-start - return sampling.rectify_image(flux, pca.predict(ocol), bpm=bpm, ocol=ocol, + return sampling.rectify_image(flux, _pca.predict(ocol), bpm=bpm, ocol=ocol, max_ocol=self.nspat-1, extract_width=extract_width, mask_threshold=mask_threshold) @@ -4777,304 +4773,6 @@ def _fill_objects_table(self, maskdef_id): self.objects['TRACEID'] = utils.index_of_x_eq_y(self.objects['MASKDEF_ID'], self.design['MASKDEF_ID'], strict=True) -# NOTE: I'd like us to keep this commented mask_refine function around -# for the time being. - # def mask_refine(self, design_file=None, allow_resync=False, debug=False): - # """ - # Use the mask design data to refine the edge trace positions. - # - # Use of this method requires: - # - a PCA decomposition is available, - # - the traces are synchronized into left-right pairs, and - # - :attr:`spectrograph` has a viable `get_slitmask` method - # to read slit mask design data from a file. That file is - # either provided directly or pulled from one of the - # files used to construct the trace image; see - # `design_file`. The result of the `get_slitmask` method - # must provide a - # :class:`pypeit.spectrographs.slitmask.SlitMask` object - # with the slit-mask design data. - # - # TODO: Traces don't need to be synchronized... - # - # Also useful, but not required, is for :attr:`spectrograph` to - # have a viable `get_detector_map` method that provides a - # :class:`pypeit.spectrograph.opticalmodel.DetectorMap` object, - # which is used to provide a guess offset between the slit-mask - # focal-plane positions and the trace pixel positions. If no - # such `get_detector_method` exists, the guess offset is:: - # - # this - # - # and the match between expected and traced slit positions may - # be unstable. - # - # The method uses - # :class:`pypeit.spectrographs.slitmask.SlitRegister` to match - # the expected and traced position and identify both missing - # and erroneous trace locations. The former are used to add new - # traces and the latter are removed. The method also constructs - # the :attr:`design` and :attr:`objects` tables, depending on - # the data accessible via the - # :class:`pypeit.spectrographs.slitmask.SlitMask` instance. - # - # Used parameters from :attr:`par` - # (:class:`pypeit.par.pypeitpar.EdgeTracePar`) are - # `left_right_pca`, `mask_reg_maxiter`, `mask_reg_maxsep`, - # `mask_reg_sigrej`, and `ignore_alignment`. - # - # Args: - # design_file (:obj:`str`, optional): - # A file with the mask design data. If None, the method - # will use the first file in :attr:`files`; if - # :attr:`files` is also None, the method will raise an - # exception. - # debug (:obj:`bool`, optional): - # Run in debug mode. - # """ - # # Still not done with this function... - # raise NotImplementedError() - # - # # Check that there are traces to refine! - # if self.is_empty: - # msgs.error('No traces to refine.') - # - # # The PCA decomposition must have already been determined - # if self.pcatype is None: - # msgs.error('Must first run the PCA analysis for the traces; run build_pca.') - # - # # Get the file to use when parsing the mask design information - # _design_file = (None if self.traceimg.files is None else self.traceimg.files[0]) \ - # if design_file is None else design_file - # if _design_file is None or not os.path.isfile(_design_file): - # msgs.error('Slit-mask design file not found or none provided.') - # - # # Get the paramters to use - # maxiter = self.par['mask_reg_maxiter'] - # maxsep = self.par['mask_reg_maxsep'] - # sigma = self.par['mask_reg_sigrej'] - # ignore_alignment = self.par['ignore_alignment'] - # - # # TODO: Set allow_resync and design_file to be a parameters, as - # # well? - # - # # Read the design data - # msgs.info('Reading slit-mask design information from: {0}'.format(_design_file)) - # if self.spectrograph.get_slitmask(_design_file) is None: - # msgs.error('Unable to read design file or no slit-mask design reader ' - # 'defined for {0}.'.format(self.spectrograph.spectrograph)) - # - # # Match both left and right edges simultaneously - # x_design = np.array([self.spectrograph.slitmask.bottom[:, 0], - # self.spectrograph.slitmask.top[:, 0]]).T.ravel() - # reference_row = self.left_pca.reference_row if self.par['left_right_pca'] \ - # else self.pca.reference_row - # x_det = self.edge_fit[reference_row, :] - # - # # Mask traces that are fully masked, except if they were - # # specifically inserted in a previous step - # # TODO: Should the BOXSLITS also be included here? - # x_det_bpm = self.fully_masked_traces(flag=self.bitmask.bad_flags, - # exclude=self.bitmask.insert_flags) - # - # # x_design = np.amin(self.spectrograph.slitmask.corners[:,:,0], axis=1) - # # side = self.traceid < 0 - # # x_det = self.edge_fit[self.pca.reference_row,side] - # - # # x_design = np.amax(self.spectrograph.slitmask.corners[:,:,0], axis=1) - # # side = self.traceid > 0 - # # x_det = self.edge_fit[self.pca.reference_row,side] - # - # # Estimate the scale in pixels/mm as the telescope platescale - # # in arcsec/mm divided by the detector platescale in - # # arcsec/pixel - # pix_per_mm = self.spectrograph.telescope.platescale() \ - # / self.traceimg.detector['platescale'] - # # / self.spectrograph.detector[self.det - 1]['platescale'] - # - # # If the traces are synchronized, use the estimated scale to - # # first mask edges that yeild slits that are too small relative - # # to the range of slit lengths in the mask file. - # if self.is_synced: - # slit_len_det = np.diff(x_det.reshape(-1, 2), axis=1).ravel() - # slit_len_mask = np.diff(x_design.reshape(-1, 2), axis=1).ravel() * pix_per_mm - # indx = (slit_len_det < np.amin(slit_len_mask) / 1.1) \ - # | (slit_len_det > np.amax(slit_len_mask) * 1.1) - # if np.any(indx): - # msgs.info('Removing {0} edges that form (an) '.format(np.sum(indx) * 2) - # + 'errantly small or large slit(s) compared to the mask design data.') - # x_det_bpm[np.repeat(indx, 2)] = True - # - # # Initial guess for the offset - # try: - # raise NotImplementedError() - # # Try using the spectrograph detector map - # self.spectrograph.get_detector_map() - # # Set the offset based on the location of this detector - # offset = self.spectrograph.detector_map.image_coordinates( - # self.spectrograph.detector_map.npix[0] / 2, - # self.spectrograph.detector_map.npix[1] / 2, - # detector=self.traceimg.detector.det, - # in_mm=False)[0][0] - self.spectrograph.detector_map.npix[0] / 2 - # # Set the bounds to some nominal fraction of the detector - # # size and pix/mm scale; allow for a +/- 10% deviation in - # # the pixel scale - # # TODO: Is 10% generally enough (for any instrument)? Make - # # this a (spectrograph-specific) parameter? - # offset_rng = [offset - 0.1 * self.spectrograph.detector_map.npix[0], - # offset + 0.1 * self.spectrograph.detector_map.npix[0]] - # except: - # # No detector map - # msgs.warn('No detector map available for {0}'.format(self.spectrograph.spectrograph) - # + '; attempting to match to slit-mask design anyway.') - # # Set the guess offset such that two sets of coordinates - # # are offset to their mean - # offset = np.mean(x_det) - np.mean(pix_per_mm * x_design) - # # Set the offset range - # offset_rng = [offset - np.absolute(np.amin(x_det) - np.amin(pix_per_mm * x_design)) * 1.1, - # offset + np.absolute(np.amax(pix_per_mm * x_design) - np.amax(x_det)) * 1.1] - # - # # import pdb - # # pdb.set_trace() - # # - # # slitmask.xc_trace(x_det, x_design, pix_per_mm) - # # - # # pdb.set_trace() - # - # # The solution can be highly dependent on the initial guess for - # # the offset, so do an initial grid search to get close to the - # # solution. - # msgs.info('Running a grid search to try to find the best starting offset.') - # # Step by 2 pixels - # off = np.arange(offset_rng[0], offset_rng[1], 2).astype(float) - # rms = np.zeros_like(off, dtype=float) - # scl = np.zeros_like(off, dtype=float) - # par = np.array([0, pix_per_mm]) - # bounds = np.array([offset_rng, [pix_per_mm / 1.1, pix_per_mm * 1.1]]) - # register = slitmask.SlitRegister(x_det, x_design, trace_mask=x_det_bpm) - # - # # NOTE: The commented approach below gets the RMS at each - # # offset point just using the estimated scale. This is faster - # # than the approach taken, but results are sensitive to the - # # accuracy of the estimated scale, which can lead to problems - # # in corner cases. - # # for i in range(off.size): - # # print('Grid point: {0}/{1}'.format(i+1, off.size), end='\r') - # # par[0] = off[i] - # # register.par = par - # # minsep = register.match(unique=True)[1] - # # rms[i] = sigma_clipped_stats(minsep, sigma=5)[2] - # # print('Grid point: {0}/{0}'.format(off.size)) - # - # # For each grid point, keep the offset fixed and find the best - # # scale. No rejection iterations are performed. - # for i in range(off.size): - # print('Grid point: {0}/{1}'.format(i + 1, off.size), end='\r') - # par[0] = off[i] - # register.find_best_match(guess=par, fix=[True, False], bounds=bounds, penalty=False) - # minsep = register.match(unique=True)[1] - # scl[i] = register.par[1] - # rms[i] = sigma_clipped_stats(minsep, sigma=5)[2] - # print('Grid point: {0}/{0}'.format(off.size)) - # - # # Use the grid point with the best RMS - # minindx = np.argmin(rms) - # offset = off[minindx] - # best_rms = rms[minindx] - # msgs.info('Minimum RMS ({0:.2f}) found with offset = {1:.2f}'.format(best_rms, offset)) - # if debug: - # # Plot the result - # ax1 = plt.subplot(211) - # ax1.scatter(off, rms, color='k', marker='.', s=100, lw=0, zorder=0) - # ax1.scatter(offset, best_rms, color='C3', marker='x', s=50, zorder=1) - # ax1.set_xlabel('Trace Offset (pix)') - # ax1.set_ylabel('RMS (det-mask; pix)') - # ax1.set_title('Grid search for initial offset') - # ax2 = plt.subplot(212, sharex=ax1) - # ax2.scatter(off, scl, color='k', marker='.', s=100, lw=0, zorder=0) - # ax2.set_ylabel('Best-fit scale') - # plt.show() - # - # # Do the final fit with some rejection iterations - # register.find_best_match(guess=[offset, pix_per_mm], bounds=bounds, penalty=False, - # maxiter=maxiter, maxsep=maxsep, sigma=sigma, debug=debug) - # - # if debug: - # register.show(minmax=[0, self.nspat], synced=True) - # - # # Find the missing, bad, and masked traces - # missing, bad = register.trace_mismatch(minmax=[0, self.nspat], synced=True) - # # masked_by_registration = np.where(register.trace_mask & np.invert(x_det_bpm))[0] - # # bad = np.append(bad, masked_by_registration) - # bad = np.append(bad, np.where(register.trace_mask | x_det_bpm)[0]) - # - # # Ignore missing alignment boxes - # if ignore_alignment: - # missing = missing[np.invert(self.spectrograph.slitmask.alignment_slit[missing // 2])] - # found_alignment_slits = register.match_index[ - # self.spectrograph.slitmask.alignment_slit[register.match_index // 2]] - # bad = np.append(bad, found_alignment_slits) - # - # # Report - # msgs.info('Best-fitting offset and scale for mask coordinates: {0:.2f} {1:.2f}'.format( - # *register.par)) - # msgs.info('Traces will {0} alignment slits'.format('exclude' if ignore_alignment - # else 'include')) - # msgs.info('Number of missing mask traces to insert: {0}'.format(len(missing))) - # msgs.info('Number of bad or alignment traces to remove: {0}'.format(len(bad))) - # - # if self.is_synced and (len(missing) - len(bad)) % 2 != 0: - # if allow_resync: - # msgs.warning('Difference in added and removed traces is odd; will resync traces.') - # else: - # msgs.error('Difference in added and removed traces desyncronizes traces.') - # - # if len(bad) > 0: - # # Remove the bad traces and rebuild the pca - # rmtrace = np.zeros(self.ntrace, dtype=bool) - # rmtrace[bad] = True - # self.remove_traces(rmtrace, rebuild_pca=True) - # - # if len(missing) > 0: - # # Even indices are lefts, odd indices are rights - # side = missing % 2 * 2 - 1 - # # Predict the traces using the PCA - # missing_traces = self.predict_traces(register.match_coo[missing], side) - # # Insert them - # self.insert_traces(side, missing_traces, mode='mask') - # - # # import pdb - # # pdb.set_trace() - # - # if len(bad) > 0 or len(missing) > 0: - # # Traces were removed and/or inserted, resync or recheck that the edges are synced. - # if (len(missing) - len(bad)) % 2 != 0 and allow_resync: - # self.sync(rebuild_pca=True) - # else: - # self.check_synced(rebuild_pca=True) - # reference_row = self.left_pca.reference_row if self.par['left_right_pca'] \ - # else self.pca.reference_row - # # Reset the match after removing/inserting traces - # x_det = self.edge_fit[reference_row, :] - # # TODO: Should the BOXSLITS also be included here? - # x_det_bpm = self.fully_masked_traces(flag=self.bitmask.bad_flags, - # exclude=self.bitmask.insert_flags) - # register = slitmask.SlitRegister(x_det, x_design, trace_mask=x_det_bpm, - # guess=[offset, pix_per_mm], bounds=bounds, - # penalty=False, maxiter=maxiter, maxsep=maxsep, - # sigma=sigma, debug=debug, fit=True) - # - # # TODO: This fit should *never* result in missing or bad - # # traces! Keep this for a while until we feel like we've - # # vetted the code well enough. - # missing, bad = register.trace_mismatch(minmax=[0, self.nspat], synced=True) - # if len(missing) != 0 or len(bad) != 0: - # msgs.error('CODING ERROR: Should never find missing or bad traces in re-fit!') - # - # # Fill the slit-design and object tables - # self._fill_design_table(register, _design_file) - # self._fill_objects_table(register) - def order_refine(self, debug=False): """ For echelle spectrographs, attempt to add any orders that are not @@ -5138,6 +4836,9 @@ def order_refine_fixed_format(self, debug=False): # requires the spatial positoion at the relevant reference row. # This needs to be checked. + embed() + exit() + # First match the expected orders spat_offset = self.match_order() @@ -5471,83 +5172,70 @@ def match_order(self): if self.spectrograph.order_spat_pos is None: msgs.error('Coding error: order_spat_pos not defined for {0}!'.format( self.spectrograph.__class__.__name__)) + if not self.is_synced: + msgs.error('EdgeTraceSet must be synced to match to orders.') offset = self.par['order_offset'] if offset is None: offset = 0.0 # Get the order centers in fractions of the detector width. This - # requires the slits to be synced! Masked elements in slit_cen are for - # bad slits. - slit_cen = self.slit_spatial_center() - - # Calculate the separation between the order and every - sep = self.spectrograph.order_spat_pos[:,None] - slit_cen[None,:] - offset - # Find the smallest offset for each order - slit_indx = np.ma.MaskedArray(np.ma.argmin(np.absolute(sep), axis=1)) + # requires the slits to be synced (checked above)! Masked elements in + # slit_cen are for bad slits or syncing. + slit_cen = self.slit_spatial_center() + offset + good_sync = np.logical_not(np.ma.getmaskarray(slit_cen)) + # "slit_indx" matches the "slit" index to the order number. I.e., + # "slit_indx" has one element per expected order, and the value at a + # given position is the index of the paired traces for the relevant + # order. + slit_indx = slitdesign_matching.match_positions_1D( + slit_cen.data[good_sync], # (Good) Measured positions + self.spectrograph.order_spat_pos, # Expected positions + tol=self.par['order_match']) # Matching tolerance + + # Boolean array selecting found orders + fnd = slit_indx > -1 + missed_orders = self.spectrograph.orders[np.logical_not(fnd)] + if not np.all(fnd): + msgs.warn(f'Did not find all orders! Missing orders: {missed_orders}') + + # Flag paired edges that were not matched to a known order + nomatch = np.setdiff1d(np.arange(np.sum(good_sync)), slit_indx[fnd]) + if nomatch.size > 0: + msgs.warn(f'Flagging {nomatch.size} trace pairs as not being matched to an order.') + # Create a vector that selects the appropriate traces. This + # *assumes* that the traces are left-right syncronized and the order + # has not changed between the order of the traces in the relevant + # array and how the centers of the synced traces are computed + # (slit_spatial_center). + flag = np.append(2*nomatch, 2*nomatch+1) + self.edge_msk[:,flag] = self.bitmask.turn_on(self.edge_msk[:,flag], 'ORDERMISMATCH') # Minimum separation between the order and its matching slit; # keep the signed value for reporting, but used the absolute # value of the difference for vetting below. - sep = sep[(np.arange(self.spectrograph.norders),slit_indx)] - med_offset = np.median(sep) - min_sep = np.absolute(sep - med_offset) + # NOTE: This includes indices for orders that were not found. This is + # largely for book-keeping purposes in the print statement below. + sep = self.spectrograph.order_spat_pos - slit_cen.data[good_sync][slit_indx] + med_offset = np.median(sep[fnd]) # Report msgs.info(f'Median offset is {med_offset:.3f}.') msgs.info('After offsetting, order-matching separations are:') - msgs.info(' {0:>6} {1:>4} {2:>6}'.format('ORDER', 'PAIR', 'SEP')) - msgs.info(' {0} {1} {2}'.format('-'*6, '-'*4, '-'*6)) + msgs.info(f' {"ORDER":>6} {"PAIR":>4} {"SEP":>6}') + msgs.info(f' {"-"*6} {"-"*4} {"-"*6}') for i in range(self.spectrograph.norders): - msgs.info(' {0:>6} {1:>4} {2:6.3f}'.format(self.spectrograph.orders[i], i+1, sep[i])) - msgs.info(' {0} {1} {2}'.format('-'*6, '-'*4, '-'*6)) - - # Single slit matched to multiple orders - uniq, cnts = np.unique(slit_indx.compressed(), return_counts=True) - for u in uniq[cnts > 1]: - # Find the unmasked and multiply-matched indices - indx = (slit_indx.data == u) & np.logical_not(np.ma.getmaskarray(slit_indx)) - - # we need a masked version of min_sep array with only relevant indices - # as the below masking of slit_indx needs to be in correct shape - min_sep_mask = np.ones_like(min_sep, dtype = bool) - min_sep_mask[indx] = False - min_sep_masked = np.ma.masked_array(min_sep, min_sep_mask) - - # Keep the one with the smallest separation and mask the rest - slit_indx[np.setdiff1d(np.where(indx), [np.argmin(min_sep_masked)])] = np.ma.masked - - # Flag orders separated by more than the provided threshold - if self.par['order_match'] is not None: - indx = (min_sep > self.par['order_match']) \ - & np.logical_not(np.ma.getmaskarray(min_sep)) - if np.any(indx): - # Flag the associated traces - _indx = np.isin(np.absolute(self.traceid), (slit_indx[indx]).compressed()+1) - self.edge_msk[:,_indx] = self.bitmask.turn_on(self.edge_msk[:,_indx], - 'ORDERMISMATCH') - # Disassociate these orders from any slit - slit_indx[indx] = np.ma.masked - - # Unmatched slits - indx = np.logical_not(np.isin(np.arange(self.nslits), slit_indx.compressed())) - if np.any(indx): - # This works because the traceids are sorted and synced - indx = np.repeat(indx, 2) - self.edge_msk[:,indx] = self.bitmask.turn_on(self.edge_msk[:,indx], 'NOORDER') - - # Warning that there are missing orders - missed_order = np.ma.getmaskarray(slit_indx) - if np.any(missed_order): - msgs.warn('Did not find all orders! Missing orders: {0}'.format( - ', '.join(self.spectrograph.orders[missed_order].astype(str)))) + if fnd[i]: + msgs.info(f' {self.spectrograph.orders[i]:>6} {i+1:>4} {sep[i]:6.3f}') + else: + msgs.info(f' {self.spectrograph.orders[i]:>6} {"N/A":>4} {"MISSED":>6}') + msgs.info(f' {"-"*6} {"-"*4} {"-"*6}') # Instantiate the order ID; 0 means the order is unassigned self.orderid = np.zeros(self.nslits*2, dtype=int) - found_orders = self.spectrograph.orders[np.logical_not(missed_order)] - nfound = len(found_orders) - indx = (2*slit_indx.compressed()[:,None] + np.tile(np.array([0,1]), (nfound,1))).ravel() - self.orderid[indx] = (np.array([-1,1])[None,:]*found_orders[:,None]).ravel() + raw_indx = np.arange(self.nslits)[good_sync][slit_indx[fnd]] + indx = (2*raw_indx[:,None] + np.tile(np.array([0,1]), (np.sum(fnd),1))).ravel() + self.orderid[indx] = (np.array([-1,1])[None,:]*self.spectrograph.orders[fnd,None]).ravel() return med_offset diff --git a/pypeit/tests/test_slitmatch.py b/pypeit/tests/test_slitmatch.py new file mode 100644 index 0000000000..fe03404c68 --- /dev/null +++ b/pypeit/tests/test_slitmatch.py @@ -0,0 +1,65 @@ + +from IPython import embed + +import numpy as np +from pypeit.core import slitdesign_matching + +def test_match_positions(): + + # Random number generator + rng = np.random.default_rng(99) + + # Fake nominal positions + nominal = np.linspace(0, 1, 5) + d = np.mean(np.diff(nominal)) + + #---------- + # Test 1: Same number of measurements + + # Vectors used to rearrange and then recover a set of "measured" positions + isrt = np.arange(nominal.size) + rng.shuffle(isrt) + srt = np.argsort(isrt) + + # Generate fake data with no matching ambiguity + measured = rng.uniform(low=-d/3, high=d/3, size=nominal.size) + nominal[isrt] + + match = slitdesign_matching.match_positions_1D(measured, nominal) + + assert np.array_equal(match, srt), 'Bad match' + + #---------- + # Test 2: Add ambiguity by appending another measurement that is a better + # match than the existing one + measured = np.append(measured, [nominal[2]]) + + match = slitdesign_matching.match_positions_1D(measured, nominal) + assert match[2] == nominal.size, 'Should use new measurement' + assert np.sum(np.absolute(nominal - measured[match])) \ + < np.sum(np.absolute(nominal - measured[srt])), 'Should get a better match' + + #---------- + # Test 3: Fake a missing measurement + measured = measured[1:5] + match = slitdesign_matching.match_positions_1D(measured, nominal) + + assert match.size == nominal.size, 'Match vector should be the same size as nominal' + assert match[isrt[0]] == -1, 'Should not match missing element' + + #---------- + # Test 4: Missing a measurement and effectively a repeat of an existing one + measured = np.append(measured, [nominal[1]]) + match = slitdesign_matching.match_positions_1D(measured, nominal) + + assert match.size == nominal.size, 'Match vector should be the same size as nominal' + assert np.all(match > -1), 'All should match' + assert np.sum(np.absolute(nominal - measured[match]) > d/2) == 1, 'Should be one large outlier' + + #---------- + # Test 5: Same as test 4, but include a tolerance that should remove the outlier + match = slitdesign_matching.match_positions_1D(measured, nominal, tol=d/2) + + assert match.size == nominal.size, 'Match vector should be the same size as nominal' + assert match[isrt[0]] == -1, 'Should not match missing element' + + From 0959658f14b306587fd817e66e0b1707b76018a3 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Mon, 11 Dec 2023 16:12:01 -0800 Subject: [PATCH 107/244] debugging --- pypeit/core/slitdesign_matching.py | 4 +- pypeit/edgetrace.py | 107 +++++++++++++++-------------- 2 files changed, 58 insertions(+), 53 deletions(-) diff --git a/pypeit/core/slitdesign_matching.py b/pypeit/core/slitdesign_matching.py index 933187afab..eeac5d2f20 100644 --- a/pypeit/core/slitdesign_matching.py +++ b/pypeit/core/slitdesign_matching.py @@ -391,8 +391,8 @@ def match_positions_1D(measured, nominal, tol=None): set (i.e., ``n < m``), any element of ``nominal`` that does not have an appropriate match in ``measured`` is given an index of -1. """ - # Calculate the (n,m) separation matrix - # NOTE: This is the brute force approach. For *lots* of measuremens, this + # Calculate the (m,n) separation matrix + # NOTE: This is the brute force approach. For *lots* of measurements, this # can be sped up by using a KDTree to build a sparse matrix with only a # subset of the distances calculated. sep = np.absolute(nominal[:,None] - measured[None,:]) diff --git a/pypeit/edgetrace.py b/pypeit/edgetrace.py index 1766532a79..9b356335c9 100644 --- a/pypeit/edgetrace.py +++ b/pypeit/edgetrace.py @@ -172,7 +172,8 @@ def __init__(self): ('ABNORMALSLIT_SHORT', 'Slit formed by left and right edge is abnormally short'), ('ABNORMALSLIT_LONG', 'Slit formed by left and right edge is abnormally long'), ('USERRMSLIT', 'Slit removed by user'), - ('NOORDER', 'Unable to associate this trace with an echelle order (echelle ' + ('NOORDER', '(DEPRECATED as of version 1.15.0; use ORDERMISMATCH). ' + 'Unable to associate this trace with an echelle order (echelle ' 'spectrographs only)'), ('ORDERMISMATCH', 'Slit traces are not well matched to any echelle order (echelle ' 'spectrographs only)'), @@ -4793,12 +4794,12 @@ def order_refine(self, debug=False): # Update the PCA self.build_pca() + reference_row = self.left_pca.reference_row if self.par['left_right_pca'] \ + else self.pca.reference_row if self.spectrograph.ech_fixed_format: - add_left, add_right = self.order_refine_fixed_format(debug=debug) + add_left, add_right = self.order_refine_fixed_format(reference_row, debug=debug) rmtraces = None else: - reference_row = self.left_pca.reference_row if self.par['left_right_pca'] \ - else self.pca.reference_row add_left, add_right, rmtraces \ = self.order_refine_free_format(reference_row, debug=debug) @@ -4819,9 +4820,9 @@ def order_refine(self, debug=False): # If fixed-format, rematch the orders if self.spectrograph.ech_fixed_format: - self.match_order() + self.match_order(reference_row=reference_row) - def order_refine_fixed_format(self, debug=False): + def order_refine_fixed_format(self, reference_row, debug=False): """ Refine the order locations for fixed-format Echelles. """ @@ -4836,11 +4837,8 @@ def order_refine_fixed_format(self, debug=False): # requires the spatial positoion at the relevant reference row. # This needs to be checked. - embed() - exit() - # First match the expected orders - spat_offset = self.match_order() + spat_offset = self.match_order(reference_row=reference_row) available_orders = self.orderid[1::2] missed_orders = np.setdiff1d(self.spectrograph.orders, available_orders) @@ -5080,16 +5078,15 @@ def slit_spatial_center(self, normalized=True, spec=None, use_center=False, Return coordinates normalized by the size of the detector. spec (:obj:`int`, optional): - Spectral position (row) at which to return the - spatial position. If ``None``, set at the central row - (i.e., ``self.nspat//2``) + Spectral position (row) at which to return the spatial position. + If ``None``, use the PCA reference row if a PCA exists or the + central row (i.e., ``self.nspec//2``), otherwise. use_center (:obj:`bool`, optional): Use the measured centroids to define the slit edges even if the slit edges have been otherwise modeled. include_box (:obj:`bool`, optional): Include box slits in the calculated coordinates. - Returns: `numpy.ma.MaskedArray`_: Spatial coordinates of the slit centers in pixels or in fractions of the detector. Masked @@ -5104,7 +5101,13 @@ def slit_spatial_center(self, normalized=True, spec=None, use_center=False, # TODO: Use reference_row by default? Except that it's only # defined if the PCA is defined. - _spec = self.nspec//2 if spec is None else spec + _spec = spec + if _spec is None: + if self.pcatype is None: + _spec = self.nspec//2 + else: + _spec = self.left_pca.reference_row if self.par['left_right_pca'] \ + else self.pca.reference_row # Synced, spatially sorted traces are always ordered in left, # right pairs @@ -5115,52 +5118,54 @@ def slit_spatial_center(self, normalized=True, spec=None, use_center=False, slit_c[good_slit] = np.mean(trace_cen[_spec,:].reshape(-1,2), axis=1) return slit_c/self.nspat if normalized else slit_c - def match_order(self): + def match_order(self, reference_row=None): """ Match synchronized slits to the expected echelle orders. This function will fault if called for non-Echelle spectrographs! - For Echelle spectrographs, this finds the best matching order - for each left-right trace pair; the traces must have already - been synchronized into left-right pairs. Currently, this is a - very simple, non-optimized match: - - - The closest order from - ``self.spectrograph.order_spat_pos`` is matched to each - slit. - - Any slit that is not matched to an order (only if the - number of slits is larger than the number of expected - orders) is flagged as ``NOORDER``. - - If multiple slits are matched to the same order, the - slit with the smallest on-detector separation is kept - and the other match is ignored. - - Any slit matched to an order with a separation above - the provided tolerance is flagged as ORDERMISMATCH. - - A warning is issued if the number of valid matches - is not identical to the number of expected orders - (``self.spectrograph.norders``). The warning includes - the list of orders that were not identified. - - The match tolerance is et by the parameter ``order_match``. - An offset can be applied to improve the match via the - parameter ``order_offset``; i.e., this should minimize the - difference between the expected order positions and - ``self.slit_spatial_center() + self.par['order_offset']``. - Both ``order_match`` and ``order_offset`` are given in - fractions of the detector size along the spatial axis. + For Echelle spectrographs, this finds the best matching order for each + left-right trace pair; the traces must have already been synchronized + into left-right pairs. Currently, this is a very simple, non-optimized + match: + + - The closest order from ``self.spectrograph.order_spat_pos`` is + matched to each slit. + - Any slit that cannot be matched to an order -- either because + there are more "slits" than orders or because the separation is + larger than the provided tolerance -- is flagged as + ``ORDERMISMATCH``. + - A warning is issued if the number of valid matches is not + identical to the number of expected orders + (``self.spectrograph.norders``). The warning includes the list of + orders that were not identified. + + The match tolerance is set by the parameter ``order_match``. An offset + can be applied to improve the match via the parameter ``order_offset``; + i.e., this should minimize the difference between the expected order + positions and ``self.slit_spatial_center() + self.par['order_offset']``. + Both ``order_match`` and ``order_offset`` are given in fractions of the + detector size along the spatial axis. The result of this method is to instantiate :attr:`orderid`. + Args: + reference_row (:obj:`int`, optional): + The spectral pixel (row) used to generate spatial positions of + the orders to match against the expected positions. If + ``None``, use the PCA reference row if a PCA exists or the + central row (i.e., ``self.nspec//2``), otherwise. + Returns: - :obj:`float`: The median offset in pixels between the archived order - positions and those measured via the edge tracing. + :obj:`float`: The median offset in the relative of the detector size + between the archived order positions and those measured via the edge + tracing. Raises: PypeItError: - Raised if the number of orders or their spatial - locations are not defined for an Echelle - spectrograph. + Raised if the number of orders, the order number, or their + spatial locations are not defined for an Echelle spectrograph. + Also raised if the edge traces are not synced. """ if self.spectrograph.norders is None: @@ -5182,7 +5187,7 @@ def match_order(self): # Get the order centers in fractions of the detector width. This # requires the slits to be synced (checked above)! Masked elements in # slit_cen are for bad slits or syncing. - slit_cen = self.slit_spatial_center() + offset + slit_cen = self.slit_spatial_center(spec=reference_row) + offset good_sync = np.logical_not(np.ma.getmaskarray(slit_cen)) # "slit_indx" matches the "slit" index to the order number. I.e., # "slit_indx" has one element per expected order, and the value at a @@ -5226,7 +5231,7 @@ def match_order(self): msgs.info(f' {"-"*6} {"-"*4} {"-"*6}') for i in range(self.spectrograph.norders): if fnd[i]: - msgs.info(f' {self.spectrograph.orders[i]:>6} {i+1:>4} {sep[i]:6.3f}') + msgs.info(f' {self.spectrograph.orders[i]:>6} {i+1:>4} {sep[i]-med_offset:6.3f}') else: msgs.info(f' {self.spectrograph.orders[i]:>6} {"N/A":>4} {"MISSED":>6}') msgs.info(f' {"-"*6} {"-"*4} {"-"*6}') From c2a29fe96e4b99e5bd14eb442386ded74fea3dbb Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Tue, 12 Dec 2023 11:08:17 +0100 Subject: [PATCH 108/244] ryan PR comments --- pypeit/core/telluric.py | 148 ++++++++++++++++++---------------------- pypeit/par/pypeitpar.py | 15 +++- 2 files changed, 77 insertions(+), 86 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 3e73f7397a..7f3d3dce01 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -39,7 +39,7 @@ # TODO These codes should probably be in a separate qso_pca module. Also pickle functionality needs to be removed. # The prior is not used (that was the reason for pickling), so the components could be stored in fits format. # npca is not actually required here. -def init_qso_pca(filename,wave_grid,redshift, npca): +def qso_init_pca(filename,wave_grid,redshift,npca): """ This routine reads in the pickle file created by coarse_pca.create_coarse_pca. The relevant pieces are the wavelengths (wave_pca_c), the PCA components (pca_comp_c), and the Gaussian mixture @@ -92,7 +92,7 @@ def qso_pca_eval(theta,qso_pca_dict): Parameter vector, where theta_pca[0] is redshift, theta_pca[1] is the normalization, and theta_pca[2:npca+1] are the PCA coefficients, where npca is the PCA dimensionality qso_pca_dict (dict): - Dictionary continaing the PCA information generated by init_qso_pca + Dictionary continaing the PCA information generated by qso_init_pca Returns: `numpy.ndarray`_: Evaluated PCA for the QSO @@ -125,7 +125,7 @@ def qso_pca_eval(theta,qso_pca_dict): # is the PCA dimensionality # qso_pca_dict (dict): # Dictionary continaing the PCA information generated by -# ``init_qso__pca`` +# ``qso_init_pca`` # # # Returns: @@ -149,11 +149,11 @@ def read_telluric_pca(filename, wave_min=None, wave_max=None, pad_frac=0.10): Args: filename (:obj:`str`): Telluric PCA filename - wave_min (:obj:`float`): + wave_min (:obj:`float`, optional): Minimum wavelength at which the grid is desired - wave_max (:obj:`float`): + wave_max (:obj:`float`, optional): Maximum wavelength at which the grid is desired. - pad_frac (:obj:`float`): + pad_frac (:obj:`float`, optional): Percentage padding to be added to the grid boundaries if ``wave_min`` or ``wave_max`` are input; ignored otherwise. The resulting grid will extend from ``(1.0 - pad_frac)*wave_min`` to @@ -174,6 +174,7 @@ def read_telluric_pca(filename, wave_min=None, wave_max=None, pad_frac=0.10): hdul = data.load_telluric_grid(filename) wave_grid_full = hdul[1].data pca_comp_full = hdul[0].data + nspec_full = wave_grid_full.size ncomp = hdul[0].header['NCOMP'] bounds = hdul[2].data model_coefs = hdul[3].data @@ -191,7 +192,7 @@ def read_telluric_pca(filename, wave_min=None, wave_max=None, pad_frac=0.10): return dict(wave_grid=wave_grid, dloglam=dloglam, tell_pad_pix=tell_pad_pix, ncomp_tell_pca=ncomp, tell_pca=pca_comp_grid, bounds_tell_pca=bounds, - coefs_tell_pca=model_coefs, teltype='PCA') + coefs_tell_pca=model_coefs, teltype='pca') def read_telluric_grid(filename, wave_min=None, wave_max=None, pad_frac=0.10): """ @@ -204,11 +205,11 @@ def read_telluric_grid(filename, wave_min=None, wave_max=None, pad_frac=0.10): Args: filename (:obj:`str`): Telluric grid filename - wave_min (:obj:`float`): + wave_min (:obj:`float`, optional): Minimum wavelength at which the grid is desired - wave_max (:obj:`float`): + wave_max (:obj:`float`, optional): Maximum wavelength at which the grid is desired. - pad_frac (:obj:`float`): + pad_frac (:obj:`float`, optional): Percentage padding to be added to the grid boundaries if ``wave_min`` or ``wave_max`` are input; ignored otherwise. The resulting grid will extend from ``(1.0 - pad_frac)*wave_min`` to @@ -366,7 +367,7 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): """ Evaluate the telluric model. - Parameters from ``theta_tell`` are: (if teltype == 'PCA') the PCA coefficients + Parameters from ``theta_tell`` are: (if teltype == 'pca') the PCA coefficients or (if teltype == 'grid') the telluric grid parameters pressure, temperature, humidity, and airmass, in both cases followed by spectral resolution, shift, and stretch. @@ -415,13 +416,11 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): teltype = tell_dict['teltype'] # FD: Currently assumes that shift and stretch are on. # TODO: Make this work without shift and stretch. - if teltype == 'PCA': + if teltype == 'pca': ntell = ntheta-3 elif teltype == 'grid': ntell = 4 - else: - msgs.error("Unsupported teltype, must be 'PCA' or 'grid'") - + # Set the wavelength range if not provided ind_lower = 0 if ind_lower is None else ind_lower ind_upper = tell_dict['wave_grid'].size - 1 if ind_upper is None else ind_upper @@ -433,7 +432,7 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): ind_lower_final = ind_lower_pad if ind_lower_pad == ind_lower else ind_lower - ind_lower_pad ind_upper_final = ind_upper_pad if ind_upper_pad == ind_upper else ind_upper - ind_upper_pad - if teltype == 'PCA': + if teltype == 'pca': # Evaluate PCA model after truncating the wavelength range tellmodel_hires = np.zeros_like(tell_dict['tell_pca'][0]) tellmodel_hires[ind_lower_pad:ind_upper_pad+1] = np.dot(np.append(1,theta_tell[:ntell]), @@ -448,8 +447,6 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): elif teltype == 'grid': # Interpolate within the telluric grid tellmodel_hires = interp_telluric_grid(theta_tell[:ntell], tell_dict) - else: - msgs.error("Unsupported teltype, must be 'PCA' or 'grid'") tellmodel_conv = conv_telluric(tellmodel_hires[ind_lower_pad:ind_upper_pad+1], tell_dict['dloglam'], theta_tell[-3]) @@ -473,7 +470,45 @@ def tellfit_chi2(theta, flux, thismask, arg_dict): Args: theta (`numpy.ndarray`_): - Parameter vector for the object + telluric model. See documentation of tellfit for a detailed description. + + Parameter vector for the object + telluric model. + + This is actually two concatenated parameter vectors, one for + the object and one for the telluric, i.e.: + + (in PCA mode) + theta_obj = theta[:-(tell_npca+3)] + theta_tell = theta[-(tell_npca+3):] + + (in grid mode) + theta_obj = theta[:-7] + theta_tell = theta[-7:] + + The telluric model theta_tell includes a either user-specified + number of PCA coefficients (in PCA mode) or ambient pressure, + temperature, humidity, and airmass (in grid mode) followed by + spectral resolution, shift, and stretch. + + That is, in PCA mode, + + pca_coeffs = theta_tell[:tell_npca] + + while in grid mode, + + pressure = theta_tell[0] + temperature = theta_tell[1] + humidity = theta_tell[2] + airmass = theta_tell[3] + + with the last three indices of the array corresponding to + + resolution = theta_tell[-3] + shift = theta_tell[-2] + stretch = theta_tell[-1] + + The object model theta_obj can have an arbitrary size and is + provided as an argument to obj_model_func + flux (`numpy.ndarray`_): The flux of the object being fit thismask (`numpy.ndarray`_, boolean): @@ -488,19 +523,16 @@ def tellfit_chi2(theta, flux, thismask, arg_dict): """ - obj_model_func = arg_dict['obj_model_func'] flux_ivar = arg_dict['ivar'] teltype = arg_dict['tell_dict']['teltype'] # TODO: make this work without shift and stretch? # Number of telluric model parameters, plus shift, stretch, and resolution - if teltype == 'PCA': + if teltype == 'pca': nfit = arg_dict['tell_npca']+3 elif teltype == 'grid': nfit = 4+3 - else: - msgs.error("Unsupported teltype, must be 'PCA' or 'grid'") theta_obj = theta[:-nfit] theta_tell = theta[-nfit:] @@ -526,47 +558,6 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): fits for any object model that the user provides. Args: - - theta (`numpy.ndarray`_): - - Parameter vector for the object + telluric model. - - This is actually two concatenated paramter vectors, one for - the object and one for the telluric, i.e.: - - (in PCA mode) - theta_obj = theta[:-(tell_npca+3)] - theta_tell = theta[-(tell_npca+3):] - - (in grid mode) - theta_obj = theta[:-7] - theta_tell = theta[-7:] - - The telluric model theta_tell includes a either user-specified - number of PCA coefficients (in PCA mode) or ambient pressure, - temperature, humidity, and airmass (in grid mode) followed by - spectral resolution, shift, and stretch. - - That is, in PCA mode, - - pca_coeffs = theta_tell[:tell_npca] - - while in grid mode, - - pressure = theta_tell[0] - temperature = theta_tell[1] - humidity = theta_tell[2] - airmass = theta_tell[3] - - with the last three indices of the array corresponding to - - resolution = theta_tell[-3] - shift = theta_tell[-2] - stretch = theta_tell[-1] - - The object model theta_obj can have an arbitrary size and is - provided as an argument to obj_model_func - flux (`numpy.ndarray`_): The flux of the object being fit thismask (`numpy.ndarray`_, boolean): @@ -594,14 +585,11 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): object model arguments which is passed to the obj_model_func - init_from_last (object): + init_from_last (object, optional): Optional. Result object returned by the differential evolution optimizer for the last iteration. If this is passed the code will initialize from the previous best-fit for faster convergence. - **kwargs_opt (dict): - Optional arguments for the differential evolution - optimization Returns: tuple: Returns three objects: @@ -630,12 +618,11 @@ def tellfit(flux, thismask, arg_dict, init_from_last=None): nsamples = arg_dict['popsize']*nparams teltype = arg_dict['tell_dict']['teltype'] # FD: Assumes shift and stretch are turned on. - if teltype == 'PCA': + if teltype == 'pca': ntheta_tell = arg_dict['tell_npca']+3 # Total number of telluric model parameters in PCA mode elif teltype == 'grid': ntheta_tell = 4+3 # Total number of telluric model parameters in grid mode - else: - msgs.error("Unsupported teltype, must be 'PCA' or 'grid'") + # Decide how to initialize if init_from_last is not None: # Use a Gaussian ball about the optimum from a previous iteration @@ -1017,7 +1004,7 @@ def init_qso_model(obj_params, iord, wave, flux, ivar, mask, tellmodel): """ - qso_pca_dict = init_qso_pca(obj_params['pca_file'], wave, obj_params['z_qso'], obj_params['npca']) + qso_pca_dict = qso_init_pca(obj_params['pca_file'], wave, obj_params['z_qso'], obj_params['npca']) qso_pca_mean = np.exp(qso_pca_dict['components'][0, :]) tell_mask = tellmodel > obj_params['tell_norm_thresh'] # Create a reference model and bogus noise @@ -2371,7 +2358,7 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode # 1) Assign arguments self.telgrid = telgridfile - self.teltype = teltype.upper() + self.teltype = teltype.lower() self.obj_params = obj_params self.init_obj_model = init_obj_model self.tell_npca = tell_npca @@ -2413,18 +2400,13 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode wave, flux, ivar, gpm) # 3) Read the telluric grid and initalize associated parameters wv_gpm = self.wave_in_arr > 1.0 - if self.teltype == 'PCA': + if self.teltype == 'pca': self.tell_dict = read_telluric_pca(self.telgrid, wave_min=self.wave_in_arr[wv_gpm].min(), wave_max=self.wave_in_arr[wv_gpm].max()) - if self.tell_npca > self.tell_dict['ncomp_tell_pca']: - msgs.error('Asked for more telluric PCA components ({}) ' \ - 'than exist in the PCA file ({}).'.format(self.tell_npca, - self.tell_dict['ncomp_tell_pca'])) elif self.teltype == 'grid': self.tell_dict = read_telluric_grid(self.telgrid, wave_min=self.wave_in_arr[wv_gpm].min(), wave_max=self.wave_in_arr[wv_gpm].max()) - else: - msgs.error('Invalid teltype -- must be `pca` or `grid`') + self.wave_grid = self.tell_dict['wave_grid'] self.ngrid = self.wave_grid.size @@ -2542,7 +2524,7 @@ def run(self, only_orders=None): inmask=self.mask_arr[self.ind_lower[iord]:self.ind_upper[iord]+1,iord], maxiter=self.maxiter, lower=self.lower, upper=self.upper, sticky=self.sticky) - if self.teltype == 'PCA': + if self.teltype == 'pca': self.theta_obj_list[iord] = self.result_list[iord].x[:-(self.tell_npca+3)] self.theta_tell_list[iord] = self.result_list[iord].x[-(self.tell_npca+3):] elif self.teltype == 'grid': @@ -2627,7 +2609,7 @@ def assign_output(self, iord): wave_in_gd = self.wave_in_arr[gdwave,iord] wave_grid_now = self.wave_grid[self.ind_lower[iord]:self.ind_upper[iord]+1] - if self.teltype == 'PCA': + if self.teltype == 'pca': ntell = self.tell_npca elif self.teltype == 'grid': ntell = 4 @@ -2756,7 +2738,7 @@ def get_bounds_tell(self): self.tell_dict['h2o_grid'].max())) bounds.append((self.tell_dict['airmass_grid'].min(), self.tell_dict['airmass_grid'].max())) - else: + else: # i.e. for teltype == 'pca' for ii in range(self.tell_npca): bounds.append((self.tell_dict['bounds_tell_pca'][0][ii+1], self.tell_dict['bounds_tell_pca'][1][ii+1])) diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index 21894a367b..445d18daba 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -2239,11 +2239,11 @@ def __init__(self, telgridfile=None, sn_clip=None, resln_guess=None, resln_frac_ dtypes['tell_npca'] = int descr['tell_npca'] = 'Number of telluric PCA components used. Can be set to any number from 1 to 10.' - defaults['teltype'] = 'PCA' + defaults['teltype'] = 'pca' dtypes['teltype'] = str - descr['teltype'] = 'Method used to evaluate telluric models, either PCA or grid. The grid option uses a ' \ + descr['teltype'] = 'Method used to evaluate telluric models, either pca or grid. The grid option uses a ' \ 'fixed grid of pre-computed HITRAN+LBLRTM atmospheric transmission models for each ' \ - 'observatory, whereas the PCA option uses principal components of a larger model grid ' \ + 'observatory, whereas the pca option uses principal components of a larger model grid ' \ 'to compute an accurate pseudo-telluric model with a much lighter telgridfile.' defaults['sn_clip'] = 30.0 @@ -2495,6 +2495,15 @@ def validate(self): """ Check the parameters are valid for the provided method. """ + + if self.data['tell_npca'] < 1 or self.data['tell_npca'] > 10: + raise ValueError('Invalid value {:d} for tell_npca '.format(self.data['tell_npca'])+ + '(must be between 1 and 10).') + + if self.data['teltype'].lower() != 'pca' and self.data['teltype'].lower() != 'grid': + raise ValueError('Invalid teltype {} '.format(self.data['teltype'])+ + ', must be "pca" or "grid".') + pass # JFH add something in here which checks that the recombination value provided is bewteen 0 and 1, although # scipy.optimize.differential_evoluiton probalby checks this. From 514b76cafb0319a4a6c9b8300169a31741df21bb Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Tue, 12 Dec 2023 11:44:36 +0100 Subject: [PATCH 109/244] resln bounds edits --- pypeit/core/telluric.py | 2 +- pypeit/par/pypeitpar.py | 6 +++--- pypeit/spectrographs/keck_hires.py | 3 +++ pypeit/spectrographs/vlt_xshooter.py | 1 + 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 7f3d3dce01..37835104af 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -1302,7 +1302,7 @@ def sensfunc_telluric(wave, counts, counts_ivar, counts_mask, exptime, airmass, telgridfile, log10_blaze_function=None, ech_orders=None, polyorder=8, tell_npca=4, teltype='PCA', mask_hydrogen_lines=True, mask_helium_lines=False, hydrogen_mask_wid=10., - resln_guess=None, resln_frac_bounds=(0.3, 1.5), pix_shift_bounds=(-5.0, 5.0), + resln_guess=None, resln_frac_bounds=(0.6, 1.4), pix_shift_bounds=(-5.0, 5.0), delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), sn_clip=30.0, ballsize=5e-4, only_orders=None, maxiter=3, lower=3.0, upper=3.0, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index 445d18daba..040a2bc0fa 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -2265,12 +2265,12 @@ def __init__(self, telgridfile=None, sn_clip=None, resln_guess=None, resln_frac_ pars['resln_frac_bounds'] = tuple_force(pars['resln_frac_bounds']) - defaults['resln_frac_bounds'] = (0.3,1.5) + defaults['resln_frac_bounds'] = (0.6,1.4) dtypes['resln_frac_bounds'] = tuple descr['resln_frac_bounds'] = 'Bounds for the resolution fit optimization which is part of the telluric model. ' \ - 'This range is in units of the resln_guess, so the (0.3, 1.5) would bound the ' \ + 'This range is in units of the resln_guess, so the (0.6, 1.4) would bound the ' \ 'spectral resolution fit to be within the range ' \ - 'bounds_resln = (0.3*resln_guess, 1.5*resln_guess)' + 'bounds_resln = (0.6*resln_guess, 1.4*resln_guess)' pars['pix_shift_bounds'] = tuple_force(pars['pix_shift_bounds']) defaults['pix_shift_bounds'] = (-5.0,5.0) diff --git a/pypeit/spectrographs/keck_hires.py b/pypeit/spectrographs/keck_hires.py index 6bb3b3c028..b03fd2359a 100644 --- a/pypeit/spectrographs/keck_hires.py +++ b/pypeit/spectrographs/keck_hires.py @@ -161,7 +161,10 @@ def default_pypeit_par(cls): par['sensfunc']['IR']['pix_shift_bounds'] = (-40.0,40.0) # Telluric parameters + # HIRES is usually oversampled, so the helio shift can be large par['telluric']['pix_shift_bounds'] = (-40.0,40.0) + # Similarly, the resolution guess is higher than it should be + par['telluric']['resln_frac_bounds'] = (0.25,1.25) # Coadding par['coadd1d']['wave_method'] = 'log10' diff --git a/pypeit/spectrographs/vlt_xshooter.py b/pypeit/spectrographs/vlt_xshooter.py index 2ebd32d72b..e8d2767c9c 100644 --- a/pypeit/spectrographs/vlt_xshooter.py +++ b/pypeit/spectrographs/vlt_xshooter.py @@ -728,6 +728,7 @@ def default_pypeit_par(cls): # Telluric parameters par['telluric']['pix_shift_bounds'] = (-8.0,8.0) + par['telluric']['resln_frac_bounds'] = (0.4,1.4) # Coadding par['coadd1d']['wave_method'] = 'log10' From 560e9deac4a48fef99ed5edb6444901d9e1e041e Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Tue, 12 Dec 2023 14:02:08 +0100 Subject: [PATCH 110/244] added to release doc --- doc/releases/1.14.1dev.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/releases/1.14.1dev.rst b/doc/releases/1.14.1dev.rst index 7ca2a44ade..7e67df68c1 100644 --- a/doc/releases/1.14.1dev.rst +++ b/doc/releases/1.14.1dev.rst @@ -41,6 +41,7 @@ Functionality/Performance Improvements and Additions to reflect the *type* of polynomial being fit. Both names point to the same code, but the name ``'polynomial'`` is deprecated and will be removed in the future. +- Introduced PCA method for telluric corrections Instrument-specific Updates --------------------------- From b1372f16bbae389969792b4aaf9309495b480139 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Tue, 12 Dec 2023 14:07:11 +0100 Subject: [PATCH 111/244] one last xshooter optimization --- pypeit/spectrographs/vlt_xshooter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/spectrographs/vlt_xshooter.py b/pypeit/spectrographs/vlt_xshooter.py index e8d2767c9c..5ed566ade0 100644 --- a/pypeit/spectrographs/vlt_xshooter.py +++ b/pypeit/spectrographs/vlt_xshooter.py @@ -728,7 +728,7 @@ def default_pypeit_par(cls): # Telluric parameters par['telluric']['pix_shift_bounds'] = (-8.0,8.0) - par['telluric']['resln_frac_bounds'] = (0.4,1.4) + par['telluric']['resln_frac_bounds'] = (0.4,2.0) # Coadding par['coadd1d']['wave_method'] = 'log10' From cbaaa9f38b915a5df86073e64b4131431da988f9 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Tue, 12 Dec 2023 16:00:29 +0100 Subject: [PATCH 112/244] much faster, and sort_telluric works again --- pypeit/core/telluric.py | 7 ++++--- pypeit/par/pypeitpar.py | 2 +- pypeit/spectrographs/vlt_xshooter.py | 9 +++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 37835104af..ab0ff418f7 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -439,11 +439,11 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): tell_dict['tell_pca'][:ntell+1][:,ind_lower_pad:ind_upper_pad+1]) # PCA model is inverse sinh of the optical depth, convert to transmission here - tellmodel_hires = np.sinh(tellmodel_hires) + tellmodel_hires[ind_lower_pad:ind_upper_pad+1] = np.sinh(tellmodel_hires[ind_lower_pad:ind_upper_pad+1]) # It should generally be very rare, but trim negative optical depths here just in case. clip = tellmodel_hires < 0 tellmodel_hires[clip] = 0 - tellmodel_hires = np.exp(-tellmodel_hires) + tellmodel_hires[ind_lower_pad:ind_upper_pad+1] = np.exp(-tellmodel_hires[ind_lower_pad:ind_upper_pad+1]) elif teltype == 'grid': # Interpolate within the telluric grid tellmodel_hires = interp_telluric_grid(theta_tell[:ntell], tell_dict) @@ -2404,6 +2404,7 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode self.tell_dict = read_telluric_pca(self.telgrid, wave_min=self.wave_in_arr[wv_gpm].min(), wave_max=self.wave_in_arr[wv_gpm].max()) elif self.teltype == 'grid': + self.tell_npca = 4 self.tell_dict = read_telluric_grid(self.telgrid, wave_min=self.wave_in_arr[wv_gpm].min(), wave_max=self.wave_in_arr[wv_gpm].max()) @@ -2775,7 +2776,7 @@ def sort_telluric(self): tell_med[iord] = np.mean(tell_model_mid) else: tell_model_mean = self.tell_dict['tell_pca'][0,self.ind_lower[iord]:self.ind_upper[iord]+1] - tell_med[iord] = np.mean(tell_model_mean) + tell_med[iord] = np.mean(np.exp(-np.sinh(tell_model_mean))) # Perform fits in order of telluric strength return tell_med.argsort() diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index 040a2bc0fa..f17b479bcd 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -2235,7 +2235,7 @@ def __init__(self, telgridfile=None, sn_clip=None, resln_guess=None, resln_frac_ 'the pypeit_install_telluric script. NOTE: This parameter no longer includes the full ' \ 'pathname to the Telluric Grid file, but is just the filename of the grid itself.' - defaults['tell_npca'] = 4 + defaults['tell_npca'] = 5 dtypes['tell_npca'] = int descr['tell_npca'] = 'Number of telluric PCA components used. Can be set to any number from 1 to 10.' diff --git a/pypeit/spectrographs/vlt_xshooter.py b/pypeit/spectrographs/vlt_xshooter.py index 5ed566ade0..01cc8fc088 100644 --- a/pypeit/spectrographs/vlt_xshooter.py +++ b/pypeit/spectrographs/vlt_xshooter.py @@ -345,10 +345,11 @@ def default_pypeit_par(cls): par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 8 par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R25000.fits' - par['sensfunc']['IR']['pix_shift_bounds'] = (-8.0,8.0) + par['sensfunc']['IR']['pix_shift_bounds'] = (-10.0,10.0) # Telluric parameters - par['telluric']['pix_shift_bounds'] = (-8.0,8.0) + par['telluric']['pix_shift_bounds'] = (-10.0,10.0) + par['telluric']['resln_frac_bounds'] = (0.4,2.0) # Coadding par['coadd1d']['wave_method'] = 'log10' @@ -724,10 +725,10 @@ def default_pypeit_par(cls): par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 8 #[9, 11, 11, 9, 9, 8, 8, 7, 7, 7, 7, 7, 7, 7, 7] par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R25000.fits' - par['sensfunc']['IR']['pix_shift_bounds'] = (-8.0,8.0) + par['sensfunc']['IR']['pix_shift_bounds'] = (-10.0,10.0) # Telluric parameters - par['telluric']['pix_shift_bounds'] = (-8.0,8.0) + par['telluric']['pix_shift_bounds'] = (-10.0,10.0) par['telluric']['resln_frac_bounds'] = (0.4,2.0) # Coadding From 328e227e2a9b5198ae405d31659997780ea90802 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Tue, 12 Dec 2023 08:21:43 -0800 Subject: [PATCH 113/244] changes --- doc/releases/1.14.1dev.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/releases/1.14.1dev.rst b/doc/releases/1.14.1dev.rst index 79cbb637e6..9bb947288e 100644 --- a/doc/releases/1.14.1dev.rst +++ b/doc/releases/1.14.1dev.rst @@ -78,6 +78,8 @@ Under-the-hood Improvements - The CoAdd3D code has been refactored into a series of core modules and PypeIt-specific routines. - Polynomial overscan fitting now uses the ``numpy.polynomial.Chebyshev`` class instead of the deprecated ``numpy.polyfit()`` function. +- Fixed some failure modes when matching measured order positions to expected + positions for fixed-format echelle spectrographs. Bug Fixes --------- From 2c978c82153755a4e23b0e109e80b877f0b850fe Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Wed, 13 Dec 2023 11:15:42 +0100 Subject: [PATCH 114/244] par cleanup --- pypeit/core/telluric.py | 2 +- pypeit/par/pypeitpar.py | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index ab0ff418f7..625bd99567 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -2358,7 +2358,7 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode # 1) Assign arguments self.telgrid = telgridfile - self.teltype = teltype.lower() + self.teltype = teltype self.obj_params = obj_params self.init_obj_model = init_obj_model self.tell_npca = tell_npca diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index f17b479bcd..7a9d681f08 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -2224,6 +2224,7 @@ def __init__(self, telgridfile=None, sn_clip=None, resln_guess=None, resln_frac_ # Initialize the other used specifications for this parameter # set defaults = OrderedDict.fromkeys(pars.keys()) + options = OrderedDict.fromkeys(pars.keys()) dtypes = OrderedDict.fromkeys(pars.keys()) descr = OrderedDict.fromkeys(pars.keys()) @@ -2240,6 +2241,7 @@ def __init__(self, telgridfile=None, sn_clip=None, resln_guess=None, resln_frac_ descr['tell_npca'] = 'Number of telluric PCA components used. Can be set to any number from 1 to 10.' defaults['teltype'] = 'pca' + options['teltype'] = TelluricPar.valid_teltype() dtypes['teltype'] = str descr['teltype'] = 'Method used to evaluate telluric models, either pca or grid. The grid option uses a ' \ 'fixed grid of pre-computed HITRAN+LBLRTM atmospheric transmission models for each ' \ @@ -2490,21 +2492,27 @@ def from_dict(cls, cfg): for pk in parkeys: kwargs[pk] = cfg[pk] if pk in k else None return cls(**kwargs) + + @staticmethod + def valid_teltype(): + """ + Return the valid telluric methods. + """ + return ['pca', 'grid'] def validate(self): """ Check the parameters are valid for the provided method. """ - if self.data['tell_npca'] < 1 or self.data['tell_npca'] > 10: raise ValueError('Invalid value {:d} for tell_npca '.format(self.data['tell_npca'])+ '(must be between 1 and 10).') - if self.data['teltype'].lower() != 'pca' and self.data['teltype'].lower() != 'grid': - raise ValueError('Invalid teltype {} '.format(self.data['teltype'])+ - ', must be "pca" or "grid".') + self.data['teltype'] = self.data['teltype'].lower() + if self.data['teltype'] not in TelluricPar.valid_teltype(): + raise ValueError('Invalid teltype "{}"'.format(self.data['teltype'])+ + ', valid options are: {}.'.format(TelluricPar.valid_teltype())) - pass # JFH add something in here which checks that the recombination value provided is bewteen 0 and 1, although # scipy.optimize.differential_evoluiton probalby checks this. From ca5d0e6d771beafd2def4cbf8a5c1cd5330cdf1a Mon Sep 17 00:00:00 2001 From: rcooke Date: Wed, 13 Dec 2023 11:02:23 +0000 Subject: [PATCH 115/244] docs --- pypeit/coadd3d.py | 128 +++------------------------------------------- 1 file changed, 6 insertions(+), 122 deletions(-) diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index fc051285a9..bd6a51c9b0 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -917,7 +917,6 @@ def get_alignments(self, spec2DObj, slits, spat_flexure=None): def load(self): """ - TODO :: Update all of the docstrings! This is the main function that loads in the data, and performs several frame-specific corrections. If the user does not wish to align or combine the individual datacubes, then this routine will also produce a spec3d file, which is a DataCube representation of a PypeIt spec2d frame for SlicerIFU data. @@ -938,19 +937,18 @@ def load(self): As well as the primary arrays that store the pixel information for multiple spec2d frames, including: - * self.all_wave * self.all_sci * self.all_ivar - * self.all_idx + * self.all_wave + * self.all_slitid * self.all_wghts - * self.all_spatpos - * self.all_specpos - * self.all_spatid * self.all_tilts * self.all_slits * self.all_align - * self.all_dar * self.all_wcs + * self.all_ra + * self.all_dec + * self.all_dar """ # Load all spec2d files and prepare the data for making a datacube for ff, fil in enumerate(self.spec2d): @@ -1136,23 +1134,6 @@ def load(self): # Get the slit image and then unset pixels in the slit image that are bad slitid_img_gpm = slitid_img_init * onslit_gpm.astype(int) - ################################## - # Astrometric alignment to HST frames - # TODO :: RJC requests this remains here... it is only used by RJC - # Get the coordinate bounds - if True: - slitlength = int(np.round(np.median(slits.get_slitlengths(initial=True, median=True)))) - numwav = int((np.max(waveimg) - wave0) / dwv) - raw_bins = self.spec.get_datacube_bins(slitlength, minmax, numwav) - # Now do the alignment - self.ra_offsets[ff], self.dec_offsets[ff] = hst_alignment(sciImg, ivar, waveimg, slitid_img_gpm, wghts, ra_img, dec_img, - self.all_wcs[ff], spec2DObj.tilts, slits, alignSplines, darcorr, - self._dspat, self._dwv, raw_bins=raw_bins) - # TODO :: Does this need to be included anywhere? - # self.ifu_ra[ff] += self.ra_offsets[ff] - # self.ifu_dec[ff] += self.dec_offsets[ff] - ################################## - # If individual frames are to be output without aligning them, # there's no need to store information, just make the cubes now if not self.combine and not self.align: @@ -1344,8 +1325,7 @@ def run(self): dspat=self._dspat, dwv=self._dwv) # Align the frames - # TODO :: NOTE THAT THIS IS 'and False' because the HST alignment is done when reading in the data. - if self.align and False: + if self.align: self.ra_offsets, self.dec_offsets = self.run_align() # Compute the relative weights on the spectra @@ -1425,99 +1405,3 @@ def run(self): final_cube = DataCube(flxcube, sigcube, bpmcube, wave, self.specname, self.blaze_wave, self.blaze_spec, sensfunc=sensfunc, fluxed=self.fluxcal) final_cube.to_file(outfile, hdr=hdr, overwrite=self.overwrite) - - -def hst_alignment(sciImg, ivar, waveimg, slitid_img_gpm, wghts, raImg, decImg, - all_wcs, tilts, slits, astrom_trans, all_dar, dspat, dwave, - spat_subpixel=10, spec_subpixel=10, slice_subpixel=1, raw_bins=None): - """ - This is currently only used by RJC. This function adds corrections to the RA and Dec pixels - to align the daatcubes to an HST image. - - Process: - * Send away pixel RA, Dec, wave, flux, error. - * ------ - * Compute emission line map - - Need to generate full cube around H I gamma - - Fit to continuum and subtract it off - - Sum all flux above continuum - - Estimate error - * MPFIT HST emission line map to - * ------ - * Return updated pixel RA, Dec - """ - ra_offset, dec_offset = 0.0, 0.0 # These must be zero when calculating the offsets - from pypeit import astrometry - niter = 1 - for ii in range(niter): - ############ - ## STEP 1 ## - Create a datacube around Hgamma - ############ - # Only use a small wavelength range - inmask = slitid_img_gpm * ((waveimg > 4346.0) & (waveimg < 4358.0)).astype(int) - # Create a WCS for this subcube - subcube_wcs, voxedge, reference_image = datacube.create_wcs(raImg, decImg, waveimg, inmask, dspat, dwave) - # Create the subcube - flxcube, sigcube, bpmcube, wave = \ - datacube.generate_cube_subpixel(subcube_wcs, voxedge, sciImg, ivar, waveimg, inmask, wghts, - all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, - overwrite=False, spec_subpixel=5, spat_subpixel=5, - slice_subpixel=slice_subpixel) - - if False: - hdu = fits.PrimaryHDU(flxcube) - hdu.writeto("tstHg.fits", overwrite=True) - - ############ - ## STEP 2 ## - Create an emission line map of Hgamma - ############ - # Compute an emission line map that is as consistent as possible to an archival HST image - HgMap, HgMapErr = astrometry.fit_cube(flxcube, sigcube ** 2, subcube_wcs, line="HIgamma") - - ############ - ## STEP 3A ## - Generate another datacube using the raw_wcs and make an image of the emission line map - # Need to generate a datacube near both Hgamma and Hdelta. - ############ - if False: - #Is DAR correction being performed here? - slice_subpixel = 10 - # FIRST DO Hdelta - inmask = slitid_img_gpm * ((waveimg > 4107.0) & (waveimg < 4119.0)).astype(int) - flxcube, sigcube, bpmcube, wave = \ - datacube.generate_cube_subpixel(all_wcs, raw_bins, sciImg, ivar, waveimg, inmask, wghts, - all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, overwrite=False, - spec_subpixel=5, spat_subpixel=5, slice_subpixel=slice_subpixel) - HdMap_raw, HdMapErr_raw = astrometry.fit_cube(flxcube, sigcube**2, all_wcs, line="HIdelta") - # THEN DO Hgamma - inmask = slitid_img_gpm * ((waveimg > 4346.0) & (waveimg < 4358.0)).astype(int) - flxcube, sigcube, bpmcube, wave = \ - datacube.generate_cube_subpixel(all_wcs, raw_bins, sciImg, ivar, waveimg, inmask, wghts, - all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset, - overwrite=False, spec_subpixel=5, spat_subpixel=5, slice_subpixel=slice_subpixel) - HgMap_raw, HgMapErr_raw = astrometry.fit_cube(flxcube, sigcube**2, all_wcs, line="HIgamma") - # Plot the emission line map - from matplotlib import pyplot as plt - plt.subplot(131) - plt.imshow(HdMap_raw, vmin=0, vmax=200, aspect=0.5) - plt.subplot(132) - plt.imshow(HgMap_raw, vmin=0, vmax=300, aspect=0.5) - plt.subplot(133) - plt.imshow(HdMap_raw*utils.inverse(HgMap_raw), vmin=0.5, vmax=0.7, aspect=0.5) - plt.show() - embed() - # np.save("Hd-Hg_preSlicerDAR.npy", HdMap_raw*utils.inverse(HgMap_raw)) - - ############ - ## STEP 3B ## - Map the emission line map to an HST image, and vice-versa - ############ - ra_corr, dec_corr = astrometry.map_image(HgMap, HgMapErr, subcube_wcs, raImg, decImg, slitid_img_gpm)#, ra_sort, dec_sort, raw_wcs) - if False: - # ra_corr, dec_corr, flx_corr = astrometry.map_image(HgMap, HgMapErr, subcube_wcs, ra_sort, dec_sort, raw_wcs, HgMap_raw, HgMapErr_raw, HdMap_raw, HdMapErr_raw) - # Loop over the slits and apply the corrections - slit_illum_ref_idx = 14 - for ss, spatid in enumerate(slits.spat_id): - idx = np.where(all_spatid == spatid)[0] - # Calculate the correction as a function of wavelength - - # embed() - return ra_corr, dec_corr \ No newline at end of file From 927d85e56efd9891f20ce8039d0311b955354c0b Mon Sep 17 00:00:00 2001 From: rcooke Date: Wed, 13 Dec 2023 11:03:48 +0000 Subject: [PATCH 116/244] docs --- pypeit/coadd3d.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index bd6a51c9b0..bd48f2dcfe 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -390,6 +390,7 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs Show QA for debugging. """ + # TODO :: Before PR merge, add information to the release notes about the slicer subpixellation. # TODO :: Consider loading all calibrations into a single variable within the main CoAdd3D parent class. # Set the variables self.spec2d = spec2dfiles From f0ed689e6bd774f6466aba9ca320ecbbcc05a011 Mon Sep 17 00:00:00 2001 From: rcooke Date: Wed, 13 Dec 2023 11:46:25 +0000 Subject: [PATCH 117/244] fix inputs --- pypeit/inputfiles.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pypeit/inputfiles.py b/pypeit/inputfiles.py index 63317231f7..5fc3a28592 100644 --- a/pypeit/inputfiles.py +++ b/pypeit/inputfiles.py @@ -935,7 +935,7 @@ def options(self): # Load coordinate offsets for each file. This is "Delta RA cos(dec)" and "Delta Dec" # Get the RA offset of each file - off_ra = self.path_and_files('ra_offset', skip_blank=False, check_exists=False) + off_ra = self.data['ra_offset'].tolist() if off_ra is None: opts['ra_offset'] = None elif len(off_ra) == 1 and len(self.filenames) > 1: @@ -945,7 +945,7 @@ def options(self): # Convert from arcsec to degrees opts['ra_offset'] = [ora/3600.0 for ora in off_ra] # Get the DEC offset of each file - off_dec = self.path_and_files('dec_offset', skip_blank=False, check_exists=False) + off_dec = self.data['dec_offset'].tolist() if off_dec is None: opts['dec_offset'] = None elif len(off_dec) == 1 and len(self.filenames) > 1: From 438e5f36725bc2c8af6ceb0a9a7877b6fe70a84e Mon Sep 17 00:00:00 2001 From: profxj Date: Thu, 14 Dec 2023 06:36:46 -0800 Subject: [PATCH 118/244] detA --- pypeit/spectrographs/keck_deimos.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pypeit/spectrographs/keck_deimos.py b/pypeit/spectrographs/keck_deimos.py index b7c711af6c..8066755a9d 100644 --- a/pypeit/spectrographs/keck_deimos.py +++ b/pypeit/spectrographs/keck_deimos.py @@ -741,7 +741,7 @@ def get_rawimage(self, raw_file, det): mosaic = None if nimg == 1 else self.get_mosaic_par(det, hdu=hdu) detectors = [self.get_detector_par(det, hdu=hdu)] if nimg == 1 else mosaic.detectors - if hdu[0].header['AMPMODE'] != 'SINGLE:B': + if hdu[0].header['AMPMODE'] not in ['SINGLE:B', 'SINGLE:A']: msgs.error('PypeIt can only reduce images with AMPMODE == SINGLE:B.') if hdu[0].header['MOSMODE'] != 'Spectral': msgs.error('PypeIt can only reduce images with MOSMODE == Spectral.') @@ -778,6 +778,7 @@ def get_rawimage(self, raw_file, det): rawdatasec_img = np.zeros_like(image, dtype=int) oscansec_img = np.zeros_like(image, dtype=int) + #embed(header='DEIMOS 754') # Loop over the chips for ii, tt in enumerate(chips): data, oscan = deimos_read_1chip(hdu, tt + 1) From c89ffd5439c202bff55422c9c761747fdfb66d67 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 14 Dec 2023 08:56:37 -0800 Subject: [PATCH 119/244] test fix --- pypeit/tests/tstutils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pypeit/tests/tstutils.py b/pypeit/tests/tstutils.py index a6393a2c15..7222d9d3b9 100644 --- a/pypeit/tests/tstutils.py +++ b/pypeit/tests/tstutils.py @@ -36,7 +36,8 @@ # Tests require the Telluric file (Mauna Kea) par = Spectrograph.default_pypeit_par() -tell_test_grid = data.Paths.telgrid / 'TelFit_MaunaKea_3100_26100_R20000.fits' +tell_test_grid = data.get_telgrid_filepath('TelFit_MaunaKea_3100_26100_R20000.fits') +#tell_test_grid = data.Paths.telgrid / 'TelFit_MaunaKea_3100_26100_R20000.fits' telluric_required = pytest.mark.skipif(not tell_test_grid.is_file(), reason='no Mauna Kea telluric file') From 44ea0b22d1be2c0fe1f259ac19312e6cbaa7585e Mon Sep 17 00:00:00 2001 From: joe Date: Thu, 14 Dec 2023 16:09:37 -1000 Subject: [PATCH 120/244] Working to get KCWI working on more modes with more options. --- pypeit/extraction.py | 9 +- pypeit/find_objects.py | 470 +++++++++++++++++++++++----------------- pypeit/par/pypeitpar.py | 1 + 3 files changed, 280 insertions(+), 200 deletions(-) diff --git a/pypeit/extraction.py b/pypeit/extraction.py index c50f9d9eb3..dd332cc9b3 100644 --- a/pypeit/extraction.py +++ b/pypeit/extraction.py @@ -879,7 +879,7 @@ def local_skysub_extract(self, global_sky, sobjs, return self.skymodel, self.objmodel, self.ivarmodel, self.outmask, self.sobjs -class IFUExtract(MultiSlitExtract): +class SlicerIFUExtract(MultiSlitExtract): """ Child of Extract for IFU reductions @@ -887,5 +887,8 @@ class IFUExtract(MultiSlitExtract): """ def __init__(self, sciImg, slits, sobjs_obj, spectrograph, par, objtype, **kwargs): - # IFU doesn't extract, and there's no need for a super call here. - return + super().__init__(sciImg, slits, sobjs_obj, spectrograph, par, objtype, **kwargs) + + #def __init__(self, sciImg, slits, sobjs_obj, spectrograph, par, objtype, **kwargs): + # # IFU doesn't extract, and there's no need for a super call here. + # return diff --git a/pypeit/find_objects.py b/pypeit/find_objects.py index 7d157e3cf4..e4c4186b79 100644 --- a/pypeit/find_objects.py +++ b/pypeit/find_objects.py @@ -482,7 +482,7 @@ def get_platescale(self, slitord_id=None): pass - def global_skysub(self, skymask=None, update_crmask=True, trim_edg = (0, 0), + def global_skysub(self, skymask=None, update_crmask=True, previous_sky=None, show_fit=False, show=False, show_objs=False, objs_not_masked=False, reinit_bpm:bool=True): @@ -496,9 +496,6 @@ def global_skysub(self, skymask=None, update_crmask=True, trim_edg = (0, 0), A 2D image indicating sky regions (1=sky) update_crmask (bool, optional): Update the crmask in the science image - trim_edg (tuple, optional): - A two tuple of ints that specify the number of pixels to trim from the slit edges - Only used by the IFU child show_fit (bool, optional): Show the sky fits? show (bool, optional): @@ -960,7 +957,275 @@ class SlicerIFUFindObjects(MultiSlitFindObjects): """ def __init__(self, sciImg, slits, spectrograph, par, objtype, **kwargs): super().__init__(sciImg, slits, spectrograph, par, objtype, **kwargs) - self.initialize_slits(slits, initial=True) + #self.initialize_slits(slits, initial=True) # below is better. This was being done twice. + + def initialize_slits(self, slits, initial=True): + """ + Gather all the :class:`~pypeit.slittrace.SlitTraceSet` attributes + that we'll use here in :class:`FindObjects. Identical to the parent but the slits are not trimmed. + + + Args: + slits (:class:`~pypeit.slittrace.SlitTraceSet`): + SlitTraceSet object containing the slit boundaries that will be initialized. + initial (:obj:`bool`, optional): + Use the initial definition of the slits. If False, + tweaked slits are used. + """ + super().initialize_slits(slits, initial=True) + + def global_skysub(self, skymask=None, update_crmask=True, + previous_sky=None, show_fit=False, show=False, show_objs=False, objs_not_masked=False, + reinit_bpm: bool = True): + """ + Perform global sky subtraction. This SlicerIFU-specific routine ensures that the + edges of the slits are not trimmed, and performs a spatial and spectral + correction using the sky spectrum, if requested. See Reduce.global_skysub() + for parameter definitions. + + See base class method for description of parameters. + + Args: + reinit_bpm (:obj:`bool`, optional): + If True (default), the bpm is reinitialized to the initial bpm + Should be False on the final run in case there was a failure + upstream and no sources were found in the slit/order + """ + + global_sky_sep = super().global_skysub(skymask=skymask, update_crmask=update_crmask, + previous_sky=previous_sky, show_fit=show_fit, show=show, + show_objs=show_objs, + objs_not_masked=objs_not_masked, reinit_bpm=reinit_bpm) + + # Check if flexure or a joint fit is requested. If not return this + if not self.par['reduce']['skysub']['joint_fit'] and self.par['flexure']['spec_method'] == 'skip': + return global_sky_sep + + if self.wv_calib is None: + msgs.error("A wavelength calibration is needed (wv_calib) if a joint sky fit is requested.") + msgs.info("Generating wavelength image") + + # Generate the waveimg which is needed if flexure is being computed + self.waveimg = self.wv_calib.build_waveimg(self.tilts, self.slits, spat_flexure=self.spat_flexure_shift) + # TODO: Can the flexure compensation be removed from this class or is it necessary for the joint skysub? + + # Calculate spectral flexure + method = self.par['flexure']['spec_method'] + # TODO :: Perhaps include a new label for IFU flexure correction - e.g. 'slitcen_relative' or 'slitcenIFU' or 'IFU' + # :: If a new label is introduced, change the other instances of 'method' (see below), and in flexure.spec_flexure_qa() + if method in ['slitcen']: + # TODO make waveimg an argument here and return values pass back the new waveimg and shifts + self.calculate_flexure(global_sky_sep) + + # If the joint fit or spec/spat sensitivity corrections are not being performed, return the separate slits sky + if not self.par['reduce']['skysub']['joint_fit']: + return global_sky_sep + + # TODO I'm not following the control flow here. Is it that we need the initial global sky to compute flexure + # for each slit so that then one can perform joint sky-subtraction with flexure corrected wavelengths? Some + # more information on the logic of this flow would make it clearer. + + # Use sky information in all slits to perform a joint sky fit + global_sky = self.joint_skysub(skymask=skymask, update_crmask=update_crmask, + show_fit=show_fit, show=show, show_objs=show_objs, + objs_not_masked=objs_not_masked) + + return global_sky + + def joint_skysub(self, skymask=None, update_crmask=True, trim_edg=(0,0), + show_fit=False, show=False, show_objs=False, adderr=0.01, objs_not_masked=False): + """ Perform a joint sky model fit to the data. See Reduce.global_skysub() + for parameter definitions. + """ + msgs.info("Performing joint global sky subtraction") + # Mask objects using the skymask? If skymask has been set by objfinding, and masking is requested, then do so + skymask_now = skymask if (skymask is not None) else np.ones_like(self.sciImg.image, dtype=bool) + _global_sky = np.zeros_like(self.sciImg.image) + thismask = (self.slitmask > 0) + inmask = (self.sciImg.select_flag(invert=True) & thismask & skymask_now).astype(bool) + # Convert the wavelength image to A/pixel, registered at pixel 0 (this gives something like + # the tilts frame, but conserves wavelength position in each slit) + wavemin = self.waveimg[self.waveimg != 0.0].min() + tilt_wave = (self.waveimg - wavemin) / (self.waveimg.max() - wavemin) + + # Parameters for a standard star + sigrej = 3.0 + if self.std_redux: + sigrej = 7.0 + update_crmask = False + if not self.par['reduce']['skysub']['global_sky_std']: + msgs.info('Skipping global sky-subtraction for standard star.') + return _global_sky + + # Use the FWHM map determined from the arc lines to convert the science frame + # to have the same effective spectral resolution. + fwhm_map = self.wv_calib.build_fwhmimg(self.tilts, self.slits, initial=True, spat_flexure=self.spat_flexure_shift) + thismask = thismask & (fwhm_map != 0.0) + # Need to include S/N for deconvolution + sciimg = skysub.convolve_skymodel(self.sciImg.image, fwhm_map, thismask) + # Iterate to use a model variance image + numiter = 4 # This is more than enough, and will probably break earlier than this + model_ivar = self.sciImg.ivar + sl_ref = self.par['calibrations']['flatfield']['slit_illum_ref_idx'] + # Prepare the slitmasks for the relative spectral illumination + slitmask = self.slits.slit_img(pad=0, initial=True, flexure=self.spat_flexure_shift) + slitmask_trim = self.slits.slit_img(pad=-3, initial=True, flexure=self.spat_flexure_shift) + for nn in range(numiter): + msgs.info("Performing iterative joint sky subtraction - ITERATION {0:d}/{1:d}".format(nn+1, numiter)) + # TODO trim_edg is in the parset so it should be passed in here via trim_edg=tuple(self.par['reduce']['trim_edge']), + _global_sky[thismask] = skysub.global_skysub(sciimg, model_ivar, tilt_wave, + thismask, self.slits_left, self.slits_right, inmask=inmask, + sigrej=sigrej, trim_edg=trim_edg, + bsp=self.par['reduce']['skysub']['bspline_spacing'], + no_poly=self.par['reduce']['skysub']['no_poly'], + pos_mask=not self.bkg_redux and not objs_not_masked, + max_mask_frac=self.par['reduce']['skysub']['max_mask_frac'], + show_fit=show_fit) + + # Calculate the relative spectral illumination + scaleImg = flat.illum_profile_spectral_poly(sciimg, self.waveimg, slitmask, slitmask_trim, _global_sky, + slit_illum_ref_idx=sl_ref, gpmask=inmask, thismask=thismask) + # Apply this scale image to the temporary science frame + sciimg /= scaleImg + + # Update the ivar image used in the sky fit + msgs.info("Updating sky noise model") + # Choose the highest counts out of sky and object + counts = _global_sky + _scale = None if self.sciImg.img_scale is None else self.sciImg.img_scale[thismask] + # NOTE: darkcurr must be a float for the call below to work. + if not self.bkg_redux: + var = procimg.variance_model(self.sciImg.base_var[thismask], counts=counts[thismask], + count_scale=_scale, noise_floor=adderr) + model_ivar[thismask] = utils.inverse(var) + else: + model_ivar[thismask] = self.sciImg.ivar[thismask] + # RJC :: Recalculating the global sky and flexure is probably overkill... but please keep this code in for now + # Recalculate the sky on each individual slit and redetermine the spectral flexure + # global_sky_sep = super().global_skysub(skymask=skymask, update_crmask=update_crmask, + # trim_edg=trim_edg, show_fit=show_fit, show=show, + # show_objs=show_objs) + # self.calculate_flexure(global_sky_sep) + + # Check if the relative scaling isn't changing much after at least 4 iterations + minv, maxv = np.min(scaleImg[thismask]), np.max(scaleImg[thismask]) + if nn >= 3 and max(abs(1/minv), abs(maxv)) < 1.005: # Relative accuracy of 0.5% is sufficient + break + + if update_crmask: + # Find CRs with sky subtraction + # NOTE: There's no need to run `sciImg.update_mask_cr` after this. + # This operation updates the mask directly! + self.sciImg.build_crmask(self.par['scienceframe']['process'], subtract_img=_global_sky) + + # Now we have a correct scale, apply it to the original science image + self.apply_relative_scale(scaleImg) + + # Recalculate the joint sky using the original image + _global_sky[thismask] = skysub.global_skysub(self.sciImg.image, model_ivar, tilt_wave, + thismask, self.slits_left, self.slits_right, inmask=inmask, + sigrej=sigrej, trim_edg=trim_edg, + bsp=self.par['reduce']['skysub']['bspline_spacing'], + no_poly=self.par['reduce']['skysub']['no_poly'], + pos_mask=not self.bkg_redux and not objs_not_masked, + max_mask_frac=self.par['reduce']['skysub']['max_mask_frac'], + show_fit=show_fit) + + # Update the ivar image used in the sky fit + msgs.info("Updating sky noise model") + # Choose the highest counts out of sky and object + counts = _global_sky + _scale = None if self.sciImg.img_scale is None else self.sciImg.img_scale[thismask] + # NOTE: darkcurr must be a float for the call below to work. + var = procimg.variance_model(self.sciImg.base_var[thismask], counts=counts[thismask], + count_scale=_scale, noise_floor=adderr) + model_ivar[thismask] = utils.inverse(var) + + # Step + self.steps.append(inspect.stack()[0][3]) + + if show: + sobjs_show = None if show_objs else self.sobjs_obj + # Global skysub is the first step in a new extraction so clear the channels here + self.show('global', global_sky=_global_sky, slits=True, sobjs=sobjs_show, clear=False) + return _global_sky + + # TODO is the flexure compensation needed in object finding. Can it be separated from this class and put somewhere + # else like a flexure compensation class or module. + def calculate_flexure(self, global_sky): + """ + TODO: Need better docs. What is being assigned in this function? Need either docs or return values. + Convenience function to calculate the flexure of the IFU + + Args: + global_sky (`numpy.ndarray`_): + Model of the sky + """ + sl_ref = self.par['calibrations']['flatfield']['slit_illum_ref_idx'] + box_rad = self.par['reduce']['extraction']['boxcar_radius'] + trace_spat = 0.5 * (self.slits_left + self.slits_right) + # Load archival sky spectrum for absolute correction + sky_spectrum, sky_fwhm_pix = flexure.get_archive_spectrum(self.par['flexure']['spectrum']) + # Get spectral FWHM (in Angstrom) if available + iwv = np.where(self.wv_calib.spat_ids == self.slits.spat_id[sl_ref])[0][0] + ref_fwhm_pix = self.wv_calib.wv_fits[iwv].fwhm + # Extract a spectrum of the sky + thismask = (self.slitmask == self.slits.spat_id[sl_ref]) + ref_skyspec = flexure.get_sky_spectrum(self.sciImg.image, self.sciImg.ivar, self.waveimg, thismask, + global_sky, box_rad, self.slits, trace_spat[:, sl_ref], + self.pypeline, self.det) + # Calculate the flexure + flex_dict_ref = flexure.spec_flex_shift(ref_skyspec, sky_spectrum, sky_fwhm_pix, spec_fwhm_pix=ref_fwhm_pix, + mxshft=self.par['flexure']['spec_maxshift'], + excess_shft=self.par['flexure']['excessive_shift'], + method="slitcen", + minwave=self.par['flexure']['minwave'], + maxwave=self.par['flexure']['maxwave']) + this_slitshift = np.zeros(self.slits.nslits) + if flex_dict_ref is not None: + msgs.warn("Only a relative spectral flexure correction will be performed") + this_slitshift = np.ones(self.slits.nslits) * flex_dict_ref['shift'] + # Now loop through all slits to calculate the additional shift relative to the reference slit + flex_list = [] + for slit_idx, slit_spat in enumerate(self.slits.spat_id): + thismask = (self.slitmask == slit_spat) + # Extract sky spectrum for this slit + this_skyspec = flexure.get_sky_spectrum(self.sciImg.image, self.sciImg.ivar, self.waveimg, thismask, + global_sky, box_rad, self.slits, trace_spat[:, slit_idx], + self.pypeline, self.det) + # Calculate the flexure + flex_dict = flexure.spec_flex_shift(this_skyspec, ref_skyspec, ref_fwhm_pix * 1.01, + spec_fwhm_pix=ref_fwhm_pix, + mxshft=self.par['flexure']['spec_maxshift'], + excess_shft=self.par['flexure']['excessive_shift'], + method="slitcen", + minwave=self.par['flexure']['minwave'], + maxwave=self.par['flexure']['maxwave']) + this_slitshift[slit_idx] += flex_dict['shift'] + flex_list.append(flex_dict.copy()) + # Replace the reference slit with the absolute shift + flex_list[sl_ref] = flex_dict_ref.copy() + # Add this flexure to the previous flexure correction + self.slitshift += this_slitshift + # Now report the flexure values + for slit_idx, slit_spat in enumerate(self.slits.spat_id): + msgs.info("Flexure correction, slit {0:d} (spat id={1:d}): {2:.3f} pixels".format(1+slit_idx, slit_spat, + self.slitshift[slit_idx])) + # Save QA + # TODO :: Need to implement QA + msgs.work("QA is not currently implemented for the flexure correction") + if False:#flex_list is not None: + basename = f'{self.basename}_global_{self.spectrograph.get_det_name(self.det)}' + out_dir = os.path.join(self.par['rdx']['redux_path'], 'QA') + slit_bpm = np.zeros(self.slits.nslits, dtype=bool) + flexure.spec_flexure_qa(self.slits.slitord_id, slit_bpm, basename, flex_list, out_dir=out_dir) + + # Recalculate the wavelength image, and the global sky taking into account the spectral flexure + msgs.info("Generating wavelength image, accounting for spectral flexure") + self.waveimg = self.wv_calib.build_waveimg(self.tilts, self.slits, spec_flexure=self.slitshift, + spat_flexure=self.spat_flexure_shift) + return + def apply_relative_scale(self, scaleImg): """Apply a relative scale to the science frame (and correct the varframe, too) @@ -1091,6 +1356,7 @@ def apply_relative_scale(self, scaleImg): # # Apply the relative scale correction # self.apply_relative_scale(spatScaleImg) + # TODO This method appears to not be used. Should be removed? def illum_profile_spectral(self, global_sky, skymask=None): """Calculate the residual spectral illumination profile using the sky regions. This uses the same routine as the flatfield spectral illumination profile. @@ -1114,125 +1380,8 @@ def illum_profile_spectral(self, global_sky, skymask=None): # Now apply the correction to the science frame self.apply_relative_scale(scaleImg) - def joint_skysub(self, skymask=None, update_crmask=True, trim_edg=(0,0), - show_fit=False, show=False, show_objs=False, adderr=0.01, objs_not_masked=False): - """ Perform a joint sky model fit to the data. See Reduce.global_skysub() - for parameter definitions. - """ - msgs.info("Performing joint global sky subtraction") - # Mask objects using the skymask? If skymask has been set by objfinding, and masking is requested, then do so - skymask_now = skymask if (skymask is not None) else np.ones_like(self.sciImg.image, dtype=bool) - _global_sky = np.zeros_like(self.sciImg.image) - thismask = (self.slitmask > 0) - inmask = (self.sciImg.select_flag(invert=True) & thismask & skymask_now).astype(bool) - # Convert the wavelength image to A/pixel, registered at pixel 0 (this gives something like - # the tilts frame, but conserves wavelength position in each slit) - wavemin = self.waveimg[self.waveimg != 0.0].min() - tilt_wave = (self.waveimg - wavemin) / (self.waveimg.max() - wavemin) - - # Parameters for a standard star - sigrej = 3.0 - if self.std_redux: - sigrej = 7.0 - update_crmask = False - if not self.par['reduce']['skysub']['global_sky_std']: - msgs.info('Skipping global sky-subtraction for standard star.') - return _global_sky - - # Use the FWHM map determined from the arc lines to convert the science frame - # to have the same effective spectral resolution. - fwhm_map = self.wv_calib.build_fwhmimg(self.tilts, self.slits, initial=True, spat_flexure=self.spat_flexure_shift) - thismask = thismask & (fwhm_map != 0.0) - # Need to include S/N for deconvolution - sciimg = skysub.convolve_skymodel(self.sciImg.image, fwhm_map, thismask) - # Iterate to use a model variance image - numiter = 4 # This is more than enough, and will probably break earlier than this - model_ivar = self.sciImg.ivar - sl_ref = self.par['calibrations']['flatfield']['slit_illum_ref_idx'] - # Prepare the slitmasks for the relative spectral illumination - slitmask = self.slits.slit_img(pad=0, initial=True, flexure=self.spat_flexure_shift) - slitmask_trim = self.slits.slit_img(pad=-3, initial=True, flexure=self.spat_flexure_shift) - for nn in range(numiter): - msgs.info("Performing iterative joint sky subtraction - ITERATION {0:d}/{1:d}".format(nn+1, numiter)) - # TODO trim_edg is in the parset so it should be passed in here via trim_edg=tuple(self.par['reduce']['trim_edge']), - _global_sky[thismask] = skysub.global_skysub(sciimg, model_ivar, tilt_wave, - thismask, self.slits_left, self.slits_right, inmask=inmask, - sigrej=sigrej, trim_edg=trim_edg, - bsp=self.par['reduce']['skysub']['bspline_spacing'], - no_poly=self.par['reduce']['skysub']['no_poly'], - pos_mask=not self.bkg_redux and not objs_not_masked, - max_mask_frac=self.par['reduce']['skysub']['max_mask_frac'], - show_fit=show_fit) - - # Calculate the relative spectral illumination - scaleImg = flat.illum_profile_spectral_poly(sciimg, self.waveimg, slitmask, slitmask_trim, _global_sky, - slit_illum_ref_idx=sl_ref, gpmask=inmask, thismask=thismask) - # Apply this scale image to the temporary science frame - sciimg /= scaleImg - # Update the ivar image used in the sky fit - msgs.info("Updating sky noise model") - # Choose the highest counts out of sky and object - counts = _global_sky - _scale = None if self.sciImg.img_scale is None else self.sciImg.img_scale[thismask] - # NOTE: darkcurr must be a float for the call below to work. - if not self.bkg_redux: - var = procimg.variance_model(self.sciImg.base_var[thismask], counts=counts[thismask], - count_scale=_scale, noise_floor=adderr) - model_ivar[thismask] = utils.inverse(var) - else: - model_ivar[thismask] = self.sciImg.ivar[thismask] - # RJC :: Recalculating the global sky and flexure is probably overkill... but please keep this code in for now - # Recalculate the sky on each individual slit and redetermine the spectral flexure - # global_sky_sep = super().global_skysub(skymask=skymask, update_crmask=update_crmask, - # trim_edg=trim_edg, show_fit=show_fit, show=show, - # show_objs=show_objs) - # self.calculate_flexure(global_sky_sep) - - # Check if the relative scaling isn't changing much after at least 4 iterations - minv, maxv = np.min(scaleImg[thismask]), np.max(scaleImg[thismask]) - if nn >= 3 and max(abs(1/minv), abs(maxv)) < 1.005: # Relative accuracy of 0.5% is sufficient - break - - if update_crmask: - # Find CRs with sky subtraction - # NOTE: There's no need to run `sciImg.update_mask_cr` after this. - # This operation updates the mask directly! - self.sciImg.build_crmask(self.par['scienceframe']['process'], subtract_img=_global_sky) - - # Now we have a correct scale, apply it to the original science image - self.apply_relative_scale(scaleImg) - - # Recalculate the joint sky using the original image - _global_sky[thismask] = skysub.global_skysub(self.sciImg.image, model_ivar, tilt_wave, - thismask, self.slits_left, self.slits_right, inmask=inmask, - sigrej=sigrej, trim_edg=trim_edg, - bsp=self.par['reduce']['skysub']['bspline_spacing'], - no_poly=self.par['reduce']['skysub']['no_poly'], - pos_mask=not self.bkg_redux and not objs_not_masked, - max_mask_frac=self.par['reduce']['skysub']['max_mask_frac'], - show_fit=show_fit) - - # Update the ivar image used in the sky fit - msgs.info("Updating sky noise model") - # Choose the highest counts out of sky and object - counts = _global_sky - _scale = None if self.sciImg.img_scale is None else self.sciImg.img_scale[thismask] - # NOTE: darkcurr must be a float for the call below to work. - var = procimg.variance_model(self.sciImg.base_var[thismask], counts=counts[thismask], - count_scale=_scale, noise_floor=adderr) - model_ivar[thismask] = utils.inverse(var) - - # Step - self.steps.append(inspect.stack()[0][3]) - - if show: - sobjs_show = None if show_objs else self.sobjs_obj - # Global skysub is the first step in a new extraction so clear the channels here - self.show('global', global_sky=_global_sky, slits=True, sobjs=sobjs_show, clear=False) - return _global_sky - - def global_skysub(self, skymask=None, update_crmask=True, trim_edg=(0,0), + def global_skysub_old(self, skymask=None, update_crmask=True, trim_edg=(0,0), previous_sky=None, show_fit=False, show=False, show_objs=False, objs_not_masked=False, reinit_bpm:bool=True): """ @@ -1245,7 +1394,7 @@ def global_skysub(self, skymask=None, update_crmask=True, trim_edg=(0,0), Args: reinit_bpm (:obj:`bool`, optional): - If True (default), the bpm is reinitialized to the initial bpm + If True (default), the bpm is reinitialized to the initial bpm Should be False on the final run in case there was a failure upstream and no sources were found in the slit/order """ @@ -1295,76 +1444,3 @@ def global_skysub(self, skymask=None, update_crmask=True, trim_edg=(0,0), objs_not_masked=objs_not_masked) return global_sky - - def calculate_flexure(self, global_sky): - """ - Convenience function to calculate the flexure of the IFU - - Args: - global_sky (`numpy.ndarray`_): - Model of the sky - """ - sl_ref = self.par['calibrations']['flatfield']['slit_illum_ref_idx'] - box_rad = self.par['reduce']['extraction']['boxcar_radius'] - trace_spat = 0.5 * (self.slits_left + self.slits_right) - # Load archival sky spectrum for absolute correction - sky_spectrum, sky_fwhm_pix = flexure.get_archive_spectrum(self.par['flexure']['spectrum']) - # Get spectral FWHM (in Angstrom) if available - iwv = np.where(self.wv_calib.spat_ids == self.slits.spat_id[sl_ref])[0][0] - ref_fwhm_pix = self.wv_calib.wv_fits[iwv].fwhm - # Extract a spectrum of the sky - thismask = (self.slitmask == self.slits.spat_id[sl_ref]) - ref_skyspec = flexure.get_sky_spectrum(self.sciImg.image, self.sciImg.ivar, self.waveimg, thismask, - global_sky, box_rad, self.slits, trace_spat[:, sl_ref], - self.pypeline, self.det) - # Calculate the flexure - flex_dict_ref = flexure.spec_flex_shift(ref_skyspec, sky_spectrum, sky_fwhm_pix, spec_fwhm_pix=ref_fwhm_pix, - mxshft=self.par['flexure']['spec_maxshift'], - excess_shft=self.par['flexure']['excessive_shift'], - method="slitcen", - minwave=self.par['flexure']['minwave'], - maxwave=self.par['flexure']['maxwave']) - this_slitshift = np.zeros(self.slits.nslits) - if flex_dict_ref is not None: - msgs.warn("Only a relative spectral flexure correction will be performed") - this_slitshift = np.ones(self.slits.nslits) * flex_dict_ref['shift'] - # Now loop through all slits to calculate the additional shift relative to the reference slit - flex_list = [] - for slit_idx, slit_spat in enumerate(self.slits.spat_id): - thismask = (self.slitmask == slit_spat) - # Extract sky spectrum for this slit - this_skyspec = flexure.get_sky_spectrum(self.sciImg.image, self.sciImg.ivar, self.waveimg, thismask, - global_sky, box_rad, self.slits, trace_spat[:, slit_idx], - self.pypeline, self.det) - # Calculate the flexure - flex_dict = flexure.spec_flex_shift(this_skyspec, ref_skyspec, ref_fwhm_pix * 1.01, - spec_fwhm_pix=ref_fwhm_pix, - mxshft=self.par['flexure']['spec_maxshift'], - excess_shft=self.par['flexure']['excessive_shift'], - method="slitcen", - minwave=self.par['flexure']['minwave'], - maxwave=self.par['flexure']['maxwave']) - this_slitshift[slit_idx] += flex_dict['shift'] - flex_list.append(flex_dict.copy()) - # Replace the reference slit with the absolute shift - flex_list[sl_ref] = flex_dict_ref.copy() - # Add this flexure to the previous flexure correction - self.slitshift += this_slitshift - # Now report the flexure values - for slit_idx, slit_spat in enumerate(self.slits.spat_id): - msgs.info("Flexure correction, slit {0:d} (spat id={1:d}): {2:.3f} pixels".format(1+slit_idx, slit_spat, - self.slitshift[slit_idx])) - # Save QA - # TODO :: Need to implement QA - msgs.work("QA is not currently implemented for the flexure correction") - if False:#flex_list is not None: - basename = f'{self.basename}_global_{self.spectrograph.get_det_name(self.det)}' - out_dir = os.path.join(self.par['rdx']['redux_path'], 'QA') - slit_bpm = np.zeros(self.slits.nslits, dtype=bool) - flexure.spec_flexure_qa(self.slits.slitord_id, slit_bpm, basename, flex_list, out_dir=out_dir) - - # Recalculate the wavelength image, and the global sky taking into account the spectral flexure - msgs.info("Generating wavelength image, accounting for spectral flexure") - self.waveimg = self.wv_calib.build_waveimg(self.tilts, self.slits, spec_flexure=self.slitshift, - spat_flexure=self.spat_flexure_shift) - return diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index 0e398a5310..74a25b7831 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -682,6 +682,7 @@ def __init__(self, method=None, pixelflat_file=None, spec_samp_fine=None, defaults['slit_illum_relative'] = False dtypes['slit_illum_relative'] = bool + # TODO Needs to be updated. use_slitillum is not a parameter anywhere in pypeit descr['slit_illum_relative'] = 'Generate an image of the relative spectral illumination ' \ 'for a multi-slit setup. If you set ``use_slitillum = ' \ 'True`` for any of the frames that use the flatfield ' \ From 5b7ca3817092a76ac6cc6ca8632acc311d8f10f2 Mon Sep 17 00:00:00 2001 From: joe Date: Sat, 16 Dec 2023 21:26:40 -1000 Subject: [PATCH 121/244] Working to get KCWI working on more modes with more options. --- pypeit/spectrographs/keck_deimos.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pypeit/spectrographs/keck_deimos.py b/pypeit/spectrographs/keck_deimos.py index 02d8758304..d9a7d934a0 100644 --- a/pypeit/spectrographs/keck_deimos.py +++ b/pypeit/spectrographs/keck_deimos.py @@ -741,8 +741,9 @@ def get_rawimage(self, raw_file, det): mosaic = None if nimg == 1 else self.get_mosaic_par(det, hdu=hdu) detectors = [self.get_detector_par(det, hdu=hdu)] if nimg == 1 else mosaic.detectors - if hdu[0].header['AMPMODE'] != 'SINGLE:B': - msgs.error('PypeIt can only reduce images with AMPMODE == SINGLE:B.') + # TODO check that that read noise and gain are the same for this amplifier mode?? + #if hdu[0].header['AMPMODE'] != 'SINGLE:B': + # msgs.error('PypeIt can only reduce images with AMPMODE == SINGLE:B.') if hdu[0].header['MOSMODE'] != 'Spectral': msgs.error('PypeIt can only reduce images with MOSMODE == Spectral.') From 6ebd3ba948af5104d46efdcb02fe7f63f3623308 Mon Sep 17 00:00:00 2001 From: joe Date: Sun, 17 Dec 2023 21:01:45 -1000 Subject: [PATCH 122/244] Got slit masking to work with alignframe. Fixed a slitspatnum bug. --- pypeit/alignframe.py | 27 +++++++++++++++++++++++---- pypeit/core/procimg.py | 5 ++++- pypeit/images/detector_container.py | 2 +- pypeit/slittrace.py | 3 ++- 4 files changed, 30 insertions(+), 7 deletions(-) diff --git a/pypeit/alignframe.py b/pypeit/alignframe.py index 598a27855c..189fbf9088 100644 --- a/pypeit/alignframe.py +++ b/pypeit/alignframe.py @@ -143,6 +143,12 @@ def __init__(self, rawalignimg, slits, spectrograph, alignpar, det=1, qa_path=No # Attributes unique to this object self._alignprof = None + + # Create a bad pixel mask + self.slit_bpm = self.slits.mask.astype(bool) + self.slit_bpm &= np.logical_not(self.slits.bitmask.flagged(self.slits.mask, flag=self.slits.bitmask.exclude_for_reducing)) + + # Completed steps self.steps = [] @@ -180,8 +186,14 @@ def build_traces(self, show_peaks=False, debug=False): slitid_img_init = self.slits.slit_img(initial=True) left, right, _ = self.slits.select_edges(initial=True) align_prof = dict({}) + # Go through the slits for slit_idx, slit_spat in enumerate(self.slits.spat_id): + if self.slit_bpm[slit_idx]: + msgs.info(f'Skipping bad slit/order {self.slits.slitord_id[slit_idx]} ({slit_idx+1}/{self.slits.nslits})') + self.slits.mask[slit_idx] = self.slits.bitmask.turn_on(self.slits.mask[slit_idx], 'BADALIGNCALIB') + continue + specobj_dict = {'SLITID': slit_idx, 'DET': self.rawalignimg.detector.name, 'OBJTYPE': "align_profile", 'PYPELINE': self.spectrograph.pypeline} msgs.info("Fitting alignment traces in slit {0:d}/{1:d}".format(slit_idx+1, self.slits.nslits)) @@ -221,12 +233,19 @@ def generate_traces(self, align_prof): nbars = len(self.alignpar['locations']) # Generate an array containing the centroid of all bars align_traces = np.zeros((self.nspec, nbars, self.nslits)) - for sl in range(self.nslits): - sls = '{0:d}'.format(sl) + + + # Go through the slits + for slit_idx, slit_spat in enumerate(self.slits.spat_id): + if self.slit_bpm[slit_idx]: + msgs.info(f'Skipping bad slit/order {self.slits.slitord_id[slit_idx]} ({slit_idx+1}/{self.slits.nslits})') + self.slits.mask[slit_idx] = self.slits.bitmask.turn_on(self.slits.mask[slit_idx], 'BADALIGNCALIB') + continue + sls = '{0:d}'.format(slit_idx) for bar in range(nbars): - if align_prof[sls][bar].SLITID != sl: + if align_prof[sls][bar].SLITID != slit_idx: msgs.error("Alignment profiling failed to generate traces") - align_traces[:, bar, sl] = align_prof[sls][bar].TRACE_SPAT + align_traces[:, bar, slit_idx] = align_prof[sls][bar].TRACE_SPAT return align_traces def run(self, show=False): diff --git a/pypeit/core/procimg.py b/pypeit/core/procimg.py index 0f20a719b8..5c29e6a609 100644 --- a/pypeit/core/procimg.py +++ b/pypeit/core/procimg.py @@ -816,7 +816,10 @@ def subtract_pattern(rawframe, datasec_img, oscansec_img, frequency=None, axis=1 sgnl = overscan[ii,:] LSfreq, power = LombScargle(pixels, sgnl).autopower(minimum_frequency=use_fr*(1-100/frame_orig.shape[1]), maximum_frequency=use_fr*(1+100/frame_orig.shape[1]), samples_per_peak=10) bst = np.argmax(power) - cc = np.polyfit(LSfreq[bst-2:bst+3],power[bst-2:bst+3],2) + imin = np.clip(bst-2,0,None) + imax = np.clip(bst+3,None,overscan.shape[1]) + # TODO is this correct?? + cc = np.polyfit(LSfreq[imin:imax],power[imin:imax],2) all_freq[ii] = -0.5*cc[1]/cc[0] cc = np.polyfit(all_rows, all_freq, 1) frq_mod = np.polyval(cc, all_rows) * (overscan.shape[1]-1) diff --git a/pypeit/images/detector_container.py b/pypeit/images/detector_container.py index d5e624a4fb..312a82b813 100644 --- a/pypeit/images/detector_container.py +++ b/pypeit/images/detector_container.py @@ -104,7 +104,7 @@ class DetectorContainer(datamodel.DataContainer): 'where the valid data sections can be obtained, one ' 'per amplifier. If defined explicitly should be in ' 'FITS format (e.g., [1:2048,10:4096]).'), - 'det': dict(otype=int, + 'det': dict(otype=(int, np.int64), descr='PypeIt designation for detector number (1-based).'), 'binning': dict(otype=str, descr='Binning in PypeIt orientation (not the original)')} diff --git a/pypeit/slittrace.py b/pypeit/slittrace.py index 74c0f1e414..4c8b4b04a1 100644 --- a/pypeit/slittrace.py +++ b/pypeit/slittrace.py @@ -40,6 +40,7 @@ def __init__(self): ('USERIGNORE', 'User has specified to ignore this slit. Not ignored for flexure.'), ('BADWVCALIB', 'Wavelength calibration failed for this slit'), ('BADTILTCALIB', 'Tilts analysis failed for this slit'), + ('BADALIGNCALIB', 'Alignment analysis failed for this slit'), ('SKIPFLATCALIB', 'Flat field generation failed for this slit. Skip flat fielding'), ('BADFLATCALIB', 'Flat field generation failed for this slit. Ignore it fully.'), ('BADREDUCE', 'Reduction failed for this slit'), # THIS IS DEPRECATED (we may remove in v1.13) BUT STAYS HERE TO ALLOW FOR BACKWARDS COMPATIBILITY @@ -57,7 +58,7 @@ def exclude_for_reducing(self): def exclude_for_flexure(self): # Ignore these flags when performing a flexure calculation # Currently they are *all* of the flags.. - return ['SHORTSLIT', 'USERIGNORE', 'BADWVCALIB', 'BADTILTCALIB', + return ['SHORTSLIT', 'USERIGNORE', 'BADWVCALIB', 'BADTILTCALIB', 'BADALIGNCALIB', 'SKIPFLATCALIB', 'BADFLATCALIB', 'BADSKYSUB', 'BADEXTRACT'] From 84c50d024a86929963c01d9c39378664a29d5fda Mon Sep 17 00:00:00 2001 From: David Jones Date: Mon, 18 Dec 2023 10:15:04 +0000 Subject: [PATCH 123/244] Update gtc_osiris.py Increased exposure time limit for GTC-OSIRIS standards to account for longer standards with high-res grisms --- pypeit/spectrographs/gtc_osiris.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/spectrographs/gtc_osiris.py b/pypeit/spectrographs/gtc_osiris.py index daa644241c..1a6864f8fa 100644 --- a/pypeit/spectrographs/gtc_osiris.py +++ b/pypeit/spectrographs/gtc_osiris.py @@ -110,7 +110,7 @@ def default_pypeit_par(cls): par['calibrations']['pinholeframe']['exprng'] = [999999, None] # No pinhole frames par['calibrations']['arcframe']['exprng'] = [None, None] # Long arc exposures par['calibrations']['arcframe']['process']['clip'] = False - par['calibrations']['standardframe']['exprng'] = [None, 180] + par['calibrations']['standardframe']['exprng'] = [None, 300] # Multiple arcs with different lamps, so can't median combine nor clip, also need to remove continuum par['calibrations']['arcframe']['process']['combine'] = 'mean' par['calibrations']['arcframe']['process']['subtract_continuum'] = True From c8028eeabf6b1d829ad052f2bbb6f3e21cd29a8c Mon Sep 17 00:00:00 2001 From: rcooke Date: Mon, 18 Dec 2023 16:15:03 +0000 Subject: [PATCH 124/244] fixes --- pypeit/coadd3d.py | 22 +++++++++++----------- pypeit/inputfiles.py | 41 ++++++++++++++++++++++------------------- 2 files changed, 33 insertions(+), 30 deletions(-) diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index bd48f2dcfe..b8b2374abd 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -428,6 +428,17 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs self.scale_corr = scale_corr self.ra_offsets = list(ra_offsets) if isinstance(ra_offsets, np.ndarray) else ra_offsets self.dec_offsets = list(dec_offsets) if isinstance(dec_offsets, np.ndarray) else dec_offsets + # If there is only one frame being "combined" AND there's no reference image, then don't compute the translation. + if self.numfiles == 1 and self.cubepar["reference_image"] is None: + if self.align: + msgs.warn("Parameter 'align' should be False when there is only one frame and no reference image") + msgs.info("Setting 'align' to False") + self.align = False + if self.ra_offsets is not None: + if not self.align: + msgs.warn("When 'ra_offset' and 'dec_offset' are set, 'align' must be True.") + msgs.info("Setting 'align' to True") + self.align = True # If no ra_offsets or dec_offsets have been provided, initialise the lists self.user_alignment = True if self.ra_offsets is None and self.dec_offsets is None: @@ -459,17 +470,6 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs self._dspat = None if self.cubepar['spatial_delta'] is None else self.cubepar['spatial_delta'] / 3600.0 # binning size on the sky (/3600 to convert to degrees) self._dwv = self.cubepar['wave_delta'] # linear binning size in wavelength direction (in Angstroms) - # If there is only one frame being "combined" AND there's no reference image, then don't compute the translation. - if self.numfiles == 1 and self.cubepar["reference_image"] is None: - if self.align: - msgs.warn("Parameter 'align' should be False when there is only one frame and no reference image") - msgs.info("Setting 'align' to False") - self.align = False - if self.ra_offsets is not None: - if not self.align: - msgs.warn("When 'ra_offset' and 'dec_offset' are set, 'align' must be True.") - msgs.info("Setting 'align' to True") - self.align = True # TODO :: The default behaviour (combine=False, align=False) produces a datacube that uses the instrument WCS # It should be possible (and perhaps desirable) to do a spatial alignment (i.e. align=True), apply this to the # RA,Dec values of each pixel, and then use the instrument WCS to save the output (or, just adjust the crval). diff --git a/pypeit/inputfiles.py b/pypeit/inputfiles.py index 5fc3a28592..6759dffae1 100644 --- a/pypeit/inputfiles.py +++ b/pypeit/inputfiles.py @@ -935,26 +935,29 @@ def options(self): # Load coordinate offsets for each file. This is "Delta RA cos(dec)" and "Delta Dec" # Get the RA offset of each file - off_ra = self.data['ra_offset'].tolist() - if off_ra is None: - opts['ra_offset'] = None - elif len(off_ra) == 1 and len(self.filenames) > 1: - # Convert from arcsec to degrees - opts['ra_offset'] = [off_ra[0]/3600.0 for _ in range(len(self.filenames))] - elif len(off_ra) != 0: - # Convert from arcsec to degrees - opts['ra_offset'] = [ora/3600.0 for ora in off_ra] + off_ra, off_dec = None, None + if 'ra_offset' in self.data.keys(): + off_ra = self.data['ra_offset'].tolist() + if off_ra is None: + opts['ra_offset'] = None + elif len(off_ra) == 1 and len(self.filenames) > 1: + # Convert from arcsec to degrees + opts['ra_offset'] = [off_ra[0]/3600.0 for _ in range(len(self.filenames))] + elif len(off_ra) != 0: + # Convert from arcsec to degrees + opts['ra_offset'] = [ora/3600.0 for ora in off_ra] # Get the DEC offset of each file - off_dec = self.data['dec_offset'].tolist() - if off_dec is None: - opts['dec_offset'] = None - elif len(off_dec) == 1 and len(self.filenames) > 1: - # Convert from arcsec to degrees - opts['dec_offset'] = [off_dec[0]/3600.0 for _ in range(len(self.filenames))] - elif len(off_dec) != 0: - # Convert from arcsec to degrees - opts['dec_offset'] = [odec/3600.0 for odec in off_dec] - # Check that both have been set + if 'dec_offset' in self.data.keys(): + off_dec = self.data['dec_offset'].tolist() + if off_dec is None: + opts['dec_offset'] = None + elif len(off_dec) == 1 and len(self.filenames) > 1: + # Convert from arcsec to degrees + opts['dec_offset'] = [off_dec[0]/3600.0 for _ in range(len(self.filenames))] + elif len(off_dec) != 0: + # Convert from arcsec to degrees + opts['dec_offset'] = [odec/3600.0 for odec in off_dec] + # Check that both have been set or both are not set if (off_ra is not None and off_dec is None) or (off_ra is None and off_dec is not None): msgs.error("You must specify both or neither of the following arguments: ra_offset, dec_offset") From 64367a5cf9e935029857f71a6ed231611921e9c0 Mon Sep 17 00:00:00 2001 From: Ryan Cooke Date: Mon, 18 Dec 2023 16:39:49 +0000 Subject: [PATCH 125/244] Update telluric.py lowercase pca --- pypeit/core/telluric.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 625bd99567..95539d6620 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -2339,7 +2339,7 @@ def empty_model_table(norders, nspec, tell_npca=4, n_obj_par=0): description='Maximum wavelength included in the fit')]) def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_model, eval_obj_model, - log10_blaze_function=None, ech_orders=None, sn_clip=30.0, teltype='PCA', tell_npca=4, + log10_blaze_function=None, ech_orders=None, sn_clip=30.0, teltype='pca', tell_npca=4, airmass_guess=1.5, resln_guess=None, resln_frac_bounds=(0.3, 1.5), pix_shift_bounds=(-5.0, 5.0), pix_stretch_bounds=(0.9,1.1), maxiter=2, sticky=True, lower=3.0, upper=3.0, seed=777, ballsize = 5e-4, tol=1e-3, diff_evol_maxiter=1000, popsize=30, From abc092e7b073366ec7145c4a1b95ff513f482d08 Mon Sep 17 00:00:00 2001 From: rcooke Date: Mon, 18 Dec 2023 19:15:24 +0000 Subject: [PATCH 126/244] PR fixes --- pypeit/core/telluric.py | 32 ++++++++++++++++---------------- pypeit/tests/tstutils.py | 2 +- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 95539d6620..f3bbc96147 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -168,7 +168,7 @@ def read_telluric_pca(filename, wave_min=None, wave_max=None, pad_frac=0.10): - tell_pca: PCA component vectors - bounds_tell_pca: Maximum/minimum coefficient - coefs_tell_pca: Set of model coefficient values (for prior in future) - - teltype: Type of telluric model, i.e. 'PCA' + - teltype: Type of telluric model, i.e. 'pca' """ # load_telluric_grid() takes care of path and existance check hdul = data.load_telluric_grid(filename) @@ -385,13 +385,13 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): Args: theta_tell (`numpy.ndarray`_): - Vector with tell_npca PCA coefficients (if teltype='PCA') + Vector with tell_npca PCA coefficients (if teltype='pca') or pressure, temperature, humidity, and airmass (if teltype='grid'), followed by spectral resolution, shift, and stretch. Final length is then tell_npca+3 or 7. tell_dict (:obj:`dict`): Dictionary containing the telluric data. See - :func:`read_telluric_pca` if teltype=='PCA'. + :func:`read_telluric_pca` if teltype=='pca'. or :func:`read_telluric_grid` if teltype=='grid'. ind_lower (:obj:`int`, optional): @@ -1300,7 +1300,7 @@ def eval_poly_model(theta, obj_dict): def sensfunc_telluric(wave, counts, counts_ivar, counts_mask, exptime, airmass, std_dict, telgridfile, log10_blaze_function=None, ech_orders=None, polyorder=8, - tell_npca=4, teltype='PCA', + tell_npca=4, teltype='pca', mask_hydrogen_lines=True, mask_helium_lines=False, hydrogen_mask_wid=10., resln_guess=None, resln_frac_bounds=(0.6, 1.4), pix_shift_bounds=(-5.0, 5.0), delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), @@ -1351,8 +1351,8 @@ def sensfunc_telluric(wave, counts, counts_ivar, counts_mask, exptime, airmass, If passed, provides the true order numbers for the spectra provided. polyorder : :obj:`int`, optional, default = 8 Polynomial order for the sensitivity function fit. - teltype : :obj:`str`, optional, default = 'PCA' - Method for evaluating telluric models, either `PCA` or `grid`. + teltype : :obj:`str`, optional, default = 'pca' + Method for evaluating telluric models, either `pca` or `grid`. tell_npca : :obj:`int`, optional, default = 4 Number of telluric PCA components used, must be <= 10 mask_hydrogen_lines : :obj:`bool`, optional @@ -1515,7 +1515,7 @@ def create_bal_mask(wave, bal_wv_min_max): def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile, npca=8, pca_lower=1220.0, pca_upper=3100.0, bal_wv_min_max=None, delta_zqso=0.1, - teltype='PCA', tell_npca=4, + teltype='pca', tell_npca=4, bounds_norm=(0.1, 3.0), tell_norm_thresh=0.9, sn_clip=30.0, only_orders=None, maxiter=3, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, pix_shift_bounds=(-5.0,5.0), debug_init=False, debug=False, show=False): @@ -1551,8 +1551,8 @@ def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile delta_zqso : :obj:`float`, optional During the fit, the QSO redshift is allowed to vary within ``+/-delta_zqso``. - teltype : :obj:`str`, optional, default = 'PCA' - Method for evaluating telluric models, either `PCA` or `grid`. + teltype : :obj:`str`, optional, default = 'pca' + Method for evaluating telluric models, either `pca` or `grid`. tell_npca : :obj:`int`, optional, default = 4 Number of telluric PCA components used, must be <=10 bounds_norm : :obj:`tuple`, optional @@ -1685,7 +1685,7 @@ def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile def star_telluric(spec1dfile, telgridfile, telloutfile, outfile, star_type=None, star_mag=None, star_ra=None, star_dec=None, func='legendre', model='exp', - polyorder=5, teltype='PCA', tell_npca=4, mask_hydrogen_lines=True, + polyorder=5, teltype='pca', tell_npca=4, mask_hydrogen_lines=True, mask_helium_lines=False, hydrogen_mask_wid=10., delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), only_orders=None, sn_clip=30.0, maxiter=3, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, @@ -1793,7 +1793,7 @@ def star_telluric(spec1dfile, telgridfile, telloutfile, outfile, star_type=None, return TelObj def poly_telluric(spec1dfile, telgridfile, telloutfile, outfile, z_obj=0.0, func='legendre', - model='exp', polyorder=3, fit_wv_min_max=None, mask_lyman_a=True, teltype='PCA', + model='exp', polyorder=3, fit_wv_min_max=None, mask_lyman_a=True, teltype='pca', tell_npca=4, delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), only_orders=None, sn_clip=30.0, maxiter=3, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, pix_shift_bounds=(-5.0,5.0), @@ -1936,13 +1936,13 @@ class Telluric(datamodel.DataContainer): The telluric model is governed by seven parameters --- for the `grid` approach: ``pressure``, ``temperature``, ``humidity``, ``airmass``; for - the `PCA` approach 4 PCA coefficients; plus ``resln``, ``shift``, and + the `pca` approach 4 PCA coefficients; plus ``resln``, ``shift``, and ``stretch`` --- where ``resln`` is the resolution of the spectrograph and ``shift`` is a shift in pixels. The ``stretch`` is a stretch of the pixel scale. The airmass of the object will be used to initalize the fit (this helps with initalizing the object model), but the models are sufficiently flexible that often the best-fit airmass actually differs from the - airmass of the spectrum. In the `PCA` approach, the number of PCA + airmass of the spectrum. In the `pca` approach, the number of PCA components used can be adjusted between 1 and 10, with a tradeoff between speed and accuracy. @@ -1983,10 +1983,10 @@ class Telluric(datamodel.DataContainer): decomposition of such models, see :class:`~pypeit.par.pypeitpar.TelluricPar`. teltype (:obj:`str`, optional): - Method for evaluating telluric models, either `PCA` or `grid`. + Method for evaluating telluric models, either `pca` or `grid`. The `grid` method directly interpolates a pre-computed grid of HITRAN atmospheric models with nearest grid-point interpolation, - while the `PCA` method instead fits for the coefficients of a + while the `pca` method instead fits for the coefficients of a PCA decomposition of HITRAN atmospheric models, enabling a more flexible and far more file-size efficient interpolation of the telluric absorption model space. @@ -2180,7 +2180,7 @@ class Telluric(datamodel.DataContainer): descr='File containing PCA components or grid of ' 'HITRAN atmosphere models'), 'teltype': dict(otype=str, - descr='Type of telluric model, `PCA` or `grid`'), + descr='Type of telluric model, `pca` or `grid`'), 'tell_npca': dict(otype=int, descr='Number of telluric PCA components used'), 'std_src': dict(otype=str, descr='Name of the standard source'), diff --git a/pypeit/tests/tstutils.py b/pypeit/tests/tstutils.py index 7222d9d3b9..c120e2f388 100644 --- a/pypeit/tests/tstutils.py +++ b/pypeit/tests/tstutils.py @@ -36,7 +36,7 @@ # Tests require the Telluric file (Mauna Kea) par = Spectrograph.default_pypeit_par() -tell_test_grid = data.get_telgrid_filepath('TelFit_MaunaKea_3100_26100_R20000.fits') +tell_test_grid = data.get_telgrid_filepath('TellPCA_3000_26000_R25000.fits') #tell_test_grid = data.Paths.telgrid / 'TelFit_MaunaKea_3100_26100_R20000.fits' telluric_required = pytest.mark.skipif(not tell_test_grid.is_file(), reason='no Mauna Kea telluric file') From 2eb2894aa1804c6525c997542f15742e4f7b1624 Mon Sep 17 00:00:00 2001 From: "J. Xavier Prochaska" Date: Mon, 18 Dec 2023 11:55:13 -0800 Subject: [PATCH 127/244] typo --- CITATION.cff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CITATION.cff b/CITATION.cff index 4cca8f5b19..c225a7c686 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -2,7 +2,7 @@ cff-version: 1.2.0 message: "If you use this software, please cite it as below." authors: - family-names: Prochaska, Hennawi, Westfall, Cooke, Wang - given-names: J. Xavier, Joseph, Kyle, Ryan + given-names: J. Xavier, Joseph, Kyle, Ryan, Feige title: "PypeIt" version: 1.10.0 doi: 10.5281/zenodo.3743493 From a223cf73ec44f5cc5ac17b5f0b875e85f12e7159 Mon Sep 17 00:00:00 2001 From: "J. Xavier Prochaska" Date: Mon, 18 Dec 2023 11:57:16 -0800 Subject: [PATCH 128/244] mo --- CITATION.cff | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index c225a7c686..02b0303597 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -1,9 +1,9 @@ cff-version: 1.2.0 message: "If you use this software, please cite it as below." authors: - - family-names: Prochaska, Hennawi, Westfall, Cooke, Wang - given-names: J. Xavier, Joseph, Kyle, Ryan, Feige + - family-names: Prochaska, Hennawi, Westfall, Cooke, Wang, Hsyu, Davies, Farina, Pelliccia + given-names: J. Xavier, Joseph, Kyle, Ryan, Feige, Tiffany, Fred, Emanuele, Debora title: "PypeIt" -version: 1.10.0 -doi: 10.5281/zenodo.3743493 -date-released: 2022-08-28 \ No newline at end of file +version: 1.14.0 +doi: 10.21105/joss.02308 +date-released: 2020-08-28 \ No newline at end of file From 2a87c54ab5b0b7e2432366df5acb2121cf646d33 Mon Sep 17 00:00:00 2001 From: joe Date: Mon, 18 Dec 2023 17:27:21 -1000 Subject: [PATCH 129/244] Working on KCWI --- pypeit/inputfiles.py | 40 ++++++++++++++++---------------- pypeit/scripts/coadd_datacube.py | 2 +- pypeit/specobjs.py | 2 ++ 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/pypeit/inputfiles.py b/pypeit/inputfiles.py index 5fc3a28592..7deb108f4f 100644 --- a/pypeit/inputfiles.py +++ b/pypeit/inputfiles.py @@ -935,28 +935,28 @@ def options(self): # Load coordinate offsets for each file. This is "Delta RA cos(dec)" and "Delta Dec" # Get the RA offset of each file - off_ra = self.data['ra_offset'].tolist() - if off_ra is None: - opts['ra_offset'] = None - elif len(off_ra) == 1 and len(self.filenames) > 1: - # Convert from arcsec to degrees - opts['ra_offset'] = [off_ra[0]/3600.0 for _ in range(len(self.filenames))] - elif len(off_ra) != 0: - # Convert from arcsec to degrees - opts['ra_offset'] = [ora/3600.0 for ora in off_ra] + #off_ra = self.data['ra_offset'].tolist() + #if off_ra is None: + # opts['ra_offset'] = None + #elif len(off_ra) == 1 and len(self.filenames) > 1: + # # Convert from arcsec to degrees + # opts['ra_offset'] = [off_ra[0]/3600.0 for _ in range(len(self.filenames))] + #elif len(off_ra) != 0: + # # Convert from arcsec to degrees + # opts['ra_offset'] = [ora/3600.0 for ora in off_ra] # Get the DEC offset of each file - off_dec = self.data['dec_offset'].tolist() - if off_dec is None: - opts['dec_offset'] = None - elif len(off_dec) == 1 and len(self.filenames) > 1: - # Convert from arcsec to degrees - opts['dec_offset'] = [off_dec[0]/3600.0 for _ in range(len(self.filenames))] - elif len(off_dec) != 0: - # Convert from arcsec to degrees - opts['dec_offset'] = [odec/3600.0 for odec in off_dec] + #off_dec = self.data['dec_offset'].tolist() + #if off_dec is None: + # opts['dec_offset'] = None + #elif len(off_dec) == 1 and len(self.filenames) > 1: + # # Convert from arcsec to degrees + # opts['dec_offset'] = [off_dec[0]/3600.0 for _ in range(len(self.filenames))] + #elif len(off_dec) != 0: + # # Convert from arcsec to degrees + # opts['dec_offset'] = [odec/3600.0 for odec in off_dec] # Check that both have been set - if (off_ra is not None and off_dec is None) or (off_ra is None and off_dec is not None): - msgs.error("You must specify both or neither of the following arguments: ra_offset, dec_offset") + #if (off_ra is not None and off_dec is None) or (off_ra is None and off_dec is not None): + # msgs.error("You must specify both or neither of the following arguments: ra_offset, dec_offset") # Return all options return opts diff --git a/pypeit/scripts/coadd_datacube.py b/pypeit/scripts/coadd_datacube.py index 26e12a6bd2..42e7809561 100644 --- a/pypeit/scripts/coadd_datacube.py +++ b/pypeit/scripts/coadd_datacube.py @@ -13,7 +13,7 @@ from pypeit.coadd3d import CoAdd3D from pypeit.spectrographs.util import load_spectrograph from pypeit.scripts import scriptbase - +from IPython import embed class CoAddDataCube(scriptbase.ScriptBase): diff --git a/pypeit/specobjs.py b/pypeit/specobjs.py index e45c155203..baf3a7f1d1 100644 --- a/pypeit/specobjs.py +++ b/pypeit/specobjs.py @@ -1035,6 +1035,8 @@ def get_std_trace(detname, std_outfile, chk_version=True): std_trace = std_trace.flatten() elif 'Echelle' in pypeline: std_trace = std_trace.T + elif 'SlicerIFU' in pypeline: + std_trace = None else: msgs.error('Unrecognized pypeline') else: From 885eeb44e187534f509164fbd874a987048c62b5 Mon Sep 17 00:00:00 2001 From: Debora Pelliccia Date: Tue, 19 Dec 2023 17:25:57 -1000 Subject: [PATCH 130/244] fix the fits file --- .../sky_LRISr_600_7500_5460_7950.fits | Bin 31680 -> 31680 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/pypeit/data/sky_spec/sky_LRISr_600_7500_5460_7950.fits b/pypeit/data/sky_spec/sky_LRISr_600_7500_5460_7950.fits index 0d926ff155caabb32a43f1ca098f3053734d7093..0231ab5a88a12ef5b4897e0f0603c46f1bbd1a4f 100644 GIT binary patch delta 838 zcmbtS!EVz)5S34WIFLijp-6P>OD?TaZ$g|zic~9hNknTb#et|7w8Wbb8n5HE4XqFg ze*ngJ1bhPdfru~S)+@V-lUM?JVWruTc6Q#peQy`n(&AeB1bZ;>>Ck6h2$2sb9j6On zgsF>}n*{gM!c+MkW(;3}iz5ut)s!RwBwyt{8hbAZ1Ahovee4k$j(r*?!EW+vKm9JL z>9MrTCA05X`k4t$D4v}5`oLFFY}qOSxr|obK(WV1$ak!+ZMJy5nFKo=>6?TeT6SmI z+FbW-GCHc%#Fbgk5-;FgP<*EhC-J4y0L5KXn~*Sk5z}#`9Dwo`zagTk(@_{P-&5Y_ zDP_>Dw%K~p?VWKv<2fxQ34-OSR;fIyYPI@yaA~Ep;FyEG0oAmO3|XZ47wPA%ALk1>U06_nj)zBN ztF3U({vSU-+w#uV_bF!FtB5e}_=D88<-)c$Vtzd3=lgl~`vb9Wi+Vga+F%c>?o? J%?{iL6an&DOpO2l From 9a217b0c437f535b790b2b5780832d749436a6bd Mon Sep 17 00:00:00 2001 From: Debora Pelliccia Date: Wed, 20 Dec 2023 11:11:52 -1000 Subject: [PATCH 131/244] added amp A gain --- .gitignore | 2 +- doc/dev/deimosconfig.rst | 4 +- doc/dev/deimosframes.rst | 2 +- .../ampA/gain_check.Direct.2008-Dec-17.log | 27 +++++++ .../ampA/gain_check.Direct.2008-Jul-09.log | 27 +++++++ .../ampA/gain_check.Direct.2008-Nov-19.log | 27 +++++++ .../ampA/gain_check.Direct.2009-Apr-15.log | 27 +++++++ .../ampA/gain_check.Direct.2009-Dec-08.log | 27 +++++++ .../ampA/gain_check.Direct.2009-Jan-21.log | 27 +++++++ .../ampA/gain_check.Direct.2009-Jun-18.log | 27 +++++++ .../ampA/gain_check.Direct.2009-Mar-17.log | 27 +++++++ .../ampA/gain_check.Direct.2009-Sep-30.log | 27 +++++++ .../ampA/gain_check.Direct.2010-Feb-02.log | 27 +++++++ .../ampA/gain_check.Direct.2010-Jan-07.log | 27 +++++++ .../ampA/gain_check.Direct.2010-Jun-06.log | 27 +++++++ .../ampA/gain_check.Direct.2010-Mar-02.log | 27 +++++++ .../ampA/gain_check.Direct.2010-May-05.log | 27 +++++++ .../ampA/gain_check.Direct.2010-Sep-01.log | 27 +++++++ .../ampA/gain_check.Direct.2011-Feb-26.log | 27 +++++++ .../ampA/gain_check.Direct.2011-Mar-22.log | 27 +++++++ .../ampA/gain_check.Direct.2011-May-25.log | 27 +++++++ .../ampA/gain_check.Direct.2011-Oct-25.log | 27 +++++++ .../ampA/gain_check.Direct.2012-Apr-12.log | 27 +++++++ .../ampA/gain_check.Direct.2012-Jun-08.log | 27 +++++++ .../ampA/gain_check.Direct.2012-Mar-14.log | 27 +++++++ .../ampA/gain_check.Direct.2012-Nov-11.log | 26 +++++++ .../ampA/gain_check.Direct.2013-Aug-26.log | 27 +++++++ .../ampA/gain_check.Direct.2013-Dec-22.log | 27 +++++++ .../ampA/gain_check.Direct.2013-Feb-04.log | 27 +++++++ .../ampA/gain_check.Direct.2013-Oct-28.log | 27 +++++++ .../ampA/gain_check.Direct.2013-Sep-26.log | 27 +++++++ .../ampA/gain_check.Direct.2014-Oct-16.log | 27 +++++++ .../ampA/gain_check.Direct.2015-Apr-10.log | 28 ++++++++ .../ampA/gain_check.Direct.2015-Dec-01.log | 27 +++++++ .../ampA/gain_check.Direct.2015-Dec-30.log | 27 +++++++ .../ampA/gain_check.Direct.2015-Sep-02.log | 27 +++++++ .../ampA/gain_check.Direct.2015-Sep-30.log | 27 +++++++ .../ampA/gain_check.Direct.2016-Dec-17.log | 27 +++++++ .../ampA/gain_check.Direct.2016-May-03.log | 27 +++++++ .../ampA/gain_check.Direct.2016-Nov-22.log | 27 +++++++ .../ampA/gain_check.Direct.2017-Dec-10.log | 27 +++++++ .../ampA/gain_check.Direct.2017-May-16.log | 27 +++++++ .../ampA/gain_check.Direct.2018-Apr-19.log | 27 +++++++ .../ampA/gain_check.Direct.2018-Aug-08.log | 27 +++++++ .../ampA/gain_check.Direct.2018-Aug-17.log | 27 +++++++ .../ampA/gain_check.Direct.2018-May-17.log | 27 +++++++ .../ampA/gain_check.Direct.2018-Nov-26.log | 27 +++++++ .../ampA/gain_check.Direct.2019-Dec-29.log | 27 +++++++ .../ampA/gain_check.Direct.2019-Nov-19.log | 27 +++++++ .../ampA/gain_check.Direct.2019-Nov-27.log | 27 +++++++ .../ampA/gain_check.Direct.2021-Jun-21.log | 27 +++++++ .../ampA/gain_check.Direct.2021-May-04.log | 27 +++++++ .../ampA/gain_check.Direct.2021-May-05.log | 27 +++++++ .../ampA/gain_check.Direct.2021-May-31.log | 27 +++++++ .../ampA/gain_check.Direct.2021-Sep-05.log | 27 +++++++ .../ampA/gain_check.Direct.2022-Apr-22.log | 27 +++++++ .../ampA/gain_check.Direct.2022-Jan-28.log | 27 +++++++ .../ampA/gain_check.Direct.2022-Jun-01.log | 27 +++++++ .../ampA/gain_check.Direct.2022-Jun-22.log | 27 +++++++ .../ampA/gain_check.Direct.2022-Nov-14.log | 27 +++++++ .../ampA/gain_check.Direct.2022-Oct-11.log | 27 +++++++ .../ampA/gain_check.Direct.2023-Aug-06.log | 27 +++++++ .../ampA/gain_check.Direct.2023-Dec-11.log | 27 +++++++ .../ampA/gain_check.Direct.2023-Jan-06.log | 27 +++++++ .../ampA/gain_check.Direct.2023-Oct-05.log | 27 +++++++ .../gain_check.Spectral.2008-Dec-17.log | 0 .../gain_check.Spectral.2008-Jul-09.log | 0 .../gain_check.Spectral.2008-Nov-19.log | 0 .../gain_check.Spectral.2009-Apr-16.log | 0 .../gain_check.Spectral.2009-Dec-08.log | 0 .../gain_check.Spectral.2009-Jun-18.log | 0 .../gain_check.Spectral.2009-Mar-17.log | 0 .../gain_check.Spectral.2009-Sep-30.log | 0 .../gain_check.Spectral.2010-Feb-02.log | 0 .../gain_check.Spectral.2010-Jan-07.log | 0 .../gain_check.Spectral.2010-Jun-06.log | 0 .../gain_check.Spectral.2010-Mar-02.log | 0 .../gain_check.Spectral.2010-May-05.log | 0 .../gain_check.Spectral.2010-Sep-01.log | 0 .../gain_check.Spectral.2011-Feb-26.log | 0 .../gain_check.Spectral.2011-Mar-22.log | 0 .../gain_check.Spectral.2011-May-25.log | 0 .../gain_check.Spectral.2011-Oct-25.log | 0 .../gain_check.Spectral.2012-Nov-11.log | 0 .../gain_check.Spectral.2013-Aug-26.log | 0 .../gain_check.Spectral.2013-Dec-22.log | 0 .../gain_check.Spectral.2013-Feb-04.log | 0 .../gain_check.Spectral.2013-Oct-28.log | 0 .../gain_check.Spectral.2013-Sep-26.log | 0 .../gain_check.Spectral.2014-Oct-16.log | 0 .../gain_check.Spectral.2015-Apr-10.log | 0 .../gain_check.Spectral.2015-Dec-01.log | 0 .../gain_check.Spectral.2015-Dec-30.log | 0 .../gain_check.Spectral.2015-Sep-02.log | 0 .../gain_check.Spectral.2015-Sep-30.log | 0 .../gain_check.Spectral.2016-Dec-17.log | 0 .../gain_check.Spectral.2016-May-03.log | 0 .../gain_check.Spectral.2016-Nov-22.log | 0 .../gain_check.Spectral.2017-Dec-10.log | 0 .../gain_check.Spectral.2017-May-16.log | 0 .../gain_check.Spectral.2018-Apr-19.log | 0 .../gain_check.Spectral.2018-May-17.log | 0 .../gain_check.Spectral.2018-Nov-26.log | 0 .../gain_check.Spectral.2019-Dec-29.log | 0 .../gain_check.Spectral.2019-Nov-19.log | 0 .../gain_check.Spectral.2019-Nov-27.log | 0 .../gain_check.Spectral.2021-Jun-21.log | 0 .../gain_check.Spectral.2021-May-04.log | 0 .../gain_check.Spectral.2021-May-05.log | 0 .../gain_check.Spectral.2021-May-31.log | 0 .../gain_check.Spectral.2021-Sep-05.log | 0 .../gain_check.Spectral.2022-Apr-22.log | 0 .../gain_check.Spectral.2022-Jan-28.log | 0 .../ampB/gain_check.Spectral.2022-Jun-01.log | 27 +++++++ .../ampB/gain_check.Spectral.2022-Jun-22.log | 27 +++++++ .../ampB/gain_check.Spectral.2022-Nov-14.log | 27 +++++++ .../ampB/gain_check.Spectral.2022-Oct-11.log | 27 +++++++ .../ampB/gain_check.Spectral.2023-Aug-06.log | 27 +++++++ .../ampB/gain_check.Spectral.2023-Dec-11.log | 27 +++++++ .../ampB/gain_check.Spectral.2023-Jan-06.log | 27 +++++++ .../ampB/gain_check.Spectral.2023-Oct-05.log | 27 +++++++ pypeit/spectrographs/keck_deimos.py | 72 ++++++++++++------- 122 files changed, 1941 insertions(+), 29 deletions(-) create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2008-Dec-17.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2008-Jul-09.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2008-Nov-19.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Apr-15.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Dec-08.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Jan-21.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Jun-18.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Mar-17.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Sep-30.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Feb-02.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Jan-07.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Jun-06.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Mar-02.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-May-05.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Sep-01.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-Feb-26.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-Mar-22.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-May-25.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-Oct-25.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Apr-12.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Jun-08.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Mar-14.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Nov-11.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Aug-26.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Dec-22.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Feb-04.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Oct-28.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Sep-26.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2014-Oct-16.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Apr-10.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Dec-01.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Dec-30.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Sep-02.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Sep-30.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2016-Dec-17.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2016-May-03.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2016-Nov-22.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2017-Dec-10.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2017-May-16.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Apr-19.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Aug-08.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Aug-17.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-May-17.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Nov-26.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2019-Dec-29.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2019-Nov-19.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2019-Nov-27.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-Jun-21.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-May-04.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-May-05.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-May-31.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-Sep-05.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Apr-22.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Jan-28.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Jun-01.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Jun-22.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Nov-14.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Oct-11.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Aug-06.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Dec-11.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Jan-06.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Oct-05.log rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2008-Dec-17.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2008-Jul-09.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2008-Nov-19.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2009-Apr-16.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2009-Dec-08.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2009-Jun-18.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2009-Mar-17.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2009-Sep-30.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2010-Feb-02.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2010-Jan-07.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2010-Jun-06.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2010-Mar-02.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2010-May-05.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2010-Sep-01.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2011-Feb-26.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2011-Mar-22.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2011-May-25.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2011-Oct-25.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2012-Nov-11.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2013-Aug-26.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2013-Dec-22.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2013-Feb-04.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2013-Oct-28.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2013-Sep-26.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2014-Oct-16.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2015-Apr-10.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2015-Dec-01.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2015-Dec-30.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2015-Sep-02.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2015-Sep-30.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2016-Dec-17.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2016-May-03.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2016-Nov-22.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2017-Dec-10.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2017-May-16.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2018-Apr-19.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2018-May-17.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2018-Nov-26.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2019-Dec-29.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2019-Nov-19.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2019-Nov-27.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2021-Jun-21.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2021-May-04.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2021-May-05.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2021-May-31.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2021-Sep-05.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2022-Apr-22.log (100%) rename pypeit/data/spectrographs/keck_deimos/gain_ronoise/{ => ampB}/gain_check.Spectral.2022-Jan-28.log (100%) create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Jun-01.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Jun-22.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Nov-14.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Oct-11.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Aug-06.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Dec-11.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Jan-06.log create mode 100644 pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Oct-05.log diff --git a/.gitignore b/.gitignore index db6b3a8c4b..11d23b8394 100644 --- a/.gitignore +++ b/.gitignore @@ -52,7 +52,7 @@ tst*.fits # Logs and databases # ###################### *.log -!pypeit/data/spectrographs/keck_deimos/gain_ronoise/*.log +!pypeit/data/spectrographs/keck_deimos/gain_ronoise/*/*.log *.sql *.sqlite diff --git a/doc/dev/deimosconfig.rst b/doc/dev/deimosconfig.rst index 3401cd76ef..995140c065 100644 --- a/doc/dev/deimosconfig.rst +++ b/doc/dev/deimosconfig.rst @@ -48,7 +48,7 @@ as determined by :func:`pypeit.metadata.PypeItMetaData.unique_configurations`. The ``AMPMODE`` value is included, even though ``PypeIt`` (currently) restricts itself to only attempting to reduce frames read by the B -amplifier; see +and A amplifiers; see :func:`~pypeit.spectrographs.keck_deimos.KeckDEIMOSSpectrograph.valid_configuration_values`. Additionally, ``PypeIt`` requires all frames to have ``MOSMODE == 'Spectral'``. Frames that do not match these header keyword @@ -178,7 +178,7 @@ The algorithm for this test is as follows: 5. Check that "cleaning" the configurations of frames that cannot be reduced by ``PypeIt`` (those with ``MOSMODE != 'Spectral'`` - or ``AMPMODE != SINGLE:B``), using + or ``AMPMODE != SINGLE:B`` or ``AMPMODE != SINGLE:A``), using :func:`~pypeit.metadata.PypeItMetaData.clean_configurations` does not remove any file because all of the dev-suite files are valid. diff --git a/doc/dev/deimosframes.rst b/doc/dev/deimosframes.rst index 5206aa2286..a1b1024a36 100644 --- a/doc/dev/deimosframes.rst +++ b/doc/dev/deimosframes.rst @@ -79,7 +79,7 @@ Also note that ``PypeIt`` will ignore frames observed under conditions that do not meet the restricted values set by :func:`~pypeit.spectrographs.keck_deimos.KeckDEIMOSSpectrograph.valid_configuration_values`. This currently requires all frames to have ``MOSMODE == 'Spectral'`` -and ``AMPMODE == 'SINGLE:B'``. Frames that do not meet these criteria +and ``AMPMODE == 'SINGLE:B'`` or ``AMPMODE == 'SINGLE:A'``. Frames that do not meet these criteria will not be included in the automatically generated :ref:`pypeit_file` created by :ref:`pypeit_setup`. diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2008-Dec-17.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2008-Dec-17.log new file mode 100644 index 0000000000..7e604036cd --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2008-Dec-17.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2008-Dec-17.log +# datestr = Wed Dec 17 16:23:10 2008 +# mode = Direct +# flatfile1 = d1218_0040.fits +# flatfile2 = d1218_0041.fits +# flatfile3 = d1218_0042.fits +# flatfile4 = d1218_0043.fits +# flatfile5 = d1218_0044.fits +# zerofile1 = d1218_0037.fits +# zerofile2 = d1218_0038.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2024 0.0024215 2.5444 0.013172 + 2 1 1 B 1.2112 0.0035957 2.5395 0.0075393 + 3 4 2 A 1.1864 0.0024307 2.4875 0.0050964 + 4 3 2 B 1.1736 0.0060077 2.4716 0.012121 + 5 6 3 A 1.1687 0.0015475 2.5022 0.032810 + 6 5 3 B 1.2725 0.0034453 2.6680 0.0072236 + 7 8 4 A 1.2260 0.0010171 2.5729 0.0035058 + 8 7 4 B 1.2137 0.00081794 2.5801 0.0037044 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2008-Jul-09.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2008-Jul-09.log new file mode 100644 index 0000000000..769cd7a7ad --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2008-Jul-09.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2008-Jul-09.log +# datestr = Wed Jul 9 20:30:03 2008 +# mode = Direct +# flatfile1 = d0710_0004.fits +# flatfile2 = d0710_0005.fits +# flatfile3 = d0710_0006.fits +# flatfile4 = d0710_0007.fits +# flatfile5 = d0710_0008.fits +# zerofile1 = d0710_0001.fits +# zerofile2 = d0710_0002.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2009 0.0064159 2.5179 0.013452 + 2 1 1 B 1.2260 0.0035678 2.5706 0.0074806 + 3 4 2 A 1.1919 0.0043200 2.4990 0.0090574 + 4 3 2 B 1.1760 0.0022843 2.4656 0.0047896 + 5 6 3 A 1.1750 0.0013504 2.4636 0.0028317 + 6 5 3 B 1.2672 0.0044587 2.6569 0.0093487 + 7 8 4 A 1.2208 0.0016011 2.5596 0.0033569 + 8 7 4 B 1.2106 0.0016594 2.5382 0.0034793 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2008-Nov-19.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2008-Nov-19.log new file mode 100644 index 0000000000..04043087ed --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2008-Nov-19.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2008-Nov-19.log +# datestr = Wed Nov 19 18:21:21 2008 +# mode = Direct +# flatfile1 = d1120_0040.fits +# flatfile2 = d1120_0041.fits +# flatfile3 = d1120_0042.fits +# flatfile4 = d1120_0043.fits +# flatfile5 = d1120_0044.fits +# zerofile1 = d1120_0037.fits +# zerofile2 = d1120_0038.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2134 0.0011958 2.5441 0.0025072 + 2 1 1 B 1.2110 0.0019706 2.5390 0.0041318 + 3 4 2 A 1.1996 0.00058394 2.5151 0.0012244 + 4 3 2 B 1.1848 0.0027476 2.4842 0.0057606 + 5 6 3 A 1.1806 0.0025219 2.4752 0.0052876 + 6 5 3 B 1.2855 0.0029711 2.6952 0.0062293 + 7 8 4 A 1.2272 0.0069947 2.5731 0.014666 + 8 7 4 B 1.2223 0.0021113 2.5627 0.0044266 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Apr-15.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Apr-15.log new file mode 100644 index 0000000000..14b08a486a --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Apr-15.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2009-Apr-15.log +# datestr = Wed Apr 15 23:58:57 2009 +# mode = Direct +# flatfile1 = d0410_0405.fits +# flatfile2 = d0410_0406.fits +# flatfile3 = d0410_0407.fits +# flatfile4 = d0410_0408.fits +# flatfile5 = d0410_0409.fits +# zerofile1 = d0410_0402.fits +# zerofile2 = d0410_0403.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2265 0.016408 2.5715 0.034402 + 2 1 1 B 1.2128 0.0032306 2.5429 0.0067733 + 3 4 2 A 1.2521 0.044030 2.6252 0.092317 + 4 3 2 B 1.1725 0.0064645 2.4584 0.013554 + 5 6 3 A 1.2533 0.045157 2.6278 0.094680 + 6 5 3 B 1.2916 0.011757 2.7080 0.024651 + 7 8 4 A 1.3476 0.074654 2.8255 0.15653 + 8 7 4 B 1.1962 0.0043081 2.5081 0.0090330 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Dec-08.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Dec-08.log new file mode 100644 index 0000000000..276da76604 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Dec-08.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2009-Dec-08.log +# datestr = Tue Dec 8 15:28:23 2009 +# mode = Direct +# flatfile1 = d1209_0011.fits +# flatfile2 = d1209_0012.fits +# flatfile3 = d1209_0013.fits +# flatfile4 = d1209_0014.fits +# flatfile5 = d1209_0015.fits +# zerofile1 = d1209_0008.fits +# zerofile2 = d1209_0009.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2127 0.011772 2.5426 0.024683 + 2 1 1 B 1.2241 0.0095496 2.5666 0.020023 + 3 4 2 A 1.1937 0.020458 2.5029 0.042894 + 4 3 2 B 1.1922 0.013859 2.4996 0.029058 + 5 6 3 A 1.1806 0.018605 2.4753 0.039008 + 6 5 3 B 1.2891 0.022812 2.7028 0.047830 + 7 8 4 A 1.2198 0.010162 2.5576 0.021306 + 8 7 4 B 1.2326 0.015397 2.5845 0.032283 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Jan-21.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Jan-21.log new file mode 100644 index 0000000000..6050af87f4 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Jan-21.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2009-Jan-21.log +# datestr = Wed Jan 21 09:40:08 2009 +# mode = Direct +# flatfile1 = d0122_0024.fits +# flatfile2 = d0122_0025.fits +# flatfile3 = d0122_0026.fits +# flatfile4 = d0122_0027.fits +# flatfile5 = d0122_0028.fits +# zerofile1 = d0122_0021.fits +# zerofile2 = d0122_0022.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2171 0.0053002 2.5519 0.011113 + 2 1 1 B 1.2170 0.0042223 2.5517 0.0088528 + 3 4 2 A 1.1975 0.0069933 2.5108 0.014663 + 4 3 2 B 1.1870 0.00068344 2.4887 0.0014330 + 5 6 3 A 1.1794 0.0041459 2.4729 0.0086926 + 6 5 3 B 1.2890 0.00057864 2.7027 0.0012135 + 7 8 4 A 1.2213 0.0015275 2.5607 0.0032028 + 8 7 4 B 1.2251 0.00097453 2.5687 0.0020431 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Jun-18.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Jun-18.log new file mode 100644 index 0000000000..551f4903a7 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Jun-18.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2009-Jun-18.log +# datestr = Thu Jun 18 18:33:56 2009 +# mode = Direct +# flatfile1 = d0619_0004.fits +# flatfile2 = d0619_0005.fits +# flatfile3 = d0619_0006.fits +# flatfile4 = d0619_0007.fits +# flatfile5 = d0619_0008.fits +# zerofile1 = d0619_0001.fits +# zerofile2 = d0619_0002.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2179 0.0032398 2.5536 0.0067927 + 2 1 1 B 1.2157 0.0061805 2.5490 0.012959 + 3 4 2 A 1.2046 0.0019324 2.5256 0.0040515 + 4 3 2 B 1.1876 0.00090507 2.4900 0.0018975 + 5 6 3 A 1.1806 0.0052213 2.4753 0.010948 + 6 5 3 B 1.2764 0.0054817 2.6763 0.011493 + 7 8 4 A 1.2302 0.0040342 2.5793 0.0084583 + 8 7 4 B 1.2193 0.00030576 2.5564 0.00064120 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Mar-17.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Mar-17.log new file mode 100644 index 0000000000..891f643ff1 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Mar-17.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2009-Mar-17.log +# datestr = Tue Mar 17 18:51:42 2009 +# mode = Direct +# flatfile1 = d0318_0013.fits +# flatfile2 = d0318_0014.fits +# flatfile3 = d0318_0015.fits +# flatfile4 = d0318_0016.fits +# flatfile5 = d0318_0017.fits +# zerofile1 = d0318_0010.fits +# zerofile2 = d0318_0011.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2151 0.0021988 2.5476 0.0046100 + 2 1 1 B 1.2082 0.0013784 2.5332 0.0028900 + 3 4 2 A 1.1972 0.0052019 2.5102 0.010907 + 4 3 2 B 1.1881 0.0012605 2.4911 0.0026426 + 5 6 3 A 1.1790 0.0035966 2.4720 0.0075410 + 6 5 3 B 1.2847 0.0057194 2.6937 0.011992 + 7 8 4 A 1.2175 0.010660 2.5527 0.022351 + 8 7 4 B 1.2256 0.0014185 2.5697 0.0029741 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Sep-30.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Sep-30.log new file mode 100644 index 0000000000..a15cf1e592 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2009-Sep-30.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2009-Sep-30.log +# datestr = Wed Sep 30 11:59:22 2009 +# mode = Direct +# flatfile1 = d1001_0004.fits +# flatfile2 = d1001_0005.fits +# flatfile3 = d1001_0006.fits +# flatfile4 = d1001_0007.fits +# flatfile5 = d1001_0008.fits +# zerofile1 = d1001_0001.fits +# zerofile2 = d1001_0002.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2181 0.014474 2.5539 0.030348 + 2 1 1 B 1.2283 0.025599 2.5754 0.053672 + 3 4 2 A 1.1979 0.013589 2.5116 0.028492 + 4 3 2 B 1.1929 0.016964 2.5011 0.035569 + 5 6 3 A 1.1819 0.011609 2.4781 0.024341 + 6 5 3 B 1.2943 0.011172 2.7138 0.023424 + 7 8 4 A 1.2275 0.014618 2.5737 0.030649 + 8 7 4 B 1.2285 0.013163 2.5757 0.027599 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Feb-02.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Feb-02.log new file mode 100644 index 0000000000..6290526c73 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Feb-02.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2010-Feb-02.log +# datestr = Tue Feb 2 15:36:23 2010 +# mode = Direct +# flatfile1 = d0203_0023.fits +# flatfile2 = d0203_0024.fits +# flatfile3 = d0203_0025.fits +# flatfile4 = d0203_0026.fits +# flatfile5 = d0203_0027.fits +# zerofile1 = d0203_0020.fits +# zerofile2 = d0203_0021.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2259 0.015300 2.5704 0.032080 + 2 1 1 B 1.2463 0.022173 2.6131 0.046489 + 3 4 2 A 1.2265 0.018076 2.5717 0.037899 + 4 3 2 B 1.2089 0.010507 2.5347 0.022030 + 5 6 3 A 1.1995 0.025882 2.5150 0.054266 + 6 5 3 B 1.3028 0.025062 2.7315 0.052547 + 7 8 4 A 1.2350 0.013071 2.5893 0.027406 + 8 7 4 B 1.2363 0.010014 2.5921 0.020996 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Jan-07.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Jan-07.log new file mode 100644 index 0000000000..5ab65bdfd1 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Jan-07.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2010-Jan-07.log +# datestr = Thu Jan 7 18:49:08 2010 +# mode = Direct +# flatfile1 = d0108_0017.fits +# flatfile2 = d0108_0018.fits +# flatfile3 = d0108_0019.fits +# flatfile4 = d0108_0020.fits +# flatfile5 = d0108_0021.fits +# zerofile1 = d0108_0014.fits +# zerofile2 = d0108_0015.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2128 0.014558 3.8075 0.052824 + 2 1 1 B 1.2333 0.015789 2.5857 0.033105 + 3 4 2 A 1.2017 0.012996 2.5830 0.11089 + 4 3 2 B 1.1966 0.013201 2.5417 0.097460 + 5 6 3 A 1.1857 0.013864 3.7070 0.072131 + 6 5 3 B 1.2848 0.021610 2.7633 0.16351 + 7 8 4 A 1.2273 0.014075 3.7922 0.94343 + 8 7 4 B 1.2287 0.017110 3.8168 0.14523 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Jun-06.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Jun-06.log new file mode 100644 index 0000000000..dc77843920 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Jun-06.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2010-Jun-06.log +# datestr = Sun Jun 6 18:32:17 2010 +# mode = Direct +# flatfile1 = d0607_0023.fits +# flatfile2 = d0607_0024.fits +# flatfile3 = d0607_0025.fits +# flatfile4 = d0607_0026.fits +# flatfile5 = d0607_0027.fits +# zerofile1 = d0607_0020.fits +# zerofile2 = d0607_0021.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2053 0.021831 2.5271 0.045774 + 2 1 1 B 1.2251 0.017808 2.5686 0.037339 + 3 4 2 A 1.1922 0.019690 2.4996 0.041285 + 4 3 2 B 1.1790 0.017103 2.4720 0.035860 + 5 6 3 A 1.1781 0.014910 2.4701 0.031260 + 6 5 3 B 1.2919 0.017971 2.7087 0.037680 + 7 8 4 A 1.2159 0.012018 2.5493 0.025198 + 8 7 4 B 1.2086 0.016910 2.5341 0.035454 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Mar-02.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Mar-02.log new file mode 100644 index 0000000000..86c15117a3 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Mar-02.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2010-Mar-02.log +# datestr = Tue Mar 2 11:56:58 2010 +# mode = Direct +# flatfile1 = d0303_0025.fits +# flatfile2 = d0303_0026.fits +# flatfile3 = d0303_0027.fits +# flatfile4 = d0303_0028.fits +# flatfile5 = d0303_0029.fits +# zerofile1 = d0303_0022.fits +# zerofile2 = d0303_0023.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2301 0.015997 2.5792 0.033540 + 2 1 1 B 1.2431 0.012607 2.6064 0.026432 + 3 4 2 A 1.2162 0.017342 2.5499 0.036360 + 4 3 2 B 1.2022 0.017339 2.5206 0.036355 + 5 6 3 A 1.1957 0.020572 2.5491 0.093522 + 6 5 3 B 1.2961 0.025214 2.7175 0.052866 + 7 8 4 A 1.2345 0.010414 2.5884 0.021834 + 8 7 4 B 1.2352 0.017529 2.5898 0.036752 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-May-05.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-May-05.log new file mode 100644 index 0000000000..9c4daa7b92 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-May-05.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2010-May-05.log +# datestr = Wed May 5 11:13:58 2010 +# mode = Direct +# flatfile1 = d0506_0025.fits +# flatfile2 = d0506_0026.fits +# flatfile3 = d0506_0027.fits +# flatfile4 = d0506_0028.fits +# flatfile5 = d0506_0029.fits +# zerofile1 = d0506_0022.fits +# zerofile2 = d0506_0023.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2130 0.017112 2.5434 0.035878 + 2 1 1 B 1.2354 0.0080350 2.5902 0.016847 + 3 4 2 A 1.1952 0.016490 2.5060 0.034575 + 4 3 2 B 1.1975 0.015944 2.5107 0.033430 + 5 6 3 A 1.1855 0.0090983 2.4855 0.019076 + 6 5 3 B 1.2856 0.019140 2.6955 0.040130 + 7 8 4 A 1.2275 0.018102 2.5736 0.037954 + 8 7 4 B 1.2193 0.012532 2.5564 0.026274 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Sep-01.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Sep-01.log new file mode 100644 index 0000000000..4c126e2be3 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2010-Sep-01.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2010-Sep-01.log +# datestr = Wed Sep 1 07:29:09 2010 +# mode = Direct +# flatfile1 = d0901_0039.fits +# flatfile2 = d0901_0040.fits +# flatfile3 = d0901_0041.fits +# flatfile4 = d0901_0042.fits +# flatfile5 = d0901_0043.fits +# zerofile1 = d0901_0036.fits +# zerofile2 = d0901_0037.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2164 0.015648 2.5504 0.032810 + 2 1 1 B 1.2322 0.019232 2.5836 0.040324 + 3 4 2 A 1.2026 0.020414 2.5215 0.042801 + 4 3 2 B 1.1900 0.010322 2.4950 0.021643 + 5 6 3 A 1.1850 0.015011 2.4845 0.031474 + 6 5 3 B 1.2883 0.019664 2.7012 0.041229 + 7 8 4 A 1.2340 0.018733 2.5873 0.039277 + 8 7 4 B 1.2239 0.010707 2.5662 0.022449 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-Feb-26.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-Feb-26.log new file mode 100644 index 0000000000..5cb616e3f0 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-Feb-26.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2011-Feb-26.log +# datestr = Sat Feb 26 21:44:10 2011 +# mode = Direct +# flatfile1 = d0227_0052.fits +# flatfile2 = d0227_0053.fits +# flatfile3 = d0227_0054.fits +# flatfile4 = d0227_0055.fits +# flatfile5 = d0227_0056.fits +# zerofile1 = d0227_0049.fits +# zerofile2 = d0227_0050.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2369 0.015895 2.5933 0.033327 + 2 1 1 B 1.2451 0.017819 2.6105 0.037362 + 3 4 2 A 1.2236 0.015896 2.5654 0.033330 + 4 3 2 B 1.2179 0.019184 2.5535 0.040223 + 5 6 3 A 1.2036 0.014354 2.5235 0.030096 + 6 5 3 B 1.2991 0.022734 2.7239 0.047666 + 7 8 4 A 1.2369 0.012008 2.5934 0.025177 + 8 7 4 B 1.2435 0.0092990 2.6073 0.019497 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-Mar-22.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-Mar-22.log new file mode 100644 index 0000000000..1beb9486e4 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-Mar-22.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2011-Mar-22.log +# datestr = Tue Mar 22 16:21:02 2011 +# mode = Direct +# flatfile1 = d0323_0024.fits +# flatfile2 = d0323_0025.fits +# flatfile3 = d0323_0026.fits +# flatfile4 = d0323_0027.fits +# flatfile5 = d0323_0028.fits +# zerofile1 = d0323_0021.fits +# zerofile2 = d0323_0022.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2192 0.018642 2.5563 0.039087 + 2 1 1 B 1.2375 0.014153 2.5946 0.029674 + 3 4 2 A 1.2025 0.020976 2.5213 0.043980 + 4 3 2 B 1.1922 0.014358 2.4997 0.030103 + 5 6 3 A 1.1836 0.018015 2.4815 0.037772 + 6 5 3 B 1.2859 0.017405 2.6961 0.036493 + 7 8 4 A 1.2277 0.011987 2.5741 0.025133 + 8 7 4 B 1.2268 0.017227 2.5723 0.036119 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-May-25.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-May-25.log new file mode 100644 index 0000000000..7e3191c310 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-May-25.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2011-May-25.log +# datestr = Wed May 25 15:27:19 2011 +# mode = Direct +# flatfile1 = d0526_0011.fits +# flatfile2 = d0526_0012.fits +# flatfile3 = d0526_0013.fits +# flatfile4 = d0526_0014.fits +# flatfile5 = d0526_0015.fits +# zerofile1 = d0526_0008.fits +# zerofile2 = d0526_0009.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2142 0.013147 2.5458 0.027565 + 2 1 1 B 1.2394 0.0086886 2.5985 0.018217 + 3 4 2 A 1.2016 0.020383 2.5193 0.042736 + 4 3 2 B 1.1954 0.014341 2.5065 0.030069 + 5 6 3 A 1.1895 0.020515 2.4940 0.043013 + 6 5 3 B 1.2912 0.019684 2.7073 0.041271 + 7 8 4 A 1.2209 0.014744 2.5598 0.030912 + 8 7 4 B 1.2300 0.016366 2.5789 0.034315 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-Oct-25.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-Oct-25.log new file mode 100644 index 0000000000..47cd9540dc --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2011-Oct-25.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2011-Oct-25.log +# datestr = Tue Oct 25 11:14:27 2011 +# mode = Direct +# flatfile1 = d1026_0011.fits +# flatfile2 = d1026_0012.fits +# flatfile3 = d1026_0013.fits +# flatfile4 = d1026_0014.fits +# flatfile5 = d1026_0015.fits +# zerofile1 = d1026_0008.fits +# zerofile2 = d1026_0009.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2100 0.011826 2.5369 0.024796 + 2 1 1 B 1.2397 0.021519 2.5992 0.045119 + 3 4 2 A 1.2029 0.016268 2.5222 0.034109 + 4 3 2 B 1.2011 0.013787 2.5184 0.028908 + 5 6 3 A 1.1895 0.019960 2.4939 0.041850 + 6 5 3 B 1.2840 0.017495 2.6921 0.036681 + 7 8 4 A 1.2334 0.013424 2.5861 0.028146 + 8 7 4 B 1.2275 0.017398 2.5737 0.036478 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Apr-12.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Apr-12.log new file mode 100644 index 0000000000..2c6f8bc0cd --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Apr-12.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2012-Apr-12.log +# datestr = Thu Apr 12 17:05:56 2012 +# mode = Direct +# flatfile1 = d0413_0010.fits +# flatfile2 = d0413_0011.fits +# flatfile3 = d0413_0012.fits +# flatfile4 = d0413_0013.fits +# flatfile5 = d0413_0014.fits +# zerofile1 = d0413_0007.fits +# zerofile2 = d0413_0008.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 0.0015744 0.10255 0.0033011 0.21502 + 2 1 1 B 1.2924 0.025125 2.7098 0.052679 + 3 4 2 A -0.49111 0.072159 -1.0297 0.15129 + 4 3 2 B -0.52790 0.073904 -1.1068 0.15495 + 5 6 3 A -0.55877 0.11742 -1.1716 0.24618 + 6 5 3 B -0.67065 0.12005 -1.4061 0.25171 + 7 8 4 A 1.2825 0.039860 2.6891 0.083574 + 8 7 4 B 0.011546 0.095399 0.024209 0.20002 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Jun-08.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Jun-08.log new file mode 100644 index 0000000000..63951d04c7 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Jun-08.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2012-Jun-08.log +# datestr = Fri Jun 8 19:18:21 2012 +# mode = Direct +# flatfile1 = d0609_0004.fits +# flatfile2 = d0609_0005.fits +# flatfile3 = d0609_0006.fits +# flatfile4 = d0609_0007.fits +# flatfile5 = d0609_0008.fits +# zerofile1 = d0609_0001.fits +# zerofile2 = d0609_0002.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2753 0.020584 2.6739 0.043158 + 2 1 1 B 1.2887 0.018639 2.7019 0.039080 + 3 4 2 A 0.011702 0.044785 0.024535 0.093901 + 4 3 2 B -0.017178 0.033837 -0.036017 0.070946 + 5 6 3 A -0.015898 0.046156 -0.033332 0.096775 + 6 5 3 B -0.064485 0.052428 -0.13520 0.10993 + 7 8 4 A 1.2819 0.015164 2.6877 0.031793 + 8 7 4 B 1.2765 1.0085 2.6765 2.1145 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Mar-14.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Mar-14.log new file mode 100644 index 0000000000..c2e01638fe --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Mar-14.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2012-Mar-14.log +# datestr = Wed Mar 14 18:33:02 2012 +# mode = Direct +# flatfile1 = d0315_0021.fits +# flatfile2 = d0315_0022.fits +# flatfile3 = d0315_0023.fits +# flatfile4 = d0315_0024.fits +# flatfile5 = d0315_0025.fits +# zerofile1 = d0315_0018.fits +# zerofile2 = d0315_0019.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2684 0.036559 2.6595 0.076653 + 2 1 1 B 1.2802 0.017248 2.6842 0.036163 + 3 4 2 A -0.0083733 0.033011 -0.017556 0.069214 + 4 3 2 B -0.029809 0.024391 -0.062501 0.051141 + 5 6 3 A -0.035762 0.037714 -0.074982 0.079074 + 6 5 3 B -0.077832 0.042671 -0.16319 0.089468 + 7 8 4 A 1.2809 0.015975 2.6856 0.033494 + 8 7 4 B -0.042963 0.18241 -0.090079 0.38246 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Nov-11.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Nov-11.log new file mode 100644 index 0000000000..ae60fe911b --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2012-Nov-11.log @@ -0,0 +1,26 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2012-Nov-11.log +# datestr = Sun Nov 11 08:45:03 2012 +# mode = Direct +# flatfile1 = d1112_0005.fits +# flatfile2 = d1112_0006.fits +# flatfile3 = d1112_0007.fits +# flatfile4 = d1112_0008.fits +# zerofile1 = d1112_0002.fits +# zerofile2 = d1112_0003.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2237 0.014181 2.5658 0.029733 + 2 1 1 B 1.2329 0.010559 2.5850 0.022139 + 3 4 2 A 1.2081 0.014732 2.5330 0.030888 + 4 3 2 B 1.1995 0.013442 2.5150 0.028185 + 5 6 3 A 1.1934 0.016256 2.5021 0.034084 + 6 5 3 B 1.2831 0.014538 2.6903 0.030481 + 7 8 4 A 1.2268 0.011326 2.5723 0.023747 + 8 7 4 B 1.2297 0.018440 2.5784 0.038663 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Aug-26.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Aug-26.log new file mode 100644 index 0000000000..acc17c7c2a --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Aug-26.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2013-Aug-26.log +# datestr = Mon Aug 26 14:04:49 2013 +# mode = Direct +# flatfile1 = d0827_0016.fits +# flatfile2 = d0827_0017.fits +# flatfile3 = d0827_0018.fits +# flatfile4 = d0827_0019.fits +# flatfile5 = d0827_0020.fits +# zerofile1 = d0827_0013.fits +# zerofile2 = d0827_0014.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2209 0.012625 2.5599 0.026470 + 2 1 1 B 1.2344 0.018013 2.5882 0.037767 + 3 4 2 A 1.2075 0.023451 2.5318 0.049169 + 4 3 2 B 1.1921 0.013585 2.4994 0.028484 + 5 6 3 A 1.1868 0.013606 2.4883 0.028528 + 6 5 3 B 1.2842 0.015307 2.6926 0.032095 + 7 8 4 A 1.2260 0.015683 2.5706 0.032883 + 8 7 4 B 1.2259 0.013076 2.5703 0.027417 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Dec-22.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Dec-22.log new file mode 100644 index 0000000000..74ba15646e --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Dec-22.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2013-Dec-22.log +# datestr = Sun Dec 22 22:06:59 2013 +# mode = Direct +# flatfile1 = d1223_0021.fits +# flatfile2 = d1223_0022.fits +# flatfile3 = d1223_0023.fits +# flatfile4 = d1223_0024.fits +# flatfile5 = d1223_0025.fits +# zerofile1 = d1223_0018.fits +# zerofile2 = d1223_0019.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2124 0.010797 2.5420 0.022638 + 2 1 1 B 1.2309 0.010179 2.5809 0.021342 + 3 4 2 A 1.1983 0.013589 2.5125 0.028491 + 4 3 2 B 1.1886 0.013343 2.4921 0.027976 + 5 6 3 A 1.1883 0.020742 2.4916 0.043490 + 6 5 3 B 1.2811 0.024592 2.6861 0.051562 + 7 8 4 A 1.2208 0.014893 2.5597 0.031225 + 8 7 4 B 1.2209 0.016619 2.5599 0.034846 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Feb-04.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Feb-04.log new file mode 100644 index 0000000000..096849e094 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Feb-04.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2013-Feb-04.log +# datestr = Mon Feb 4 16:52:46 2013 +# mode = Direct +# flatfile1 = d0205_0020.fits +# flatfile2 = d0205_0021.fits +# flatfile3 = d0205_0022.fits +# flatfile4 = d0205_0023.fits +# flatfile5 = d0205_0024.fits +# zerofile1 = d0205_0017.fits +# zerofile2 = d0205_0018.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2120 0.017747 2.5412 0.037209 + 2 1 1 B 1.2377 0.0076416 2.5951 0.016022 + 3 4 2 A 1.2042 0.018348 2.5249 0.038470 + 4 3 2 B 1.1985 0.014051 2.5129 0.029461 + 5 6 3 A 1.1873 0.0083574 2.4893 0.017523 + 6 5 3 B 1.2953 0.021348 2.7158 0.044761 + 7 8 4 A 1.2288 0.016836 2.5765 0.035299 + 8 7 4 B 1.2299 0.019779 2.5788 0.041470 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Oct-28.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Oct-28.log new file mode 100644 index 0000000000..702beadbe5 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Oct-28.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2013-Oct-28.log +# datestr = Mon Oct 28 10:30:38 2013 +# mode = Direct +# flatfile1 = d1029_0028.fits +# flatfile2 = d1029_0029.fits +# flatfile3 = d1029_0030.fits +# flatfile4 = d1029_0031.fits +# flatfile5 = d1029_0032.fits +# zerofile1 = d1029_0025.fits +# zerofile2 = d1029_0026.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2116 0.018703 2.5403 0.039215 + 2 1 1 B 1.2291 0.011658 2.5771 0.024444 + 3 4 2 A 1.1976 0.020518 2.5109 0.043019 + 4 3 2 B 1.1872 0.0095606 2.4891 0.020046 + 5 6 3 A 1.1863 0.010872 2.4872 0.022794 + 6 5 3 B 1.2785 0.017024 2.6807 0.035694 + 7 8 4 A 1.2288 0.018051 2.5763 0.037848 + 8 7 4 B 1.2298 0.010007 2.5785 0.020982 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Sep-26.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Sep-26.log new file mode 100644 index 0000000000..504a3c25d1 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2013-Sep-26.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2013-Sep-26.log +# datestr = Thu Sep 26 17:13:18 2013 +# mode = Direct +# flatfile1 = d0927_0011.fits +# flatfile2 = d0927_0012.fits +# flatfile3 = d0927_0013.fits +# flatfile4 = d0927_0014.fits +# flatfile5 = d0927_0015.fits +# zerofile1 = d0927_0008.fits +# zerofile2 = d0927_0009.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2159 0.010762 2.5494 0.022563 + 2 1 1 B 1.2344 0.0091386 2.5882 0.019160 + 3 4 2 A 1.1994 0.017440 2.5147 0.036565 + 4 3 2 B 1.1972 0.012260 2.5101 0.025705 + 5 6 3 A 1.1881 0.019040 2.4911 0.039920 + 6 5 3 B 1.2892 0.014831 2.7030 0.031096 + 7 8 4 A 1.2381 0.016243 2.5960 0.034057 + 8 7 4 B 1.2314 0.010438 2.5818 0.021886 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2014-Oct-16.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2014-Oct-16.log new file mode 100644 index 0000000000..3082731279 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2014-Oct-16.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2014-Oct-16.log +# datestr = Thu Oct 16 09:14:41 2014 +# mode = Direct +# flatfile1 = d1002_0110.fits +# flatfile2 = d1002_0111.fits +# flatfile3 = d1002_0112.fits +# flatfile4 = d1002_0113.fits +# flatfile5 = d1002_0114.fits +# zerofile1 = d1002_0107.fits +# zerofile2 = d1002_0108.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2113 0.021651 2.5398 0.045396 + 2 1 1 B 1.2293 0.011356 2.5775 0.023809 + 3 4 2 A 1.2018 0.018134 2.5197 0.038021 + 4 3 2 B 1.1962 0.018497 2.5080 0.038781 + 5 6 3 A 0.38248 0.078593 0.0000 0.0000 + 6 5 3 B 1.2907 0.012124 2.7063 0.025420 + 7 8 4 A 1.2242 0.020894 2.5668 0.043808 + 8 7 4 B 1.2264 0.014217 2.5713 0.029809 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Apr-10.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Apr-10.log new file mode 100644 index 0000000000..395ec320fd --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Apr-10.log @@ -0,0 +1,28 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2015-Apr-10.log +# datestr = Fri Apr 10 17:17:57 2015 +# mode = Direct +# flatfile1 = d0411_0019.fits +# flatfile2 = d0411_0024.fits +# flatfile3 = d0411_0025.fits +# flatfile4 = d0411_0026.fits +# flatfile5 = d0411_0027.fits +# flatfile6 = d0411_0028.fits +# zerofile1 = d0411_0016.fits +# zerofile2 = d0411_0017.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2088 0.023120 2.5345 0.048474 + 2 1 1 B 1.2237 0.010551 2.5657 0.022122 + 3 4 2 A 1.1874 0.015721 2.4897 0.032962 + 4 3 2 B 1.1833 0.012724 2.4811 0.026678 + 5 6 3 A 6.6175 3.3380 0.0000 0.0000 + 6 5 3 B 1.2717 0.017433 2.6663 0.036551 + 7 8 4 A 1.2306 0.019980 2.5801 0.041891 + 8 7 4 B 1.2173 0.015029 2.5523 0.031512 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Dec-01.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Dec-01.log new file mode 100644 index 0000000000..827ec1df34 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Dec-01.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2015-Dec-01.log +# datestr = Tue Dec 1 13:13:02 2015 +# mode = Direct +# flatfile1 = d1202_0011.fits +# flatfile2 = d1202_0012.fits +# flatfile3 = d1202_0013.fits +# flatfile4 = d1202_0014.fits +# flatfile5 = d1202_0015.fits +# zerofile1 = d1202_0008.fits +# zerofile2 = d1202_0009.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2029 0.019667 2.5220 0.041235 + 2 1 1 B 1.2224 0.018197 2.5630 0.038153 + 3 4 2 A 1.1753 0.019515 2.4641 0.040917 + 4 3 2 B 1.1812 0.016200 2.4765 0.033966 + 5 6 3 A 1.1378 0.020328 2.3856 0.042621 + 6 5 3 B 1.2443 0.024999 2.6089 0.052415 + 7 8 4 A 1.3916 0.011417 4.3765 0.035906 + 8 7 4 B 1.2168 0.010457 2.5512 0.021926 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Dec-30.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Dec-30.log new file mode 100644 index 0000000000..131911fd1a --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Dec-30.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2015-Dec-30.log +# datestr = Wed Dec 30 12:16:34 2015 +# mode = Direct +# flatfile1 = d1231_0012.fits +# flatfile2 = d1231_0013.fits +# flatfile3 = d1231_0014.fits +# flatfile4 = d1231_0015.fits +# flatfile5 = d1231_0016.fits +# zerofile1 = d1231_0009.fits +# zerofile2 = d1231_0010.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2026 0.013645 2.5215 0.028609 + 2 1 1 B 1.2266 0.016573 2.5717 0.034748 + 3 4 2 A 1.1678 0.011537 2.4486 0.024190 + 4 3 2 B 1.1840 0.015072 2.4825 0.031601 + 5 6 3 A 1.1448 0.014461 2.4003 0.030319 + 6 5 3 B 1.2446 0.016256 2.6095 0.034084 + 7 8 4 A 1.2200 0.012824 2.5580 0.026888 + 8 7 4 B 1.2223 0.010710 2.5627 0.022455 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Sep-02.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Sep-02.log new file mode 100644 index 0000000000..227f036f95 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Sep-02.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2015-Sep-02.log +# datestr = Wed Sep 2 10:13:48 2015 +# mode = Direct +# flatfile1 = d0903_0012.fits +# flatfile2 = d0903_0013.fits +# flatfile3 = d0903_0014.fits +# flatfile4 = d0903_0015.fits +# flatfile5 = d0903_0016.fits +# zerofile1 = d0903_0009.fits +# zerofile2 = d0903_0010.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.1941 0.024805 2.5037 0.052008 + 2 1 1 B 1.2292 0.015682 2.5771 0.032881 + 3 4 2 A 1.1656 0.028640 2.4439 0.060050 + 4 3 2 B 1.1856 0.010798 2.4859 0.022639 + 5 6 3 A 1.1427 0.010662 2.3959 0.022356 + 6 5 3 B 1.2428 0.023203 2.6057 0.048650 + 7 8 4 A 1.2297 0.018093 2.5783 0.037935 + 8 7 4 B 1.2209 0.016638 2.5598 0.034885 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Sep-30.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Sep-30.log new file mode 100644 index 0000000000..ddb0122bcd --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2015-Sep-30.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2015-Sep-30.log +# datestr = Wed Sep 30 14:46:52 2015 +# mode = Direct +# flatfile1 = d0929_0026.fits +# flatfile2 = d0929_0027.fits +# flatfile3 = d0929_0028.fits +# flatfile4 = d0929_0029.fits +# flatfile5 = d0929_0030.fits +# zerofile1 = d0929_0023.fits +# zerofile2 = d0929_0024.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2129 0.016825 2.5431 0.035277 + 2 1 1 B 1.2280 0.016373 2.5747 0.034329 + 3 4 2 A 1.1693 0.022456 2.4516 0.047082 + 4 3 2 B 1.1905 0.013078 2.4962 0.027420 + 5 6 3 A 1.1396 0.018878 2.3893 0.039581 + 6 5 3 B 1.2513 0.015501 2.6235 0.032501 + 7 8 4 A 1.2288 0.0099079 2.5764 0.020774 + 8 7 4 B 1.2205 0.013599 2.5591 0.028512 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2016-Dec-17.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2016-Dec-17.log new file mode 100644 index 0000000000..c9a7aa9b37 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2016-Dec-17.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2016-Dec-17.log +# datestr = Sat Dec 17 12:43:30 2016 +# mode = Direct +# flatfile1 = d1218_0005.fits +# flatfile2 = d1218_0006.fits +# flatfile3 = d1218_0007.fits +# flatfile4 = d1218_0008.fits +# flatfile5 = d1218_0009.fits +# zerofile1 = d1218_0002.fits +# zerofile2 = d1218_0003.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.1966 0.017150 2.5090 0.035959 + 2 1 1 B 1.2216 0.014696 2.5613 0.030814 + 3 4 2 A 1.1670 0.019917 2.4468 0.041759 + 4 3 2 B 1.1852 0.013697 2.4851 0.028719 + 5 6 3 A 1.1583 0.018198 2.4287 0.038155 + 6 5 3 B 1.2700 0.012915 2.6629 0.027079 + 7 8 4 A 1.2209 0.014479 2.5597 0.030357 + 8 7 4 B 1.2202 0.0095583 2.5583 0.020041 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2016-May-03.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2016-May-03.log new file mode 100644 index 0000000000..afaaf5973e --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2016-May-03.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2016-May-03.log +# datestr = Tue May 3 11:48:18 2016 +# mode = Direct +# flatfile1 = d0504_0011.fits +# flatfile2 = d0504_0012.fits +# flatfile3 = d0504_0013.fits +# flatfile4 = d0504_0014.fits +# flatfile5 = d0504_0015.fits +# zerofile1 = d0504_0008.fits +# zerofile2 = d0504_0009.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2062 0.017440 2.5290 0.036566 + 2 1 1 B 1.2305 0.016079 2.5800 0.033713 + 3 4 2 A 1.1762 0.015266 2.4661 0.032007 + 4 3 2 B 1.1904 0.015536 2.4959 0.032573 + 5 6 3 A 1.1435 0.019940 2.3976 0.041808 + 6 5 3 B 1.2439 0.025989 2.6081 0.054490 + 7 8 4 A 1.2267 0.011221 2.5720 0.023528 + 8 7 4 B 1.2226 0.016266 2.5634 0.034104 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2016-Nov-22.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2016-Nov-22.log new file mode 100644 index 0000000000..75fe5a8b51 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2016-Nov-22.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2016-Nov-22.log +# datestr = Tue Nov 22 16:05:21 2016 +# mode = Direct +# flatfile1 = d1123_0012.fits +# flatfile2 = d1123_0013.fits +# flatfile3 = d1123_0014.fits +# flatfile4 = d1123_0015.fits +# flatfile5 = d1123_0016.fits +# zerofile1 = d1123_0009.fits +# zerofile2 = d1123_0010.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2037 0.023999 2.5237 0.050319 + 2 1 1 B 1.2204 0.014640 2.5587 0.030696 + 3 4 2 A 1.1761 0.024087 2.4660 0.050502 + 4 3 2 B 1.1876 0.015070 2.4900 0.031597 + 5 6 3 A 1.1586 0.018038 2.4292 0.037819 + 6 5 3 B 1.2594 0.021622 2.6406 0.045335 + 7 8 4 A 1.2283 0.014526 2.5753 0.030456 + 8 7 4 B 1.2185 0.020491 2.5548 0.042963 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2017-Dec-10.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2017-Dec-10.log new file mode 100644 index 0000000000..bd92eb2d8e --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2017-Dec-10.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2017-Dec-10.log +# datestr = Sun Dec 10 12:33:50 2017 +# mode = Direct +# flatfile1 = d1211_0050.fits +# flatfile2 = d1211_0051.fits +# flatfile3 = d1211_0052.fits +# flatfile4 = d1211_0053.fits +# flatfile5 = d1211_0054.fits +# zerofile1 = d1211_0047.fits +# zerofile2 = d1211_0048.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2062 0.015234 2.5291 0.031940 + 2 1 1 B 1.2210 0.016928 2.5601 0.035493 + 3 4 2 A 8.1527 3.4832 -0.0000 0.0000 + 4 3 2 B 1.1877 0.013289 2.4902 0.027862 + 5 6 3 A 1.1668 0.010371 2.4465 0.021744 + 6 5 3 B 1.2499 0.017126 2.6206 0.035907 + 7 8 4 A 1.2169 0.014226 2.5514 0.029827 + 8 7 4 B 1.2276 0.014758 2.5739 0.030942 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2017-May-16.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2017-May-16.log new file mode 100644 index 0000000000..43f98b221f --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2017-May-16.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2017-May-16.log +# datestr = Tue May 16 14:17:38 2017 +# mode = Direct +# flatfile1 = d0517_0032.fits +# flatfile2 = d0517_0033.fits +# flatfile3 = d0517_0034.fits +# flatfile4 = d0517_0035.fits +# flatfile5 = d0517_0036.fits +# zerofile1 = d0517_0029.fits +# zerofile2 = d0517_0030.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.1967 0.023154 2.5091 0.048547 + 2 1 1 B 1.2186 0.013135 2.5551 0.027541 + 3 4 2 A 1.1712 0.022452 2.4557 0.047075 + 4 3 2 B 1.1863 0.015390 2.4872 0.032268 + 5 6 3 A 1.1582 0.012982 2.4284 0.027220 + 6 5 3 B 1.2717 0.022158 2.6663 0.046459 + 7 8 4 A 1.2231 0.018996 2.5644 0.039827 + 8 7 4 B 1.2132 0.018976 2.5436 0.039787 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Apr-19.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Apr-19.log new file mode 100644 index 0000000000..5b53ac384f --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Apr-19.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2018-Apr-19.log +# datestr = Thu Apr 19 20:15:59 2018 +# mode = Direct +# flatfile1 = d0420_0017.fits +# flatfile2 = d0420_0018.fits +# flatfile3 = d0420_0019.fits +# flatfile4 = d0420_0020.fits +# flatfile5 = d0420_0021.fits +# zerofile1 = d0420_0014.fits +# zerofile2 = d0420_0015.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2087 0.015297 2.5342 0.032073 + 2 1 1 B 1.2272 0.014454 2.5731 0.030306 + 3 4 2 A 0.59913 0.39172 -0.0000 0.0000 + 4 3 2 B 1.1865 0.018092 2.4869 0.035813 + 5 6 3 A 1.1743 0.020004 2.4621 0.041942 + 6 5 3 B 1.2492 0.029381 2.6191 0.061603 + 7 8 4 A 1.2210 0.016258 2.5600 0.034089 + 8 7 4 B 1.2174 0.015866 2.5526 0.033266 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Aug-08.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Aug-08.log new file mode 100644 index 0000000000..9b3d7ac21a --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Aug-08.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2018-Aug-08.log +# datestr = Wed Aug 8 18:58:48 2018 +# mode = Direct +# flatfile1 = d0809_0033.fits +# flatfile2 = d0809_0034.fits +# flatfile3 = d0809_0035.fits +# flatfile4 = d0809_0036.fits +# flatfile5 = d0809_0037.fits +# zerofile1 = d0809_0030.fits +# zerofile2 = d0809_0031.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2084 0.020639 2.5336 0.043273 + 2 1 1 B 1.2260 0.020071 2.5706 0.042083 + 3 4 2 A 1.1712 0.022483 2.4556 0.047140 + 4 3 2 B 1.1896 0.013173 2.4942 0.027620 + 5 6 3 A 1.1629 0.013157 2.4383 0.027586 + 6 5 3 B 1.2542 0.019223 2.6296 0.040305 + 7 8 4 A 1.2256 0.014612 2.5696 0.030637 + 8 7 4 B 1.2261 0.012793 2.5707 0.026824 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Aug-17.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Aug-17.log new file mode 100644 index 0000000000..dd0d8a43c5 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Aug-17.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2018-Aug-17.log +# datestr = Fri Aug 17 13:45:31 2018 +# mode = Direct +# flatfile1 = d0818_0041.fits +# flatfile2 = d0818_0042.fits +# flatfile3 = d0818_0043.fits +# flatfile4 = d0818_0044.fits +# flatfile5 = d0818_0045.fits +# zerofile1 = d0818_0038.fits +# zerofile2 = d0818_0039.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2543 0.018228 2.6300 0.038218 + 2 1 1 B 1.2221 0.012989 2.5623 0.027235 + 3 4 2 A 1.1697 0.014521 2.4524 0.030445 + 4 3 2 B 1.1861 0.014467 2.4869 0.030332 + 5 6 3 A 1.1648 0.010588 2.4422 0.022199 + 6 5 3 B 1.2526 0.023916 2.6264 0.050145 + 7 8 4 A 1.2341 0.017459 2.5875 0.036606 + 8 7 4 B 1.2207 0.012766 2.5594 0.026766 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-May-17.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-May-17.log new file mode 100644 index 0000000000..f78c342af6 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-May-17.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2018-May-17.log +# datestr = Thu May 17 14:54:11 2018 +# mode = Direct +# flatfile1 = d0518_0024.fits +# flatfile2 = d0518_0025.fits +# flatfile3 = d0518_0026.fits +# flatfile4 = d0518_0027.fits +# flatfile5 = d0518_0028.fits +# zerofile1 = d0518_0021.fits +# zerofile2 = d0518_0022.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2080 0.019749 2.5328 0.041407 + 2 1 1 B 1.2269 0.017901 2.5724 0.037533 + 3 4 2 A 0.63905 0.42127 -0.0000 0.0000 + 4 3 2 B 1.1866 0.013973 2.4879 0.029297 + 5 6 3 A 1.1672 0.014287 2.4473 0.029956 + 6 5 3 B 1.2478 0.031201 2.6163 0.065418 + 7 8 4 A 1.2277 0.016168 2.5741 0.033898 + 8 7 4 B 1.2200 0.020422 2.5579 0.042818 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Nov-26.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Nov-26.log new file mode 100644 index 0000000000..ffaab01329 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2018-Nov-26.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2018-Nov-26.log +# datestr = Mon Nov 26 18:00:30 2018 +# mode = Direct +# flatfile1 = d1127_0014.fits +# flatfile2 = d1127_0015.fits +# flatfile3 = d1127_0016.fits +# flatfile4 = d1127_0017.fits +# flatfile5 = d1127_0018.fits +# zerofile1 = d1127_0011.fits +# zerofile2 = d1127_0012.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2735 0.011005 2.6702 0.023074 + 2 1 1 B 1.2007 0.014247 2.5175 0.029870 + 3 4 2 A 1.1747 0.018535 2.4630 0.038862 + 4 3 2 B 1.1866 0.012016 2.4879 0.025195 + 5 6 3 A 1.1681 0.014539 3.6401 0.17244 + 6 5 3 B 1.2536 0.019390 2.6284 0.040655 + 7 8 4 A 1.2299 0.010285 2.5788 0.021565 + 8 7 4 B 1.2175 0.014690 2.5526 0.030799 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2019-Dec-29.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2019-Dec-29.log new file mode 100644 index 0000000000..dcba891348 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2019-Dec-29.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2019-Dec-29.log +# datestr = Sun Dec 29 13:16:14 2019 +# mode = Direct +# flatfile1 = d1230_0012.fits +# flatfile2 = d1230_0013.fits +# flatfile3 = d1230_0014.fits +# flatfile4 = d1230_0015.fits +# flatfile5 = d1230_0016.fits +# zerofile1 = d1230_0009.fits +# zerofile2 = d1230_0010.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2648 0.020301 2.6519 0.042565 + 2 1 1 B 1.1952 0.016133 2.5059 0.033827 + 3 4 2 A 1.1621 0.014632 2.4366 0.030679 + 4 3 2 B 1.1835 0.016240 2.4814 0.034049 + 5 6 3 A 1.1513 0.014941 2.4141 0.037324 + 6 5 3 B 1.2599 0.024316 2.6417 0.050982 + 7 8 4 A 1.2474 0.012977 3.9171 0.056409 + 8 7 4 B 1.2192 0.018556 2.5564 0.038906 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2019-Nov-19.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2019-Nov-19.log new file mode 100644 index 0000000000..441a7f3804 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2019-Nov-19.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2019-Nov-19.log +# datestr = Tue Nov 19 08:15:56 2019 +# mode = Direct +# flatfile1 = d1119_0004.fits +# flatfile2 = d1119_0005.fits +# flatfile3 = d1119_0006.fits +# flatfile4 = d1119_0007.fits +# flatfile5 = d1119_0008.fits +# zerofile1 = d1119_0001.fits +# zerofile2 = d1119_0002.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2611 0.020239 2.6442 0.042434 + 2 1 1 B 1.1931 0.014274 2.5017 0.029927 + 3 4 2 A 1.1634 0.016668 2.4392 0.034946 + 4 3 2 B 1.1765 0.016175 2.4667 0.033913 + 5 6 3 A 1.1584 0.011443 2.4287 0.023991 + 6 5 3 B 1.2621 0.020207 2.6463 0.042368 + 7 8 4 A 1.2450 0.010563 2.6217 0.038713 + 8 7 4 B 1.2111 0.012566 2.5393 0.026347 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2019-Nov-27.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2019-Nov-27.log new file mode 100644 index 0000000000..33b82cac0c --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2019-Nov-27.log @@ -0,0 +1,27 @@ +# logfile = /home/dmoseng/data/gain/gain_check.Direct.2019-Nov-27.log +# datestr = Wed Nov 27 07:15:01 2019 +# mode = Direct +# flatfile1 = d1127_0025.fits +# flatfile2 = d1127_0026.fits +# flatfile3 = d1127_0027.fits +# flatfile4 = d1127_0028.fits +# flatfile5 = d1127_0029.fits +# zerofile1 = d1127_0022.fits +# zerofile2 = d1127_0023.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2673 0.015794 2.6571 0.033116 + 2 1 1 B 1.1956 0.014366 2.5068 0.030121 + 3 4 2 A 1.1723 0.015917 2.4580 0.033373 + 4 3 2 B 1.1809 0.017559 2.4759 0.036815 + 5 6 3 A 1.1549 0.012052 2.4215 0.025268 + 6 5 3 B 1.2643 0.018608 2.6508 0.039015 + 7 8 4 A 1.2407 0.011724 2.6469 0.10091 + 8 7 4 B 1.2115 0.013294 2.5401 0.027873 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-Jun-21.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-Jun-21.log new file mode 100644 index 0000000000..93a1440e3e --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-Jun-21.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2021-Jun-21.log +# datestr = Mon Jun 21 15:43:33 2021 +# mode = Direct +# flatfile1 = /sdata1005/deimos5/2021jun22/d0622_0020.fits +# flatfile2 = /sdata1005/deimos5/2021jun22/d0622_0021.fits +# flatfile3 = /sdata1005/deimos5/2021jun22/d0622_0022.fits +# flatfile4 = /sdata1005/deimos5/2021jun22/d0622_0023.fits +# flatfile5 = /sdata1005/deimos5/2021jun22/d0622_0024.fits +# zerofile1 = /sdata1005/deimos5/2021jun22/d0622_0017.fits +# zerofile2 = /sdata1005/deimos5/2021jun22/d0622_0018.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2320 0.017068 2.5831 0.035785 + 2 1 1 B 1.1796 0.016234 2.4732 0.034038 + 3 4 2 A 1.7137 0.027804 1.7966 0.029148 + 4 3 2 B 1.1728 0.012499 2.4589 0.026207 + 5 6 3 A 1.1609 0.013320 2.4341 0.027928 + 6 5 3 B 1.2614 0.020335 2.6447 0.042635 + 7 8 4 A 1.2460 0.013255 3.9177 0.040593 + 8 7 4 B 1.2156 0.011986 2.5486 0.025132 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-May-04.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-May-04.log new file mode 100644 index 0000000000..a4cdebb01f --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-May-04.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2021-May-04.log +# datestr = Tue May 4 23:23:33 2021 +# mode = Direct +# flatfile1 = /sdata1005/deimos10/2021may05/d0505_0026.fits +# flatfile2 = /sdata1005/deimos10/2021may05/d0505_0027.fits +# flatfile3 = /sdata1005/deimos10/2021may05/d0505_0028.fits +# flatfile4 = /sdata1005/deimos10/2021may05/d0505_0029.fits +# flatfile5 = /sdata1005/deimos10/2021may05/d0505_0030.fits +# zerofile1 = /sdata1005/deimos10/2021may05/d0505_0023.fits +# zerofile2 = /sdata1005/deimos10/2021may05/d0505_0024.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2345 0.016751 2.5885 0.035122 + 2 1 1 B 1.1730 0.012694 2.4594 0.026616 + 3 4 2 A 1.7085 0.033074 1.7911 0.034673 + 4 3 2 B 1.1736 0.012521 2.4606 0.026252 + 5 6 3 A 1.1542 0.016727 2.4201 0.035072 + 6 5 3 B 1.2689 0.017654 2.6605 0.037016 + 7 8 4 A 1.2468 0.013576 2.6293 0.038898 + 8 7 4 B 1.2085 0.016786 2.5338 0.035195 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-May-05.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-May-05.log new file mode 100644 index 0000000000..b10516f3f2 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-May-05.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2021-May-05.log +# datestr = Wed May 5 13:36:39 2021 +# mode = Direct +# flatfile1 = /sdata1005/deimos6/2021may06/d0506_0022.fits +# flatfile2 = /sdata1005/deimos6/2021may06/d0506_0023.fits +# flatfile3 = /sdata1005/deimos6/2021may06/d0506_0024.fits +# flatfile4 = /sdata1005/deimos6/2021may06/d0506_0025.fits +# flatfile5 = /sdata1005/deimos6/2021may06/d0506_0026.fits +# zerofile1 = /sdata1005/deimos6/2021may06/d0506_0019.fits +# zerofile2 = /sdata1005/deimos6/2021may06/d0506_0020.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2299 0.019747 2.5787 0.041404 + 2 1 1 B 1.1742 0.017435 2.4619 0.036555 + 3 4 2 A 1.7006 0.033702 1.7828 0.035331 + 4 3 2 B 1.1712 0.013545 2.4556 0.028400 + 5 6 3 A 1.1579 0.017180 2.4277 0.036021 + 6 5 3 B 1.2627 0.017114 2.6475 0.035882 + 7 8 4 A 1.2534 0.013603 3.9364 0.055853 + 8 7 4 B 1.2141 0.014372 2.5455 0.030135 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-May-31.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-May-31.log new file mode 100644 index 0000000000..093551d80e --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-May-31.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2021-May-31.log +# datestr = Mon May 31 17:24:42 2021 +# mode = Direct +# flatfile1 = /sdata1005/deimos6/2021jun01/d0601_0034.fits +# flatfile2 = /sdata1005/deimos6/2021jun01/d0601_0035.fits +# flatfile3 = /sdata1005/deimos6/2021jun01/d0601_0036.fits +# flatfile4 = /sdata1005/deimos6/2021jun01/d0601_0037.fits +# flatfile5 = /sdata1005/deimos6/2021jun01/d0601_0038.fits +# zerofile1 = /sdata1005/deimos6/2021jun01/d0601_0031.fits +# zerofile2 = /sdata1005/deimos6/2021jun01/d0601_0032.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2284 0.020443 2.5755 0.042862 + 2 1 1 B 1.1759 0.014111 2.4654 0.029586 + 3 4 2 A 1.7021 0.033337 1.7843 0.034948 + 4 3 2 B 1.1763 0.014020 2.4663 0.029395 + 5 6 3 A 1.1623 0.014713 2.4373 0.034536 + 6 5 3 B 1.2711 0.017036 2.6651 0.035719 + 7 8 4 A 1.2472 0.016559 3.9223 0.052079 + 8 7 4 B 1.2208 0.0096946 2.5597 0.020327 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-Sep-05.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-Sep-05.log new file mode 100644 index 0000000000..66fd4a03d2 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2021-Sep-05.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2021-Sep-05.log +# datestr = Sun Sep 5 21:52:11 2021 +# mode = Direct +# flatfile1 = /sdata1004/deimos5/2021sep06/d0906_0023.fits +# flatfile2 = /sdata1004/deimos5/2021sep06/d0906_0024.fits +# flatfile3 = /sdata1004/deimos5/2021sep06/d0906_0025.fits +# flatfile4 = /sdata1004/deimos5/2021sep06/d0906_0026.fits +# flatfile5 = /sdata1004/deimos5/2021sep06/d0906_0027.fits +# zerofile1 = /sdata1004/deimos5/2021sep06/d0906_0020.fits +# zerofile2 = /sdata1004/deimos5/2021sep06/d0906_0021.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2314 0.020328 2.5818 0.042621 + 2 1 1 B 1.1796 0.017172 2.4732 0.036005 + 3 4 2 A 1.6996 0.030561 1.7818 0.032039 + 4 3 2 B 1.1707 0.015098 2.4545 0.031656 + 5 6 3 A 1.1663 0.015360 2.4454 0.032204 + 6 5 3 B 1.2680 0.014573 2.6587 0.030556 + 7 8 4 A 1.2415 0.015028 3.9047 0.047264 + 8 7 4 B 1.2145 0.013076 2.5463 0.027416 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Apr-22.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Apr-22.log new file mode 100644 index 0000000000..a8f2a404ec --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Apr-22.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2022-Apr-22.log +# datestr = Fri Apr 22 12:02:48 2022 +# mode = Direct +# flatfile1 = /sdata1005/deimos2/2022apr23/d0423_0007.fits +# flatfile2 = /sdata1005/deimos2/2022apr23/d0423_0008.fits +# flatfile3 = /sdata1005/deimos2/2022apr23/d0423_0009.fits +# flatfile4 = /sdata1005/deimos2/2022apr23/d0423_0010.fits +# flatfile5 = /sdata1005/deimos2/2022apr23/d0423_0011.fits +# zerofile1 = /sdata1005/deimos2/2022apr23/d0423_0004.fits +# zerofile2 = /sdata1005/deimos2/2022apr23/d0423_0005.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2296 0.015107 2.5781 0.031676 + 2 1 1 B 1.1701 0.015093 2.4533 0.031646 + 3 4 2 A 1.6979 0.036449 1.7799 0.038211 + 4 3 2 B 1.1699 0.016766 2.4530 0.035153 + 5 6 3 A 1.1625 0.011392 2.4373 0.023885 + 6 5 3 B 1.2509 0.020297 2.6227 0.042556 + 7 8 4 A 1.2287 0.016077 2.5762 0.033707 + 8 7 4 B 1.2257 0.012202 2.5700 0.025585 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Jan-28.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Jan-28.log new file mode 100644 index 0000000000..04b488d1ff --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Jan-28.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2022-Jan-28.log +# datestr = Fri Jan 28 11:59:35 2022 +# mode = Direct +# flatfile1 = /sdata1005/dmoseng/2022jan29/d0129_0020.fits +# flatfile2 = /sdata1005/dmoseng/2022jan29/d0129_0021.fits +# flatfile3 = /sdata1005/dmoseng/2022jan29/d0129_0022.fits +# flatfile4 = /sdata1005/dmoseng/2022jan29/d0129_0023.fits +# flatfile5 = /sdata1005/dmoseng/2022jan29/d0129_0024.fits +# zerofile1 = /sdata1005/dmoseng/2022jan29/d0129_0017.fits +# zerofile2 = /sdata1005/dmoseng/2022jan29/d0129_0018.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2300 0.014013 2.5790 0.029381 + 2 1 1 B 1.1732 0.013404 2.4599 0.028103 + 3 4 2 A 1.6990 0.025338 1.7811 0.026563 + 4 3 2 B 1.1722 0.0099498 2.4577 0.020861 + 5 6 3 A 1.1477 0.010857 2.4063 0.022763 + 6 5 3 B 1.2589 0.021481 2.6395 0.045038 + 7 8 4 A 1.2485 0.013253 3.9267 0.041681 + 8 7 4 B 1.2103 0.012897 2.5375 0.027041 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Jun-01.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Jun-01.log new file mode 100644 index 0000000000..53b6ff9077 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Jun-01.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2022-Jun-01.log +# datestr = Wed Jun 1 23:21:27 2022 +# mode = Direct +# flatfile1 = /sdata1005/deimos3/2022jun02/d0602_0019.fits +# flatfile2 = /sdata1005/deimos3/2022jun02/d0602_0020.fits +# flatfile3 = /sdata1005/deimos3/2022jun02/d0602_0021.fits +# flatfile4 = /sdata1005/deimos3/2022jun02/d0602_0022.fits +# flatfile5 = /sdata1005/deimos3/2022jun02/d0602_0023.fits +# zerofile1 = /sdata1005/deimos3/2022jun02/d0602_0017.fits +# zerofile2 = /sdata1005/deimos3/2022jun02/d0602_0018.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2344 0.022140 2.5882 0.046421 + 2 1 1 B 1.1725 0.023057 2.4584 0.048343 + 3 4 2 A 1.6891 0.032117 1.7708 0.033669 + 4 3 2 B 1.1657 0.016579 2.4442 0.034760 + 5 6 3 A 1.1581 0.0090475 2.4282 0.018970 + 6 5 3 B 1.2565 0.021344 2.6345 0.044751 + 7 8 4 A 1.2287 0.016826 2.5763 0.035278 + 8 7 4 B 1.2097 0.012957 2.5364 0.027168 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Jun-22.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Jun-22.log new file mode 100644 index 0000000000..2e80c06315 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Jun-22.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2022-Jun-22.log +# datestr = Wed Jun 22 18:43:19 2022 +# mode = Direct +# flatfile1 = /sdata1005/deimos4/2022jun23/d0623_0027.fits +# flatfile2 = /sdata1005/deimos4/2022jun23/d0623_0028.fits +# flatfile3 = /sdata1005/deimos4/2022jun23/d0623_0029.fits +# flatfile4 = /sdata1005/deimos4/2022jun23/d0623_0030.fits +# flatfile5 = /sdata1005/deimos4/2022jun23/d0623_0031.fits +# zerofile1 = /sdata1005/deimos4/2022jun23/d0623_0024.fits +# zerofile2 = /sdata1005/deimos4/2022jun23/d0623_0025.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2378 0.012998 2.5953 0.027252 + 2 1 1 B 1.1759 0.014031 2.4656 0.029418 + 3 4 2 A 1.7069 0.042328 1.7894 0.044375 + 4 3 2 B 1.1753 0.018892 2.4643 0.039611 + 5 6 3 A 1.1654 0.018260 2.4435 0.038287 + 6 5 3 B 1.2532 0.020820 2.6275 0.043652 + 7 8 4 A 1.2308 0.012811 2.5806 0.026860 + 8 7 4 B 1.2233 0.0088112 2.5648 0.018474 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Nov-14.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Nov-14.log new file mode 100644 index 0000000000..32782d4831 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Nov-14.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2022-Nov-14.log +# datestr = Mon Nov 14 00:21:39 2022 +# mode = Direct +# flatfile1 = /sdata1004/dmoseng/2022nov14/d1114_0005.fits +# flatfile2 = /sdata1004/dmoseng/2022nov14/d1114_0006.fits +# flatfile3 = /sdata1004/dmoseng/2022nov14/d1114_0007.fits +# flatfile4 = /sdata1004/dmoseng/2022nov14/d1114_0008.fits +# flatfile5 = /sdata1004/dmoseng/2022nov14/d1114_0009.fits +# zerofile1 = /sdata1004/dmoseng/2022nov14/d1114_0002.fits +# zerofile2 = /sdata1004/dmoseng/2022nov14/d1114_0003.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2350 0.019341 2.5894 0.040552 + 2 1 1 B 1.1713 0.013260 2.4559 0.027802 + 3 4 2 A 1.7019 0.038561 1.7842 0.040425 + 4 3 2 B 1.1708 0.014360 2.4548 0.030109 + 5 6 3 A 1.1575 0.014873 2.4269 0.031185 + 6 5 3 B 1.2581 0.025157 2.6379 0.052747 + 7 8 4 A 1.2304 0.018116 2.5798 0.037984 + 8 7 4 B 1.2196 0.012423 2.5572 0.026048 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Oct-11.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Oct-11.log new file mode 100644 index 0000000000..339034a145 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2022-Oct-11.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2022-Oct-11.log +# datestr = Tue Oct 11 15:00:05 2022 +# mode = Direct +# flatfile1 = /sdata1004/dmoseng/2022oct12/d1012_0011.fits +# flatfile2 = /sdata1004/dmoseng/2022oct12/d1012_0012.fits +# flatfile3 = /sdata1004/dmoseng/2022oct12/d1012_0013.fits +# flatfile4 = /sdata1004/dmoseng/2022oct12/d1012_0014.fits +# flatfile5 = /sdata1004/dmoseng/2022oct12/d1012_0015.fits +# zerofile1 = /sdata1004/dmoseng/2022oct12/d1012_0008.fits +# zerofile2 = /sdata1004/dmoseng/2022oct12/d1012_0009.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2304 0.022985 2.5797 0.048192 + 2 1 1 B 1.1774 0.014896 2.4687 0.031232 + 3 4 2 A 1.7045 0.027092 1.7869 0.028402 + 4 3 2 B 1.1813 0.016740 2.4767 0.035098 + 5 6 3 A 1.1603 0.018360 2.4328 0.038495 + 6 5 3 B 1.2589 0.023527 2.6395 0.049329 + 7 8 4 A 1.2265 0.015310 2.5716 0.032099 + 8 7 4 B 1.2167 0.014626 2.5510 0.030665 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Aug-06.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Aug-06.log new file mode 100644 index 0000000000..d956adbf97 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Aug-06.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2023-Aug-06.log +# datestr = Sun Aug 6 20:10:00 2023 +# mode = Direct +# flatfile1 = /sdata1005/dmoseng/2023aug07/d0728_0036.fits +# flatfile2 = /sdata1005/dmoseng/2023aug07/d0728_0037.fits +# flatfile3 = /sdata1005/dmoseng/2023aug07/d0728_0038.fits +# flatfile4 = /sdata1005/dmoseng/2023aug07/d0728_0039.fits +# flatfile5 = /sdata1005/dmoseng/2023aug07/d0728_0040.fits +# zerofile1 = /sdata1005/dmoseng/2023aug07/d0728_0033.fits +# zerofile2 = /sdata1005/dmoseng/2023aug07/d0728_0034.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2301 0.021649 2.5792 0.045392 + 2 1 1 B 1.1708 0.013904 2.4549 0.029152 + 3 4 2 A 1.6935 0.026653 1.7753 0.027941 + 4 3 2 B 1.1726 0.011941 2.4585 0.025037 + 5 6 3 A 1.1530 0.013522 2.4174 0.028350 + 6 5 3 B 1.2652 0.027525 2.6526 0.057711 + 7 8 4 A 1.2423 0.015752 3.9071 0.049542 + 8 7 4 B 1.2170 0.012133 2.5516 0.025440 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Dec-11.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Dec-11.log new file mode 100644 index 0000000000..f1f322d6e8 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Dec-11.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2023-Dec-11.log +# datestr = Mon Dec 11 01:23:00 2023 +# mode = Direct +# flatfile1 = /sdata1005/deimos2/2023dec11/d1211_0014.fits +# flatfile2 = /sdata1005/deimos2/2023dec11/d1211_0015.fits +# flatfile3 = /sdata1005/deimos2/2023dec11/d1211_0016.fits +# flatfile4 = /sdata1005/deimos2/2023dec11/d1211_0017.fits +# flatfile5 = /sdata1005/deimos2/2023dec11/d1211_0018.fits +# zerofile1 = /sdata1005/deimos2/2023dec11/d1211_0011.fits +# zerofile2 = /sdata1005/deimos2/2023dec11/d1211_0012.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2298 0.016408 2.5785 0.034402 + 2 1 1 B 1.1769 0.016055 2.4676 0.033662 + 3 4 2 A 1.7054 0.022554 1.7879 0.023644 + 4 3 2 B 1.1711 0.021072 2.4555 0.044182 + 5 6 3 A 1.1532 0.010453 2.4180 0.021917 + 6 5 3 B 1.2657 0.024084 3.7961 0.49716 + 7 8 4 A 1.2384 0.019650 3.8947 0.061800 + 8 7 4 B 1.2181 0.014519 2.5541 0.030443 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Jan-06.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Jan-06.log new file mode 100644 index 0000000000..2c8539a6ff --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Jan-06.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2023-Jan-06.log +# datestr = Fri Jan 6 22:27:30 2023 +# mode = Direct +# flatfile1 = /sdata1005/deimos2/2023jan07/d0107_0029.fits +# flatfile2 = /sdata1005/deimos2/2023jan07/d0107_0030.fits +# flatfile3 = /sdata1005/deimos2/2023jan07/d0107_0031.fits +# flatfile4 = /sdata1005/deimos2/2023jan07/d0107_0032.fits +# flatfile5 = /sdata1005/deimos2/2023jan07/d0107_0033.fits +# zerofile1 = /sdata1005/deimos2/2023jan07/d0107_0026.fits +# zerofile2 = /sdata1005/deimos2/2023jan07/d0107_0027.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2306 0.016749 2.5802 0.035117 + 2 1 1 B 1.1764 0.013286 2.4666 0.027856 + 3 4 2 A 1.7039 0.028774 1.7863 0.030164 + 4 3 2 B 1.1705 0.016283 2.4542 0.034140 + 5 6 3 A 1.1522 0.019050 2.4157 0.039942 + 6 5 3 B 1.2689 0.021091 2.6604 0.044221 + 7 8 4 A 1.2447 0.013178 3.9145 0.041445 + 8 7 4 B 1.2209 0.015194 2.5599 0.031858 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Oct-05.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Oct-05.log new file mode 100644 index 0000000000..bffa0eb30f --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampA/gain_check.Direct.2023-Oct-05.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Direct.2023-Oct-05.log +# datestr = Thu Oct 5 12:54:46 2023 +# mode = Direct +# flatfile1 = /sdata1004/dmoseng/2023oct05/d1005_0038.fits +# flatfile2 = /sdata1004/dmoseng/2023oct05/d1005_0039.fits +# flatfile3 = /sdata1004/dmoseng/2023oct05/d1005_0040.fits +# flatfile4 = /sdata1004/dmoseng/2023oct05/d1005_0041.fits +# flatfile5 = /sdata1004/dmoseng/2023oct05/d1005_0042.fits +# zerofile1 = /sdata1004/dmoseng/2023oct05/d1005_0035.fits +# zerofile2 = /sdata1004/dmoseng/2023oct05/d1005_0036.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 1 2 1 A 1.2279 0.015704 2.5745 0.032925 + 2 1 1 B 1.1784 0.014061 2.4708 0.029482 + 3 4 2 A 1.7079 0.036212 1.7904 0.037962 + 4 3 2 B 1.1721 0.012944 2.4575 0.027139 + 5 6 3 A 1.1637 0.0096046 2.4400 0.020138 + 6 5 3 B 1.2597 0.014713 2.6412 0.030848 + 7 8 4 A 1.2471 0.015236 3.9221 0.047917 + 8 7 4 B 1.2130 0.013148 2.5433 0.027566 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2008-Dec-17.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2008-Dec-17.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2008-Dec-17.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2008-Dec-17.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2008-Jul-09.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2008-Jul-09.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2008-Jul-09.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2008-Jul-09.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2008-Nov-19.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2008-Nov-19.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2008-Nov-19.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2008-Nov-19.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2009-Apr-16.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2009-Apr-16.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2009-Apr-16.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2009-Apr-16.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2009-Dec-08.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2009-Dec-08.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2009-Dec-08.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2009-Dec-08.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2009-Jun-18.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2009-Jun-18.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2009-Jun-18.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2009-Jun-18.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2009-Mar-17.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2009-Mar-17.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2009-Mar-17.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2009-Mar-17.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2009-Sep-30.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2009-Sep-30.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2009-Sep-30.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2009-Sep-30.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2010-Feb-02.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2010-Feb-02.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2010-Feb-02.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2010-Feb-02.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2010-Jan-07.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2010-Jan-07.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2010-Jan-07.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2010-Jan-07.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2010-Jun-06.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2010-Jun-06.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2010-Jun-06.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2010-Jun-06.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2010-Mar-02.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2010-Mar-02.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2010-Mar-02.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2010-Mar-02.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2010-May-05.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2010-May-05.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2010-May-05.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2010-May-05.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2010-Sep-01.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2010-Sep-01.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2010-Sep-01.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2010-Sep-01.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2011-Feb-26.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2011-Feb-26.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2011-Feb-26.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2011-Feb-26.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2011-Mar-22.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2011-Mar-22.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2011-Mar-22.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2011-Mar-22.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2011-May-25.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2011-May-25.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2011-May-25.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2011-May-25.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2011-Oct-25.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2011-Oct-25.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2011-Oct-25.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2011-Oct-25.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2012-Nov-11.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2012-Nov-11.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2012-Nov-11.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2012-Nov-11.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2013-Aug-26.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2013-Aug-26.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2013-Aug-26.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2013-Aug-26.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2013-Dec-22.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2013-Dec-22.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2013-Dec-22.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2013-Dec-22.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2013-Feb-04.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2013-Feb-04.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2013-Feb-04.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2013-Feb-04.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2013-Oct-28.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2013-Oct-28.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2013-Oct-28.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2013-Oct-28.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2013-Sep-26.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2013-Sep-26.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2013-Sep-26.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2013-Sep-26.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2014-Oct-16.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2014-Oct-16.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2014-Oct-16.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2014-Oct-16.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2015-Apr-10.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2015-Apr-10.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2015-Apr-10.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2015-Apr-10.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2015-Dec-01.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2015-Dec-01.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2015-Dec-01.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2015-Dec-01.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2015-Dec-30.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2015-Dec-30.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2015-Dec-30.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2015-Dec-30.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2015-Sep-02.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2015-Sep-02.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2015-Sep-02.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2015-Sep-02.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2015-Sep-30.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2015-Sep-30.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2015-Sep-30.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2015-Sep-30.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2016-Dec-17.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2016-Dec-17.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2016-Dec-17.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2016-Dec-17.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2016-May-03.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2016-May-03.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2016-May-03.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2016-May-03.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2016-Nov-22.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2016-Nov-22.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2016-Nov-22.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2016-Nov-22.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2017-Dec-10.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2017-Dec-10.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2017-Dec-10.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2017-Dec-10.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2017-May-16.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2017-May-16.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2017-May-16.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2017-May-16.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2018-Apr-19.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2018-Apr-19.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2018-Apr-19.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2018-Apr-19.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2018-May-17.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2018-May-17.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2018-May-17.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2018-May-17.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2018-Nov-26.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2018-Nov-26.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2018-Nov-26.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2018-Nov-26.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2019-Dec-29.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2019-Dec-29.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2019-Dec-29.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2019-Dec-29.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2019-Nov-19.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2019-Nov-19.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2019-Nov-19.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2019-Nov-19.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2019-Nov-27.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2019-Nov-27.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2019-Nov-27.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2019-Nov-27.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2021-Jun-21.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2021-Jun-21.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2021-Jun-21.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2021-Jun-21.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2021-May-04.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2021-May-04.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2021-May-04.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2021-May-04.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2021-May-05.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2021-May-05.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2021-May-05.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2021-May-05.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2021-May-31.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2021-May-31.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2021-May-31.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2021-May-31.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2021-Sep-05.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2021-Sep-05.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2021-Sep-05.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2021-Sep-05.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2022-Apr-22.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Apr-22.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2022-Apr-22.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Apr-22.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2022-Jan-28.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Jan-28.log similarity index 100% rename from pypeit/data/spectrographs/keck_deimos/gain_ronoise/gain_check.Spectral.2022-Jan-28.log rename to pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Jan-28.log diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Jun-01.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Jun-01.log new file mode 100644 index 0000000000..5719f819e8 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Jun-01.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Spectral.2022-Jun-01.log +# datestr = Wed Jun 1 23:22:28 2022 +# mode = Spectral +# flatfile1 = /sdata1005/deimos3/2022jun02/d0602_0027.fits +# flatfile2 = /sdata1005/deimos3/2022jun02/d0602_0028.fits +# flatfile3 = /sdata1005/deimos3/2022jun02/d0602_0029.fits +# flatfile4 = /sdata1005/deimos3/2022jun02/d0602_0030.fits +# flatfile5 = /sdata1005/deimos3/2022jun02/d0602_0031.fits +# zerofile1 = /sdata1005/deimos3/2022jun02/d0602_0024.fits +# zerofile2 = /sdata1005/deimos3/2022jun02/d0602_0025.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 2 1 1 B 1.1689 0.017343 2.4508 0.036363 + 4 3 2 B 1.1647 0.017516 2.4420 0.036727 + 6 5 3 B 1.2477 0.020783 2.6161 0.043576 + 8 7 4 B 1.2056 0.020708 2.5278 0.043419 + 10 10 5 B 1.2199 0.016986 2.5577 0.035614 + 12 12 6 B 1.2028 0.013458 2.5219 0.028217 + 14 14 7 B 1.1980 0.018849 2.5118 0.039520 + 16 16 8 B 1.2148 0.018771 2.5471 0.039356 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Jun-22.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Jun-22.log new file mode 100644 index 0000000000..a38ff21a5f --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Jun-22.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Spectral.2022-Jun-22.log +# datestr = Wed Jun 22 18:44:36 2022 +# mode = Spectral +# flatfile1 = /sdata1005/deimos4/2022jun23/d0623_0035.fits +# flatfile2 = /sdata1005/deimos4/2022jun23/d0623_0036.fits +# flatfile3 = /sdata1005/deimos4/2022jun23/d0623_0037.fits +# flatfile4 = /sdata1005/deimos4/2022jun23/d0623_0038.fits +# flatfile5 = /sdata1005/deimos4/2022jun23/d0623_0039.fits +# zerofile1 = /sdata1005/deimos4/2022jun23/d0623_0032.fits +# zerofile2 = /sdata1005/deimos4/2022jun23/d0623_0033.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 2 1 1 B 1.1826 0.016517 2.4796 0.034631 + 4 3 2 B 1.1710 0.014819 2.4553 0.031070 + 6 5 3 B 1.2479 0.021303 2.6165 0.044665 + 8 7 4 B 1.2141 0.014918 2.5457 0.031280 + 10 10 5 B 1.2165 0.015308 3.8256 0.047888 + 12 12 6 B 1.2035 0.018497 2.5234 0.038782 + 14 14 7 B 1.2006 0.019129 2.5172 0.040108 + 16 16 8 B 1.2167 0.022545 2.5510 0.047271 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Nov-14.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Nov-14.log new file mode 100644 index 0000000000..13553d0881 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Nov-14.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Spectral.2022-Nov-14.log +# datestr = Mon Nov 14 00:22:22 2022 +# mode = Spectral +# flatfile1 = /sdata1004/dmoseng/2022nov14/d1114_0013.fits +# flatfile2 = /sdata1004/dmoseng/2022nov14/d1114_0014.fits +# flatfile3 = /sdata1004/dmoseng/2022nov14/d1114_0015.fits +# flatfile4 = /sdata1004/dmoseng/2022nov14/d1114_0016.fits +# flatfile5 = /sdata1004/dmoseng/2022nov14/d1114_0017.fits +# zerofile1 = /sdata1004/dmoseng/2022nov14/d1114_0010.fits +# zerofile2 = /sdata1004/dmoseng/2022nov14/d1114_0011.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 2 1 1 B 1.1784 0.016103 2.4708 0.033763 + 4 3 2 B 1.1722 0.015694 2.4578 0.032905 + 6 5 3 B 1.2531 0.015877 2.6275 0.033289 + 8 7 4 B 1.2187 0.015008 2.5552 0.031466 + 10 10 5 B 1.2158 0.013563 2.5492 0.028437 + 12 12 6 B 1.2030 0.018836 2.5223 0.039492 + 14 14 7 B 1.2054 0.014269 2.5274 0.029917 + 16 16 8 B 1.2544 0.022474 2.6302 0.047122 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Oct-11.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Oct-11.log new file mode 100644 index 0000000000..0080d5652f --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2022-Oct-11.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Spectral.2022-Oct-11.log +# datestr = Tue Oct 11 15:00:40 2022 +# mode = Spectral +# flatfile1 = /sdata1004/dmoseng/2022oct12/d1012_0019.fits +# flatfile2 = /sdata1004/dmoseng/2022oct12/d1012_0020.fits +# flatfile3 = /sdata1004/dmoseng/2022oct12/d1012_0021.fits +# flatfile4 = /sdata1004/dmoseng/2022oct12/d1012_0022.fits +# flatfile5 = /sdata1004/dmoseng/2022oct12/d1012_0023.fits +# zerofile1 = /sdata1004/dmoseng/2022oct12/d1012_0016.fits +# zerofile2 = /sdata1004/dmoseng/2022oct12/d1012_0017.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 2 1 1 B 1.1797 0.017066 2.4735 0.035782 + 4 3 2 B 1.1797 0.017516 2.4736 0.036725 + 6 5 3 B 1.2532 0.016484 2.6276 0.034562 + 8 7 4 B 1.2138 0.012262 2.5449 0.025710 + 10 10 5 B 1.2171 0.017933 2.5518 0.037600 + 12 12 6 B 1.2034 0.017107 2.5231 0.035869 + 14 14 7 B 1.2060 0.017311 2.5287 0.036296 + 16 16 8 B 1.2179 0.012596 2.5535 0.026409 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Aug-06.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Aug-06.log new file mode 100644 index 0000000000..79cc78b548 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Aug-06.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Spectral.2023-Aug-06.log +# datestr = Sun Aug 6 20:07:19 2023 +# mode = Spectral +# flatfile1 = /sdata1005/dmoseng/2023aug07/d0728_0044.fits +# flatfile2 = /sdata1005/dmoseng/2023aug07/d0728_0045.fits +# flatfile3 = /sdata1005/dmoseng/2023aug07/d0728_0046.fits +# flatfile4 = /sdata1005/dmoseng/2023aug07/d0728_0047.fits +# flatfile5 = /sdata1005/dmoseng/2023aug07/d0728_0048.fits +# zerofile1 = /sdata1005/dmoseng/2023aug07/d0728_0041.fits +# zerofile2 = /sdata1005/dmoseng/2023aug07/d0728_0042.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 2 1 1 B 1.1784 0.014323 2.4708 0.030031 + 4 3 2 B 1.1687 0.012534 2.4504 0.026280 + 6 5 3 B 1.2621 0.019232 2.6463 0.040324 + 8 7 4 B 1.2110 0.016884 2.5390 0.035401 + 10 10 5 B 1.2421 0.019382 2.6044 0.040637 + 12 12 6 B 1.2009 0.018459 2.5178 0.038702 + 14 14 7 B 1.1947 0.016139 2.5049 0.033839 + 16 16 8 B 1.2446 0.022439 2.6096 0.047049 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Dec-11.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Dec-11.log new file mode 100644 index 0000000000..8a1ae1af1e --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Dec-11.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Spectral.2023-Dec-11.log +# datestr = Mon Dec 11 01:24:09 2023 +# mode = Spectral +# flatfile1 = /sdata1005/deimos2/2023dec11/d1211_0022.fits +# flatfile2 = /sdata1005/deimos2/2023dec11/d1211_0023.fits +# flatfile3 = /sdata1005/deimos2/2023dec11/d1211_0024.fits +# flatfile4 = /sdata1005/deimos2/2023dec11/d1211_0025.fits +# flatfile5 = /sdata1005/deimos2/2023dec11/d1211_0026.fits +# zerofile1 = /sdata1005/deimos2/2023dec11/d1211_0019.fits +# zerofile2 = /sdata1005/deimos2/2023dec11/d1211_0020.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 2 1 1 B 1.1800 0.013007 2.4740 0.027271 + 4 3 2 B 1.1719 0.015773 2.4572 0.033070 + 6 5 3 B 1.2648 0.019444 3.9777 0.061153 + 8 7 4 B 1.2095 0.018859 2.5360 0.039542 + 10 10 5 B 1.2327 0.017411 2.5846 0.036505 + 12 12 6 B 1.1974 0.013983 2.5106 0.029318 + 14 14 7 B 1.1943 0.018010 2.5041 0.037762 + 16 16 8 B 1.2449 0.016423 2.6101 0.034435 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Jan-06.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Jan-06.log new file mode 100644 index 0000000000..52b8822d48 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Jan-06.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Spectral.2023-Jan-06.log +# datestr = Fri Jan 6 22:28:15 2023 +# mode = Spectral +# flatfile1 = /sdata1005/deimos2/2023jan07/d0107_0037.fits +# flatfile2 = /sdata1005/deimos2/2023jan07/d0107_0038.fits +# flatfile3 = /sdata1005/deimos2/2023jan07/d0107_0039.fits +# flatfile4 = /sdata1005/deimos2/2023jan07/d0107_0040.fits +# flatfile5 = /sdata1005/deimos2/2023jan07/d0107_0041.fits +# zerofile1 = /sdata1005/deimos2/2023jan07/d0107_0034.fits +# zerofile2 = /sdata1005/deimos2/2023jan07/d0107_0035.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 2 1 1 B 1.1790 0.013500 2.4721 0.028305 + 4 3 2 B 1.1731 0.018365 2.4596 0.038507 + 6 5 3 B 1.2702 0.015348 3.9949 0.048269 + 8 7 4 B 1.2170 0.014134 2.5517 0.029635 + 10 10 5 B 1.2471 0.023396 7.7994 0.26533 + 12 12 6 B 1.1977 0.017206 2.5112 0.036076 + 14 14 7 B 1.1976 0.014123 2.5109 0.029612 + 16 16 8 B 1.2485 0.021585 2.6177 0.045256 diff --git a/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Oct-05.log b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Oct-05.log new file mode 100644 index 0000000000..81ae1c4cc8 --- /dev/null +++ b/pypeit/data/spectrographs/keck_deimos/gain_ronoise/ampB/gain_check.Spectral.2023-Oct-05.log @@ -0,0 +1,27 @@ +# logfile = /s/sdata1004/ccd_gain/gain_check.Spectral.2023-Oct-05.log +# datestr = Thu Oct 5 12:57:16 2023 +# mode = Spectral +# flatfile1 = /sdata1004/dmoseng/2023oct05/d1005_0046.fits +# flatfile2 = /sdata1004/dmoseng/2023oct05/d1005_0047.fits +# flatfile3 = /sdata1004/dmoseng/2023oct05/d1005_0048.fits +# flatfile4 = /sdata1004/dmoseng/2023oct05/d1005_0049.fits +# flatfile5 = /sdata1004/dmoseng/2023oct05/d1005_0050.fits +# zerofile1 = /sdata1004/dmoseng/2023oct05/d1005_0043.fits +# zerofile2 = /sdata1004/dmoseng/2023oct05/d1005_0044.fits +# column1 = videoinp +# column2 = amploc +# column3 = ccdloc +# column4 = amptype +# column5 = gain [e-/DN] +# column6 = sigma_gain [e-/DN] +# column7 = readnoise [e-] +# column8 = sigma_readnoise [e-] +#---------------------------------------- + 2 1 1 B 1.1799 0.014565 2.4740 0.030537 + 4 3 2 B 1.1741 0.015021 2.4618 0.031494 + 6 5 3 B 1.2625 0.015967 2.6471 0.033478 + 8 7 4 B 1.2105 0.016401 2.5381 0.034388 + 10 10 5 B 1.2344 0.014263 2.5882 0.029905 + 12 12 6 B 1.2006 0.014541 2.5172 0.030488 + 14 14 7 B 1.1942 0.015844 2.5038 0.033219 + 16 16 8 B 1.2538 0.016214 2.6288 0.033996 diff --git a/pypeit/spectrographs/keck_deimos.py b/pypeit/spectrographs/keck_deimos.py index 8066755a9d..25f9576a61 100644 --- a/pypeit/spectrographs/keck_deimos.py +++ b/pypeit/spectrographs/keck_deimos.py @@ -210,10 +210,12 @@ def get_detector_par(self, det, hdu=None): )) if hdu is not None: + amp = self.get_meta_value(self.get_headarr(hdu), 'amp') + amp_folder = "ampA" if amp == 'SINGLE:A' else "ampB" # raw frame date in mjd date = time.Time(self.get_meta_value(self.get_headarr(hdu), 'mjd'), format='mjd').value # get the measurements files - measure_files = sorted((data.Paths.spectrographs / "keck_deimos" / "gain_ronoise").glob("*")) + measure_files = sorted((data.Paths.spectrographs / "keck_deimos" / "gain_ronoise" / amp_folder).glob("*")) # Parse the dates recorded in the name of the files measure_dates = np.array([f.name.split('.')[2] for f in measure_files]) # convert into datetime format @@ -225,28 +227,43 @@ def get_detector_par(self, det, hdu=None): # get measurements tab_measure = Table.read(measure_files[close_idx], format='ascii') measured_det = tab_measure['col3'] + measured_amptype = tab_measure['col4'] measured_gain = tab_measure['col5'] # [e-/DN] measured_ronoise = tab_measure['col7'] # [e-] - msgs.info(f"We are using DEIMOS gain/RN values based on WMKO estimates on {measure_dates[close_idx]}.") + msgs.info(f"We are using DEIMOS gain/RN values for AMPMODE = {amp} " + f"based on WMKO estimates on {measure_dates[close_idx]}.") + # find values for this amp and each detector + this_amp = measured_amptype == 'A' if amp == 'SINGLE:A' else measured_amptype == 'B' + det_1 = this_amp & (measured_det == 1) + det_2 = this_amp & (measured_det == 2) + det_3 = this_amp & (measured_det == 3) + det_4 = this_amp & (measured_det == 4) + # we don't have measurements for the red detectors when amp = 'SINGLE:A', + # therefore we use the values for the blue detectors + det_5 = this_amp & (measured_det == 5 if amp == 'SINGLE:B' else measured_det == 1) + det_6 = this_amp & (measured_det == 6 if amp == 'SINGLE:B' else measured_det == 2) + det_7 = this_amp & (measured_det == 7 if amp == 'SINGLE:B' else measured_det == 3) + det_8 = this_amp & (measured_det == 8 if amp == 'SINGLE:B' else measured_det == 4) + # get gain - detector_dict1['gain'] = measured_gain[measured_det == 1] - detector_dict2['gain'] = measured_gain[measured_det == 2] - detector_dict3['gain'] = measured_gain[measured_det == 3] - detector_dict4['gain'] = measured_gain[measured_det == 4] - detector_dict5['gain'] = measured_gain[measured_det == 5] - detector_dict6['gain'] = measured_gain[measured_det == 6] - detector_dict7['gain'] = measured_gain[measured_det == 7] - detector_dict8['gain'] = measured_gain[measured_det == 8] + detector_dict1['gain'] = measured_gain[det_1].data + detector_dict2['gain'] = measured_gain[det_2].data + detector_dict3['gain'] = measured_gain[det_3].data + detector_dict4['gain'] = measured_gain[det_4].data + detector_dict5['gain'] = measured_gain[det_5].data + detector_dict6['gain'] = measured_gain[det_6].data + detector_dict7['gain'] = measured_gain[det_7].data + detector_dict8['gain'] = measured_gain[det_8].data # get ronoise - detector_dict1['ronoise'] = measured_ronoise[measured_det == 1] - detector_dict2['ronoise'] = measured_ronoise[measured_det == 2] - detector_dict3['ronoise'] = measured_ronoise[measured_det == 3] - detector_dict4['ronoise'] = measured_ronoise[measured_det == 4] - detector_dict5['ronoise'] = measured_ronoise[measured_det == 5] - detector_dict6['ronoise'] = measured_ronoise[measured_det == 6] - detector_dict7['ronoise'] = measured_ronoise[measured_det == 7] - detector_dict8['ronoise'] = measured_ronoise[measured_det == 8] + detector_dict1['ronoise'] = measured_ronoise[det_1].data + detector_dict2['ronoise'] = measured_ronoise[det_2].data + detector_dict3['ronoise'] = measured_ronoise[det_3].data + detector_dict4['ronoise'] = measured_ronoise[det_4].data + detector_dict5['ronoise'] = measured_ronoise[det_5].data + detector_dict6['ronoise'] = measured_ronoise[det_6].data + detector_dict7['ronoise'] = measured_ronoise[det_7].data + detector_dict8['ronoise'] = measured_ronoise[det_8].data detectors = [detector_dict1, detector_dict2, detector_dict3, detector_dict4, detector_dict5, detector_dict6, detector_dict7, detector_dict8] @@ -325,9 +342,14 @@ def config_specific_par(self, scifile, inp_par=None): headarr = self.get_headarr(scifile) - # When using LVM mask reduce only detectors 3,7 - if 'LVMslit' in self.get_meta_value(headarr, 'decker'): - par['rdx']['detnum'] = [(3,7)] + # When using LVM mask or AMPMODE = SINGLE:A reduce only detectors 3,7 + if ('LVMslit' in self.get_meta_value(headarr, 'decker') or + self.get_meta_value(headarr, 'amp') == 'SINGLE:A'): + # give an info message if AMPMODE = SINGLE:A + if self.get_meta_value(headarr, 'amp') == 'SINGLE:A': + msgs.info('Data taken with AMPMODE = SINGLE:A. Only detectors 3,7 will be reduced. To change this,' + ' modify the detnum parameter in the pypeit file.') + par['rdx']['detnum'] = [(3, 7)] # Turn PCA off for long slits # TODO: I'm a bit worried that this won't catch all @@ -542,7 +564,7 @@ def valid_configuration_values(self): and their associated discrete set of valid values. If there are no restrictions on configuration values, None is returned. """ - return {'amp': ['SINGLE:B'], 'mode':['Spectral']} + return {'amp': ['SINGLE:B', 'SINGLE:A'], 'mode':['Spectral']} def config_independent_frames(self): """ @@ -696,9 +718,9 @@ def get_rawimage(self, raw_file, det): .. warning:: PypeIt currently *cannot* reduce images produced by - reading the DEIMOS CCDs with the A amplifier or those + reading the DEIMOS CCDs with the A+B amplifier or those taken in imaging mode. All image handling assumes DEIMOS - images have been read with the B amplifier in the + images have been read with the B or A amplifier in the "Spectral" observing mode. This method will fault if this is not true based on the header keywords MOSMODE and AMPMODE. @@ -742,7 +764,7 @@ def get_rawimage(self, raw_file, det): detectors = [self.get_detector_par(det, hdu=hdu)] if nimg == 1 else mosaic.detectors if hdu[0].header['AMPMODE'] not in ['SINGLE:B', 'SINGLE:A']: - msgs.error('PypeIt can only reduce images with AMPMODE == SINGLE:B.') + msgs.error('PypeIt can only reduce images with AMPMODE == SINGLE:B or AMPMODE == SINGLE:A.') if hdu[0].header['MOSMODE'] != 'Spectral': msgs.error('PypeIt can only reduce images with MOSMODE == Spectral.') From 2e94b93ce3f386909733e4a79ea6457ad5d1e4a7 Mon Sep 17 00:00:00 2001 From: jhennawi Date: Thu, 21 Dec 2023 14:10:40 -0800 Subject: [PATCH 132/244] Fixed a coadding issue. --- pypeit/coadd1d.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pypeit/coadd1d.py b/pypeit/coadd1d.py index fddadbf6ea..aba3360810 100644 --- a/pypeit/coadd1d.py +++ b/pypeit/coadd1d.py @@ -35,7 +35,7 @@ def get_instance(cls, spec1dfiles, objids, spectrograph=None, par=None, sensfunc argument descriptions. """ pypeline = fits.getheader(spec1dfiles[0])['PYPELINE'] + 'CoAdd1D' - return next(c for c in cls.__subclasses__() if c.__name__ == pypeline)( + return next(c for c in utils.all_subclasses(CoAdd1D) if c.__name__ == pypeline)( spec1dfiles, objids, spectrograph=spectrograph, par=par, sensfuncfile=sensfuncfile, setup_id=setup_id, debug=debug, show=show) @@ -518,3 +518,14 @@ def load(self): return waves, fluxes, ivars, gpms, weights_sens, headers +class SlicerIFUCoAdd1D(MultiSlitCoAdd1D): + """ + Child of MultiSlitCoAdd1d for SlicerIFU reductions. + """ + + def __init__(self, spec1dfiles, objids, spectrograph=None, par=None, sensfuncfile=None, setup_id=None, debug=False, show=False): + """ + See :class:`CoAdd1D` instantiation for argument descriptions. + """ + super().__init__(spec1dfiles, objids, spectrograph=spectrograph, par = par, sensfuncfile = sensfuncfile, + setup_id=setup_id, debug = debug, show = show) From 098f3fb5ad2019bb53b49e5106ce90499212a91d Mon Sep 17 00:00:00 2001 From: jhennawi Date: Thu, 21 Dec 2023 14:31:52 -0800 Subject: [PATCH 133/244] Fixed a coadding issue. --- pypeit/core/coadd.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pypeit/core/coadd.py b/pypeit/core/coadd.py index 98ed7e886c..0becda7d61 100644 --- a/pypeit/core/coadd.py +++ b/pypeit/core/coadd.py @@ -771,8 +771,8 @@ def calc_snr(fluxes, ivars, gpms): Returns ------- - rms_sn (np.ndarray): - Array of shape (nexp,) of root-mean-square S/N value for each input spectra where nexp=len(fluxes). + rms_sn : list + List of length nexp root-mean-square S/N value for each input spectra where nexp=len(fluxes). sn_val : list List of length nexp containing the wavelength dependent S/N arrays for each input spectrum, i.e. each element contains the array flux*sqrt(ivar) @@ -794,7 +794,8 @@ def calc_snr(fluxes, ivars, gpms): sn2.append(sn2_iexp) rms_sn.append(np.sqrt(sn2_iexp)) # Root Mean S/N**2 value for all spectra - return np.array(rms_sn), sn_val + return rms_sn, sn_val + def sn_weights(fluxes, ivars, gpms, sn_smooth_npix=None, weight_method='auto', verbose=False): From 3204184bc3a6bdd53acb37b70c67f3a4215f0c1f Mon Sep 17 00:00:00 2001 From: jhennawi Date: Thu, 21 Dec 2023 14:24:10 -0800 Subject: [PATCH 134/244] Fixed a coadding issue. --- pypeit/core/datacube.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index 475db4f93a..a788ca51d8 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -665,10 +665,9 @@ def set_voxel_sampling(spatscale, specscale, dspat=None, dwv=None): def wcs_bounds(all_ra, all_dec, all_wave, ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None): """ - Calculate the bounds of the WCS and the expected edges of the voxels, based - on user-specified parameters or the extremities of the data. This is a - convenience function that calls the core function in - :mod:`~pypeit.core.datacube`. + Create a WCS and the expected edges of the voxels, based on user-specified + parameters or the extremities of the data. This is a convenience function + that calls the core function in `pypeit.core.datacube`_. Parameters ---------- From f887f5adf102f6613a47b671a798dd67d7e7b577 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 21 Dec 2023 14:36:32 -0800 Subject: [PATCH 135/244] make show_2dspec more robust against bitmask changes --- pypeit/bitmask.py | 25 +++++++++++++++++++++++++ pypeit/pypmsgs.py | 3 +++ pypeit/scripts/show_2dspec.py | 4 ++-- pypeit/slittrace.py | 12 ++++++------ pypeit/tests/test_bitmask.py | 17 +++++++++++++++++ 5 files changed, 53 insertions(+), 8 deletions(-) diff --git a/pypeit/bitmask.py b/pypeit/bitmask.py index caa92c96d5..a7b666d057 100644 --- a/pypeit/bitmask.py +++ b/pypeit/bitmask.py @@ -532,5 +532,30 @@ def parse_bits_from_hdr(hdr, prefix): values += [i] descr += [hdr.comments[k]] return keys, values, descr + + def correct_flag_order(self, flags): + """ + Check if the provided flags are in the correct order compared to the + current definition of the object. + + Args: + flags (:obj:`list`): + A list of strings that *must* be in the order of the bit + numbers. I.e., bit 0 uses the string in the first element of + the list, bit 1 uses the second element, etc. The number of + flags does not need to exactly match the current set of flags. + It can be longer or shorter, so long as it begins with the first + flag. + + Returns: + :obj:`bool`: Indicates if the provides flags are in the correct order. + """ + cls_flags = list(self.keys()) + for i, flag in enumerate(flags): + if i >= self.nbits: + break + if cls_flags[i] != flag: + return False + return True diff --git a/pypeit/pypmsgs.py b/pypeit/pypmsgs.py index 0755b00fd6..36caf265b4 100644 --- a/pypeit/pypmsgs.py +++ b/pypeit/pypmsgs.py @@ -31,6 +31,9 @@ class PypeItError(Exception): pass +class PypeItBitMaskError(PypeItError): + pass + class PypeItDataModelError(PypeItError): pass diff --git a/pypeit/scripts/show_2dspec.py b/pypeit/scripts/show_2dspec.py index 5a9c13029b..66ad5e66f8 100644 --- a/pypeit/scripts/show_2dspec.py +++ b/pypeit/scripts/show_2dspec.py @@ -20,7 +20,7 @@ from pypeit import io from pypeit import utils from pypeit import __version__ -from pypeit.pypmsgs import PypeItError, PypeItDataModelError +from pypeit.pypmsgs import PypeItDataModelError, PypeItBitMaskError from pypeit.display import display from pypeit.images.imagebitmask import ImageBitMask @@ -138,7 +138,7 @@ def main(args): # for the datamodel version to be different try: spec2DObj = spec2dobj.Spec2DObj.from_file(args.file, detname, chk_version=False) - except PypeItDataModelError: + except (PypeItDataModelError, PypeItBitMaskError): try: # Try to get the pypeit version used to write this file file_pypeit_version = fits.getval(args.file, 'VERSPYP', 0) diff --git a/pypeit/slittrace.py b/pypeit/slittrace.py index 4c8b4b04a1..fd7943c3cf 100644 --- a/pypeit/slittrace.py +++ b/pypeit/slittrace.py @@ -17,6 +17,7 @@ from astropy.stats import sigma_clipped_stats from scipy.interpolate import RegularGridInterpolator, interp1d +from pypeit.pypmsgs import PypeItBitMaskError from pypeit import msgs from pypeit import datamodel from pypeit import calibframe @@ -32,7 +33,9 @@ class SlitTraceBitMask(BitMask): version = '1.0.1' def __init__(self): - # Only ever append new bits (and don't remove old ones) + # !!!!!!!!!! + # IMPORTANT: Only ever *append* new bits, and don't remove old ones + # !!!!!!!!!! mask = dict([ ('SHORTSLIT', 'Slit formed by left and right edge is too short. Not ignored for flexure'), ('BOXSLIT', 'Slit formed by left and right edge is valid (large enough to be a valid ' @@ -245,11 +248,8 @@ def _validate(self): self.slitbitm = ','.join(list(self.bitmask.keys())) else: # Validate -- All of the keys must be present and in current order, but new ones can exist - bitms = self.slitbitm.split(',') - curbitm = list(self.bitmask.keys()) - for kk, bit in enumerate(bitms): - if curbitm[kk] != bit: - msgs.error("Input BITMASK keys differ from current data model!") + if not self.bitmask.correct_flag_order(self.slitbitm.split(',')): + raise PypeItBitMaskError('Input BITMASK keys differ from current data model!') # Update to current, no matter what self.slitbitm = ','.join(list(self.bitmask.keys())) # Mask diff --git a/pypeit/tests/test_bitmask.py b/pypeit/tests/test_bitmask.py index 2b870dd02a..bea18c71d9 100644 --- a/pypeit/tests/test_bitmask.py +++ b/pypeit/tests/test_bitmask.py @@ -107,3 +107,20 @@ def test_wrong_bits(): assert numpy.sum(image_bm.flagged(mask, flag='COSMIC')) == numpy.sum(cosmics_indx) +def test_flag_order(): + + bm = ImageBitMask() + + flags = bm.keys() + assert bm.correct_flag_order(flags), 'Flags should not be mismatched' + + flags += ['NEWBIT'] + assert bm.correct_flag_order(flags), 'Appending flags should be fine' + + flags = bm.keys()[:-1] + assert bm.correct_flag_order(flags), 'Checking a subset of the flags should be fine' + + flags = bm.keys()[::-1] + assert not bm.correct_flag_order(flags), 'Reordering the flags is not okay' + + From 4416eeec6e8dd0758881003c2c5756f7f8b31a0a Mon Sep 17 00:00:00 2001 From: jhennawi Date: Thu, 21 Dec 2023 14:49:12 -0800 Subject: [PATCH 136/244] Fixed a coadding issue. --- pypeit/specutils/pypeit_loaders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/specutils/pypeit_loaders.py b/pypeit/specutils/pypeit_loaders.py index 6c529bb26f..937d2fc796 100644 --- a/pypeit/specutils/pypeit_loaders.py +++ b/pypeit/specutils/pypeit_loaders.py @@ -269,7 +269,7 @@ def pypeit_onespec_loader(filename, grid=False, strict=True, **kwargs): except AttributeError: name = '' - # TODO We should be dealing with masking here + # TODO We should be dealing with masking here. return Spectrum1D(flux=astropy.units.Quantity(spec.flux * flux_unit), uncertainty=None if spec.sigma is None else astropy.units.Quantity(spec.sigma * flux_unit), # else astropy.nddata.InverseVariance(spec.ivar / flux_unit**2), From fab890c0ad97e4ddc305ab9daa237ceadb68cf4d Mon Sep 17 00:00:00 2001 From: jhennawi Date: Thu, 21 Dec 2023 16:12:24 -0800 Subject: [PATCH 137/244] addressed PR comments. --- pypeit/coadd2d.py | 42 +++++++++++++++++++++++++++++++++-------- pypeit/onespec.py | 9 --------- pypeit/par/pypeitpar.py | 39 ++++++++++++++++++++++++++++++++++---- 3 files changed, 69 insertions(+), 21 deletions(-) diff --git a/pypeit/coadd2d.py b/pypeit/coadd2d.py index 4f792fd7a2..025ca4eea9 100644 --- a/pypeit/coadd2d.py +++ b/pypeit/coadd2d.py @@ -480,17 +480,43 @@ def optimal_weights(self, slitorderid, objid, weight_method='auto'): Array of object indices with shape = (nexp,) of the brightest object whose S/N will be used to determine the weight for each frame. - const_weights : :obj:`bool` - Use constant weights for coadding the exposures. - Default=False + weight_method: (`str`, optional) + Weight method to be used in `coadd.sn_weights`. Default is 'auto'. + Options are 'auto', 'constant', 'relative', or 'ivar'. The defaulti is'auto'. + Behavior is as follows: + 'auto': + Use constant weights if rms_sn < 3.0, otherwise use wavelength dependent. + 'constant': + Constant weights based on rms_sn**2 + 'uniform': + Uniform weighting. + 'wave_dependent': + Wavelength dependent weights will be used irrespective of the rms_sn ratio. This option + will not work well at low S/N ratio although it is useful for objects where only a small + fraction of the spectral coverage has high S/N ratio (like high-z quasars). + 'relative': + Calculate weights by fitting to the ratio of spectra? Note, relative + weighting will only work well when there is at least one spectrum with a + reasonable S/N, and a continuum. RJC note - This argument may only be + better when the object being used has a strong continuum + emission + lines. The reference spectrum is assigned a value of 1 for all + wavelengths, and the weights of all other spectra will be determined + relative to the reference spectrum. This is particularly useful if you + are dealing with highly variable spectra (e.g. emission lines) and + require a precision better than ~1 per cent. + 'ivar': + Use inverse variance weighting. This is not well tested and should probably be deprecated. + Returns ------- - rms_sn : ndarray, shape = (len(specobjs_list),) - Root mean square S/N value for each input spectra - weights : ndarray, shape (len(specobjs_list),) - Weights to be applied to the spectra. These are - signal-to-noise squared weights. + rms_sn : `numpy.ndarray`_ + Array of root-mean-square S/N value for each input spectra. Shape = (nexp,) + weights : list + List of len(nexp) containing the signal-to-noise squared weights to be + applied to the spectra. This output is aligned with the vector (or + vectors) provided in waves which is read in by this routine, i.e. it is a + list of arrays of type `numpy.ndarray`_ with the same shape as those in waves. """ nexp = len(self.stack_dict['specobjs_list']) diff --git a/pypeit/onespec.py b/pypeit/onespec.py index a381174a53..c71f74b527 100644 --- a/pypeit/onespec.py +++ b/pypeit/onespec.py @@ -120,15 +120,6 @@ def _bundle(self): """ return super()._bundle(ext='SPECTRUM') - #@property - #def sig(self): - # """ Return the 1-sigma array - # - # Returns: - # `numpy.ndarray`_: error array - # """ - # return np.sqrt(utils.inverse(self.ivar)) - def to_file(self, ofile, primary_hdr=None, history=None, **kwargs): """ Over-load :func:`pypeit.datamodel.DataContainer.to_file` diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index c997cb7ec7..87f5e01afe 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -1170,6 +1170,7 @@ def __init__(self, ex_value=None, flux_value=None, nmaskedge=None, defaults['wave_method'] = 'linear' dtypes['wave_method'] = str + options['wave_method'] = Coadd1DPar.valid_wave_methods() descr['wave_method'] = "Method used to construct wavelength grid for coadding spectra. The routine that creates " \ "the wavelength is :func:`~pypeit.core.wavecal.wvutils.get_wave_grid`. The options are:" \ " "\ @@ -1228,6 +1229,7 @@ def __init__(self, ex_value=None, flux_value=None, nmaskedge=None, defaults['scale_method'] = 'auto' dtypes['scale_method'] = str + options['scale_method'] = Coadd1DPar.valid_scale_methods() descr['scale_method'] = "Method used to rescale the spectra prior to coadding. The options are:" \ " "\ "'auto' -- Determine the scaling method automatically based on the S/N ratio which works well. "\ @@ -1246,6 +1248,7 @@ def __init__(self, ex_value=None, flux_value=None, nmaskedge=None, descr['sn_min_polyscale'] = "For scale method set to ``auto``, this sets the minimum SNR for which polynomial scaling is attempted." defaults['weight_method'] = 'auto' + options['weight_method'] = Coadd1DPar.valid_weight_methods() dtypes['weight_method'] = str descr['weight_method'] = "Method used to rescale the spectra prior to coadding. The options are:" \ " " \ @@ -1356,13 +1359,21 @@ def validate(self): """ Check the parameters are valid for the provided method. """ - allowed_scale_methods = ['auto', 'poly', 'median', 'none', 'hand'] + allowed_extensions = self.valid_ex() + if self.data['ex_value'] not in allowed_extensions: + raise ValueError("'ex_value' must be one of:\n" + ", ".join(allowed_extensions)) + + allowed_wave_methods = self.valid_wave_methods() + if self.data['ex_value'] not in allowed_wave_methods: + raise ValueError("'wave_method' must be one of:\n" + ", ".join(allowed_wave_methods)) + + allowed_scale_methods = self.valid_scale_methods() if self.data['scale_method'] not in allowed_scale_methods: - raise ValueError("If 'wave_method' is not None it must be one of:\n" + ", ".join(allowed_scale_methods)) + raise ValueError("'scale_method' must be one of:\n" + ", ".join(allowed_scale_methods)) - allowed_weight_methods = ['auto', 'constant', 'uniform', 'wave_dependent', 'relative', 'ivar'] + allowed_weight_methods = self.valid_weight_methods() if self.data['weight_method'] not in allowed_scale_methods: - raise ValueError("If 'weight_method' is not None it must be one of:\n" + ", ".join(allowed_weight_methods)) + raise ValueError("'weight_method' must be one of:\n" + ", ".join(allowed_weight_methods)) @@ -1374,6 +1385,26 @@ def valid_ex(): return ['BOX', 'OPT'] + @staticmethod + def valid_wave_methods(): + """ Return the valid options for the wavelength grid of spectra. """ + + return ['iref', 'velocity', 'log10', 'linear', 'concatenate'] + + @staticmethod + def valid_scale_methods(): + """ Return the valid options for the scaling of spectra. """ + + return ['auto', 'poly', 'median', 'none', 'hand'] + + @staticmethod + def valid_weight_methods(): + """ Return the valid options for the weighting of spectra. """ + + return ['auto', 'constant', 'uniform', 'wave_dependent', 'relative', 'ivar'] + + + class Coadd2DPar(ParSet): """ A parameter set holding the arguments for how to perform 2D coadds From 64176114a431a20641f4458da1ecfe232d442345 Mon Sep 17 00:00:00 2001 From: jhennawi Date: Thu, 21 Dec 2023 16:28:39 -0800 Subject: [PATCH 138/244] addressed PR comments. --- pypeit/par/pypeitpar.py | 2 +- pypeit/specutils/pypeit_loaders.py | 1 - pypeit/tests/test_onespec.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index 87f5e01afe..8b54708841 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -1364,7 +1364,7 @@ def validate(self): raise ValueError("'ex_value' must be one of:\n" + ", ".join(allowed_extensions)) allowed_wave_methods = self.valid_wave_methods() - if self.data['ex_value'] not in allowed_wave_methods: + if self.data['wave_method'] not in allowed_wave_methods: raise ValueError("'wave_method' must be one of:\n" + ", ".join(allowed_wave_methods)) allowed_scale_methods = self.valid_scale_methods() diff --git a/pypeit/specutils/pypeit_loaders.py b/pypeit/specutils/pypeit_loaders.py index 937d2fc796..533e005a85 100644 --- a/pypeit/specutils/pypeit_loaders.py +++ b/pypeit/specutils/pypeit_loaders.py @@ -272,7 +272,6 @@ def pypeit_onespec_loader(filename, grid=False, strict=True, **kwargs): # TODO We should be dealing with masking here. return Spectrum1D(flux=astropy.units.Quantity(spec.flux * flux_unit), uncertainty=None if spec.sigma is None else astropy.units.Quantity(spec.sigma * flux_unit), - # else astropy.nddata.InverseVariance(spec.ivar / flux_unit**2), meta={'name': name, 'extract': spec.ext_mode, 'fluxed': spec.fluxed, 'grid': grid}, spectral_axis=astropy.units.Quantity(wave * astropy.units.angstrom), diff --git a/pypeit/tests/test_onespec.py b/pypeit/tests/test_onespec.py index 02c51747c1..4e9d479fd4 100644 --- a/pypeit/tests/test_onespec.py +++ b/pypeit/tests/test_onespec.py @@ -21,7 +21,7 @@ def test_init(): assert spec.spectrograph is None, 'Spectrograph should not be set' spec = onespec.OneSpec(wave, wave, flux, ivar=2*np.ones_like(flux)) - assert np.allclose(spec.sig, 1/np.sqrt(2)), 'Conversion to sigma is wrong' + #assert np.allclose(spec.sigma, 1/np.sqrt(2)), 'Conversion to sigma is wrong' def test_io(): From d16e5a12b0428e03ae9c6fdc5f6ee13e23abd44f Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 21 Dec 2023 16:33:24 -0800 Subject: [PATCH 139/244] add new writer for SlitTraceSet --- pypeit/edgetrace.py | 4 +- pypeit/slittrace.py | 95 +++++++++++++++++++++++++++++++++++---------- 2 files changed, 77 insertions(+), 22 deletions(-) diff --git a/pypeit/edgetrace.py b/pypeit/edgetrace.py index 684a13d5d8..613f6d4b2f 100644 --- a/pypeit/edgetrace.py +++ b/pypeit/edgetrace.py @@ -1135,7 +1135,7 @@ def _base_header(self, hdr=None): `astropy.io.fits.Header`_: Header object to include in all HDU extensions. """ - _hdr = super(EdgeTraceSet, self)._base_header(hdr=hdr) + _hdr = super()._base_header(hdr=hdr) _hdr['QAPATH'] = 'None' if self.qa_path is None else str(self.qa_path) self.par.to_header(_hdr) self.bitmask.to_header(_hdr) @@ -1259,7 +1259,7 @@ def from_hdu(cls, hdu, hdu_prefix=None, chk_version=True): # parse traceimg because it's not a single-extension # DataContainer. It *will* parse pca, left_pca, and right_pca, # if they exist, but not their model components. - d, version_passed, type_passed, parsed_hdus = super(EdgeTraceSet, cls)._parse(hdu) + d, version_passed, type_passed, parsed_hdus = cls._parse(hdu) if not type_passed: msgs.error('The HDU(s) cannot be parsed by a {0} object!'.format(cls.__name__)) if not version_passed: diff --git a/pypeit/slittrace.py b/pypeit/slittrace.py index 5554ac753b..4ee62dfa0a 100644 --- a/pypeit/slittrace.py +++ b/pypeit/slittrace.py @@ -30,6 +30,7 @@ class SlitTraceBitMask(BitMask): Mask bits used during slit tracing. """ version = '1.0.1' + # TODO: Need a unique bit prefix? def __init__(self): # Only ever append new bits (and don't remove old ones) @@ -83,7 +84,7 @@ class SlitTraceSet(calibframe.CalibFrame): calib_file_format = 'fits.gz' """File format for the calibration frame file.""" - version = '1.1.4' + version = '1.1.5' """SlitTraceSet data model version.""" bitmask = SlitTraceBitMask() @@ -151,7 +152,6 @@ class SlitTraceSet(calibframe.CalibFrame): 'mask': dict(otype=np.ndarray, atype=np.integer, descr='Bit mask for slits (fully good slits have 0 value). Shape ' 'is Nslits.'), - 'slitbitm': dict(otype=str, descr='List of BITMASK keys from SlitTraceBitMask'), 'specmin': dict(otype=np.ndarray, atype=np.floating, descr='Minimum spectral position (pixel units) allowed for each slit/order. ' 'Shape is Nslits.'), @@ -168,7 +168,7 @@ def __init__(self, left_init, right_init, pypeline, detname=None, nspec=None, ns pad=0, spat_id=None, maskdef_id=None, maskdef_designtab=None, maskfile=None, maskdef_posx_pa=None, maskdef_offset=None, maskdef_objpos=None, maskdef_slitcen=None, ech_order=None, nslits=None, left_tweak=None, - right_tweak=None, center=None, mask=None, slitbitm=None): + right_tweak=None, center=None, mask=None): # Instantiate the DataContainer args, _, _, values = inspect.getargvalues(inspect.currentframe()) @@ -240,21 +240,40 @@ def _validate(self): self.mask_init = np.atleast_1d(self.mask_init) self.specmin = np.atleast_1d(self.specmin) self.specmax = np.atleast_1d(self.specmax) - if self.slitbitm is None: - self.slitbitm = ','.join(list(self.bitmask.keys())) - else: - # Validate -- All of the keys must be present and in current order, but new ones can exist - bitms = self.slitbitm.split(',') - curbitm = list(self.bitmask.keys()) - for kk, bit in enumerate(bitms): - if curbitm[kk] != bit: - msgs.error("Input BITMASK keys differ from current data model!") - # Update to current, no matter what - self.slitbitm = ','.join(list(self.bitmask.keys())) +# if self.slitbitm is None: +# self.slitbitm = ','.join(list(self.bitmask.keys())) +# else: +# # Validate -- All of the keys must be present and in current order, but new ones can exist +# bitms = self.slitbitm.split(',') +# curbitm = list(self.bitmask.keys()) +# for kk, bit in enumerate(bitms): +# if curbitm[kk] != bit: +# msgs.error("Input BITMASK keys differ from current data model!") +# # Update to current, no matter what +# self.slitbitm = ','.join(list(self.bitmask.keys())) # Mask if self.mask is None: self.mask = self.mask_init.copy() + def _base_header(self, hdr=None): + """ + Construct the baseline header for all HDU extensions. + + This appends the :class:`SlitTraceBitMask` data to the supplied header. + + Args: + hdr (`astropy.io.fits.Header`_, optional): + Baseline header for additional data. If None, set by + :func:`pypeit.io.initialize_header()`. + + Returns: + `astropy.io.fits.Header`_: Header object to include in + all HDU extensions. + """ + _hdr = super()._base_header(hdr=hdr) + self.bitmask.to_header(_hdr) + return _hdr + def _bundle(self): """ Bundle the data in preparation for writing to a fits file. @@ -298,6 +317,45 @@ def _parse(cls, hdu, hdu_prefix=None, **kwargs): except KeyError: return super()._parse(hdu, ext='SLITS', transpose_table_arrays=True) + @classmethod + def from_hdu(cls, hdu, hdu_prefix=None, chk_version=True): + """ + Instantiate the object from an HDU extension. + + This overrides the base-class method, only to add checks (or not) for + the bitmask. + + Args: + hdu (`astropy.io.fits.HDUList`_, `astropy.io.fits.ImageHDU`_, `astropy.io.fits.BinTableHDU`_): + The HDU(s) with the data to use for instantiation. + hdu_prefix (:obj:`str`, optional): + Maintained for consistency with the base class but is + not used by this method. + chk_version (:obj:`bool`, optional): + If True, raise an error if the datamodel version or + type check failed. If False, throw a warning only. + """ + # Run the default parser + d, version_passed, type_passed, parsed_hdus = cls._parse(hdu) + if not type_passed: + msgs.error('The HDU(s) cannot be parsed by a {0} object!'.format(cls.__name__)) + if not version_passed: + _f = msgs.error if chk_version else msgs.warn + _f('Current version of {0} object in code (v{1})'.format(cls.__name__, cls.version) + + ' does not match version used to write your HDU(s)!') + + # Instantiate + self = super().from_dict(d=d) + + # Check the bitmasks. Bits should have been written to *any* header + # associated with the object + hdr_bitmask = BitMask.from_header(hdu[parsed_hdus[0]].header) + if chk_version and hdr_bitmask.bits != self.bitmask.bits: + msgs.error('The bitmask in this fits file appear to be out of date! Recreate this ' + 'file either by rerunning run_pypeit or pypeit_trace_edges.') + + return self + def init_tweaked(self): """ Initialize the tweaked slits. @@ -383,10 +441,9 @@ def slitord_to_zero(self, slitord): """ if self.pypeline in ['MultiSlit', 'SlicerIFU']: return np.where(self.spat_id == slitord)[0][0] - elif self.pypeline in ['Echelle']: + if self.pypeline in ['Echelle']: return np.where(self.ech_order == slitord)[0][0] - else: - msgs.error('Unrecognized Pypeline {:}'.format(self.pypeline)) + msgs.error('Unrecognized Pypeline {:}'.format(self.pypeline)) def get_slitlengths(self, initial=False, median=False): """ @@ -412,9 +469,7 @@ def get_slitlengths(self, initial=False, median=False): """ left, right, _ = self.select_edges(initial=initial) slitlen = right - left - if median is True: - slitlen = np.median(slitlen, axis=1) - return slitlen + return np.median(slitlen, axis=1) if median else slitlen def get_radec_image(self, wcs, alignSplines, tilts, initial=True, flexure=None): """Generate an RA and DEC image for every pixel in the frame From 20b2351f752e30ac9cb1a9608cfd5647b7708592 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 21 Dec 2023 16:50:17 -0800 Subject: [PATCH 140/244] test fix --- pypeit/specutils/pypeit_loaders.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pypeit/specutils/pypeit_loaders.py b/pypeit/specutils/pypeit_loaders.py index 533e005a85..f500574337 100644 --- a/pypeit/specutils/pypeit_loaders.py +++ b/pypeit/specutils/pypeit_loaders.py @@ -36,6 +36,7 @@ from pypeit import msgs from pypeit import specobjs from pypeit import onespec +from pypeit import utils def _enforce_monotonic_wavelengths(wave, flux, ivar, strict=True): @@ -49,7 +50,9 @@ def _enforce_monotonic_wavelengths(wave, flux, ivar, strict=True): flux : `numpy.ndarray`_ Spectrum flux ivar : `numpy.ndarray`_ - Spectrum inverse variance. Can be None. + Spectrum inverse variance. Can be None or the standard deviation. The + only operation on this and the ``flux`` vector is to downselect the + monotonically increasing values. strict : bool, optional Check that the wavelength vector is monotonically increasing. If not, raise an error (as would be done by the `specutils.SpectrumList`_ class). @@ -202,10 +205,11 @@ def pypeit_spec1d_loader(filename, extract=None, fluxed=True, strict=True, **kwa continue _wave, _flux, _ivar = _enforce_monotonic_wavelengths(_wave[_gpm], _flux[_gpm], _ivar[_gpm], strict=strict) + _sigma = np.sqrt(utils.inverse(_ivar)) flux_unit = astropy.units.Unit("1e-17 erg/(s cm^2 Angstrom)" if _cal else "electron") spec += \ [Spectrum1D(flux=astropy.units.Quantity(_flux * flux_unit), - uncertainty=astropy.nddata.InverseVariance(_ivar / flux_unit**2), + uncertainty=astropy.nddata.StdDevUncertainty(_sigma * flux_unit), meta={'name': sobj.NAME, 'extract': _ext, 'fluxed': _cal}, spectral_axis=astropy.units.Quantity(_wave * astropy.units.angstrom), velocity_convention="doppler_optical", @@ -256,7 +260,7 @@ def pypeit_onespec_loader(filename, grid=False, strict=True, **kwargs): flux_unit = astropy.units.Unit("1e-17 erg/(s cm^2 Angstrom)" if spec.fluxed else "ct/s") wave = spec.wave_grid_mid if grid else spec.wave - wave, flux, ivar = _enforce_monotonic_wavelengths(wave, spec.flux, spec.ivar, strict=strict) + wave, flux, sigma = _enforce_monotonic_wavelengths(wave, spec.flux, spec.sigma, strict=strict) # If the input filename is actually a string, assign it as the spectrum # name. Otherwise, try assuming it's a _io.FileIO object, and if that @@ -270,8 +274,9 @@ def pypeit_onespec_loader(filename, grid=False, strict=True, **kwargs): name = '' # TODO We should be dealing with masking here. - return Spectrum1D(flux=astropy.units.Quantity(spec.flux * flux_unit), - uncertainty=None if spec.sigma is None else astropy.units.Quantity(spec.sigma * flux_unit), + return Spectrum1D(flux=astropy.units.Quantity(flux * flux_unit), + uncertainty=None if spec.sigma is None + else astropy.nddata.StdDevUncertainty(sigma * flux_unit), meta={'name': name, 'extract': spec.ext_mode, 'fluxed': spec.fluxed, 'grid': grid}, spectral_axis=astropy.units.Quantity(wave * astropy.units.angstrom), From 58951df2dc5ccb239b4839449c754cae7c138c39 Mon Sep 17 00:00:00 2001 From: jhennawi Date: Thu, 21 Dec 2023 16:52:44 -0800 Subject: [PATCH 141/244] Fixed test failures. --- pypeit/core/datacube.py | 64 +++++++++++++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 8 deletions(-) diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index a788ca51d8..e2018a3d8c 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -864,7 +864,7 @@ def generate_WCS(crval, cdelt, equinox=2000.0, name="PYP_SPEC"): def compute_weights_frompix(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, dspat, dwv, mnmx_wv, all_wghts, all_spatpos, all_specpos, all_spatid, all_tilts, all_slits, all_align, all_dar, ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None, - sn_smooth_npix=None, relative_weights=False, reference_image=None, whitelight_range=None, + sn_smooth_npix=None, weight_method='autod', reference_image=None, whitelight_range=None, specname="PYPSPEC"): r""" Calculate wavelength dependent optimal weights. The weighting is currently @@ -940,8 +940,32 @@ def compute_weights_frompix(all_ra, all_dec, all_wave, all_sci, all_ivar, all_id Number of pixels used for determining smoothly varying S/N ratio weights. This is currently not required, since a relative weighting scheme with a polynomial fit is used to calculate the S/N weights. - relative_weights (bool, optional): - Calculate weights by fitting to the ratio of spectra? + weight_method: (`str`, optional) + Weight method to be used in `coadd.sn_weights`. Default is 'auto'. + Options are 'auto', 'constant', 'relative', or 'ivar'. The defaulti is'auto'. + Behavior is as follows: + 'auto': + Use constant weights if rms_sn < 3.0, otherwise use wavelength dependent. + 'constant': + Constant weights based on rms_sn**2 + 'uniform': + Uniform weighting. + 'wave_dependent': + Wavelength dependent weights will be used irrespective of the rms_sn ratio. This option + will not work well at low S/N ratio although it is useful for objects where only a small + fraction of the spectral coverage has high S/N ratio (like high-z quasars). + 'relative': + Calculate weights by fitting to the ratio of spectra? Note, relative + weighting will only work well when there is at least one spectrum with a + reasonable S/N, and a continuum. RJC note - This argument may only be + better when the object being used has a strong continuum + emission + lines. The reference spectrum is assigned a value of 1 for all + wavelengths, and the weights of all other spectra will be determined + relative to the reference spectrum. This is particularly useful if you + are dealing with highly variable spectra (e.g. emission lines) and + require a precision better than ~1 per cent. + 'ivar': + Use inverse variance weighting. This is not well tested and should probably be deprecated. reference_image (`numpy.ndarray`_): Reference image to use for the determination of the highest S/N spaxel in the image. specname (str): @@ -972,11 +996,11 @@ def compute_weights_frompix(all_ra, all_dec, all_wave, all_sci, all_ivar, all_id voxedge, all_idx=all_idx, spec_subpixel=1, spat_subpixel=1, combine=True) # Compute the weights return compute_weights(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, wl_full[:, :, 0], dspat, dwv, - sn_smooth_npix=sn_smooth_npix, relative_weights=relative_weights) + sn_smooth_npix=sn_smooth_npix, weight_method=weight_method) def compute_weights(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, whitelight_img, dspat, dwv, - sn_smooth_npix=None, relative_weights=False): + sn_smooth_npix=None, weight_method='auto'): r""" Calculate wavelength dependent optimal weights. The weighting is currently based on a relative :math:`(S/N)^2` at each wavelength @@ -1014,8 +1038,32 @@ def compute_weights(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, white Number of pixels used for determining smoothly varying S/N ratio weights. This is currently not required, since a relative weighting scheme with a polynomial fit is used to calculate the S/N weights. - relative_weights (bool, optional): - Calculate weights by fitting to the ratio of spectra? + weight_method: (`str`, optional) + Weight method to be used in `coadd.sn_weights`. Default is 'auto'. + Options are 'auto', 'constant', 'relative', or 'ivar'. The defaulti is'auto'. + Behavior is as follows: + 'auto': + Use constant weights if rms_sn < 3.0, otherwise use wavelength dependent. + 'constant': + Constant weights based on rms_sn**2 + 'uniform': + Uniform weighting. + 'wave_dependent': + Wavelength dependent weights will be used irrespective of the rms_sn ratio. This option + will not work well at low S/N ratio although it is useful for objects where only a small + fraction of the spectral coverage has high S/N ratio (like high-z quasars). + 'relative': + Calculate weights by fitting to the ratio of spectra? Note, relative + weighting will only work well when there is at least one spectrum with a + reasonable S/N, and a continuum. RJC note - This argument may only be + better when the object being used has a strong continuum + emission + lines. The reference spectrum is assigned a value of 1 for all + wavelengths, and the weights of all other spectra will be determined + relative to the reference spectrum. This is particularly useful if you + are dealing with highly variable spectra (e.g. emission lines) and + require a precision better than ~1 per cent. + 'ivar': + Use inverse variance weighting. This is not well tested and should probably be deprecated. Returns: `numpy.ndarray`_ : a 1D array the same size as all_sci, containing @@ -1067,7 +1115,7 @@ def compute_weights(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, white if sn_smooth_npix is None: sn_smooth_npix = int(np.round(0.1 * wave_spec.size)) rms_sn, weights = coadd.sn_weights(utils.array_to_explist(flux_stack), utils.array_to_explist(ivar_stack), utils.array_to_explist(mask_stack), - sn_smooth_npix=sn_smooth_npix, relative_weights=relative_weights) + sn_smooth_npix=sn_smooth_npix, weight_method=weight_method) # Because we pass back a weights array, we need to interpolate to assign each detector pixel a weight all_wghts = np.ones(all_idx.size) From 4810e50acffc6e4bd5f50ad3ed8485dc6f158525 Mon Sep 17 00:00:00 2001 From: rcooke Date: Tue, 26 Dec 2023 14:37:12 +0000 Subject: [PATCH 142/244] typo --- pypeit/core/procimg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/core/procimg.py b/pypeit/core/procimg.py index 0f20a719b8..d0fdae48c9 100644 --- a/pypeit/core/procimg.py +++ b/pypeit/core/procimg.py @@ -832,7 +832,7 @@ def subtract_pattern(rawframe, datasec_img, oscansec_img, frequency=None, axis=1 # Convert result to amplitude and phase amps = (np.abs(tmpamp))[idx] * (2.0 / overscan.shape[1]) - # STEP 2 - Using th emodel frequency, calculate how amplitude depends on pixel row (usually constant) + # STEP 2 - Using the model frequency, calculate how amplitude depends on pixel row (usually constant) # Use the above to as initial guess parameters for a chi-squared minimisation of the amplitudes msgs.info("Measuring amplitude-pixel dependence of amplifier {0:d}".format(amp)) nspec = overscan.shape[0] From bd5ae1708de894239c6384891db5fcc71bd42e21 Mon Sep 17 00:00:00 2001 From: rcooke Date: Tue, 26 Dec 2023 14:40:06 +0000 Subject: [PATCH 143/244] nonlinear todo --- pypeit/images/rawimage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pypeit/images/rawimage.py b/pypeit/images/rawimage.py index 853c96f8bc..6b8555252b 100644 --- a/pypeit/images/rawimage.py +++ b/pypeit/images/rawimage.py @@ -612,6 +612,7 @@ def process(self, par, bpm=None, scattlight=None, flatimages=None, bias=None, sl self.subtract_bias(bias) # TODO: Checking for count (well-depth) saturation should be done here. + # TODO :: Non-linearity correction should be done here. # - Create the dark current image(s). The dark-current image *always* # includes the tabulated dark current and the call below ensures From bafd2264393fdb27c99bbad50dcc72e1678a6435 Mon Sep 17 00:00:00 2001 From: jhennawi Date: Wed, 27 Dec 2023 17:45:48 -0800 Subject: [PATCH 144/244] Changed output in calc_snr and fixed a bug in coadd1d. --- pypeit/core/coadd.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pypeit/core/coadd.py b/pypeit/core/coadd.py index 0becda7d61..c34ad25c74 100644 --- a/pypeit/core/coadd.py +++ b/pypeit/core/coadd.py @@ -771,8 +771,8 @@ def calc_snr(fluxes, ivars, gpms): Returns ------- - rms_sn : list - List of length nexp root-mean-square S/N value for each input spectra where nexp=len(fluxes). + rms_sn : `numpy.ndarray`_ + Array of shape (nexp,) of root-mean-square S/N value for each input spectra where nexp=len(fluxes). sn_val : list List of length nexp containing the wavelength dependent S/N arrays for each input spectrum, i.e. each element contains the array flux*sqrt(ivar) @@ -782,10 +782,10 @@ def calc_snr(fluxes, ivars, gpms): # Calculate S/N sn_val, rms_sn, sn2 = [], [], [] - for iexp in range(nexp): - sn_val_iexp = fluxes[iexp]*np.sqrt(ivars[iexp]) + for iexp, (flux, ivar, gpm) in enumerate(zip(fluxes, ivars, gpms)): + sn_val_iexp = flux*np.sqrt(ivar) sn_val.append(sn_val_iexp) - sn_val_ma = np.ma.array(sn_val_iexp, mask=np.logical_not(gpms[iexp])) + sn_val_ma = np.ma.array(sn_val_iexp, mask=np.logical_not(gpm)) sn_sigclip = stats.sigma_clip(sn_val_ma, sigma=3, maxiters=5) sn2_iexp = sn_sigclip.mean()**2 # S/N^2 value for each spectrum if sn2_iexp is np.ma.masked: @@ -794,7 +794,7 @@ def calc_snr(fluxes, ivars, gpms): sn2.append(sn2_iexp) rms_sn.append(np.sqrt(sn2_iexp)) # Root Mean S/N**2 value for all spectra - return rms_sn, sn_val + return np.array(rms_sn), sn_val def sn_weights(fluxes, ivars, gpms, sn_smooth_npix=None, weight_method='auto', verbose=False): From eba6afa689c03604e45b1bf025f1958c078754c4 Mon Sep 17 00:00:00 2001 From: jhennawi Date: Wed, 27 Dec 2023 18:11:50 -0800 Subject: [PATCH 145/244] Addressed PR comments and fixed some tests. --- pypeit/coadd1d.py | 1 - pypeit/core/datacube.py | 8 ++++---- pypeit/par/pypeitpar.py | 39 +++++++++++++++++++++++++++++++-------- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/pypeit/coadd1d.py b/pypeit/coadd1d.py index aba3360810..1ac0a1d255 100644 --- a/pypeit/coadd1d.py +++ b/pypeit/coadd1d.py @@ -125,7 +125,6 @@ def save(self, coaddfile, telluric=None, obj_model=None, overwrite=True): Overwrite existing file? """ self.coaddfile = coaddfile - #wave_gpm = self.wave_grid_mid > 1.0 # Generate the spectrum container object onespec = OneSpec(wave=self.wave_coadd, wave_grid_mid=self.wave_grid_mid, flux=self.flux_coadd, PYP_SPEC=self.spectrograph.name, ivar=self.ivar_coadd, diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index e2018a3d8c..f754f6fc48 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -665,7 +665,7 @@ def set_voxel_sampling(spatscale, specscale, dspat=None, dwv=None): def wcs_bounds(all_ra, all_dec, all_wave, ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None): """ - Create a WCS and the expected edges of the voxels, based on user-specified + Compute the bounds of the WCS and the expected edges of the voxels, based on user-specified parameters or the extremities of the data. This is a convenience function that calls the core function in `pypeit.core.datacube`_. @@ -864,7 +864,7 @@ def generate_WCS(crval, cdelt, equinox=2000.0, name="PYP_SPEC"): def compute_weights_frompix(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, dspat, dwv, mnmx_wv, all_wghts, all_spatpos, all_specpos, all_spatid, all_tilts, all_slits, all_align, all_dar, ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None, - sn_smooth_npix=None, weight_method='autod', reference_image=None, whitelight_range=None, + sn_smooth_npix=None, weight_method='auto', reference_image=None, whitelight_range=None, specname="PYPSPEC"): r""" Calculate wavelength dependent optimal weights. The weighting is currently @@ -942,7 +942,7 @@ def compute_weights_frompix(all_ra, all_dec, all_wave, all_sci, all_ivar, all_id scheme with a polynomial fit is used to calculate the S/N weights. weight_method: (`str`, optional) Weight method to be used in `coadd.sn_weights`. Default is 'auto'. - Options are 'auto', 'constant', 'relative', or 'ivar'. The defaulti is'auto'. + Options are 'auto', 'constant', 'uniform', 'wave_dependent', 'relative', or 'ivar'. The defaulti is 'auto'. Behavior is as follows: 'auto': Use constant weights if rms_sn < 3.0, otherwise use wavelength dependent. @@ -1040,7 +1040,7 @@ def compute_weights(all_ra, all_dec, all_wave, all_sci, all_ivar, all_idx, white scheme with a polynomial fit is used to calculate the S/N weights. weight_method: (`str`, optional) Weight method to be used in `coadd.sn_weights`. Default is 'auto'. - Options are 'auto', 'constant', 'relative', or 'ivar'. The defaulti is'auto'. + Options are 'auto', 'constant', 'uniform', 'wave_dependent', 'relative', or 'ivar'. The defaulti is 'auto'. Behavior is as follows: 'auto': Use constant weights if rms_sn < 3.0, otherwise use wavelength dependent. diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index 8b54708841..44f88f1888 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -1250,7 +1250,7 @@ def __init__(self, ex_value=None, flux_value=None, nmaskedge=None, defaults['weight_method'] = 'auto' options['weight_method'] = Coadd1DPar.valid_weight_methods() dtypes['weight_method'] = str - descr['weight_method'] = "Method used to rescale the spectra prior to coadding. The options are:" \ + descr['weight_method'] = "Method used to weight the spectra for coadding. The options are:" \ " " \ "'auto' -- Use constant weights if rms_sn < 3.0, otherwise use wavelength dependent." \ "'constant' -- Constant weights based on rms_sn**2" \ @@ -1549,7 +1549,7 @@ class CubePar(ParSet): see :ref:`parameters`. """ - def __init__(self, slit_spec=None, relative_weights=None, align=None, combine=None, output_filename=None, + def __init__(self, slit_spec=None, weight_method=None, align=None, combine=None, output_filename=None, standard_cube=None, reference_image=None, save_whitelight=None, whitelight_range=None, method=None, ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None, spatial_delta=None, wave_delta=None, astrometric=None, grating_corr=None, scale_corr=None, @@ -1576,12 +1576,35 @@ def __init__(self, slit_spec=None, relative_weights=None, align=None, combine=No descr['slit_spec'] = 'If the data use slits in one spatial direction, set this to True. ' \ 'If the data uses fibres for all spaxels, set this to False.' - defaults['relative_weights'] = False - dtypes['relative_weights'] = [bool] - descr['relative_weights'] = 'If set to True, the combined frames will use a relative weighting scheme. ' \ - 'This only works well if there is a common continuum source in the field of ' \ - 'view of all input observations, and is generally only required if high ' \ - 'relative precision is desired.' + defaults['weight_method'] = 'auto' + options['weight_method'] = Coadd1DPar.valid_weight_methods() + dtypes['weight_method'] = str + descr['weight_method'] = "Method used to weight the spectra for coadding. The options are:" \ + " " \ + "'auto' -- Use constant weights if rms_sn < 3.0, otherwise use wavelength dependent." \ + "'constant' -- Constant weights based on rms_sn**2" \ + "'uniform' -- Uniform weighting" \ + "'wave_dependent' -- Wavelength dependent weights will be used irrespective of the rms_" \ + "sn ratio. This option will not work well at low S/N ratio although it is useful for " \ + "objects where only a small fraction of the spectral coverage has high S/N ratio " \ + "(like high-z quasars)." \ + "'relative' -- Apply relative weights implying one reference exposure will receive unit " \ + "weight at all wavelengths and all others receive relatively wavelength dependent "\ + "weights . Note, relative weighting will only work well " \ + "when there is at least one spectrum with a reasonable S/N, and a continuum. " \ + "This option may only be better when the object being used has a strong " \ + "continuum + emission lines. This is particularly useful if you " \ + "are dealing with highly variable spectra (e.g. emission lines) and" \ + "require a precision better than ~1 per cent." \ + "'ivar' -- Use inverse variance weighting. This is not well tested and should probably be deprecated." + + + #defaults['relative_weights'] = False + #dtypes['relative_weights'] = [bool] + #descr['relative_weights'] = 'If set to True, the combined frames will use a relative weighting scheme. ' \ + # 'This only works well if there is a common continuum source in the field of ' \ + # 'view of all input observations, and is generally only required if high ' \ + # 'relative precision is desired.' defaults['align'] = False dtypes['align'] = [bool] From 6003eaac027d941ba18fa37cb1f4dc48e3777b8a Mon Sep 17 00:00:00 2001 From: jhennawi Date: Wed, 27 Dec 2023 22:10:33 -0800 Subject: [PATCH 146/244] Fixed bug in coadd3d introduced by this PR. --- pypeit/coadd3d.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index 3119b8edd6..7801f42aaa 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -1272,7 +1272,7 @@ def compute_weights(self): ra_min=self.cubepar['ra_min'], ra_max=self.cubepar['ra_max'], dec_min=self.cubepar['dec_min'], dec_max=self.cubepar['dec_max'], wave_min=self.cubepar['wave_min'], wave_max=self.cubepar['wave_max'], - relative_weights=self.cubepar['relative_weights'], + weight_method=self.cubepar['weight_method'], whitelight_range=self.cubepar['whitelight_range'], reference_image=self.cubepar['reference_image'], specname=self.specname) From 351d4ee2a3e11310ef74cbf335c011902d77d9e8 Mon Sep 17 00:00:00 2001 From: jhennawi Date: Thu, 28 Dec 2023 00:14:23 -0800 Subject: [PATCH 147/244] Fixed bug in par and added validation. --- pypeit/par/pypeitpar.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index 44f88f1888..c2504d8b61 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -1372,7 +1372,7 @@ def validate(self): raise ValueError("'scale_method' must be one of:\n" + ", ".join(allowed_scale_methods)) allowed_weight_methods = self.valid_weight_methods() - if self.data['weight_method'] not in allowed_scale_methods: + if self.data['weight_method'] not in allowed_weight_methods: raise ValueError("'weight_method' must be one of:\n" + ", ".join(allowed_weight_methods)) @@ -1804,6 +1804,11 @@ def validate(self): if len(self.data['whitelight_range']) != 2: raise ValueError("The 'whitelight_range' must be a two element list of either NoneType or float") + allowed_weight_methods = Coadd1DPar.valid_weight_methods() + if self.data['weight_method'] not in allowed_weight_methods: + raise ValueError("'weight_method' must be one of:\n" + ", ".join(allowed_weight_methods)) + + class FluxCalibratePar(ParSet): """ From 356e9ab156fe7b0b010c47485833d4493c586ec7 Mon Sep 17 00:00:00 2001 From: jhennawi Date: Thu, 28 Dec 2023 00:15:38 -0800 Subject: [PATCH 148/244] Fixed bug in par and added validation. --- pypeit/par/pypeitpar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index c2504d8b61..fdf07656e4 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -1781,7 +1781,7 @@ def from_dict(cls, cfg): # Basic keywords parkeys = ['slit_spec', 'output_filename', 'standard_cube', 'reference_image', 'save_whitelight', 'method', 'spec_subpixel', 'spat_subpixel', 'ra_min', 'ra_max', 'dec_min', 'dec_max', - 'wave_min', 'wave_max', 'spatial_delta', 'wave_delta', 'relative_weights', 'align', 'combine', + 'wave_min', 'wave_max', 'spatial_delta', 'wave_delta', 'weight_method', 'align', 'combine', 'astrometric', 'grating_corr', 'scale_corr', 'skysub_frame', 'whitelight_range'] badkeys = np.array([pk not in parkeys for pk in k]) From 84c3e3863b9306109ea2a62a3ca8bf439380b6f7 Mon Sep 17 00:00:00 2001 From: rcooke Date: Thu, 28 Dec 2023 13:20:47 +0000 Subject: [PATCH 149/244] cleanup --- pypeit/core/procimg.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pypeit/core/procimg.py b/pypeit/core/procimg.py index d0fdae48c9..b006837424 100644 --- a/pypeit/core/procimg.py +++ b/pypeit/core/procimg.py @@ -705,7 +705,6 @@ def subtract_overscan(rawframe, datasec_img, oscansec_img, method='savgol', para no_overscan[data_slice][:,0::2] -= even[:,None] else: msgs.error('Not ready for this approach, please contact the Developers') - # Subtract along the appropriate axis no_overscan[data_slice] -= (ossub[:, None] if compress_axis == 1 else ossub[None, :]) From 35e50731c4e85311cf2d578e5cc7c1133e573d46 Mon Sep 17 00:00:00 2001 From: rcooke Date: Thu, 28 Dec 2023 19:38:08 +0000 Subject: [PATCH 150/244] add docs --- pypeit/core/wavecal/autoid.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pypeit/core/wavecal/autoid.py b/pypeit/core/wavecal/autoid.py index d9a2cfcfb7..02a56a1735 100644 --- a/pypeit/core/wavecal/autoid.py +++ b/pypeit/core/wavecal/autoid.py @@ -52,7 +52,7 @@ def arc_fit_qa(waveFit, outfile (:obj:`str`, optional): Name of output file or 'show' to show on screen ids_only (bool, optional): - ?? + Only show the main panel with the arc spectrum and the identified lines title (:obj:`str`, optional): Add a title to the spectrum plot log (:obj:`bool`, optional): @@ -142,6 +142,8 @@ def arc_fit_qa(waveFit, # Title if title is not None: fig.suptitle(title, fontsize='x-large', va='top') + + # If we're only plotting the ID panel, save the figure and return if ids_only: plt.tight_layout(pad=0.2, h_pad=0.0, w_pad=0.0) if outfile is None: From c90f84a431e15491ebaf1a22c2254d198c2fa9c3 Mon Sep 17 00:00:00 2001 From: rcooke Date: Sat, 30 Dec 2023 17:01:35 +0000 Subject: [PATCH 151/244] fix QA title --- pypeit/flatfield.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/flatfield.py b/pypeit/flatfield.py index a0a3a4cf32..0ce5a018e2 100644 --- a/pypeit/flatfield.py +++ b/pypeit/flatfield.py @@ -1747,7 +1747,7 @@ def detector_structure_qa(det_resp, det_resp_model, outfile=None, title="Detecto # Axes showing the residual of the detector response fit ax_resd = plt.subplot(gs[2]) ax_resd.imshow(det_resp-det_resp_model, origin='lower', vmin=vmin-1, vmax=vmax-1) - ax_resd.set_xlabel("data-model", fontsize='medium') + ax_resd.set_xlabel("1+data-model", fontsize='medium') ax_resd.axes.xaxis.set_ticks([]) ax_resd.axes.yaxis.set_ticks([]) # Add a colorbar From 242f16c48b8ef00d92598a1b3164524d0fff716a Mon Sep 17 00:00:00 2001 From: rcooke Date: Sat, 30 Dec 2023 17:01:53 +0000 Subject: [PATCH 152/244] reduce FWHM slit range --- pypeit/core/wavecal/autoid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/core/wavecal/autoid.py b/pypeit/core/wavecal/autoid.py index 02a56a1735..fcd00cd5d2 100644 --- a/pypeit/core/wavecal/autoid.py +++ b/pypeit/core/wavecal/autoid.py @@ -913,7 +913,7 @@ def map_fwhm(image, gpm, slits_left, slits_right, slitmask, npixel=None, nsample msgs.info(f"Calculating spectral resolution of slit {sl + 1}/{nslits}") # Fraction along the slit in the spatial direction to sample the arc line width nmeas = int(0.5+slit_lengths[sl]/_npixel) if nsample is None else nsample - slitsamp = np.linspace(0.01, 0.99, nmeas) + slitsamp = np.linspace(0.05, 0.95, nmeas) this_samp, this_cent, this_fwhm = np.array([]), np.array([]), np.array([]) for ss in range(nmeas): spat_vec = np.atleast_2d((1-slitsamp[ss]) * slits_left[:, sl] + slitsamp[ss] * slits_right[:, sl]).T From c85609fa9ace007b6a321c33cd93fbaee5820ee0 Mon Sep 17 00:00:00 2001 From: rcooke Date: Wed, 3 Jan 2024 21:30:37 +0000 Subject: [PATCH 153/244] medfilt finecorr --- pypeit/flatfield.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pypeit/flatfield.py b/pypeit/flatfield.py index 0ce5a018e2..02f9624d0d 100644 --- a/pypeit/flatfield.py +++ b/pypeit/flatfield.py @@ -8,7 +8,7 @@ import inspect import numpy as np -from scipy import interpolate +from scipy import interpolate, ndimage from matplotlib import pyplot as plt from matplotlib import gridspec @@ -1628,6 +1628,9 @@ def spatillum_finecorr_qa(normed, finecorr, left, right, ypos, cut, outfile=None fcor_cut = finecorr[xmn:xmx, ymn:ymx] vmin, vmax = max(0.95, np.min(fcor_cut)), min(1.05, np.max(fcor_cut)) # Show maximum corrections of ~5% + # For display/visual purposes, apply a median filter to the data + norm_cut = ndimage.median_filter(norm_cut, size=(normed.shape[0]//100, 5)) + # Plot fighght = 8.5 cutrat = fighght*norm_cut.shape[1]/norm_cut.shape[0] From bb68d670a50a0b3d5398f0620f6654acc0ffc330 Mon Sep 17 00:00:00 2001 From: rcooke Date: Wed, 3 Jan 2024 21:31:24 +0000 Subject: [PATCH 154/244] add finecorr method --- pypeit/par/pypeitpar.py | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index b16fde59f0..966ccc11c8 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -1019,7 +1019,7 @@ class ScatteredLightPar(ParSet): see :ref:`parameters`. """ - def __init__(self, method=None, finecorr=None, finecorr_pad=None, finecorr_order=None, finecorr_mask=None): + def __init__(self, method=None, finecorr_method=None, finecorr_pad=None, finecorr_order=None, finecorr_mask=None): # Grab the parameter names and values from the function # arguments @@ -1040,7 +1040,7 @@ def __init__(self, method=None, finecorr=None, finecorr_pad=None, finecorr_order options['method'] = ScatteredLightPar.valid_scattlight_methods() dtypes['method'] = str descr['method'] = 'Method used to fit the overscan. ' \ - 'Options are: {0}'.format(', '.join(options['method'])) + '.' + \ + 'Options are: {0}'.format(', '.join(options['method'])) + '. ' + \ '\'model\' will the scattered light model parameters derived from a ' \ 'user-specified frame during their reduction (note, you will need to make sure ' \ 'that you set appropriate scattlight frames in your .pypeit file for this option). ' \ @@ -1049,15 +1049,21 @@ def __init__(self, method=None, finecorr=None, finecorr_pad=None, finecorr_order '\'archive\' will use an archival model parameter solution for the scattered ' \ 'light (note that this option is not currently available for all spectrographs).' - defaults['finecorr'] = True - dtypes['finecorr'] = bool - descr['finecorr'] = 'If True, a fine correction to the scattered light will be performed. However, the ' \ - 'fine correction will only be applied if the model/frame/archive correction is performed.' - - defaults['finecorr_pad'] = 2 + defaults['finecorr_method'] = None + options['finecorr_method'] = ScatteredLightPar.valid_finecorr_scattlight_methods() + dtypes['finecorr_method'] = str + descr['finecorr_method'] = 'If None, a fine correction to the scattered light will not be performed. ' \ + 'Otherwise, the allowed methods include: ' \ + '{0}'.format(', '.join(options['finecorr_method'])) + '. ' + \ + '\'median\' will subtract a constant value from an entire CCD row, based on a ' \ + 'median of the pixels that are not on slits (see also, \'finecorr_pad\'). ' \ + '\'poly\' will fit a polynomial to the scattered light in each row, based ' \ + 'on the pixels that are not on slits (see also, \'finecorr_pad\').' + + defaults['finecorr_pad'] = 4 dtypes['finecorr_pad'] = int - descr['finecorr_pad'] = 'Number of unbinned pixels to extend the slit edges by when masking the slits for the' \ - 'fine correction to the scattered light.' + descr['finecorr_pad'] = 'Number of unbinned pixels to extend the slit edges by when masking the slits for ' \ + 'the fine correction to the scattered light.' defaults['finecorr_order'] = 2 dtypes['finecorr_order'] = int @@ -1086,7 +1092,7 @@ def __init__(self, method=None, finecorr=None, finecorr_pad=None, finecorr_order @classmethod def from_dict(cls, cfg): k = np.array([*cfg.keys()]) - parkeys = ['method', 'finecorr', 'finecorr_pad', 'finecorr_order', 'finecorr_mask'] + parkeys = ['method', 'finecorr_method', 'finecorr_pad', 'finecorr_order', 'finecorr_mask'] badkeys = np.array([pk not in parkeys for pk in k]) if np.any(badkeys): @@ -1101,7 +1107,10 @@ def validate(self): """ Check the parameters are valid for the provided method. """ - pass + if self.data['method'] is not None and self.data['method'] not in self.valid_scattlight_methods(): + raise ValueError("If 'method' is not None it must be one of:\n"+", ".join(self.valid_scattlight_methods())) + if self.data['finecorr_method'] is not None and self.data['finecorr_method'] not in self.valid_finecorr_scattlight_methods(): + raise ValueError("If 'finecorr_method' is not None it must be one of:\n"+", ".join(self.valid_finecorr_scattlight_methods())) @staticmethod def valid_scattlight_methods(): @@ -1110,6 +1119,13 @@ def valid_scattlight_methods(): """ return ['model', 'frame', 'archive'] + @staticmethod + def valid_finecorr_scattlight_methods(): + """ + Return the valid scattered light methods. + """ + return ['median', 'poly'] + class Coadd1DPar(ParSet): """ From 9c188dc17facee99996c11e8ce541b8a9c3959be Mon Sep 17 00:00:00 2001 From: rcooke Date: Wed, 3 Jan 2024 21:33:46 +0000 Subject: [PATCH 155/244] pad scattlight model --- pypeit/core/scattlight.py | 118 +++++++++++++++++++++++++++++++------- pypeit/scattlight.py | 2 +- 2 files changed, 97 insertions(+), 23 deletions(-) diff --git a/pypeit/core/scattlight.py b/pypeit/core/scattlight.py index 03376ed88d..a8fc336598 100644 --- a/pypeit/core/scattlight.py +++ b/pypeit/core/scattlight.py @@ -12,6 +12,68 @@ from pypeit import msgs, utils +def pad_frame(_frame, detpad=300): + """ + Clean the edges of the input frame and then pad the frame to avoid edge effects. + + Parameters + ---------- + _frame : `numpy.ndarray`_ + Frame to be padded + detpad : int + Number of pixels to pad the frame on each side + + Returns + ------- + _frame_pad : `numpy.ndarray`_ + Padded frame + """ + _frame[0, :] = np.median(_frame[0:10, :], axis=0) + _frame[-1, :] = np.median(_frame[-10:, :], axis=0) + return np.pad(_frame, detpad, mode='edge') # Model should be generated on padded data + + +def scattered_light_model_pad(param, img, detpad=300): + """ + Construct a scattered light model for the input image, with the model parameters + defined by param. This function is used to generate a model of the scattered light, + based on a set of model parameters that have first been optimized using scattered_light(). + The model is generated on a padded version of the input image, and then trimmed to + match the input image size. + + Parameters + ---------- + param : `numpy.ndarray`_ + Model parameters that determine the scattered light based on the input img. + Here is a list of the individual parameter meanings: + + # TODO :: I think this is out of date... + * param[0] = Gaussian kernel width in the spectral direction + * param[1] = Gaussian kernel width in the spatial direction + * param[2] = Lorentzian kernel width in the spectral direction + * param[3] = Lorentzian kernel width in the spatial direction + * param[4] = Pixel shift of the scattered light in the spectral direction + * param[5] = Pixel shift of the scattered light in the spatial direction + * param[6] = Zoom factor of the scattered light (~1) + * param[7] = Kernel angle + * param[8] = Relative importance of Gaussian vs Lorentzian. + 0 < value < 1 means Lorentzian is weighted more + value > 1 means Gaussian is weighted more. + * param[9:] = Polynomial scaling coefficients + img : `numpy.ndarray`_ + Image used to generate the scattered light model + detpad : int + Number of pixels to pad the frame on each side + + Returns + ------- + model : `numpy.ndarray`_ + Model of the scattered light for the input + """ + _frame_pad = pad_frame(img, detpad=detpad) + return scattered_light_model(param, _frame_pad)[detpad:-detpad, detpad:-detpad] + + def scattered_light_model(param, img): """ Model used to calculate the scattered light. @@ -27,6 +89,7 @@ def scattered_light_model(param, img): Model parameters that determine the scattered light based on the input img. Here is a list of the individual parameter meanings: + # TODO :: I think this is out of date... * param[0] = Gaussian kernel width in the spectral direction * param[1] = Gaussian kernel width in the spatial direction * param[2] = Lorentzian kernel width in the spectral direction @@ -151,9 +214,7 @@ def scattered_light(frame, bpm, offslitmask, x0, bounds, detpad=300, debug=False _frame = utils.replace_bad(frame, bpm) # Pad the edges of the data - _frame[0, :] = np.median(_frame[0:10, :], axis=0) - _frame[-1, :] = np.median(_frame[-10:, :], axis=0) - _frame_pad = np.pad(_frame, detpad, mode='edge') # Model should be generated on padded data + _frame_pad = pad_frame(_frame, detpad) offslitmask_pad = np.pad(offslitmask * gpm, detpad, mode='constant', constant_values=0) # but don't include padded data in the fit # Grab the pixels to be included in the fit wpix = np.where(offslitmask_pad) @@ -252,7 +313,7 @@ def mask_slit_regions(offslitmask, centrace, mask_regions=None): return offslitmask & np.logical_not(bad_mask) -def fine_correction(frame, bpm, offslitmask, polyord=2, debug=False): +def fine_correction(frame, bpm, offslitmask, method='median', polyord=2, debug=False): """ Calculate a fine correction to the residual scattered light of the input frame. Parameters @@ -265,6 +326,11 @@ def fine_correction(frame, bpm, offslitmask, polyord=2, debug=False): 2D boolean array indicating the bad pixels (True=bad), same shape as frame offslitmask : `numpy.ndarray`_ A boolean mask indicating the pixels that are on/off the slit (True = off the slit), same shape as frame + method : :obj:`str`, optional + Method to use to determine the fine correction to the scattered light. Options are: + - 'median': Use the median of the off-slit pixels to determine the scattered light + - 'poly': Use a polynomial fit to the off-slit pixels to determine the scattered light. If this + option is chosen, the polynomial order must be specified. See `polyord` below. polyord : :obj:`int`, optional Polynomial order to use for fitting the residual scattered light in the spatial direction. debug : :obj:`bool`, optional @@ -275,25 +341,33 @@ def fine_correction(frame, bpm, offslitmask, polyord=2, debug=False): scatt_img : `numpy.ndarray`_ A 2D image (nspec, nspat) of the fine correction to the scattered light determined from the input frame. """ - msgs.info("Performing a fine correction to the scattered light") - # Convert the BPM to a GPM for convenience - gpm = np.logical_not(bpm) - - # Define some useful variables + if method not in ['median', 'poly']: + msgs.error("Unrecognized method to determine the fine correction to the scattered light: {:s}".format(method)) + msgs.info("Performing a fine correction to the scattered light using the {:s} method".format(method)) nspec, nspat = frame.shape - xspat = np.linspace(0, 1, nspat) - model = np.zeros_like(frame) - - # Loop over the residual scattered light in the spectral direction and perform - # a low order polynomial fit to the scattered light in the spatial direction. - for yy in range(nspec): - ext = frame[yy, :] - gd = np.where(offslitmask[yy, :] & gpm[yy, :]) - coeff = np.polyfit(xspat[gd], ext[gd], polyord) - model[yy, :] = np.polyval(coeff, xspat) - # Median filter in the spectral direction to smooth out irregularities in the fine correction - model_med = ndimage.median_filter(model, size=(50, 1)) # Median filter to get rid of CRs - scatt_light_fine = ndimage.gaussian_filter(model_med, sigma=10) # Gaussian filter to smooth median filter + if method == 'median': + # Use the median of the off-slit pixels to determine the scattered light + mask = bpm | np.logical_not(offslitmask) + frame_msk = np.ma.array(frame, mask=mask) + scatt_light_fine = np.repeat(np.ma.median(frame_msk, axis=1).data[:, np.newaxis], nspat, axis=1) + elif method == 'poly': + # Convert the BPM to a GPM for convenience + gpm = np.logical_not(bpm) + + # Define some useful variables + xspat = np.linspace(0, 1, nspat) + model = np.zeros_like(frame) + + # Loop over the residual scattered light in the spectral direction and perform + # a low order polynomial fit to the scattered light in the spatial direction. + for yy in range(nspec): + ext = frame[yy, :] + gd = np.where(offslitmask[yy, :] & gpm[yy, :]) + coeff = np.polyfit(xspat[gd], ext[gd], polyord) + model[yy, :] = np.polyval(coeff, xspat) + # Median filter in the spectral direction to smooth out irregularities in the fine correction + model_med = ndimage.median_filter(model, size=(50, 1)) # Median filter to get rid of CRs + scatt_light_fine = ndimage.gaussian_filter(model_med, sigma=10) # Gaussian filter to smooth median filter if debug: from matplotlib import pyplot as plt vmin, vmax = -np.max(scatt_light_fine), np.max(scatt_light_fine) diff --git a/pypeit/scattlight.py b/pypeit/scattlight.py index 6ed95d82b4..21edde1f7e 100644 --- a/pypeit/scattlight.py +++ b/pypeit/scattlight.py @@ -105,7 +105,7 @@ def get_model(self, image): msgs.warn("No scattered light parameters are available") return np.zeros_like(image) # Return the model of the scattered light - return scattlight.scattered_light_model(self.scattlight_param, image) + return scattlight.scattered_light_model_pad(self.scattlight_param, image) def show(self, image=None, slits=None, mask=False, wcs_match=True): """ Display the master scattered light frame, the model, and data-model. From ac1ce48460289f5e49e275348229bfc19f86c6af Mon Sep 17 00:00:00 2001 From: rcooke Date: Wed, 3 Jan 2024 21:34:45 +0000 Subject: [PATCH 156/244] pad scattlight model add scattlight finecorr method --- pypeit/images/rawimage.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pypeit/images/rawimage.py b/pypeit/images/rawimage.py index 6b8555252b..08553813df 100644 --- a/pypeit/images/rawimage.py +++ b/pypeit/images/rawimage.py @@ -1224,10 +1224,10 @@ def subtract_scattlight(self, msscattlight, slits, debug=False): this_modpar = msscattlight.scattlight_param.copy() this_modpar[8] = 0.0 # This is the zero-level of the scattlight frame. The zero-level is determined by the finecorr # Apply the requested method for the scattered light - do_finecorr = self.par["scattlight"]["finecorr"] + do_finecorr = self.par["scattlight"]["finecorr_method"] is not None if self.par["scattlight"]["method"] == "model": # Use predefined model parameters - scatt_img = scattlight.scattered_light_model(this_modpar, _img) + scatt_img = scattlight.scattered_light_model_pad(this_modpar, _img) if debug: specbin, spatbin = parse.parse_binning(self.detector[0]['binning']) tmp = msscattlight.scattlight_param.copy() @@ -1305,9 +1305,11 @@ def subtract_scattlight(self, msscattlight, slits, debug=False): offslitmask = scattlight.mask_slit_regions(offslitmask, centrace, mask_regions=self.par['scattlight']['finecorr_mask']) # Calculate the fine correction to the scattered light image, and add it to the full model - scatt_img += scattlight.fine_correction(_img-scatt_img, full_bpm, offslitmask, - polyord=self.par['scattlight']['finecorr_order']) - # Subtract the scattered light model from the image + fine_corr_scatt_light = scattlight.fine_correction(_img-scatt_img, full_bpm, offslitmask, + method=self.par['scattlight']['finecorr_method'], + polyord=self.par['scattlight']['finecorr_order']) + scatt_img += fine_corr_scatt_light + # Subtract the total scattered light model from the image self.image[ii, ...] -= scatt_img self.steps[step] = True From 3026fd352da120860711697a6eefe4f3f569ea24 Mon Sep 17 00:00:00 2001 From: rcooke Date: Wed, 3 Jan 2024 21:35:49 +0000 Subject: [PATCH 157/244] quadratic detector structure amplitude add finecorr method --- pypeit/spectrographs/keck_kcwi.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pypeit/spectrographs/keck_kcwi.py b/pypeit/spectrographs/keck_kcwi.py index 191d158701..9513a07b6c 100644 --- a/pypeit/spectrographs/keck_kcwi.py +++ b/pypeit/spectrographs/keck_kcwi.py @@ -871,7 +871,6 @@ def init_meta(self): self.meta['dispangle'] = dict(ext=0, card='BGRANGLE', rtol=0.01) self.meta['cenwave'] = dict(ext=0, card='BCWAVE', rtol=0.01) - def raw_header_cards(self): """ Return additional raw header cards to be propagated in @@ -919,9 +918,10 @@ def default_pypeit_par(cls): par['calibrations']['pixelflatframe']['process']['subtract_scattlight'] = True par['calibrations']['illumflatframe']['process']['subtract_scattlight'] = True par['scienceframe']['process']['subtract_scattlight'] = True - par['scienceframe']['process']['scattlight']['finecorr_pad'] = 2 + par['scienceframe']['process']['scattlight']['finecorr_method'] = 'median' + par['scienceframe']['process']['scattlight']['finecorr_pad'] = 4 # This is the unbinned number of pixels to pad par['scienceframe']['process']['scattlight']['finecorr_order'] = 2 - par['scienceframe']['process']['scattlight']['finecorr_mask'] = 12 # Mask the middle inter-slit region. It contains a strange scattered light feature that doesn't appear to affect any other inter-slit regions + # par['scienceframe']['process']['scattlight']['finecorr_mask'] = 12 # Mask the middle inter-slit region. It contains a strange scattered light feature that doesn't appear to affect any other inter-slit regions # Correct the illumflat for pixel-to-pixel sensitivity variations par['calibrations']['illumflatframe']['process']['use_pixelflat'] = True @@ -1167,13 +1167,13 @@ def fit_2d_det_response(self, det_resp, gpmask): msgs.info("Performing a 2D fit to the detector response") # Define a 2D sine function, which is a good description of KCWI data - def sinfunc2d(x, amp, scl, phase, wavelength, angle): + def sinfunc2d(x, amp, scl, quad, phase, wavelength, angle): """ 2D Sine function """ xx, yy = x angle *= np.pi / 180.0 - return 1 + (amp + xx * scl) * np.sin( + return 1 + (amp + xx*scl + xx*xx*quad) * np.sin( 2 * np.pi * (xx * np.cos(angle) + yy * np.sin(angle)) / wavelength + phase) x = np.arange(det_resp.shape[0]) @@ -1181,11 +1181,12 @@ def sinfunc2d(x, amp, scl, phase, wavelength, angle): xx, yy = np.meshgrid(x, y, indexing='ij') # Prepare the starting parameters amp = 0.02 # Roughly a 2% effect - scale = 0.0 # Assume the amplitude is constant over the detector + scale, quad = 0.0, 0.0 # Assume the amplitude is constant over the detector wavelength = np.sqrt(det_resp.shape[0] ** 2 + det_resp.shape[1] ** 2) / 31.5 # 31-32 cycles of the pattern from corner to corner phase, angle = 0.0, -45.34 # No phase, and a decent guess at the angle - p0 = [amp, scale, phase, wavelength, angle] - popt, pcov = curve_fit(sinfunc2d, (xx[gpmask], yy[gpmask]), det_resp[gpmask], p0=p0) + p0 = [amp, scale, quad, phase, wavelength, angle] + this_bpm = gpmask & (np.abs(det_resp-1) < 0.1) # Only expect this to be a 5% effect + popt, pcov = curve_fit(sinfunc2d, (xx[this_bpm], yy[this_bpm]), det_resp[this_bpm], p0=p0) return sinfunc2d((xx, yy), *popt) From 9648138dc509c5d78df63d38a77fd1e999202df5 Mon Sep 17 00:00:00 2001 From: rcooke Date: Wed, 3 Jan 2024 21:38:55 +0000 Subject: [PATCH 158/244] updated docstring --- pypeit/core/scattlight.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pypeit/core/scattlight.py b/pypeit/core/scattlight.py index a8fc336598..394af7166d 100644 --- a/pypeit/core/scattlight.py +++ b/pypeit/core/scattlight.py @@ -47,7 +47,6 @@ def scattered_light_model_pad(param, img, detpad=300): Model parameters that determine the scattered light based on the input img. Here is a list of the individual parameter meanings: - # TODO :: I think this is out of date... * param[0] = Gaussian kernel width in the spectral direction * param[1] = Gaussian kernel width in the spatial direction * param[2] = Lorentzian kernel width in the spectral direction @@ -55,11 +54,12 @@ def scattered_light_model_pad(param, img, detpad=300): * param[4] = Pixel shift of the scattered light in the spectral direction * param[5] = Pixel shift of the scattered light in the spatial direction * param[6] = Zoom factor of the scattered light (~1) - * param[7] = Kernel angle - * param[8] = Relative importance of Gaussian vs Lorentzian. - 0 < value < 1 means Lorentzian is weighted more - value > 1 means Gaussian is weighted more. - * param[9:] = Polynomial scaling coefficients + * param[7] = constant offset for scattered light (independent of img) + * param[8] = Kernel angle + * param[9] = Relative importance of Gaussian vs Lorentzian. + 0 < value < 1 means Lorentzian is weighted more + value > 1 means Gaussian is weighted more. + * param[10:] = Polynomial scaling coefficients img : `numpy.ndarray`_ Image used to generate the scattered light model detpad : int @@ -89,7 +89,6 @@ def scattered_light_model(param, img): Model parameters that determine the scattered light based on the input img. Here is a list of the individual parameter meanings: - # TODO :: I think this is out of date... * param[0] = Gaussian kernel width in the spectral direction * param[1] = Gaussian kernel width in the spatial direction * param[2] = Lorentzian kernel width in the spectral direction @@ -97,11 +96,12 @@ def scattered_light_model(param, img): * param[4] = Pixel shift of the scattered light in the spectral direction * param[5] = Pixel shift of the scattered light in the spatial direction * param[6] = Zoom factor of the scattered light (~1) - * param[7] = Kernel angle - * param[8] = Relative importance of Gaussian vs Lorentzian. - 0 < value < 1 means Lorentzian is weighted more - value > 1 means Gaussian is weighted more. - * param[9:] = Polynomial scaling coefficients + * param[7] = constant offset for scattered light (independent of img) + * param[8] = Kernel angle + * param[9] = Relative importance of Gaussian vs Lorentzian. + 0 < value < 1 means Lorentzian is weighted more + value > 1 means Gaussian is weighted more. + * param[10:] = Polynomial scaling coefficients img : `numpy.ndarray`_ Raw image that you want to compute the scattered light model. shape is (nspec, nspat) From 5eeb4b0e3d8ca70b9a3a647c1b3b202574964dde Mon Sep 17 00:00:00 2001 From: jhennawi Date: Wed, 3 Jan 2024 20:57:50 -0800 Subject: [PATCH 159/244] Fixed bug in par and added validation. --- pypeit/par/pypeitpar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index 8159e65eee..a20d380805 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -1787,7 +1787,7 @@ def from_dict(cls, cfg): # Basic keywords parkeys = ['slit_spec', 'output_filename', 'standard_cube', 'reference_image', 'save_whitelight', - 'method', 'spec_subpixel', 'spat_subpixel', 'ra_min', 'ra_max', 'dec_min', 'dec_max', + 'method', 'spec_subpixel', 'spat_subpixel', 'slice_subpixel', 'ra_min', 'ra_max', 'dec_min', 'dec_max', 'wave_min', 'wave_max', 'spatial_delta', 'wave_delta', 'weight_method', 'align', 'combine', 'astrometric', 'grating_corr', 'scale_corr', 'skysub_frame', 'whitelight_range'] From bd9bcb885786913b8f76e4dc885ec61b1dfa1634 Mon Sep 17 00:00:00 2001 From: rcooke Date: Thu, 4 Jan 2024 08:03:35 +0000 Subject: [PATCH 160/244] cleanup --- pypeit/images/rawimage.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pypeit/images/rawimage.py b/pypeit/images/rawimage.py index 08553813df..aa4f47d35f 100644 --- a/pypeit/images/rawimage.py +++ b/pypeit/images/rawimage.py @@ -1305,10 +1305,9 @@ def subtract_scattlight(self, msscattlight, slits, debug=False): offslitmask = scattlight.mask_slit_regions(offslitmask, centrace, mask_regions=self.par['scattlight']['finecorr_mask']) # Calculate the fine correction to the scattered light image, and add it to the full model - fine_corr_scatt_light = scattlight.fine_correction(_img-scatt_img, full_bpm, offslitmask, - method=self.par['scattlight']['finecorr_method'], - polyord=self.par['scattlight']['finecorr_order']) - scatt_img += fine_corr_scatt_light + scatt_img += scattlight.fine_correction(_img-scatt_img, full_bpm, offslitmask, + method=self.par['scattlight']['finecorr_method'], + polyord=self.par['scattlight']['finecorr_order']) # Subtract the total scattered light model from the image self.image[ii, ...] -= scatt_img self.steps[step] = True From 76996b8f405756ee121337de5417a7bce52277d8 Mon Sep 17 00:00:00 2001 From: jhennawi Date: Thu, 4 Jan 2024 00:14:24 -0800 Subject: [PATCH 161/244] Fixed a brittle argmax call on whitelight image in datacube.compute_weights --- pypeit/core/coadd.py | 3 ++- pypeit/core/datacube.py | 12 ++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/pypeit/core/coadd.py b/pypeit/core/coadd.py index c34ad25c74..8085c05010 100644 --- a/pypeit/core/coadd.py +++ b/pypeit/core/coadd.py @@ -747,6 +747,7 @@ def smooth_weights(inarr, gdmsk, sn_smooth_npix): sig_res = np.fmax(sn_smooth_npix / 10.0, 3.0) gauss_kernel = convolution.Gaussian1DKernel(sig_res) sn_conv = convolution.convolve(sn_med2, gauss_kernel, boundary='extend') + # TODO Should we be setting a floor on the weights to prevent tiny numbers? return sn_conv # TODO add a wave_min, wave_max option here to deal with objects like high-z QSOs etc. which only have flux in a given wavelength range. @@ -796,7 +797,7 @@ def calc_snr(fluxes, ivars, gpms): return np.array(rms_sn), sn_val - +# TODO add a wave_min, wave_max option here to deal with objects like high-z QSOs etc. which only have flux in a given wavelength range. def sn_weights(fluxes, ivars, gpms, sn_smooth_npix=None, weight_method='auto', verbose=False): """ diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index 735ce8977d..97a7271e5e 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -10,6 +10,7 @@ from astropy.coordinates import AltAz, SkyCoord from astropy.io import fits import scipy.optimize as opt +from scipy import signal from scipy.interpolate import interp1d import numpy as np @@ -1207,7 +1208,12 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, ra_min=ra_min, ra_max=ra_max, dec_min=dec_min, dec_max=dec_max, wave_min=wave_min, wave_max=wave_max) # Find the location of the object with the highest S/N in the combined white light image - idx_max = np.unravel_index(np.argmax(whitelight_img), whitelight_img.shape) + med_filt_whitelight = signal.medfilt2d(whitelight_img, kernel_size=3) + idx_max = np.unravel_index(np.argmax(med_filt_whitelight), med_filt_whitelight.shape) + # TODO: Taking the maximum pixel of the whitelight image is extremely brittle to the case where + # their are hot pixels in the white light image, which there are plenty of since the edges of the slits are very + # poorly behaved. + #idx_max = np.unravel_index(np.argmax(whitelight_img), whitelight_img.shape) msgs.info("Highest S/N object located at spaxel (x, y) = {0:d}, {1:d}".format(idx_max[0], idx_max[1])) # Generate a 2D WCS to register all frames @@ -1367,7 +1373,7 @@ def generate_cube_subpixel(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_im """ Save a datacube using the subpixel algorithm. Refer to the subpixellate() docstring for further details about this algorithm - + # TODO These docs need to indicate the shapes of the inputs Args: output_wcs (`astropy.wcs.WCS`_): Output world coordinate system. @@ -1441,6 +1447,7 @@ def generate_cube_subpixel(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_im flux should have mean=0 and std=1. Returns: + TODO: These docs need to indicate the shapes of the returned arrays :obj:`tuple`: Four `numpy.ndarray`_ objects containing (1) the datacube generated from the subpixellated inputs, (2) the corresponding error cube (standard deviation), @@ -1493,6 +1500,7 @@ def generate_cube_subpixel(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_im msgs.info("Saving white light image as: {0:s}".format(out_whitelight)) img_hdu = fits.PrimaryHDU(whitelight_img.T, header=whitelight_wcs.to_header()) img_hdu.writeto(out_whitelight, overwrite=overwrite) + # TODO :: Avoid transposing these large cubes return flxcube.T, np.sqrt(varcube.T), bpmcube.T, wave From d87e2ccf5ae83c2d895c10b14f1f7cf23613c449 Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 5 Jan 2024 18:51:25 +0000 Subject: [PATCH 162/244] improved finecorr --- pypeit/flatfield.py | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/pypeit/flatfield.py b/pypeit/flatfield.py index 02f9624d0d..85cbfd5cc6 100644 --- a/pypeit/flatfield.py +++ b/pypeit/flatfield.py @@ -1152,8 +1152,11 @@ def fit(self, spat_illum_only=False, doqa=True, debug=False): # Perform a fine correction to the spatial illumination profile spat_illum_fine = 1 # Default value if the fine correction is not performed if exit_status <= 1 and self.flatpar['slit_illum_finecorr']: - spat_illum = spat_bspl.value(spat_coo_final[onslit_tweak])[0] - self.spatial_fit_finecorr(spat_illum, onslit_tweak, slit_idx, slit_spat, gpm, doqa=doqa) + spat_model = np.ones_like(spec_model) + spat_model[onslit_padded] = spat_bspl.value(spat_coo_final[onslit_padded])[0]#np.exp(spec_bspl.value(spec_coo[onslit_padded])[0]) + specspat_illum = np.fmax(spec_model, 1.0) * spat_model + norm_spatspec = rawflat / specspat_illum + self.spatial_fit_finecorr(norm_spatspec, onslit_tweak, slit_idx, slit_spat, gpm, doqa=doqa) # ---------------------------------------------------------- # Construct the illumination profile with the tweaked edges @@ -1368,7 +1371,7 @@ def spatial_fit(self, norm_spec, spat_coo, median_slit_width, spat_gpm, gpm, deb return exit_status, spat_coo_data, spat_flat_data, spat_bspl, spat_gpm_fit, \ spat_flat_fit, spat_flat_data_raw - def spatial_fit_finecorr(self, spat_illum, onslit_tweak, slit_idx, slit_spat, gpm, slit_trim=3, doqa=False): + def spatial_fit_finecorr(self, normed, onslit_tweak, slit_idx, slit_spat, gpm, slit_trim=3, doqa=False): """ Generate a relative scaling image for a slicer IFU. All slits are scaled relative to a reference slit, specified in @@ -1376,8 +1379,8 @@ def spatial_fit_finecorr(self, spat_illum, onslit_tweak, slit_idx, slit_spat, gp Parameters ---------- - spat_illum : `numpy.ndarray`_ - An image containing the generated spatial illumination profile for all slits. + normed : `numpy.ndarray`_ + Raw flat field image, normalized by the spectral and spatial illuminations. onslit_tweak : `numpy.ndarray`_ mask indicating which pixels are on the slit (True = on slit) slit_idx : int @@ -1406,10 +1409,6 @@ def spatial_fit_finecorr(self, spat_illum, onslit_tweak, slit_idx, slit_spat, gp onslit_tweak_trim = self.slits.slit_img(pad=-slit_trim, slitidx=slit_idx, initial=False) == slit_spat # Setup slitimg = (slit_spat + 1) * onslit_tweak.astype(int) - 1 # Need to +1 and -1 so that slitimg=-1 when off the slit - normed = self.rawflatimg.image.copy() - ivarnrm = self.rawflatimg.ivar.copy() - normed[onslit_tweak] *= utils.inverse(spat_illum) - ivarnrm[onslit_tweak] *= spat_illum**2 left, right, msk = self.slits.select_edges(initial=True, flexure=self.wavetilts.spat_flexure) this_left = left[:, slit_idx] this_right = right[:, slit_idx] @@ -1432,18 +1431,6 @@ def spatial_fit_finecorr(self, spat_illum, onslit_tweak, slit_idx, slit_spat, gp ypos = (this_wave - wave_min) / (wave_max - wave_min) # Need to use the same wave_min and wave_max as the fitting coordinates xpos = xpos_img[this_slit] - # Normalise the image - delta = 0.5/self.slits.nspec # include the endpoints - bins = np.linspace(0.0-delta, 1.0+delta, self.slits.nspec+1) - censpec, _ = np.histogram(ypos_fit, bins=bins, weights=normed[this_slit_trim]) - nrm, _ = np.histogram(ypos_fit, bins=bins) - censpec *= utils.inverse(nrm) - tiltspl = interpolate.interp1d(0.5*(bins[1:]+bins[:-1]), censpec, kind='linear', - bounds_error=False, fill_value='extrapolate') - nrm_vals = tiltspl(ypos_fit) - normed[this_slit_trim] *= utils.inverse(nrm_vals) - ivarnrm[this_slit_trim] *= nrm_vals**2 - # Mask the edges and fit gpmfit = gpm[this_slit_trim] # Trim by 5% of the slit length, or at least slit_trim pixels From 71b7f7f9d28fe64bd2b96708e82dc539134afc03 Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 5 Jan 2024 18:51:33 +0000 Subject: [PATCH 163/244] fix tests --- pypeit/tests/test_slittrace.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pypeit/tests/test_slittrace.py b/pypeit/tests/test_slittrace.py index bf105dd9ca..8d4b3571c7 100644 --- a/pypeit/tests/test_slittrace.py +++ b/pypeit/tests/test_slittrace.py @@ -14,8 +14,17 @@ def test_bits(): # Make sure bits are correct bm = SlitTraceBitMask() + assert bm.bits['SHORTSLIT'] == 0, 'Bits changed' + assert bm.bits['BOXSLIT'] == 1, 'Bits changed' assert bm.bits['USERIGNORE'] == 2, 'Bits changed' - assert bm.bits['BADFLATCALIB'] == 6, 'Bits changed' + assert bm.bits['BADWVCALIB'] == 3, 'Bits changed' + assert bm.bits['BADTILTCALIB'] == 4, 'Bits changed' + assert bm.bits['BADALIGNCALIB'] == 5, 'Bits changed' + assert bm.bits['SKIPFLATCALIB'] == 6, 'Bits changed' + assert bm.bits['BADFLATCALIB'] == 7, 'Bits changed' + assert bm.bits['BADREDUCE'] == 8, 'Bits changed' + assert bm.bits['BADSKYSUB'] == 9, 'Bits changed' + assert bm.bits['BADEXTRACT'] == 10, 'Bits changed' def test_init(): From d68b890ebc7fb3116f7918054c8f6bd76bcdbc1b Mon Sep 17 00:00:00 2001 From: jhennawi Date: Fri, 5 Jan 2024 21:38:23 -0800 Subject: [PATCH 164/244] Just deleted some commented out code. --- pypeit/core/coadd.py | 4 ++-- pypeit/par/pypeitpar.py | 7 ------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/pypeit/core/coadd.py b/pypeit/core/coadd.py index c34ad25c74..593bdeed75 100644 --- a/pypeit/core/coadd.py +++ b/pypeit/core/coadd.py @@ -2626,8 +2626,8 @@ def ech_combspec(waves_arr_setup, fluxes_arr_setup, ivars_arr_setup, gpms_arr_se # nspec, norder, nexp = shape # Decide how much to smooth the spectra by if this number was not passed in - nspec_good = [] - ngood = [] + #nspec_good = [] + #ngood = [] #if sn_smooth_npix is None: # # Loop over setups # for wave, norder, nexp in zip(waves_arr_setup, norders, nexps): diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index fdf07656e4..8409519610 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -1599,13 +1599,6 @@ def __init__(self, slit_spec=None, weight_method=None, align=None, combine=None, "'ivar' -- Use inverse variance weighting. This is not well tested and should probably be deprecated." - #defaults['relative_weights'] = False - #dtypes['relative_weights'] = [bool] - #descr['relative_weights'] = 'If set to True, the combined frames will use a relative weighting scheme. ' \ - # 'This only works well if there is a common continuum source in the field of ' \ - # 'view of all input observations, and is generally only required if high ' \ - # 'relative precision is desired.' - defaults['align'] = False dtypes['align'] = [bool] descr['align'] = 'If set to True, the input frames will be spatially aligned by cross-correlating the ' \ From 9655fd96193398c639438d2c46869f39c2c67244 Mon Sep 17 00:00:00 2001 From: rcooke Date: Sun, 7 Jan 2024 13:21:33 +0000 Subject: [PATCH 165/244] fine spatial profile --- pypeit/spectrographs/keck_kcwi.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pypeit/spectrographs/keck_kcwi.py b/pypeit/spectrographs/keck_kcwi.py index 9513a07b6c..4d6008a9a6 100644 --- a/pypeit/spectrographs/keck_kcwi.py +++ b/pypeit/spectrographs/keck_kcwi.py @@ -941,6 +941,7 @@ def default_pypeit_par(cls): # Alter the method used to combine pixel flats par['calibrations']['pixelflatframe']['process']['combine'] = 'median' par['calibrations']['flatfield']['spec_samp_coarse'] = 20.0 + par['calibrations']['flatfield']['spat_samp'] = 1.0 # This should give 1% accuracy in the spatial illumination correction for 2x2 binning, and <0.5% accuracy for 1x1 binning #par['calibrations']['flatfield']['tweak_slits'] = False # Do not tweak the slit edges (we want to use the full slit) par['calibrations']['flatfield']['tweak_slits_thresh'] = 0.0 # Make sure the full slit is used (i.e. when the illumination fraction is > 0.5) par['calibrations']['flatfield']['tweak_slits_maxfrac'] = 0.0 # Make sure the full slit is used (i.e. no padding) From 8bc8bcbfafa542e63a51ab0a02246e81cc3c0676 Mon Sep 17 00:00:00 2001 From: rcooke Date: Sun, 7 Jan 2024 13:33:54 +0000 Subject: [PATCH 166/244] update docstrings --- pypeit/core/datacube.py | 51 ++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index 97a7271e5e..54966b0e4d 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -1373,7 +1373,7 @@ def generate_cube_subpixel(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_im """ Save a datacube using the subpixel algorithm. Refer to the subpixellate() docstring for further details about this algorithm - # TODO These docs need to indicate the shapes of the inputs + Args: output_wcs (`astropy.wcs.WCS`_): Output world coordinate system. @@ -1381,23 +1381,28 @@ def generate_cube_subpixel(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_im A 3-tuple (x,y,z) containing the histogram bin edges in x,y spatial and z wavelength coordinates sciImg (`numpy.ndarray`_, list): - A list of 2D array containing the counts of each pixel + A list of 2D array containing the counts of each pixel. If a list, + the shape of each numpy array is (nspec, nspat). ivarImg (`numpy.ndarray`_, list): - A list of 2D array containing the inverse variance of each pixel + A list of 2D array containing the inverse variance of each pixel. If a list, + the shape of each numpy array is (nspec, nspat). waveImg (`numpy.ndarray`_, list): - A list of 2D array containing the wavelength of each pixel + A list of 2D array containing the wavelength of each pixel. If a list, + the shape of each numpy array is (nspec, nspat). slitid_img_gpm (`numpy.ndarray`_, list): - A list of 2D array containing the slitmask of each pixel. + A list of 2D array containing the slitmask of each pixel. If a list, + the shape of each numpy array is (nspec, nspat). A zero value indicates that a pixel is either not on a slit or it is a bad pixel. All other values are the slit spatial ID number. wghtImg (`numpy.ndarray`_, list): A list of 2D array containing the weights of each pixel to be used in the - combination + combination. If a list, the shape of each numpy array is (nspec, nspat). all_wcs (`astropy.wcs.WCS`_, list): - A list of `astropy.wcs.WCS`_ objects, one for each spec2d file + A list of `astropy.wcs.WCS`_ objects, one for each spec2d file. tilts (list): A list of `numpy.ndarray`_ objects, one for each spec2d file, - containing the tilts of each pixel + containing the tilts of each pixel. The shape of each numpy array + is (nspec, nspat). slits (:class:`pypeit.slittrace.SlitTraceSet`, list): A list of `pypeit.slittrace.SlitTraceSet`_ objects, one for each spec2d file, containing the properties of the slit for each spec2d file @@ -1447,12 +1452,15 @@ def generate_cube_subpixel(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_im flux should have mean=0 and std=1. Returns: - TODO: These docs need to indicate the shapes of the returned arrays :obj:`tuple`: Four `numpy.ndarray`_ objects containing - (1) the datacube generated from the subpixellated inputs, - (2) the corresponding error cube (standard deviation), - (3) the corresponding bad pixel mask cube, and - (4) a 1D array containing the wavelength at each spectral coordinate of the datacube. + (1) the datacube generated from the subpixellated inputs. The shape of + the datacube is (nwave, nspat1, nspat2). + (2) the corresponding error cube (standard deviation). The shape of the + error cube is (nwave, nspat1, nspat2). + (3) the corresponding bad pixel mask cube. The shape of the bad pixel + mask cube is (nwave, nspat1, nspat2). + (4) a 1D array containing the wavelength at each spectral coordinate of the datacube. The + shape of the wavelength array is (nwave,). """ # Check the inputs if whitelight_range is not None or debug: @@ -1529,23 +1537,28 @@ def subpixellate(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wgh A 3-tuple (x,y,z) containing the histogram bin edges in x,y spatial and z wavelength coordinates sciImg (`numpy.ndarray`_, list): - A list of 2D array containing the counts of each pixel + A list of 2D array containing the counts of each pixel. The shape of + each 2D array is (nspec, nspat). ivarImg (`numpy.ndarray`_, list): - A list of 2D array containing the inverse variance of each pixel + A list of 2D array containing the inverse variance of each pixel. The shape of + each 2D array is (nspec, nspat). waveImg (`numpy.ndarray`_, list): - A list of 2D array containing the wavelength of each pixel + A list of 2D array containing the wavelength of each pixel. The shape of + each 2D array is (nspec, nspat). slitid_img_gpm (`numpy.ndarray`_, list): - A list of 2D array containing the slitmask of each pixel. + A list of 2D array containing the slitmask of each pixel. The shape of + each 2D array is (nspec, nspat). A zero value indicates that a pixel is either not on a slit or it is a bad pixel. All other values are the slit spatial ID number. wghtImg (`numpy.ndarray`_, list): A list of 2D array containing the weights of each pixel to be used in the - combination + combination. The shape of each 2D array is (nspec, nspat). all_wcs (`astropy.wcs.WCS`_, list): A list of `astropy.wcs.WCS`_ objects, one for each spec2d file tilts (list): A list of `numpy.ndarray`_ objects, one for each spec2d file, - containing the tilts of each pixel + containing the tilts of each pixel. The shape of each 2D array is + (nspec, nspat). slits (:class:`pypeit.slittrace.SlitTraceSet`, list): A list of `pypeit.slittrace.SlitTraceSet`_ objects, one for each spec2d file, containing the properties of the slit for each spec2d file From 160daa4f437cc4a40651e287329fe43fbec12ace Mon Sep 17 00:00:00 2001 From: rcooke Date: Sun, 7 Jan 2024 13:57:05 +0000 Subject: [PATCH 167/244] fix use_specillum descr --- pypeit/par/pypeitpar.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index e7b7a02ed7..27c0e77753 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -682,9 +682,8 @@ def __init__(self, method=None, pixelflat_file=None, spec_samp_fine=None, defaults['slit_illum_relative'] = False dtypes['slit_illum_relative'] = bool - # TODO Needs to be updated. use_slitillum is not a parameter anywhere in pypeit descr['slit_illum_relative'] = 'Generate an image of the relative spectral illumination ' \ - 'for a multi-slit setup. If you set ``use_slitillum = ' \ + 'for a multi-slit setup. If you set ``use_specillum = ' \ 'True`` for any of the frames that use the flatfield ' \ 'model, this *must* be set to True. Currently, this is ' \ 'only used for SlicerIFU reductions.' From 3f4a65e3b4a1e3fa5c220496102311616702c11a Mon Sep 17 00:00:00 2001 From: rcooke Date: Sun, 7 Jan 2024 13:57:24 +0000 Subject: [PATCH 168/244] deprecate spline_map --- pypeit/deprecated/flat.py | 99 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/pypeit/deprecated/flat.py b/pypeit/deprecated/flat.py index ac9451216e..f59334af70 100644 --- a/pypeit/deprecated/flat.py +++ b/pypeit/deprecated/flat.py @@ -1025,3 +1025,102 @@ def norm_slits(mstrace, datasec_img, lordloc, rordloc, pixwid, # # return slit_profiles, mstracenrm, extrap_blz +# Deprecated because it is slow, inefficient, and needs more work. The idea is to use a spline fit to control points +# along the spectral direction to construct a map between modelimg and rawimg. See core.flat.poly_map for a faster +# approach. +def spline_map(rawimg, rawivar, waveimg, slitmask, slitmask_trim, modelimg, + slit_illum_ref_idx=0, gpmask=None, thismask=None, nbins=20, debug=False): + """ + Use a spline fit to control points along the spectral direction to construct a map between modelimg and + rawimg. Currently, this routine is only used for image slicer IFUs. + + This problem needs to be recast into a smoothing problem, that can take advantage of the following: + https://docs.scipy.org/doc/scipy/reference/generated/scipy.interpolate.UnivariateSpline.html#scipy.interpolate.UnivariateSpline + where: + y = science/model + w = model/science_error + resid = w*(y-f(x)) + Then, iterate over this. The reason to iterate is that there should be a mapping between science and + science_error. Once we have an estimate of f(x), we can do the following: + spl = spline(science, science_error) + new_science_error = spl(model*f(x)) + Now iterate a few times until new_science_error (and the fit) is converged. + + Parameters + ---------- + rawimg : `numpy.ndarray`_ + Image data that will be used to estimate the spectral relative sensitivity + rawivar : `numpy.ndarray`_ + Inverse variance image of rawimg + waveimg : `numpy.ndarray`_ + Wavelength image + slitmask : `numpy.ndarray`_ + A 2D int mask, the same shape as rawimg, indicating which pixels are on a slit. A -1 value + indicates not on a slit, while any pixels on a slit should have the value of the slit spatial ID + number. + slitmask_trim : `numpy.ndarray`_ + Same as slitmask, but the slit edges are trimmed. + modelimg : `numpy.ndarray`_ + A model of the rawimg data. + slit_illum_ref_idx : :obj:`int` + Index of slit that is used as the reference. + gpmask : `numpy.ndarray`_, optional + Boolean good pixel mask (True = Good) + thismask : `numpy.ndarray`_, optional + A boolean mask (True = good) that indicates all pixels where the scaleImg should be constructed. + If None, the slitmask that is generated by this routine will be used. + nbins : :obj:`int` + Number of bins in the spectral direction to sample the relative spectral sensitivity + debug : :obj:`bool` + If True, some plots will be output to test if the fitting is working correctly. + + Returns + ------- + scale_model: `numpy.ndarray`_ + An image containing the appropriate scaling + """ + # Some variables to consider putting as function arguments + numiter = 4 + + # Start by calculating a ratio of the raming and the modelimg + nspec = rawimg.shape[0] + _ratio = rawimg * utils.inverse(modelimg) + _ratio_ivar = rawivar * modelimg**2 + _fit_wghts = modelimg * np.sqrt(rawivar) + + # Generate the mask + _thismask = thismask if (thismask is not None) else (slitmask > 0) + gpm = gpmask if (gpmask is not None) else np.ones_like(rawimg, dtype=bool) + # Extract the list of spatial IDs from the slitmask + slitmask_spatid = np.unique(slitmask) + slitmask_spatid = np.sort(slitmask_spatid[slitmask_spatid > 0]) + + # Create a spline between the raw data and the error + # TODO :: This is slow and inefficient. + embed() + flxsrt = np.argsort(np.ravel(rawimg)) + spl = scipy.interpolate.interp1d(np.ravel(rawimg)[flxsrt], np.ravel(rawivar)[flxsrt], kind='linear', + bounds_error=False, fill_value=0.0, assume_sorted=True) + modelmap = np.zeros_like(rawimg) + for sl, spatid in enumerate(slitmask_spatid): + print("sl") + # Prepare the masks, edges, and fitting variables + this_slit = (slitmask == spatid) + this_slit_trim = (slitmask_trim == spatid) + this_slit_mask = gpm & this_slit_trim + this_wave = waveimg[this_slit_mask] + this_wghts = _fit_wghts[this_slit_mask] + asrt = np.argsort(this_wave) + for ii in range(numiter): + # Generate the map between model and data + splmap = scipy.interpolate.UnivariateSpline(this_wave[asrt], _ratio[this_slit_mask][asrt], w=this_wghts[asrt], + bbox=[None, None], k=3, s=nspec, ext=0, check_finite=False) + # Construct the mapping, and use this to make a model of the rawdata + this_modmap = splmap(this_wave[this_slit_mask]) + this_modflx = modelimg[this_slit_mask] * this_modmap + # Update the fit weights + this_wghts = modelimg[this_slit_mask] * np.sqrt(spl(this_modflx)) + # Produce the final model for this slit + modelmap[this_slit] = splmap(this_wave[this_slit]) + return modelmap + From 567c08312c998a7a533823dc427f6667b3b2c8ef Mon Sep 17 00:00:00 2001 From: rcooke Date: Sun, 7 Jan 2024 13:57:55 +0000 Subject: [PATCH 169/244] cleanup --- pypeit/core/procimg.py | 1 - pypeit/images/rawimage.py | 1 - 2 files changed, 2 deletions(-) diff --git a/pypeit/core/procimg.py b/pypeit/core/procimg.py index 5f15b01d97..9133bd6eac 100644 --- a/pypeit/core/procimg.py +++ b/pypeit/core/procimg.py @@ -817,7 +817,6 @@ def subtract_pattern(rawframe, datasec_img, oscansec_img, frequency=None, axis=1 bst = np.argmax(power) imin = np.clip(bst-2,0,None) imax = np.clip(bst+3,None,overscan.shape[1]) - # TODO is this correct?? cc = np.polyfit(LSfreq[imin:imax],power[imin:imax],2) all_freq[ii] = -0.5*cc[1]/cc[0] cc = np.polyfit(all_rows, all_freq, 1) diff --git a/pypeit/images/rawimage.py b/pypeit/images/rawimage.py index aa4f47d35f..247deb6128 100644 --- a/pypeit/images/rawimage.py +++ b/pypeit/images/rawimage.py @@ -1243,7 +1243,6 @@ def subtract_scattlight(self, msscattlight, slits, debug=False): f" {tmp[10]}, # Relative kernel scale (>1 means the kernel is more Gaussian, >0 but <1 makes the profile more lorentzian)\n" + \ f" {tmp[11]}, {tmp[12]}, # Polynomial terms (coefficients of \"spat\" and \"spat*spec\")\n" + \ f" {tmp[13]}, {tmp[14]}, {tmp[15]}]) # Polynomial terms (coefficients of spec**index)\n" - # f" {tmp[13]}, {tmp[14]}]) # Polynomial terms (coefficients of spec**index)\n" print(strprint) pad = msscattlight.pad // spatbin offslitmask = slits.slit_img(pad=pad, initial=True, flexure=None) == -1 From 96a3cf8640dcd1af3db4f213f8f73dec3050d7ba Mon Sep 17 00:00:00 2001 From: rcooke Date: Sun, 7 Jan 2024 13:58:13 +0000 Subject: [PATCH 170/244] rm spline_map --- pypeit/core/flat.py | 99 +-------------------------------------------- 1 file changed, 2 insertions(+), 97 deletions(-) diff --git a/pypeit/core/flat.py b/pypeit/core/flat.py index c88e94ad7e..d938f79e84 100644 --- a/pypeit/core/flat.py +++ b/pypeit/core/flat.py @@ -361,6 +361,8 @@ def illum_profile_spectral_poly(rawimg, waveimg, slitmask, slitmask_trim, model, return scaleImg +# TODO:: See pypeit/deprecated/flat.py for a spline version. The following polynomial version is faster, but +# the spline version is more versatile. def poly_map(rawimg, rawivar, waveimg, slitmask, slitmask_trim, modelimg, deg=3, slit_illum_ref_idx=0, gpmask=None, thismask=None, debug=False): """ @@ -465,103 +467,6 @@ def poly_map(rawimg, rawivar, waveimg, slitmask, slitmask_trim, modelimg, deg=3, return modelmap, relscale -def spline_map(rawimg, rawivar, waveimg, slitmask, slitmask_trim, modelimg, - slit_illum_ref_idx=0, gpmask=None, thismask=None, nbins=20, debug=False): - """ - Use a spline fit to control points along the spectral direction to construct a map between modelimg and - rawimg. Currently, this routine is only used for image slicer IFUs. - - This problem needs to be recast into a smoothing problem, that can take advantage of the following: - https://docs.scipy.org/doc/scipy/reference/generated/scipy.interpolate.UnivariateSpline.html#scipy.interpolate.UnivariateSpline - where: - y = science/model - w = model/science_error - resid = w*(y-f(x)) - Then, iterate over this. The reason to iterate is that there should be a mapping between science and - science_error. Once we have an estimate of f(x), we can do the following: - spl = spline(science, science_error) - new_science_error = spl(model*f(x)) - Now iterate a few times until new_science_error (and the fit) is converged. - - Parameters - ---------- - rawimg : `numpy.ndarray`_ - Image data that will be used to estimate the spectral relative sensitivity - rawivar : `numpy.ndarray`_ - Inverse variance image of rawimg - waveimg : `numpy.ndarray`_ - Wavelength image - slitmask : `numpy.ndarray`_ - A 2D int mask, the same shape as rawimg, indicating which pixels are on a slit. A -1 value - indicates not on a slit, while any pixels on a slit should have the value of the slit spatial ID - number. - slitmask_trim : `numpy.ndarray`_ - Same as slitmask, but the slit edges are trimmed. - modelimg : `numpy.ndarray`_ - A model of the rawimg data. - slit_illum_ref_idx : :obj:`int` - Index of slit that is used as the reference. - gpmask : `numpy.ndarray`_, optional - Boolean good pixel mask (True = Good) - thismask : `numpy.ndarray`_, optional - A boolean mask (True = good) that indicates all pixels where the scaleImg should be constructed. - If None, the slitmask that is generated by this routine will be used. - nbins : :obj:`int` - Number of bins in the spectral direction to sample the relative spectral sensitivity - debug : :obj:`bool` - If True, some plots will be output to test if the fitting is working correctly. - - Returns - ------- - scale_model: `numpy.ndarray`_ - An image containing the appropriate scaling - """ - # Some variables to consider putting as function arguments - numiter = 4 - - # Start by calculating a ratio of the raming and the modelimg - nspec = rawimg.shape[0] - _ratio = rawimg * utils.inverse(modelimg) - _ratio_ivar = rawivar * modelimg**2 - _fit_wghts = modelimg * np.sqrt(rawivar) - - # Generate the mask - _thismask = thismask if (thismask is not None) else (slitmask > 0) - gpm = gpmask if (gpmask is not None) else np.ones_like(rawimg, dtype=bool) - # Extract the list of spatial IDs from the slitmask - slitmask_spatid = np.unique(slitmask) - slitmask_spatid = np.sort(slitmask_spatid[slitmask_spatid > 0]) - - # Create a spline between the raw data and the error - # TODO :: This is slow and inefficient. - embed() - flxsrt = np.argsort(np.ravel(rawimg)) - spl = scipy.interpolate.interp1d(np.ravel(rawimg)[flxsrt], np.ravel(rawivar)[flxsrt], kind='linear', - bounds_error=False, fill_value=0.0, assume_sorted=True) - modelmap = np.zeros_like(rawimg) - for sl, spatid in enumerate(slitmask_spatid): - print("sl") - # Prepare the masks, edges, and fitting variables - this_slit = (slitmask == spatid) - this_slit_trim = (slitmask_trim == spatid) - this_slit_mask = gpm & this_slit_trim - this_wave = waveimg[this_slit_mask] - this_wghts = _fit_wghts[this_slit_mask] - asrt = np.argsort(this_wave) - for ii in range(numiter): - # Generate the map between model and data - splmap = scipy.interpolate.UnivariateSpline(this_wave[asrt], _ratio[this_slit_mask][asrt], w=this_wghts[asrt], - bbox=[None, None], k=3, s=nspec, ext=0, check_finite=False) - # Construct the mapping, and use this to make a model of the rawdata - this_modmap = splmap(this_wave[this_slit_mask]) - this_modflx = modelimg[this_slit_mask] * this_modmap - # Update the fit weights - this_wghts = modelimg[this_slit_mask] * np.sqrt(spl(this_modflx)) - # Produce the final model for this slit - modelmap[this_slit] = splmap(this_wave[this_slit]) - return modelmap - - # TODO: See pypeit/deprecated/flat.py for the previous version. We need # to continue to vet this algorithm to make sure there are no # unforeseen corner cases that cause errors. From e7b72cad293f5e80368edc6f9d69b255e22111c3 Mon Sep 17 00:00:00 2001 From: rcooke Date: Sun, 7 Jan 2024 14:44:21 +0000 Subject: [PATCH 171/244] updated IFU find_objects + flexure correction + docs --- pypeit/deprecated/find_objects.py | 205 +++++++++++++++++++++++++ pypeit/find_objects.py | 247 ++++-------------------------- 2 files changed, 231 insertions(+), 221 deletions(-) diff --git a/pypeit/deprecated/find_objects.py b/pypeit/deprecated/find_objects.py index 695ab55698..46263e217a 100644 --- a/pypeit/deprecated/find_objects.py +++ b/pypeit/deprecated/find_objects.py @@ -160,3 +160,208 @@ def main(args): printout=True, slit_ids=slits.id) ofgui = gui_object_find.initialise(args.det, frame, tslits_dict, None, printout=True, slit_ids=slits.id) + + + +def illum_profile_spatial(self, skymask=None, trim_edg=(0, 0), debug=False): + """ + Calculate the residual spatial illumination profile using the sky regions. + + The residual is calculated using the differential: + + .. code-block:: python + + correction = amplitude * (1 + spatial_shift * (dy/dx)/y) + + where ``y`` is the spatial profile determined from illumflat, and + spatial_shift is the residual spatial flexure shift in units of pixels. + + Args: + skymask (`numpy.ndarray`_): + Mask of sky regions where the spatial illumination will be determined + trim_edg (:obj:`tuple`): + A tuple of two ints indicated how much of the slit edges should be + trimmed when fitting to the spatial profile. + debug (:obj:`bool`): + Show debugging plots? + """ + + msgs.info("Performing spatial sensitivity correction") + # Setup some helpful parameters + skymask_now = skymask if (skymask is not None) else np.ones_like(self.sciImg.image, dtype=bool) + hist_trim = 0 # Trim the edges of the histogram to take into account edge effects + gpm = self.sciImg.select_flag(invert=True) + slitid_img_init = self.slits.slit_img(pad=0, initial=True, flexure=self.spat_flexure_shift) + spatScaleImg = np.ones_like(self.sciImg.image) + # For each slit, grab the spatial coordinates and a spline + # representation of the spatial profile from the illumflat + rawimg = self.sciImg.image.copy() + numbins = int(np.max(self.slits.get_slitlengths(initial=True, median=True))) + spatbins = np.linspace(0.0, 1.0, numbins + 1) + spat_slit = 0.5 * (spatbins[1:] + spatbins[:-1]) + slitlength = np.median(self.slits.get_slitlengths(median=True)) + coeff_fit = np.zeros((self.slits.nslits, 2)) + for sl, slitnum in enumerate(self.slits.spat_id): + msgs.info("Deriving spatial correction for slit {0:d}/{1:d}".format(sl + 1, self.slits.spat_id.size)) + # Get the initial slit locations + onslit_b_init = (slitid_img_init == slitnum) + + # Synthesize ximg, and edgmask from slit boundaries. Doing this outside this + # routine would save time. But this is pretty fast, so we just do it here to make the interface simpler. + spatcoord, edgmask = pixels.ximg_and_edgemask(self.slits_left[:, sl], self.slits_right[:, sl], + onslit_b_init, trim_edg=trim_edg) + + # Make the model histogram + xspl = np.linspace(0.0, 1.0, 10 * int(slitlength)) # Sub sample each pixel with 10 subpixels + # TODO: caliBrate is no longer a dependency. If you need these b-splines pass them in. + modspl = self.caliBrate.flatimages.illumflat_spat_bsplines[sl].value(xspl)[0] + gradspl = interpolate.interp1d(xspl, np.gradient(modspl) / modspl, kind='linear', bounds_error=False, + fill_value='extrapolate') + + # Ignore skymask + coord_msk = onslit_b_init & gpm + hist, _ = np.histogram(spatcoord[coord_msk], bins=spatbins, weights=rawimg[coord_msk]) + cntr, _ = np.histogram(spatcoord[coord_msk], bins=spatbins) + hist_slit_all = hist / (cntr + (cntr == 0)) + histmod, _ = np.histogram(spatcoord[coord_msk], bins=spatbins, weights=gradspl(spatcoord[coord_msk])) + hist_model = histmod / (cntr + (cntr == 0)) + + # Repeat with skymask + coord_msk = onslit_b_init & gpm & skymask_now + hist, _ = np.histogram(spatcoord[coord_msk], bins=spatbins, weights=rawimg[coord_msk]) + cntr, _ = np.histogram(spatcoord[coord_msk], bins=spatbins) + hist_slit = hist / (cntr + (cntr == 0)) + + # Prepare for fit - take the non-zero elements and trim slit edges + if hist_trim == 0: + ww = (hist_slit != 0) + xfit = spat_slit[ww] + yfit = hist_slit_all[ww] + mfit = hist_model[ww] + else: + ww = (hist_slit[hist_trim:-hist_trim] != 0) + xfit = spat_slit[hist_trim:-hist_trim][ww] + yfit = hist_slit_all[hist_trim:-hist_trim][ww] + mfit = hist_model[hist_trim:-hist_trim][ww] + + # Fit the function + spat_func = lambda par, ydata, model: par[0]*(1 + par[1] * model) - ydata + res_lsq = least_squares(spat_func, [np.median(yfit), 0.0], args=(yfit, mfit)) + spatnorm = spat_func(res_lsq.x, 0.0, gradspl(spatcoord[onslit_b_init])) + spatnorm /= spat_func(res_lsq.x, 0.0, gradspl(0.5)) + # Set the scaling factor + spatScaleImg[onslit_b_init] = spatnorm + coeff_fit[sl, :] = res_lsq.x + + if debug: + from matplotlib import pyplot as plt + xplt = np.arange(24) + plt.subplot(121) + plt.plot(xplt[0::2], coeff_fit[::2, 0], 'rx') + plt.plot(xplt[1::2], coeff_fit[1::2, 0], 'bx') + plt.subplot(122) + plt.plot(xplt[0::2], coeff_fit[::2, 1]/10, 'rx') + plt.plot(xplt[1::2], coeff_fit[1::2, 1]/10, 'bx') + plt.show() + plt.imshow(spatScaleImg, vmin=0.99, vmax=1.01) + plt.show() + plt.subplot(133) + plt.plot(xplt[0::2], coeff_fit[::2, 2], 'rx') + plt.plot(xplt[1::2], coeff_fit[1::2, 2], 'bx') + plt.show() + # Apply the relative scale correction + self.apply_relative_scale(spatScaleImg) + +def illum_profile_spectral(self, global_sky, skymask=None): + """Calculate the residual spectral illumination profile using the sky regions. + This uses the same routine as the flatfield spectral illumination profile. + + Args: + global_sky (`numpy.ndarray`_): + Model of the sky + skymask (`numpy.ndarray`_, optional): + Mask of sky regions where the spectral illumination will be determined + """ + trim = self.par['calibrations']['flatfield']['slit_trim'] + sl_ref = self.par['calibrations']['flatfield']['slit_illum_ref_idx'] + smooth_npix = self.par['calibrations']['flatfield']['slit_illum_smooth_npix'] + gpm = self.sciImg.select_flag(invert=True) + # Note :: Need to provide wavelength to illum_profile_spectral (not the tilts) so that the + # relative spectral sensitivity is calculated at a given wavelength for all slits simultaneously. + scaleImg = flatfield.illum_profile_spectral(self.sciImg.image.copy(), self.waveimg, self.slits, + slit_illum_ref_idx=sl_ref, model=global_sky, gpmask=gpm, + skymask=skymask, trim=trim, flexure=self.spat_flexure_shift, + smooth_npix=smooth_npix) + # Now apply the correction to the science frame + self.apply_relative_scale(scaleImg) + + + +def global_skysub_old(self, skymask=None, update_crmask=True, trim_edg=(0,0), + previous_sky=None, show_fit=False, show=False, show_objs=False, objs_not_masked=False, + reinit_bpm:bool=True): + """ + Perform global sky subtraction. This SlicerIFU-specific routine ensures that the + edges of the slits are not trimmed, and performs a spatial and spectral + correction using the sky spectrum, if requested. See Reduce.global_skysub() + for parameter definitions. + + See base class method for description of parameters. + + Args: + reinit_bpm (:obj:`bool`, optional): + If True (default), the bpm is reinitialized to the initial bpm + Should be False on the final run in case there was a failure + upstream and no sources were found in the slit/order + """ + if self.par['reduce']['findobj']['skip_skysub']: + msgs.info("Skipping global sky sub as per user request") + return np.zeros_like(self.sciImg.image) + + # Generate a global sky sub for all slits separately + global_sky_sep = super().global_skysub(skymask=skymask, update_crmask=update_crmask, + trim_edg=trim_edg, show_fit=show_fit, show=show, + show_objs=show_objs, reinit_bpm=reinit_bpm) + # Check if any slits failed + if np.any(global_sky_sep[self.slitmask >= 0] == 0) and not self.bkg_redux: + # Cannot continue without a sky model for all slits + msgs.error("Global sky subtraction has failed for at least one slit.") + + # Check if flexure or a joint fit is requested + if not self.par['reduce']['skysub']['joint_fit'] and self.par['flexure']['spec_method'] == 'skip': + return global_sky_sep + if self.wv_calib is None: + msgs.error("A wavelength calibration is needed (wv_calib) if a joint sky fit is requested.") + msgs.info("Generating wavelength image") + + self.waveimg = self.wv_calib.build_waveimg(self.tilts, self.slits, spat_flexure=self.spat_flexure_shift) + # Calculate spectral flexure + method = self.par['flexure']['spec_method'] + # TODO :: Perhaps include a new label for IFU flexure correction - e.g. 'slitcen_relative' or 'slitcenIFU' or 'IFU' + # :: If a new label is introduced, change the other instances of 'method' (see below), and in flexure.spec_flexure_qa() + if method in ['slitcen']: + self.slitshift = self.calculate_flexure(global_sky_sep) + # Recalculate the wavelength image, and the global sky taking into account the spectral flexure + msgs.info("Generating wavelength image, now accounting for spectral flexure") + self.waveimg = self.wv_calib.build_waveimg(self.tilts, self.slits, spec_flexure=self.slitshift, + spat_flexure=self.spat_flexure_shift) + + + # If the joint fit or spec/spat sensitivity corrections are not being performed, return the separate slits sky + if not self.par['reduce']['skysub']['joint_fit']: + return global_sky_sep + + # Do the spatial scaling first + # if self.par['scienceframe']['process']['use_illumflat']: + # # Perform the correction + # self.illum_profile_spatial(skymask=skymask) + # # Re-generate a global sky sub for all slits separately + # global_sky_sep = Reduce.global_skysub(self, skymask=skymask, update_crmask=update_crmask, trim_edg=trim_edg, + # show_fit=show_fit, show=show, show_objs=show_objs) + + # Use sky information in all slits to perform a joint sky fit + global_sky = self.joint_skysub(skymask=skymask, update_crmask=update_crmask, trim_edg=trim_edg, + show_fit=show_fit, show=show, show_objs=show_objs, + objs_not_masked=objs_not_masked) + + return global_sky diff --git a/pypeit/find_objects.py b/pypeit/find_objects.py index e4c4186b79..756015ed84 100644 --- a/pypeit/find_objects.py +++ b/pypeit/find_objects.py @@ -957,7 +957,6 @@ class SlicerIFUFindObjects(MultiSlitFindObjects): """ def __init__(self, sciImg, slits, spectrograph, par, objtype, **kwargs): super().__init__(sciImg, slits, spectrograph, par, objtype, **kwargs) - #self.initialize_slits(slits, initial=True) # below is better. This was being done twice. def initialize_slits(self, slits, initial=True): """ @@ -1007,24 +1006,25 @@ def global_skysub(self, skymask=None, update_crmask=True, # Generate the waveimg which is needed if flexure is being computed self.waveimg = self.wv_calib.build_waveimg(self.tilts, self.slits, spat_flexure=self.spat_flexure_shift) - # TODO: Can the flexure compensation be removed from this class or is it necessary for the joint skysub? - # Calculate spectral flexure + # Calculate spectral flexure, so that we can align all slices of the IFU. We need to do this on the model + # sky spectrum. It must be performed in this class if the joint sky fit is requested, because all of + # the wavelengths need to be aligned for different slits before the sky is fit. method = self.par['flexure']['spec_method'] # TODO :: Perhaps include a new label for IFU flexure correction - e.g. 'slitcen_relative' or 'slitcenIFU' or 'IFU' # :: If a new label is introduced, change the other instances of 'method' (see below), and in flexure.spec_flexure_qa() if method in ['slitcen']: - # TODO make waveimg an argument here and return values pass back the new waveimg and shifts - self.calculate_flexure(global_sky_sep) + self.slitshift = self.calculate_flexure(global_sky_sep) + # Recalculate the wavelength image, and the global sky taking into account the spectral flexure + msgs.info("Generating wavelength image, accounting for spectral flexure") + self.waveimg = self.wv_calib.build_waveimg(self.tilts, self.slits, spec_flexure=self.slitshift, + spat_flexure=self.spat_flexure_shift) # If the joint fit or spec/spat sensitivity corrections are not being performed, return the separate slits sky if not self.par['reduce']['skysub']['joint_fit']: return global_sky_sep - # TODO I'm not following the control flow here. Is it that we need the initial global sky to compute flexure - # for each slit so that then one can perform joint sky-subtraction with flexure corrected wavelengths? Some - # more information on the logic of this flow would make it clearer. - + # If we reach this point in the code, a joint skysub has been requested. # Use sky information in all slits to perform a joint sky fit global_sky = self.joint_skysub(skymask=skymask, update_crmask=update_crmask, show_fit=show_fit, show=show, show_objs=show_objs, @@ -1154,12 +1154,22 @@ def joint_skysub(self, skymask=None, update_crmask=True, trim_edg=(0,0), # else like a flexure compensation class or module. def calculate_flexure(self, global_sky): """ - TODO: Need better docs. What is being assigned in this function? Need either docs or return values. - Convenience function to calculate the flexure of the IFU + Convenience function to calculate the flexure of an IFU. The flexure is calculated by cross-correlating the + sky model of a reference slit with an archival sky spectrum. This gives an "absolute" flexure correction for + the reference slit in pixels. Then, the flexure for all other slits is calculated by cross-correlating the + sky model of each slit with the sky model of the reference slit. This gives a "relative" flexure correction + for each slit in pixels. The relative flexure is then added to the absolute flexure to give the total flexure + correction for each slit in pixels. - Args: - global_sky (`numpy.ndarray`_): - Model of the sky + Parameters + ---------- + global_sky : ndarray + Sky model + + Returns + ------- + new_slitshift: ndarray + The flexure in pixels """ sl_ref = self.par['calibrations']['flatfield']['slit_illum_ref_idx'] box_rad = self.par['reduce']['extraction']['boxcar_radius'] @@ -1206,7 +1216,7 @@ def calculate_flexure(self, global_sky): # Replace the reference slit with the absolute shift flex_list[sl_ref] = flex_dict_ref.copy() # Add this flexure to the previous flexure correction - self.slitshift += this_slitshift + new_slitshift = self.slitshift + this_slitshift # Now report the flexure values for slit_idx, slit_spat in enumerate(self.slits.spat_id): msgs.info("Flexure correction, slit {0:d} (spat id={1:d}): {2:.3f} pixels".format(1+slit_idx, slit_spat, @@ -1219,13 +1229,7 @@ def calculate_flexure(self, global_sky): out_dir = os.path.join(self.par['rdx']['redux_path'], 'QA') slit_bpm = np.zeros(self.slits.nslits, dtype=bool) flexure.spec_flexure_qa(self.slits.slitord_id, slit_bpm, basename, flex_list, out_dir=out_dir) - - # Recalculate the wavelength image, and the global sky taking into account the spectral flexure - msgs.info("Generating wavelength image, accounting for spectral flexure") - self.waveimg = self.wv_calib.build_waveimg(self.tilts, self.slits, spec_flexure=self.slitshift, - spat_flexure=self.spat_flexure_shift) - return - + return new_slitshift def apply_relative_scale(self, scaleImg): """Apply a relative scale to the science frame (and correct the varframe, too) @@ -1245,202 +1249,3 @@ def apply_relative_scale(self, scaleImg): if np.any(_bpm): self.sciImg.update_mask('BADSCALE', indx=_bpm) self.sciImg.ivar = utils.inverse(varImg) - - # RJC :: THIS FUNCTION IS NOT CURRENTLY USED, BUT RJC REQUESTS TO KEEP THIS CODE HERE FOR THE TIME BEING. - # def illum_profile_spatial(self, skymask=None, trim_edg=(0, 0), debug=False): - # """ - # Calculate the residual spatial illumination profile using the sky regions. - # - # The residual is calculated using the differential: - # - # .. code-block:: python - # - # correction = amplitude * (1 + spatial_shift * (dy/dx)/y) - # - # where ``y`` is the spatial profile determined from illumflat, and - # spatial_shift is the residual spatial flexure shift in units of pixels. - # - # Args: - # skymask (`numpy.ndarray`_): - # Mask of sky regions where the spatial illumination will be determined - # trim_edg (:obj:`tuple`): - # A tuple of two ints indicated how much of the slit edges should be - # trimmed when fitting to the spatial profile. - # debug (:obj:`bool`): - # Show debugging plots? - # """ - # - # msgs.info("Performing spatial sensitivity correction") - # # Setup some helpful parameters - # skymask_now = skymask if (skymask is not None) else np.ones_like(self.sciImg.image, dtype=bool) - # hist_trim = 0 # Trim the edges of the histogram to take into account edge effects - # gpm = self.sciImg.select_flag(invert=True) - # slitid_img_init = self.slits.slit_img(pad=0, initial=True, flexure=self.spat_flexure_shift) - # spatScaleImg = np.ones_like(self.sciImg.image) - # # For each slit, grab the spatial coordinates and a spline - # # representation of the spatial profile from the illumflat - # rawimg = self.sciImg.image.copy() - # numbins = int(np.max(self.slits.get_slitlengths(initial=True, median=True))) - # spatbins = np.linspace(0.0, 1.0, numbins + 1) - # spat_slit = 0.5 * (spatbins[1:] + spatbins[:-1]) - # slitlength = np.median(self.slits.get_slitlengths(median=True)) - # coeff_fit = np.zeros((self.slits.nslits, 2)) - # for sl, slitnum in enumerate(self.slits.spat_id): - # msgs.info("Deriving spatial correction for slit {0:d}/{1:d}".format(sl + 1, self.slits.spat_id.size)) - # # Get the initial slit locations - # onslit_b_init = (slitid_img_init == slitnum) - # - # # Synthesize ximg, and edgmask from slit boundaries. Doing this outside this - # # routine would save time. But this is pretty fast, so we just do it here to make the interface simpler. - # spatcoord, edgmask = pixels.ximg_and_edgemask(self.slits_left[:, sl], self.slits_right[:, sl], - # onslit_b_init, trim_edg=trim_edg) - # - # # Make the model histogram - # xspl = np.linspace(0.0, 1.0, 10 * int(slitlength)) # Sub sample each pixel with 10 subpixels - # # TODO: caliBrate is no longer a dependency. If you need these b-splines pass them in. - # modspl = self.caliBrate.flatimages.illumflat_spat_bsplines[sl].value(xspl)[0] - # gradspl = interpolate.interp1d(xspl, np.gradient(modspl) / modspl, kind='linear', bounds_error=False, - # fill_value='extrapolate') - # - # # Ignore skymask - # coord_msk = onslit_b_init & gpm - # hist, _ = np.histogram(spatcoord[coord_msk], bins=spatbins, weights=rawimg[coord_msk]) - # cntr, _ = np.histogram(spatcoord[coord_msk], bins=spatbins) - # hist_slit_all = hist / (cntr + (cntr == 0)) - # histmod, _ = np.histogram(spatcoord[coord_msk], bins=spatbins, weights=gradspl(spatcoord[coord_msk])) - # hist_model = histmod / (cntr + (cntr == 0)) - # - # # Repeat with skymask - # coord_msk = onslit_b_init & gpm & skymask_now - # hist, _ = np.histogram(spatcoord[coord_msk], bins=spatbins, weights=rawimg[coord_msk]) - # cntr, _ = np.histogram(spatcoord[coord_msk], bins=spatbins) - # hist_slit = hist / (cntr + (cntr == 0)) - # - # # Prepare for fit - take the non-zero elements and trim slit edges - # if hist_trim == 0: - # ww = (hist_slit != 0) - # xfit = spat_slit[ww] - # yfit = hist_slit_all[ww] - # mfit = hist_model[ww] - # else: - # ww = (hist_slit[hist_trim:-hist_trim] != 0) - # xfit = spat_slit[hist_trim:-hist_trim][ww] - # yfit = hist_slit_all[hist_trim:-hist_trim][ww] - # mfit = hist_model[hist_trim:-hist_trim][ww] - # - # # Fit the function - # spat_func = lambda par, ydata, model: par[0]*(1 + par[1] * model) - ydata - # res_lsq = least_squares(spat_func, [np.median(yfit), 0.0], args=(yfit, mfit)) - # spatnorm = spat_func(res_lsq.x, 0.0, gradspl(spatcoord[onslit_b_init])) - # spatnorm /= spat_func(res_lsq.x, 0.0, gradspl(0.5)) - # # Set the scaling factor - # spatScaleImg[onslit_b_init] = spatnorm - # coeff_fit[sl, :] = res_lsq.x - # - # if debug: - # from matplotlib import pyplot as plt - # xplt = np.arange(24) - # plt.subplot(121) - # plt.plot(xplt[0::2], coeff_fit[::2, 0], 'rx') - # plt.plot(xplt[1::2], coeff_fit[1::2, 0], 'bx') - # plt.subplot(122) - # plt.plot(xplt[0::2], coeff_fit[::2, 1]/10, 'rx') - # plt.plot(xplt[1::2], coeff_fit[1::2, 1]/10, 'bx') - # plt.show() - # plt.imshow(spatScaleImg, vmin=0.99, vmax=1.01) - # plt.show() - # plt.subplot(133) - # plt.plot(xplt[0::2], coeff_fit[::2, 2], 'rx') - # plt.plot(xplt[1::2], coeff_fit[1::2, 2], 'bx') - # plt.show() - # # Apply the relative scale correction - # self.apply_relative_scale(spatScaleImg) - - # TODO This method appears to not be used. Should be removed? - def illum_profile_spectral(self, global_sky, skymask=None): - """Calculate the residual spectral illumination profile using the sky regions. - This uses the same routine as the flatfield spectral illumination profile. - - Args: - global_sky (`numpy.ndarray`_): - Model of the sky - skymask (`numpy.ndarray`_, optional): - Mask of sky regions where the spectral illumination will be determined - """ - trim = self.par['calibrations']['flatfield']['slit_trim'] - sl_ref = self.par['calibrations']['flatfield']['slit_illum_ref_idx'] - smooth_npix = self.par['calibrations']['flatfield']['slit_illum_smooth_npix'] - gpm = self.sciImg.select_flag(invert=True) - # Note :: Need to provide wavelength to illum_profile_spectral (not the tilts) so that the - # relative spectral sensitivity is calculated at a given wavelength for all slits simultaneously. - scaleImg = flatfield.illum_profile_spectral(self.sciImg.image.copy(), self.waveimg, self.slits, - slit_illum_ref_idx=sl_ref, model=global_sky, gpmask=gpm, - skymask=skymask, trim=trim, flexure=self.spat_flexure_shift, - smooth_npix=smooth_npix) - # Now apply the correction to the science frame - self.apply_relative_scale(scaleImg) - - - def global_skysub_old(self, skymask=None, update_crmask=True, trim_edg=(0,0), - previous_sky=None, show_fit=False, show=False, show_objs=False, objs_not_masked=False, - reinit_bpm:bool=True): - """ - Perform global sky subtraction. This SlicerIFU-specific routine ensures that the - edges of the slits are not trimmed, and performs a spatial and spectral - correction using the sky spectrum, if requested. See Reduce.global_skysub() - for parameter definitions. - - See base class method for description of parameters. - - Args: - reinit_bpm (:obj:`bool`, optional): - If True (default), the bpm is reinitialized to the initial bpm - Should be False on the final run in case there was a failure - upstream and no sources were found in the slit/order - """ - if self.par['reduce']['findobj']['skip_skysub']: - msgs.info("Skipping global sky sub as per user request") - return np.zeros_like(self.sciImg.image) - - # Generate a global sky sub for all slits separately - global_sky_sep = super().global_skysub(skymask=skymask, update_crmask=update_crmask, - trim_edg=trim_edg, show_fit=show_fit, show=show, - show_objs=show_objs, reinit_bpm=reinit_bpm) - # Check if any slits failed - if np.any(global_sky_sep[self.slitmask >= 0] == 0) and not self.bkg_redux: - # Cannot continue without a sky model for all slits - msgs.error("Global sky subtraction has failed for at least one slit.") - - # Check if flexure or a joint fit is requested - if not self.par['reduce']['skysub']['joint_fit'] and self.par['flexure']['spec_method'] == 'skip': - return global_sky_sep - if self.wv_calib is None: - msgs.error("A wavelength calibration is needed (wv_calib) if a joint sky fit is requested.") - msgs.info("Generating wavelength image") - - self.waveimg = self.wv_calib.build_waveimg(self.tilts, self.slits, spat_flexure=self.spat_flexure_shift) - # Calculate spectral flexure - method = self.par['flexure']['spec_method'] - # TODO :: Perhaps include a new label for IFU flexure correction - e.g. 'slitcen_relative' or 'slitcenIFU' or 'IFU' - # :: If a new label is introduced, change the other instances of 'method' (see below), and in flexure.spec_flexure_qa() - if method in ['slitcen']: - self.calculate_flexure(global_sky_sep) - - # If the joint fit or spec/spat sensitivity corrections are not being performed, return the separate slits sky - if not self.par['reduce']['skysub']['joint_fit']: - return global_sky_sep - - # Do the spatial scaling first - # if self.par['scienceframe']['process']['use_illumflat']: - # # Perform the correction - # self.illum_profile_spatial(skymask=skymask) - # # Re-generate a global sky sub for all slits separately - # global_sky_sep = Reduce.global_skysub(self, skymask=skymask, update_crmask=update_crmask, trim_edg=trim_edg, - # show_fit=show_fit, show=show, show_objs=show_objs) - - # Use sky information in all slits to perform a joint sky fit - global_sky = self.joint_skysub(skymask=skymask, update_crmask=update_crmask, trim_edg=trim_edg, - show_fit=show_fit, show=show, show_objs=show_objs, - objs_not_masked=objs_not_masked) - - return global_sky From d96f99b4cde973bf756b379996a9c826fa2096ed Mon Sep 17 00:00:00 2001 From: rcooke Date: Sun, 7 Jan 2024 14:45:55 +0000 Subject: [PATCH 172/244] updated IFU find_objects + flexure correction + docs --- pypeit/find_objects.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pypeit/find_objects.py b/pypeit/find_objects.py index 756015ed84..a8bca7a557 100644 --- a/pypeit/find_objects.py +++ b/pypeit/find_objects.py @@ -1150,8 +1150,7 @@ def joint_skysub(self, skymask=None, update_crmask=True, trim_edg=(0,0), self.show('global', global_sky=_global_sky, slits=True, sobjs=sobjs_show, clear=False) return _global_sky - # TODO is the flexure compensation needed in object finding. Can it be separated from this class and put somewhere - # else like a flexure compensation class or module. + # TODO :: This function should be removed from the find_objects() class, once the flexure code has been tidied up. def calculate_flexure(self, global_sky): """ Convenience function to calculate the flexure of an IFU. The flexure is calculated by cross-correlating the @@ -1222,7 +1221,8 @@ def calculate_flexure(self, global_sky): msgs.info("Flexure correction, slit {0:d} (spat id={1:d}): {2:.3f} pixels".format(1+slit_idx, slit_spat, self.slitshift[slit_idx])) # Save QA - # TODO :: Need to implement QA + # TODO :: Need to implement QA once the flexure code has been tidied up, and this routine has been moved + # out of the find_objects() class. msgs.work("QA is not currently implemented for the flexure correction") if False:#flex_list is not None: basename = f'{self.basename}_global_{self.spectrograph.get_det_name(self.det)}' From b096b71b42b93979a2d6f06ed59ad0804c8d73d5 Mon Sep 17 00:00:00 2001 From: jhennawi Date: Mon, 8 Jan 2024 09:20:16 -0800 Subject: [PATCH 173/244] added default telluric grid file. --- pypeit/spectrographs/keck_kcwi.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pypeit/spectrographs/keck_kcwi.py b/pypeit/spectrographs/keck_kcwi.py index 4d6008a9a6..5743eaa6eb 100644 --- a/pypeit/spectrographs/keck_kcwi.py +++ b/pypeit/spectrographs/keck_kcwi.py @@ -313,6 +313,9 @@ def default_pypeit_par(cls): # Flux calibration parameters par['sensfunc']['UVIS']['extinct_correct'] = False # This must be False - the extinction correction is performed when making the datacube + # If telluric is triggered + par['sensfunc']['IR']['telgridfile'] = 'TellPCA_3000_26000_R15000.fits' + return par def pypeit_file_keys(self): From ac1cdc619bbdf88fd7001f50ff29383b45ddc8af Mon Sep 17 00:00:00 2001 From: jhennawi Date: Mon, 8 Jan 2024 09:26:40 -0800 Subject: [PATCH 174/244] Enabling 1d sensunfunc for SlicerIFUs --- pypeit/specobjs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pypeit/specobjs.py b/pypeit/specobjs.py index baf3a7f1d1..8578d382d6 100644 --- a/pypeit/specobjs.py +++ b/pypeit/specobjs.py @@ -572,7 +572,9 @@ def apply_flux_calib(self, par, spectrograph, sens): _extinct_correct = (True if sens.algorithm == 'UVIS' else False) \ if par['extinct_correct'] is None else par['extinct_correct'] - if spectrograph.pypeline == 'MultiSlit': + # TODO enbaling this for now in case someone wants to treat the IFU as a slit spectrograph + # (not recommnneded but useful for quick reductions where you don't want to construct cubes and don't care about DAR). + if spectrograph.pypeline in ['Multislit','SlicerIFU']: for ii, sci_obj in enumerate(self.specobjs): if sens.wave.shape[1] == 1: sci_obj.apply_flux_calib(sens.wave[:, 0], sens.zeropoint[:, 0], From 83184c1be5fe0af319df0994c6359cd1184326d9 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Mon, 8 Jan 2024 12:16:02 -0600 Subject: [PATCH 175/244] doc fixes --- doc/help/run_pypeit.rst | 2 +- .../class_datamodel_detectorcontainer.rst | 46 +-- doc/include/class_datamodel_onespec.rst | 3 +- doc/include/class_datamodel_telluric.rst | 4 +- doc/include/datamodel_onespec.rst | 2 +- doc/include/datamodel_sensfunc.rst | 47 ++- doc/include/datamodel_telluric.rst | 47 ++- doc/pypeit_par.rst | 239 ++++++------ pypeit/coadd2d.py | 69 ++-- pypeit/core/coadd.py | 58 +-- pypeit/core/datacube.py | 342 ++++++++++-------- pypeit/core/telluric.py | 2 +- pypeit/find_objects.py | 9 +- pypeit/images/detector_container.py | 2 +- 14 files changed, 466 insertions(+), 406 deletions(-) diff --git a/doc/help/run_pypeit.rst b/doc/help/run_pypeit.rst index 6a5293f4c7..23b9ade3c4 100644 --- a/doc/help/run_pypeit.rst +++ b/doc/help/run_pypeit.rst @@ -4,7 +4,7 @@ usage: run_pypeit [-h] [-v VERBOSITY] [-r REDUX_PATH] [-m] [-s] [-o] [-c] pypeit_file - ## PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.14.1.dev477+g4a02a85ca + ## PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.14.1.dev768+gac1cdc619.d20240108 ## ## Available spectrographs include: ## bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, diff --git a/doc/include/class_datamodel_detectorcontainer.rst b/doc/include/class_datamodel_detectorcontainer.rst index 9154cbe8d9..e5a4d49b7b 100644 --- a/doc/include/class_datamodel_detectorcontainer.rst +++ b/doc/include/class_datamodel_detectorcontainer.rst @@ -1,26 +1,26 @@ **Version**: 1.0.1 -================= ================ ================= ======================================================================================================================================================================================================================================================== -Attribute Type Array Type Description -================= ================ ================= ======================================================================================================================================================================================================================================================== -``binning`` str Binning in PypeIt orientation (not the original) -``darkcurr`` int, float Dark current (e-/pixel/hour) -``dataext`` int Index of fits extension containing data -``datasec`` `numpy.ndarray`_ str Either the data sections or the header keyword where the valid data sections can be obtained, one per amplifier. If defined explicitly should be in FITS format (e.g., [1:2048,10:4096]). -``det`` int PypeIt designation for detector number (1-based). -``gain`` `numpy.ndarray`_ `numpy.floating`_ Inverse gain (e-/ADU). A list should be provided if a detector contains more than one amplifier. -``mincounts`` int, float Counts (e-) in a pixel below this value will be ignored as being unphysical. -``nonlinear`` int, float Percentage of detector range which is linear (i.e. everything above ``nonlinear*saturation`` will be flagged as saturated) -``numamplifiers`` int Number of amplifiers -``oscansec`` `numpy.ndarray`_ str Either the overscan section or the header keyword where the valid data sections can be obtained, one per amplifier. If defined explicitly should be in FITS format (e.g., [1:2048,10:4096]). -``platescale`` int, float arcsec per pixel in the spatial dimension for an unbinned pixel -``ronoise`` `numpy.ndarray`_ `numpy.floating`_ Read-out noise (e-). A list should be provided if a detector contains more than one amplifier. If any element of this list is <=0, the readout noise will be determined from the overscan regions defined by oscansec. -``saturation`` int, float The detector saturation level in ADU/DN -``spatflip`` bool If this is True then the spatial dimension will be flipped. PypeIt expects echelle orders to increase with increasing pixel number. I.e., setting spatflip=True can reorder images so that blue orders appear on the left and red orders on the right. -``specaxis`` int Spectra are dispersed along this axis. Allowed values are 0 (first dimension for a numpy array shape) or 1 (second dimension for numpy array shape). -``specflip`` bool If this is True then the dispersion dimension (specified by the specaxis) will be flipped. PypeIt expects wavelengths to increase with increasing pixel number. If this is not the case for this instrument, set specflip to True. -``xgap`` int, float Gap between the square detector pixels (expressed as a fraction of the x pixel size -- x is predominantly the spatial axis) -``ygap`` int, float Gap between the square detector pixels (expressed as a fraction of the y pixel size -- y is predominantly the spectral axis) -``ysize`` int, float The size of a pixel in the y-direction as a multiple of the x pixel size (i.e. xsize = 1.0 -- x is predominantly the dispersion axis) -================= ================ ================= ======================================================================================================================================================================================================================================================== +================= ===================== ================= ======================================================================================================================================================================================================================================================== +Attribute Type Array Type Description +================= ===================== ================= ======================================================================================================================================================================================================================================================== +``binning`` str Binning in PypeIt orientation (not the original) +``darkcurr`` int, float Dark current (e-/pixel/hour) +``dataext`` int Index of fits extension containing data +``datasec`` `numpy.ndarray`_ str Either the data sections or the header keyword where the valid data sections can be obtained, one per amplifier. If defined explicitly should be in FITS format (e.g., [1:2048,10:4096]). +``det`` int, `numpy.integer`_ PypeIt designation for detector number (1-based). +``gain`` `numpy.ndarray`_ `numpy.floating`_ Inverse gain (e-/ADU). A list should be provided if a detector contains more than one amplifier. +``mincounts`` int, float Counts (e-) in a pixel below this value will be ignored as being unphysical. +``nonlinear`` int, float Percentage of detector range which is linear (i.e. everything above ``nonlinear*saturation`` will be flagged as saturated) +``numamplifiers`` int Number of amplifiers +``oscansec`` `numpy.ndarray`_ str Either the overscan section or the header keyword where the valid data sections can be obtained, one per amplifier. If defined explicitly should be in FITS format (e.g., [1:2048,10:4096]). +``platescale`` int, float arcsec per pixel in the spatial dimension for an unbinned pixel +``ronoise`` `numpy.ndarray`_ `numpy.floating`_ Read-out noise (e-). A list should be provided if a detector contains more than one amplifier. If any element of this list is <=0, the readout noise will be determined from the overscan regions defined by oscansec. +``saturation`` int, float The detector saturation level in ADU/DN +``spatflip`` bool If this is True then the spatial dimension will be flipped. PypeIt expects echelle orders to increase with increasing pixel number. I.e., setting spatflip=True can reorder images so that blue orders appear on the left and red orders on the right. +``specaxis`` int Spectra are dispersed along this axis. Allowed values are 0 (first dimension for a numpy array shape) or 1 (second dimension for numpy array shape). +``specflip`` bool If this is True then the dispersion dimension (specified by the specaxis) will be flipped. PypeIt expects wavelengths to increase with increasing pixel number. If this is not the case for this instrument, set specflip to True. +``xgap`` int, float Gap between the square detector pixels (expressed as a fraction of the x pixel size -- x is predominantly the spatial axis) +``ygap`` int, float Gap between the square detector pixels (expressed as a fraction of the y pixel size -- y is predominantly the spectral axis) +``ysize`` int, float The size of a pixel in the y-direction as a multiple of the x pixel size (i.e. xsize = 1.0 -- x is predominantly the dispersion axis) +================= ===================== ================= ======================================================================================================================================================================================================================================================== diff --git a/doc/include/class_datamodel_onespec.rst b/doc/include/class_datamodel_onespec.rst index eff409ec4d..a773421d33 100644 --- a/doc/include/class_datamodel_onespec.rst +++ b/doc/include/class_datamodel_onespec.rst @@ -1,5 +1,5 @@ -**Version**: 1.0.1 +**Version**: 1.0.2 ================= ================ ================= ========================================================================================================================================== Attribute Type Array Type Description @@ -11,6 +11,7 @@ Attribute Type Array Type Description ``ivar`` `numpy.ndarray`_ `numpy.floating`_ Inverse variance array (matches units of flux) ``mask`` `numpy.ndarray`_ `numpy.integer`_ Mask array (1=Good,0=Bad) ``obj_model`` `numpy.ndarray`_ `numpy.floating`_ Object model for tellurics +``sigma`` `numpy.ndarray`_ `numpy.floating`_ One sigma noise array, equivalent to 1/sqrt(ivar) (matches units of flux) ``spect_meta`` dict header dict ``telluric`` `numpy.ndarray`_ `numpy.floating`_ Telluric model ``wave`` `numpy.ndarray`_ `numpy.floating`_ Wavelength array (angstroms in vacuum), weighted by pixel contributions diff --git a/doc/include/class_datamodel_telluric.rst b/doc/include/class_datamodel_telluric.rst index 7143c793ac..f9f749f007 100644 --- a/doc/include/class_datamodel_telluric.rst +++ b/doc/include/class_datamodel_telluric.rst @@ -20,8 +20,10 @@ Attribute Type Array Type Description ``std_name`` str Type of standard source ``std_ra`` float RA of the standard source ``std_src`` str Name of the standard source -``telgrid`` str File containing grid of HITRAN atmosphere models +``telgrid`` str File containing PCA components or grid of HITRAN atmosphere models ``tell_norm_thresh`` float ?? +``tell_npca`` int Number of telluric PCA components used +``teltype`` str Type of telluric model, `pca` or `grid` ``tol`` float Relative tolerance for converage of the differential evolution optimization. ``ubound_norm`` float Flux normalization upper bound ``z_qso`` float Redshift of the QSO diff --git a/doc/include/datamodel_onespec.rst b/doc/include/datamodel_onespec.rst index 6eac2e340a..302e3a1fe5 100644 --- a/doc/include/datamodel_onespec.rst +++ b/doc/include/datamodel_onespec.rst @@ -1,5 +1,5 @@ -Version 1.0.1 +Version 1.0.2 ============ ============================== ========= =========================================================== HDU Name HDU Type Data Type Description diff --git a/doc/include/datamodel_sensfunc.rst b/doc/include/datamodel_sensfunc.rst index 2d9e9b5bf2..165d92e3b9 100644 --- a/doc/include/datamodel_sensfunc.rst +++ b/doc/include/datamodel_sensfunc.rst @@ -15,31 +15,28 @@ HDU Name HDU Type Data Type Description TELLURIC table (if present) -================= ========= =========================================================== -Column Data Type Description -================= ========= =========================================================== -``WAVE`` float64 Wavelength vector -``TELLURIC`` float64 Best-fitting telluric model spectrum -``OBJ_MODEL`` float64 Best-fitting object model spectrum -``TELL_THETA`` float64 Best-fitting telluric model parameters -``TELL_PRESS`` float64 Best-fitting telluric model pressure -``TELL_TEMP`` float64 Best-fitting telluric model temperature -``TELL_H2O`` float64 Best-fitting telluric model humidity -``TELL_AIRMASS`` float64 Best-fitting telluric model airmass -``TELL_RESLN`` float64 Best-fitting telluric model spectral resolution -``TELL_SHIFT`` float64 Best-fitting shift applied to the telluric model spectrum -``TELL_STRETCH`` float64 Best-fitting stretch applied to the telluric model spectrum -``OBJ_THETA`` float64 Best-fitting object model parameters -``CHI2`` float64 Chi-square of the best-fit model -``SUCCESS`` bool Flag that fit was successful -``NITER`` int64 Number of fit iterations -``ECH_ORDERS`` int64 Echelle order for this specrum (echelle data only) -``POLYORDER_VEC`` int64 Polynomial order for each slit/echelle (if applicable) -``IND_LOWER`` int64 Lowest index of a spectral pixel included in the fit -``IND_UPPER`` int64 Highest index of a spectral pixel included in the fit -``WAVE_MIN`` float64 Minimum wavelength included in the fit -``WAVE_MAX`` float64 Maximum wavelength included in the fit -================= ========= =========================================================== +================= ========= ================================================================ +Column Data Type Description +================= ========= ================================================================ +``WAVE`` float64 Wavelength vector +``TELLURIC`` float64 Best-fitting telluric model spectrum +``OBJ_MODEL`` float64 Best-fitting object model spectrum +``TELL_THETA`` float64 Best-fitting telluric model parameters +``TELL_PARAM`` float64 Best-fitting telluric atmospheric parameters or PCA coefficients +``TELL_RESLN`` float64 Best-fitting telluric model spectral resolution +``TELL_SHIFT`` float64 Best-fitting shift applied to the telluric model spectrum +``TELL_STRETCH`` float64 Best-fitting stretch applied to the telluric model spectrum +``OBJ_THETA`` float64 Best-fitting object model parameters +``CHI2`` float64 Chi-square of the best-fit model +``SUCCESS`` bool Flag that fit was successful +``NITER`` int64 Number of fit iterations +``ECH_ORDERS`` int64 Echelle order for this specrum (echelle data only) +``POLYORDER_VEC`` int64 Polynomial order for each slit/echelle (if applicable) +``IND_LOWER`` int64 Lowest index of a spectral pixel included in the fit +``IND_UPPER`` int64 Highest index of a spectral pixel included in the fit +``WAVE_MIN`` float64 Minimum wavelength included in the fit +``WAVE_MAX`` float64 Maximum wavelength included in the fit +================= ========= ================================================================ SENS table diff --git a/doc/include/datamodel_telluric.rst b/doc/include/datamodel_telluric.rst index aac5007b4e..d22cbf732c 100644 --- a/doc/include/datamodel_telluric.rst +++ b/doc/include/datamodel_telluric.rst @@ -11,28 +11,25 @@ HDU Name HDU Type Data Type Description TELLURIC table -================= ========= =========================================================== -Column Data Type Description -================= ========= =========================================================== -``WAVE`` float64 Wavelength vector -``TELLURIC`` float64 Best-fitting telluric model spectrum -``OBJ_MODEL`` float64 Best-fitting object model spectrum -``TELL_THETA`` float64 Best-fitting telluric model parameters -``TELL_PRESS`` float64 Best-fitting telluric model pressure -``TELL_TEMP`` float64 Best-fitting telluric model temperature -``TELL_H2O`` float64 Best-fitting telluric model humidity -``TELL_AIRMASS`` float64 Best-fitting telluric model airmass -``TELL_RESLN`` float64 Best-fitting telluric model spectral resolution -``TELL_SHIFT`` float64 Best-fitting shift applied to the telluric model spectrum -``TELL_STRETCH`` float64 Best-fitting stretch applied to the telluric model spectrum -``OBJ_THETA`` float64 Best-fitting object model parameters -``CHI2`` float64 Chi-square of the best-fit model -``SUCCESS`` bool Flag that fit was successful -``NITER`` int64 Number of fit iterations -``ECH_ORDERS`` int64 Echelle order for this specrum (echelle data only) -``POLYORDER_VEC`` int64 Polynomial order for each slit/echelle (if applicable) -``IND_LOWER`` int64 Lowest index of a spectral pixel included in the fit -``IND_UPPER`` int64 Highest index of a spectral pixel included in the fit -``WAVE_MIN`` float64 Minimum wavelength included in the fit -``WAVE_MAX`` float64 Maximum wavelength included in the fit -================= ========= =========================================================== +================= ========= ================================================================ +Column Data Type Description +================= ========= ================================================================ +``WAVE`` float64 Wavelength vector +``TELLURIC`` float64 Best-fitting telluric model spectrum +``OBJ_MODEL`` float64 Best-fitting object model spectrum +``TELL_THETA`` float64 Best-fitting telluric model parameters +``TELL_PARAM`` float64 Best-fitting telluric atmospheric parameters or PCA coefficients +``TELL_RESLN`` float64 Best-fitting telluric model spectral resolution +``TELL_SHIFT`` float64 Best-fitting shift applied to the telluric model spectrum +``TELL_STRETCH`` float64 Best-fitting stretch applied to the telluric model spectrum +``OBJ_THETA`` float64 Best-fitting object model parameters +``CHI2`` float64 Chi-square of the best-fit model +``SUCCESS`` bool Flag that fit was successful +``NITER`` int64 Number of fit iterations +``ECH_ORDERS`` int64 Echelle order for this specrum (echelle data only) +``POLYORDER_VEC`` int64 Polynomial order for each slit/echelle (if applicable) +``IND_LOWER`` int64 Lowest index of a spectral pixel included in the fit +``IND_UPPER`` int64 Highest index of a spectral pixel included in the fit +``WAVE_MIN`` float64 Minimum wavelength included in the fit +``WAVE_MAX`` float64 Maximum wavelength included in the fit +================= ========= ================================================================ diff --git a/doc/pypeit_par.rst b/doc/pypeit_par.rst index da16794add..fd0ad9835f 100644 --- a/doc/pypeit_par.rst +++ b/doc/pypeit_par.rst @@ -323,7 +323,7 @@ Key Type Options ``slit_illum_finecorr`` bool .. True If True, a fine correction to the spatial illumination profile will be performed. The fine correction is a low order 2D polynomial fit to account for a gradual change to the spatial illumination profile as a function of wavelength. ``slit_illum_pad`` int, float .. 5.0 The number of pixels to pad the slit edges when constructing the slit-illumination profile. Single value applied to both edges. ``slit_illum_ref_idx`` int .. 0 The index of a reference slit (0-indexed) used for estimating the relative spectral sensitivity (or the relative blaze). This parameter is only used if ``slit_illum_relative = True``. -``slit_illum_relative`` bool .. False Generate an image of the relative spectral illumination for a multi-slit setup. If you set ``use_slitillum = True`` for any of the frames that use the flatfield model, this *must* be set to True. Currently, this is only used for SlicerIFU reductions. +``slit_illum_relative`` bool .. False Generate an image of the relative spectral illumination for a multi-slit setup. If you set ``use_specillum = True`` for any of the frames that use the flatfield model, this *must* be set to True. Currently, this is only used for SlicerIFU reductions. ``slit_illum_smooth_npix`` int .. 10 The number of pixels used to determine smoothly varying relative weights is given by ``nspec/slit_illum_smooth_npix``, where nspec is the number of spectral pixels. ``slit_trim`` int, float, tuple .. 3.0 The number of pixels to trim each side of the slit when selecting pixels to use for fitting the spectral response function. Single values are used for both slit edges; a two-tuple can be used to trim the left and right sides differently. ``spat_samp`` int, float .. 5.0 Spatial sampling for slit illumination function. This is the width of the median filter in pixels used to determine the slit illumination function, and thus sets the minimum scale on which the illumination function will have features. @@ -506,40 +506,41 @@ Coadd1DPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.Coadd1DPar` -==================== ========== ======= ========== ========================================================================================================================================================================================================================================================================================================================================================================================================================================= -Key Type Options Default Description -==================== ========== ======= ========== ========================================================================================================================================================================================================================================================================================================================================================================================================================================= -``chk_version`` bool .. True If True enforce strict PypeIt version checking to ensure that spec1d*.fits files were createdwith the current version of PypeIt -``coaddfile`` str .. .. Output filename -``dloglam`` int, float .. .. Dispersion in units of log10(wave) in case you want to specify it in the get_wave_grid (for the 'velocity' or 'log10' options), otherwise a median value is computed from the data. -``dv`` int, float .. .. Dispersion in units of km/s in case you want to specify it in the get_wave_grid (for the 'velocity' option), otherwise a median value is computed from the data. -``dwave`` int, float .. .. Dispersion in Angstroms in case you want to specify it in the get_wave_grid (for the 'linear' option), otherwise a median value is computed from the data. -``ex_value`` str .. ``OPT`` The extraction to coadd, i.e. optimal or boxcar. Must be either 'OPT' or 'BOX' -``filter`` str .. ``none`` Filter for scaling. See flux_calib.load_fitler_file() for naming. Ignore if none -``filter_mag`` float .. .. Magnitude of the source in the given filter -``filter_mask`` str, list .. .. List of wavelength regions to mask when doing the scaling (`i.e.`, occasional junk pixels). Colon and comma separateed, e.g. 5552:5559,6010:6030 -``flux_value`` bool .. True If True (default), the code will coadd the fluxed spectra (i.e. the FLAM) in the spec1d files. If False, it will coadd the counts. -``lower`` int, float .. 3.0 Lower rejection threshold used for rejecting pixels when combining spectra in units of sigma. -``mag_type`` str .. ``AB`` Magnitude type. AB is the only option currently allowed -``maxiter_reject`` int .. 5 Maximum number of iterations for stacking and rejection. The code stops iterating either when the output mask does not change betweeen successive iterations or when maxiter_reject is reached. -``maxiter_scale`` int .. 5 Maximum number of iterations performed for rescaling spectra. -``maxrej`` int .. .. Coadding performs iterative rejection by comparing each exposure to a preliminary stack of all the exposures. If this parameter is set then it will not reject more than maxrej pixels per iteration of this rejection. The default is None, which means no maximum on rejected pixels. -``nbests`` list, int .. .. Number of orders to use for estimating the per exposure weights. Default is None, which will just use one fourth of the total number of orders. This is only used for Echelle -``nmaskedge`` int .. 2 Number of edge pixels to mask. This should be removed/fixed. -``ref_percentile`` int, float .. 70.0 Percentile used for selecting the minimum SNR cut from a reference spectrum used to robustly determine the median ratio between spectra. This parameter is used by coadd1d.robust_median_ratio as part of the automatic rescaling procedure. Pixels above this percentile cut are deemed the "good" pixels and are used to compute the ratio of two spectra. This must be a number between 0 and 100. -``scale_method`` str .. ``auto`` Method used to rescale the spectra prior to coadding. The options are: 'auto' -- Determine the scaling method automatically based on the S/N ratio which works well. 'poly' -- Polynomial rescaling. 'median' -- Median rescaling 'none' -- Do not rescale. 'hand' -- Pass in hand scaling factors. This option is not well tested. -``sigrej_exp`` int, float .. .. Rejection threshold used for rejecting exposures with S/N more than sigrej_exp*sigma above the median S/N. If None (the default), no rejection is performed. Currently, only available for multi-slit observations. -``sigrej_scale`` int, float .. 3.0 Rejection threshold used for rejecting pixels when rescaling spectra with scale_spec. -``sn_clip`` int, float .. 30.0 Errors are capped during rejection so that the S/N is never greater than sn_clip. This prevents overly aggressive rejection in high S/N ratio spectrum which neverthless differ at a level greater than the formal S/N due to systematics. -``sn_min_medscale`` int, float .. 0.5 For scale method set to ``auto``, this sets the minimum SNR for which median scaling is attempted. -``sn_min_polyscale`` int, float .. 2.0 For scale method set to ``auto``, this sets the minimum SNR for which polynomial scaling is attempted. -``sn_smooth_npix`` int, float .. .. Number of pixels to median filter by when computing S/N used to decide how to scale and weight spectra. If set to None (default), the code will determine the effective number of good pixels per spectrum in the stack that is being co-added and use 10% of this neff. -``spec_samp_fact`` float .. 1.0 Make the wavelength grid sampling finer (spec_samp_fact < 1.0) or coarser (spec_samp_fact > 1.0) by this sampling factor. This basically multiples the 'native' spectral pixels by spec_samp_fact, i.e. units spec_samp_fact are pixels. -``upper`` int, float .. 3.0 Upper rejection threshold used for rejecting pixels when combining spectra in units of sigma. -``wave_grid_max`` int, float .. .. Used in case you want to specify the maximum wavelength in your wavelength grid, default=None computes from data -``wave_grid_min`` int, float .. .. Used in case you want to specify the minimum wavelength in your wavelength grid, default=None computes from data -``wave_method`` str .. ``linear`` Method used to construct wavelength grid for coadding spectra. The routine that creates the wavelength is :func:`~pypeit.core.wavecal.wvutils.get_wave_grid`. The options are: 'iref' -- Use the first wavelength array. 'velocity' -- Grid is uniform in velocity. 'log10' -- Grid is uniform in log10(wave). This is the same as velocity. 'linear' -- Grid is uniform in lambda. 'concatenate' -- Meld the input wavelength arrays -==================== ========== ======= ========== ========================================================================================================================================================================================================================================================================================================================================================================================================================================= +==================== ========== ======= ========== ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= +Key Type Options Default Description +==================== ========== ======= ========== ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= +``chk_version`` bool .. True If True enforce strict PypeIt version checking to ensure that spec1d*.fits files were createdwith the current version of PypeIt +``coaddfile`` str .. .. Output filename +``dloglam`` int, float .. .. Dispersion in units of log10(wave) in case you want to specify it in the get_wave_grid (for the 'velocity' or 'log10' options), otherwise a median value is computed from the data. +``dv`` int, float .. .. Dispersion in units of km/s in case you want to specify it in the get_wave_grid (for the 'velocity' option), otherwise a median value is computed from the data. +``dwave`` int, float .. .. Dispersion in Angstroms in case you want to specify it in the get_wave_grid (for the 'linear' option), otherwise a median value is computed from the data. +``ex_value`` str .. ``OPT`` The extraction to coadd, i.e. optimal or boxcar. Must be either 'OPT' or 'BOX' +``filter`` str .. ``none`` Filter for scaling. See flux_calib.load_fitler_file() for naming. Ignore if none +``filter_mag`` float .. .. Magnitude of the source in the given filter +``filter_mask`` str, list .. .. List of wavelength regions to mask when doing the scaling (`i.e.`, occasional junk pixels). Colon and comma separateed, e.g. 5552:5559,6010:6030 +``flux_value`` bool .. True If True (default), the code will coadd the fluxed spectra (i.e. the FLAM) in the spec1d files. If False, it will coadd the counts. +``lower`` int, float .. 3.0 Lower rejection threshold used for rejecting pixels when combining spectra in units of sigma. +``mag_type`` str .. ``AB`` Magnitude type. AB is the only option currently allowed +``maxiter_reject`` int .. 5 Maximum number of iterations for stacking and rejection. The code stops iterating either when the output mask does not change betweeen successive iterations or when maxiter_reject is reached. +``maxiter_scale`` int .. 5 Maximum number of iterations performed for rescaling spectra. +``maxrej`` int .. .. Coadding performs iterative rejection by comparing each exposure to a preliminary stack of all the exposures. If this parameter is set then it will not reject more than maxrej pixels per iteration of this rejection. The default is None, which means no maximum on rejected pixels. +``nbests`` list, int .. .. Number of orders to use for estimating the per exposure weights. Default is None, which will just use one fourth of the total number of orders. This is only used for Echelle +``nmaskedge`` int .. 2 Number of edge pixels to mask. This should be removed/fixed. +``ref_percentile`` int, float .. 70.0 Percentile used for selecting the minimum SNR cut from a reference spectrum used to robustly determine the median ratio between spectra. This parameter is used by coadd1d.robust_median_ratio as part of the automatic rescaling procedure. Pixels above this percentile cut are deemed the "good" pixels and are used to compute the ratio of two spectra. This must be a number between 0 and 100. +``scale_method`` str .. ``auto`` Method used to rescale the spectra prior to coadding. The options are: 'auto' -- Determine the scaling method automatically based on the S/N ratio which works well. 'poly' -- Polynomial rescaling. 'median' -- Median rescaling 'none' -- Do not rescale. 'hand' -- Pass in hand scaling factors. This option is not well tested. +``sigrej_exp`` int, float .. .. Rejection threshold used for rejecting exposures with S/N more than sigrej_exp*sigma above the median S/N. If None (the default), no rejection is performed. Currently, only available for multi-slit observations. +``sigrej_scale`` int, float .. 3.0 Rejection threshold used for rejecting pixels when rescaling spectra with scale_spec. +``sn_clip`` int, float .. 30.0 Errors are capped during rejection so that the S/N is never greater than sn_clip. This prevents overly aggressive rejection in high S/N ratio spectrum which neverthless differ at a level greater than the formal S/N due to systematics. +``sn_min_medscale`` int, float .. 0.5 For scale method set to ``auto``, this sets the minimum SNR for which median scaling is attempted. +``sn_min_polyscale`` int, float .. 2.0 For scale method set to ``auto``, this sets the minimum SNR for which polynomial scaling is attempted. +``sn_smooth_npix`` int, float .. .. Number of pixels to median filter by when computing S/N used to decide how to scale and weight spectra. If set to None (default), the code will determine the effective number of good pixels per spectrum in the stack that is being co-added and use 10% of this neff. +``spec_samp_fact`` float .. 1.0 Make the wavelength grid sampling finer (spec_samp_fact < 1.0) or coarser (spec_samp_fact > 1.0) by this sampling factor. This basically multiples the 'native' spectral pixels by spec_samp_fact, i.e. units spec_samp_fact are pixels. +``upper`` int, float .. 3.0 Upper rejection threshold used for rejecting pixels when combining spectra in units of sigma. +``wave_grid_max`` int, float .. .. Used in case you want to specify the maximum wavelength in your wavelength grid, default=None computes from data +``wave_grid_min`` int, float .. .. Used in case you want to specify the minimum wavelength in your wavelength grid, default=None computes from data +``wave_method`` str .. ``linear`` Method used to construct wavelength grid for coadding spectra. The routine that creates the wavelength is :func:`~pypeit.core.wavecal.wvutils.get_wave_grid`. The options are: 'iref' -- Use the first wavelength array. 'velocity' -- Grid is uniform in velocity. 'log10' -- Grid is uniform in log10(wave). This is the same as velocity. 'linear' -- Grid is uniform in lambda. 'concatenate' -- Meld the input wavelength arrays +``weight_method`` str .. ``auto`` Method used to weight the spectra for coadding. The options are: 'auto' -- Use constant weights if rms_sn < 3.0, otherwise use wavelength dependent.'constant' -- Constant weights based on rms_sn**2'uniform' -- Uniform weighting'wave_dependent' -- Wavelength dependent weights will be used irrespective of the rms_sn ratio. This option will not work well at low S/N ratio although it is useful for objects where only a small fraction of the spectral coverage has high S/N ratio (like high-z quasars).'relative' -- Apply relative weights implying one reference exposure will receive unit weight at all wavelengths and all others receive relatively wavelength dependent weights . Note, relative weighting will only work well when there is at least one spectrum with a reasonable S/N, and a continuum. This option may only be better when the object being used has a strong continuum + emission lines. This is particularly useful if you are dealing with highly variable spectra (e.g. emission lines) andrequire a precision better than ~1 per cent.'ivar' -- Use inverse variance weighting. This is not well tested and should probably be deprecated. +==================== ========== ======= ========== ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= ---- @@ -690,34 +691,35 @@ CubePar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.CubePar` -==================== ===== ===================== ============ =========================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== -Key Type Options Default Description -==================== ===== ===================== ============ =========================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== -``align`` bool .. False If set to True, the input frames will be spatially aligned by cross-correlating the whitelight images with either a reference image (see ``reference_image``) or the whitelight image that is generated using the first spec2d listed in the coadd3d file. Alternatively, the user can specify the offsets (i.e. Delta RA x cos(dec) and Delta Dec, both in arcsec) in the spec2d block of the coadd3d file. See the documentation for examples of this usage. -``astrometric`` bool .. True If true, an astrometric correction will be applied using the alignment frames. -``combine`` bool .. False If set to True, the input frames will be combined. Otherwise, a separate datacube will be generated for each input spec2d file, and will be saved as a spec3d file. -``dec_max`` float .. .. Maximum DEC to use when generating the WCS. If None, the default is maximum DEC based on the WCS of all spaxels. Units should be degrees. -``dec_min`` float .. .. Minimum DEC to use when generating the WCS. If None, the default is minimum DEC based on the WCS of all spaxels. Units should be degrees. -``grating_corr`` bool .. True This option performs a small correction for the relative blaze function of all input frames that have (even slightly) different grating angles, or if you are flux calibrating your science data with a standard star that was observed with a slightly different setup. -``method`` str ``subpixel``, ``ngp`` ``subpixel`` What method should be used to generate the datacube. There are currently two options: (1) "subpixel" (default) - this algorithm divides each pixel in the spec2d frames into subpixels, and assigns each subpixel to a voxel of the datacube. Flux is conserved, but voxels are correlated, and the error spectrum does not account for covariance between adjacent voxels. See also, spec_subpixel and spat_subpixel. (2) "ngp" (nearest grid point) - this algorithm is effectively a 3D histogram. Flux is conserved, voxels are not correlated, however this option suffers the same downsides as any histogram; the choice of bin sizes can change how the datacube appears. This algorithm takes each pixel on the spec2d frame and puts the flux of this pixel into one voxel in the datacube. Depending on the binning used, some voxels may be empty (zero flux) while a neighboring voxel might contain the flux from two spec2d pixels. Note that all spec2d pixels that contribute to the same voxel are inverse variance weighted (e.g. if two pixels have the same variance, the voxel would be assigned the average flux of the two pixels). -``output_filename`` str .. .. If combining multiple frames, this string sets the output filename of the combined datacube. If combine=False, the output filenames will be prefixed with ``spec3d_*`` -``ra_max`` float .. .. Maximum RA to use when generating the WCS. If None, the default is maximum RA based on the WCS of all spaxels. Units should be degrees. -``ra_min`` float .. .. Minimum RA to use when generating the WCS. If None, the default is minimum RA based on the WCS of all spaxels. Units should be degrees. -``reference_image`` str .. .. White light image of a previously combined datacube. The white light image will be used as a reference when calculating the offsets of the input spec2d files. Ideally, the reference image should have the same shape as the data to be combined (i.e. set the ra_min, ra_max etc. params so they are identical to the reference image). -``relative_weights`` bool .. False If set to True, the combined frames will use a relative weighting scheme. This only works well if there is a common continuum source in the field of view of all input observations, and is generally only required if high relative precision is desired. -``save_whitelight`` bool .. False Save a white light image of the combined datacube. The output filename will be given by the "output_filename" variable with a suffix "_whitelight". Note that the white light image collapses the flux along the wavelength axis, so some spaxels in the 2D white light image may have different wavelength ranges. To set the wavelength range, use the "whitelight_range" parameter. If combine=False, the individual spec3d files will have a suffix "_whitelight". -``scale_corr`` str .. .. This option performs a small correction for the relative spectral illumination scale of different spec2D files. Specify the relative path+file to the spec2D file that you would like to use for the relative scaling. If you want to perform this correction, it is best to use the spec2d file with the highest S/N sky spectrum. You should choose the same frame for both the standards and science frames. -``skysub_frame`` str .. ``image`` Set the sky subtraction to be implemented. The default behaviour is to subtract the sky using the model that is derived from each individual image (i.e. set this parameter to "image"). To turn off sky subtraction completely, set this parameter to "none" (all lowercase). Finally, if you want to use a different frame for the sky subtraction, specify the relative path+file to the spec2D file that you would like to use for the sky subtraction. The model fit to the sky of the specified frame will be used. Note, the sky and science frames do not need to have the same exposure time; the sky model will be scaled to the science frame based on the relative exposure time. -``slit_spec`` bool .. True If the data use slits in one spatial direction, set this to True. If the data uses fibres for all spaxels, set this to False. -``spat_subpixel`` int .. 5 When method=subpixel, spat_subpixel sets the subpixellation scale of each detector pixel in the spatial direction. The total number of subpixels in each pixel is given by spec_subpixel x spat_subpixel. The default option is to divide each spec2d pixel into 25 subpixels during datacube creation. See also, spec_subpixel. -``spatial_delta`` float .. .. The spatial size of each spaxel to use when generating the WCS (in arcsec). If None, the default is set by the spectrograph file. -``spec_subpixel`` int .. 5 When method=subpixel, spec_subpixel sets the subpixellation scale of each detector pixel in the spectral direction. The total number of subpixels in each pixel is given by spec_subpixel x spat_subpixel. The default option is to divide each spec2d pixel into 25 subpixels during datacube creation. See also, spat_subpixel. -``standard_cube`` str .. .. Filename of a standard star datacube. This cube will be used to correct the relative scales of the slits, and to flux calibrate the science datacube. -``wave_delta`` float .. .. The wavelength step to use when generating the WCS (in Angstroms). If None, the default is set by the wavelength solution. -``wave_max`` float .. .. Maximum wavelength to use when generating the WCS. If None, the default is maximum wavelength based on the WCS of all spaxels. Units should be Angstroms. -``wave_min`` float .. .. Minimum wavelength to use when generating the WCS. If None, the default is minimum wavelength based on the WCS of all spaxels. Units should be Angstroms. -``whitelight_range`` list .. None, None A two element list specifying the wavelength range over which to generate the white light image. The first (second) element is the minimum (maximum) wavelength to use. If either of these elements are None, PypeIt will automatically use a wavelength range that ensures all spaxels have the same wavelength coverage. Note, if you are using a reference_image to align all frames, it is preferable to use the same white light wavelength range for all white light images. For example, you may wish to use an emission line map to register two frames. -==================== ===== ===================== ============ =========================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== +==================== ===== =============================================================================== ============ ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= +Key Type Options Default Description +==================== ===== =============================================================================== ============ ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= +``align`` bool .. False If set to True, the input frames will be spatially aligned by cross-correlating the whitelight images with either a reference image (see ``reference_image``) or the whitelight image that is generated using the first spec2d listed in the coadd3d file. Alternatively, the user can specify the offsets (i.e. Delta RA x cos(dec) and Delta Dec, both in arcsec) in the spec2d block of the coadd3d file. See the documentation for examples of this usage. +``astrometric`` bool .. True If true, an astrometric correction will be applied using the alignment frames. +``combine`` bool .. False If set to True, the input frames will be combined. Otherwise, a separate datacube will be generated for each input spec2d file, and will be saved as a spec3d file. +``dec_max`` float .. .. Maximum DEC to use when generating the WCS. If None, the default is maximum DEC based on the WCS of all spaxels. Units should be degrees. +``dec_min`` float .. .. Minimum DEC to use when generating the WCS. If None, the default is minimum DEC based on the WCS of all spaxels. Units should be degrees. +``grating_corr`` bool .. True This option performs a small correction for the relative blaze function of all input frames that have (even slightly) different grating angles, or if you are flux calibrating your science data with a standard star that was observed with a slightly different setup. +``method`` str ``subpixel``, ``ngp`` ``subpixel`` What method should be used to generate the datacube. There are currently two options: (1) "subpixel" (default) - this algorithm divides each pixel in the spec2d frames into subpixels, and assigns each subpixel to a voxel of the datacube. Flux is conserved, but voxels are correlated, and the error spectrum does not account for covariance between adjacent voxels. See also, spec_subpixel and spat_subpixel. (2) "ngp" (nearest grid point) - this algorithm is effectively a 3D histogram. Flux is conserved, voxels are not correlated, however this option suffers the same downsides as any histogram; the choice of bin sizes can change how the datacube appears. This algorithm takes each pixel on the spec2d frame and puts the flux of this pixel into one voxel in the datacube. Depending on the binning used, some voxels may be empty (zero flux) while a neighboring voxel might contain the flux from two spec2d pixels. Note that all spec2d pixels that contribute to the same voxel are inverse variance weighted (e.g. if two pixels have the same variance, the voxel would be assigned the average flux of the two pixels). +``output_filename`` str .. .. If combining multiple frames, this string sets the output filename of the combined datacube. If combine=False, the output filenames will be prefixed with ``spec3d_*`` +``ra_max`` float .. .. Maximum RA to use when generating the WCS. If None, the default is maximum RA based on the WCS of all spaxels. Units should be degrees. +``ra_min`` float .. .. Minimum RA to use when generating the WCS. If None, the default is minimum RA based on the WCS of all spaxels. Units should be degrees. +``reference_image`` str .. .. White light image of a previously combined datacube. The white light image will be used as a reference when calculating the offsets of the input spec2d files. Ideally, the reference image should have the same shape as the data to be combined (i.e. set the ra_min, ra_max etc. params so they are identical to the reference image). +``save_whitelight`` bool .. False Save a white light image of the combined datacube. The output filename will be given by the "output_filename" variable with a suffix "_whitelight". Note that the white light image collapses the flux along the wavelength axis, so some spaxels in the 2D white light image may have different wavelength ranges. To set the wavelength range, use the "whitelight_range" parameter. If combine=False, the individual spec3d files will have a suffix "_whitelight". +``scale_corr`` str .. .. This option performs a small correction for the relative spectral illumination scale of different spec2D files. Specify the relative path+file to the spec2D file that you would like to use for the relative scaling. If you want to perform this correction, it is best to use the spec2d file with the highest S/N sky spectrum. You should choose the same frame for both the standards and science frames. +``skysub_frame`` str .. ``image`` Set the sky subtraction to be implemented. The default behaviour is to subtract the sky using the model that is derived from each individual image (i.e. set this parameter to "image"). To turn off sky subtraction completely, set this parameter to "none" (all lowercase). Finally, if you want to use a different frame for the sky subtraction, specify the relative path+file to the spec2D file that you would like to use for the sky subtraction. The model fit to the sky of the specified frame will be used. Note, the sky and science frames do not need to have the same exposure time; the sky model will be scaled to the science frame based on the relative exposure time. +``slice_subpixel`` int .. 5 When method=subpixel, slice_subpixel sets the subpixellation scale of each IFU slice. The default option is to divide each slice into 5 sub-slices during datacube creation. See also, spec_subpixel and spat_subpixel. +``slit_spec`` bool .. True If the data use slits in one spatial direction, set this to True. If the data uses fibres for all spaxels, set this to False. +``spat_subpixel`` int .. 5 When method=subpixel, spat_subpixel sets the subpixellation scale of each detector pixel in the spatial direction. The total number of subpixels in each pixel is given by spec_subpixel x spat_subpixel. The default option is to divide each spec2d pixel into 25 subpixels during datacube creation. See also, spec_subpixel and slice_subpixel. +``spatial_delta`` float .. .. The spatial size of each spaxel to use when generating the WCS (in arcsec). If None, the default is set by the spectrograph file. +``spec_subpixel`` int .. 5 When method=subpixel, spec_subpixel sets the subpixellation scale of each detector pixel in the spectral direction. The total number of subpixels in each pixel is given by spec_subpixel x spat_subpixel. The default option is to divide each spec2d pixel into 25 subpixels during datacube creation. See also, spat_subpixel and slice_subpixel. +``standard_cube`` str .. .. Filename of a standard star datacube. This cube will be used to correct the relative scales of the slits, and to flux calibrate the science datacube. +``wave_delta`` float .. .. The wavelength step to use when generating the WCS (in Angstroms). If None, the default is set by the wavelength solution. +``wave_max`` float .. .. Maximum wavelength to use when generating the WCS. If None, the default is maximum wavelength based on the WCS of all spaxels. Units should be Angstroms. +``wave_min`` float .. .. Minimum wavelength to use when generating the WCS. If None, the default is minimum wavelength based on the WCS of all spaxels. Units should be Angstroms. +``weight_method`` str ``auto``, ``constant``, ``uniform``, ``wave_dependent``, ``relative``, ``ivar`` ``auto`` Method used to weight the spectra for coadding. The options are: 'auto' -- Use constant weights if rms_sn < 3.0, otherwise use wavelength dependent.'constant' -- Constant weights based on rms_sn**2'uniform' -- Uniform weighting'wave_dependent' -- Wavelength dependent weights will be used irrespective of the rms_sn ratio. This option will not work well at low S/N ratio although it is useful for objects where only a small fraction of the spectral coverage has high S/N ratio (like high-z quasars).'relative' -- Apply relative weights implying one reference exposure will receive unit weight at all wavelengths and all others receive relatively wavelength dependent weights . Note, relative weighting will only work well when there is at least one spectrum with a reasonable S/N, and a continuum. This option may only be better when the object being used has a strong continuum + emission lines. This is particularly useful if you are dealing with highly variable spectra (e.g. emission lines) andrequire a precision better than ~1 per cent.'ivar' -- Use inverse variance weighting. This is not well tested and should probably be deprecated. +``whitelight_range`` list .. None, None A two element list specifying the wavelength range over which to generate the white light image. The first (second) element is the minimum (maximum) wavelength to use. If either of these elements are None, PypeIt will automatically use a wavelength range that ensures all spaxels have the same wavelength coverage. Note, if you are using a reference_image to align all frames, it is preferable to use the same white light wavelength range for all white light images. For example, you may wish to use an emission line map to register two frames. +==================== ===== =============================================================================== ============ ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= ---- @@ -901,15 +903,15 @@ ScatteredLightPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.ScatteredLightPar` -================== ========= ================================= ========= =============================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== -Key Type Options Default Description -================== ========= ================================= ========= =============================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== -``finecorr`` bool .. True If True, a fine correction to the scattered light will be performed. However, the fine correction will only be applied if the model/frame/archive correction is performed. -``finecorr_mask`` int, list .. .. A list containing the inter-slit regions that the user wishes to mask during the fine correction to the scattered light. Each integer corresponds to an inter-slit region. For example, "0" corresponds to all pixels left of the leftmost slit, while a value of "1" corresponds to all pixels between the first and second slit (counting from the left). It should be either a single integer value, or a list of integer values. The default (None) means that no inter-slit regions will be masked. -``finecorr_order`` int .. 2 Polynomial order to use for the fine correction to the scattered light subtraction. It should be a low value. -``finecorr_pad`` int .. 2 Number of unbinned pixels to extend the slit edges by when masking the slits for thefine correction to the scattered light. -``method`` str ``model``, ``frame``, ``archive`` ``model`` Method used to fit the overscan. Options are: model, frame, archive.'model' will the scattered light model parameters derived from a user-specified frame during their reduction (note, you will need to make sure that you set appropriate scattlight frames in your .pypeit file for this option). 'frame' will use each individual frame to determine the scattered light that affects this frame. 'archive' will use an archival model parameter solution for the scattered light (note that this option is not currently available for all spectrographs). -================== ========= ================================= ========= =============================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== +=================== ========= ================================= ========= ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================ +Key Type Options Default Description +=================== ========= ================================= ========= ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================ +``finecorr_mask`` int, list .. .. A list containing the inter-slit regions that the user wishes to mask during the fine correction to the scattered light. Each integer corresponds to an inter-slit region. For example, "0" corresponds to all pixels left of the leftmost slit, while a value of "1" corresponds to all pixels between the first and second slit (counting from the left). It should be either a single integer value, or a list of integer values. The default (None) means that no inter-slit regions will be masked. +``finecorr_method`` str ``median``, ``poly`` .. If None, a fine correction to the scattered light will not be performed. Otherwise, the allowed methods include: median, poly. 'median' will subtract a constant value from an entire CCD row, based on a median of the pixels that are not on slits (see also, 'finecorr_pad'). 'poly' will fit a polynomial to the scattered light in each row, based on the pixels that are not on slits (see also, 'finecorr_pad'). +``finecorr_order`` int .. 2 Polynomial order to use for the fine correction to the scattered light subtraction. It should be a low value. +``finecorr_pad`` int .. 4 Number of unbinned pixels to extend the slit edges by when masking the slits for the fine correction to the scattered light. +``method`` str ``model``, ``frame``, ``archive`` ``model`` Method used to fit the overscan. Options are: model, frame, archive. 'model' will the scattered light model parameters derived from a user-specified frame during their reduction (note, you will need to make sure that you set appropriate scattlight frames in your .pypeit file for this option). 'frame' will use each individual frame to determine the scattered light that affects this frame. 'archive' will use an archival model parameter solution for the scattered light (note that this option is not currently available for all spectrographs). +=================== ========= ================================= ========= ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================ ---- @@ -1006,7 +1008,7 @@ Key Type Options Default ``popsize`` int .. 30 A multiplier for setting the total population size for the differential evolution optimization. See scipy.optimize.differential_evolution for details. ``recombination`` int, float .. 0.7 The recombination constant for the differential evolution optimization. This should be in the range [0, 1]. See scipy.optimize.differential_evolution for details. ``redshift`` int, float .. 0.0 The redshift for the object model. This is currently only used by objmodel=qso -``resln_frac_bounds`` tuple .. (0.5, 1.5) Bounds for the resolution fit optimization which is part of the telluric model. This range is in units of the resln_guess, so the (0.5, 1.5) would bound the spectral resolution fit to be within the range bounds_resln = (0.5*resln_guess, 1.5*resln_guess) +``resln_frac_bounds`` tuple .. (0.6, 1.4) Bounds for the resolution fit optimization which is part of the telluric model. This range is in units of the resln_guess, so the (0.6, 1.4) would bound the spectral resolution fit to be within the range bounds_resln = (0.6*resln_guess, 1.4*resln_guess) ``resln_guess`` int, float .. .. A guess for the resolution of your spectrum expressed as lambda/dlambda. The resolution is fit explicitly as part of the telluric model fitting, but this guess helps determine the bounds for the optimization (see next). If not provided, the wavelength sampling of your spectrum will be used and the resolution calculated using a typical sampling of 3 spectral pixels per resolution element. ``seed`` int .. 777 An initial seed for the differential evolution optimization, which is a random process. The default is a seed = 777 which will be used to generate a unique seed for every order. A specific seed is used because otherwise the random number generator will use the time for the seed, and the results will not be reproducible. ``sn_clip`` int, float .. 30.0 This adds an error floor to the ivar, preventing too much rejection at high-S/N (`i.e.`, standard stars, bright objects) using the function utils.clip_ivar. A small erorr is added to the input ivar so that the output ivar_out will never give S/N greater than sn_clip. This prevents overly aggressive rejection in high S/N ratio spectra which neverthless differ at a level greater than the formal S/N due to the fact that our telluric models are only good to about 3%. @@ -1017,6 +1019,8 @@ Key Type Options Default ``sticky`` bool .. True Sticky parameter for the utils.djs_reject algorithm for iterative model fit rejection. If set to True then points rejected from a previous iteration are kept rejected, in other words the bad pixel mask is the OR of all previous iterations and rejected pixels accumulate. If set to False, the bad pixel mask is the mask from the previous iteration, and if the model fit changes between iterations, points can alternate from being rejected to not rejected. At present this code only performs optimizations with differential evolution and experience shows that sticky needs to be True in order for these to converge. This is because the outliers can be so large that they dominate the loss function, and one never iteratively converges to a good model fit. In other words, the deformations in the model between iterations with sticky=False are too small to approach a reasonable fit. ``telgridfile`` str .. .. File containing the telluric grid for the observatory in question. These grids are generated from HITRAN models for each observatory using nominal site parameters. They must be downloaded from the GoogleDrive and installed in your PypeIt installation via the pypeit_install_telluric script. NOTE: This parameter no longer includes the full pathname to the Telluric Grid file, but is just the filename of the grid itself. ``tell_norm_thresh`` int, float .. 0.9 Threshold of telluric absorption region +``tell_npca`` int .. 5 Number of telluric PCA components used. Can be set to any number from 1 to 10. +``teltype`` str .. ``pca`` Method used to evaluate telluric models, either pca or grid. The grid option uses a fixed grid of pre-computed HITRAN+LBLRTM atmospheric transmission models for each observatory, whereas the pca option uses principal components of a larger model grid to compute an accurate pseudo-telluric model with a much lighter telgridfile. ``tol`` float .. 0.001 Relative tolerance for converage of the differential evolution optimization. See scipy.optimize.differential_evolution for details. ``upper`` int, float .. 3.0 Upper rejection threshold in units of sigma_corr*sigma, where sigma is the formal noise of the spectrum, and sigma_corr is an empirically determined correction to the formal error. See above for description. ======================= ================== ======= ========================== ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= @@ -1425,7 +1429,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-gemini_gmos_north_e2v: @@ -1786,7 +1790,7 @@ Alterations to the default parameters are: [sensfunc] algorithm = IR [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-gemini_gnirs_echelle: @@ -1915,7 +1919,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 6 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-gemini_gnirs_ifu: @@ -2058,7 +2062,7 @@ Alterations to the default parameters are: [[UVIS]] extinct_correct = False [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-gtc_maat: @@ -2134,7 +2138,7 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = None, 180, + exprng = None, 300, [[[process]]] mask_cr = True noise_floor = 0.01 @@ -2347,7 +2351,7 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = None, 180, + exprng = None, 300, [[[process]]] mask_cr = True noise_floor = 0.01 @@ -2665,7 +2669,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R15000.fits .. _instr_par-keck_esi: @@ -2930,7 +2934,11 @@ Alterations to the default parameters are: [sensfunc] algorithm = IR [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_10500_R120000.fits + pix_shift_bounds = (-40.0, 40.0) + [telluric] + resln_frac_bounds = (0.25, 1.25) + pix_shift_bounds = (-40.0, 40.0) .. _instr_par-keck_kcrm: @@ -3035,6 +3043,8 @@ Alterations to the default parameters are: [sensfunc] [[UVIS]] extinct_correct = False + [[IR]] + telgridfile = TellPCA_3000_26000_R15000.fits .. _instr_par-keck_kcwi: @@ -3117,6 +3127,7 @@ Alterations to the default parameters are: use_pattern = True [[flatfield]] spec_samp_coarse = 20.0 + spat_samp = 1.0 tweak_slits_thresh = 0.0 tweak_slits_maxfrac = 0.0 slit_illum_relative = True @@ -3139,7 +3150,7 @@ Alterations to the default parameters are: use_pattern = True subtract_scattlight = True [[[scattlight]]] - finecorr_mask = 12 + finecorr_method = median [reduce] [[extraction]] skip_extraction = True @@ -3148,6 +3159,8 @@ Alterations to the default parameters are: [sensfunc] [[UVIS]] extinct_correct = False + [[IR]] + telgridfile = TellPCA_3000_26000_R15000.fits .. _instr_par-keck_lris_blue: @@ -3249,7 +3262,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_lris_blue_orig: @@ -3351,7 +3364,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_lris_red: @@ -3464,7 +3477,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 9 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_lris_red_mark4: @@ -3577,7 +3590,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 9 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_lris_red_orig: @@ -3690,7 +3703,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 9 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_mosfire: @@ -3817,7 +3830,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 13 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_nires: @@ -3960,7 +3973,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_nirspec_low: @@ -4089,7 +4102,10 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R25000.fits + pix_shift_bounds = (-8.0, 8.0) + [telluric] + pix_shift_bounds = (-8.0, 8.0) .. _instr_par-lbt_luci1: @@ -5018,7 +5034,7 @@ Alterations to the default parameters are: [sensfunc] algorithm = IR [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R15000.fits .. _instr_par-magellan_fire_long: @@ -5148,7 +5164,7 @@ Alterations to the default parameters are: find_trim_edge = 50, 50, [sensfunc] [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-magellan_mage: @@ -5665,7 +5681,7 @@ Alterations to the default parameters are: [sensfunc] polyorder = 7 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-mmt_bluechannel: @@ -5914,7 +5930,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-not_alfosc: @@ -6411,7 +6427,7 @@ Alterations to the default parameters are: [[UVIS]] polycorrect = False [[IR]] - telgridfile = TelFit_Lick_3100_11100_R10000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-p200_tspec: @@ -6557,7 +6573,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-shane_kast_blue: @@ -6658,7 +6674,7 @@ Alterations to the default parameters are: spectrum = sky_kastb_600.fits [sensfunc] [[IR]] - telgridfile = TelFit_Lick_3100_11100_R10000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-shane_kast_red: @@ -6750,7 +6766,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_Lick_3100_11100_R10000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-shane_kast_red_ret: @@ -6844,7 +6860,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_Lick_3100_11100_R10000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-soar_goodman_blue: @@ -6947,7 +6963,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R15000.fits .. _instr_par-soar_goodman_red: @@ -7052,7 +7068,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R15000.fits .. _instr_par-tng_dolores: @@ -7242,7 +7258,7 @@ Alterations to the default parameters are: [sensfunc] algorithm = IR [[IR]] - telgridfile = TelFit_Paranal_VIS_9800_25000_R25000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-vlt_sinfoni: @@ -7383,7 +7399,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 7 [[IR]] - telgridfile = TelFit_Paranal_NIR_9800_25000_R25000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-vlt_xshooter_nir: @@ -7534,7 +7550,11 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_Paranal_NIR_9800_25000_R25000.fits + telgridfile = TellPCA_3000_26000_R25000.fits + pix_shift_bounds = (-10.0, 10.0) + [telluric] + resln_frac_bounds = (0.4, 2.0) + pix_shift_bounds = (-10.0, 10.0) .. _instr_par-vlt_xshooter_uvb: @@ -7676,7 +7696,10 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R25000.fits + pix_shift_bounds = (-8.0, 8.0) + [telluric] + pix_shift_bounds = (-8.0, 8.0) .. _instr_par-vlt_xshooter_vis: @@ -7820,7 +7843,11 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_Paranal_VIS_4900_11100_R25000.fits + telgridfile = TellPCA_3000_26000_R25000.fits + pix_shift_bounds = (-10.0, 10.0) + [telluric] + resln_frac_bounds = (0.4, 2.0) + pix_shift_bounds = (-10.0, 10.0) .. _instr_par-wht_isis_blue: diff --git a/pypeit/coadd2d.py b/pypeit/coadd2d.py index 025ca4eea9..53683756a0 100644 --- a/pypeit/coadd2d.py +++ b/pypeit/coadd2d.py @@ -474,39 +474,44 @@ def optimal_weights(self, slitorderid, objid, weight_method='auto'): Parameters ---------- slitorderid : :obj:`int` - The slit or order id that has the brightest object whose - S/N will be used to determine the weight for each frame. + The slit or order id that has the brightest object whose + S/N will be used to determine the weight for each frame. objid : `numpy.ndarray`_ - Array of object indices with shape = (nexp,) of the - brightest object whose S/N will be used to determine the - weight for each frame. - weight_method: (`str`, optional) - Weight method to be used in `coadd.sn_weights`. Default is 'auto'. - Options are 'auto', 'constant', 'relative', or 'ivar'. The defaulti is'auto'. - Behavior is as follows: - 'auto': - Use constant weights if rms_sn < 3.0, otherwise use wavelength dependent. - 'constant': - Constant weights based on rms_sn**2 - 'uniform': - Uniform weighting. - 'wave_dependent': - Wavelength dependent weights will be used irrespective of the rms_sn ratio. This option - will not work well at low S/N ratio although it is useful for objects where only a small - fraction of the spectral coverage has high S/N ratio (like high-z quasars). - 'relative': - Calculate weights by fitting to the ratio of spectra? Note, relative - weighting will only work well when there is at least one spectrum with a - reasonable S/N, and a continuum. RJC note - This argument may only be - better when the object being used has a strong continuum + emission - lines. The reference spectrum is assigned a value of 1 for all - wavelengths, and the weights of all other spectra will be determined - relative to the reference spectrum. This is particularly useful if you - are dealing with highly variable spectra (e.g. emission lines) and - require a precision better than ~1 per cent. - 'ivar': - Use inverse variance weighting. This is not well tested and should probably be deprecated. - + Array of object indices with shape = (nexp,) of the + brightest object whose S/N will be used to determine the + weight for each frame. + weight_method : `str`, optional + Weight method to be used in :func:`~pypeit.coadd.sn_weights`. + Options are ``'auto'``, ``'constant'``, ``'relative'``, or + ``'ivar'``. The default is ``'auto'``. Behavior is as follows: + + - ``'auto'``: Use constant weights if rms_sn < 3.0, otherwise + use wavelength dependent. + + - ``'constant'``: Constant weights based on rms_sn**2 + + - ``'uniform'``: Uniform weighting. + + - ``'wave_dependent'``: Wavelength dependent weights will be + used irrespective of the rms_sn ratio. This option will not + work well at low S/N ratio although it is useful for objects + where only a small fraction of the spectral coverage has high + S/N ratio (like high-z quasars). + + - ``'relative'``: Calculate weights by fitting to the ratio of + spectra? Note, relative weighting will only work well when + there is at least one spectrum with a reasonable S/N, and a + continuum. RJC note - This argument may only be better when + the object being used has a strong continuum + emission lines. + The reference spectrum is assigned a value of 1 for all + wavelengths, and the weights of all other spectra will be + determined relative to the reference spectrum. This is + particularly useful if you are dealing with highly variable + spectra (e.g. emission lines) and require a precision better + than ~1 per cent. + + - ``'ivar'``: Use inverse variance weighting. This is not well + tested and should probably be deprecated. Returns ------- diff --git a/pypeit/core/coadd.py b/pypeit/core/coadd.py index f28669c332..779b3c5673 100644 --- a/pypeit/core/coadd.py +++ b/pypeit/core/coadd.py @@ -819,31 +819,39 @@ def sn_weights(fluxes, ivars, gpms, sn_smooth_npix=None, weight_method='auto', v Number of pixels used for determining smoothly varying S/N ratio weights. This must be passed for all weight methods excpet for weight_method='constant' or 'uniform', since then wavelength dependent weights are not used. - weight_method (str): - The weighting method to be used. Options are 'auto', 'constant', 'relative', or 'ivar'. The defaulti is'auto'. - Behavior is as follows: - 'auto': - Use constant weights if rms_sn < 3.0, otherwise use wavelength dependent. - 'constant': - Constant weights based on rms_sn**2 - 'uniform': - Uniform weighting. - 'wave_dependent': - Wavelength dependent weights will be used irrespective of the rms_sn ratio. This option - will not work well at low S/N ratio although it is useful for objects where only a small - fraction of the spectral coverage has high S/N ratio (like high-z quasars). - 'relative': - Calculate weights by fitting to the ratio of spectra? Note, relative - weighting will only work well when there is at least one spectrum with a - reasonable S/N, and a continuum. RJC note - This argument may only be - better when the object being used has a strong continuum + emission - lines. The reference spectrum is assigned a value of 1 for all - wavelengths, and the weights of all other spectra will be determined - relative to the reference spectrum. This is particularly useful if you - are dealing with highly variable spectra (e.g. emission lines) and - require a precision better than ~1 per cent. - 'ivar': - Use inverse variance weighting. This is not well tested and should probably be deprecated. + weight_method : str, optional + + The weighting method to be used. Options are ``'auto'``, ``'constant'``, + ``'relative'``, or ``'ivar'``. The default is ``'auto'``. Behavior is + as follows: + + - ``'auto'``: Use constant weights if rms_sn < 3.0, otherwise use + wavelength dependent. + + - ``'constant'``: Constant weights based on rms_sn**2 + + - ``'uniform'``: Uniform weighting. + + - ``'wave_dependent'``: Wavelength dependent weights will be used + irrespective of the rms_sn ratio. This option will not work well + at low S/N ratio although it is useful for objects where only a + small fraction of the spectral coverage has high S/N ratio (like + high-z quasars). + + - ``'relative'``: Calculate weights by fitting to the ratio of + spectra? Note, relative weighting will only work well when there + is at least one spectrum with a reasonable S/N, and a continuum. + RJC note - This argument may only be better when the object being + used has a strong continuum + emission lines. The reference + spectrum is assigned a value of 1 for all wavelengths, and the + weights of all other spectra will be determined relative to the + reference spectrum. This is particularly useful if you are + dealing with highly variable spectra (e.g. emission lines) and + require a precision better than ~1 per cent. + + - ``'ivar'``: Use inverse variance weighting. This is not well + tested and should probably be deprecated. + verbose : bool, optional Verbosity of print out. diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index 54966b0e4d..6050993672 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -990,95 +990,106 @@ def compute_weights_frompix(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, first prepares a whitelight image, and then calls compute_weights() to determine the appropriate weights of each pixel. - Args: - raImg : (`numpy.ndarray`_, list): - A list of 2D array containing the RA of each pixel, with shape (nspec, nspat) - decImg : (`numpy.ndarray`_, list): - A list of 2D array containing the Dec of each pixel, with shape (nspec, nspat) - waveImg (`numpy.ndarray`_, list): - A list of 2D array containing the wavelength of each pixel, with shape (nspec, nspat) - sciImg (`numpy.ndarray`_, list): - A list of 2D array containing the science image of each pixel, with shape (nspec, nspat) - ivarImg (`numpy.ndarray`_, list): - A list of 2D array containing the inverse variance image of each pixel, with shape (nspec, nspat) - slitidImg (`numpy.ndarray`_, list): - A list of 2D array containing the slit ID of each pixel, with shape (nspec, nspat) - dspat (float): - The size of each spaxel on the sky (in degrees) - dwv (float): - The size of each wavelength pixel (in Angstroms) - mnmx_wv (`numpy.ndarray`_): - The minimum and maximum wavelengths of every slit and frame. The shape is (Nframes, Nslits, 2), - The minimum and maximum wavelengths are stored in the [:,:,0] and [:,:,1] indices, respectively. - wghtsImg (`numpy.ndarray`_, list): - A list of 2D array containing the weights of each pixel, with shape (nspec, nspat) - all_wcs (`astropy.wcs.WCS`_, list): - A list of WCS objects, one for each frame. - all_tilts (`numpy.ndarray`_, list): - 2D wavelength tilts frame, or a list of tilt frames - all_slits (:class:`~pypeit.slittrace.SlitTraceSet`, list): - Information stored about the slits, or a list of SlitTraceSet objects - all_align (:class:`~pypeit.alignframe.AlignmentSplines`, list): - A Class containing the transformation between detector pixel - coordinates and WCS pixel coordinates, or a list of Alignment - Splines. - all_dar (:class:`~pypeit.coadd3d.DARcorrection`, list): - A Class containing the DAR correction information, or a list of DARcorrection - classes. If a list, it must be the same length as astrom_trans. - ra_offsets (float, list): - RA offsets for each frame in units of degrees - dec_offsets (float, list): - Dec offsets for each frame in units of degrees - ra_min (float, optional): - Minimum RA of the WCS (degrees) - ra_max (float, optional): - Maximum RA of the WCS (degrees) - dec_min (float, optional): - Minimum Dec of the WCS (degrees) - dec_max (float, optional): - Maximum Dec of the WCS (degrees) - wave_min (float, optional): - Minimum wavelength of the WCS (degrees) - wave_max (float, optional): - Maximum wavelength of the WCS (degrees) - sn_smooth_npix (float, optional): - Number of pixels used for determining smoothly varying S/N ratio - weights. This is currently not required, since a relative weighting - scheme with a polynomial fit is used to calculate the S/N weights. - weight_method: (`str`, optional) - Weight method to be used in `coadd.sn_weights`. Default is 'auto'. - Options are 'auto', 'constant', 'uniform', 'wave_dependent', 'relative', or 'ivar'. The defaulti is 'auto'. - Behavior is as follows: - 'auto': - Use constant weights if rms_sn < 3.0, otherwise use wavelength dependent. - 'constant': - Constant weights based on rms_sn**2 - 'uniform': - Uniform weighting. - 'wave_dependent': - Wavelength dependent weights will be used irrespective of the rms_sn ratio. This option - will not work well at low S/N ratio although it is useful for objects where only a small - fraction of the spectral coverage has high S/N ratio (like high-z quasars). - 'relative': - Calculate weights by fitting to the ratio of spectra? Note, relative - weighting will only work well when there is at least one spectrum with a - reasonable S/N, and a continuum. RJC note - This argument may only be - better when the object being used has a strong continuum + emission - lines. The reference spectrum is assigned a value of 1 for all - wavelengths, and the weights of all other spectra will be determined - relative to the reference spectrum. This is particularly useful if you - are dealing with highly variable spectra (e.g. emission lines) and - require a precision better than ~1 per cent. - 'ivar': - Use inverse variance weighting. This is not well tested and should probably be deprecated. - reference_image (`numpy.ndarray`_): - Reference image to use for the determination of the highest S/N spaxel in the image. - specname (str): - Name of the spectrograph + Parameters + ---------- - Returns: - `numpy.ndarray`_ : a 1D array the same size as all_sci, containing - relative wavelength dependent weights of each input pixel. + raImg : `numpy.ndarray`_, list + A list of 2D array containing the RA of each pixel, with shape (nspec, nspat) + decImg : `numpy.ndarray`_, list + A list of 2D array containing the Dec of each pixel, with shape (nspec, nspat) + waveImg : `numpy.ndarray`_, list + A list of 2D array containing the wavelength of each pixel, with shape (nspec, nspat) + sciImg : `numpy.ndarray`_, list + A list of 2D array containing the science image of each pixel, with shape (nspec, nspat) + ivarImg : `numpy.ndarray`_, list + A list of 2D array containing the inverse variance image of each pixel, with shape (nspec, nspat) + slitidImg : `numpy.ndarray`_, list + A list of 2D array containing the slit ID of each pixel, with shape (nspec, nspat) + dspat : float + The size of each spaxel on the sky (in degrees) + dwv : float + The size of each wavelength pixel (in Angstroms) + mnmx_wv : `numpy.ndarray`_ + The minimum and maximum wavelengths of every slit and frame. The shape is (Nframes, Nslits, 2), + The minimum and maximum wavelengths are stored in the [:,:,0] and [:,:,1] indices, respectively. + wghtsImg : `numpy.ndarray`_, list + A list of 2D array containing the weights of each pixel, with shape (nspec, nspat) + all_wcs : `astropy.wcs.WCS`_, list + A list of WCS objects, one for each frame. + all_tilts : `numpy.ndarray`_, list + 2D wavelength tilts frame, or a list of tilt frames + all_slits : :class:`~pypeit.slittrace.SlitTraceSet`, list + Information stored about the slits, or a list of SlitTraceSet objects + all_align : :class:`~pypeit.alignframe.AlignmentSplines`, list + A Class containing the transformation between detector pixel + coordinates and WCS pixel coordinates, or a list of Alignment + Splines. + all_dar : :class:`~pypeit.coadd3d.DARcorrection`, list + A Class containing the DAR correction information, or a list of DARcorrection + classes. If a list, it must be the same length as astrom_trans. + ra_offsets : float, list + RA offsets for each frame in units of degrees + dec_offsets : float, list + Dec offsets for each frame in units of degrees + ra_min : float, optional + Minimum RA of the WCS (degrees) + ra_max : float, optional + Maximum RA of the WCS (degrees) + dec_min : float, optional + Minimum Dec of the WCS (degrees) + dec_max : float, optional + Maximum Dec of the WCS (degrees) + wave_min : float, optional + Minimum wavelength of the WCS (degrees) + wave_max : float, optional + Maximum wavelength of the WCS (degrees) + sn_smooth_npix : float, optional + Number of pixels used for determining smoothly varying S/N ratio + weights. This is currently not required, since a relative weighting + scheme with a polynomial fit is used to calculate the S/N weights. + weight_method : `str`, optional + Weight method to be used in :func:`~pypeit.coadd.sn_weights`. + Options are ``'auto'``, ``'constant'``, ``'relative'``, or + ``'ivar'``. The default is ``'auto'``. Behavior is as follows: + + - ``'auto'``: Use constant weights if rms_sn < 3.0, otherwise + use wavelength dependent. + + - ``'constant'``: Constant weights based on rms_sn**2 + + - ``'uniform'``: Uniform weighting. + + - ``'wave_dependent'``: Wavelength dependent weights will be + used irrespective of the rms_sn ratio. This option will not + work well at low S/N ratio although it is useful for objects + where only a small fraction of the spectral coverage has high + S/N ratio (like high-z quasars). + + - ``'relative'``: Calculate weights by fitting to the ratio of + spectra? Note, relative weighting will only work well when + there is at least one spectrum with a reasonable S/N, and a + continuum. RJC note - This argument may only be better when + the object being used has a strong continuum + emission lines. + The reference spectrum is assigned a value of 1 for all + wavelengths, and the weights of all other spectra will be + determined relative to the reference spectrum. This is + particularly useful if you are dealing with highly variable + spectra (e.g. emission lines) and require a precision better + than ~1 per cent. + + - ``'ivar'``: Use inverse variance weighting. This is not well + tested and should probably be deprecated. + + reference_image : `numpy.ndarray`_ + Reference image to use for the determination of the highest S/N spaxel in the image. + specname : str + Name of the spectrograph + + Returns + ------- + weights : `numpy.ndarray`_ + a 1D array the same size as all_sci, containing relative wavelength + dependent weights of each input pixel. """ # Find the wavelength range where all frames overlap min_wl, max_wl = get_whitelight_range(np.max(mnmx_wv[:, :, 0]), # The max blue wavelength @@ -1117,77 +1128,88 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, Calculate wavelength dependent optimal weights. The weighting is currently based on a relative :math:`(S/N)^2` at each wavelength - Args: - raImg : (`numpy.ndarray`_, list): - A list of 2D array containing the RA of each pixel, with shape (nspec, nspat) - decImg : (`numpy.ndarray`_, list): - A list of 2D array containing the Dec of each pixel, with shape (nspec, nspat) - waveImg (`numpy.ndarray`_, list): - A list of 2D array containing the wavelength of each pixel, with shape (nspec, nspat) - sciImg (`numpy.ndarray`_, list): - A list of 2D array containing the science image of each pixel, with shape (nspec, nspat) - ivarImg (`numpy.ndarray`_, list): - A list of 2D array containing the inverse variance image of each pixel, with shape (nspec, nspat) - slitidImg (`numpy.ndarray`_, list): - A list of 2D array containing the slit ID of each pixel, with shape (nspec, nspat) - all_wcs (`astropy.wcs.WCS`_, list): - A list of WCS objects, one for each frame. - all_tilts (`numpy.ndarray`_, list): - 2D wavelength tilts frame, or a list of tilt frames - all_slits (:class:`~pypeit.slittrace.SlitTraceSet`, list): - Information stored about the slits, or a list of SlitTraceSet objects - all_align (:class:`~pypeit.alignframe.AlignmentSplines`, list): - A Class containing the transformation between detector pixel - coordinates and WCS pixel coordinates, or a list of Alignment - Splines. - all_dar (:class:`~pypeit.coadd3d.DARcorrection`, list): - A Class containing the DAR correction information, or a list of DARcorrection - classes. If a list, it must be the same length as astrom_trans. - ra_offsets (float, list): - RA offsets for each frame in units of degrees - dec_offsets (float, list): - Dec offsets for each frame in units of degrees - whitelight_img (`numpy.ndarray`_): - A 2D array containing a white light image, that was created with the - input ``all`` arrays. - dspat (float): - The size of each spaxel on the sky (in degrees) - dwv (float): - The size of each wavelength pixel (in Angstroms) - sn_smooth_npix (float, optional): - Number of pixels used for determining smoothly varying S/N ratio - weights. This is currently not required, since a relative weighting - scheme with a polynomial fit is used to calculate the S/N weights. - weight_method: (`str`, optional) - Weight method to be used in `coadd.sn_weights`. Default is 'auto'. - Options are 'auto', 'constant', 'uniform', 'wave_dependent', 'relative', or 'ivar'. The defaulti is 'auto'. - Behavior is as follows: - 'auto': - Use constant weights if rms_sn < 3.0, otherwise use wavelength dependent. - 'constant': - Constant weights based on rms_sn**2 - 'uniform': - Uniform weighting. - 'wave_dependent': - Wavelength dependent weights will be used irrespective of the rms_sn ratio. This option - will not work well at low S/N ratio although it is useful for objects where only a small - fraction of the spectral coverage has high S/N ratio (like high-z quasars). - 'relative': - Calculate weights by fitting to the ratio of spectra? Note, relative - weighting will only work well when there is at least one spectrum with a - reasonable S/N, and a continuum. RJC note - This argument may only be - better when the object being used has a strong continuum + emission - lines. The reference spectrum is assigned a value of 1 for all - wavelengths, and the weights of all other spectra will be determined - relative to the reference spectrum. This is particularly useful if you - are dealing with highly variable spectra (e.g. emission lines) and - require a precision better than ~1 per cent. - 'ivar': - Use inverse variance weighting. This is not well tested and should probably be deprecated. + Parameters + ---------- - Returns: - list: Either a 2D `numpy.ndarray`_ or a list of 2D `numpy.ndarray`_ arrays containing the optimal - weights of each pixel for all frames, with shape (nspec, nspat). + raImg : `numpy.ndarray`_, list + A list of 2D array containing the RA of each pixel, with shape (nspec, nspat) + decImg : `numpy.ndarray`_, list + A list of 2D array containing the Dec of each pixel, with shape (nspec, nspat) + waveImg : `numpy.ndarray`_, list + A list of 2D array containing the wavelength of each pixel, with shape (nspec, nspat) + sciImg : `numpy.ndarray`_, list + A list of 2D array containing the science image of each pixel, with shape (nspec, nspat) + ivarImg : `numpy.ndarray`_, list + A list of 2D array containing the inverse variance image of each pixel, with shape (nspec, nspat) + slitidImg : `numpy.ndarray`_, list + A list of 2D array containing the slit ID of each pixel, with shape (nspec, nspat) + all_wcs : `astropy.wcs.WCS`_, list + A list of WCS objects, one for each frame. + all_tilts : `numpy.ndarray`_, list + 2D wavelength tilts frame, or a list of tilt frames + all_slits : :class:`~pypeit.slittrace.SlitTraceSet`, list + Information stored about the slits, or a list of SlitTraceSet objects + all_align : :class:`~pypeit.alignframe.AlignmentSplines`, list + A Class containing the transformation between detector pixel + coordinates and WCS pixel coordinates, or a list of Alignment + Splines. + all_dar : :class:`~pypeit.coadd3d.DARcorrection`, list + A Class containing the DAR correction information, or a list of DARcorrection + classes. If a list, it must be the same length as astrom_trans. + ra_offsets : float, list + RA offsets for each frame in units of degrees + dec_offsets : float, list + Dec offsets for each frame in units of degrees + whitelight_img : `numpy.ndarray`_ + A 2D array containing a white light image, that was created with the + input ``all`` arrays. + dspat : float + The size of each spaxel on the sky (in degrees) + dwv : float + The size of each wavelength pixel (in Angstroms) + sn_smooth_npix : float, optional + Number of pixels used for determining smoothly varying S/N ratio + weights. This is currently not required, since a relative weighting + scheme with a polynomial fit is used to calculate the S/N weights. + weight_method : `str`, optional + Weight method to be used in :func:`~pypeit.coadd.sn_weights`. + Options are ``'auto'``, ``'constant'``, ``'relative'``, or + ``'ivar'``. The default is ``'auto'``. Behavior is as follows: + + - ``'auto'``: Use constant weights if rms_sn < 3.0, otherwise + use wavelength dependent. + + - ``'constant'``: Constant weights based on rms_sn**2 + + - ``'uniform'``: Uniform weighting. + + - ``'wave_dependent'``: Wavelength dependent weights will be + used irrespective of the rms_sn ratio. This option will not + work well at low S/N ratio although it is useful for objects + where only a small fraction of the spectral coverage has high + S/N ratio (like high-z quasars). + + - ``'relative'``: Calculate weights by fitting to the ratio of + spectra? Note, relative weighting will only work well when + there is at least one spectrum with a reasonable S/N, and a + continuum. RJC note - This argument may only be better when + the object being used has a strong continuum + emission lines. + The reference spectrum is assigned a value of 1 for all + wavelengths, and the weights of all other spectra will be + determined relative to the reference spectrum. This is + particularly useful if you are dealing with highly variable + spectra (e.g. emission lines) and require a precision better + than ~1 per cent. + + - ``'ivar'``: Use inverse variance weighting. This is not well + tested and should probably be deprecated. + + Returns + ------- + all_wghts: list + Either a 2D `numpy.ndarray`_ or a list of 2D `numpy.ndarray`_ arrays + containing the optimal weights of each pixel for all frames, with shape + (nspec, nspat). """ msgs.info("Calculating the optimal weights of each pixel") # Check the inputs for combinations of lists or not, and then determine the number of frames @@ -1404,7 +1426,7 @@ def generate_cube_subpixel(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_im containing the tilts of each pixel. The shape of each numpy array is (nspec, nspat). slits (:class:`pypeit.slittrace.SlitTraceSet`, list): - A list of `pypeit.slittrace.SlitTraceSet`_ objects, one for each + A list of :class:`pypeit.slittrace.SlitTraceSet` objects, one for each spec2d file, containing the properties of the slit for each spec2d file astrom_trans (:class:`~pypeit.alignframe.AlignmentSplines`, list): A Class containing the transformation between detector pixel @@ -1560,7 +1582,7 @@ def subpixellate(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wgh containing the tilts of each pixel. The shape of each 2D array is (nspec, nspat). slits (:class:`pypeit.slittrace.SlitTraceSet`, list): - A list of `pypeit.slittrace.SlitTraceSet`_ objects, one for each + A list of :class:`pypeit.slittrace.SlitTraceSet` objects, one for each spec2d file, containing the properties of the slit for each spec2d file astrom_trans (:class:`~pypeit.alignframe.AlignmentSplines`, list): A Class containing the transformation between detector pixel diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 0c1ecf401e..13ce1b46c5 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -408,7 +408,7 @@ def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): Returns: `numpy.ndarray`_: Telluric model evaluated at the desired location theta_tell in model atmosphere parameter space. Shape is given by - the size of `wave_grid' plus `tell_pad_pix' padding from the input + the size of ``wave_grid`` plus ``tell_pad_pix`` padding from the input tell_dict. """ diff --git a/pypeit/find_objects.py b/pypeit/find_objects.py index a8bca7a557..59f5dede8a 100644 --- a/pypeit/find_objects.py +++ b/pypeit/find_objects.py @@ -960,13 +960,14 @@ def __init__(self, sciImg, slits, spectrograph, par, objtype, **kwargs): def initialize_slits(self, slits, initial=True): """ - Gather all the :class:`~pypeit.slittrace.SlitTraceSet` attributes - that we'll use here in :class:`FindObjects. Identical to the parent but the slits are not trimmed. - + Gather all the :class:`~pypeit.slittrace.SlitTraceSet` attributes that + we'll use here in :class:`FindObjects`. Identical to the parent but the + slits are not trimmed. Args: slits (:class:`~pypeit.slittrace.SlitTraceSet`): - SlitTraceSet object containing the slit boundaries that will be initialized. + SlitTraceSet object containing the slit boundaries that will be + initialized. initial (:obj:`bool`, optional): Use the initial definition of the slits. If False, tweaked slits are used. diff --git a/pypeit/images/detector_container.py b/pypeit/images/detector_container.py index 312a82b813..dcebd6ccf2 100644 --- a/pypeit/images/detector_container.py +++ b/pypeit/images/detector_container.py @@ -104,7 +104,7 @@ class DetectorContainer(datamodel.DataContainer): 'where the valid data sections can be obtained, one ' 'per amplifier. If defined explicitly should be in ' 'FITS format (e.g., [1:2048,10:4096]).'), - 'det': dict(otype=(int, np.int64), + 'det': dict(otype=(int, np.integer), descr='PypeIt designation for detector number (1-based).'), 'binning': dict(otype=str, descr='Binning in PypeIt orientation (not the original)')} From ff7b2775ab65f4d81016281848f423192c4e2e9e Mon Sep 17 00:00:00 2001 From: Dusty Reichwein Date: Fri, 15 Dec 2023 11:32:03 -0800 Subject: [PATCH 176/244] Fix setup gui type annotations to be Python 3.9 compatible. --- pypeit/setup_gui/dialog_helpers.py | 4 ++-- pypeit/setup_gui/model.py | 5 +++-- pypeit/setup_gui/text_viewer.py | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pypeit/setup_gui/dialog_helpers.py b/pypeit/setup_gui/dialog_helpers.py index f257c7a7c2..54e6c2749d 100644 --- a/pypeit/setup_gui/dialog_helpers.py +++ b/pypeit/setup_gui/dialog_helpers.py @@ -2,7 +2,7 @@ from __future__ import annotations import enum -from typing import Optional +from typing import Optional,Union from pathlib import Path from dataclasses import dataclass @@ -93,7 +93,7 @@ def __init__(self, caption : str, file_mode : QFileDialog.FileMode, file_type : Optional[FileType] =None, - default_file : Optional[str|Path] = None, + default_file : Optional[Union[str,Path]] = None, history : Optional[QStringListModel] = None, save : bool = False, ask_for_all : bool = False): diff --git a/pypeit/setup_gui/model.py b/pypeit/setup_gui/model.py index 639613e1ba..ca423ce50c 100644 --- a/pypeit/setup_gui/model.py +++ b/pypeit/setup_gui/model.py @@ -14,6 +14,7 @@ import numpy as np import astropy.table import io +import typing from pathlib import Path from functools import partial from qtpy.QtCore import QAbstractTableModel, QAbstractProxyModel, QAbstractItemModel, QAbstractListModel, QModelIndex, Qt, Signal, QObject, QThread, QStringListModel @@ -218,7 +219,7 @@ class PypeItMetadataModel(QAbstractTableModel): metadata: The PypeItMetaData object being wrapped. If this is None, the model is in a "NEW" state. """ - def __init__(self, metadata : PypeItMetaData | None): + def __init__(self, metadata : typing.Union[PypeItMetaData, None]): super().__init__() self.metadata = metadata @@ -448,7 +449,7 @@ def getDefaultColumns(self): else: return ['filename', 'frametype', 'ra', 'dec', 'target', 'dispname', 'decker', 'binning', 'mjd', 'airmass', 'exptime'] - def getStringColumnSize(self, colname: str) -> int | None: + def getStringColumnSize(self, colname: str) -> typing.Union[int,None]: """ Return the maximum size of a string column. diff --git a/pypeit/setup_gui/text_viewer.py b/pypeit/setup_gui/text_viewer.py index dc823e61d1..1f527f4328 100644 --- a/pypeit/setup_gui/text_viewer.py +++ b/pypeit/setup_gui/text_viewer.py @@ -1,6 +1,6 @@ from pathlib import Path import io -from typing import Optional +from typing import Optional,Union from qtpy.QtWidgets import QHBoxLayout, QVBoxLayout, QFileDialog, QWidget, QPlainTextEdit, QPushButton @@ -29,7 +29,7 @@ class TextViewerWindow(QWidget): """Signal sent when the window is closed.""" - def __init__(self, title : str, width : int, height : int, text_stream : io.TextIOBase, start_at_top : bool, filename: Optional[str|Path] = None, file_type : FileType = FileType("Text Files", ".txt")): + def __init__(self, title : str, width : int, height : int, text_stream : io.TextIOBase, start_at_top : bool, filename: Optional[Union[str,Path]] = None, file_type : FileType = FileType("Text Files", ".txt")): super().__init__() self._text_stream = text_stream self._file_type = file_type From 302a30a7e5a18fe323d4e2304554395193ca31ed Mon Sep 17 00:00:00 2001 From: Debora Pelliccia Date: Mon, 15 Jan 2024 13:22:48 -1000 Subject: [PATCH 177/244] comments --- pypeit/spectrographs/keck_deimos.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pypeit/spectrographs/keck_deimos.py b/pypeit/spectrographs/keck_deimos.py index b383d27e2a..55f40b0890 100644 --- a/pypeit/spectrographs/keck_deimos.py +++ b/pypeit/spectrographs/keck_deimos.py @@ -211,6 +211,8 @@ def get_detector_par(self, det, hdu=None): if hdu is not None: amp = self.get_meta_value(self.get_headarr(hdu), 'amp') + if amp == 'DUAL:A+B': + msgs.error('PypeIt can only reduce images with AMPMODE == SINGLE:B or AMPMODE == SINGLE:A.') amp_folder = "ampA" if amp == 'SINGLE:A' else "ampB" # raw frame date in mjd date = time.Time(self.get_meta_value(self.get_headarr(hdu), 'mjd'), format='mjd').value @@ -800,7 +802,6 @@ def get_rawimage(self, raw_file, det): rawdatasec_img = np.zeros_like(image, dtype=int) oscansec_img = np.zeros_like(image, dtype=int) - #embed(header='DEIMOS 754') # Loop over the chips for ii, tt in enumerate(chips): data, oscan = deimos_read_1chip(hdu, tt + 1) From 1588ff7fca29b0b5884360a1bf48e787d40be383 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Tue, 16 Jan 2024 08:28:44 -0800 Subject: [PATCH 178/244] turn off bookends in obslog --- pypeit/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/metadata.py b/pypeit/metadata.py index 36d1856e7c..b45cca9815 100644 --- a/pypeit/metadata.py +++ b/pypeit/metadata.py @@ -1914,7 +1914,7 @@ def write(self, output=None, rows=None, columns=None, sort_col=None, overwrite=F # Always write the table in ascii format with io.StringIO() as ff: - output_tbl.write(ff, format='ascii.fixed_width') + output_tbl.write(ff, format='ascii.fixed_width', bookend=False) data_lines = ff.getvalue().split('\n')[:-1] if ofile is None: From 346617ca92e36fb4cd67cbe958048cda60e0b272 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Tue, 16 Jan 2024 15:26:36 -0800 Subject: [PATCH 179/244] bitmask update --- pypeit/alignframe.py | 8 ++++---- pypeit/bitmask.py | 35 +++++++++++++++++++++++++---------- pypeit/tests/test_bitmask.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 14 deletions(-) diff --git a/pypeit/alignframe.py b/pypeit/alignframe.py index 189fbf9088..d085a5b345 100644 --- a/pypeit/alignframe.py +++ b/pypeit/alignframe.py @@ -143,11 +143,11 @@ def __init__(self, rawalignimg, slits, spectrograph, alignpar, det=1, qa_path=No # Attributes unique to this object self._alignprof = None - # Create a bad pixel mask - self.slit_bpm = self.slits.mask.astype(bool) - self.slit_bpm &= np.logical_not(self.slits.bitmask.flagged(self.slits.mask, flag=self.slits.bitmask.exclude_for_reducing)) - + self.slit_bpm = self.slits.bitmask.flagged(self.slits.mask, + expunge=self.slits.bitmask.exclude_for_reducing) +# self.slit_bpm = self.slits.mask.astype(bool) +# self.slit_bpm &= np.logical_not(self.slits.bitmask.flagged(self.slits.mask,flag=self.slits.bitmask.exclude_for_reducing)) # Completed steps self.steps = [] diff --git a/pypeit/bitmask.py b/pypeit/bitmask.py index a7b666d057..195b157e58 100644 --- a/pypeit/bitmask.py +++ b/pypeit/bitmask.py @@ -216,7 +216,7 @@ def minimum_dtype(self, asuint=False): return numpy.uint32 if asuint else numpy.int32 return numpy.uint64 if asuint else numpy.int64 - def flagged(self, value, flag=None): + def flagged(self, value, flag=None, exclude=None, expunge=None): """ Determine if a bit is on in the provided bitmask value. The function can be used to determine if any individual bit is on or @@ -229,11 +229,21 @@ def flagged(self, value, flag=None): flag (str, array-like, optional): One or more bit names to check. If None, then it checks if *any* bit is on. + exclude (str, array-like, optional): + One or more bit names to *exclude* from the check. If None, all + flags are included in the check. + expunge (str, array-like, optional): + One or more bit names to expunge from the check. If any of + these bits are flagged, the element in the returned array is as + if was not flagged, even if other flags not in this list *are* + flagged. I.e., the returned value can only be True if one of + these "expunge" bits are *not* flagged. If None, no flags are + expunged. Returns: - bool: Boolean flags that the provided flags (or any flag) is - on for the provided bitmask value. Shape is the same as - `value`. + bool, `numpy.ndarray`_: Boolean flags that the provided flags (or + any flag) is on for the provided bitmask value. If a numpy array, + the shape is the same as ``value``. Raises: KeyError: Raised by the dict data type if the input *flag* @@ -242,15 +252,20 @@ def flagged(self, value, flag=None): one or more strings. """ _flag = self._prep_flags(flag) + if exclude is not None: + # Remove the bits to exclude + _exclude = numpy.atleast_1d(exclude) + if not numpy.all(numpy.isin(_exclude, self.keys())): + raise ValueError(f'Not all exclude flags are valid: {exclude}') + _flag = numpy.setdiff1d(_flag, _exclude) - out = value & (1 << self.bits[_flag[0]]) != 0 - if len(_flag) == 1: - return out + # Bits to expunge + _exp = None if expunge is None else self.flagged(value, expunge) - nn = len(_flag) - for i in range(1,nn): + out = value & (1 << self.bits[_flag[0]]) != 0 + for i in range(1,len(_flag)): out |= (value & (1 << self.bits[_flag[i]]) != 0) - return out + return out if _exp is None else out & numpy.logical_not(_exp) def flagged_bits(self, value): """ diff --git a/pypeit/tests/test_bitmask.py b/pypeit/tests/test_bitmask.py index bea18c71d9..d826d0de61 100644 --- a/pypeit/tests/test_bitmask.py +++ b/pypeit/tests/test_bitmask.py @@ -124,3 +124,34 @@ def test_flag_order(): assert not bm.correct_flag_order(flags), 'Reordering the flags is not okay' +def test_exclude_expunge(): + + n = 1024 + shape = (n,n) + + rng = numpy.random.default_rng(99) + + image_bm = ImageBitMask() + mask = numpy.zeros(shape, dtype=image_bm.minimum_dtype()) + + cosmics_indx = numpy.zeros(shape, dtype=bool) + cosmics_indx[rng.integers(0,high=n,size=9000), rng.integers(0,high=n,size=9000)] = True + mask[cosmics_indx] = image_bm.turn_on(mask[cosmics_indx], 'COSMIC') + + saturated_indx = numpy.zeros(shape, dtype=bool) + saturated_indx[rng.integers(0,high=n,size=9000), rng.integers(0,high=n,size=9000)] = True + mask[saturated_indx] = image_bm.turn_on(mask[saturated_indx], 'SATURATED') + + # NOTE: Want to make sure there are pixels flagged as both COSMIC and + # SATURATED. Otherwise the `expunge` test is not useful. + assert numpy.sum(cosmics_indx & saturated_indx) > 0, 'Bad test setup' + + assert numpy.array_equal(image_bm.flagged(mask), cosmics_indx | saturated_indx), \ + 'Mask incorrect' + assert numpy.array_equal(image_bm.flagged(mask, exclude='SATURATED'), cosmics_indx), \ + 'Exclude incorrect' + + assert numpy.array_equal(image_bm.flagged(mask, expunge='SATURATED'), + cosmics_indx & numpy.logical_not(saturated_indx)), 'Expunge incorrect' + + From 2257d058b72097e858d434b28394c4abc29787be Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Wed, 17 Jan 2024 07:52:29 -0800 Subject: [PATCH 180/244] rename --- pypeit/alignframe.py | 2 +- pypeit/bitmask.py | 20 ++++++++++---------- pypeit/tests/test_bitmask.py | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pypeit/alignframe.py b/pypeit/alignframe.py index d085a5b345..cc0ce7a3d7 100644 --- a/pypeit/alignframe.py +++ b/pypeit/alignframe.py @@ -145,7 +145,7 @@ def __init__(self, rawalignimg, slits, spectrograph, alignpar, det=1, qa_path=No # Create a bad pixel mask self.slit_bpm = self.slits.bitmask.flagged(self.slits.mask, - expunge=self.slits.bitmask.exclude_for_reducing) + and_not=self.slits.bitmask.exclude_for_reducing) # self.slit_bpm = self.slits.mask.astype(bool) # self.slit_bpm &= np.logical_not(self.slits.bitmask.flagged(self.slits.mask,flag=self.slits.bitmask.exclude_for_reducing)) diff --git a/pypeit/bitmask.py b/pypeit/bitmask.py index 195b157e58..26d40078c1 100644 --- a/pypeit/bitmask.py +++ b/pypeit/bitmask.py @@ -216,7 +216,7 @@ def minimum_dtype(self, asuint=False): return numpy.uint32 if asuint else numpy.int32 return numpy.uint64 if asuint else numpy.int64 - def flagged(self, value, flag=None, exclude=None, expunge=None): + def flagged(self, value, flag=None, exclude=None, and_not=None): """ Determine if a bit is on in the provided bitmask value. The function can be used to determine if any individual bit is on or @@ -232,13 +232,13 @@ def flagged(self, value, flag=None, exclude=None, expunge=None): exclude (str, array-like, optional): One or more bit names to *exclude* from the check. If None, all flags are included in the check. - expunge (str, array-like, optional): - One or more bit names to expunge from the check. If any of - these bits are flagged, the element in the returned array is as - if was not flagged, even if other flags not in this list *are* - flagged. I.e., the returned value can only be True if one of - these "expunge" bits are *not* flagged. If None, no flags are - expunged. + and_not (str, array-like, optional): + One or more bit names to ensure are *not* selected by the check. + If any of these bits are flagged, the element in the returned + array is as if it was not flagged, even if other flags not in + this list *are* flagged. I.e., the returned value can only be + True if one of these "and_not" bits are *not* flagged. If None, + functionality ignored. Returns: bool, `numpy.ndarray`_: Boolean flags that the provided flags (or @@ -260,12 +260,12 @@ def flagged(self, value, flag=None, exclude=None, expunge=None): _flag = numpy.setdiff1d(_flag, _exclude) # Bits to expunge - _exp = None if expunge is None else self.flagged(value, expunge) + _and_not = None if and_not is None else self.flagged(value, and_not) out = value & (1 << self.bits[_flag[0]]) != 0 for i in range(1,len(_flag)): out |= (value & (1 << self.bits[_flag[i]]) != 0) - return out if _exp is None else out & numpy.logical_not(_exp) + return out if _and_not is None else out & numpy.logical_not(_and_not) def flagged_bits(self, value): """ diff --git a/pypeit/tests/test_bitmask.py b/pypeit/tests/test_bitmask.py index d826d0de61..0f443f5de1 100644 --- a/pypeit/tests/test_bitmask.py +++ b/pypeit/tests/test_bitmask.py @@ -143,7 +143,7 @@ def test_exclude_expunge(): mask[saturated_indx] = image_bm.turn_on(mask[saturated_indx], 'SATURATED') # NOTE: Want to make sure there are pixels flagged as both COSMIC and - # SATURATED. Otherwise the `expunge` test is not useful. + # SATURATED. Otherwise the `and_not` test is not useful. assert numpy.sum(cosmics_indx & saturated_indx) > 0, 'Bad test setup' assert numpy.array_equal(image_bm.flagged(mask), cosmics_indx | saturated_indx), \ @@ -151,7 +151,7 @@ def test_exclude_expunge(): assert numpy.array_equal(image_bm.flagged(mask, exclude='SATURATED'), cosmics_indx), \ 'Exclude incorrect' - assert numpy.array_equal(image_bm.flagged(mask, expunge='SATURATED'), + assert numpy.array_equal(image_bm.flagged(mask, and_not='SATURATED'), cosmics_indx & numpy.logical_not(saturated_indx)), 'Expunge incorrect' From e93c484d9eaebb62d0df47d27df768566b28d6c2 Mon Sep 17 00:00:00 2001 From: rcooke Date: Wed, 17 Jan 2024 18:10:55 +0000 Subject: [PATCH 181/244] Addresses PR comments --- pypeit/coadd3d.py | 26 ++++++++---------- pypeit/core/flat.py | 37 ++++++++++++++++++++++++++ pypeit/flatfield.py | 26 +++++------------- pypeit/images/rawimage.py | 56 --------------------------------------- pypeit/inputfiles.py | 8 ++---- 5 files changed, 57 insertions(+), 96 deletions(-) diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index 2513a7241a..6854e23bb5 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -406,18 +406,14 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs self.combine = self.cubepar['combine'] self.align = self.cubepar['align'] # Do some quick checks on the input options - if skysub_frame is not None: - if len(skysub_frame) != self.numfiles: - msgs.error("The skysub_frame list should be identical length to the spec2dfiles list") - if scale_corr is not None: - if len(scale_corr) != self.numfiles: - msgs.error("The scale_corr list should be identical length to the spec2dfiles list") - if ra_offsets is not None: - if len(ra_offsets) != self.numfiles: - msgs.error("The ra_offsets list should be identical length to the spec2dfiles list") - if dec_offsets is not None: - if len(dec_offsets) != self.numfiles: - msgs.error("The dec_offsets list should be identical length to the spec2dfiles list") + if skysub_frame is not None and len(skysub_frame) != self.numfiles: + msgs.error("The skysub_frame list should be identical length to the spec2dfiles list") + if scale_corr is not None and len(scale_corr) != self.numfiles: + msgs.error("The scale_corr list should be identical length to the spec2dfiles list") + if ra_offsets is not None and len(ra_offsets) != self.numfiles: + msgs.error("The ra_offsets list should be identical length to the spec2dfiles list") + if dec_offsets is not None and len(dec_offsets) != self.numfiles: + msgs.error("The dec_offsets list should be identical length to the spec2dfiles list") # Make sure both ra_offsets and dec_offsets are either both None or both lists if ra_offsets is None and dec_offsets is not None: msgs.error("If you provide dec_offsets, you must also provide ra_offsets") @@ -448,8 +444,8 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs # User offsets are not provided, so turn off the user_alignment self.user_alignment = False # Initialise the lists of ra_offsets and dec_offsets - self.ra_offsets = [0.0 for _ in range(self.numfiles)] - self.dec_offsets = [0.0 for _ in range(self.numfiles)] + self.ra_offsets = [0.0]*self.numfiles + self.dec_offsets = [0.0]*self.numfiles # Check on Spectrograph input if spectrograph is None: @@ -1202,7 +1198,7 @@ def run_align(self): # Grab cos(dec) for convenience cosdec = np.cos(np.mean(self.ifu_dec[0]) * np.pi / 180.0) # Initialize the RA and Dec offset arrays - ra_offsets, dec_offsets = [0 for _ in range(self.numfiles)], [0 for _ in range(self.numfiles)] + ra_offsets, dec_offsets = [0.0]*self.numfiles, [0.0]*self.numfiles # Register spatial offsets between all frames if self.user_alignment: # The user has specified offsets - update these values accounting for the difference in header RA/DEC diff --git a/pypeit/core/flat.py b/pypeit/core/flat.py index d938f79e84..ffc6b09a1d 100644 --- a/pypeit/core/flat.py +++ b/pypeit/core/flat.py @@ -361,6 +361,43 @@ def illum_profile_spectral_poly(rawimg, waveimg, slitmask, slitmask_trim, model, return scaleImg +def smooth_scale(arr, wave_ref=None, polydeg=None, sn_smooth_npix=None): + """ + Smooth the relative sensitivity array using a polynomial fit or a boxcar filter. + + Parameters + ---------- + arr : `numpy.ndarray`_ + Array containing the relative sensitivity + wave_ref : `numpy.ndarray`_, optional + Wavelength array corresponding to the relative sensitivity array. Only used if polydeg is not None. + polydeg : :obj:`int`, optional + Degree of the polynomial fit to the relative sensitivity array. If None, a boxcar filter will be used. + sn_smooth_npix : :obj:`int`, optional + Number of pixels to use for the boxcar filter. Only used if polydeg is None. + + Returns + ------- + arr_smooth : `numpy.ndarray` + Smoothed relative sensitivity array + """ + # Do some checks on the input + if polydeg is not None and wave_ref is None: + msgs.error("Must provide a wavelength array if polydeg is not None") + if polydeg is None and sn_smooth_npix is None: + msgs.error("Must provide either polydeg or sn_smooth_npix") + # Smooth the relative sensitivity array + if polydeg is not None: + gd = (arr != 0) + wave_norm = (wave_ref - wave_ref[0]) / (wave_ref[1] - wave_ref[0]) + coeff = np.polyfit(wave_norm[gd], arr[gd], polydeg) + ref_relscale = np.polyval(coeff, wave_norm) + else: + ref_relscale = coadd.smooth_weights(arr, (arr != 0), sn_smooth_npix) + # Return the smoothed relative sensitivity array + return ref_relscale + + # TODO:: See pypeit/deprecated/flat.py for a spline version. The following polynomial version is faster, but # the spline version is more versatile. def poly_map(rawimg, rawivar, waveimg, slitmask, slitmask_trim, modelimg, deg=3, diff --git a/pypeit/flatfield.py b/pypeit/flatfield.py index 85cbfd5cc6..460e165376 100644 --- a/pypeit/flatfield.py +++ b/pypeit/flatfield.py @@ -1153,7 +1153,7 @@ def fit(self, spat_illum_only=False, doqa=True, debug=False): spat_illum_fine = 1 # Default value if the fine correction is not performed if exit_status <= 1 and self.flatpar['slit_illum_finecorr']: spat_model = np.ones_like(spec_model) - spat_model[onslit_padded] = spat_bspl.value(spat_coo_final[onslit_padded])[0]#np.exp(spec_bspl.value(spec_coo[onslit_padded])[0]) + spat_model[onslit_padded] = spat_bspl.value(spat_coo_final[onslit_padded])[0] specspat_illum = np.fmax(spec_model, 1.0) * spat_model norm_spatspec = rawflat / specspat_illum self.spatial_fit_finecorr(norm_spatspec, onslit_tweak, slit_idx, slit_spat, gpm, doqa=doqa) @@ -1795,11 +1795,10 @@ def show_flats(image_list, wcs_match=True, slits=None, waveimg=None): clear = False +# TODO :: This could possibly be moved to core.flat def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_npix=None, polydeg=None, model=None, gpmask=None, skymask=None, trim=3, flexure=None, maxiter=5): """ - TODO :: This could possibly be moved to core.flat - Determine the relative spectral illumination of all slits. Currently only used for image slicer IFUs. @@ -1899,17 +1898,12 @@ def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_ if (ii == 1) and (slits.spat_id[wvsrt[ss]] == slit_illum_ref_idx): # This must be the first element of the loop by construction, but throw an error just in case if ss != 0: - msgs.error("An error has occurred in the relative spectral illumination. Please contact the developers.") + msgs.error("CODING ERROR - An error has occurred in the relative spectral illumination." + + msgs.newline() + "Please contact the developers.") tmp_cntr = cntr * spec_ref tmp_arr = hist * utils.inverse(tmp_cntr) # Calculate a smooth version of the relative response - if polydeg is not None: - gd = (tmp_arr != 0) - wave_norm = (wave_ref-wave_ref[0]) / (wave_ref[1]-wave_ref[0]) - coeff = np.polyfit(wave_norm[gd], tmp_arr[gd], polydeg) - ref_relscale = np.polyval(coeff, wave_norm) - else: - ref_relscale = coadd.smooth_weights(tmp_arr, (tmp_arr != 0), sn_smooth_npix) + ref_relscale = flat.smooth_scale(tmp_arr, wave_ref=wave_ref, polydeg=polydeg, sn_smooth_npix=sn_smooth_npix) # Update the reference spectrum spec_ref /= ref_relscale # Normalise by the reference spectrum @@ -1917,20 +1911,14 @@ def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_ norm = utils.inverse(cntr) arr = hist * norm # Calculate a smooth version of the relative response - if polydeg is not None: - gd = (arr != 0) - wave_norm = (wave_ref - wave_ref[0]) / (wave_ref[1] - wave_ref[0]) - coeff = np.polyfit(wave_norm[gd], arr[gd], polydeg) - relscale = np.polyval(coeff, wave_norm) - else: - relscale = coadd.smooth_weights(arr, (arr != 0), sn_smooth_npix) + relscale = flat.smooth_scale(arr, wave_ref=wave_ref, polydeg=polydeg, sn_smooth_npix=sn_smooth_npix) # Store the result relscl_model[onslit_b_init] = interpolate.interp1d(wave_ref, relscale, kind='linear', bounds_error=False, fill_value="extrapolate")(waveimg[onslit_b_init]) # Build a new reference spectrum to increase wavelength coverage of the reference spectrum (and improve S/N) if ii == 0: - onslit_ref_trim = onslit_ref_trim | (onslit_b & gpm & skymask_now) + onslit_ref_trim |= (onslit_b & gpm & skymask_now) hist, edge = np.histogram(waveimg[onslit_ref_trim], bins=wavebins, weights=modelimg_copy[onslit_ref_trim]/relscl_model[onslit_ref_trim]) cntr, edge = np.histogram(waveimg[onslit_ref_trim], bins=wavebins) cntr = cntr.astype(float) diff --git a/pypeit/images/rawimage.py b/pypeit/images/rawimage.py index 247deb6128..8108c4b828 100644 --- a/pypeit/images/rawimage.py +++ b/pypeit/images/rawimage.py @@ -838,62 +838,6 @@ def flatfield(self, flatimages, slits=None, force=False, debug=False): # Apply flat-field correction # NOTE: Using flat.flatfield to effectively multiply image*img_scale is # a bit overkill... - if debug: - iput = self.image[0].copy() - total_flat = flatimages.pixelflat_norm * illum_flat * spec_illum - self.img_scale = np.expand_dims(utils.inverse(total_flat), 0) - oput, flat_bpm = flat.flatfield(self.image[0], total_flat) - from matplotlib import pyplot as plt - from pypeit import edgetrace, wavetilts, wavecalib - from scipy.signal import savgol_filter - - edges_file = "Calibrations/Edges_A_0_DET01.fits.gz" - tilts_file = "Calibrations/Tilts_A_0_DET01.fits" - wvcalib_file = "Calibrations/WaveCalib_A_0_DET01.fits" - flex = None - slits = edgetrace.EdgeTraceSet.from_file(edges_file).get_slits() - wvtilts = wavetilts.WaveTilts.from_file(tilts_file) - wv_calib = wavecalib.WaveCalib.from_file(wvcalib_file) - - slitmask = slits.slit_img(initial=True, flexure=flex) - tilts = wvtilts.fit2tiltimg(slitmask, flexure=flex) - waveimg = wv_calib.build_waveimg(tilts, slits, spat_flexure=flex) - - # extract a spectrum down the centre of each slit - left, right, _ = slits.select_edges(initial=True) - cen = np.round(0.5 * (left + right)).astype(int) - lcen = np.round(0.75*left + 0.25*right).astype(int) - rcen = np.round(0.25*left + 0.75*right).astype(int) - - colors = plt.cm.jet(np.linspace(0, 1, slits.nslits)) - navg = 21 - for ss in range(slits.nslits): - print("slit {0:d}".format(ss+1)) - inarr = np.zeros((iput.shape[0], navg)) - outarra = np.zeros((iput.shape[0], navg)) - outarrb = np.zeros((iput.shape[0], navg)) - outarrc = np.zeros((iput.shape[0], navg)) - for ll in range(navg): - ww = (np.arange(iput.shape[0]), cen[:, ss] + ll - navg // 2) - inarr[:, ll] = iput[ww] - outarra[:, ll] = oput[ww] - ww = (np.arange(iput.shape[0]), lcen[:, ss] + ll - navg // 2) - outarrb[:, ll] = oput[ww] - ww = (np.arange(iput.shape[0]), rcen[:, ss] + ll - navg // 2) - outarrc[:, ll] = oput[ww] - wa = (np.arange(iput.shape[0]), cen[:, ss]) - wb = (np.arange(iput.shape[0]), lcen[:, ss]) - wc = (np.arange(iput.shape[0]), rcen[:, ss]) - plt.subplot(221) - plt.plot(waveimg[wa], np.median(inarr, axis=1), color=colors[ss]) - plt.subplot(222) - plt.plot(waveimg[wa], savgol_filter(np.median(outarra, axis=1), 25, 3), color=colors[ss]) - plt.subplot(223) - plt.plot(waveimg[wb], savgol_filter(np.median(outarrb, axis=1), 25, 3), color=colors[ss]) - plt.subplot(224) - plt.plot(waveimg[wc], savgol_filter(np.median(outarrc, axis=1), 25, 3), color=colors[ss]) - # plt.plot(waveimg[ww], np.median(outarr, axis=1), color=colors[ss]) - plt.show() total_flat = flatimages.pixelflat_norm * illum_flat * spec_illum self.img_scale = np.expand_dims(utils.inverse(total_flat), 0) self.image[0], flat_bpm = flat.flatfield(self.image[0], total_flat) diff --git a/pypeit/inputfiles.py b/pypeit/inputfiles.py index 6759dffae1..332822081f 100644 --- a/pypeit/inputfiles.py +++ b/pypeit/inputfiles.py @@ -938,9 +938,7 @@ def options(self): off_ra, off_dec = None, None if 'ra_offset' in self.data.keys(): off_ra = self.data['ra_offset'].tolist() - if off_ra is None: - opts['ra_offset'] = None - elif len(off_ra) == 1 and len(self.filenames) > 1: + if len(off_ra) == 1 and len(self.filenames) > 1: # Convert from arcsec to degrees opts['ra_offset'] = [off_ra[0]/3600.0 for _ in range(len(self.filenames))] elif len(off_ra) != 0: @@ -949,9 +947,7 @@ def options(self): # Get the DEC offset of each file if 'dec_offset' in self.data.keys(): off_dec = self.data['dec_offset'].tolist() - if off_dec is None: - opts['dec_offset'] = None - elif len(off_dec) == 1 and len(self.filenames) > 1: + if len(off_dec) == 1 and len(self.filenames) > 1: # Convert from arcsec to degrees opts['dec_offset'] = [off_dec[0]/3600.0 for _ in range(len(self.filenames))] elif len(off_dec) != 0: From cccf31a5ed8c3b7a5ef36b3a9d3210a9f4af7299 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Wed, 17 Jan 2024 21:53:30 +0100 Subject: [PATCH 182/244] updated default telluric file to PCA mode --- pypeit/scripts/tellfit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pypeit/scripts/tellfit.py b/pypeit/scripts/tellfit.py index d0e0626f2f..08192e5da7 100644 --- a/pypeit/scripts/tellfit.py +++ b/pypeit/scripts/tellfit.py @@ -126,8 +126,8 @@ def main(args): if par['sensfunc']['IR']['telgridfile'] is not None: par['telluric']['telgridfile'] = par['sensfunc']['IR']['telgridfile'] else: - par['telluric']['telgridfile'] = 'TelFit_MaunaKea_3100_26100_R20000.fits' - msgs.warn(f"No telluric grid file given. Using {par['telluric']['telgridfile']}.") + par['telluric']['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' + msgs.warn(f"No telluric file given. Using PCA method with {par['telluric']['telgridfile']}.") # Checks if par['telluric']['telgridfile'] is None: From 12ac602422fbbe012317003e3758678f98a53988 Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Wed, 17 Jan 2024 22:03:01 +0100 Subject: [PATCH 183/244] added default to parset --- pypeit/par/pypeitpar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index 8409519610..3217e5cb29 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -2313,7 +2313,7 @@ def __init__(self, telgridfile=None, sn_clip=None, resln_guess=None, resln_frac_ dtypes = OrderedDict.fromkeys(pars.keys()) descr = OrderedDict.fromkeys(pars.keys()) - defaults['telgridfile'] = None + defaults['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' dtypes['telgridfile'] = str descr['telgridfile'] = 'File containing the telluric grid for the observatory in question. These grids are ' \ 'generated from HITRAN models for each observatory using nominal site parameters. They ' \ From 4833158426349fd39536fa5633dbcda59a476279 Mon Sep 17 00:00:00 2001 From: rcooke Date: Wed, 17 Jan 2024 22:42:41 +0000 Subject: [PATCH 184/244] fixed typo --- pypeit/specobjs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/specobjs.py b/pypeit/specobjs.py index 8578d382d6..3037a652c6 100644 --- a/pypeit/specobjs.py +++ b/pypeit/specobjs.py @@ -574,7 +574,7 @@ def apply_flux_calib(self, par, spectrograph, sens): # TODO enbaling this for now in case someone wants to treat the IFU as a slit spectrograph # (not recommnneded but useful for quick reductions where you don't want to construct cubes and don't care about DAR). - if spectrograph.pypeline in ['Multislit','SlicerIFU']: + if spectrograph.pypeline in ['MultiSlit','SlicerIFU']: for ii, sci_obj in enumerate(self.specobjs): if sens.wave.shape[1] == 1: sci_obj.apply_flux_calib(sens.wave[:, 0], sens.zeropoint[:, 0], From 7c26fbe7965317e63a80d45a7c796da59fa54862 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 18 Jan 2024 08:42:46 -0800 Subject: [PATCH 185/244] comment edits --- pypeit/edgetrace.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pypeit/edgetrace.py b/pypeit/edgetrace.py index 9b356335c9..dde1775ab0 100644 --- a/pypeit/edgetrace.py +++ b/pypeit/edgetrace.py @@ -5099,8 +5099,7 @@ def slit_spatial_center(self, normalized=True, spec=None, use_center=False, gpm = self.good_traces(include_box=include_box) good_slit = np.all(gpm.reshape(-1,2), axis=1) - # TODO: Use reference_row by default? Except that it's only - # defined if the PCA is defined. + # Set the spectral position to use as a reference. _spec = spec if _spec is None: if self.pcatype is None: @@ -5217,7 +5216,7 @@ def match_order(self, reference_row=None): self.edge_msk[:,flag] = self.bitmask.turn_on(self.edge_msk[:,flag], 'ORDERMISMATCH') # Minimum separation between the order and its matching slit; - # keep the signed value for reporting, but used the absolute + # keep the signed value for reporting, but use the absolute # value of the difference for vetting below. # NOTE: This includes indices for orders that were not found. This is # largely for book-keeping purposes in the print statement below. From 34059b0de4c9a9d8fedc315ec5e29a1eea30a98d Mon Sep 17 00:00:00 2001 From: rcooke Date: Thu, 18 Jan 2024 17:08:39 +0000 Subject: [PATCH 186/244] fix coadd throw warning --- pypeit/coadd3d.py | 7 ++++--- pypeit/core/flat.py | 1 + pypeit/flatfield.py | 1 - 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index 6854e23bb5..770ee8d6f5 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -787,11 +787,12 @@ def add_grating_corr(self, flatfile, waveimg, slits, spat_flexure=None): spat_flexure : :obj:`float`, optional: Spatial flexure in pixels """ + # Check if the Flat file exists + if not os.path.exists(flatfile): + msgs.warn("Grating correction requested, but the following file does not exist:" + msgs.newline() + flatfile) + return if flatfile not in self.flat_splines.keys(): msgs.info("Calculating relative sensitivity for grating correction") - # Check if the Flat file exists - if not os.path.exists(flatfile): - msgs.error("Grating correction requested, but the following file does not exist:" + msgs.newline() + flatfile) # Load the Flat file flatimages = flatfield.FlatImages.from_file(flatfile) total_illum = flatimages.fit2illumflat(slits, finecorr=False, frametype='illum', initial=True, spat_flexure=spat_flexure) * \ diff --git a/pypeit/core/flat.py b/pypeit/core/flat.py index ffc6b09a1d..1c104d1d96 100644 --- a/pypeit/core/flat.py +++ b/pypeit/core/flat.py @@ -19,6 +19,7 @@ from pypeit.core import parse from pypeit.core import pixels from pypeit.core import tracewave +from pypeit.core import coadd from pypeit import utils from pypeit.core import pydl diff --git a/pypeit/flatfield.py b/pypeit/flatfield.py index 460e165376..ed5e3880a0 100644 --- a/pypeit/flatfield.py +++ b/pypeit/flatfield.py @@ -28,7 +28,6 @@ from pypeit.core import tracewave from pypeit.core import basis from pypeit.core import fitting -from pypeit.core import coadd from pypeit import slittrace From 25321d934e26146055961c3a8636d71da6d8c3f7 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 18 Jan 2024 10:49:30 -0800 Subject: [PATCH 187/244] add _check_parsed function --- pypeit/calibframe.py | 11 +++------- pypeit/core/telluric.py | 1 + pypeit/core/wavecal/wv_fitting.py | 2 +- pypeit/datamodel.py | 36 ++++++++++++++++++++++--------- pypeit/edgetrace.py | 18 ++++++---------- pypeit/images/pypeitimage.py | 9 ++------ pypeit/sensfunc.py | 10 +++------ pypeit/slittrace.py | 10 +++------ pypeit/tracepca.py | 9 ++++---- 9 files changed, 50 insertions(+), 56 deletions(-) diff --git a/pypeit/calibframe.py b/pypeit/calibframe.py index 30826d8683..7acc033dff 100644 --- a/pypeit/calibframe.py +++ b/pypeit/calibframe.py @@ -169,15 +169,10 @@ def from_hdu(cls, hdu, chk_version=True, **kwargs): **kwargs: Passed directly to :func:`~pypeit.datamodel.DataContainer._parse`. """ + # Parse d, dm_version_passed, dm_type_passed, parsed_hdus = cls._parse(hdu, **kwargs) - # Check version and type? - if not dm_type_passed: - msgs.error(f'The HDU(s) cannot be parsed by a {cls.__name__} object!') - if not dm_version_passed: - _f = msgs.error if chk_version else msgs.warn - _f(f'Current version of {cls.__name__} object in code ({cls.version}) ' - 'does not match version used to write your HDU(s)!') - + # Check + cls._check_parsed(dm_version_passed, dm_type_passed, chk_version=chk_version) # Instantiate self = cls.from_dict(d=d) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 0c1ecf401e..feedda488a 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -756,6 +756,7 @@ def general_spec_reader(specfile, ret_flam=False): head = spec.head0 else: sobjs = specobjs.SpecObjs.from_fitsfile(specfile, chk_version=False) + # TODO: What bug? Is it fixed now? How can we test if it's fixed? if np.sum(sobjs.OPT_WAVE) is None: raise ValueError("This is an ugly hack until the DataContainer bug is fixed") head = sobjs.header diff --git a/pypeit/core/wavecal/wv_fitting.py b/pypeit/core/wavecal/wv_fitting.py index 7ab7fc40f0..8333647f04 100644 --- a/pypeit/core/wavecal/wv_fitting.py +++ b/pypeit/core/wavecal/wv_fitting.py @@ -153,7 +153,7 @@ def from_hdu(cls, hdu, **kwargs): else hdu.header['SPAT_ID'] hdu_prefix = cls.hduext_prefix_from_spatid(spat_id) # Run the default parser to get the data - return super(WaveFit, cls).from_hdu(hdu, hdu_prefix=hdu_prefix, **kwargs) + return super().from_hdu(hdu, hdu_prefix=hdu_prefix, **kwargs) @property def ions(self): diff --git a/pypeit/datamodel.py b/pypeit/datamodel.py index 9101420aa5..b352198af7 100644 --- a/pypeit/datamodel.py +++ b/pypeit/datamodel.py @@ -1129,6 +1129,30 @@ def _parse(cls, hdu, ext=None, ext_pseudo=None, transpose_table_arrays=False, return _d, dm_version_passed and found_data, dm_type_passed and found_data, \ np.unique(parsed_hdus).tolist() + @classmethod + def _check_parsed(cls, version_passed, type_passed, chk_version=True): + """ + Convenience function that issues the warnings/errors caused by parsing a + file into a datamodel. + + Args: + version_passed (:obj:`bool`): + Flag that the datamodel version is correct. + type_passed (:obj:`bool`): + Flag that the datamodel class type is correct. + chk_version (:obj:`bool`, optional): + Flag to impose strict version checking. + """ + if not type_passed: + msgs.error(f'The HDU(s) cannot be parsed by a {cls.__name__} object!', + cls='PypeItDataModelError') + if not version_passed: + msg = f'Current version of {cls.__name__} object in code ({cls.version}) ' \ + 'does not match version used to write your HDU(s)!' + if chk_version: + msgs.error(msg, cls='PypeItDataModelError') + else: + msgs.warn(msg) def __getattr__(self, item): """Maps values to attributes. @@ -1424,18 +1448,10 @@ def from_hdu(cls, hdu, chk_version=True, **kwargs): **kwargs: Passed directly to :func:`_parse`. """ + # Parse the data d, dm_version_passed, dm_type_passed, parsed_hdus = cls._parse(hdu, **kwargs) # Check version and type? - if not dm_type_passed: - msgs.error(f'The HDU(s) cannot be parsed by a {cls.__name__} object!', - cls='PypeItDataModelError') - if not dm_version_passed: - msg = f'Current version of {cls.__name__} object in code ({cls.version}) ' \ - 'does not match version used to write your HDU(s)!' - if chk_version: - msgs.error(msg, cls='PypeItDataModelError') - else: - msgs.warn(msg) + cls._check_parsed(dm_version_passed, dm_type_passed, chk_version=chk_version) # Instantiate # NOTE: We can't use `cls(d)`, where `d` is the dictionary returned by diff --git a/pypeit/edgetrace.py b/pypeit/edgetrace.py index 613f6d4b2f..e1edf149c8 100644 --- a/pypeit/edgetrace.py +++ b/pypeit/edgetrace.py @@ -184,7 +184,7 @@ def __init__(self): 'order missed by the automated tracing'), ('LARGELENGTHCHANGE', 'Large difference in the slit length as a function of ' 'wavelength.')]) - super(EdgeTraceBitMask, self).__init__(list(mask.keys()), descr=list(mask.values())) + super().__init__(list(mask.keys()), descr=list(mask.values())) @property def bad_flags(self): @@ -497,7 +497,7 @@ def __init__(self, traceimg, spectrograph, par, qa_path=None, auto=False, debug= show_stages=False): # Instantiate as an empty DataContainer - super(EdgeTraceSet, self).__init__() + super().__init__() # Check input types if not isinstance(traceimg, TraceImage): @@ -1207,7 +1207,7 @@ def to_hdu(self, **kwargs): # Do not need to change the default behavior if the PCA # doesn't exist or there is only one PCA for both left and # right edges. - return super(EdgeTraceSet, self).to_hdu(**kwargs) + return super().to_hdu(**kwargs) # TODO: We need a better solution for multiple levels of nested # DataContainers. Here the commpication is that we're writing @@ -1220,7 +1220,7 @@ def to_hdu(self, **kwargs): self.left_pca, self.right_pca = None, None # Run the default (with add_primary = False) - hdu = super(EdgeTraceSet, self).to_hdu(**kwargs) + hdu = super().to_hdu(**kwargs) # Reset them self.left_pca, self.right_pca = _left_pca, _right_pca @@ -1260,12 +1260,8 @@ def from_hdu(cls, hdu, hdu_prefix=None, chk_version=True): # DataContainer. It *will* parse pca, left_pca, and right_pca, # if they exist, but not their model components. d, version_passed, type_passed, parsed_hdus = cls._parse(hdu) - if not type_passed: - msgs.error('The HDU(s) cannot be parsed by a {0} object!'.format(cls.__name__)) - if not version_passed: - _f = msgs.error if chk_version else msgs.warn - _f('Current version of {0} object in code (v{1})'.format(cls.__name__, cls.version) - + ' does not match version used to write your HDU(s)!') + # Check + cls._check_parsed(version_passed, type_passed, chk_version=chk_version) # Instantiate the TraceImage from the header d['traceimg'] = TraceImage.from_hdu(hdu, chk_version=chk_version) @@ -1292,7 +1288,7 @@ def from_hdu(cls, hdu, hdu_prefix=None, chk_version=True): else cls.datamodel[key]['atype']) # Instantiate - self = super(EdgeTraceSet, cls).from_dict(d=d) + self = super().from_dict(d=d) # Calibration frame attributes # NOTE: If multiple HDUs are parsed, this assumes that the information diff --git a/pypeit/images/pypeitimage.py b/pypeit/images/pypeitimage.py index 35bc7d9862..fe123d26f8 100644 --- a/pypeit/images/pypeitimage.py +++ b/pypeit/images/pypeitimage.py @@ -284,13 +284,8 @@ def from_hdu(cls, hdu, chk_version=True, hdu_prefix=None): # Parse everything *but* the mask extension d, version_passed, type_passed, parsed_hdus \ - = super()._parse(hdu, ext=ext, hdu_prefix=_hdu_prefix) - if not type_passed: - msgs.error(f'The HDU(s) cannot be parsed by a {cls.__name__} object!') - if not version_passed: - _f = msgs.error if chk_version else msgs.warn - _f(f'Current version of {cls.__name__} object in code (v{cls.version})' - ' does not match version used to write your HDU(s)!') + = cls._parse(hdu, ext=ext, hdu_prefix=_hdu_prefix) + cls._check_parsed(version_passed, type_passed, chk_version=chk_version) if mask_ext in hdu: # If the mask extension exists, parse it diff --git a/pypeit/sensfunc.py b/pypeit/sensfunc.py index 1719545160..a6637046a2 100644 --- a/pypeit/sensfunc.py +++ b/pypeit/sensfunc.py @@ -410,13 +410,9 @@ def from_hdu(cls, hdu, hdu_prefix=None, chk_version=True): """ # Run the default parser to get most of the data. This correctly parses # everything except for the Telluric.model data table. - d, version_passed, type_passed, parsed_hdus = super()._parse(hdu, allow_subclasses=True) - if not type_passed: - msgs.error('The HDU(s) cannot be parsed by a {0} object!'.format(cls.__name__)) - if not version_passed: - _f = msgs.error if chk_version else msgs.warn - _f('Current version of {0} object in code (v{1})'.format(cls.__name__, cls.version) - + ' does not match version used to write your HDU(s)!') + d, version_passed, type_passed, parsed_hdus = cls._parse(hdu, allow_subclasses=True) + # Check + cls._check_parsed(version_passed, type_passed, chk_version=chk_version) # Load the telluric model, if it exists if 'TELLURIC' in [h.name for h in hdu]: diff --git a/pypeit/slittrace.py b/pypeit/slittrace.py index 4ee62dfa0a..286cbc37e1 100644 --- a/pypeit/slittrace.py +++ b/pypeit/slittrace.py @@ -337,12 +337,8 @@ def from_hdu(cls, hdu, hdu_prefix=None, chk_version=True): """ # Run the default parser d, version_passed, type_passed, parsed_hdus = cls._parse(hdu) - if not type_passed: - msgs.error('The HDU(s) cannot be parsed by a {0} object!'.format(cls.__name__)) - if not version_passed: - _f = msgs.error if chk_version else msgs.warn - _f('Current version of {0} object in code (v{1})'.format(cls.__name__, cls.version) - + ' does not match version used to write your HDU(s)!') + # Check + cls._check_parsed(version_passed, type_passed, chk_version=chk_version) # Instantiate self = super().from_dict(d=d) @@ -441,7 +437,7 @@ def slitord_to_zero(self, slitord): """ if self.pypeline in ['MultiSlit', 'SlicerIFU']: return np.where(self.spat_id == slitord)[0][0] - if self.pypeline in ['Echelle']: + if self.pypeline == 'Echelle': return np.where(self.ech_order == slitord)[0][0] msgs.error('Unrecognized Pypeline {:}'.format(self.pypeline)) diff --git a/pypeit/tracepca.py b/pypeit/tracepca.py index 2f59ff6e69..14ef7a3a44 100644 --- a/pypeit/tracepca.py +++ b/pypeit/tracepca.py @@ -112,7 +112,7 @@ def __init__(self, trace_cen=None, npca=None, pca_explained_var=99.0, reference_ coo=None): # Instantiate as an empty DataContainer - super(TracePCA, self).__init__() + super().__init__() self.is_empty = True # Only do the decomposition if the trace coordinates are provided. @@ -240,7 +240,7 @@ def predict(self, x): def _bundle(self, ext='PCA'): """Bundle the data for writing.""" - d = super(TracePCA, self)._bundle(ext=ext) + d = super()._bundle(ext=ext) if self.pca_coeffs_model is None: return d @@ -264,8 +264,7 @@ def _parse(cls, hdu, hdu_prefix=None, **kwargs): argument descriptions. """ # Run the default parser to get most of the data - d, version_passed, type_passed, parsed_hdus \ - = super(TracePCA, cls)._parse(hdu, hdu_prefix=hdu_prefix) + d, version_passed, type_passed, parsed_hdus = super()._parse(hdu, hdu_prefix=hdu_prefix) # This should only ever read one hdu! if len(parsed_hdus) > 1: @@ -291,7 +290,7 @@ def from_dict(cls, d=None): :class:`~pypeit.datamodel.DataContainer.from_dict` that appropriately toggles :attr:`is_empty`. """ - self = super(TracePCA, cls).from_dict(d=d) + self = super().from_dict(d=d) self.is_empty = False return self From ced1eb618a2ec1bf6047334a36d9d3a40f753162 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 18 Jan 2024 10:49:52 -0800 Subject: [PATCH 188/244] comment --- pypeit/onespec.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pypeit/onespec.py b/pypeit/onespec.py index c71f74b527..89f961724b 100644 --- a/pypeit/onespec.py +++ b/pypeit/onespec.py @@ -84,6 +84,7 @@ class OneSpec(datamodel.DataContainer): 'spect_meta', 'history'] + # TODO: Add chk_version to this? @classmethod def from_file(cls, ifile): """ From 0523ad9517454a9067a0d83f048c0915c5d70212 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 18 Jan 2024 11:11:37 -0800 Subject: [PATCH 189/244] propagate bitmask checks --- pypeit/coadd3d.py | 1 + pypeit/edgetrace.py | 3 ++- pypeit/images/bitmaskarray.py | 37 +++++++++++++++++++++++++++++++++++ pypeit/pypmsgs.py | 3 +++ pypeit/slittrace.py | 3 ++- 5 files changed, 45 insertions(+), 2 deletions(-) diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index 7801f42aaa..2710883039 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -168,6 +168,7 @@ def to_file(self, ofile, primary_hdr=None, hdr=None, **kwargs): # Do it super(DataCube, self).to_file(ofile, primary_hdr=primary_hdr, hdr=hdr, **kwargs) + # TODO: Pass in chk_version here? @classmethod def from_file(cls, ifile): """ diff --git a/pypeit/edgetrace.py b/pypeit/edgetrace.py index e1edf149c8..9b5d5d36a8 100644 --- a/pypeit/edgetrace.py +++ b/pypeit/edgetrace.py @@ -1314,7 +1314,8 @@ def from_hdu(cls, hdu, hdu_prefix=None, chk_version=True): hdr_bitmask = BitMask.from_header(hdu['SOBELSIG'].header) if chk_version and hdr_bitmask.bits != self.bitmask.bits: msgs.error('The bitmask in this fits file appear to be out of date! Recreate this ' - 'file either by rerunning run_pypeit or pypeit_trace_edges.') + 'file by re-running the relevant script or set chk_version=False.', + cls='PypeItBitMaskError') return self diff --git a/pypeit/images/bitmaskarray.py b/pypeit/images/bitmaskarray.py index db9c6898b6..ca728f9a6f 100644 --- a/pypeit/images/bitmaskarray.py +++ b/pypeit/images/bitmaskarray.py @@ -14,6 +14,7 @@ import numpy as np from pypeit.datamodel import DataContainer +from pypeit.bitmask import BitMask from pypeit import msgs @@ -149,6 +150,42 @@ def _bundle(self): d[0].update(self.bitmask.to_dict()) return d + @classmethod + def from_hdu(cls, hdu, hdu_prefix=None, chk_version=True): + """ + Instantiate the object from an HDU extension. + + This overrides the base-class method, only to add checks (or not) for + the bitmask. + + Args: + hdu (`astropy.io.fits.HDUList`_, `astropy.io.fits.ImageHDU`_, `astropy.io.fits.BinTableHDU`_): + The HDU(s) with the data to use for instantiation. + hdu_prefix (:obj:`str`, optional): + Maintained for consistency with the base class but is + not used by this method. + chk_version (:obj:`bool`, optional): + If True, raise an error if the datamodel version or + type check failed. If False, throw a warning only. + """ + # Run the default parser + d, version_passed, type_passed, parsed_hdus = cls._parse(hdu) + # Check + cls._check_parsed(version_passed, type_passed, chk_version=chk_version) + + # Instantiate + self = super().from_dict(d=d) + + # Check the bitmasks. Bits should have been written to *any* header + # associated with the object + hdr_bitmask = BitMask.from_header(hdu[parsed_hdus[0]].header) + if chk_version and hdr_bitmask.bits != self.bitmask.bits: + msgs.error('The bitmask in this fits file appear to be out of date! Recreate this ' + 'file by re-running the relevant script or set chk_version=False.', + cls='PypeItBitMaskError') + + return self + def copy(self): """Create a deep copy.""" _self = super().__new__(self.__class__) diff --git a/pypeit/pypmsgs.py b/pypeit/pypmsgs.py index 0755b00fd6..36caf265b4 100644 --- a/pypeit/pypmsgs.py +++ b/pypeit/pypmsgs.py @@ -31,6 +31,9 @@ class PypeItError(Exception): pass +class PypeItBitMaskError(PypeItError): + pass + class PypeItDataModelError(PypeItError): pass diff --git a/pypeit/slittrace.py b/pypeit/slittrace.py index 286cbc37e1..6836cecb31 100644 --- a/pypeit/slittrace.py +++ b/pypeit/slittrace.py @@ -348,7 +348,8 @@ def from_hdu(cls, hdu, hdu_prefix=None, chk_version=True): hdr_bitmask = BitMask.from_header(hdu[parsed_hdus[0]].header) if chk_version and hdr_bitmask.bits != self.bitmask.bits: msgs.error('The bitmask in this fits file appear to be out of date! Recreate this ' - 'file either by rerunning run_pypeit or pypeit_trace_edges.') + 'file by re-running the relevant script or set chk_version=False.', + cls='PypeItBitMaskError') return self From 5a680e6f9528578ed2e3028fbed4f1d931df9f23 Mon Sep 17 00:00:00 2001 From: Jack O'Donnell Date: Thu, 18 Jan 2024 11:42:42 -0800 Subject: [PATCH 190/244] tiny change to detector list --- pypeit/spectrographs/gemini_gmos.py | 58 ++++++++++++++--------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/pypeit/spectrographs/gemini_gmos.py b/pypeit/spectrographs/gemini_gmos.py index 1015bff45a..61e16481cc 100644 --- a/pypeit/spectrographs/gemini_gmos.py +++ b/pypeit/spectrographs/gemini_gmos.py @@ -10,7 +10,7 @@ from astropy.coordinates import SkyCoord from astropy import units from astropy.wcs import wcs -from astropy.io import fits +from astropy.io import fits from pypeit import msgs from pypeit.spectrographs import spectrograph @@ -39,7 +39,7 @@ class GeminiGMOSMosaicLookUp: .. code-block:: python from geminidr.gmos.lookups.geometry_conf import geometry - + Updating to any changes made to the DRAGONS version requires by-hand editing of the PypeIt code. """ @@ -248,7 +248,7 @@ def check_frame_type(self, ftype, fitstbl, exprng=None): def default_pypeit_par(cls): """ Return the default parameters to use for this instrument. - + Returns: :class:`~pypeit.par.pypeitpar.PypeItPar`: Parameters required by all of PypeIt methods. @@ -541,9 +541,9 @@ def list_detectors(self, mosaic=False): Detectors separated along the dispersion direction should be ordered along the first axis of the returned array. For example, Keck/DEIMOS returns: - + .. code-block:: python - + dets = np.array([['DET01', 'DET02', 'DET03', 'DET04'], ['DET05', 'DET06', 'DET07', 'DET08']]) @@ -565,13 +565,13 @@ def list_detectors(self, mosaic=False): return np.array([self.get_det_name(_det) for _det in self.allowed_mosaics]) return np.array([detector_container.DetectorContainer.get_name(i+1) - for i in range(self.ndet)]).reshape(2,-1) + for i in range(self.ndet)]).reshape(3,1) @property def default_mosaic(self): return self.allowed_mosaics[0] - + def get_slitmask(self, filename): """ Parse the slitmask data from a MOSFIRE file into :attr:`slitmask`, a @@ -596,15 +596,15 @@ def get_slitmask(self, filename): # Projected distance (in arcsec) of the object from the left and right (top and bot) edges of the slit slit_length = mask_tbl['slitsize_y'].to('arcsec').value # arcsec - topdist = np.round(slit_length/2. - + topdist = np.round(slit_length/2. - mask_tbl['slitpos_y'].to('arcsec').value, 3) - botdist = np.round(slit_length/2. + + botdist = np.round(slit_length/2. + mask_tbl['slitpos_y'].to('arcsec').value, 3) # Coordinates # WARNING -- GMOS TABLE IS ONLY IN FLOAT32!!! - obj_ra = mask_tbl['RA'].value * 15. - obj_dec = mask_tbl['DEC'].value + obj_ra = mask_tbl['RA'].value * 15. + obj_dec = mask_tbl['DEC'].value objname = mask_tbl['ID'].value.astype(str) slitID = mask_tbl['ID'].value # Slit and objects are the same @@ -620,7 +620,7 @@ def get_slitmask(self, filename): topdist, botdist]).T # Mask pointing - mask_coord = SkyCoord(mask_tbl.meta['RA_IMAG'], mask_tbl.meta['DEC_IMAG'], + mask_coord = SkyCoord(mask_tbl.meta['RA_IMAG'], mask_tbl.meta['DEC_IMAG'], unit=("hourangle", "deg")) # PA corresponding to positive x on detector (spatial) @@ -631,8 +631,8 @@ def get_slitmask(self, filename): # Slit positions obj_coord = SkyCoord(ra=obj_ra, dec=obj_dec, unit='deg') offsets = np.sqrt( - mask_tbl['slitpos_x'].to('arcsec').value**2 + - mask_tbl['slitpos_y'].to('arcsec').value**2) + mask_tbl['slitpos_x'].to('arcsec').value**2 + + mask_tbl['slitpos_y'].to('arcsec').value**2) # NOT READY FOR TILTS if np.any(np.invert(np.isclose(mask_tbl['slittilt'].value, 0.))): msgs.error('NOT READY FOR TILTED SLITS') @@ -648,7 +648,7 @@ def get_slitmask(self, filename): slit_pa*units.deg, off_sign*offset*units.arcsec) slit_ra.append(slit_coord.ra.deg) slit_dec.append(slit_coord.dec.deg) - + # Instantiate the slit mask object and return it self.slitmask = SlitMask( @@ -660,12 +660,12 @@ def get_slitmask(self, filename): np.zeros(slitID.size), np.zeros(slitID.size), np.zeros(slitID.size), - np.zeros(slitID.size)]).T.reshape(-1,4,2), + np.zeros(slitID.size)]).T.reshape(-1,4,2), slitid=np.array(slitID, dtype=int), align=mask_tbl['priority'].value == b'0', science=mask_tbl['priority'].value != b'0', onsky=np.array([ - slit_ra, slit_dec, + slit_ra, slit_dec, np.array(mask_tbl['slitsize_y'].to('arcsec').value, dtype=float), np.array(mask_tbl['slitsize_x'].to('arcsec').value, dtype=float), slit_pas]).T, @@ -684,7 +684,7 @@ def get_maskdef_slitedges(self, ccdnum=None, filename=None, debug=None, Args: binning (_type_, optional): _description_. Defaults to None. binning(str, optional): spec,spat binning of the flat field image - filename (:obj:`list`, optional): Names + filename (:obj:`list`, optional): Names the mask design info and wcs_file in that order debug (:obj:`bool`, optional): Debug ccdnum (:obj:`int`, optional): detector number @@ -710,10 +710,10 @@ def get_maskdef_slitedges(self, ccdnum=None, filename=None, debug=None, self.get_slitmask(maskfile) # Binning of flat - _, bin_spat= parse.parse_binning(binning) + _, bin_spat= parse.parse_binning(binning) # Slit center - slit_coords = SkyCoord(ra=self.slitmask.onsky[:,0], + slit_coords = SkyCoord(ra=self.slitmask.onsky[:,0], dec=self.slitmask.onsky[:,1], unit='deg') mask_coord = SkyCoord(ra=self.slitmask.mask_radec[0], dec=self.slitmask.mask_radec[1], unit='deg') @@ -721,7 +721,7 @@ def get_maskdef_slitedges(self, ccdnum=None, filename=None, debug=None, # Load up the acquisition image (usually a sciframe) hdul_acq = fits.open(wcs_file) acq_binning = self.get_meta_value(self.get_headarr(hdul_acq), 'binning') - _, bin_spat_acq = parse.parse_binning(acq_binning) + _, bin_spat_acq = parse.parse_binning(acq_binning) wcss = [wcs.WCS(hdul_acq[i].header) for i in range(1, len(hdul_acq))] left_edges = [] @@ -736,7 +736,7 @@ def get_maskdef_slitedges(self, ccdnum=None, filename=None, debug=None, right_coord = slit_coords[islit].directional_offset_by( self.slitmask.onsky[islit,4]*units.deg, self.slitmask.onsky[islit,2]*units.arcsec/2.) - + got_it = False for kk, iwcs in enumerate(wcss): pix_xy = iwcs.world_to_pixel(left_coord) @@ -753,8 +753,8 @@ def get_maskdef_slitedges(self, ccdnum=None, filename=None, debug=None, # DEBUGGING # tbl = Table() -# tbl['left'] = left_edges -# tbl['right'] = right_edges +# tbl['left'] = left_edges +# tbl['right'] = right_edges # tbl['ID'] = self.slitmask.slitid # tbl.sort('left') # embed(header='641 of gemini_gmos') @@ -866,7 +866,7 @@ def get_detector_par(self, det, hdu=None): def default_pypeit_par(cls): """ Return the default parameters to use for this instrument. - + Returns: :class:`~pypeit.par.pypeitpar.PypeItPar`: Parameters required by all of PypeIt methods. @@ -922,7 +922,7 @@ def bpm(self, filename, det, shape=None, msbias=None): hdrs = self.get_headarr(filename) binning = self.get_meta_value(hdrs, 'binning') obs_epoch = self.get_meta_value(hdrs, 'mjd') - bin_spec, bin_spat= parse.parse_binning(binning) + bin_spec, bin_spat= parse.parse_binning(binning) # Add the detector-specific, hard-coded bad columns if 1 in _det: @@ -944,7 +944,7 @@ def bpm(self, filename, det, shape=None, msbias=None): # Bad amp as of January 28, 2022 # https://gemini.edu/sciops/instruments/gmos/GMOS-S_badamp5_ops_3.pdf if obs_epoch > 2022.07: - badr = (768*2)//bin_spec + badr = (768*2)//bin_spec _bpm_img[i,badr:,:] = 1 if 3 in _det: msgs.info("Using hard-coded BPM for det=3 on GMOSs") @@ -983,7 +983,7 @@ def config_specific_par(self, scifile, inp_par=None): # The bad amp needs a larger follow_span for slit edge tracing obs_epoch = self.get_meta_value(scifile, 'mjd') binning = self.get_meta_value(scifile, 'binning') - bin_spec, bin_spat= parse.parse_binning(binning) + bin_spec, bin_spat= parse.parse_binning(binning) if obs_epoch > 2022.07: par['calibrations']['slitedges']['follow_span'] = 290*bin_spec # @@ -1349,7 +1349,7 @@ def config_specific_par(self, scifile, inp_par=None): par['calibrations']['wavelengths']['reid_arxiv'] = 'gemini_gmos_r400_e2v_mosaic.fits' # The blue wavelengths are *faint* # But redder observations may prefer something closer to the default - par['calibrations']['wavelengths']['sigdetect'] = 1. + par['calibrations']['wavelengths']['sigdetect'] = 1. # Return return par From f5dc9091b3f501ce870b111c5566c4b8a025942e Mon Sep 17 00:00:00 2001 From: Frederick Davies Date: Thu, 18 Jan 2024 21:15:50 +0100 Subject: [PATCH 191/244] revert default telgridfile --- pypeit/par/pypeitpar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index 3217e5cb29..8409519610 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -2313,7 +2313,7 @@ def __init__(self, telgridfile=None, sn_clip=None, resln_guess=None, resln_frac_ dtypes = OrderedDict.fromkeys(pars.keys()) descr = OrderedDict.fromkeys(pars.keys()) - defaults['telgridfile'] = 'TellPCA_3000_26000_R10000.fits' + defaults['telgridfile'] = None dtypes['telgridfile'] = str descr['telgridfile'] = 'File containing the telluric grid for the observatory in question. These grids are ' \ 'generated from HITRAN models for each observatory using nominal site parameters. They ' \ From b5ba28e8c1bda21c79b256fbf23f1d67f12b719b Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 18 Jan 2024 13:34:32 -0800 Subject: [PATCH 192/244] census of where to put chk_version --- pypeit/calibrations.py | 11 +++++ pypeit/coadd3d.py | 25 ++++++++--- pypeit/core/coadd.py | 53 ---------------------- pypeit/core/telluric.py | 1 + pypeit/core/wavecal/templates.py | 1 + pypeit/data/utils.py | 1 + pypeit/flatfield.py | 1 + pypeit/images/bitmaskarray.py | 16 ++++--- pypeit/onespec.py | 37 ++++++++-------- pypeit/scattlight.py | 1 + pypeit/scripts/arxiv_solution.py | 1 + pypeit/scripts/chk_alignments.py | 1 + pypeit/scripts/chk_noise_1dspec.py | 1 + pypeit/scripts/chk_scattlight.py | 11 +++-- pypeit/scripts/chk_wavecalib.py | 2 + pypeit/scripts/edge_inspector.py | 1 + pypeit/scripts/flux_calib.py | 1 + pypeit/scripts/identify.py | 3 ++ pypeit/scripts/parse_slits.py | 2 + pypeit/scripts/ql.py | 1 + pypeit/scripts/show_2dspec.py | 1 + pypeit/scripts/show_wvcalib.py | 2 + pypeit/scripts/skysub_regions.py | 1 + pypeit/sensfunc.py | 4 +- pypeit/slittrace.py | 70 +++++++++++++++--------------- pypeit/spec2dobj.py | 29 ++++++------- pypeit/specobjs.py | 4 +- pypeit/wavetilts.py | 3 ++ 28 files changed, 146 insertions(+), 139 deletions(-) diff --git a/pypeit/calibrations.py b/pypeit/calibrations.py index cdcc3318d5..83f21490a6 100644 --- a/pypeit/calibrations.py +++ b/pypeit/calibrations.py @@ -314,6 +314,7 @@ def get_arc(self): # If a processed calibration frame exists and we want to reuse it, do # so: if cal_file.exists() and self.reuse_calibs: + # TODO: Somehow pass chk_version here? self.msarc = frame['class'].from_file(cal_file) return self.msarc @@ -357,6 +358,7 @@ def get_tiltimg(self): # If a processed calibration frame exists and we want to reuse it, do # so: if cal_file.exists() and self.reuse_calibs: + # TODO: Somehow pass chk_version here? self.mstilt = frame['class'].from_file(cal_file) return self.mstilt @@ -405,6 +407,7 @@ def get_align(self): # If a processed calibration frame exists and we want to reuse it, do # so: if cal_file.exists() and self.reuse_calibs: + # TODO: Somehow pass chk_version here? self.alignments = frame['class'].from_file(cal_file) self.alignments.is_synced(self.slits) return self.alignments @@ -456,6 +459,7 @@ def get_bias(self): # If a processed calibration frame exists and we want to reuse it, do # so: if cal_file.exists() and self.reuse_calibs: + # TODO: Somehow pass chk_version here? self.msbias = frame['class'].from_file(cal_file) return self.msbias @@ -495,6 +499,7 @@ def get_dark(self): # If a processed calibration frame exists and we want to reuse it, do # so: if cal_file.exists() and self.reuse_calibs: + # TODO: Somehow pass chk_version here? self.msdark = frame['class'].from_file(cal_file) return self.msdark @@ -571,6 +576,7 @@ def get_scattlight(self): # If a processed calibration frame exists and we want to reuse it, do # so: if cal_file.exists() and self.reuse_calibs: + # TODO: Somehow pass chk_version here? self.msscattlight = frame['class'].from_file(cal_file) return self.msscattlight @@ -689,6 +695,7 @@ def get_flats(self): setup = illum_setup if pixel_setup is None else pixel_setup calib_id = illum_calib_id if pixel_calib_id is None else pixel_calib_id if cal_file.exists() and self.reuse_calibs: + # TODO: Somehow pass chk_version here? self.flatimages = flatfield.FlatImages.from_file(cal_file) self.flatimages.is_synced(self.slits) # Load user defined files @@ -845,6 +852,7 @@ def get_slits(self): # If a processed calibration frame exists and we want to reuse it, do # so: if cal_file.exists() and self.reuse_calibs: + # TODO: Somehow pass chk_version here? self.slits = frame['class'].from_file(cal_file) self.slits.mask = self.slits.mask_init.copy() if self.user_slits is not None: @@ -858,6 +866,7 @@ def get_slits(self): # If so, reuse it? if edges_file.exists() and self.reuse_calibs: # Yep! Load it and parse it into slits. + # TODO: Somehow pass chk_version here? self.slits = edgetrace.EdgeTraceSet.from_file(edges_file).get_slits() # Write the slits calibration file self.slits.to_file() @@ -961,6 +970,7 @@ def get_wv_calib(self): # we want to reuse it, do so (or just load it): if cal_file.exists() and self.reuse_calibs: # Load the file + # TODO: Somehow pass chk_version here? self.wv_calib = wavecalib.WaveCalib.from_file(cal_file) self.wv_calib.chk_synced(self.slits) self.slits.mask_wvcalib(self.wv_calib) @@ -1033,6 +1043,7 @@ def get_tilts(self): # If a processed calibration frame exists and we want to reuse it, do # so: if cal_file.exists() and self.reuse_calibs: + # TODO: Somehow pass chk_version here? self.wavetilts = wavetilts.WaveTilts.from_file(cal_file) self.wavetilts.is_synced(self.slits) self.slits.mask_wavetilts(self.wavetilts) diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index 2710883039..070549afb1 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -168,19 +168,27 @@ def to_file(self, ofile, primary_hdr=None, hdr=None, **kwargs): # Do it super(DataCube, self).to_file(ofile, primary_hdr=primary_hdr, hdr=hdr, **kwargs) - # TODO: Pass in chk_version here? @classmethod - def from_file(cls, ifile): + def from_file(cls, ifile, verbose=True, chk_version=True, **kwargs): """ + Instantiate the object from an extension in the specified fits file. + Over-load :func:`~pypeit.datamodel.DataContainer.from_file` to deal with the header - + Args: - ifile (str): Filename holding the object + ifile (:obj:`str`, `Path`_): + Fits file with the data to read + verbose (:obj:`bool`, optional): + Print informational messages (not currently used) + chk_version (:obj:`bool`, optional): + Passed to :func:`from_hdu`. + kwargs (:obj:`dict`, optional): + Arguments passed directly to :func:`from_hdu`. """ with io.fits_open(ifile) as hdu: # Read using the base class - self = super().from_hdu(hdu) + self = cls.from_hdu(hdu, chk_version=chk_version, **kwargs) # Internals self.filename = ifile self.head0 = hdu[1].header # Actually use the first extension here, since it contains the WCS @@ -579,6 +587,7 @@ def set_default_scalecorr(self): msgs.info("Loading default scale image for relative spectral illumination correction:" + msgs.newline() + self.cubepar['scale_corr']) try: + # TODO: Somehow pass chk_version here? spec2DObj = spec2dobj.Spec2DObj.from_file(self.cubepar['scale_corr'], self.detname) except Exception as e: msgs.warn(f'Loading spec2d file raised {type(e).__name__}:\n{str(e)}') @@ -640,6 +649,7 @@ def get_current_scalecorr(self, spec2DObj, scalecorr=None): msgs.info("Loading the following frame for the relative spectral illumination correction:" + msgs.newline() + scalecorr) try: + # TODO: Somehow pass chk_version here? spec2DObj_scl = spec2dobj.Spec2DObj.from_file(scalecorr, self.detname) except Exception as e: msgs.warn(f'Loading spec2d file raised {type(e).__name__}:\n{str(e)}') @@ -671,6 +681,7 @@ def set_default_skysub(self): msgs.info("Loading default image for sky subtraction:" + msgs.newline() + self.cubepar['skysub_frame']) try: + # TODO: Somehow pass chk_version here? spec2DObj = spec2dobj.Spec2DObj.from_file(self.cubepar['skysub_frame'], self.detname) skysub_exptime = self.spec.get_meta_value([spec2DObj.head0], 'exptime') except: @@ -741,6 +752,7 @@ def get_current_skysub(self, spec2DObj, exptime, opts_skysub=None): # Load a user specified frame for sky subtraction msgs.info("Loading skysub frame:" + msgs.newline() + opts_skysub) try: + # TODO: Somehow pass chk_version here? spec2DObj_sky = spec2dobj.Spec2DObj.from_file(opts_skysub, self.detname) skysub_exptime = self.spec.get_meta_value([spec2DObj_sky.head0], 'exptime') except: @@ -779,6 +791,7 @@ def add_grating_corr(self, flatfile, waveimg, slits, spat_flexure=None): if not os.path.exists(flatfile): msgs.error("Grating correction requested, but the following file does not exist:" + msgs.newline() + flatfile) # Load the Flat file + # TODO: Somehow pass chk_version here? flatimages = flatfield.FlatImages.from_file(flatfile) total_illum = flatimages.fit2illumflat(slits, finecorr=False, frametype='illum', initial=True, spat_flexure=spat_flexure) * \ flatimages.fit2illumflat(slits, finecorr=True, frametype='illum', initial=True, spat_flexure=spat_flexure) @@ -882,6 +895,7 @@ def get_alignments(self, spec2DObj, slits, spat_flexure=None): alignfile = os.path.join(spec2DObj.calibs['DIR'], spec2DObj.calibs[key]) if os.path.exists(alignfile) and self.cubepar['astrometric']: msgs.info("Loading alignments") + # TODO: Somehow pass chk_version here? alignments = alignframe.Alignments.from_file(alignfile) else: msgs.warn(f'Processed alignment frame not recorded or not found!') @@ -941,6 +955,7 @@ def load(self): for ff, fil in enumerate(self.spec2d): # Load it up msgs.info("Loading PypeIt spec2d frame:" + msgs.newline() + fil) + # TODO: Somehow pass chk_version here? spec2DObj = spec2dobj.Spec2DObj.from_file(fil, self.detname) detector = spec2DObj.detector spat_flexure = None # spec2DObj.sci_spat_flexure diff --git a/pypeit/core/coadd.py b/pypeit/core/coadd.py index 593bdeed75..5687cde32f 100644 --- a/pypeit/core/coadd.py +++ b/pypeit/core/coadd.py @@ -924,59 +924,6 @@ def sn_weights(fluxes, ivars, gpms, sn_smooth_npix=None, weight_method='auto', v return np.array(rms_sn), weights -# TODO: This was commented out and would need to be refactored if brought back -# because of changes to the SensFunc and Telluric datamodels. -## TODO Rename this function to something sensfunc related -#def get_tell_from_file(sensfile, waves, masks, iord=None): -# ''' -# Get the telluric model from the sensfile. -# -# Args: -# sensfile (str): the name of your fits format sensfile -# waves (ndarray): wavelength grid for your output telluric model -# masks (ndarray, bool): mask for the wave -# iord (int or None): if None returns telluric model for all orders, otherwise return the order you want -# -# Returns: -# ndarray: telluric model on your wavelength grid -# ''' -# -# -# sens_param = Table.read(sensfile, 1) -# sens_table = Table.read(sensfile, 2) -# telluric = np.zeros_like(waves) -# -# if (waves.ndim == 1) and (iord is None): -# msgs.info('Loading Telluric from Longslit sensfiles.') -# tell_interp = scipy.interpolate.interp1d(sens_table[0]['WAVE'], sens_table[0]['TELLURIC'], kind='cubic', -# bounds_error=False, fill_value=np.nan)(waves[masks]) -# telluric[masks] = tell_interp -# elif (waves.ndim == 1) and (iord is not None): -# msgs.info('Loading order {:} Telluric from Echelle sensfiles.'.format(iord)) -# wave_tell_iord = sens_table[iord]['WAVE'] -# tell_mask = (wave_tell_iord > 1.0) -# tell_iord = sens_table[iord]['TELLURIC'] -# tell_iord_interp = scipy.interpolate.interp1d(wave_tell_iord[tell_mask], tell_iord[tell_mask], kind='cubic', -# bounds_error=False, fill_value=np.nan)(waves[masks]) -# telluric[masks] = tell_iord_interp -# else: -# norder = np.shape(waves)[1] -# for iord in range(norder): -# wave_iord = waves[:, iord] -# mask_iord = masks[:, iord] -# -# # Interpolate telluric to the same grid with waves -# # Since it will be only used for plotting, I just simply interpolate it rather than evaluate it based on the model -# wave_tell_iord = sens_table[iord]['WAVE'] -# tell_mask = (wave_tell_iord > 1.0) -# tell_iord = sens_table[iord]['TELLURIC'] -# tell_iord_interp = scipy.interpolate.interp1d(wave_tell_iord[tell_mask], tell_iord[tell_mask], kind='cubic', -# bounds_error=False, fill_value=np.nan)(wave_iord[mask_iord]) -# telluric[mask_iord, iord] = tell_iord_interp -# -# return telluric - - def robust_median_ratio(flux, ivar, flux_ref, ivar_ref, mask=None, mask_ref=None, ref_percentile=70.0, min_good=0.05, maxiters=5, sigrej=3.0, max_factor=10.0, snr_do_not_rescale=1.0, verbose=False): diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index feedda488a..30908d8495 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -742,6 +742,7 @@ def general_spec_reader(specfile, ret_flam=False): hdul = fits.open(specfile) if 'DMODCLS' in hdul[1].header and hdul[1].header['DMODCLS'] == 'OneSpec': # Load + # TODO: Somehow pass chk_version here? spec = onespec.OneSpec.from_file(specfile) # Unpack wave = spec.wave diff --git a/pypeit/core/wavecal/templates.py b/pypeit/core/wavecal/templates.py index 893bc9489d..5c612e280a 100644 --- a/pypeit/core/wavecal/templates.py +++ b/pypeit/core/wavecal/templates.py @@ -266,6 +266,7 @@ def pypeit_arcspec(in_file, slit, binspec, binning=None): # minx=iwv_calib['fmin'], maxx=iwv_calib['fmax']) flux = np.array(iwv_calib['spec']).flatten() elif '.fits' in in_file: + # TODO: Don't default to chk_version=False ... wvcalib = wavecalib.WaveCalib.from_file(in_file, chk_version=False) idx = np.where(wvcalib.spat_ids == slit)[0][0] flux = wvcalib.arc_spectra[:,idx] diff --git a/pypeit/data/utils.py b/pypeit/data/utils.py index cba62b0153..2434625fdd 100644 --- a/pypeit/data/utils.py +++ b/pypeit/data/utils.py @@ -790,3 +790,4 @@ def load_sky_spectrum(sky_file: str) -> xspectrum1d.XSpectrum1D: (`linetools.spectra.xspectrum1d.XSpectrum1D`_): Sky spectrum """ return xspectrum1d.XSpectrum1D.from_file(str(Paths.sky_spec / sky_file)) + diff --git a/pypeit/flatfield.py b/pypeit/flatfield.py index ea13b2c3c3..cda9afbb49 100644 --- a/pypeit/flatfield.py +++ b/pypeit/flatfield.py @@ -414,6 +414,7 @@ def show(self, frametype='all', slits=None, wcs_match=True): slits_file = slittrace.SlitTraceSet.construct_file_name(self.calib_key, calib_dir=self.calib_dir) try: + # TODO: Somehow pass chk_version here? slits = slittrace.SlitTraceSet.from_file(slits_file) except (FileNotFoundError, PypeItError): msgs.warn('Could not load slits to include when showing flat-field images. File ' diff --git a/pypeit/images/bitmaskarray.py b/pypeit/images/bitmaskarray.py index ca728f9a6f..51e91bb823 100644 --- a/pypeit/images/bitmaskarray.py +++ b/pypeit/images/bitmaskarray.py @@ -13,6 +13,8 @@ import numpy as np +from astropy.io import fits + from pypeit.datamodel import DataContainer from pypeit.bitmask import BitMask from pypeit import msgs @@ -151,7 +153,7 @@ def _bundle(self): return d @classmethod - def from_hdu(cls, hdu, hdu_prefix=None, chk_version=True): + def from_hdu(cls, hdu, chk_version=True, **kwargs): """ Instantiate the object from an HDU extension. @@ -161,24 +163,24 @@ def from_hdu(cls, hdu, hdu_prefix=None, chk_version=True): Args: hdu (`astropy.io.fits.HDUList`_, `astropy.io.fits.ImageHDU`_, `astropy.io.fits.BinTableHDU`_): The HDU(s) with the data to use for instantiation. - hdu_prefix (:obj:`str`, optional): - Maintained for consistency with the base class but is - not used by this method. chk_version (:obj:`bool`, optional): If True, raise an error if the datamodel version or type check failed. If False, throw a warning only. + **kwargs: + Passed directly to :func:`_parse`. """ # Run the default parser - d, version_passed, type_passed, parsed_hdus = cls._parse(hdu) + d, version_passed, type_passed, parsed_hdus = cls._parse(hdu, **kwargs) # Check cls._check_parsed(version_passed, type_passed, chk_version=chk_version) # Instantiate self = super().from_dict(d=d) - + # Check the bitmasks. Bits should have been written to *any* header # associated with the object - hdr_bitmask = BitMask.from_header(hdu[parsed_hdus[0]].header) + hdr = hdu[parsed_hdus[0]].header if isinstance(hdu, fits.HDUList) else hdu.header + hdr_bitmask = BitMask.from_header(hdr) if chk_version and hdr_bitmask.bits != self.bitmask.bits: msgs.error('The bitmask in this fits file appear to be out of date! Recreate this ' 'file by re-running the relevant script or set chk_version=False.', diff --git a/pypeit/onespec.py b/pypeit/onespec.py index 89f961724b..6eb9ef83e0 100644 --- a/pypeit/onespec.py +++ b/pypeit/onespec.py @@ -84,28 +84,31 @@ class OneSpec(datamodel.DataContainer): 'spect_meta', 'history'] - # TODO: Add chk_version to this? @classmethod - def from_file(cls, ifile): + def from_file(cls, ifile, verbose=True, chk_version=True, **kwargs): """ - Over-load :func:`pypeit.datamodel.DataContainer.from_file` - to deal with the header + Instantiate the object from an extension in the specified fits file. + Over-load :func:`~pypeit.datamodel.DataContainer.from_file` + to deal with the header + Args: - ifile (str): - Filename holding the object + ifile (:obj:`str`, `Path`_): + Fits file with the data to read + verbose (:obj:`bool`, optional): + Print informational messages (not currently used) + chk_version (:obj:`bool`, optional): + Passed to :func:`from_hdu`. + kwargs (:obj:`dict`, optional): + Arguments passed directly to :func:`from_hdu`. """ - hdul = io.fits_open(ifile) - slf = super().from_hdu(hdul) - - # Internals - slf.filename = ifile - slf.head0 = hdul[0].header - # Meta - slf.spectrograph = load_spectrograph(slf.PYP_SPEC) - slf.spect_meta = slf.spectrograph.parse_spec_header(slf.head0) - # - return slf + with io.fits_open(ifile) as hdu: + self = cls.from_hdu(hdu, chk_version=chk_version, **kwargs) + self.filename = ifile + self.head0 = hdu[0].header + self.spectrograph = load_spectrograph(self.PYP_SPEC) + self.spect_meta = self.spectrograph.parse_spec_header(self.head0) + return self def __init__(self, wave, wave_grid_mid, flux, PYP_SPEC=None, ivar=None, sigma=None, mask=None, telluric=None, obj_model=None, ext_mode=None, fluxed=None): diff --git a/pypeit/scattlight.py b/pypeit/scattlight.py index 6ed95d82b4..34c6fbe751 100644 --- a/pypeit/scattlight.py +++ b/pypeit/scattlight.py @@ -141,3 +141,4 @@ def show(self, image=None, slits=None, mask=False, wcs_match=True): # Display frames display.show_scattered_light(image_list, slits=slits, wcs_match=wcs_match) + diff --git a/pypeit/scripts/arxiv_solution.py b/pypeit/scripts/arxiv_solution.py index a608d5e2dd..797b92cc6a 100644 --- a/pypeit/scripts/arxiv_solution.py +++ b/pypeit/scripts/arxiv_solution.py @@ -43,6 +43,7 @@ def main(args): msgs.error("The following MasterWaveCalib file does not exist:" + msgs.newline() + args.file) # Load the wavelength calibration file + # TODO: Pass chk_version here? wv_calib = WaveCalib.from_file(args.file) # Check if a wavelength solution exists if wv_calib['wv_fits'][args.slit]['wave_soln'] is None: diff --git a/pypeit/scripts/chk_alignments.py b/pypeit/scripts/chk_alignments.py index a484d494e7..43d3738761 100644 --- a/pypeit/scripts/chk_alignments.py +++ b/pypeit/scripts/chk_alignments.py @@ -28,6 +28,7 @@ def main(pargs): from pypeit import alignframe # Load + # TODO: Pass chk_version here? alignments = alignframe.Alignments.from_file(pargs.file) # Show alignments.show() diff --git a/pypeit/scripts/chk_noise_1dspec.py b/pypeit/scripts/chk_noise_1dspec.py index 33123e5533..db338f80a9 100644 --- a/pypeit/scripts/chk_noise_1dspec.py +++ b/pypeit/scripts/chk_noise_1dspec.py @@ -198,6 +198,7 @@ def main(args): head = fits.getheader(file) # I/O spec object + # TODO: Pass chk_version to OneSpec too? specObjs = [OneSpec.from_file(file)] if args.fileformat == 'coadd1d' else \ specobjs.SpecObjs.from_fitsfile(file, chk_version=False) diff --git a/pypeit/scripts/chk_scattlight.py b/pypeit/scripts/chk_scattlight.py index f7446f17ae..f067d6d5b1 100644 --- a/pypeit/scripts/chk_scattlight.py +++ b/pypeit/scripts/chk_scattlight.py @@ -6,10 +6,6 @@ """ from pypeit.scripts import scriptbase -from pypeit import msgs -from pypeit.pypmsgs import PypeItError, PypeItDataModelError -from pypeit.images.detector_container import DetectorContainer -from pypeit import io class ChkScattLight(scriptbase.ScriptBase): @@ -37,6 +33,10 @@ def get_parser(cls, width=None): def main(args): from pypeit import scattlight, spec2dobj, slittrace + from pypeit import msgs + from pypeit.pypmsgs import PypeItError, PypeItDataModelError + from pypeit.images.detector_container import DetectorContainer + from pypeit import io # Parse the detector name try: @@ -47,9 +47,11 @@ def main(args): detname = DetectorContainer.get_name(det) # Load scattered light calibration frame + # TODO: Pass chk_version here? ScattLightImage = scattlight.ScatteredLight.from_file(args.file) # Load slits information + # TODO: Pass chk_version here? slits = slittrace.SlitTraceSet.from_file(args.slits) # Load the alternate file if requested @@ -58,6 +60,7 @@ def main(args): msgs.error("displaying the spec2d scattered light is not currently supported") try: # TODO :: the spec2d file may have already had the scattered light removed, so this is not correct. This script only works when the scattered light is turned off for the spec2d file + # TODO: Do not default to chk_version=False spec2D = spec2dobj.Spec2DObj.from_file(args.spec2d, detname, chk_version=False) except PypeItDataModelError: msgs.warn(f"Error loading spec2d file {args.spec2d} - attempting to load science image from fits") diff --git a/pypeit/scripts/chk_wavecalib.py b/pypeit/scripts/chk_wavecalib.py index f06c94b992..ac547346c0 100644 --- a/pypeit/scripts/chk_wavecalib.py +++ b/pypeit/scripts/chk_wavecalib.py @@ -44,11 +44,13 @@ def main(args): msgs.error("Bad file type input!") if file_type == 'WaveCalib': + # TODO: Do not always set chk_version to False waveCalib = wavecalib.WaveCalib.from_file(in_file, chk_version=False) waveCalib.wave_diagnostics(print_diag=True) continue elif file_type == 'AllSpec2D': + # TODO: Do not always set chk_version to False allspec2D = spec2dobj.AllSpec2DObj.from_fits(in_file, chk_version=False) for det in allspec2D.detectors: print('') diff --git a/pypeit/scripts/edge_inspector.py b/pypeit/scripts/edge_inspector.py index ac942e640b..b0aeb38905 100644 --- a/pypeit/scripts/edge_inspector.py +++ b/pypeit/scripts/edge_inspector.py @@ -30,6 +30,7 @@ def main(args): # Set the file name to the full path trace_file = Path(args.trace_file).resolve() # Load + # TODO: Pass chk_version here? edges = edgetrace.EdgeTraceSet.from_file(trace_file) # Inspector object pointer = edge_inspector.EdgeInspectorGUI(edges) diff --git a/pypeit/scripts/flux_calib.py b/pypeit/scripts/flux_calib.py index 2405fd0cdc..f4f12ed269 100644 --- a/pypeit/scripts/flux_calib.py +++ b/pypeit/scripts/flux_calib.py @@ -77,6 +77,7 @@ def main(args): msgs.set_logfile_and_verbosity('flux_calib', args.verbosity) # Load the file + # TODO: Pass chk_version here? fluxFile = inputfiles.FluxFile.from_file(args.flux_file) # Read in spectrograph from spec1dfile header diff --git a/pypeit/scripts/identify.py b/pypeit/scripts/identify.py index 8102b4f32b..ab78cbbe48 100644 --- a/pypeit/scripts/identify.py +++ b/pypeit/scripts/identify.py @@ -62,6 +62,7 @@ def main(args): msgs.set_logfile_and_verbosity('identify', args.verbosity) # Load the Arc file + # TODO: Pass chk_version here? msarc = ArcImage.from_file(args.arc_file) # Load the spectrograph @@ -78,12 +79,14 @@ def main(args): par['lamps'] = lamps # Load the slits + # TODO: Pass chk_version here? slits = slittrace.SlitTraceSet.from_file(args.slits_file) # Reset the mask slits.mask = slits.mask_init # Check if a solution exists solnname = WaveCalib.construct_file_name(msarc.calib_key, calib_dir=msarc.calib_dir) + # TODO: Pass chk_version here? wv_calib = WaveCalib.from_file(solnname) \ if os.path.exists(solnname) and args.solution else None diff --git a/pypeit/scripts/parse_slits.py b/pypeit/scripts/parse_slits.py index e11d1785f2..7d7f4caf9e 100644 --- a/pypeit/scripts/parse_slits.py +++ b/pypeit/scripts/parse_slits.py @@ -79,11 +79,13 @@ def main(pargs): msgs.error("Bad file type input!") if file_type == 'Slits': + # TODO: Do not always set chk_version to False slits = slittrace.SlitTraceSet.from_file(pargs.input_file, chk_version=False) print('') print_slits(slits) elif file_type == 'AllSpec2D': + # TODO: Do not always set chk_version to False allspec2D = spec2dobj.AllSpec2DObj.from_fits(pargs.input_file, chk_version=False) # Loop on Detectors for det in allspec2D.detectors: diff --git a/pypeit/scripts/ql.py b/pypeit/scripts/ql.py index f1a2224d3b..b9a9742c81 100644 --- a/pypeit/scripts/ql.py +++ b/pypeit/scripts/ql.py @@ -350,6 +350,7 @@ def generate_sci_pypeitfile(redux_path:str, # Iterate through each file to find the one with the relevant mask ID. detname = None for sliittrace_file in slittrace_files: + # TODO: Somehow pass chk_version here? slitTrace = SlitTraceSet.from_file(sliittrace_file) if maskID in slitTrace.maskdef_id: detname = slitTrace.detname diff --git a/pypeit/scripts/show_2dspec.py b/pypeit/scripts/show_2dspec.py index 5a9c13029b..2bdd0b4c11 100644 --- a/pypeit/scripts/show_2dspec.py +++ b/pypeit/scripts/show_2dspec.py @@ -137,6 +137,7 @@ def main(args): # Try to read the Spec2DObj using the current datamodel, but allowing # for the datamodel version to be different try: + # TODO: Do not always set chk_version to False spec2DObj = spec2dobj.Spec2DObj.from_file(args.file, detname, chk_version=False) except PypeItDataModelError: try: diff --git a/pypeit/scripts/show_wvcalib.py b/pypeit/scripts/show_wvcalib.py index 78d484b1ac..832af3e72d 100644 --- a/pypeit/scripts/show_wvcalib.py +++ b/pypeit/scripts/show_wvcalib.py @@ -36,8 +36,10 @@ def main(pargs, unit_test=False): from matplotlib import pyplot as plt # Load + # TODO: Pass chk_version here? wvcalib = wavecalib.WaveCalib.from_file(pargs.file) if pargs.slit_file is not None: + # TODO: Pass chk_version here? slits = slittrace.SlitTraceSet.from_file(pargs.slit_file) # Parse diff --git a/pypeit/scripts/skysub_regions.py b/pypeit/scripts/skysub_regions.py index f1347f40c8..7366b2f62c 100644 --- a/pypeit/scripts/skysub_regions.py +++ b/pypeit/scripts/skysub_regions.py @@ -55,6 +55,7 @@ def main(args): detname = DetectorContainer.get_name(det) # Load it up + # TODO: Pass chk_version here? Do not always set it to True spec2DObj = spec2dobj.Spec2DObj.from_file(args.file, detname, chk_version=True) frame = spec2DObj.sciimg hdr = fits.open(args.file)[0].header diff --git a/pypeit/sensfunc.py b/pypeit/sensfunc.py index a6637046a2..85fc11dcfc 100644 --- a/pypeit/sensfunc.py +++ b/pypeit/sensfunc.py @@ -293,8 +293,7 @@ def compute_blaze(self, wave, trace_spec, trace_spat, flatfile, box_radius=10.0, `numpy.ndarray`_: The log10 blaze function. Shape = (nspec, norddet) if norddet > 1, else shape = (nspec,) """ - - + # TODO: Somehow pass chk_version here? flatImages = flatfield.FlatImages.from_file(flatfile) pixelflat_raw = flatImages.pixelflat_raw @@ -805,6 +804,7 @@ def sensfunc_weights(cls, sensfile, waves, debug=False, extrap_sens=True): `numpy.ndarray`_: sensfunc weights evaluated on the input waves wavelength grid, shape = same as waves """ + # TODO: Somehow pass chk_version here? sens = cls.from_file(sensfile) # wave, zeropoint, meta_table, out_table, header_sens = sensfunc.SensFunc.load(sensfile) diff --git a/pypeit/slittrace.py b/pypeit/slittrace.py index 6836cecb31..4c49aee0dc 100644 --- a/pypeit/slittrace.py +++ b/pypeit/slittrace.py @@ -12,10 +12,10 @@ import numpy as np from astropy.table import Table -from astropy.coordinates import SkyCoord, Angle +from astropy.coordinates import SkyCoord from astropy import units from astropy.stats import sigma_clipped_stats -from scipy.interpolate import RegularGridInterpolator, interp1d +from astropy.io import fits from pypeit import msgs from pypeit import datamodel @@ -115,13 +115,18 @@ class SlitTraceSet(calibframe.CalibFrame): descr='Slit ID number from SPAT measured at half way point.'), 'maskdef_id': dict(otype=np.ndarray, atype=(int,np.integer), descr='Slit ID number slitmask'), - 'maskdef_designtab': dict(otype=Table, descr='Table with slitmask design and object info'), + 'maskdef_designtab': dict(otype=Table, + descr='Table with slitmask design and object info'), 'maskfile': dict(otype=str, descr='Data file that yielded the slitmask info'), - 'maskdef_posx_pa': dict(otype=float, descr='PA that aligns with spatial dimension of the detector'), - 'maskdef_offset': dict(otype=float, descr='Slitmask offset (pixels) from position expected ' - 'by the slitmask design'), + 'maskdef_posx_pa': dict(otype=float, + descr='PA that aligns with spatial dimension of the ' + 'detector'), + 'maskdef_offset': dict(otype=float, + descr='Slitmask offset (pixels) from position expected ' + 'by the slitmask design'), 'maskdef_objpos': dict(otype=np.ndarray, atype=np.floating, - descr='Object positions expected by the slitmask design [relative pixels]'), + descr='Object positions expected by the slitmask design ' + '[relative pixels]'), 'maskdef_slitcen': dict(otype=np.ndarray, atype=np.floating, descr='Slit centers expected by the slitmask design'), 'ech_order': dict(otype=np.ndarray, atype=(int,np.integer), @@ -153,11 +158,11 @@ class SlitTraceSet(calibframe.CalibFrame): descr='Bit mask for slits (fully good slits have 0 value). Shape ' 'is Nslits.'), 'specmin': dict(otype=np.ndarray, atype=np.floating, - descr='Minimum spectral position (pixel units) allowed for each slit/order. ' - 'Shape is Nslits.'), + descr='Minimum spectral position (pixel units) allowed for each ' + 'slit/order. Shape is Nslits.'), 'specmax': dict(otype=np.ndarray, atype=np.floating, - descr='Maximum spectral position (pixel units) allowed for each slit/order. ' - 'Shape is Nslits.')} + descr='Maximum spectral position (pixel units) allowed for each ' + 'slit/order. Shape is Nslits.')} """Provides the class data model.""" # TODO: Allow tweaked edges to be arguments? @@ -240,17 +245,6 @@ def _validate(self): self.mask_init = np.atleast_1d(self.mask_init) self.specmin = np.atleast_1d(self.specmin) self.specmax = np.atleast_1d(self.specmax) -# if self.slitbitm is None: -# self.slitbitm = ','.join(list(self.bitmask.keys())) -# else: -# # Validate -- All of the keys must be present and in current order, but new ones can exist -# bitms = self.slitbitm.split(',') -# curbitm = list(self.bitmask.keys()) -# for kk, bit in enumerate(bitms): -# if curbitm[kk] != bit: -# msgs.error("Input BITMASK keys differ from current data model!") -# # Update to current, no matter what -# self.slitbitm = ','.join(list(self.bitmask.keys())) # Mask if self.mask is None: self.mask = self.mask_init.copy() @@ -345,7 +339,8 @@ def from_hdu(cls, hdu, hdu_prefix=None, chk_version=True): # Check the bitmasks. Bits should have been written to *any* header # associated with the object - hdr_bitmask = BitMask.from_header(hdu[parsed_hdus[0]].header) + hdr = hdu[parsed_hdus[0]].header if isinstance(hdu, fits.HDUList) else hdu.header + hdr_bitmask = BitMask.from_header(hdr) if chk_version and hdr_bitmask.bits != self.bitmask.bits: msgs.error('The bitmask in this fits file appear to be out of date! Recreate this ' 'file by re-running the relevant script or set chk_version=False.', @@ -400,7 +395,8 @@ def slitord_id(self): @property def slitord_txt(self): """ - Return string indicating if the logs/QA should use "slit" (MultiSlit, SlicerIFU) or "order" (Echelle) + Return string indicating if the logs/QA should use "slit" (MultiSlit, + SlicerIFU) or "order" (Echelle). Returns: str: Either 'slit' or 'order' @@ -510,14 +506,16 @@ def get_radec_image(self, wcs, alignSplines, tilts, initial=True, flexure=None): onslit = (slitid_img_init == spatid) onslit_init = np.where(onslit) if self.mask[slit_idx] != 0: - msgs.error(f"Slit {spatid} ({slit_idx+1}/{self.spat_id.size}) is masked. Cannot generate RA/DEC image.") + msgs.error(f'Slit {spatid} ({slit_idx+1}/{self.spat_id.size}) is masked. Cannot ' + 'generate RA/DEC image.') # Retrieve the pixel offset from the central trace evalpos = alignSplines.transform(slit_idx, onslit_init[1], onslit_init[0]) minmax[slit_idx, 0] = np.min(evalpos) minmax[slit_idx, 1] = np.max(evalpos) # Calculate the WCS from the pixel positions slitID = np.ones(evalpos.size) * slit_idx - wcs.wcs.crpix[0] - world_ra, world_dec, _ = wcs.wcs_pix2world(slitID, evalpos, tilts[onslit_init]*(self.nspec-1), 0) + world_ra, world_dec, _ \ + = wcs.wcs_pix2world(slitID, evalpos, tilts[onslit_init]*(self.nspec-1), 0) # Set the RA first and DEC next raimg[onslit] = world_ra.copy() decimg[onslit] = world_dec.copy() @@ -801,7 +799,8 @@ def slit_spat_pos(left, right, nspat): def mask_add_missing_obj(self, sobjs, spat_flexure, fwhm, boxcar_rad): """ - Generate new SpecObj and add them into the SpecObjs object for any slits missing the targeted source. + Generate new SpecObj and add them into the SpecObjs object for any slits + missing the targeted source. Args: sobjs (:class:`pypeit.specobjs.SpecObjs`): @@ -814,16 +813,19 @@ def mask_add_missing_obj(self, sobjs, spat_flexure, fwhm, boxcar_rad): BOX_RADIUS in pixels to be used in the boxcar extraction Returns: - :class:`pypeit.specobjs.SpecObjs`: Updated list of SpecObj that have been found and traced + :class:`pypeit.specobjs.SpecObjs`: Updated list of SpecObj that have + been found and traced """ msgs.info('Add undetected objects at the expected location from slitmask design.') if fwhm is None: - msgs.error('A FWHM for the optimal extraction must be provided. See `find_fwhm` in `FindObjPar`.') + msgs.error('A FWHM for the optimal extraction must be provided. See `find_fwhm` in ' + '`FindObjPar`.') if self.maskdef_objpos is None: - msgs.error('An array with the object positions expected from slitmask design is missing.') + msgs.error('An array with the object positions expected from slitmask design is ' + 'missing.') if self.maskdef_offset is None: msgs.error('A value for the slitmask offset must be provided.') @@ -835,10 +837,10 @@ def mask_add_missing_obj(self, sobjs, spat_flexure, fwhm, boxcar_rad): else: cut_sobjs = sobjs - # get slits edges init - left_init, _, _ = self.select_edges(initial=True, flexure=spat_flexure) # includes flexure - # get slits edges tweaked - left_tweak, right_tweak, _ = self.select_edges(initial=False, flexure=spat_flexure) # includes flexure + # get slits edges init; includes flexure + left_init, _, _ = self.select_edges(initial=True, flexure=spat_flexure) + # get slits edges tweaked; includes flexure + left_tweak, right_tweak, _ = self.select_edges(initial=False, flexure=spat_flexure) # midpoint in the spectral direction specmid = left_init[:,0].size//2 diff --git a/pypeit/spec2dobj.py b/pypeit/spec2dobj.py index fd6906288d..8cf3709939 100644 --- a/pypeit/spec2dobj.py +++ b/pypeit/spec2dobj.py @@ -109,31 +109,30 @@ class Spec2DObj(datamodel.DataContainer): 'head0' # Raw header ] - # TODO: Allow for **kwargs here? @classmethod - def from_file(cls, file, detname, chk_version=True): + def from_file(cls, ifile, verbose=True, chk_version=True, **kwargs): """ - Override base-class :func:`~pypeit.datamodel.DataContainer.from_file` to - specify detector to read. + Instantiate the object from an extension in the specified fits file. + Over-load :func:`~pypeit.datamodel.DataContainer.from_file` + to specify detector to read. + Args: - file (:obj:`str`): - File name to read. - detname (:obj:`str`): - The string identifier for the detector or mosaic used to select - the data that is read. + ifile (:obj:`str`, `Path`_): + Fits file with the data to read + verbose (:obj:`bool`, optional): + Print informational messages (not currently used) chk_version (:obj:`bool`, optional): - If False, allow a mismatch in datamodel to proceed - - Returns: - :class:`~pypeit.spec2dobj.Spec2DObj`: 2D spectra object. + Passed to :func:`from_hdu`. + kwargs (:obj:`dict`, optional): + Arguments passed directly to :func:`from_hdu`. """ - with io.fits_open(file) as hdu: + with io.fits_open(ifile) as hdu: # Check detname is valid detnames = np.unique([h.name.split('-')[0] for h in hdu[1:]]) if detname not in detnames: msgs.error(f'Your --det={detname} is not available. \n Choose from: {detnames}') - return cls.from_hdu(hdu, detname, chk_version=chk_version) + return cls.from_hdu(hdu, detname, chk_version=chk_version, **kwargs) # TODO: Allow for **kwargs here? @classmethod diff --git a/pypeit/specobjs.py b/pypeit/specobjs.py index e45c155203..3031b08657 100644 --- a/pypeit/specobjs.py +++ b/pypeit/specobjs.py @@ -24,10 +24,10 @@ from pypeit.spectrographs.util import load_spectrograph from pypeit.core import parse from pypeit.images.detector_container import DetectorContainer -from pypeit.images.mosaic import Mosaic -from pypeit import slittrace from pypeit import utils + +# TODO: Make this a DataContainer class SpecObjs: """ Object to hold a set of :class:`~pypeit.specobj.SpecObj` objects diff --git a/pypeit/wavetilts.py b/pypeit/wavetilts.py index 7b4e805db9..fb70fbc98e 100644 --- a/pypeit/wavetilts.py +++ b/pypeit/wavetilts.py @@ -191,6 +191,7 @@ def show(self, waveimg=None, wcs_match=True, in_ginga=True, show_traces=False): """ # get tilt_img_dict if (Path(self.calib_dir).resolve() / self.tiltimg_filename).exists(): + # TODO: Somehow pass chk_version here? tilt_img_dict = buildimage.TiltImage.from_file(Path(self.calib_dir).resolve() / self.tiltimg_filename) else: msgs.error(f'Tilt image {str((Path(self.calib_dir).resolve() / self.tiltimg_filename))} NOT FOUND.') @@ -198,6 +199,7 @@ def show(self, waveimg=None, wcs_match=True, in_ginga=True, show_traces=False): # get slits slitmask = None if (Path(self.calib_dir).resolve() / self.slits_filename).exists(): + # TODO: Somehow pass chk_version here? slits = slittrace.SlitTraceSet.from_file(Path(self.calib_dir).resolve() / self.slits_filename) _slitmask = slits.slit_img(initial=True, flexure=self.spat_flexure) _left, _right, _mask = slits.select_edges(flexure=self.spat_flexure) @@ -215,6 +217,7 @@ def show(self, waveimg=None, wcs_match=True, in_ginga=True, show_traces=False): if waveimg is None and slits is not None and same_size and in_ginga: wv_calib_name = wavecalib.WaveCalib.construct_file_name(self.calib_key, calib_dir=self.calib_dir) if Path(wv_calib_name).resolve().exists(): + # TODO: Somehow pass chk_version here? wv_calib = wavecalib.WaveCalib.from_file(wv_calib_name) tilts = self.fit2tiltimg(slitmask, flexure=self.spat_flexure) waveimg = wv_calib.build_waveimg(tilts, slits, spat_flexure=self.spat_flexure) From 071988d550be6dfb14ed0d030d5be80e9e9a2ecf Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 18 Jan 2024 13:37:39 -0800 Subject: [PATCH 193/244] forgot to save --- pypeit/specutils/pypeit_loaders.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pypeit/specutils/pypeit_loaders.py b/pypeit/specutils/pypeit_loaders.py index f500574337..c26b095e67 100644 --- a/pypeit/specutils/pypeit_loaders.py +++ b/pypeit/specutils/pypeit_loaders.py @@ -249,6 +249,7 @@ def pypeit_onespec_loader(filename, grid=False, strict=True, **kwargs): """ # Try to load the file and ignoring any version mismatch try: + # TODO: Pass chk_version here? spec = onespec.OneSpec.from_file(filename) except PypeItError: file_pypeit_version = astropy.io.fits.getval(filename, 'VERSPYP', 'PRIMARY') From 363c1dcffa426ad8af5afd5f3d9389add389d3d0 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 18 Jan 2024 13:49:05 -0800 Subject: [PATCH 194/244] fixes --- pypeit/core/wavecal/wv_fitting.py | 3 +-- pypeit/slittrace.py | 9 ++++----- pypeit/spec2dobj.py | 12 +++++------- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/pypeit/core/wavecal/wv_fitting.py b/pypeit/core/wavecal/wv_fitting.py index 8333647f04..2481f89d50 100644 --- a/pypeit/core/wavecal/wv_fitting.py +++ b/pypeit/core/wavecal/wv_fitting.py @@ -96,8 +96,7 @@ def _bundle(self, **kwargs): # Extension prefix (for being unique with slits) hdue_pref = self.hduext_prefix_from_spatid(self.spat_id) # Without PypeItFit - _d = super(WaveFit, self)._bundle( - ext=hdue_pref+'WAVEFIT', **kwargs) + _d = super()._bundle(ext=hdue_pref+'WAVEFIT', **kwargs) # Deal with PypeItFit if _d[0][hdue_pref+'WAVEFIT']['pypeitfit'] is not None: _d.append({hdue_pref+'PYPEITFIT': _d[0][hdue_pref + 'WAVEFIT'].pop('pypeitfit')}) diff --git a/pypeit/slittrace.py b/pypeit/slittrace.py index 4c49aee0dc..1127558e07 100644 --- a/pypeit/slittrace.py +++ b/pypeit/slittrace.py @@ -312,7 +312,7 @@ def _parse(cls, hdu, hdu_prefix=None, **kwargs): return super()._parse(hdu, ext='SLITS', transpose_table_arrays=True) @classmethod - def from_hdu(cls, hdu, hdu_prefix=None, chk_version=True): + def from_hdu(cls, hdu, chk_version=True, **kwargs): """ Instantiate the object from an HDU extension. @@ -322,15 +322,14 @@ def from_hdu(cls, hdu, hdu_prefix=None, chk_version=True): Args: hdu (`astropy.io.fits.HDUList`_, `astropy.io.fits.ImageHDU`_, `astropy.io.fits.BinTableHDU`_): The HDU(s) with the data to use for instantiation. - hdu_prefix (:obj:`str`, optional): - Maintained for consistency with the base class but is - not used by this method. chk_version (:obj:`bool`, optional): If True, raise an error if the datamodel version or type check failed. If False, throw a warning only. + **kwargs: + Passed directly to :func:`_parse`. """ # Run the default parser - d, version_passed, type_passed, parsed_hdus = cls._parse(hdu) + d, version_passed, type_passed, parsed_hdus = cls._parse(hdu, **kwargs) # Check cls._check_parsed(version_passed, type_passed, chk_version=chk_version) diff --git a/pypeit/spec2dobj.py b/pypeit/spec2dobj.py index 8cf3709939..9b5eaff94b 100644 --- a/pypeit/spec2dobj.py +++ b/pypeit/spec2dobj.py @@ -110,7 +110,7 @@ class Spec2DObj(datamodel.DataContainer): ] @classmethod - def from_file(cls, ifile, verbose=True, chk_version=True, **kwargs): + def from_file(cls, ifile, detname, chk_version=True): """ Instantiate the object from an extension in the specified fits file. @@ -120,21 +120,19 @@ def from_file(cls, ifile, verbose=True, chk_version=True, **kwargs): Args: ifile (:obj:`str`, `Path`_): Fits file with the data to read - verbose (:obj:`bool`, optional): - Print informational messages (not currently used) + detname (:obj:`str`): + The string identifier for the detector or mosaic used to select + the data that is read. chk_version (:obj:`bool`, optional): Passed to :func:`from_hdu`. - kwargs (:obj:`dict`, optional): - Arguments passed directly to :func:`from_hdu`. """ with io.fits_open(ifile) as hdu: # Check detname is valid detnames = np.unique([h.name.split('-')[0] for h in hdu[1:]]) if detname not in detnames: msgs.error(f'Your --det={detname} is not available. \n Choose from: {detnames}') - return cls.from_hdu(hdu, detname, chk_version=chk_version, **kwargs) + return cls.from_hdu(hdu, detname, chk_version=chk_version) - # TODO: Allow for **kwargs here? @classmethod def from_hdu(cls, hdu, detname, chk_version=True): """ From 413f6bd5dbb637ac144c77b97f7fcd0872a3d54b Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 18 Jan 2024 14:36:04 -0800 Subject: [PATCH 195/244] propagate to scripts --- pypeit/scripts/arxiv_solution.py | 5 +++-- pypeit/scripts/chk_alignments.py | 7 ++++--- pypeit/scripts/chk_edges.py | 16 +++++++++------- pypeit/scripts/chk_noise_1dspec.py | 8 +++++--- pypeit/scripts/chk_noise_2dspec.py | 5 ++++- pypeit/scripts/chk_scattlight.py | 13 +++++++------ pypeit/scripts/chk_tilts.py | 6 +++--- pypeit/scripts/chk_wavecalib.py | 15 ++++++++------- pypeit/scripts/edge_inspector.py | 5 +++-- pypeit/scripts/flux_calib.py | 7 ++++--- pypeit/scripts/identify.py | 11 +++++------ pypeit/scripts/multislit_flexure.py | 7 +++---- pypeit/scripts/parse_slits.py | 12 ++++++++---- pypeit/scripts/ql.py | 12 ++++++++---- pypeit/scripts/show_2dspec.py | 6 ++++-- pypeit/scripts/show_wvcalib.py | 19 ++++++++++--------- pypeit/scripts/skysub_regions.py | 6 ++++-- 17 files changed, 92 insertions(+), 68 deletions(-) diff --git a/pypeit/scripts/arxiv_solution.py b/pypeit/scripts/arxiv_solution.py index 797b92cc6a..bf4f7139fc 100644 --- a/pypeit/scripts/arxiv_solution.py +++ b/pypeit/scripts/arxiv_solution.py @@ -25,6 +25,8 @@ def get_parser(cls, width=None): parser.add_argument('-v', '--verbosity', type=int, default=1, help='Verbosity level between 0 [none] and 2 [all]. Default: 1. ' 'Level 2 writes a log with filename make_arxiv_solution_YYYYMMDD-HHMM.log') + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @staticmethod @@ -43,8 +45,7 @@ def main(args): msgs.error("The following MasterWaveCalib file does not exist:" + msgs.newline() + args.file) # Load the wavelength calibration file - # TODO: Pass chk_version here? - wv_calib = WaveCalib.from_file(args.file) + wv_calib = WaveCalib.from_file(args.file, chk_version=(not args.try_old)) # Check if a wavelength solution exists if wv_calib['wv_fits'][args.slit]['wave_soln'] is None: gd_slits = [] diff --git a/pypeit/scripts/chk_alignments.py b/pypeit/scripts/chk_alignments.py index 43d3738761..a9a3127187 100644 --- a/pypeit/scripts/chk_alignments.py +++ b/pypeit/scripts/chk_alignments.py @@ -21,15 +21,16 @@ def get_parser(cls, width=None): help='PypeIt Alignment file [e.g. Alignment_A_1_DET01.fits]') parser.add_argument('--chname', default='Alignments', type=str, help='Channel name for image in Ginga') + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @staticmethod - def main(pargs): + def main(args): from pypeit import alignframe # Load - # TODO: Pass chk_version here? - alignments = alignframe.Alignments.from_file(pargs.file) + alignments = alignframe.Alignments.from_file(args.file, chk_version=(not args.try_old)) # Show alignments.show() diff --git a/pypeit/scripts/chk_edges.py b/pypeit/scripts/chk_edges.py index 990a607ed9..2d8a8e593c 100644 --- a/pypeit/scripts/chk_edges.py +++ b/pypeit/scripts/chk_edges.py @@ -6,7 +6,6 @@ """ from pypeit.scripts import scriptbase -from pathlib import Path class ChkEdges(scriptbase.ScriptBase): @@ -30,21 +29,24 @@ def get_parser(cls, width=None): return parser @staticmethod - def main(pargs): + def main(args): + + from pathlib import Path + from pypeit import edgetrace, slittrace, msgs # Load - edges = edgetrace.EdgeTraceSet.from_file(pargs.trace_file, chk_version=(not pargs.try_old)) + edges = edgetrace.EdgeTraceSet.from_file(args.trace_file, chk_version=(not args.try_old)) - if pargs.mpl: + if args.mpl: edges.show(thin=10, include_img=True, idlabel=True) return # Set the Slits file name - slit_filename = pargs.slits_file + slit_filename = args.slits_file if slit_filename is not None: # File provided by user - slit_filename = Path(pargs.slits_file).resolve() + slit_filename = Path(args.slits_file).resolve() if not slit_filename.exists(): # But doesn't exist msgs.warn(f'{slit_filename} does not exist!') @@ -58,7 +60,7 @@ def main(pargs): msgs.warn(f'{slit_filename} does not exist!') # NOTE: At this point, slit_filename *must* be a Path object - slits = slittrace.SlitTraceSet.from_file(slit_filename, chk_version=(not pargs.try_old)) \ + slits = slittrace.SlitTraceSet.from_file(slit_filename, chk_version=(not args.try_old)) \ if slit_filename.exists() else None edges.show(thin=10, in_ginga=True, slits=slits) diff --git a/pypeit/scripts/chk_noise_1dspec.py b/pypeit/scripts/chk_noise_1dspec.py index db338f80a9..bc75550c95 100644 --- a/pypeit/scripts/chk_noise_1dspec.py +++ b/pypeit/scripts/chk_noise_1dspec.py @@ -168,6 +168,8 @@ def get_parser(cls, width=None): 'save, a folder called spec1d*_noisecheck' ' will be created and all the relevant ' 'plot will be placed there.') + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @@ -198,9 +200,9 @@ def main(args): head = fits.getheader(file) # I/O spec object - # TODO: Pass chk_version to OneSpec too? - specObjs = [OneSpec.from_file(file)] if args.fileformat == 'coadd1d' else \ - specobjs.SpecObjs.from_fitsfile(file, chk_version=False) + specObjs = [OneSpec.from_file(file, chk_version=(not args.try_old))] \ + if args.fileformat == 'coadd1d' else \ + specobjs.SpecObjs.from_fitsfile(file, chk_version=(not args.try_old)) # loop on the spectra for spec in specObjs: diff --git a/pypeit/scripts/chk_noise_2dspec.py b/pypeit/scripts/chk_noise_2dspec.py index 6dc8372af9..1401b73d88 100644 --- a/pypeit/scripts/chk_noise_2dspec.py +++ b/pypeit/scripts/chk_noise_2dspec.py @@ -171,6 +171,8 @@ def get_parser(cls, width=None): 'noise values to be printed in the terminal.') parser.add_argument('--list', default=False, help='List the extensions only?', action='store_true') + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @@ -205,7 +207,8 @@ def main(args): if args.list: io.fits_open(file).info() continue - spec2DObj = spec2dobj.Spec2DObj.from_file(file, detname, chk_version=False) + spec2DObj = spec2dobj.Spec2DObj.from_file(file, detname, + chk_version=(not args.try_old)) # Deal with redshifts if args.z is not None: diff --git a/pypeit/scripts/chk_scattlight.py b/pypeit/scripts/chk_scattlight.py index f067d6d5b1..8171841f22 100644 --- a/pypeit/scripts/chk_scattlight.py +++ b/pypeit/scripts/chk_scattlight.py @@ -27,6 +27,8 @@ def get_parser(cls, width=None): parser.add_argument('--mask', type=bool, default=False, help='If True, the detector pixels that are considered on the slit will be ' 'masked to highlight the scattered light regions') + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @staticmethod @@ -47,12 +49,11 @@ def main(args): detname = DetectorContainer.get_name(det) # Load scattered light calibration frame - # TODO: Pass chk_version here? - ScattLightImage = scattlight.ScatteredLight.from_file(args.file) + ScattLightImage = scattlight.ScatteredLight.from_file(args.file, + chk_version=(not args.try_old)) # Load slits information - # TODO: Pass chk_version here? - slits = slittrace.SlitTraceSet.from_file(args.slits) + slits = slittrace.SlitTraceSet.from_file(args.slits, chk_version=(not args.try_old)) # Load the alternate file if requested display_frame = None # The default is to display the frame used to calculate the scattered light model @@ -60,8 +61,8 @@ def main(args): msgs.error("displaying the spec2d scattered light is not currently supported") try: # TODO :: the spec2d file may have already had the scattered light removed, so this is not correct. This script only works when the scattered light is turned off for the spec2d file - # TODO: Do not default to chk_version=False - spec2D = spec2dobj.Spec2DObj.from_file(args.spec2d, detname, chk_version=False) + spec2D = spec2dobj.Spec2DObj.from_file(args.spec2d, detname, + chk_version=(not args.try_old)) except PypeItDataModelError: msgs.warn(f"Error loading spec2d file {args.spec2d} - attempting to load science image from fits") spec2D = None diff --git a/pypeit/scripts/chk_tilts.py b/pypeit/scripts/chk_tilts.py index e85bdbac40..53632789c1 100644 --- a/pypeit/scripts/chk_tilts.py +++ b/pypeit/scripts/chk_tilts.py @@ -31,12 +31,12 @@ def get_parser(cls, width=None): return parser @staticmethod - def main(pargs): + def main(args): from pypeit import wavetilts # Load - tilts = wavetilts.WaveTilts.from_file(pargs.file, chk_version=(not pargs.try_old)) - tilts.show(in_ginga=np.logical_not(pargs.mpl), show_traces=pargs.show_traces) + tilts = wavetilts.WaveTilts.from_file(args.file, chk_version=(not args.try_old)) + tilts.show(in_ginga=np.logical_not(args.mpl), show_traces=args.show_traces) diff --git a/pypeit/scripts/chk_wavecalib.py b/pypeit/scripts/chk_wavecalib.py index ac547346c0..af9f20cd70 100644 --- a/pypeit/scripts/chk_wavecalib.py +++ b/pypeit/scripts/chk_wavecalib.py @@ -18,15 +18,16 @@ def get_parser(cls, width=None): parser.add_argument('input_file', type=str, nargs='+', help='One or more PypeIt WaveCalib file [e.g. WaveCalib_A_1_DET01.fits] or ' 'spec2d file') + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @staticmethod def main(args): from IPython import embed - from pypeit import wavecalib, spec2dobj, msgs - from pypeit.pypmsgs import PypeItError from astropy.io import fits + from pypeit import wavecalib, spec2dobj, msgs # Loop over the input files for in_file in args.input_file: @@ -44,19 +45,19 @@ def main(args): msgs.error("Bad file type input!") if file_type == 'WaveCalib': - # TODO: Do not always set chk_version to False - waveCalib = wavecalib.WaveCalib.from_file(in_file, chk_version=False) + waveCalib = wavecalib.WaveCalib.from_file(in_file, chk_version=(not args.try_old)) waveCalib.wave_diagnostics(print_diag=True) continue elif file_type == 'AllSpec2D': - # TODO: Do not always set chk_version to False - allspec2D = spec2dobj.AllSpec2DObj.from_fits(in_file, chk_version=False) + allspec2D = spec2dobj.AllSpec2DObj.from_fits(in_file, + chk_version=(not args.try_old)) for det in allspec2D.detectors: print('') print('='*50 + f'{det:^7}' + '='*51) wave_diag = allspec2D[det].wavesol - for colname in ['minWave', 'Wave_cen', 'maxWave', 'IDs_Wave_cov(%)', 'mesured_fwhm']: + for colname in ['minWave', 'Wave_cen', 'maxWave', 'IDs_Wave_cov(%)', + 'mesured_fwhm']: wave_diag[colname].format = '0.1f' for colname in ['dWave', 'RMS']: wave_diag[colname].format = '0.3f' diff --git a/pypeit/scripts/edge_inspector.py b/pypeit/scripts/edge_inspector.py index b0aeb38905..0874d881db 100644 --- a/pypeit/scripts/edge_inspector.py +++ b/pypeit/scripts/edge_inspector.py @@ -17,6 +17,8 @@ def get_parser(cls, width=None): width=width) parser.add_argument('trace_file', type=str, default=None, help='PypeIt Edges file [e.g. Edges_A_0_DET01.fits.gz]') + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @staticmethod @@ -30,8 +32,7 @@ def main(args): # Set the file name to the full path trace_file = Path(args.trace_file).resolve() # Load - # TODO: Pass chk_version here? - edges = edgetrace.EdgeTraceSet.from_file(trace_file) + edges = edgetrace.EdgeTraceSet.from_file(trace_file, chk_version=(not args.try_old)) # Inspector object pointer = edge_inspector.EdgeInspectorGUI(edges) # Run. Ends when window is closed diff --git a/pypeit/scripts/flux_calib.py b/pypeit/scripts/flux_calib.py index f4f12ed269..2962c0c822 100644 --- a/pypeit/scripts/flux_calib.py +++ b/pypeit/scripts/flux_calib.py @@ -64,9 +64,10 @@ def get_parser(cls, width=None): parser.add_argument('-v', '--verbosity', type=int, default=1, help='Verbosity level between 0 [none] and 2 [all]. Default: 1. ' 'Level 2 writes a log with filename flux_calib_YYYYMMDD-HHMM.log') - # parser.add_argument("--plot", default=False, action="store_true", # help="Show the sensitivity function?") + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @staticmethod @@ -77,7 +78,6 @@ def main(args): msgs.set_logfile_and_verbosity('flux_calib', args.verbosity) # Load the file - # TODO: Pass chk_version here? fluxFile = inputfiles.FluxFile.from_file(args.flux_file) # Read in spectrograph from spec1dfile header @@ -110,7 +110,8 @@ def main(args): 'Run pypeit_flux_calib --help for information on the format') # Instantiate - fluxcalibrate.flux_calibrate(fluxFile.filenames, sensfiles, par=par['fluxcalib']) + fluxcalibrate.flux_calibrate(fluxFile.filenames, sensfiles, par=par['fluxcalib'], + chk_version=(not args.try_old)) msgs.info('Flux calibration complete') return 0 diff --git a/pypeit/scripts/identify.py b/pypeit/scripts/identify.py index ab78cbbe48..41bde34050 100644 --- a/pypeit/scripts/identify.py +++ b/pypeit/scripts/identify.py @@ -42,6 +42,8 @@ def get_parser(cls, width=None): parser.add_argument('-v', '--verbosity', type=int, default=1, help='Verbosity level between 0 [none] and 2 [all]. Default: 1. ' 'Level 2 writes a log with filename identify_YYYYMMDD-HHMM.log') + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @staticmethod @@ -62,8 +64,7 @@ def main(args): msgs.set_logfile_and_verbosity('identify', args.verbosity) # Load the Arc file - # TODO: Pass chk_version here? - msarc = ArcImage.from_file(args.arc_file) + msarc = ArcImage.from_file(args.arc_file, chk_version=(not args.try_old)) # Load the spectrograph spec = load_spectrograph(msarc.PYP_SPEC) @@ -79,15 +80,13 @@ def main(args): par['lamps'] = lamps # Load the slits - # TODO: Pass chk_version here? - slits = slittrace.SlitTraceSet.from_file(args.slits_file) + slits = slittrace.SlitTraceSet.from_file(args.slits_file, chk_version=(not args.try_old)) # Reset the mask slits.mask = slits.mask_init # Check if a solution exists solnname = WaveCalib.construct_file_name(msarc.calib_key, calib_dir=msarc.calib_dir) - # TODO: Pass chk_version here? - wv_calib = WaveCalib.from_file(solnname) \ + wv_calib = WaveCalib.from_file(solnname, chk_version=(not args.try_old)) \ if os.path.exists(solnname) and args.solution else None # Load the calibration frame (if it exists and is desired). Bad-pixel mask diff --git a/pypeit/scripts/multislit_flexure.py b/pypeit/scripts/multislit_flexure.py index be36bc47c6..98e836196b 100644 --- a/pypeit/scripts/multislit_flexure.py +++ b/pypeit/scripts/multislit_flexure.py @@ -95,12 +95,12 @@ def get_parser(cls, width=None): return parser @staticmethod - def main(pargs): + def main(args): from astropy.io import fits # Load the file - flexFile = inputfiles.FlexureFile.from_file(pargs.flex_file) + flexFile = inputfiles.FlexureFile.from_file(args.flex_file) # Read in spectrograph from spec1dfile header header = fits.getheader(flexFile.filenames[0]) @@ -141,8 +141,7 @@ def main(pargs): # Write msgs.info("Write to disk") - mdFlex.to_file(pargs.outroot+root+'.fits', - overwrite=pargs.clobber) + mdFlex.to_file(args.outroot+root+'.fits', overwrite=args.clobber) # Apply?? diff --git a/pypeit/scripts/parse_slits.py b/pypeit/scripts/parse_slits.py index 7d7f4caf9e..4774782d9a 100644 --- a/pypeit/scripts/parse_slits.py +++ b/pypeit/scripts/parse_slits.py @@ -58,14 +58,16 @@ def get_parser(cls, width=None): parser = super().get_parser(description='Print info on slits from a input file', width=width) parser.add_argument('input_file', type=str, help='Either a spec2D or Slits filename') + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @staticmethod - def main(pargs): + def main(args): # What kind of file are we?? - hdul = fits.open(pargs.input_file) + hdul = fits.open(args.input_file) head0 = hdul[0].header head1 = hdul[1].header file_type = None @@ -80,13 +82,15 @@ def main(pargs): if file_type == 'Slits': # TODO: Do not always set chk_version to False - slits = slittrace.SlitTraceSet.from_file(pargs.input_file, chk_version=False) + slits = slittrace.SlitTraceSet.from_file(args.input_file, + chk_version=(not args.try_old)) print('') print_slits(slits) elif file_type == 'AllSpec2D': # TODO: Do not always set chk_version to False - allspec2D = spec2dobj.AllSpec2DObj.from_fits(pargs.input_file, chk_version=False) + allspec2D = spec2dobj.AllSpec2DObj.from_fits(args.input_file, + chk_version=(not args.try_old)) # Loop on Detectors for det in allspec2D.detectors: print('') diff --git a/pypeit/scripts/ql.py b/pypeit/scripts/ql.py index b9a9742c81..1b91d9d798 100644 --- a/pypeit/scripts/ql.py +++ b/pypeit/scripts/ql.py @@ -211,7 +211,8 @@ def generate_sci_pypeitfile(redux_path:str, slitspatnum:str=None, maskID:str=None, boxcar_radius:float=None, - snr_thresh:float=None): + snr_thresh:float=None, + chk_version:bool=True): """ Prepare to reduce the science frames: @@ -350,8 +351,7 @@ def generate_sci_pypeitfile(redux_path:str, # Iterate through each file to find the one with the relevant mask ID. detname = None for sliittrace_file in slittrace_files: - # TODO: Somehow pass chk_version here? - slitTrace = SlitTraceSet.from_file(sliittrace_file) + slitTrace = SlitTraceSet.from_file(sliittrace_file, chk_version=chk_version) if maskID in slitTrace.maskdef_id: detname = slitTrace.detname # Mosaic? @@ -745,6 +745,9 @@ def get_parser(cls, width=None): 'factor. For a finer grid, set value to <1.0; for coarser ' 'sampling, set value to >1.0).') + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') + return parser @@ -961,7 +964,8 @@ def main(args): slitspatnum=args.slitspatnum, maskID=args.maskID, boxcar_radius=args.boxcar_radius, - snr_thresh=args.snr_thresh) + snr_thresh=args.snr_thresh, + chk_version=(not args.try_old)) # Run it pypeIt = pypeit.PypeIt(sci_pypeit_file, reuse_calibs=True) diff --git a/pypeit/scripts/show_2dspec.py b/pypeit/scripts/show_2dspec.py index 2bdd0b4c11..8e1c98ced7 100644 --- a/pypeit/scripts/show_2dspec.py +++ b/pypeit/scripts/show_2dspec.py @@ -109,6 +109,8 @@ def get_parser(cls, width=None): help='Do *not* clear all existing tabs') parser.add_argument('-v', '--verbosity', type=int, default=1, help='Verbosity level between 0 [none] and 2 [all]') + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @staticmethod @@ -137,8 +139,8 @@ def main(args): # Try to read the Spec2DObj using the current datamodel, but allowing # for the datamodel version to be different try: - # TODO: Do not always set chk_version to False - spec2DObj = spec2dobj.Spec2DObj.from_file(args.file, detname, chk_version=False) + spec2DObj = spec2dobj.Spec2DObj.from_file(args.file, detname, + chk_version=(not args.try_old)) except PypeItDataModelError: try: # Try to get the pypeit version used to write this file diff --git a/pypeit/scripts/show_wvcalib.py b/pypeit/scripts/show_wvcalib.py index 832af3e72d..fd6d07b9ac 100644 --- a/pypeit/scripts/show_wvcalib.py +++ b/pypeit/scripts/show_wvcalib.py @@ -26,27 +26,28 @@ def get_parser(cls, width=None): parser.add_argument("--slit_file", type=str, help="Slit file") parser.add_argument("--is_order", default=False, action="store_true", help="Input slit/order is an order") + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @staticmethod - def main(pargs, unit_test=False): + def main(args, unit_test=False): """ Shows the spectrum """ from matplotlib import pyplot as plt # Load - # TODO: Pass chk_version here? - wvcalib = wavecalib.WaveCalib.from_file(pargs.file) - if pargs.slit_file is not None: - # TODO: Pass chk_version here? - slits = slittrace.SlitTraceSet.from_file(pargs.slit_file) + wvcalib = wavecalib.WaveCalib.from_file(args.file, chk_version=(not args.try_old)) + if args.slit_file is not None: + slits = slittrace.SlitTraceSet.from_file(args.slit_file, + chk_version=(not args.try_old)) # Parse - if pargs.is_order: - idx = np.where(slits.ech_order == pargs.slit_order)[0][0] + if args.is_order: + idx = np.where(slits.ech_order == args.slit_order)[0][0] else: - idx = np.where(wvcalib.spat_ids == pargs.slit_order)[0][0] + idx = np.where(wvcalib.spat_ids == args.slit_order)[0][0] # Grab it spec = wvcalib.arc_spectra[:,idx] diff --git a/pypeit/scripts/skysub_regions.py b/pypeit/scripts/skysub_regions.py index 7366b2f62c..d492271a09 100644 --- a/pypeit/scripts/skysub_regions.py +++ b/pypeit/scripts/skysub_regions.py @@ -31,6 +31,8 @@ def get_parser(cls, width=None): parser.add_argument('-v', '--verbosity', type=int, default=1, help='Verbosity level between 0 [none] and 2 [all]. Default: 1. ' 'Level 2 writes a log with filename skysub_regions_YYYYMMDD-HHMM.log') + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') return parser @staticmethod @@ -55,8 +57,8 @@ def main(args): detname = DetectorContainer.get_name(det) # Load it up - # TODO: Pass chk_version here? Do not always set it to True - spec2DObj = spec2dobj.Spec2DObj.from_file(args.file, detname, chk_version=True) + spec2DObj = spec2dobj.Spec2DObj.from_file(args.file, detname, + chk_version=(not args.try_old)) frame = spec2DObj.sciimg hdr = fits.open(args.file)[0].header fname = hdr['FILENAME'] From 946b89e16895cd4f6d6985f0d6c44b095191f26c Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 18 Jan 2024 14:49:41 -0800 Subject: [PATCH 196/244] clean up --- pypeit/scripts/parse_slits.py | 2 -- pypeit/slittrace.py | 4 ++-- pypeit/wavecalib.py | 1 + 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pypeit/scripts/parse_slits.py b/pypeit/scripts/parse_slits.py index 4774782d9a..b99f33e5d6 100644 --- a/pypeit/scripts/parse_slits.py +++ b/pypeit/scripts/parse_slits.py @@ -81,14 +81,12 @@ def main(args): msgs.error("Bad file type input!") if file_type == 'Slits': - # TODO: Do not always set chk_version to False slits = slittrace.SlitTraceSet.from_file(args.input_file, chk_version=(not args.try_old)) print('') print_slits(slits) elif file_type == 'AllSpec2D': - # TODO: Do not always set chk_version to False allspec2D = spec2dobj.AllSpec2DObj.from_fits(args.input_file, chk_version=(not args.try_old)) # Loop on Detectors diff --git a/pypeit/slittrace.py b/pypeit/slittrace.py index 1127558e07..b37673d0f1 100644 --- a/pypeit/slittrace.py +++ b/pypeit/slittrace.py @@ -802,7 +802,7 @@ def mask_add_missing_obj(self, sobjs, spat_flexure, fwhm, boxcar_rad): missing the targeted source. Args: - sobjs (:class:`pypeit.specobjs.SpecObjs`): + sobjs (:class:`~pypeit.specobjs.SpecObjs`): List of SpecObj that have been found and traced spat_flexure (:obj:`float`): Shifts, in spatial pixels, between this image and SlitTrace @@ -812,7 +812,7 @@ def mask_add_missing_obj(self, sobjs, spat_flexure, fwhm, boxcar_rad): BOX_RADIUS in pixels to be used in the boxcar extraction Returns: - :class:`pypeit.specobjs.SpecObjs`: Updated list of SpecObj that have + :class:`~pypeit.specobjs.SpecObjs`: Updated list of SpecObj that have been found and traced """ diff --git a/pypeit/wavecalib.py b/pypeit/wavecalib.py index 3c5a3be0d1..a9287bd431 100644 --- a/pypeit/wavecalib.py +++ b/pypeit/wavecalib.py @@ -401,6 +401,7 @@ def wave_diagnostics(self, print_diag=False): diag['IDs_Wave_cov(%)'] = lines_cov diag['IDs_Wave_cov(%)'].format = '0.1f' # FWHM + # TODO: Why not me*a*sured_fwhm? diag['mesured_fwhm'] = [0. if wvfit.fwhm is None else wvfit.fwhm for wvfit in self.wv_fits] diag['mesured_fwhm'].format = '0.1f' # RMS From b4e4d932878c2a68c3d62cbf579b2a88c4932da4 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 18 Jan 2024 17:05:04 -0800 Subject: [PATCH 197/244] propagate and consolidate chk_version --- pypeit/calibrations.py | 43 ++++++++++------------ pypeit/coadd1d.py | 50 ++++++++++++------------- pypeit/coadd3d.py | 59 ++++++++++++++++-------------- pypeit/core/telluric.py | 33 ++++++++++++----- pypeit/core/wavecal/templates.py | 2 +- pypeit/flatfield.py | 10 +++-- pypeit/par/pypeitpar.py | 50 +++++++++++++++++-------- pypeit/scripts/chk_flats.py | 2 +- pypeit/scripts/chk_tilts.py | 3 +- pypeit/scripts/coadd_1dspec.py | 4 +- pypeit/scripts/coadd_datacube.py | 6 ++- pypeit/scripts/collate_1d.py | 28 +++++++++----- pypeit/scripts/sensfunc.py | 3 +- pypeit/scripts/tellfit.py | 13 +++++-- pypeit/sensfunc.py | 55 +++++++++++++++++----------- pypeit/specutils/pypeit_loaders.py | 17 +++++++-- pypeit/wavetilts.py | 20 ++++++---- 17 files changed, 240 insertions(+), 158 deletions(-) diff --git a/pypeit/calibrations.py b/pypeit/calibrations.py index 83f21490a6..07f15a32cf 100644 --- a/pypeit/calibrations.py +++ b/pypeit/calibrations.py @@ -64,6 +64,11 @@ class Calibrations: user_slits (:obj:`dict`, optional): A limited set of slits selected by the user for analysis. See :func:`~pypeit.slittrace.SlitTraceSet.user_mask`. + chk_version (:obj:`bool`, optional): + When reading in existing files written by PypeIt, perform strict + version checking to ensure a valid file. If False, the code will + try to keep going, but this may lead to faults and quiet failures. + User beware! Attributes: fitstbl (:class:`~pypeit.metadata.PypeItMetaData`): @@ -134,7 +139,7 @@ def get_instance(fitstbl, par, spectrograph, caldir, **kwargs): return calibclass(fitstbl, par, spectrograph, caldir, **kwargs) def __init__(self, fitstbl, par, spectrograph, caldir, qadir=None, - reuse_calibs=False, show=False, user_slits=None): + reuse_calibs=False, show=False, user_slits=None, chk_version=True): # Check the types # TODO -- Remove this None option once we have data models for all the Calibrations @@ -153,6 +158,7 @@ def __init__(self, fitstbl, par, spectrograph, caldir, qadir=None, # Calibrations self.reuse_calibs = reuse_calibs + self.chk_version = chk_version self.calib_dir = Path(caldir).resolve() if not self.calib_dir.exists(): self.calib_dir.mkdir(parents=True) @@ -314,8 +320,7 @@ def get_arc(self): # If a processed calibration frame exists and we want to reuse it, do # so: if cal_file.exists() and self.reuse_calibs: - # TODO: Somehow pass chk_version here? - self.msarc = frame['class'].from_file(cal_file) + self.msarc = frame['class'].from_file(cal_file, chk_version=self.chk_version) return self.msarc # Reset the BPM @@ -358,8 +363,7 @@ def get_tiltimg(self): # If a processed calibration frame exists and we want to reuse it, do # so: if cal_file.exists() and self.reuse_calibs: - # TODO: Somehow pass chk_version here? - self.mstilt = frame['class'].from_file(cal_file) + self.mstilt = frame['class'].from_file(cal_file, chk_version=self.chk_version) return self.mstilt # Reset the BPM @@ -407,8 +411,7 @@ def get_align(self): # If a processed calibration frame exists and we want to reuse it, do # so: if cal_file.exists() and self.reuse_calibs: - # TODO: Somehow pass chk_version here? - self.alignments = frame['class'].from_file(cal_file) + self.alignments = frame['class'].from_file(cal_file, chk_version=self.chk_version) self.alignments.is_synced(self.slits) return self.alignments @@ -459,8 +462,7 @@ def get_bias(self): # If a processed calibration frame exists and we want to reuse it, do # so: if cal_file.exists() and self.reuse_calibs: - # TODO: Somehow pass chk_version here? - self.msbias = frame['class'].from_file(cal_file) + self.msbias = frame['class'].from_file(cal_file, chk_version=self.chk_version) return self.msbias # Otherwise, create the processed file. @@ -499,8 +501,7 @@ def get_dark(self): # If a processed calibration frame exists and we want to reuse it, do # so: if cal_file.exists() and self.reuse_calibs: - # TODO: Somehow pass chk_version here? - self.msdark = frame['class'].from_file(cal_file) + self.msdark = frame['class'].from_file(cal_file, chk_version=self.chk_version) return self.msdark # TODO: If a bias has been constructed and it will be subtracted from @@ -576,8 +577,7 @@ def get_scattlight(self): # If a processed calibration frame exists and we want to reuse it, do # so: if cal_file.exists() and self.reuse_calibs: - # TODO: Somehow pass chk_version here? - self.msscattlight = frame['class'].from_file(cal_file) + self.msscattlight = frame['class'].from_file(cal_file, chk_version=self.chk_version) return self.msscattlight # Scattered light model does not exist or we're not reusing it. @@ -695,8 +695,8 @@ def get_flats(self): setup = illum_setup if pixel_setup is None else pixel_setup calib_id = illum_calib_id if pixel_calib_id is None else pixel_calib_id if cal_file.exists() and self.reuse_calibs: - # TODO: Somehow pass chk_version here? - self.flatimages = flatfield.FlatImages.from_file(cal_file) + self.flatimages = flatfield.FlatImages.from_file(cal_file, + chk_version=self.chk_version) self.flatimages.is_synced(self.slits) # Load user defined files if self.par['flatfield']['pixelflat_file'] is not None: @@ -852,8 +852,7 @@ def get_slits(self): # If a processed calibration frame exists and we want to reuse it, do # so: if cal_file.exists() and self.reuse_calibs: - # TODO: Somehow pass chk_version here? - self.slits = frame['class'].from_file(cal_file) + self.slits = frame['class'].from_file(cal_file, chk_version=self.chk_version) self.slits.mask = self.slits.mask_init.copy() if self.user_slits is not None: self.slits.user_mask(detname, self.user_slits) @@ -866,8 +865,8 @@ def get_slits(self): # If so, reuse it? if edges_file.exists() and self.reuse_calibs: # Yep! Load it and parse it into slits. - # TODO: Somehow pass chk_version here? - self.slits = edgetrace.EdgeTraceSet.from_file(edges_file).get_slits() + self.slits = edgetrace.EdgeTraceSet.from_file(edges_file, + chk_version=self.chk_version).get_slits() # Write the slits calibration file self.slits.to_file() if self.user_slits is not None: @@ -970,8 +969,7 @@ def get_wv_calib(self): # we want to reuse it, do so (or just load it): if cal_file.exists() and self.reuse_calibs: # Load the file - # TODO: Somehow pass chk_version here? - self.wv_calib = wavecalib.WaveCalib.from_file(cal_file) + self.wv_calib = wavecalib.WaveCalib.from_file(cal_file, chk_version=self.chk_version) self.wv_calib.chk_synced(self.slits) self.slits.mask_wvcalib(self.wv_calib) # Return @@ -1043,8 +1041,7 @@ def get_tilts(self): # If a processed calibration frame exists and we want to reuse it, do # so: if cal_file.exists() and self.reuse_calibs: - # TODO: Somehow pass chk_version here? - self.wavetilts = wavetilts.WaveTilts.from_file(cal_file) + self.wavetilts = wavetilts.WaveTilts.from_file(cal_file, chk_version=self.chk_version) self.wavetilts.is_synced(self.slits) self.slits.mask_wavetilts(self.wavetilts) return self.wavetilts diff --git a/pypeit/coadd1d.py b/pypeit/coadd1d.py index 1ac0a1d255..cd1f7c1944 100644 --- a/pypeit/coadd1d.py +++ b/pypeit/coadd1d.py @@ -28,19 +28,19 @@ class CoAdd1D: @classmethod - def get_instance(cls, spec1dfiles, objids, spectrograph=None, par=None, sensfuncfile=None, setup_id=None, - debug=False, show=False): + def get_instance(cls, spec1dfiles, objids, spectrograph=None, par=None, sensfuncfile=None, + setup_id=None, debug=False, show=False, chk_version=True): """ - Superclass factory method which generates the subclass instance. See :class:`CoAdd1D` instantiation for - argument descriptions. + Superclass factory method which generates the subclass instance. See + :class:`CoAdd1D` instantiation for argument descriptions. """ pypeline = fits.getheader(spec1dfiles[0])['PYPELINE'] + 'CoAdd1D' return next(c for c in utils.all_subclasses(CoAdd1D) if c.__name__ == pypeline)( - spec1dfiles, objids, spectrograph=spectrograph, par=par, sensfuncfile=sensfuncfile, setup_id=setup_id, - debug=debug, show=show) + spec1dfiles, objids, spectrograph=spectrograph, par=par, sensfuncfile=sensfuncfile, + setup_id=setup_id, debug=debug, show=show, chk_version=chk_version) - def __init__(self, spec1dfiles, objids, spectrograph=None, par=None, sensfuncfile=None, setup_id=None, - debug=False, show=False): + def __init__(self, spec1dfiles, objids, spectrograph=None, par=None, sensfuncfile=None, + setup_id=None, debug=False, show=False, chk_version=True): """ Args: @@ -66,6 +66,11 @@ def __init__(self, spec1dfiles, objids, spectrograph=None, par=None, sensfuncfil Debug. Default = False show (bool, optional): Debug. Default = True + chk_version (:obj:`bool`, optional): + When reading in existing files written by PypeIt, perform strict + version checking to ensure a valid file. If False, the code + will try to keep going, but this may lead to faults and quiet + failures. User beware! """ # Instantiate attributes self.spec1dfiles = spec1dfiles @@ -84,6 +89,7 @@ def __init__(self, spec1dfiles, objids, spectrograph=None, par=None, sensfuncfil # self.debug = debug self.show = show + self.chk_version = chk_version self.nexp = len(self.spec1dfiles) # Number of exposures self.coaddfile = None self.gpm_exp = np.ones(self.nexp, dtype=bool).tolist() # list of bool indicating the exposures that have been coadded @@ -159,14 +165,6 @@ class MultiSlitCoAdd1D(CoAdd1D): Child of CoAdd1d for Multislit and Longslit reductions. """ - def __init__(self, spec1dfiles, objids, spectrograph=None, par=None, sensfuncfile=None, setup_id=None, debug=False, show=False): - """ - See :class:`CoAdd1D` instantiation for argument descriptions. - """ - super().__init__(spec1dfiles, objids, spectrograph=spectrograph, par = par, sensfuncfile = sensfuncfile, - setup_id=setup_id, debug = debug, show = show) - - def load(self): """ Load the arrays we need for performing coadds. @@ -190,7 +188,8 @@ def load(self): """ waves, fluxes, ivars, gpms, headers = [], [], [], [], [] for iexp in range(self.nexp): - sobjs = specobjs.SpecObjs.from_fitsfile(self.spec1dfiles[iexp], chk_version=self.par['chk_version']) + sobjs = specobjs.SpecObjs.from_fitsfile(self.spec1dfiles[iexp], + chk_version=self.chk_version) indx = sobjs.name_indices(self.objids[iexp]) if not np.any(indx): msgs.error( @@ -333,15 +332,14 @@ class EchelleCoAdd1D(CoAdd1D): Child of CoAdd1d for Echelle reductions. """ - def __init__(self, spec1dfiles, objids, spectrograph=None, par=None, sensfuncfile=None, setup_id=None, - debug=False, show=False): + def __init__(self, spec1dfiles, objids, spectrograph=None, par=None, sensfuncfile=None, + setup_id=None, debug=False, show=False, chk_version=True): """ See :class:`CoAdd1D` instantiation for argument descriptions. - - """ - super().__init__(spec1dfiles, objids, spectrograph=spectrograph, par = par, sensfuncfile = sensfuncfile, - setup_id=setup_id, debug = debug, show = show) + super().__init__(spec1dfiles, objids, spectrograph=spectrograph, par=par, + sensfuncfile=sensfuncfile, setup_id=setup_id, debug=debug, show=show, + chk_version=chk_version) if sensfuncfile is None: msgs.error('sensfuncfile is a required argument for echelle coadding') @@ -435,7 +433,7 @@ def load_ech_arrays(self, spec1dfiles, objids, sensfuncfiles): """ nexp = len(spec1dfiles) for iexp in range(nexp): - sobjs = specobjs.SpecObjs.from_fitsfile(spec1dfiles[iexp], chk_version=self.par['chk_version']) + sobjs = specobjs.SpecObjs.from_fitsfile(spec1dfiles[iexp], chk_version=self.chk_version) indx = sobjs.name_indices(objids[iexp]) if not np.any(indx): msgs.error("No matching objects for {:s}. Odds are you input the wrong OBJID".format(objids[iexp])) @@ -446,7 +444,9 @@ def load_ech_arrays(self, spec1dfiles, objids, sensfuncfiles): # echelle coadding code to combine echelle and multislit data if wave_iexp.ndim == 1: wave_iexp, flux_iexp, ivar_iexp, gpm_iexp = np.atleast_2d(wave_iexp).T, np.atleast_2d(flux_iexp).T, np.atleast_2d(ivar_iexp).T, np.atleast_2d(gpm_iexp).T - weights_sens_iexp = sensfunc.SensFunc.sensfunc_weights(sensfuncfiles[iexp], wave_iexp, debug=self.debug) + weights_sens_iexp = sensfunc.SensFunc.sensfunc_weights(sensfuncfiles[iexp], wave_iexp, + debug=self.debug, + chk_version=self.chk_version) # Allocate arrays on first iteration # TODO :: We should refactor to use a list of numpy arrays, instead of a 2D numpy array. if iexp == 0: diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index 070549afb1..55b921844b 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -340,8 +340,9 @@ class CoAdd3D: """ # Superclass factory method generates the subclass instance @classmethod - def get_instance(cls, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offsets=None, dec_offsets=None, - spectrograph=None, det=1, overwrite=False, show=False, debug=False): + def get_instance(cls, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offsets=None, + dec_offsets=None, spectrograph=None, det=1, overwrite=False, show=False, + debug=False): """ Instantiate the subclass appropriate for the provided spectrograph. @@ -354,15 +355,15 @@ def get_instance(cls, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_o :class:`CoAdd3D`: One of the subclasses with :class:`CoAdd3D` as its base. """ - return next(c for c in cls.__subclasses__() if c.__name__ == (spectrograph.pypeline + 'CoAdd3D'))( - spec2dfiles, par, skysub_frame=skysub_frame, scale_corr=scale_corr, ra_offsets=ra_offsets, - dec_offsets=dec_offsets, spectrograph=spectrograph, det=det, overwrite=overwrite, - show=show, debug=debug) + spec2dfiles, par, skysub_frame=skysub_frame, scale_corr=scale_corr, + ra_offsets=ra_offsets, dec_offsets=dec_offsets, spectrograph=spectrograph, + det=det, overwrite=overwrite, show=show, debug=debug) - def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offsets=None, dec_offsets=None, - spectrograph=None, det=None, overwrite=False, show=False, debug=False): + def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offsets=None, + dec_offsets=None, spectrograph=None, det=None, overwrite=False, show=False, + debug=False): """ Args: @@ -397,13 +398,13 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs Show results in ginga debug (:obj:`bool`, optional): Show QA for debugging. - """ # TODO :: Consider loading all calibrations into a single variable within the main CoAdd3D parent class. self.spec2d = spec2dfiles self.numfiles = len(spec2dfiles) self.par = par self.overwrite = overwrite + self.chk_version = self.par['rdx']['chk_version'] # Do some quick checks on the input options if skysub_frame is not None: if len(skysub_frame) != len(spec2dfiles): @@ -587,8 +588,9 @@ def set_default_scalecorr(self): msgs.info("Loading default scale image for relative spectral illumination correction:" + msgs.newline() + self.cubepar['scale_corr']) try: - # TODO: Somehow pass chk_version here? - spec2DObj = spec2dobj.Spec2DObj.from_file(self.cubepar['scale_corr'], self.detname) + spec2DObj = spec2dobj.Spec2DObj.from_file(self.cubepar['scale_corr'], + self.detname, + chk_version=self.chk_version) except Exception as e: msgs.warn(f'Loading spec2d file raised {type(e).__name__}:\n{str(e)}') msgs.warn("Could not load scaleimg from spec2d file:" + msgs.newline() + @@ -649,8 +651,8 @@ def get_current_scalecorr(self, spec2DObj, scalecorr=None): msgs.info("Loading the following frame for the relative spectral illumination correction:" + msgs.newline() + scalecorr) try: - # TODO: Somehow pass chk_version here? - spec2DObj_scl = spec2dobj.Spec2DObj.from_file(scalecorr, self.detname) + spec2DObj_scl = spec2dobj.Spec2DObj.from_file(scalecorr, self.detname, + chk_version=self.chk_version) except Exception as e: msgs.warn(f'Loading spec2d file raised {type(e).__name__}:\n{str(e)}') msgs.error("Could not load skysub image from spec2d file:" + msgs.newline() + scalecorr) @@ -681,8 +683,9 @@ def set_default_skysub(self): msgs.info("Loading default image for sky subtraction:" + msgs.newline() + self.cubepar['skysub_frame']) try: - # TODO: Somehow pass chk_version here? - spec2DObj = spec2dobj.Spec2DObj.from_file(self.cubepar['skysub_frame'], self.detname) + spec2DObj = spec2dobj.Spec2DObj.from_file(self.cubepar['skysub_frame'], + self.detname, + chk_version=self.chk_version) skysub_exptime = self.spec.get_meta_value([spec2DObj.head0], 'exptime') except: msgs.error("Could not load skysub image from spec2d file:" + msgs.newline() + self.cubepar['skysub_frame']) @@ -752,8 +755,8 @@ def get_current_skysub(self, spec2DObj, exptime, opts_skysub=None): # Load a user specified frame for sky subtraction msgs.info("Loading skysub frame:" + msgs.newline() + opts_skysub) try: - # TODO: Somehow pass chk_version here? - spec2DObj_sky = spec2dobj.Spec2DObj.from_file(opts_skysub, self.detname) + spec2DObj_sky = spec2dobj.Spec2DObj.from_file(opts_skysub, self.detname, + chk_version=self.chk_version) skysub_exptime = self.spec.get_meta_value([spec2DObj_sky.head0], 'exptime') except: msgs.error("Could not load skysub image from spec2d file:" + msgs.newline() + opts_skysub) @@ -791,8 +794,7 @@ def add_grating_corr(self, flatfile, waveimg, slits, spat_flexure=None): if not os.path.exists(flatfile): msgs.error("Grating correction requested, but the following file does not exist:" + msgs.newline() + flatfile) # Load the Flat file - # TODO: Somehow pass chk_version here? - flatimages = flatfield.FlatImages.from_file(flatfile) + flatimages = flatfield.FlatImages.from_file(flatfile, chk_version=self.chk_version) total_illum = flatimages.fit2illumflat(slits, finecorr=False, frametype='illum', initial=True, spat_flexure=spat_flexure) * \ flatimages.fit2illumflat(slits, finecorr=True, frametype='illum', initial=True, spat_flexure=spat_flexure) flatframe = flatimages.pixelflat_raw / total_illum @@ -857,11 +859,12 @@ class SlicerIFUCoAdd3D(CoAdd3D): - White light images are also produced, if requested. """ - def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offsets=None, dec_offsets=None, - spectrograph=None, det=1, overwrite=False, show=False, debug=False): - super().__init__(spec2dfiles, par, skysub_frame=skysub_frame, scale_corr=scale_corr, ra_offsets=ra_offsets, - dec_offsets=dec_offsets, spectrograph=spectrograph, det=det, overwrite=overwrite, - show=show, debug=debug) + def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offsets=None, + dec_offsets=None, spectrograph=None, det=1, overwrite=False, show=False, + debug=False): + super().__init__(spec2dfiles, par, skysub_frame=skysub_frame, scale_corr=scale_corr, + ra_offsets=ra_offsets, dec_offsets=dec_offsets, spectrograph=spectrograph, + det=det, overwrite=overwrite, show=show, debug=debug) self.mnmx_wv = None # Will be used to store the minimum and maximum wavelengths of every slit and frame. self._spatscale = np.zeros((self.numfiles, 2)) # index 0, 1 = pixel scale, slicer scale self._specscale = np.zeros(self.numfiles) @@ -895,8 +898,8 @@ def get_alignments(self, spec2DObj, slits, spat_flexure=None): alignfile = os.path.join(spec2DObj.calibs['DIR'], spec2DObj.calibs[key]) if os.path.exists(alignfile) and self.cubepar['astrometric']: msgs.info("Loading alignments") - # TODO: Somehow pass chk_version here? - alignments = alignframe.Alignments.from_file(alignfile) + alignments = alignframe.Alignments.from_file(alignfile, + chk_version=self.chk_version) else: msgs.warn(f'Processed alignment frame not recorded or not found!') msgs.info("Using slit edges for astrometric transform") @@ -955,8 +958,8 @@ def load(self): for ff, fil in enumerate(self.spec2d): # Load it up msgs.info("Loading PypeIt spec2d frame:" + msgs.newline() + fil) - # TODO: Somehow pass chk_version here? - spec2DObj = spec2dobj.Spec2DObj.from_file(fil, self.detname) + spec2DObj = spec2dobj.Spec2DObj.from_file(fil, self.detname, + chk_version=self.chk_version) detector = spec2DObj.detector spat_flexure = None # spec2DObj.sci_spat_flexure diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 30908d8495..a66e587b1e 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -719,7 +719,7 @@ def unpack_orders(sobjs, ret_flam=False): # TODO: This function needs to be revisited. Better yet, it would useful to # brainstorm about whether or not it's worth revisiting the spec1d datamodel. -def general_spec_reader(specfile, ret_flam=False): +def general_spec_reader(specfile, ret_flam=False, chk_version=True): """ Read a spec1d file or a coadd spectrum file. @@ -728,6 +728,11 @@ def general_spec_reader(specfile, ret_flam=False): File with the data ret_flam (:obj:`bool`, optional): Return FLAM instead of COUNTS. + chk_version (:obj:`bool`, optional): + When reading in existing files written by PypeIt, perform strict + version checking to ensure a valid file. If False, the code will + try to keep going, but this may lead to faults and quiet failures. + User beware! Returns: :obj:`tuple`: Seven objects are returned. The first five are @@ -742,8 +747,7 @@ def general_spec_reader(specfile, ret_flam=False): hdul = fits.open(specfile) if 'DMODCLS' in hdul[1].header and hdul[1].header['DMODCLS'] == 'OneSpec': # Load - # TODO: Somehow pass chk_version here? - spec = onespec.OneSpec.from_file(specfile) + spec = onespec.OneSpec.from_file(specfile, chk_version=chk_version) # Unpack wave = spec.wave # wavelength grid evaluated at the bin centers, uniformly-spaced in lambda or log10-lambda/velocity. @@ -756,7 +760,7 @@ def general_spec_reader(specfile, ret_flam=False): spect_dict = spec.spect_meta head = spec.head0 else: - sobjs = specobjs.SpecObjs.from_fitsfile(specfile, chk_version=False) + sobjs = specobjs.SpecObjs.from_fitsfile(specfile, chk_version=chk_version) # TODO: What bug? Is it fixed now? How can we test if it's fixed? if np.sum(sobjs.OPT_WAVE) is None: raise ValueError("This is an ugly hack until the DataContainer bug is fixed") @@ -1521,7 +1525,8 @@ def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile teltype='pca', tell_npca=4, bounds_norm=(0.1, 3.0), tell_norm_thresh=0.9, sn_clip=30.0, only_orders=None, maxiter=3, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, - pix_shift_bounds=(-5.0,5.0), debug_init=False, debug=False, show=False): + pix_shift_bounds=(-5.0,5.0), debug_init=False, debug=False, show=False, + chk_version=True): """ Fit and correct a QSO spectrum for telluric absorption. @@ -1610,6 +1615,10 @@ def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile fits. show : :obj:`bool`, optional Show a QA plot of the final fit. + chk_version (:obj:`bool`, optional): + When reading in existing files written by PypeIt, perform strict version + checking to ensure a valid file. If False, the code will try to keep + going, but this may lead to faults and quiet failures. User beware! Returns ------- @@ -1628,7 +1637,8 @@ def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile 'lbound_norm', 'ubound_norm', 'tell_norm_thresh'), debug_init=debug_init) - wave, wave_grid_mid, flux, ivar, mask, meta_spec, header = general_spec_reader(spec1dfile, ret_flam=True) + wave, wave_grid_mid, flux, ivar, mask, meta_spec, header \ + = general_spec_reader(spec1dfile, ret_flam=True, chk_version=chk_version) header = fits.getheader(spec1dfile) # clean this up! # Mask the IGM and mask wavelengths that extend redward of our PCA @@ -1692,7 +1702,8 @@ def star_telluric(spec1dfile, telgridfile, telloutfile, outfile, star_type=None, mask_helium_lines=False, hydrogen_mask_wid=10., delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), only_orders=None, sn_clip=30.0, maxiter=3, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, - pix_shift_bounds=(-5.0,5.0), debug_init=False, debug=False, show=False): + pix_shift_bounds=(-5.0,5.0), debug_init=False, debug=False, show=False, + chk_version=True): """ This needs a doc string. @@ -1709,7 +1720,8 @@ def star_telluric(spec1dfile, telgridfile, telloutfile, outfile, star_type=None, disp = True # Read in the data - wave, wave_grid_mid, flux, ivar, mask, meta_spec, header = general_spec_reader(spec1dfile, ret_flam=False) + wave, wave_grid_mid, flux, ivar, mask, meta_spec, header \ + = general_spec_reader(spec1dfile, ret_flam=False, chk_version=chk_version) # Read in standard star dictionary and interpolate onto regular telluric wave_grid star_ra = meta_spec['core']['RA'] if star_ra is None else star_ra star_dec = meta_spec['core']['DEC'] if star_dec is None else star_dec @@ -1800,7 +1812,7 @@ def poly_telluric(spec1dfile, telgridfile, telloutfile, outfile, z_obj=0.0, func tell_npca=4, delta_coeff_bounds=(-20.0, 20.0), minmax_coeff_bounds=(-5.0, 5.0), only_orders=None, sn_clip=30.0, maxiter=3, tol=1e-3, popsize=30, recombination=0.7, polish=True, disp=False, pix_shift_bounds=(-5.0,5.0), - debug_init=False, debug=False, show=False): + debug_init=False, debug=False, show=False, chk_version=True): """ This needs a doc string. @@ -1814,7 +1826,8 @@ def poly_telluric(spec1dfile, telgridfile, telloutfile, outfile, z_obj=0.0, func disp = True # Read in the data - wave, wave_grid_mid, flux, ivar, mask, meta_spec, header = general_spec_reader(spec1dfile, ret_flam=False) + wave, wave_grid_mid, flux, ivar, mask, meta_spec, header \ + = general_spec_reader(spec1dfile, ret_flam=False, chk_version=chk_version) if flux.ndim == 2: norders = flux.shape[1] diff --git a/pypeit/core/wavecal/templates.py b/pypeit/core/wavecal/templates.py index 5c612e280a..44bb9ca0be 100644 --- a/pypeit/core/wavecal/templates.py +++ b/pypeit/core/wavecal/templates.py @@ -266,7 +266,7 @@ def pypeit_arcspec(in_file, slit, binspec, binning=None): # minx=iwv_calib['fmin'], maxx=iwv_calib['fmax']) flux = np.array(iwv_calib['spec']).flatten() elif '.fits' in in_file: - # TODO: Don't default to chk_version=False ... + # TODO: This should not default to chk_version=False ... wvcalib = wavecalib.WaveCalib.from_file(in_file, chk_version=False) idx = np.where(wvcalib.spat_ids == slit)[0][0] flux = wvcalib.arc_spectra[:,idx] diff --git a/pypeit/flatfield.py b/pypeit/flatfield.py index cda9afbb49..cb7c5e8a31 100644 --- a/pypeit/flatfield.py +++ b/pypeit/flatfield.py @@ -389,7 +389,7 @@ def fit2illumflat(self, slits, frametype='illum', finecorr=False, initial=False, # TODO -- Update the internal one? Or remove it altogether?? return illumflat - def show(self, frametype='all', slits=None, wcs_match=True): + def show(self, frametype='all', slits=None, wcs_match=True, chk_version=True): """ Simple wrapper to :func:`show_flats`. @@ -403,6 +403,11 @@ def show(self, frametype='all', slits=None, wcs_match=True): wcs_match (:obj:`bool`, optional): (Attempt to) Match the WCS coordinates of the output images in the `ginga`_ viewer. + chk_version (:obj:`bool`, optional): + When reading in existing files written by PypeIt, perform strict + version checking to ensure a valid file. If False, the code + will try to keep going, but this may lead to faults and quiet + failures. User beware! """ illumflat_pixel, illumflat_illum = None, None pixelflat_finecorr, illumflat_finecorr = None, None @@ -414,8 +419,7 @@ def show(self, frametype='all', slits=None, wcs_match=True): slits_file = slittrace.SlitTraceSet.construct_file_name(self.calib_key, calib_dir=self.calib_dir) try: - # TODO: Somehow pass chk_version here? - slits = slittrace.SlitTraceSet.from_file(slits_file) + slits = slittrace.SlitTraceSet.from_file(slits_file, chk_version=chk_version) except (FileNotFoundError, PypeItError): msgs.warn('Could not load slits to include when showing flat-field images. File ' 'was either not provided directly, or it could not be read based on its ' diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index 8409519610..fc3305b43c 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -1124,7 +1124,7 @@ def __init__(self, ex_value=None, flux_value=None, nmaskedge=None, sigrej_scale=None, scale_method=None, sn_min_medscale=None, sn_min_polyscale=None, weight_method=None, maxiter_reject=None, lower=None, upper=None, maxrej=None, sn_clip=None, nbests=None, coaddfile=None, - mag_type=None, filter=None, filter_mag=None, filter_mask=None, chk_version=None): + mag_type=None, filter=None, filter_mag=None, filter_mask=None): # Grab the parameter names and values from the function # arguments @@ -1321,12 +1321,16 @@ def __init__(self, ex_value=None, flux_value=None, nmaskedge=None, dtypes['coaddfile'] = str descr['coaddfile'] = 'Output filename' - - # Version checking - defaults['chk_version'] = True - dtypes['chk_version'] = bool - descr['chk_version'] = 'If True enforce strict PypeIt version checking to ensure that spec1d*.fits files were created' \ - 'with the current version of PypeIt' + # NOTE: Deprecated in favor of the parameter in ReduxPar! +# defaults['chk_version'] = True +# dtypes['chk_version'] = bool +# descr['chk_version'] = 'If True, enforce strict PypeIt version checking to ensure that ' \ +# 'all reused PypeIt files were created with the current version ' \ +# 'of PypeIt. If set to False, the code will attempt to read ' \ +# 'out-of-date files and keep going. Beware (!!) that this can ' \ +# 'lead to unforeseen bugs that either cause the code to crash or ' \ +# 'produce erroneous results. I.e., you really need to know what ' \ +# 'you are doing if you set this to False!' # Instantiate the parameter set super(Coadd1DPar, self).__init__(list(pars.keys()), @@ -1343,7 +1347,7 @@ def from_dict(cls, cfg): 'wave_method', 'dv', 'dwave', 'dloglam', 'wave_grid_min', 'wave_grid_max', 'spec_samp_fact', 'ref_percentile', 'maxiter_scale', 'sigrej_scale', 'scale_method', 'sn_min_medscale', 'sn_min_polyscale', 'weight_method', 'maxiter_reject', 'lower', 'upper', - 'maxrej', 'sn_clip', 'nbests', 'coaddfile', 'chk_version', + 'maxrej', 'sn_clip', 'nbests', 'coaddfile', 'filter', 'mag_type', 'filter_mag', 'filter_mask'] badkeys = np.array([pk not in parkeys for pk in k]) @@ -2616,7 +2620,7 @@ class ReduxPar(ParSet): """ def __init__(self, spectrograph=None, detnum=None, sortroot=None, calwin=None, scidir=None, qadir=None, redux_path=None, ignore_bad_headers=None, slitspatnum=None, - maskIDs=None, quicklook=None): + maskIDs=None, quicklook=None, chk_version=None): # Grab the parameter names and values from the function # arguments @@ -2696,6 +2700,17 @@ def __init__(self, spectrograph=None, detnum=None, sortroot=None, calwin=None, s descr['redux_path'] = 'Path to folder for performing reductions. Default is the ' \ 'current working directory.' + # Version checking + defaults['chk_version'] = True + dtypes['chk_version'] = bool + descr['chk_version'] = 'If True enforce strict PypeIt version checking to ensure that ' \ + 'all files were created with the current version of PypeIt. If ' \ + 'set to False, the code will attempt to read out-of-date files ' \ + 'and keep going. Beware (!!) that this can lead to unforeseen ' \ + 'bugs that either cause the code to crash or lead to erroneous ' \ + 'results. I.e., you really need to know what you are doing if ' \ + 'you set this to False!' + # Instantiate the parameter set super(ReduxPar, self).__init__(list(pars.keys()), values=list(pars.values()), @@ -2711,7 +2726,7 @@ def from_dict(cls, cfg): # Basic keywords parkeys = [ 'spectrograph', 'quicklook', 'detnum', 'sortroot', 'calwin', 'scidir', 'qadir', - 'redux_path', 'ignore_bad_headers', 'slitspatnum', 'maskIDs'] + 'redux_path', 'ignore_bad_headers', 'slitspatnum', 'maskIDs', 'chk_version'] badkeys = np.array([pk not in parkeys for pk in k]) if np.any(badkeys): @@ -5165,7 +5180,9 @@ class Collate1DPar(ParSet): For a table with the current keywords, defaults, and descriptions, see :ref:`parameters`. """ - def __init__(self, tolerance=None, dry_run=None, ignore_flux=None, flux=None, match_using=None, exclude_slit_trace_bm=[], exclude_serendip=False, wv_rms_thresh=None, outdir=None, spec1d_outdir=None, refframe=None, chk_version=False): + def __init__(self, tolerance=None, dry_run=None, ignore_flux=None, flux=None, match_using=None, + exclude_slit_trace_bm=[], exclude_serendip=False, wv_rms_thresh=None, outdir=None, + spec1d_outdir=None, refframe=None): # Grab the parameter names and values from the function # arguments @@ -5244,9 +5261,10 @@ def __init__(self, tolerance=None, dry_run=None, ignore_flux=None, flux=None, ma descr['refframe'] = 'Perform reference frame correction prior to coadding. ' \ 'Options are: {0}'.format(', '.join(options['refframe'])) - defaults['chk_version'] = False - dtypes['chk_version'] = bool - descr['chk_version'] = "Whether to check the data model versions of spec1d files and sensfunc files." + # NOTE: Deprecated in favor of the parameter in ReduxPar! +# defaults['chk_version'] = False +# dtypes['chk_version'] = bool +# descr['chk_version'] = "Whether to check the data model versions of spec1d files and sensfunc files." # Instantiate the parameter set super(Collate1DPar, self).__init__(list(pars.keys()), @@ -5259,7 +5277,9 @@ def __init__(self, tolerance=None, dry_run=None, ignore_flux=None, flux=None, ma @classmethod def from_dict(cls, cfg): k = [*cfg.keys()] - parkeys = ['tolerance', 'dry_run', 'ignore_flux', 'flux', 'match_using', 'exclude_slit_trace_bm', 'exclude_serendip', 'outdir', 'spec1d_outdir', 'wv_rms_thresh', 'refframe', 'chk_version'] + parkeys = ['tolerance', 'dry_run', 'ignore_flux', 'flux', 'match_using', + 'exclude_slit_trace_bm', 'exclude_serendip', 'outdir', 'spec1d_outdir', + 'wv_rms_thresh', 'refframe'] badkeys = np.array([pk not in parkeys for pk in k]) if np.any(badkeys): diff --git a/pypeit/scripts/chk_flats.py b/pypeit/scripts/chk_flats.py index 92211ca894..e6c53d28c6 100644 --- a/pypeit/scripts/chk_flats.py +++ b/pypeit/scripts/chk_flats.py @@ -30,6 +30,6 @@ def main(args): # Load flatImages = flatfield.FlatImages.from_file(args.file, chk_version=(not args.try_old)) # Show - flatImages.show(args.type) + flatImages.show(args.type, chk_version=(not args.try_old)) diff --git a/pypeit/scripts/chk_tilts.py b/pypeit/scripts/chk_tilts.py index 53632789c1..4bc9903d47 100644 --- a/pypeit/scripts/chk_tilts.py +++ b/pypeit/scripts/chk_tilts.py @@ -36,7 +36,8 @@ def main(args): # Load tilts = wavetilts.WaveTilts.from_file(args.file, chk_version=(not args.try_old)) - tilts.show(in_ginga=np.logical_not(args.mpl), show_traces=args.show_traces) + tilts.show(in_ginga=np.logical_not(args.mpl), show_traces=args.show_traces, + chk_version=(not args.try_old)) diff --git a/pypeit/scripts/coadd_1dspec.py b/pypeit/scripts/coadd_1dspec.py index e6e2c604c7..7763f9b905 100644 --- a/pypeit/scripts/coadd_1dspec.py +++ b/pypeit/scripts/coadd_1dspec.py @@ -247,7 +247,6 @@ def main(args): # sensfile = os.path.join(args.test_spec_path, sensfile) # coaddfile = os.path.join(args.test_spec_path, coaddfile) - if coaddfile is None: coaddfile = build_coadd_file_name(coadd1dFile.filenames, spectrograph) @@ -258,7 +257,8 @@ def main(args): par=par['coadd1d'], sensfuncfile=coadd1dFile.sensfiles, setup_id=coadd1dFile.setup_id, - debug=args.debug, show=args.show) + debug=args.debug, show=args.show, + chk_version=par['rdx']['chk_version']) # Run coAdd1d.run() # Save to file diff --git a/pypeit/scripts/coadd_datacube.py b/pypeit/scripts/coadd_datacube.py index 26e12a6bd2..b4f86887f7 100644 --- a/pypeit/scripts/coadd_datacube.py +++ b/pypeit/scripts/coadd_datacube.py @@ -47,6 +47,7 @@ def main(args): spectrograph_def_par = spectrograph.default_pypeit_par() parset = par.PypeItPar.from_cfg_lines(cfg_lines=spectrograph_def_par.to_config(), merge_with=(coadd3dfile.cfg_lines,)) + # If detector was passed as an argument override whatever was in the coadd3d file if args.det is not None: msgs.info("Restricting to detector={}".format(args.det)) @@ -60,8 +61,9 @@ def main(args): # Instantiate CoAdd3d tstart = time.time() - coadd = CoAdd3D.get_instance(coadd3dfile.filenames, parset, skysub_frame=skysub_frame, scale_corr=scale_corr, - ra_offsets=ra_offsets, dec_offsets=dec_offsets, spectrograph=spectrograph, + coadd = CoAdd3D.get_instance(coadd3dfile.filenames, parset, skysub_frame=skysub_frame, + scale_corr=scale_corr, ra_offsets=ra_offsets, + dec_offsets=dec_offsets, spectrograph=spectrograph, det=args.det, overwrite=args.overwrite) # Coadd the files diff --git a/pypeit/scripts/collate_1d.py b/pypeit/scripts/collate_1d.py index 6e99c789ad..4bfaa7a948 100644 --- a/pypeit/scripts/collate_1d.py +++ b/pypeit/scripts/collate_1d.py @@ -105,7 +105,7 @@ def find_slits_to_exclude(spec2d_files, par): Args: spec2d_files (:obj:`list`): List of spec2d files to build the map from. - par (:class:`~pypeit.par.pypeitpar.Collate1DPar`): + par (:class:`~pypeit.par.pypeitpar.PypeItPar`): Parameters from a ``.collate1d`` file Returns: @@ -124,7 +124,9 @@ def find_slits_to_exclude(spec2d_files, par): exclude_map = dict() for spec2d_file in spec2d_files: - allspec2d = AllSpec2DObj.from_fits(spec2d_file, chk_version=par['collate1d']['chk_version']) + allspec2d = AllSpec2DObj.from_fits(spec2d_file, + chk_version=par['rdx']['chk_version']) +# chk_version=par['collate1d']['chk_version']) for sobj2d in [allspec2d[det] for det in allspec2d.detectors]: for (slit_id, mask, slit_mask_id) in sobj2d['slits'].slit_info: for flag in exclude_flags: @@ -247,7 +249,9 @@ def read_spec1d_files(par, spec1d_files, failure_msgs): good_spec1d_files = [] for spec1d_file in spec1d_files: try: - sobjs = SpecObjs.from_fitsfile(spec1d_file, chk_version = par['collate1d']['chk_version']) + sobjs = SpecObjs.from_fitsfile(spec1d_file, + chk_version=par['rdx']['chk_version']) +# chk_version=par['collate1d']['chk_version']) specobjs_list.append(sobjs) good_spec1d_files.append(spec1d_file) except Exception as e: @@ -301,7 +305,9 @@ def flux(par, spectrograph, spec1d_files, failed_fluxing_msgs): # Flux calibrate the spec1d file try: msgs.info(f"Running flux calibrate on {spec1d_file}") - FxCalib = fluxcalibrate.flux_calibrate([spec1d_file], [sens_file], par=par['fluxcalib'], chk_version=par['collate1d']['chk_version']) + FxCalib = fluxcalibrate.flux_calibrate([spec1d_file], [sens_file], par=par['fluxcalib'], + chk_version=par['rdx']['chk_version']) +# chk_version=par['collate1d']['chk_version']) flux_calibrated_files.append(spec1d_file) except Exception: @@ -352,7 +358,7 @@ def refframe_correction(par, spectrograph, spec1d_files, spec1d_failure_msgs): for spec1d in spec1d_files: # Get values from the fits header needed to calculate the correction try: - sobjs = SpecObjs.from_fitsfile(spec1d) + sobjs = SpecObjs.from_fitsfile(spec1d, chk_version=par['rdx']['chk_version']) hdr_ra = sobjs.header['RA'] hdr_dec = sobjs.header['DEC'] hdr_radec = ltu.radec_to_coord((hdr_ra, hdr_dec)) @@ -425,8 +431,8 @@ def coadd(par, coaddfile, source): # Set destination file for coadding par['coadd1d']['coaddfile'] = coaddfile - # Whether to be forgiving of data model versions - par['coadd1d']['chk_version'] = par['collate1d']['chk_version'] +# # Whether to be forgiving of data model versions +# par['coadd1d']['chk_version'] = par['collate1d']['chk_version'] # Determine if we should coadd flux calibrated data flux_key = par['coadd1d']['ex_value'] + "_FLAM" @@ -610,7 +616,8 @@ def build_parameters(args): params['collate1d']['refframe'] = args.refframe if args.chk_version is True: - params['collate1d']['chk_version'] = True +# params['collate1d']['chk_version'] = True + params['rdx']['chk_version'] = True return params, spectrograph, spec1d_files @@ -670,6 +677,9 @@ def get_parser(cls, width=None): 'nights and prepare a directory for the KOA.', width=width, formatter=scriptbase.SmartFormatter) +# 'F| chk_version If true, spec1ds and archival sensfuncs must match the currently\n' +# 'F| supported versions. If false (the default) version numbers are not checked.\n' + # TODO: Is the file optional? If so, shouldn't the first argument start # with '--'? parser.add_argument('input_file', type=str, @@ -694,8 +704,6 @@ def get_parser(cls, width=None): 'F| value are skipped, else all wavelength rms values are accepted.\n' 'F| refframe Perform reference frame correction prior to coadding.\n' f'F| Options are {pypeitpar.WavelengthSolutionPar.valid_reference_frames()}. Defaults to None.\n' - 'F| chk_version If true, spec1ds and archival sensfuncs must match the currently\n' - 'F| supported versions. If false (the default) version numbers are not checked.\n' '\n' 'F|spec1d read\n' 'F|\n' diff --git a/pypeit/scripts/sensfunc.py b/pypeit/scripts/sensfunc.py index d6325f7a03..bc4406711c 100644 --- a/pypeit/scripts/sensfunc.py +++ b/pypeit/scripts/sensfunc.py @@ -179,7 +179,8 @@ def main(args): if args.outfile is None else args.outfile # Instantiate the relevant class for the requested algorithm sensobj = sensfunc.SensFunc.get_instance(args.spec1dfile, outfile, par['sensfunc'], - debug=args.debug) + debug=args.debug, + chk_version=par['rdx']['chk_version']) # Generate the sensfunc sensobj.run() # Write it out to a file, including the new primary FITS header diff --git a/pypeit/scripts/tellfit.py b/pypeit/scripts/tellfit.py index d0e0626f2f..846f5eb13d 100644 --- a/pypeit/scripts/tellfit.py +++ b/pypeit/scripts/tellfit.py @@ -72,6 +72,10 @@ def get_parser(cls, width=None): parser.add_argument('-v', '--verbosity', type=int, default=1, help='Verbosity level between 0 [none] and 2 [all]. Default: 1. ' 'Level 2 writes a log with filename tellfit_YYYYMMDD-HHMM.log') + parser.add_argument('--chk_version', default=False, action='store_true', + help='Ensure the datamodels are from the current PypeIt version. ' + 'By default (consistent with previous functionality) this is ' + 'not enforced and crashes may ensue ...') return parser @staticmethod @@ -167,7 +171,8 @@ def main(args): popsize=par['telluric']['popsize'], tol=par['telluric']['tol'], debug_init=args.debug, disp=args.debug, - debug=args.debug, show=args.plot) + debug=args.debug, show=args.plot, + chk_version=args.chk_version) elif par['telluric']['objmodel']=='star': TelStar = telluric.star_telluric(args.spec1dfile, par['telluric']['telgridfile'], modelfile, outfile, @@ -190,7 +195,8 @@ def main(args): popsize=par['telluric']['popsize'], tol=par['telluric']['tol'], debug_init=args.debug, disp=args.debug, - debug=args.debug, show=args.plot) + debug=args.debug, show=args.plot, + chk_version=args.chk_version) elif par['telluric']['objmodel']=='poly': TelPoly = telluric.poly_telluric(args.spec1dfile, par['telluric']['telgridfile'], modelfile, outfile, @@ -209,7 +215,8 @@ def main(args): popsize=par['telluric']['popsize'], tol=par['telluric']['tol'], debug_init=args.debug, disp=args.debug, - debug=args.debug, show=args.plot) + debug=args.debug, show=args.plot, + chk_version=args.chk_version) else: msgs.error("Object model is not supported yet. Must be 'qso', 'star', or 'poly'.") diff --git a/pypeit/sensfunc.py b/pypeit/sensfunc.py index 85fc11dcfc..a405526896 100644 --- a/pypeit/sensfunc.py +++ b/pypeit/sensfunc.py @@ -189,20 +189,21 @@ def empty_sensfunc_table(norders, nspec, nspec_in, ncoeff=1): table.Column(name='SENS_FLUXED_STD_MASK', dtype=bool, length=norders, shape=(nspec_in,), description='The good pixel mask for the fluxed standard star spectrum ')]) - - # Superclass factory method generates the subclass instance @classmethod - def get_instance(cls, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False): + def get_instance(cls, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False, + chk_version=True): """ Instantiate the relevant subclass based on the algorithm provided in ``par``. """ return next(c for c in cls.__subclasses__() - if c.__name__ == f"{par['algorithm']}SensFunc")(spec1dfile, sensfile, par, - par_fluxcalib=par_fluxcalib, debug=debug) + if c.__name__ == f"{par['algorithm']}SensFunc")( + spec1dfile, sensfile, par, par_fluxcalib=par_fluxcalib, debug=debug, + chk_version=chk_version) - def __init__(self, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False): + def __init__(self, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False, + chk_version=True): # Instantiate as an empty DataContainer super().__init__() @@ -211,6 +212,7 @@ def __init__(self, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False): self.spec1df = spec1dfile self.sensfile = sensfile self.par = par + self.chk_version = chk_version # Spectrograph header = fits.getheader(self.spec1df) self.PYP_SPEC = header['PYP_SPEC'] @@ -220,7 +222,8 @@ def __init__(self, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False): # spectrograph objects with configuration specific information from # spec1d files. self.spectrograph.dispname = header['DISPNAME'] - self.par_fluxcalib = self.spectrograph.default_pypeit_par()['fluxcalib'] if par_fluxcalib is None else par_fluxcalib + self.par_fluxcalib = self.spectrograph.default_pypeit_par()['fluxcalib'] \ + if par_fluxcalib is None else par_fluxcalib # Set the algorithm in the datamodel self.algorithm = self.__class__._algorithm @@ -238,17 +241,21 @@ def __init__(self, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False): self.splice_multi_det = True if self.par['multi_spec_det'] is not None else False # Read in the Standard star data - self.sobjs_std = specobjs.SpecObjs.from_fitsfile(self.spec1df).get_std(multi_spec_det=self.par['multi_spec_det']) + self.sobjs_std = specobjs.SpecObjs.from_fitsfile( + self.spec1df, chk_version=self.chk_version).get_std( + multi_spec_det=self.par['multi_spec_det']) if self.sobjs_std is None: - msgs.error('There is a problem with your standard star spec1d file: {:s}'.format(self.spec1df)) + msgs.error(f'There is a problem with your standard star spec1d file: {self.spec1df}') # Unpack standard - wave, counts, counts_ivar, counts_mask, trace_spec, trace_spat, self.meta_spec, header = self.sobjs_std.unpack_object(ret_flam=False) + wave, counts, counts_ivar, counts_mask, trace_spec, trace_spat, self.meta_spec, header \ + = self.sobjs_std.unpack_object(ret_flam=False) # Compute the blaze function # TODO Make the blaze function optional - log10_blaze_function = self.compute_blaze(wave, trace_spec, trace_spat, par['flatfile']) if par['flatfile'] is not None else None + log10_blaze_function = self.compute_blaze(wave, trace_spec, trace_spat, par['flatfile']) \ + if par['flatfile'] is not None else None # Perform any instrument tweaks wave_twk, counts_twk, counts_ivar_twk, counts_mask_twk, log10_blaze_function_twk = \ @@ -269,7 +276,8 @@ def __init__(self, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False): star_mag=self.par['star_mag'], ra=star_ra, dec=star_dec) - def compute_blaze(self, wave, trace_spec, trace_spat, flatfile, box_radius=10.0, min_blaze_value=1e-3, debug=False): + def compute_blaze(self, wave, trace_spec, trace_spat, flatfile, box_radius=10.0, + min_blaze_value=1e-3, debug=False): """ Compute the blaze function from a flat field image. @@ -293,8 +301,7 @@ def compute_blaze(self, wave, trace_spec, trace_spat, flatfile, box_radius=10.0, `numpy.ndarray`_: The log10 blaze function. Shape = (nspec, norddet) if norddet > 1, else shape = (nspec,) """ - # TODO: Somehow pass chk_version here? - flatImages = flatfield.FlatImages.from_file(flatfile) + flatImages = flatfield.FlatImages.from_file(flatfile, chk_version=self.chk_version) pixelflat_raw = flatImages.pixelflat_raw pixelflat_norm = flatImages.pixelflat_norm @@ -308,7 +315,8 @@ def compute_blaze(self, wave, trace_spec, trace_spat, flatfile, box_radius=10.0, mask_box = (pixmsk != pixtot) & np.isfinite(wave) & (wave > 0.0) # TODO This is ugly and redundant with spec_atleast_2d, but the order of operations compels me to do it this way - blaze_function = (np.clip(flux_box*mask_box, 1e-3, 1e9)).reshape(-1,1) if flux_box.ndim == 1 else flux_box*mask_box + blaze_function = (np.clip(flux_box*mask_box, 1e-3, 1e9)).reshape(-1,1) \ + if flux_box.ndim == 1 else flux_box*mask_box wave_debug = wave.reshape(-1,1) if wave.ndim == 1 else wave log10_blaze_function = np.zeros_like(blaze_function) norddet = log10_blaze_function.shape[1] @@ -459,9 +467,11 @@ def flux_std(self): # TODO assign this to the data model # Unpack the fluxed standard - _wave, _flam, _flam_ivar, _flam_mask, _, _, _, _ = self.sobjs_std.unpack_object(ret_flam=True) + _wave, _flam, _flam_ivar, _flam_mask, _, _, _, _ \ + = self.sobjs_std.unpack_object(ret_flam=True) # Reshape to 2d arrays - wave, flam, flam_ivar, flam_mask, _, _, _ = utils.spec_atleast_2d(_wave, _flam, _flam_ivar, _flam_mask) + wave, flam, flam_ivar, flam_mask, _, _, _ \ + = utils.spec_atleast_2d(_wave, _flam, _flam_ivar, _flam_mask) # Store in the sens table self.sens['SENS_FLUXED_STD_WAVE'] = wave.T self.sens['SENS_FLUXED_STD_FLAM'] = flam.T @@ -787,7 +797,7 @@ def write_QA(self): @classmethod - def sensfunc_weights(cls, sensfile, waves, debug=False, extrap_sens=True): + def sensfunc_weights(cls, sensfile, waves, debug=False, extrap_sens=True, chk_version=True): """ Get the weights based on the sensfunc @@ -799,14 +809,17 @@ def sensfunc_weights(cls, sensfile, waves, debug=False, extrap_sens=True): norders, nexp) or (nspec, norders). debug (bool): default=False show the weights QA + chk_version (:obj:`bool`, optional): + When reading in existing files written by PypeIt, perform strict + version checking to ensure a valid file. If False, the code + will try to keep going, but this may lead to faults and quiet + failures. User beware! Returns: `numpy.ndarray`_: sensfunc weights evaluated on the input waves wavelength grid, shape = same as waves """ - # TODO: Somehow pass chk_version here? - sens = cls.from_file(sensfile) - # wave, zeropoint, meta_table, out_table, header_sens = sensfunc.SensFunc.load(sensfile) + sens = cls.from_file(sensfile, chk_version=chk_version) if waves.ndim == 2: nspec, norder = waves.shape diff --git a/pypeit/specutils/pypeit_loaders.py b/pypeit/specutils/pypeit_loaders.py index c26b095e67..c6fc2bce4a 100644 --- a/pypeit/specutils/pypeit_loaders.py +++ b/pypeit/specutils/pypeit_loaders.py @@ -150,7 +150,8 @@ def identify_pypeit_onespec(origin, *args, **kwargs): priority=10, dtype=SpectrumList, autogenerate_spectrumlist=False) -def pypeit_spec1d_loader(filename, extract=None, fluxed=True, strict=True, **kwargs): +def pypeit_spec1d_loader(filename, extract=None, fluxed=True, strict=True, chk_version=True, + **kwargs): """ Load spectra from a PypeIt spec1d file into a SpectrumList. @@ -172,6 +173,10 @@ def pypeit_spec1d_loader(filename, extract=None, fluxed=True, strict=True, **kwa raise an error (as would be done by the `specutils.SpectrumList`_ class). If False, wavelengths that are *not* monotonically increasing are masked in the construction of the returned `specutils.SpectrumList`_ object. + chk_version : :obj:`bool`, optional + When reading in existing files written by PypeIt, perform strict version + checking to ensure a valid file. If False, the code will try to keep + going, but this may lead to faults and quiet failures. User beware! kwargs : dict, optional **Ignored!** Used to catch spurious arguments passed to the base class that are ignored by this function. @@ -183,7 +188,7 @@ def pypeit_spec1d_loader(filename, extract=None, fluxed=True, strict=True, **kwa """ # Try to load the file and ignoring any version mismatch try: - sobjs = specobjs.SpecObjs.from_fitsfile(filename, chk_version=False) + sobjs = specobjs.SpecObjs.from_fitsfile(filename, chk_version=chk_version) except PypeItError: file_pypeit_version = astropy.io.fits.getval(filename, 'VERSPYP', 'PRIMARY') msgs.error(f'Unable to ingest {filename.name} using pypeit.specobjs module from your version ' @@ -222,7 +227,7 @@ def pypeit_spec1d_loader(filename, extract=None, fluxed=True, strict=True, **kwa extensions=["fits"], priority=10, dtype=Spectrum1D) -def pypeit_onespec_loader(filename, grid=False, strict=True, **kwargs): +def pypeit_onespec_loader(filename, grid=False, strict=True, chk_version=True, **kwargs): """ Load a spectrum from a PypeIt OneSpec file into a Spectrum1D object. @@ -238,6 +243,10 @@ def pypeit_onespec_loader(filename, grid=False, strict=True, **kwargs): raise an error (as would be done by the `specutils.Spectrum1D`_ class). If False, wavelengths that are *not* monotonically increasing are masked in the construction of the returned `specutils.Spectrum1D`_ object. + chk_version : :obj:`bool`, optional + When reading in existing files written by PypeIt, perform strict version + checking to ensure a valid file. If False, the code will try to keep + going, but this may lead to faults and quiet failures. User beware! kwargs : dict, optional **Ignored!** Used to catch spurious arguments passed to the base class that are ignored by this function. @@ -250,7 +259,7 @@ def pypeit_onespec_loader(filename, grid=False, strict=True, **kwargs): # Try to load the file and ignoring any version mismatch try: # TODO: Pass chk_version here? - spec = onespec.OneSpec.from_file(filename) + spec = onespec.OneSpec.from_file(filename, chk_version=chk_version) except PypeItError: file_pypeit_version = astropy.io.fits.getval(filename, 'VERSPYP', 'PRIMARY') msgs.error(f'Unable to ingest {filename.name} using pypeit.specobjs module from your version ' diff --git a/pypeit/wavetilts.py b/pypeit/wavetilts.py index fb70fbc98e..e7188afd13 100644 --- a/pypeit/wavetilts.py +++ b/pypeit/wavetilts.py @@ -173,7 +173,8 @@ def spatid_to_zero(self, spat_id): mtch = self.spat_id == spat_id return np.where(mtch)[0][0] - def show(self, waveimg=None, wcs_match=True, in_ginga=True, show_traces=False): + def show(self, waveimg=None, wcs_match=True, in_ginga=True, show_traces=False, + chk_version=True): """ Show in ginga or mpl Tiltimg with the tilts traced and fitted overlaid @@ -187,20 +188,24 @@ def show(self, waveimg=None, wcs_match=True, in_ginga=True, show_traces=False): If True, show the image in ginga. Otherwise, use matplotlib. show_traces (bool, optional): If True, show the traces of the tilts on the image. - + chk_version (:obj:`bool`, optional): + When reading in existing files written by PypeIt, perform strict + version checking to ensure a valid file. If False, the code + will try to keep going, but this may lead to faults and quiet + failures. User beware! """ # get tilt_img_dict if (Path(self.calib_dir).resolve() / self.tiltimg_filename).exists(): - # TODO: Somehow pass chk_version here? - tilt_img_dict = buildimage.TiltImage.from_file(Path(self.calib_dir).resolve() / self.tiltimg_filename) + cal_file = Path(self.calib_dir).resolve() / self.tiltimg_filename + tilt_img_dict = buildimage.TiltImage.from_file(cal_file, chk_version=chk_version) else: msgs.error(f'Tilt image {str((Path(self.calib_dir).resolve() / self.tiltimg_filename))} NOT FOUND.') # get slits slitmask = None if (Path(self.calib_dir).resolve() / self.slits_filename).exists(): - # TODO: Somehow pass chk_version here? - slits = slittrace.SlitTraceSet.from_file(Path(self.calib_dir).resolve() / self.slits_filename) + cal_file = Path(self.calib_dir).resolve() / self.slits_filename + slits = slittrace.SlitTraceSet.from_file(cal_file, chk_version=chk_version) _slitmask = slits.slit_img(initial=True, flexure=self.spat_flexure) _left, _right, _mask = slits.select_edges(flexure=self.spat_flexure) gpm = _mask == 0 @@ -217,8 +222,7 @@ def show(self, waveimg=None, wcs_match=True, in_ginga=True, show_traces=False): if waveimg is None and slits is not None and same_size and in_ginga: wv_calib_name = wavecalib.WaveCalib.construct_file_name(self.calib_key, calib_dir=self.calib_dir) if Path(wv_calib_name).resolve().exists(): - # TODO: Somehow pass chk_version here? - wv_calib = wavecalib.WaveCalib.from_file(wv_calib_name) + wv_calib = wavecalib.WaveCalib.from_file(wv_calib_name, chk_version=chk_version) tilts = self.fit2tiltimg(slitmask, flexure=self.spat_flexure) waveimg = wv_calib.build_waveimg(tilts, slits, spat_flexure=self.spat_flexure) else: From 749e09555049cd6a9e5debdf6871392af32c57fe Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 18 Jan 2024 17:15:01 -0800 Subject: [PATCH 198/244] clean up --- pypeit/core/telluric.py | 2 +- pypeit/specutils/pypeit_loaders.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index a66e587b1e..79d99160b1 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -1615,7 +1615,7 @@ def qso_telluric(spec1dfile, telgridfile, pca_file, z_qso, telloutfile, outfile fits. show : :obj:`bool`, optional Show a QA plot of the final fit. - chk_version (:obj:`bool`, optional): + chk_version : :obj:`bool`, optional When reading in existing files written by PypeIt, perform strict version checking to ensure a valid file. If False, the code will try to keep going, but this may lead to faults and quiet failures. User beware! diff --git a/pypeit/specutils/pypeit_loaders.py b/pypeit/specutils/pypeit_loaders.py index c6fc2bce4a..3b4aed3df8 100644 --- a/pypeit/specutils/pypeit_loaders.py +++ b/pypeit/specutils/pypeit_loaders.py @@ -258,7 +258,6 @@ def pypeit_onespec_loader(filename, grid=False, strict=True, chk_version=True, * """ # Try to load the file and ignoring any version mismatch try: - # TODO: Pass chk_version here? spec = onespec.OneSpec.from_file(filename, chk_version=chk_version) except PypeItError: file_pypeit_version = astropy.io.fits.getval(filename, 'VERSPYP', 'PRIMARY') From 11209496ab2f1fa014a290098af9d05a18eb0de5 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 18 Jan 2024 17:57:13 -0800 Subject: [PATCH 199/244] bug fix --- pypeit/sensfunc.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pypeit/sensfunc.py b/pypeit/sensfunc.py index a405526896..e7ccadad1b 100644 --- a/pypeit/sensfunc.py +++ b/pypeit/sensfunc.py @@ -128,7 +128,8 @@ class SensFunc(datamodel.DataContainer): 'steps', 'splice_multi_det', 'meta_spec', - 'std_dict' + 'std_dict', + 'chk_version' ] _algorithm = None @@ -1028,8 +1029,10 @@ class UVISSensFunc(SensFunc): _algorithm = 'UVIS' """Algorithm used for the sensitivity calculation.""" - def __init__(self, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False): - super().__init__(spec1dfile, sensfile, par, par_fluxcalib=par_fluxcalib, debug=debug) + def __init__(self, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False, + chk_version=True): + super().__init__(spec1dfile, sensfile, par, par_fluxcalib=par_fluxcalib, debug=debug, + chk_version=chk_version) # Add some cards to the meta spec. These should maybe just be added # already in unpack object From 40ad5e9ad1986dcacf35a1790b9f17ce283040d7 Mon Sep 17 00:00:00 2001 From: Dusty Reichwein Date: Thu, 18 Jan 2024 18:58:23 -0800 Subject: [PATCH 200/244] Fix test_collate_1d.py unit test failure because of the chk_version argument to from_fitsfile --- pypeit/tests/test_collate_1d.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pypeit/tests/test_collate_1d.py b/pypeit/tests/test_collate_1d.py index 3a4db0ae1f..de5a097fab 100644 --- a/pypeit/tests/test_collate_1d.py +++ b/pypeit/tests/test_collate_1d.py @@ -623,7 +623,9 @@ def mock_geomotion_correct(*args, **kwargs): # Test where one VEL_CORR is already set, and the SpecObj objects have no RA/DEC so the header RA/DEC must be used instead sobjs = MockSpecObjs("spec1d_file4") - monkeypatch.setattr(specobjs.SpecObjs, "from_fitsfile", lambda x: sobjs) + def mock_from_fitsfile(*args, **kwargs): + return sobjs + monkeypatch.setattr(specobjs.SpecObjs, "from_fitsfile", mock_from_fitsfile) refframe_correction(par, spectrograph, spec1d_files, spec1d_failure_msgs) assert len(spec1d_failure_msgs) == 1 From 2fd5a53bb4caab979f64955e72af926822d88189 Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 19 Jan 2024 10:20:45 +0000 Subject: [PATCH 201/244] Added new coadd options --- pypeit/core/coadd.py | 4 ++-- pypeit/core/datacube.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pypeit/core/coadd.py b/pypeit/core/coadd.py index 779b3c5673..0ac4ac9585 100644 --- a/pypeit/core/coadd.py +++ b/pypeit/core/coadd.py @@ -821,7 +821,7 @@ def sn_weights(fluxes, ivars, gpms, sn_smooth_npix=None, weight_method='auto', v wavelength dependent weights are not used. weight_method : str, optional - The weighting method to be used. Options are ``'auto'``, ``'constant'``, + The weighting method to be used. Options are ``'auto'``, ``'constant'``, ``'uniform'``, ``'wave_dependent'``, ``'relative'``, or ``'ivar'``. The default is ``'auto'``. Behavior is as follows: @@ -865,7 +865,7 @@ def sn_weights(fluxes, ivars, gpms, sn_smooth_npix=None, weight_method='auto', v vectors) provided in waves, i.e. it is a list of arrays of type `numpy.ndarray`_ with the same shape as those in waves. """ - if weight_method not in ['auto', 'constant', 'wave_dependent', 'relative', 'ivar']: + if weight_method not in ['auto', 'constant', 'uniform', 'wave_dependent', 'relative', 'ivar']: msgs.error('Unrecognized option for weight_method=%s').format(weight_method) nexp = len(fluxes) diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index 6050993672..2105e2d0a1 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -1049,7 +1049,7 @@ def compute_weights_frompix(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, scheme with a polynomial fit is used to calculate the S/N weights. weight_method : `str`, optional Weight method to be used in :func:`~pypeit.coadd.sn_weights`. - Options are ``'auto'``, ``'constant'``, ``'relative'``, or + Options are ``'auto'``, ``'constant'``, ``'uniform'``, ``'wave_dependent'``, ``'relative'``, or ``'ivar'``. The default is ``'auto'``. Behavior is as follows: - ``'auto'``: Use constant weights if rms_sn < 3.0, otherwise @@ -1173,7 +1173,7 @@ def compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, scheme with a polynomial fit is used to calculate the S/N weights. weight_method : `str`, optional Weight method to be used in :func:`~pypeit.coadd.sn_weights`. - Options are ``'auto'``, ``'constant'``, ``'relative'``, or + Options are ``'auto'``, ``'constant'``, ``'uniform'``, ``'wave_dependent'``, ``'relative'``, or ``'ivar'``. The default is ``'auto'``. Behavior is as follows: - ``'auto'``: Use constant weights if rms_sn < 3.0, otherwise From 1ac5d96bb67993bc610bc1a7f38dfb62abef5a71 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Fri, 19 Jan 2024 08:18:38 -0800 Subject: [PATCH 202/244] doc update + bug fix --- doc/help/pypeit_chk_alignments.rst | 4 +- doc/help/pypeit_chk_noise_1dspec.rst | 3 + doc/help/pypeit_chk_noise_2dspec.rst | 4 +- doc/help/pypeit_chk_scattlight.rst | 3 + doc/help/pypeit_chk_wavecalib.rst | 4 +- doc/help/pypeit_collate_1d.rst | 12 +- doc/help/pypeit_edge_inspector.rst | 4 +- doc/help/pypeit_flux_calib.rst | 5 +- doc/help/pypeit_identify.rst | 3 + doc/help/pypeit_parse_slits.rst | 4 +- doc/help/pypeit_ql.rst | 4 +- doc/help/pypeit_show_2dspec.rst | 4 +- doc/help/pypeit_show_wvcalib.rst | 4 +- doc/help/pypeit_skysub_regions.rst | 3 + doc/help/pypeit_tellfit.rst | 5 + doc/help/run_pypeit.rst | 2 +- doc/include/class_datamodel_onespec.rst | 3 +- doc/include/class_datamodel_slittraceset.rst | 3 +- doc/include/class_datamodel_telluric.rst | 4 +- doc/include/datamodel_onespec.rst | 2 +- doc/include/datamodel_sensfunc.rst | 47 ++-- doc/include/datamodel_slittraceset.rst | 2 +- doc/include/datamodel_telluric.rst | 47 ++-- doc/pypeit_par.rst | 242 ++++++++++--------- pypeit/scripts/collate_1d.py | 5 +- 25 files changed, 240 insertions(+), 183 deletions(-) diff --git a/doc/help/pypeit_chk_alignments.rst b/doc/help/pypeit_chk_alignments.rst index a68e12c7ca..2f58ffa5b7 100644 --- a/doc/help/pypeit_chk_alignments.rst +++ b/doc/help/pypeit_chk_alignments.rst @@ -1,7 +1,7 @@ .. code-block:: console $ pypeit_chk_alignments -h - usage: pypeit_chk_alignments [-h] [--chname CHNAME] file + usage: pypeit_chk_alignments [-h] [--chname CHNAME] [--try_old] file Display Alignment image and the trace data @@ -11,4 +11,6 @@ options: -h, --help show this help message and exit --chname CHNAME Channel name for image in Ginga (default: Alignments) + --try_old Attempt to load old datamodel versions. A crash may ensue.. + (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_chk_noise_1dspec.rst b/doc/help/pypeit_chk_noise_1dspec.rst index 80fa4e2613..51b54fbfaf 100644 --- a/doc/help/pypeit_chk_noise_1dspec.rst +++ b/doc/help/pypeit_chk_noise_1dspec.rst @@ -6,6 +6,7 @@ [--z [Z ...]] [--maskdef_objname MASKDEF_OBJNAME] [--pypeit_name PYPEIT_NAME] [--wavemin WAVEMIN] [--wavemax WAVEMAX] [--plot_or_save PLOT_OR_SAVE] + [--try_old] [files ...] Examine the noise in a PypeIt spectrum @@ -41,4 +42,6 @@ window. If you choose save, a folder called spec1d*_noisecheck will be created and all the relevant plot will be placed there. (default: plot) + --try_old Attempt to load old datamodel versions. A crash may + ensue.. (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_chk_noise_2dspec.rst b/doc/help/pypeit_chk_noise_2dspec.rst index 7fc7fd75d5..d0a6662125 100644 --- a/doc/help/pypeit_chk_noise_2dspec.rst +++ b/doc/help/pypeit_chk_noise_2dspec.rst @@ -5,7 +5,7 @@ [--maskdef_id MASKDEF_ID] [--pypeit_id PYPEIT_ID] [--pad PAD] [--aspect_ratio ASPECT_RATIO] [--wavemin WAVEMIN] [--wavemax WAVEMAX] - [--mode MODE] [--list] + [--mode MODE] [--list] [--try_old] [files ...] Examine the noise in a PypeIt slit/order @@ -43,4 +43,6 @@ placed. "print" will cause the check noise values to be printed in the terminal. (default: plot) --list List the extensions only? (default: False) + --try_old Attempt to load old datamodel versions. A crash may + ensue.. (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_chk_scattlight.rst b/doc/help/pypeit_chk_scattlight.rst index 73f9cddab3..08cd077c40 100644 --- a/doc/help/pypeit_chk_scattlight.rst +++ b/doc/help/pypeit_chk_scattlight.rst @@ -2,6 +2,7 @@ $ pypeit_chk_scattlight -h usage: pypeit_chk_scattlight [-h] [--spec2d SPEC2D] [--det DET] [--mask MASK] + [--try_old] file slits Display the scattered light image in a Ginga viewer @@ -21,4 +22,6 @@ --mask MASK If True, the detector pixels that are considered on the slit will be masked to highlight the scattered light regions (default: False) + --try_old Attempt to load old datamodel versions. A crash may ensue.. + (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_chk_wavecalib.rst b/doc/help/pypeit_chk_wavecalib.rst index 51d396d998..fe49166847 100644 --- a/doc/help/pypeit_chk_wavecalib.rst +++ b/doc/help/pypeit_chk_wavecalib.rst @@ -1,7 +1,7 @@ .. code-block:: console $ pypeit_chk_wavecalib -h - usage: pypeit_chk_wavecalib [-h] input_file [input_file ...] + usage: pypeit_chk_wavecalib [-h] [--try_old] input_file [input_file ...] Print QA on Wavelength Calib to the screen @@ -11,4 +11,6 @@ options: -h, --help show this help message and exit + --try_old Attempt to load old datamodel versions. A crash may ensue.. + (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_collate_1d.rst b/doc/help/pypeit_collate_1d.rst index d61fa3a918..1054c1ee55 100644 --- a/doc/help/pypeit_collate_1d.rst +++ b/doc/help/pypeit_collate_1d.rst @@ -36,8 +36,6 @@ value are skipped, else all wavelength rms values are accepted. refframe Perform reference frame correction prior to coadding. Options are ['observed', 'heliocentric', 'barycentric']. Defaults to None. - chk_version If true, spec1ds and archival sensfuncs must match the currently - supported versions. If false (the default) version numbers are not checked. spec1d read @@ -88,8 +86,14 @@ --refframe {observed,heliocentric,barycentric} Perform reference frame correction prior to coadding. Options are: observed, heliocentric, barycentric - --chk_version Whether to check the data model versions of spec1d files - and sensfunc files. + --chk_version If True enforce strict PypeIt version checking to ensure + that all files were created with the current version of + PypeIt. If set to False, the code will attempt to read + out-of-date files and keep going. Beware (!!) that this + can lead to unforeseen bugs that either cause the code + to crash or lead to erroneous results. I.e., you really + need to know what you are doing if you set this to + False! -v VERBOSITY, --verbosity VERBOSITY Verbosity level between 0 [none] and 2 [all]. Default: 1. Level 2 writes a log with filename diff --git a/doc/help/pypeit_edge_inspector.rst b/doc/help/pypeit_edge_inspector.rst index f975fedd18..ca482dfafa 100644 --- a/doc/help/pypeit_edge_inspector.rst +++ b/doc/help/pypeit_edge_inspector.rst @@ -1,7 +1,7 @@ .. code-block:: console $ pypeit_edge_inspector -h - usage: pypeit_edge_inspector [-h] trace_file + usage: pypeit_edge_inspector [-h] [--try_old] trace_file Interactively inspect/edit slit edge traces @@ -10,4 +10,6 @@ options: -h, --help show this help message and exit + --try_old Attempt to load old datamodel versions. A crash may ensue.. + (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_flux_calib.rst b/doc/help/pypeit_flux_calib.rst index 9bdb6d363f..162dab294c 100644 --- a/doc/help/pypeit_flux_calib.rst +++ b/doc/help/pypeit_flux_calib.rst @@ -1,7 +1,8 @@ .. code-block:: console $ pypeit_flux_calib -h - usage: pypeit_flux_calib [-h] [--par_outfile] [-v VERBOSITY] flux_file + usage: pypeit_flux_calib [-h] [--par_outfile] [-v VERBOSITY] [--try_old] + flux_file Flux calibrate 1D spectra produced by PypeIt @@ -53,4 +54,6 @@ Verbosity level between 0 [none] and 2 [all]. Default: 1. Level 2 writes a log with filename flux_calib_YYYYMMDD-HHMM.log + --try_old Attempt to load old datamodel versions. A crash may + ensue.. \ No newline at end of file diff --git a/doc/help/pypeit_identify.rst b/doc/help/pypeit_identify.rst index da9e7a8bab..51874f8498 100644 --- a/doc/help/pypeit_identify.rst +++ b/doc/help/pypeit_identify.rst @@ -5,6 +5,7 @@ [--slit SLIT] [--det DET] [--rmstol RMSTOL] [--fwhm FWHM] [--sigdetect SIGDETECT] [--pixtol PIXTOL] [--linear] [--force_save] [--rescale_resid] [-v VERBOSITY] + [--try_old] arc_file slits_file Launch PypeIt identify tool, display extracted Arc, and load linelist. @@ -38,4 +39,6 @@ Verbosity level between 0 [none] and 2 [all]. Default: 1. Level 2 writes a log with filename identify_YYYYMMDD- HHMM.log (default: 1) + --try_old Attempt to load old datamodel versions. A crash may + ensue.. (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_parse_slits.rst b/doc/help/pypeit_parse_slits.rst index f58c4230f4..19dbd2c1dd 100644 --- a/doc/help/pypeit_parse_slits.rst +++ b/doc/help/pypeit_parse_slits.rst @@ -1,7 +1,7 @@ .. code-block:: console $ pypeit_parse_slits -h - usage: pypeit_parse_slits [-h] input_file + usage: pypeit_parse_slits [-h] [--try_old] input_file Print info on slits from a input file @@ -10,4 +10,6 @@ options: -h, --help show this help message and exit + --try_old Attempt to load old datamodel versions. A crash may ensue.. + (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_ql.rst b/doc/help/pypeit_ql.rst index 609095f598..5e84f548a1 100644 --- a/doc/help/pypeit_ql.rst +++ b/doc/help/pypeit_ql.rst @@ -11,7 +11,7 @@ [--ignore_std] [--skip_display] [--coadd2d] [--only_slits ONLY_SLITS [ONLY_SLITS ...]] [--offsets OFFSETS] [--weights WEIGHTS] [--spec_samp_fact SPEC_SAMP_FACT] - [--spat_samp_fact SPAT_SAMP_FACT] + [--spat_samp_fact SPAT_SAMP_FACT] [--try_old] spectrograph Script to produce quick-look PypeIt reductions @@ -133,4 +133,6 @@ If coadding, adjust the spatial grid sampling by this factor. For a finer grid, set value to <1.0; for coarser sampling, set value to >1.0). (default: 1.0) + --try_old Attempt to load old datamodel versions. A crash may + ensue.. (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_show_2dspec.rst b/doc/help/pypeit_show_2dspec.rst index 77d84129bd..a4362b431b 100644 --- a/doc/help/pypeit_show_2dspec.rst +++ b/doc/help/pypeit_show_2dspec.rst @@ -5,7 +5,7 @@ [--maskID MASKID] [--showmask] [--removetrace] [--embed] [--ignore_extract_mask] [--channels CHANNELS] [--prefix PREFIX] [--no_clear] - [-v VERBOSITY] + [-v VERBOSITY] [--try_old] file Display sky subtracted, spec2d image in a ginga viewer. @@ -38,4 +38,6 @@ -v VERBOSITY, --verbosity VERBOSITY Verbosity level between 0 [none] and 2 [all] (default: 1) + --try_old Attempt to load old datamodel versions. A crash may + ensue.. (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_show_wvcalib.rst b/doc/help/pypeit_show_wvcalib.rst index 43684e9cc2..b993d3fd27 100644 --- a/doc/help/pypeit_show_wvcalib.rst +++ b/doc/help/pypeit_show_wvcalib.rst @@ -1,7 +1,7 @@ .. code-block:: console $ pypeit_show_wvcalib -h - usage: pypeit_show_wvcalib [-h] [--slit_file SLIT_FILE] [--is_order] + usage: pypeit_show_wvcalib [-h] [--slit_file SLIT_FILE] [--is_order] [--try_old] file slit_order Show the result of wavelength calibration @@ -15,4 +15,6 @@ --slit_file SLIT_FILE Slit file (default: None) --is_order Input slit/order is an order (default: False) + --try_old Attempt to load old datamodel versions. A crash may + ensue.. (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_skysub_regions.rst b/doc/help/pypeit_skysub_regions.rst index 92b9b0ff8b..99e96054be 100644 --- a/doc/help/pypeit_skysub_regions.rst +++ b/doc/help/pypeit_skysub_regions.rst @@ -2,6 +2,7 @@ $ pypeit_skysub_regions -h usage: pypeit_skysub_regions [-h] [--det DET] [-o] [-i] [-f] [-s] [-v VERBOSITY] + [--try_old] file Display a spec2d frame and interactively define the sky regions using a GUI. Run @@ -22,4 +23,6 @@ Verbosity level between 0 [none] and 2 [all]. Default: 1. Level 2 writes a log with filename skysub_regions_YYYYMMDD-HHMM.log (default: 1) + --try_old Attempt to load old datamodel versions. A crash may + ensue.. (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_tellfit.rst b/doc/help/pypeit_tellfit.rst index 28158d4068..134ba35556 100644 --- a/doc/help/pypeit_tellfit.rst +++ b/doc/help/pypeit_tellfit.rst @@ -4,6 +4,7 @@ usage: pypeit_tellfit [-h] [--objmodel {qso,star,poly}] [-r REDSHIFT] [-g TELL_GRID] [-p PCA_FILE] [-t TELL_FILE] [--debug] [--plot] [--par_outfile PAR_OUTFILE] [-v VERBOSITY] + [--chk_version] spec1dfile Telluric correct a spectrum @@ -74,4 +75,8 @@ Verbosity level between 0 [none] and 2 [all]. Default: 1. Level 2 writes a log with filename tellfit_YYYYMMDD- HHMM.log + --chk_version Ensure the datamodels are from the current PypeIt + version. By default (consistent with previous + functionality) this is not enforced and crashes may + ensue ... \ No newline at end of file diff --git a/doc/help/run_pypeit.rst b/doc/help/run_pypeit.rst index 6a5293f4c7..f65bce9629 100644 --- a/doc/help/run_pypeit.rst +++ b/doc/help/run_pypeit.rst @@ -4,7 +4,7 @@ usage: run_pypeit [-h] [-v VERBOSITY] [-r REDUX_PATH] [-m] [-s] [-o] [-c] pypeit_file - ## PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.14.1.dev477+g4a02a85ca + ## PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.14.1.dev696+g00535ea2e.d20240117 ## ## Available spectrographs include: ## bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, diff --git a/doc/include/class_datamodel_onespec.rst b/doc/include/class_datamodel_onespec.rst index eff409ec4d..a773421d33 100644 --- a/doc/include/class_datamodel_onespec.rst +++ b/doc/include/class_datamodel_onespec.rst @@ -1,5 +1,5 @@ -**Version**: 1.0.1 +**Version**: 1.0.2 ================= ================ ================= ========================================================================================================================================== Attribute Type Array Type Description @@ -11,6 +11,7 @@ Attribute Type Array Type Description ``ivar`` `numpy.ndarray`_ `numpy.floating`_ Inverse variance array (matches units of flux) ``mask`` `numpy.ndarray`_ `numpy.integer`_ Mask array (1=Good,0=Bad) ``obj_model`` `numpy.ndarray`_ `numpy.floating`_ Object model for tellurics +``sigma`` `numpy.ndarray`_ `numpy.floating`_ One sigma noise array, equivalent to 1/sqrt(ivar) (matches units of flux) ``spect_meta`` dict header dict ``telluric`` `numpy.ndarray`_ `numpy.floating`_ Telluric model ``wave`` `numpy.ndarray`_ `numpy.floating`_ Wavelength array (angstroms in vacuum), weighted by pixel contributions diff --git a/doc/include/class_datamodel_slittraceset.rst b/doc/include/class_datamodel_slittraceset.rst index f245b38ad4..8b3f4e5bb9 100644 --- a/doc/include/class_datamodel_slittraceset.rst +++ b/doc/include/class_datamodel_slittraceset.rst @@ -1,5 +1,5 @@ -**Version**: 1.1.4 +**Version**: 1.1.5 ===================== ============================ ===================== ==================================================================================================================================================== Attribute Type Array Type Description @@ -28,7 +28,6 @@ Attribute Type Array Type Desc ``pypeline`` str PypeIt pypeline name ``right_init`` `numpy.ndarray`_ `numpy.floating`_ Spatial coordinates (pixel indices) of all right edges, one per slit. Derived from the TraceImage. Shape is Nspec by Nslits. ``right_tweak`` `numpy.ndarray`_ `numpy.floating`_ Spatial coordinates (pixel indices) of all right edges, one per slit. These traces have been adjusted by the flat-field. Shape is Nspec by Nslits. -``slitbitm`` str List of BITMASK keys from SlitTraceBitMask ``spat_id`` `numpy.ndarray`_ int, `numpy.integer`_ Slit ID number from SPAT measured at half way point. ``specmax`` `numpy.ndarray`_ `numpy.floating`_ Maximum spectral position (pixel units) allowed for each slit/order. Shape is Nslits. ``specmin`` `numpy.ndarray`_ `numpy.floating`_ Minimum spectral position (pixel units) allowed for each slit/order. Shape is Nslits. diff --git a/doc/include/class_datamodel_telluric.rst b/doc/include/class_datamodel_telluric.rst index 7143c793ac..f9f749f007 100644 --- a/doc/include/class_datamodel_telluric.rst +++ b/doc/include/class_datamodel_telluric.rst @@ -20,8 +20,10 @@ Attribute Type Array Type Description ``std_name`` str Type of standard source ``std_ra`` float RA of the standard source ``std_src`` str Name of the standard source -``telgrid`` str File containing grid of HITRAN atmosphere models +``telgrid`` str File containing PCA components or grid of HITRAN atmosphere models ``tell_norm_thresh`` float ?? +``tell_npca`` int Number of telluric PCA components used +``teltype`` str Type of telluric model, `pca` or `grid` ``tol`` float Relative tolerance for converage of the differential evolution optimization. ``ubound_norm`` float Flux normalization upper bound ``z_qso`` float Redshift of the QSO diff --git a/doc/include/datamodel_onespec.rst b/doc/include/datamodel_onespec.rst index 6eac2e340a..302e3a1fe5 100644 --- a/doc/include/datamodel_onespec.rst +++ b/doc/include/datamodel_onespec.rst @@ -1,5 +1,5 @@ -Version 1.0.1 +Version 1.0.2 ============ ============================== ========= =========================================================== HDU Name HDU Type Data Type Description diff --git a/doc/include/datamodel_sensfunc.rst b/doc/include/datamodel_sensfunc.rst index 2d9e9b5bf2..165d92e3b9 100644 --- a/doc/include/datamodel_sensfunc.rst +++ b/doc/include/datamodel_sensfunc.rst @@ -15,31 +15,28 @@ HDU Name HDU Type Data Type Description TELLURIC table (if present) -================= ========= =========================================================== -Column Data Type Description -================= ========= =========================================================== -``WAVE`` float64 Wavelength vector -``TELLURIC`` float64 Best-fitting telluric model spectrum -``OBJ_MODEL`` float64 Best-fitting object model spectrum -``TELL_THETA`` float64 Best-fitting telluric model parameters -``TELL_PRESS`` float64 Best-fitting telluric model pressure -``TELL_TEMP`` float64 Best-fitting telluric model temperature -``TELL_H2O`` float64 Best-fitting telluric model humidity -``TELL_AIRMASS`` float64 Best-fitting telluric model airmass -``TELL_RESLN`` float64 Best-fitting telluric model spectral resolution -``TELL_SHIFT`` float64 Best-fitting shift applied to the telluric model spectrum -``TELL_STRETCH`` float64 Best-fitting stretch applied to the telluric model spectrum -``OBJ_THETA`` float64 Best-fitting object model parameters -``CHI2`` float64 Chi-square of the best-fit model -``SUCCESS`` bool Flag that fit was successful -``NITER`` int64 Number of fit iterations -``ECH_ORDERS`` int64 Echelle order for this specrum (echelle data only) -``POLYORDER_VEC`` int64 Polynomial order for each slit/echelle (if applicable) -``IND_LOWER`` int64 Lowest index of a spectral pixel included in the fit -``IND_UPPER`` int64 Highest index of a spectral pixel included in the fit -``WAVE_MIN`` float64 Minimum wavelength included in the fit -``WAVE_MAX`` float64 Maximum wavelength included in the fit -================= ========= =========================================================== +================= ========= ================================================================ +Column Data Type Description +================= ========= ================================================================ +``WAVE`` float64 Wavelength vector +``TELLURIC`` float64 Best-fitting telluric model spectrum +``OBJ_MODEL`` float64 Best-fitting object model spectrum +``TELL_THETA`` float64 Best-fitting telluric model parameters +``TELL_PARAM`` float64 Best-fitting telluric atmospheric parameters or PCA coefficients +``TELL_RESLN`` float64 Best-fitting telluric model spectral resolution +``TELL_SHIFT`` float64 Best-fitting shift applied to the telluric model spectrum +``TELL_STRETCH`` float64 Best-fitting stretch applied to the telluric model spectrum +``OBJ_THETA`` float64 Best-fitting object model parameters +``CHI2`` float64 Chi-square of the best-fit model +``SUCCESS`` bool Flag that fit was successful +``NITER`` int64 Number of fit iterations +``ECH_ORDERS`` int64 Echelle order for this specrum (echelle data only) +``POLYORDER_VEC`` int64 Polynomial order for each slit/echelle (if applicable) +``IND_LOWER`` int64 Lowest index of a spectral pixel included in the fit +``IND_UPPER`` int64 Highest index of a spectral pixel included in the fit +``WAVE_MIN`` float64 Minimum wavelength included in the fit +``WAVE_MAX`` float64 Maximum wavelength included in the fit +================= ========= ================================================================ SENS table diff --git a/doc/include/datamodel_slittraceset.rst b/doc/include/datamodel_slittraceset.rst index e542c3e310..afd6996d75 100644 --- a/doc/include/datamodel_slittraceset.rst +++ b/doc/include/datamodel_slittraceset.rst @@ -1,5 +1,5 @@ -Version 1.1.4 +Version 1.1.5 ===================== ============================== ========= ============================================================================================================= HDU Name HDU Type Data Type Description diff --git a/doc/include/datamodel_telluric.rst b/doc/include/datamodel_telluric.rst index aac5007b4e..d22cbf732c 100644 --- a/doc/include/datamodel_telluric.rst +++ b/doc/include/datamodel_telluric.rst @@ -11,28 +11,25 @@ HDU Name HDU Type Data Type Description TELLURIC table -================= ========= =========================================================== -Column Data Type Description -================= ========= =========================================================== -``WAVE`` float64 Wavelength vector -``TELLURIC`` float64 Best-fitting telluric model spectrum -``OBJ_MODEL`` float64 Best-fitting object model spectrum -``TELL_THETA`` float64 Best-fitting telluric model parameters -``TELL_PRESS`` float64 Best-fitting telluric model pressure -``TELL_TEMP`` float64 Best-fitting telluric model temperature -``TELL_H2O`` float64 Best-fitting telluric model humidity -``TELL_AIRMASS`` float64 Best-fitting telluric model airmass -``TELL_RESLN`` float64 Best-fitting telluric model spectral resolution -``TELL_SHIFT`` float64 Best-fitting shift applied to the telluric model spectrum -``TELL_STRETCH`` float64 Best-fitting stretch applied to the telluric model spectrum -``OBJ_THETA`` float64 Best-fitting object model parameters -``CHI2`` float64 Chi-square of the best-fit model -``SUCCESS`` bool Flag that fit was successful -``NITER`` int64 Number of fit iterations -``ECH_ORDERS`` int64 Echelle order for this specrum (echelle data only) -``POLYORDER_VEC`` int64 Polynomial order for each slit/echelle (if applicable) -``IND_LOWER`` int64 Lowest index of a spectral pixel included in the fit -``IND_UPPER`` int64 Highest index of a spectral pixel included in the fit -``WAVE_MIN`` float64 Minimum wavelength included in the fit -``WAVE_MAX`` float64 Maximum wavelength included in the fit -================= ========= =========================================================== +================= ========= ================================================================ +Column Data Type Description +================= ========= ================================================================ +``WAVE`` float64 Wavelength vector +``TELLURIC`` float64 Best-fitting telluric model spectrum +``OBJ_MODEL`` float64 Best-fitting object model spectrum +``TELL_THETA`` float64 Best-fitting telluric model parameters +``TELL_PARAM`` float64 Best-fitting telluric atmospheric parameters or PCA coefficients +``TELL_RESLN`` float64 Best-fitting telluric model spectral resolution +``TELL_SHIFT`` float64 Best-fitting shift applied to the telluric model spectrum +``TELL_STRETCH`` float64 Best-fitting stretch applied to the telluric model spectrum +``OBJ_THETA`` float64 Best-fitting object model parameters +``CHI2`` float64 Chi-square of the best-fit model +``SUCCESS`` bool Flag that fit was successful +``NITER`` int64 Number of fit iterations +``ECH_ORDERS`` int64 Echelle order for this specrum (echelle data only) +``POLYORDER_VEC`` int64 Polynomial order for each slit/echelle (if applicable) +``IND_LOWER`` int64 Lowest index of a spectral pixel included in the fit +``IND_UPPER`` int64 Highest index of a spectral pixel included in the fit +``WAVE_MIN`` float64 Minimum wavelength included in the fit +``WAVE_MAX`` float64 Maximum wavelength included in the fit +================= ========= ================================================================ diff --git a/doc/pypeit_par.rst b/doc/pypeit_par.rst index da16794add..548f0e5d8a 100644 --- a/doc/pypeit_par.rst +++ b/doc/pypeit_par.rst @@ -506,40 +506,40 @@ Coadd1DPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.Coadd1DPar` -==================== ========== ======= ========== ========================================================================================================================================================================================================================================================================================================================================================================================================================================= -Key Type Options Default Description -==================== ========== ======= ========== ========================================================================================================================================================================================================================================================================================================================================================================================================================================= -``chk_version`` bool .. True If True enforce strict PypeIt version checking to ensure that spec1d*.fits files were createdwith the current version of PypeIt -``coaddfile`` str .. .. Output filename -``dloglam`` int, float .. .. Dispersion in units of log10(wave) in case you want to specify it in the get_wave_grid (for the 'velocity' or 'log10' options), otherwise a median value is computed from the data. -``dv`` int, float .. .. Dispersion in units of km/s in case you want to specify it in the get_wave_grid (for the 'velocity' option), otherwise a median value is computed from the data. -``dwave`` int, float .. .. Dispersion in Angstroms in case you want to specify it in the get_wave_grid (for the 'linear' option), otherwise a median value is computed from the data. -``ex_value`` str .. ``OPT`` The extraction to coadd, i.e. optimal or boxcar. Must be either 'OPT' or 'BOX' -``filter`` str .. ``none`` Filter for scaling. See flux_calib.load_fitler_file() for naming. Ignore if none -``filter_mag`` float .. .. Magnitude of the source in the given filter -``filter_mask`` str, list .. .. List of wavelength regions to mask when doing the scaling (`i.e.`, occasional junk pixels). Colon and comma separateed, e.g. 5552:5559,6010:6030 -``flux_value`` bool .. True If True (default), the code will coadd the fluxed spectra (i.e. the FLAM) in the spec1d files. If False, it will coadd the counts. -``lower`` int, float .. 3.0 Lower rejection threshold used for rejecting pixels when combining spectra in units of sigma. -``mag_type`` str .. ``AB`` Magnitude type. AB is the only option currently allowed -``maxiter_reject`` int .. 5 Maximum number of iterations for stacking and rejection. The code stops iterating either when the output mask does not change betweeen successive iterations or when maxiter_reject is reached. -``maxiter_scale`` int .. 5 Maximum number of iterations performed for rescaling spectra. -``maxrej`` int .. .. Coadding performs iterative rejection by comparing each exposure to a preliminary stack of all the exposures. If this parameter is set then it will not reject more than maxrej pixels per iteration of this rejection. The default is None, which means no maximum on rejected pixels. -``nbests`` list, int .. .. Number of orders to use for estimating the per exposure weights. Default is None, which will just use one fourth of the total number of orders. This is only used for Echelle -``nmaskedge`` int .. 2 Number of edge pixels to mask. This should be removed/fixed. -``ref_percentile`` int, float .. 70.0 Percentile used for selecting the minimum SNR cut from a reference spectrum used to robustly determine the median ratio between spectra. This parameter is used by coadd1d.robust_median_ratio as part of the automatic rescaling procedure. Pixels above this percentile cut are deemed the "good" pixels and are used to compute the ratio of two spectra. This must be a number between 0 and 100. -``scale_method`` str .. ``auto`` Method used to rescale the spectra prior to coadding. The options are: 'auto' -- Determine the scaling method automatically based on the S/N ratio which works well. 'poly' -- Polynomial rescaling. 'median' -- Median rescaling 'none' -- Do not rescale. 'hand' -- Pass in hand scaling factors. This option is not well tested. -``sigrej_exp`` int, float .. .. Rejection threshold used for rejecting exposures with S/N more than sigrej_exp*sigma above the median S/N. If None (the default), no rejection is performed. Currently, only available for multi-slit observations. -``sigrej_scale`` int, float .. 3.0 Rejection threshold used for rejecting pixels when rescaling spectra with scale_spec. -``sn_clip`` int, float .. 30.0 Errors are capped during rejection so that the S/N is never greater than sn_clip. This prevents overly aggressive rejection in high S/N ratio spectrum which neverthless differ at a level greater than the formal S/N due to systematics. -``sn_min_medscale`` int, float .. 0.5 For scale method set to ``auto``, this sets the minimum SNR for which median scaling is attempted. -``sn_min_polyscale`` int, float .. 2.0 For scale method set to ``auto``, this sets the minimum SNR for which polynomial scaling is attempted. -``sn_smooth_npix`` int, float .. .. Number of pixels to median filter by when computing S/N used to decide how to scale and weight spectra. If set to None (default), the code will determine the effective number of good pixels per spectrum in the stack that is being co-added and use 10% of this neff. -``spec_samp_fact`` float .. 1.0 Make the wavelength grid sampling finer (spec_samp_fact < 1.0) or coarser (spec_samp_fact > 1.0) by this sampling factor. This basically multiples the 'native' spectral pixels by spec_samp_fact, i.e. units spec_samp_fact are pixels. -``upper`` int, float .. 3.0 Upper rejection threshold used for rejecting pixels when combining spectra in units of sigma. -``wave_grid_max`` int, float .. .. Used in case you want to specify the maximum wavelength in your wavelength grid, default=None computes from data -``wave_grid_min`` int, float .. .. Used in case you want to specify the minimum wavelength in your wavelength grid, default=None computes from data -``wave_method`` str .. ``linear`` Method used to construct wavelength grid for coadding spectra. The routine that creates the wavelength is :func:`~pypeit.core.wavecal.wvutils.get_wave_grid`. The options are: 'iref' -- Use the first wavelength array. 'velocity' -- Grid is uniform in velocity. 'log10' -- Grid is uniform in log10(wave). This is the same as velocity. 'linear' -- Grid is uniform in lambda. 'concatenate' -- Meld the input wavelength arrays -==================== ========== ======= ========== ========================================================================================================================================================================================================================================================================================================================================================================================================================================= +==================== ========== ======= ========== ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= +Key Type Options Default Description +==================== ========== ======= ========== ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= +``coaddfile`` str .. .. Output filename +``dloglam`` int, float .. .. Dispersion in units of log10(wave) in case you want to specify it in the get_wave_grid (for the 'velocity' or 'log10' options), otherwise a median value is computed from the data. +``dv`` int, float .. .. Dispersion in units of km/s in case you want to specify it in the get_wave_grid (for the 'velocity' option), otherwise a median value is computed from the data. +``dwave`` int, float .. .. Dispersion in Angstroms in case you want to specify it in the get_wave_grid (for the 'linear' option), otherwise a median value is computed from the data. +``ex_value`` str .. ``OPT`` The extraction to coadd, i.e. optimal or boxcar. Must be either 'OPT' or 'BOX' +``filter`` str .. ``none`` Filter for scaling. See flux_calib.load_fitler_file() for naming. Ignore if none +``filter_mag`` float .. .. Magnitude of the source in the given filter +``filter_mask`` str, list .. .. List of wavelength regions to mask when doing the scaling (`i.e.`, occasional junk pixels). Colon and comma separateed, e.g. 5552:5559,6010:6030 +``flux_value`` bool .. True If True (default), the code will coadd the fluxed spectra (i.e. the FLAM) in the spec1d files. If False, it will coadd the counts. +``lower`` int, float .. 3.0 Lower rejection threshold used for rejecting pixels when combining spectra in units of sigma. +``mag_type`` str .. ``AB`` Magnitude type. AB is the only option currently allowed +``maxiter_reject`` int .. 5 Maximum number of iterations for stacking and rejection. The code stops iterating either when the output mask does not change betweeen successive iterations or when maxiter_reject is reached. +``maxiter_scale`` int .. 5 Maximum number of iterations performed for rescaling spectra. +``maxrej`` int .. .. Coadding performs iterative rejection by comparing each exposure to a preliminary stack of all the exposures. If this parameter is set then it will not reject more than maxrej pixels per iteration of this rejection. The default is None, which means no maximum on rejected pixels. +``nbests`` list, int .. .. Number of orders to use for estimating the per exposure weights. Default is None, which will just use one fourth of the total number of orders. This is only used for Echelle +``nmaskedge`` int .. 2 Number of edge pixels to mask. This should be removed/fixed. +``ref_percentile`` int, float .. 70.0 Percentile used for selecting the minimum SNR cut from a reference spectrum used to robustly determine the median ratio between spectra. This parameter is used by coadd1d.robust_median_ratio as part of the automatic rescaling procedure. Pixels above this percentile cut are deemed the "good" pixels and are used to compute the ratio of two spectra. This must be a number between 0 and 100. +``scale_method`` str .. ``auto`` Method used to rescale the spectra prior to coadding. The options are: 'auto' -- Determine the scaling method automatically based on the S/N ratio which works well. 'poly' -- Polynomial rescaling. 'median' -- Median rescaling 'none' -- Do not rescale. 'hand' -- Pass in hand scaling factors. This option is not well tested. +``sigrej_exp`` int, float .. .. Rejection threshold used for rejecting exposures with S/N more than sigrej_exp*sigma above the median S/N. If None (the default), no rejection is performed. Currently, only available for multi-slit observations. +``sigrej_scale`` int, float .. 3.0 Rejection threshold used for rejecting pixels when rescaling spectra with scale_spec. +``sn_clip`` int, float .. 30.0 Errors are capped during rejection so that the S/N is never greater than sn_clip. This prevents overly aggressive rejection in high S/N ratio spectrum which neverthless differ at a level greater than the formal S/N due to systematics. +``sn_min_medscale`` int, float .. 0.5 For scale method set to ``auto``, this sets the minimum SNR for which median scaling is attempted. +``sn_min_polyscale`` int, float .. 2.0 For scale method set to ``auto``, this sets the minimum SNR for which polynomial scaling is attempted. +``sn_smooth_npix`` int, float .. .. Number of pixels to median filter by when computing S/N used to decide how to scale and weight spectra. If set to None (default), the code will determine the effective number of good pixels per spectrum in the stack that is being co-added and use 10% of this neff. +``spec_samp_fact`` float .. 1.0 Make the wavelength grid sampling finer (spec_samp_fact < 1.0) or coarser (spec_samp_fact > 1.0) by this sampling factor. This basically multiples the 'native' spectral pixels by spec_samp_fact, i.e. units spec_samp_fact are pixels. +``upper`` int, float .. 3.0 Upper rejection threshold used for rejecting pixels when combining spectra in units of sigma. +``wave_grid_max`` int, float .. .. Used in case you want to specify the maximum wavelength in your wavelength grid, default=None computes from data +``wave_grid_min`` int, float .. .. Used in case you want to specify the minimum wavelength in your wavelength grid, default=None computes from data +``wave_method`` str .. ``linear`` Method used to construct wavelength grid for coadding spectra. The routine that creates the wavelength is :func:`~pypeit.core.wavecal.wvutils.get_wave_grid`. The options are: 'iref' -- Use the first wavelength array. 'velocity' -- Grid is uniform in velocity. 'log10' -- Grid is uniform in log10(wave). This is the same as velocity. 'linear' -- Grid is uniform in lambda. 'concatenate' -- Meld the input wavelength arrays +``weight_method`` str .. ``auto`` Method used to weight the spectra for coadding. The options are: 'auto' -- Use constant weights if rms_sn < 3.0, otherwise use wavelength dependent.'constant' -- Constant weights based on rms_sn**2'uniform' -- Uniform weighting'wave_dependent' -- Wavelength dependent weights will be used irrespective of the rms_sn ratio. This option will not work well at low S/N ratio although it is useful for objects where only a small fraction of the spectral coverage has high S/N ratio (like high-z quasars).'relative' -- Apply relative weights implying one reference exposure will receive unit weight at all wavelengths and all others receive relatively wavelength dependent weights . Note, relative weighting will only work well when there is at least one spectrum with a reasonable S/N, and a continuum. This option may only be better when the object being used has a strong continuum + emission lines. This is particularly useful if you are dealing with highly variable spectra (e.g. emission lines) andrequire a precision better than ~1 per cent.'ivar' -- Use inverse variance weighting. This is not well tested and should probably be deprecated. +==================== ========== ======= ========== ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= ---- @@ -578,7 +578,6 @@ Class Instantiation: :class:`~pypeit.par.pypeitpar.Collate1DPar` ========================= ========== ======= ============================================ ================================================================================================================================================================================================================================================================================================================================================================================================================== Key Type Options Default Description ========================= ========== ======= ============================================ ================================================================================================================================================================================================================================================================================================================================================================================================================== -``chk_version`` bool .. False Whether to check the data model versions of spec1d files and sensfunc files. ``dry_run`` bool .. False If set, the script will display the matching File and Object Ids but will not flux, coadd or archive. ``exclude_serendip`` bool .. False Whether to exclude SERENDIP objects from collating. ``exclude_slit_trace_bm`` list, str .. [] A list of slit trace bitmask bits that should be excluded. @@ -643,21 +642,22 @@ ReduxPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.ReduxPar` -====================== ============== ======= ============================================ =============================================================================================================================================================================================================================================================================================================================================================== -Key Type Options Default Description -====================== ============== ======= ============================================ =============================================================================================================================================================================================================================================================================================================================================================== -``calwin`` int, float .. 0 The window of time in hours to search for calibration frames for a science frame -``detnum`` int, list .. .. Restrict reduction to a list of detector indices. In case of mosaic reduction (currently only available for Gemini/GMOS and Keck/DEIMOS) ``detnum`` should be a list of tuples of the detector indices that are mosaiced together. E.g., for Gemini/GMOS ``detnum`` would be ``[(1,2,3)]`` and for Keck/DEIMOS it would be ``[(1, 5), (2, 6), (3, 7), (4, 8)]`` -``ignore_bad_headers`` bool .. False Ignore bad headers (NOT recommended unless you know it is safe). -``maskIDs`` str, int, list .. .. Restrict reduction to a set of slitmask IDs Example syntax -- ``maskIDs = 818006,818015`` This must be used with detnum (for now). -``qadir`` str .. ``QA`` Directory relative to calling directory to write quality assessment files. -``quicklook`` bool .. False Run a quick look reduction? This is usually good if you want to quickly reduce the data (usually at the telescope in real time) to get an initial estimate of the data quality. -``redux_path`` str .. ``/Users/westfall/Work/packages/pypeit/doc`` Path to folder for performing reductions. Default is the current working directory. -``scidir`` str .. ``Science`` Directory relative to calling directory to write science files. -``slitspatnum`` str, list .. .. Restrict reduction to a set of slit DET:SPAT values (closest slit is used). Example syntax -- slitspatnum = DET01:175,DET01:205 or MSC02:2234 If you are re-running the code, (i.e. modifying one slit) you *must* have the precise SPAT_ID index. -``sortroot`` str .. .. A filename given to output the details of the sorted files. If None, the default is the root name of the pypeit file. If off, no output is produced. -``spectrograph`` str .. .. Spectrograph that provided the data to be reduced. See :ref:`instruments` for valid options. -====================== ============== ======= ============================================ =============================================================================================================================================================================================================================================================================================================================================================== +====================== ============== ======= ============================================ ========================================================================================================================================================================================================================================================================================================================================================================================================== +Key Type Options Default Description +====================== ============== ======= ============================================ ========================================================================================================================================================================================================================================================================================================================================================================================================== +``calwin`` int, float .. 0 The window of time in hours to search for calibration frames for a science frame +``chk_version`` bool .. True If True enforce strict PypeIt version checking to ensure that all files were created with the current version of PypeIt. If set to False, the code will attempt to read out-of-date files and keep going. Beware (!!) that this can lead to unforeseen bugs that either cause the code to crash or lead to erroneous results. I.e., you really need to know what you are doing if you set this to False! +``detnum`` int, list .. .. Restrict reduction to a list of detector indices. In case of mosaic reduction (currently only available for Gemini/GMOS and Keck/DEIMOS) ``detnum`` should be a list of tuples of the detector indices that are mosaiced together. E.g., for Gemini/GMOS ``detnum`` would be ``[(1,2,3)]`` and for Keck/DEIMOS it would be ``[(1, 5), (2, 6), (3, 7), (4, 8)]`` +``ignore_bad_headers`` bool .. False Ignore bad headers (NOT recommended unless you know it is safe). +``maskIDs`` str, int, list .. .. Restrict reduction to a set of slitmask IDs Example syntax -- ``maskIDs = 818006,818015`` This must be used with detnum (for now). +``qadir`` str .. ``QA`` Directory relative to calling directory to write quality assessment files. +``quicklook`` bool .. False Run a quick look reduction? This is usually good if you want to quickly reduce the data (usually at the telescope in real time) to get an initial estimate of the data quality. +``redux_path`` str .. ``/Users/westfall/Work/packages/pypeit/doc`` Path to folder for performing reductions. Default is the current working directory. +``scidir`` str .. ``Science`` Directory relative to calling directory to write science files. +``slitspatnum`` str, list .. .. Restrict reduction to a set of slit DET:SPAT values (closest slit is used). Example syntax -- slitspatnum = DET01:175,DET01:205 or MSC02:2234 If you are re-running the code, (i.e. modifying one slit) you *must* have the precise SPAT_ID index. +``sortroot`` str .. .. A filename given to output the details of the sorted files. If None, the default is the root name of the pypeit file. If off, no output is produced. +``spectrograph`` str .. .. Spectrograph that provided the data to be reduced. See :ref:`instruments` for valid options. +====================== ============== ======= ============================================ ========================================================================================================================================================================================================================================================================================================================================================================================================== ---- @@ -690,34 +690,34 @@ CubePar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.CubePar` -==================== ===== ===================== ============ =========================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== -Key Type Options Default Description -==================== ===== ===================== ============ =========================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== -``align`` bool .. False If set to True, the input frames will be spatially aligned by cross-correlating the whitelight images with either a reference image (see ``reference_image``) or the whitelight image that is generated using the first spec2d listed in the coadd3d file. Alternatively, the user can specify the offsets (i.e. Delta RA x cos(dec) and Delta Dec, both in arcsec) in the spec2d block of the coadd3d file. See the documentation for examples of this usage. -``astrometric`` bool .. True If true, an astrometric correction will be applied using the alignment frames. -``combine`` bool .. False If set to True, the input frames will be combined. Otherwise, a separate datacube will be generated for each input spec2d file, and will be saved as a spec3d file. -``dec_max`` float .. .. Maximum DEC to use when generating the WCS. If None, the default is maximum DEC based on the WCS of all spaxels. Units should be degrees. -``dec_min`` float .. .. Minimum DEC to use when generating the WCS. If None, the default is minimum DEC based on the WCS of all spaxels. Units should be degrees. -``grating_corr`` bool .. True This option performs a small correction for the relative blaze function of all input frames that have (even slightly) different grating angles, or if you are flux calibrating your science data with a standard star that was observed with a slightly different setup. -``method`` str ``subpixel``, ``ngp`` ``subpixel`` What method should be used to generate the datacube. There are currently two options: (1) "subpixel" (default) - this algorithm divides each pixel in the spec2d frames into subpixels, and assigns each subpixel to a voxel of the datacube. Flux is conserved, but voxels are correlated, and the error spectrum does not account for covariance between adjacent voxels. See also, spec_subpixel and spat_subpixel. (2) "ngp" (nearest grid point) - this algorithm is effectively a 3D histogram. Flux is conserved, voxels are not correlated, however this option suffers the same downsides as any histogram; the choice of bin sizes can change how the datacube appears. This algorithm takes each pixel on the spec2d frame and puts the flux of this pixel into one voxel in the datacube. Depending on the binning used, some voxels may be empty (zero flux) while a neighboring voxel might contain the flux from two spec2d pixels. Note that all spec2d pixels that contribute to the same voxel are inverse variance weighted (e.g. if two pixels have the same variance, the voxel would be assigned the average flux of the two pixels). -``output_filename`` str .. .. If combining multiple frames, this string sets the output filename of the combined datacube. If combine=False, the output filenames will be prefixed with ``spec3d_*`` -``ra_max`` float .. .. Maximum RA to use when generating the WCS. If None, the default is maximum RA based on the WCS of all spaxels. Units should be degrees. -``ra_min`` float .. .. Minimum RA to use when generating the WCS. If None, the default is minimum RA based on the WCS of all spaxels. Units should be degrees. -``reference_image`` str .. .. White light image of a previously combined datacube. The white light image will be used as a reference when calculating the offsets of the input spec2d files. Ideally, the reference image should have the same shape as the data to be combined (i.e. set the ra_min, ra_max etc. params so they are identical to the reference image). -``relative_weights`` bool .. False If set to True, the combined frames will use a relative weighting scheme. This only works well if there is a common continuum source in the field of view of all input observations, and is generally only required if high relative precision is desired. -``save_whitelight`` bool .. False Save a white light image of the combined datacube. The output filename will be given by the "output_filename" variable with a suffix "_whitelight". Note that the white light image collapses the flux along the wavelength axis, so some spaxels in the 2D white light image may have different wavelength ranges. To set the wavelength range, use the "whitelight_range" parameter. If combine=False, the individual spec3d files will have a suffix "_whitelight". -``scale_corr`` str .. .. This option performs a small correction for the relative spectral illumination scale of different spec2D files. Specify the relative path+file to the spec2D file that you would like to use for the relative scaling. If you want to perform this correction, it is best to use the spec2d file with the highest S/N sky spectrum. You should choose the same frame for both the standards and science frames. -``skysub_frame`` str .. ``image`` Set the sky subtraction to be implemented. The default behaviour is to subtract the sky using the model that is derived from each individual image (i.e. set this parameter to "image"). To turn off sky subtraction completely, set this parameter to "none" (all lowercase). Finally, if you want to use a different frame for the sky subtraction, specify the relative path+file to the spec2D file that you would like to use for the sky subtraction. The model fit to the sky of the specified frame will be used. Note, the sky and science frames do not need to have the same exposure time; the sky model will be scaled to the science frame based on the relative exposure time. -``slit_spec`` bool .. True If the data use slits in one spatial direction, set this to True. If the data uses fibres for all spaxels, set this to False. -``spat_subpixel`` int .. 5 When method=subpixel, spat_subpixel sets the subpixellation scale of each detector pixel in the spatial direction. The total number of subpixels in each pixel is given by spec_subpixel x spat_subpixel. The default option is to divide each spec2d pixel into 25 subpixels during datacube creation. See also, spec_subpixel. -``spatial_delta`` float .. .. The spatial size of each spaxel to use when generating the WCS (in arcsec). If None, the default is set by the spectrograph file. -``spec_subpixel`` int .. 5 When method=subpixel, spec_subpixel sets the subpixellation scale of each detector pixel in the spectral direction. The total number of subpixels in each pixel is given by spec_subpixel x spat_subpixel. The default option is to divide each spec2d pixel into 25 subpixels during datacube creation. See also, spat_subpixel. -``standard_cube`` str .. .. Filename of a standard star datacube. This cube will be used to correct the relative scales of the slits, and to flux calibrate the science datacube. -``wave_delta`` float .. .. The wavelength step to use when generating the WCS (in Angstroms). If None, the default is set by the wavelength solution. -``wave_max`` float .. .. Maximum wavelength to use when generating the WCS. If None, the default is maximum wavelength based on the WCS of all spaxels. Units should be Angstroms. -``wave_min`` float .. .. Minimum wavelength to use when generating the WCS. If None, the default is minimum wavelength based on the WCS of all spaxels. Units should be Angstroms. -``whitelight_range`` list .. None, None A two element list specifying the wavelength range over which to generate the white light image. The first (second) element is the minimum (maximum) wavelength to use. If either of these elements are None, PypeIt will automatically use a wavelength range that ensures all spaxels have the same wavelength coverage. Note, if you are using a reference_image to align all frames, it is preferable to use the same white light wavelength range for all white light images. For example, you may wish to use an emission line map to register two frames. -==================== ===== ===================== ============ =========================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== +==================== ===== =============================================================================== ============ ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= +Key Type Options Default Description +==================== ===== =============================================================================== ============ ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= +``align`` bool .. False If set to True, the input frames will be spatially aligned by cross-correlating the whitelight images with either a reference image (see ``reference_image``) or the whitelight image that is generated using the first spec2d listed in the coadd3d file. Alternatively, the user can specify the offsets (i.e. Delta RA x cos(dec) and Delta Dec, both in arcsec) in the spec2d block of the coadd3d file. See the documentation for examples of this usage. +``astrometric`` bool .. True If true, an astrometric correction will be applied using the alignment frames. +``combine`` bool .. False If set to True, the input frames will be combined. Otherwise, a separate datacube will be generated for each input spec2d file, and will be saved as a spec3d file. +``dec_max`` float .. .. Maximum DEC to use when generating the WCS. If None, the default is maximum DEC based on the WCS of all spaxels. Units should be degrees. +``dec_min`` float .. .. Minimum DEC to use when generating the WCS. If None, the default is minimum DEC based on the WCS of all spaxels. Units should be degrees. +``grating_corr`` bool .. True This option performs a small correction for the relative blaze function of all input frames that have (even slightly) different grating angles, or if you are flux calibrating your science data with a standard star that was observed with a slightly different setup. +``method`` str ``subpixel``, ``ngp`` ``subpixel`` What method should be used to generate the datacube. There are currently two options: (1) "subpixel" (default) - this algorithm divides each pixel in the spec2d frames into subpixels, and assigns each subpixel to a voxel of the datacube. Flux is conserved, but voxels are correlated, and the error spectrum does not account for covariance between adjacent voxels. See also, spec_subpixel and spat_subpixel. (2) "ngp" (nearest grid point) - this algorithm is effectively a 3D histogram. Flux is conserved, voxels are not correlated, however this option suffers the same downsides as any histogram; the choice of bin sizes can change how the datacube appears. This algorithm takes each pixel on the spec2d frame and puts the flux of this pixel into one voxel in the datacube. Depending on the binning used, some voxels may be empty (zero flux) while a neighboring voxel might contain the flux from two spec2d pixels. Note that all spec2d pixels that contribute to the same voxel are inverse variance weighted (e.g. if two pixels have the same variance, the voxel would be assigned the average flux of the two pixels). +``output_filename`` str .. .. If combining multiple frames, this string sets the output filename of the combined datacube. If combine=False, the output filenames will be prefixed with ``spec3d_*`` +``ra_max`` float .. .. Maximum RA to use when generating the WCS. If None, the default is maximum RA based on the WCS of all spaxels. Units should be degrees. +``ra_min`` float .. .. Minimum RA to use when generating the WCS. If None, the default is minimum RA based on the WCS of all spaxels. Units should be degrees. +``reference_image`` str .. .. White light image of a previously combined datacube. The white light image will be used as a reference when calculating the offsets of the input spec2d files. Ideally, the reference image should have the same shape as the data to be combined (i.e. set the ra_min, ra_max etc. params so they are identical to the reference image). +``save_whitelight`` bool .. False Save a white light image of the combined datacube. The output filename will be given by the "output_filename" variable with a suffix "_whitelight". Note that the white light image collapses the flux along the wavelength axis, so some spaxels in the 2D white light image may have different wavelength ranges. To set the wavelength range, use the "whitelight_range" parameter. If combine=False, the individual spec3d files will have a suffix "_whitelight". +``scale_corr`` str .. .. This option performs a small correction for the relative spectral illumination scale of different spec2D files. Specify the relative path+file to the spec2D file that you would like to use for the relative scaling. If you want to perform this correction, it is best to use the spec2d file with the highest S/N sky spectrum. You should choose the same frame for both the standards and science frames. +``skysub_frame`` str .. ``image`` Set the sky subtraction to be implemented. The default behaviour is to subtract the sky using the model that is derived from each individual image (i.e. set this parameter to "image"). To turn off sky subtraction completely, set this parameter to "none" (all lowercase). Finally, if you want to use a different frame for the sky subtraction, specify the relative path+file to the spec2D file that you would like to use for the sky subtraction. The model fit to the sky of the specified frame will be used. Note, the sky and science frames do not need to have the same exposure time; the sky model will be scaled to the science frame based on the relative exposure time. +``slit_spec`` bool .. True If the data use slits in one spatial direction, set this to True. If the data uses fibres for all spaxels, set this to False. +``spat_subpixel`` int .. 5 When method=subpixel, spat_subpixel sets the subpixellation scale of each detector pixel in the spatial direction. The total number of subpixels in each pixel is given by spec_subpixel x spat_subpixel. The default option is to divide each spec2d pixel into 25 subpixels during datacube creation. See also, spec_subpixel. +``spatial_delta`` float .. .. The spatial size of each spaxel to use when generating the WCS (in arcsec). If None, the default is set by the spectrograph file. +``spec_subpixel`` int .. 5 When method=subpixel, spec_subpixel sets the subpixellation scale of each detector pixel in the spectral direction. The total number of subpixels in each pixel is given by spec_subpixel x spat_subpixel. The default option is to divide each spec2d pixel into 25 subpixels during datacube creation. See also, spat_subpixel. +``standard_cube`` str .. .. Filename of a standard star datacube. This cube will be used to correct the relative scales of the slits, and to flux calibrate the science datacube. +``wave_delta`` float .. .. The wavelength step to use when generating the WCS (in Angstroms). If None, the default is set by the wavelength solution. +``wave_max`` float .. .. Maximum wavelength to use when generating the WCS. If None, the default is maximum wavelength based on the WCS of all spaxels. Units should be Angstroms. +``wave_min`` float .. .. Minimum wavelength to use when generating the WCS. If None, the default is minimum wavelength based on the WCS of all spaxels. Units should be Angstroms. +``weight_method`` str ``auto``, ``constant``, ``uniform``, ``wave_dependent``, ``relative``, ``ivar`` ``auto`` Method used to weight the spectra for coadding. The options are: 'auto' -- Use constant weights if rms_sn < 3.0, otherwise use wavelength dependent.'constant' -- Constant weights based on rms_sn**2'uniform' -- Uniform weighting'wave_dependent' -- Wavelength dependent weights will be used irrespective of the rms_sn ratio. This option will not work well at low S/N ratio although it is useful for objects where only a small fraction of the spectral coverage has high S/N ratio (like high-z quasars).'relative' -- Apply relative weights implying one reference exposure will receive unit weight at all wavelengths and all others receive relatively wavelength dependent weights . Note, relative weighting will only work well when there is at least one spectrum with a reasonable S/N, and a continuum. This option may only be better when the object being used has a strong continuum + emission lines. This is particularly useful if you are dealing with highly variable spectra (e.g. emission lines) andrequire a precision better than ~1 per cent.'ivar' -- Use inverse variance weighting. This is not well tested and should probably be deprecated. +``whitelight_range`` list .. None, None A two element list specifying the wavelength range over which to generate the white light image. The first (second) element is the minimum (maximum) wavelength to use. If either of these elements are None, PypeIt will automatically use a wavelength range that ensures all spaxels have the same wavelength coverage. Note, if you are using a reference_image to align all frames, it is preferable to use the same white light wavelength range for all white light images. For example, you may wish to use an emission line map to register two frames. +==================== ===== =============================================================================== ============ ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= ---- @@ -1006,7 +1006,7 @@ Key Type Options Default ``popsize`` int .. 30 A multiplier for setting the total population size for the differential evolution optimization. See scipy.optimize.differential_evolution for details. ``recombination`` int, float .. 0.7 The recombination constant for the differential evolution optimization. This should be in the range [0, 1]. See scipy.optimize.differential_evolution for details. ``redshift`` int, float .. 0.0 The redshift for the object model. This is currently only used by objmodel=qso -``resln_frac_bounds`` tuple .. (0.5, 1.5) Bounds for the resolution fit optimization which is part of the telluric model. This range is in units of the resln_guess, so the (0.5, 1.5) would bound the spectral resolution fit to be within the range bounds_resln = (0.5*resln_guess, 1.5*resln_guess) +``resln_frac_bounds`` tuple .. (0.6, 1.4) Bounds for the resolution fit optimization which is part of the telluric model. This range is in units of the resln_guess, so the (0.6, 1.4) would bound the spectral resolution fit to be within the range bounds_resln = (0.6*resln_guess, 1.4*resln_guess) ``resln_guess`` int, float .. .. A guess for the resolution of your spectrum expressed as lambda/dlambda. The resolution is fit explicitly as part of the telluric model fitting, but this guess helps determine the bounds for the optimization (see next). If not provided, the wavelength sampling of your spectrum will be used and the resolution calculated using a typical sampling of 3 spectral pixels per resolution element. ``seed`` int .. 777 An initial seed for the differential evolution optimization, which is a random process. The default is a seed = 777 which will be used to generate a unique seed for every order. A specific seed is used because otherwise the random number generator will use the time for the seed, and the results will not be reproducible. ``sn_clip`` int, float .. 30.0 This adds an error floor to the ivar, preventing too much rejection at high-S/N (`i.e.`, standard stars, bright objects) using the function utils.clip_ivar. A small erorr is added to the input ivar so that the output ivar_out will never give S/N greater than sn_clip. This prevents overly aggressive rejection in high S/N ratio spectra which neverthless differ at a level greater than the formal S/N due to the fact that our telluric models are only good to about 3%. @@ -1017,6 +1017,8 @@ Key Type Options Default ``sticky`` bool .. True Sticky parameter for the utils.djs_reject algorithm for iterative model fit rejection. If set to True then points rejected from a previous iteration are kept rejected, in other words the bad pixel mask is the OR of all previous iterations and rejected pixels accumulate. If set to False, the bad pixel mask is the mask from the previous iteration, and if the model fit changes between iterations, points can alternate from being rejected to not rejected. At present this code only performs optimizations with differential evolution and experience shows that sticky needs to be True in order for these to converge. This is because the outliers can be so large that they dominate the loss function, and one never iteratively converges to a good model fit. In other words, the deformations in the model between iterations with sticky=False are too small to approach a reasonable fit. ``telgridfile`` str .. .. File containing the telluric grid for the observatory in question. These grids are generated from HITRAN models for each observatory using nominal site parameters. They must be downloaded from the GoogleDrive and installed in your PypeIt installation via the pypeit_install_telluric script. NOTE: This parameter no longer includes the full pathname to the Telluric Grid file, but is just the filename of the grid itself. ``tell_norm_thresh`` int, float .. 0.9 Threshold of telluric absorption region +``tell_npca`` int .. 5 Number of telluric PCA components used. Can be set to any number from 1 to 10. +``teltype`` str .. ``pca`` Method used to evaluate telluric models, either pca or grid. The grid option uses a fixed grid of pre-computed HITRAN+LBLRTM atmospheric transmission models for each observatory, whereas the pca option uses principal components of a larger model grid to compute an accurate pseudo-telluric model with a much lighter telgridfile. ``tol`` float .. 0.001 Relative tolerance for converage of the differential evolution optimization. See scipy.optimize.differential_evolution for details. ``upper`` int, float .. 3.0 Upper rejection threshold in units of sigma_corr*sigma, where sigma is the formal noise of the spectrum, and sigma_corr is an empirically determined correction to the formal error. See above for description. ======================= ================== ======= ========================== ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= @@ -1425,7 +1427,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-gemini_gmos_north_e2v: @@ -1786,7 +1788,7 @@ Alterations to the default parameters are: [sensfunc] algorithm = IR [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-gemini_gnirs_echelle: @@ -1915,7 +1917,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 6 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-gemini_gnirs_ifu: @@ -2058,7 +2060,7 @@ Alterations to the default parameters are: [[UVIS]] extinct_correct = False [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-gtc_maat: @@ -2134,7 +2136,7 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = None, 180, + exprng = None, 300, [[[process]]] mask_cr = True noise_floor = 0.01 @@ -2347,7 +2349,7 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = None, 180, + exprng = None, 300, [[[process]]] mask_cr = True noise_floor = 0.01 @@ -2665,7 +2667,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R15000.fits .. _instr_par-keck_esi: @@ -2930,7 +2932,11 @@ Alterations to the default parameters are: [sensfunc] algorithm = IR [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_10500_R120000.fits + pix_shift_bounds = (-40.0, 40.0) + [telluric] + resln_frac_bounds = (0.25, 1.25) + pix_shift_bounds = (-40.0, 40.0) .. _instr_par-keck_kcrm: @@ -3249,7 +3255,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_lris_blue_orig: @@ -3351,7 +3357,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_lris_red: @@ -3464,7 +3470,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 9 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_lris_red_mark4: @@ -3577,7 +3583,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 9 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_lris_red_orig: @@ -3690,7 +3696,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 9 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_mosfire: @@ -3817,7 +3823,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 13 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_nires: @@ -3960,7 +3966,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_nirspec_low: @@ -4089,7 +4095,10 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R25000.fits + pix_shift_bounds = (-8.0, 8.0) + [telluric] + pix_shift_bounds = (-8.0, 8.0) .. _instr_par-lbt_luci1: @@ -5018,7 +5027,7 @@ Alterations to the default parameters are: [sensfunc] algorithm = IR [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R15000.fits .. _instr_par-magellan_fire_long: @@ -5148,7 +5157,7 @@ Alterations to the default parameters are: find_trim_edge = 50, 50, [sensfunc] [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-magellan_mage: @@ -5665,7 +5674,7 @@ Alterations to the default parameters are: [sensfunc] polyorder = 7 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-mmt_bluechannel: @@ -5914,7 +5923,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-not_alfosc: @@ -6411,7 +6420,7 @@ Alterations to the default parameters are: [[UVIS]] polycorrect = False [[IR]] - telgridfile = TelFit_Lick_3100_11100_R10000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-p200_tspec: @@ -6557,7 +6566,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-shane_kast_blue: @@ -6658,7 +6667,7 @@ Alterations to the default parameters are: spectrum = sky_kastb_600.fits [sensfunc] [[IR]] - telgridfile = TelFit_Lick_3100_11100_R10000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-shane_kast_red: @@ -6750,7 +6759,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_Lick_3100_11100_R10000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-shane_kast_red_ret: @@ -6844,7 +6853,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_Lick_3100_11100_R10000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-soar_goodman_blue: @@ -6947,7 +6956,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R15000.fits .. _instr_par-soar_goodman_red: @@ -7052,7 +7061,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R15000.fits .. _instr_par-tng_dolores: @@ -7242,7 +7251,7 @@ Alterations to the default parameters are: [sensfunc] algorithm = IR [[IR]] - telgridfile = TelFit_Paranal_VIS_9800_25000_R25000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-vlt_sinfoni: @@ -7383,7 +7392,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 7 [[IR]] - telgridfile = TelFit_Paranal_NIR_9800_25000_R25000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-vlt_xshooter_nir: @@ -7534,7 +7543,11 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_Paranal_NIR_9800_25000_R25000.fits + telgridfile = TellPCA_3000_26000_R25000.fits + pix_shift_bounds = (-10.0, 10.0) + [telluric] + resln_frac_bounds = (0.4, 2.0) + pix_shift_bounds = (-10.0, 10.0) .. _instr_par-vlt_xshooter_uvb: @@ -7676,7 +7689,10 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R25000.fits + pix_shift_bounds = (-8.0, 8.0) + [telluric] + pix_shift_bounds = (-8.0, 8.0) .. _instr_par-vlt_xshooter_vis: @@ -7820,7 +7836,11 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_Paranal_VIS_4900_11100_R25000.fits + telgridfile = TellPCA_3000_26000_R25000.fits + pix_shift_bounds = (-10.0, 10.0) + [telluric] + resln_frac_bounds = (0.4, 2.0) + pix_shift_bounds = (-10.0, 10.0) .. _instr_par-wht_isis_blue: diff --git a/pypeit/scripts/collate_1d.py b/pypeit/scripts/collate_1d.py index 4bfaa7a948..4d0744c483 100644 --- a/pypeit/scripts/collate_1d.py +++ b/pypeit/scripts/collate_1d.py @@ -671,7 +671,8 @@ class Collate1D(scriptbase.ScriptBase): @classmethod def get_parser(cls, width=None): # A blank Colate1DPar to avoid duplicating the help text. - blank_par = pypeitpar.Collate1DPar() + blank_pypar = pypeitpar.PypeItPar() + blank_par = blank_pypar['coadd1d'] parser = super().get_parser(description='Flux/Coadd multiple 1d spectra from multiple ' 'nights and prepare a directory for the KOA.', @@ -730,7 +731,7 @@ def get_parser(cls, width=None): parser.add_argument("--wv_rms_thresh", type=float, default = None, help=blank_par.descr['wv_rms_thresh']) parser.add_argument("--refframe", type=str, default = None, choices = pypeitpar.WavelengthSolutionPar.valid_reference_frames(), help=blank_par.descr['refframe']) - parser.add_argument('--chk_version', action = 'store_true', help=blank_par.descr['chk_version']) + parser.add_argument('--chk_version', action = 'store_true', help=blank_pypar['rdx'].descr['chk_version']) parser.add_argument('-v', '--verbosity', type=int, default=1, help='Verbosity level between 0 [none] and 2 [all]. Default: 1. ' 'Level 2 writes a log with filename collate_1d_YYYYMMDD-HHMM.log') From 35f7b898d8802ee925cc0ec3b09a0aab28a3aafa Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Fri, 19 Jan 2024 08:24:48 -0800 Subject: [PATCH 203/244] bug --- pypeit/scripts/collate_1d.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/scripts/collate_1d.py b/pypeit/scripts/collate_1d.py index 4d0744c483..6bdb60903f 100644 --- a/pypeit/scripts/collate_1d.py +++ b/pypeit/scripts/collate_1d.py @@ -672,7 +672,7 @@ class Collate1D(scriptbase.ScriptBase): def get_parser(cls, width=None): # A blank Colate1DPar to avoid duplicating the help text. blank_pypar = pypeitpar.PypeItPar() - blank_par = blank_pypar['coadd1d'] + blank_par = blank_pypar['collate1d'] parser = super().get_parser(description='Flux/Coadd multiple 1d spectra from multiple ' 'nights and prepare a directory for the KOA.', From 74e1c7a576c7ce0aff77166656677f0940e3daac Mon Sep 17 00:00:00 2001 From: Jack O'Donnell Date: Fri, 19 Jan 2024 12:08:40 -0800 Subject: [PATCH 204/244] As Debora recommended, remove list-detectors entirely. --- pypeit/spectrographs/gemini_gmos.py | 37 ----------------------------- 1 file changed, 37 deletions(-) diff --git a/pypeit/spectrographs/gemini_gmos.py b/pypeit/spectrographs/gemini_gmos.py index 61e16481cc..7f810c2aab 100644 --- a/pypeit/spectrographs/gemini_gmos.py +++ b/pypeit/spectrographs/gemini_gmos.py @@ -530,43 +530,6 @@ def allowed_mosaics(self): """ return [(1,2,3)] - def list_detectors(self, mosaic=False): - """ - List the *names* of the detectors in this spectrograph. - - This is primarily used :func:`~pypeit.slittrace.average_maskdef_offset` - to measure the mean offset between the measured and expected slit - locations. - - Detectors separated along the dispersion direction should be ordered - along the first axis of the returned array. For example, Keck/DEIMOS - returns: - - .. code-block:: python - - dets = np.array([['DET01', 'DET02', 'DET03', 'DET04'], - ['DET05', 'DET06', 'DET07', 'DET08']]) - - such that all the bluest detectors are in ``dets[0]``, and the slits - found in detectors 1 and 5 are just from the blue and red counterparts - of the same slit. - - Args: - mosaic (:obj:`bool`, optional): - Is this a mosaic reduction? - It is used to determine how to list the detector, i.e., 'DET' or 'MSC'. - - Returns: - `numpy.ndarray`_: The list of detectors in a `numpy.ndarray`_. If - the array is 2D, there are detectors separated along the dispersion - axis. - """ - if mosaic: - return np.array([self.get_det_name(_det) for _det in self.allowed_mosaics]) - - return np.array([detector_container.DetectorContainer.get_name(i+1) - for i in range(self.ndet)]).reshape(3,1) - @property def default_mosaic(self): return self.allowed_mosaics[0] From a772c49593eff30ba7887fa7a061a0f5dfe89bc5 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Sat, 20 Jan 2024 13:21:36 -0800 Subject: [PATCH 205/244] examples, and propagate changes to other modules --- doc/help/run_pypeit.rst | 2 +- pypeit/alignframe.py | 2 - pypeit/bitmask.py | 88 ++++++++++++++++++++++++++++++++++--- pypeit/coadd2d.py | 9 ++-- pypeit/deprecated/reduce.py | 34 ++++++++++---- pypeit/extraction.py | 5 ++- pypeit/find_objects.py | 22 +++++++--- pypeit/spec2dobj.py | 12 +++-- pypeit/wavecalib.py | 5 ++- 9 files changed, 142 insertions(+), 37 deletions(-) diff --git a/doc/help/run_pypeit.rst b/doc/help/run_pypeit.rst index 23b9ade3c4..f65bce9629 100644 --- a/doc/help/run_pypeit.rst +++ b/doc/help/run_pypeit.rst @@ -4,7 +4,7 @@ usage: run_pypeit [-h] [-v VERBOSITY] [-r REDUX_PATH] [-m] [-s] [-o] [-c] pypeit_file - ## PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.14.1.dev768+gac1cdc619.d20240108 + ## PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.14.1.dev696+g00535ea2e.d20240117 ## ## Available spectrographs include: ## bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, diff --git a/pypeit/alignframe.py b/pypeit/alignframe.py index cc0ce7a3d7..9a28491d21 100644 --- a/pypeit/alignframe.py +++ b/pypeit/alignframe.py @@ -146,8 +146,6 @@ def __init__(self, rawalignimg, slits, spectrograph, alignpar, det=1, qa_path=No # Create a bad pixel mask self.slit_bpm = self.slits.bitmask.flagged(self.slits.mask, and_not=self.slits.bitmask.exclude_for_reducing) -# self.slit_bpm = self.slits.mask.astype(bool) -# self.slit_bpm &= np.logical_not(self.slits.bitmask.flagged(self.slits.mask,flag=self.slits.bitmask.exclude_for_reducing)) # Completed steps self.steps = [] diff --git a/pypeit/bitmask.py b/pypeit/bitmask.py index 26d40078c1..fd35e61c0f 100644 --- a/pypeit/bitmask.py +++ b/pypeit/bitmask.py @@ -218,9 +218,10 @@ def minimum_dtype(self, asuint=False): def flagged(self, value, flag=None, exclude=None, and_not=None): """ - Determine if a bit is on in the provided bitmask value. The - function can be used to determine if any individual bit is on or - any one of many bits is on. + + Determine if a bit is on in the provided integer(s). The function can + be used to determine if any individual bit is on or any one of many bits + is on. Args: value (int, array-like): @@ -234,10 +235,8 @@ def flagged(self, value, flag=None, exclude=None, and_not=None): flags are included in the check. and_not (str, array-like, optional): One or more bit names to ensure are *not* selected by the check. - If any of these bits are flagged, the element in the returned - array is as if it was not flagged, even if other flags not in - this list *are* flagged. I.e., the returned value can only be - True if one of these "and_not" bits are *not* flagged. If None, + I.e., if this bit is flagged the returned value is false, even + if other selected bits *are* flagged. See examples. If None, functionality ignored. Returns: @@ -250,6 +249,81 @@ def flagged(self, value, flag=None, exclude=None, and_not=None): is not one of the valid bitmask names. TypeError: Raised if the provided *flag* does not contain one or more strings. + + Example: + + Let the BitMask object have two flags, 'A' and 'B', and a bit + array ``v``, such that: + + >>> import numpy + >>> from pypeit.bitmask import BitMask + >>> bm = BitMask(['A', 'B']) + >>> v = numpy.arange(4).astype(numpy.int16) + >>> v + array([0, 1, 2, 3], dtype=int16) + >>> print([bm.flagged_bits(_v) for _v in v]) + [[], ['A'], ['B'], ['A', 'B']] + + This function will return a boolean array that indicates where flags + are turned on in each bit value. + + - To find if a specific bit is on: + + >>> bm.flagged(v, flag='A') + array([False, True, False, True]) + + - To find if *any* bit is on: + + >>> bm.flagged(v) + array([False, True, True, True]) + + This is identical to: + + >>> bm.flagged(v, flag=['A', 'B']) + array([False, True, True, True]) + + or a logical-or combination of all the bits: + + >>> bm.flagged(v, flag='A') | bm.flagged(v, flag='B') + array([False, True, True, True]) + + - To find if any bit is on except for a given subset, use the + ``exclude`` keyword: + + >>> bm.flagged(v, exclude='A') + array([False, False, True, True]) + + This is identical to: + + >>> bm.flagged(v, flag='B') + array([False, False, True, True]) + + Obviously, this is more useful when there are many flags and + you're only trying to exclude a few. + + - To find if a set of bits are on *and* a different set of bits are + *not* on, use the ``and_not`` keyword: + + >>> bm.flagged(v, and_not='B') + array([False, True, False, False]) + + This is equivalent to: + + >>> bm.flagged(v, flag=['A', 'B'], and_not='B') + array([False, True, False, False]) + + and: + + >>> bm.flagged(v) & numpy.logical_not(bm.flagged(v, flag='B')) + array([False, True, False, False]) + + Note that the returned values are False if the bit indicates that + flag 'B' is on, even though flag 'B' was in the list provided to + the ``flag`` keyword; i.e., the use of ``and_not`` supersedes any + elements of ``flag``. + + - Currently there is no functionality that performs a logical-and + combination of flags. """ _flag = self._prep_flags(flag) if exclude is not None: diff --git a/pypeit/coadd2d.py b/pypeit/coadd2d.py index 53683756a0..73edac0f7d 100644 --- a/pypeit/coadd2d.py +++ b/pypeit/coadd2d.py @@ -420,13 +420,14 @@ def good_slitindx(self, only_slits=None, exclude_slits=None): # This creates a unified bpm common to all frames slits0 = self.stack_dict['slits_list'][0] # bpm for the first frame - reduce_bpm = (slits0.mask > 0) & (np.invert(slits0.bitmask.flagged(slits0.mask, - flag=slits0.bitmask.exclude_for_reducing))) + + reduce_bpm = slits0.bitmask.flagged(slits0.mask, + and_not=slits0.bitmask.exclude_for_reducing) for i in range(1, self.nexp): # update bpm with the info from the other frames slits = self.stack_dict['slits_list'][i] - reduce_bpm |= (slits.mask > 0) & (np.invert(slits.bitmask.flagged(slits.mask, - flag=slits.bitmask.exclude_for_reducing))) + reduce_bpm |= slits.bitmask.flagged(slits.mask, + and_not=slits.bitmask.exclude_for_reducing) # these are the good slit index according to the bpm mask good_slitindx = np.where(np.logical_not(reduce_bpm))[0] diff --git a/pypeit/deprecated/reduce.py b/pypeit/deprecated/reduce.py index 0205b6cd7f..0542116283 100644 --- a/pypeit/deprecated/reduce.py +++ b/pypeit/deprecated/reduce.py @@ -173,8 +173,19 @@ def __init__(self, sciImg, spectrograph, par, caliBrate, # Internal bpm mask # We want to keep the 'BOXSLIT', which has bpm=2. But we don't want to keep 'BOXSLIT' # with other bad flag (for which bpm>2) - self.reduce_bpm = (self.slits.mask > 2) & (np.invert(self.slits.bitmask.flagged( - self.slits.mask, flag=self.slits.bitmask.exclude_for_reducing))) + # TODO: To my mind, we should never be using the value of the bit to + # check for flags. We should be using the BitMask functions. I *think* + # what you want is this: + self.reduce_bpm = self.slits.bitmask.flagged( + self.slits.mask, + exclude='BOXSLIT', + and_not=self.slits.bitmask.exclude_for_reducing) +# self.reduce_bpm = (self.slits.mask > 2) & (np.invert(self.slits.bitmask.flagged( +# self.slits.mask, flag=self.slits.bitmask.exclude_for_reducing))) + # I.e., mask anything *except* slits flagged by only 'BOXSLIT', and also + # make sure any of the `exclude_for_reducing` flags are not on. This + # previous code may also have included slits that were flagged as + # SHORTSLIT. Was that on purpose? self.reduce_bpm_init = self.reduce_bpm.copy() # These may be None (i.e. COADD2D) @@ -371,9 +382,12 @@ def prepare_extraction(self, global_sky): """ Prepare the masks and wavelength image for extraction. """ # Update bpm mask to remove `BOXSLIT`, i.e., we don't want to extract those - self.reduce_bpm = (self.slits.mask > 0) & \ - (np.invert(self.slits.bitmask.flagged(self.slits.mask, - flag=self.slits.bitmask.exclude_for_reducing))) + self.reduce_bpm = self.slits.bitmask.flagged( + self.slits.mask, + and_not=self.slits.bitmask.exclude_for_reducing) +# self.reduce_bpm = (self.slits.mask > 0) & \ +# (np.invert(self.slits.bitmask.flagged(self.slits.mask, +# flag=self.slits.bitmask.exclude_for_reducing))) # Update Slitmask to remove `BOXSLIT`, i.e., we don't want to extract those self.slitmask = self.slits.slit_img(flexure=self.spat_flexure_shift, exclude_flag=self.slits.bitmask.exclude_for_reducing) @@ -654,10 +668,12 @@ def global_skysub(self, skymask=None, update_crmask=True, trim_edg=(3,3), sigrej = 3.0 # We use this tmp bpm so that we exclude the BOXSLITS during the global_skysub - tmp_bpm = (self.slits.mask > 0) & \ - (np.invert(self.slits.bitmask.flagged(self.slits.mask, - flag=self.slits.bitmask.exclude_for_reducing))) - gdslits = np.where(np.invert(tmp_bpm))[0] + tmp_bpm = self.slits.bitmask.flagged(self.slits.mask, + and_not=self.slits.bitmask.exclude_for_reducing) +# tmp_bpm = (self.slits.mask > 0) & \ +# (np.invert(self.slits.bitmask.flagged(self.slits.mask, +# flag=self.slits.bitmask.exclude_for_reducing))) + gdslits = np.where(np.logical_not(tmp_bpm))[0] # Mask objects using the skymask? If skymask has been set by objfinding, and masking is requested, then do so skymask_now = skymask if (skymask is not None) else np.ones_like(self.sciImg.image, dtype=bool) diff --git a/pypeit/extraction.py b/pypeit/extraction.py index dd332cc9b3..461f82b1b9 100644 --- a/pypeit/extraction.py +++ b/pypeit/extraction.py @@ -164,8 +164,9 @@ def __init__(self, sciImg, slits, sobjs_obj, spectrograph, par, objtype, global_ self.initialize_slits(slits) # Internal bpm mask - self.extract_bpm = (self.slits.mask > 0) & (np.logical_not(self.slits.bitmask.flagged( - self.slits.mask, flag=self.slits.bitmask.exclude_for_reducing))) + self.extract_bpm = self.slits.bitmask.flagged( + self.slits.mask, + and_not=self.slits.bitmask.exclude_for_reducing) self.extract_bpm_init = self.extract_bpm.copy() # These may be None (i.e. COADD2D) diff --git a/pypeit/find_objects.py b/pypeit/find_objects.py index 59f5dede8a..1ba353e92c 100644 --- a/pypeit/find_objects.py +++ b/pypeit/find_objects.py @@ -166,8 +166,19 @@ def __init__(self, sciImg, slits, spectrograph, par, objtype, wv_calib=None, wav # Internal bpm mask # We want to keep the 'BOXSLIT', which has bpm=2. But we don't want to keep 'BOXSLIT' # with other bad flag (for which bpm>2) - self.reduce_bpm = (self.slits.mask > 2) & (np.invert(self.slits.bitmask.flagged( - self.slits.mask, flag=self.slits.bitmask.exclude_for_reducing))) + # TODO: To my mind, we should never be using the value of the bit to + # check for flags. We should be using the BitMask functions. I *think* + # what you want is this: + self.reduce_bpm = self.slits.bitmask.flagged( + self.slits.mask, + exclude='BOXSLIT', + and_not=self.slits.bitmask.exclude_for_reducing) + # I.e., mask anything *except* slits flagged by only 'BOXSLIT', and also + # make sure any of the `exclude_for_reducing` flags are not on. This + # previous code may also have included slits that were flagged as + # SHORTSLIT. Was that on purpose? +# self.reduce_bpm = (self.slits.mask > 2) & (np.invert(self.slits.bitmask.flagged( +# self.slits.mask, flag=self.slits.bitmask.exclude_for_reducing))) self.reduce_bpm_init = self.reduce_bpm.copy() # Load up other input items @@ -534,10 +545,9 @@ def global_skysub(self, skymask=None, update_crmask=True, sigrej = 3.0 # We use this tmp bpm so that we exclude the BOXSLITS during the global_skysub - tmp_bpm = (self.slits.mask > 0) & \ - (np.invert(self.slits.bitmask.flagged(self.slits.mask, - flag=self.slits.bitmask.exclude_for_reducing))) - gdslits = np.where(np.invert(tmp_bpm))[0] + tmp_bpm = self.slits.bitmask.flagged(self.slits.mask, + and_not=self.slits.bitmask.exclude_for_reducing) + gdslits = np.where(np.logical_not(tmp_bpm))[0] # Mask objects using the skymask? If skymask has been set by objfinding, and masking is requested, then do so skymask_now = skymask if (skymask is not None) else np.ones_like(self.sciImg.image, dtype=bool) diff --git a/pypeit/spec2dobj.py b/pypeit/spec2dobj.py index fd6906288d..72240be1af 100644 --- a/pypeit/spec2dobj.py +++ b/pypeit/spec2dobj.py @@ -319,10 +319,14 @@ def update_slits(self, spec2DObj): msgs.error("SPAT_IDs are not in sync!") # Find the good ones on the input object - bpm = spec2DObj.slits.mask.astype(bool) - exc_reduce = np.invert(spec2DObj.slits.bitmask.flagged( - spec2DObj.slits.mask, flag=spec2DObj.slits.bitmask.exclude_for_reducing)) - gpm = np.invert(bpm & exc_reduce) +# bpm = spec2DObj.slits.mask.astype(bool) +# exc_reduce = np.invert(spec2DObj.slits.bitmask.flagged( +# spec2DObj.slits.mask, flag=spec2DObj.slits.bitmask.exclude_for_reducing)) +# gpm = np.invert(bpm & exc_reduce) + bpm = spec2DObj.slits.bitmask.flagged( + spec2DObj.slits.mask, + and_not=spec2DObj.slits.bitmask.exclude_for_reducing) + gpm = np.logical_not(bpm) # Update slits.mask self.slits.mask[gpm] = spec2DObj.slits.mask[gpm] diff --git a/pypeit/wavecalib.py b/pypeit/wavecalib.py index 3c5a3be0d1..98685010a2 100644 --- a/pypeit/wavecalib.py +++ b/pypeit/wavecalib.py @@ -304,8 +304,9 @@ def build_waveimg(self, tilts, slits, spat_flexure=None, spec_flexure=None): # Setup #ok_slits = slits.mask == 0 - bpm = slits.mask.astype(bool) - bpm &= np.logical_not(slits.bitmask.flagged(slits.mask, flag=slits.bitmask.exclude_for_reducing)) +# bpm = slits.mask.astype(bool) +# bpm &= np.logical_not(slits.bitmask.flagged(slits.mask, flag=slits.bitmask.exclude_for_reducing)) + bpm = slits.bitmask.flagged(slits.mask, and_not=slits.bitmask.exclude_for_reducing) ok_slits = np.logical_not(bpm) # image = np.zeros_like(tilts) From 192fd85727b7c2021c89103d47427b0b57599b09 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Sat, 20 Jan 2024 13:24:35 -0800 Subject: [PATCH 206/244] revert changes to deprecated file --- pypeit/deprecated/reduce.py | 34 +++++++++------------------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/pypeit/deprecated/reduce.py b/pypeit/deprecated/reduce.py index 0542116283..0205b6cd7f 100644 --- a/pypeit/deprecated/reduce.py +++ b/pypeit/deprecated/reduce.py @@ -173,19 +173,8 @@ def __init__(self, sciImg, spectrograph, par, caliBrate, # Internal bpm mask # We want to keep the 'BOXSLIT', which has bpm=2. But we don't want to keep 'BOXSLIT' # with other bad flag (for which bpm>2) - # TODO: To my mind, we should never be using the value of the bit to - # check for flags. We should be using the BitMask functions. I *think* - # what you want is this: - self.reduce_bpm = self.slits.bitmask.flagged( - self.slits.mask, - exclude='BOXSLIT', - and_not=self.slits.bitmask.exclude_for_reducing) -# self.reduce_bpm = (self.slits.mask > 2) & (np.invert(self.slits.bitmask.flagged( -# self.slits.mask, flag=self.slits.bitmask.exclude_for_reducing))) - # I.e., mask anything *except* slits flagged by only 'BOXSLIT', and also - # make sure any of the `exclude_for_reducing` flags are not on. This - # previous code may also have included slits that were flagged as - # SHORTSLIT. Was that on purpose? + self.reduce_bpm = (self.slits.mask > 2) & (np.invert(self.slits.bitmask.flagged( + self.slits.mask, flag=self.slits.bitmask.exclude_for_reducing))) self.reduce_bpm_init = self.reduce_bpm.copy() # These may be None (i.e. COADD2D) @@ -382,12 +371,9 @@ def prepare_extraction(self, global_sky): """ Prepare the masks and wavelength image for extraction. """ # Update bpm mask to remove `BOXSLIT`, i.e., we don't want to extract those - self.reduce_bpm = self.slits.bitmask.flagged( - self.slits.mask, - and_not=self.slits.bitmask.exclude_for_reducing) -# self.reduce_bpm = (self.slits.mask > 0) & \ -# (np.invert(self.slits.bitmask.flagged(self.slits.mask, -# flag=self.slits.bitmask.exclude_for_reducing))) + self.reduce_bpm = (self.slits.mask > 0) & \ + (np.invert(self.slits.bitmask.flagged(self.slits.mask, + flag=self.slits.bitmask.exclude_for_reducing))) # Update Slitmask to remove `BOXSLIT`, i.e., we don't want to extract those self.slitmask = self.slits.slit_img(flexure=self.spat_flexure_shift, exclude_flag=self.slits.bitmask.exclude_for_reducing) @@ -668,12 +654,10 @@ def global_skysub(self, skymask=None, update_crmask=True, trim_edg=(3,3), sigrej = 3.0 # We use this tmp bpm so that we exclude the BOXSLITS during the global_skysub - tmp_bpm = self.slits.bitmask.flagged(self.slits.mask, - and_not=self.slits.bitmask.exclude_for_reducing) -# tmp_bpm = (self.slits.mask > 0) & \ -# (np.invert(self.slits.bitmask.flagged(self.slits.mask, -# flag=self.slits.bitmask.exclude_for_reducing))) - gdslits = np.where(np.logical_not(tmp_bpm))[0] + tmp_bpm = (self.slits.mask > 0) & \ + (np.invert(self.slits.bitmask.flagged(self.slits.mask, + flag=self.slits.bitmask.exclude_for_reducing))) + gdslits = np.where(np.invert(tmp_bpm))[0] # Mask objects using the skymask? If skymask has been set by objfinding, and masking is requested, then do so skymask_now = skymask if (skymask is not None) else np.ones_like(self.sciImg.image, dtype=bool) From 91822a4202ceaf79f9d7cdb49f85a40d6431f03f Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Mon, 22 Jan 2024 15:34:36 -0800 Subject: [PATCH 207/244] PR comments --- doc/help/pypeit_chk_tilts.rst | 2 +- doc/releases/1.14.1dev.rst | 6 +++++- pypeit/scripts/arxiv_solution.py | 4 +++- pypeit/scripts/chk_alignments.py | 3 ++- pypeit/scripts/chk_edges.py | 6 ++++-- pypeit/scripts/chk_flats.py | 6 ++++-- pypeit/scripts/chk_noise_1dspec.py | 7 +++++-- pypeit/scripts/chk_noise_2dspec.py | 5 +++-- pypeit/scripts/chk_scattlight.py | 9 +++++---- pypeit/scripts/chk_tilts.py | 8 +++++--- pypeit/scripts/chk_wavecalib.py | 7 ++++--- pypeit/scripts/edge_inspector.py | 4 +++- pypeit/scripts/flux_calib.py | 5 ++++- pypeit/scripts/identify.py | 8 +++++--- pypeit/scripts/parse_slits.py | 8 ++++---- pypeit/scripts/ql.py | 5 +++-- pypeit/scripts/show_2dspec.py | 5 +++-- pypeit/scripts/show_wvcalib.py | 7 ++++--- pypeit/scripts/skysub_regions.py | 5 +++-- pypeit/slittrace.py | 8 +++++++- pypeit/wavecalib.py | 8 ++++---- 21 files changed, 81 insertions(+), 45 deletions(-) diff --git a/doc/help/pypeit_chk_tilts.rst b/doc/help/pypeit_chk_tilts.rst index b286323586..a7440699a2 100644 --- a/doc/help/pypeit_chk_tilts.rst +++ b/doc/help/pypeit_chk_tilts.rst @@ -17,5 +17,5 @@ Ginga). If not set, only the fitted, masked and rejected in the fit tilts are shown. (default: False) --try_old Attempt to load old datamodel versions. A crash may ensue.. - (default: True) + (default: False) \ No newline at end of file diff --git a/doc/releases/1.14.1dev.rst b/doc/releases/1.14.1dev.rst index 0133908452..76bf406169 100644 --- a/doc/releases/1.14.1dev.rst +++ b/doc/releases/1.14.1dev.rst @@ -85,7 +85,9 @@ Datamodel Changes ----------------- - A wavelength array is now stored for DataCube() -- Wavecalib and Wavefit datacontainers now store information about echelle order number, if applicable. +- Wavecalib and Wavefit datacontainers now store information about echelle order + number, if applicable. +- Change to how Slit datamodel stores and checks bit flag values. Under-the-hood Improvements --------------------------- @@ -96,6 +98,8 @@ Under-the-hood Improvements - Improvements to echelle wavelength calibration. Code changes in: ``pypeit/core/wavecal/wvutils.py``, ``pypeit/core/wavecal/echelle.py``, ``pypeit/core/wavecal/autoid.py``, ``pypeit/wavecalib.py``. +- More extensive propagation of turning off datamodel version checking (using + ``try_old`` and ``chk_version``) throughout the code. Bug Fixes --------- diff --git a/pypeit/scripts/arxiv_solution.py b/pypeit/scripts/arxiv_solution.py index bf4f7139fc..160a622a1a 100644 --- a/pypeit/scripts/arxiv_solution.py +++ b/pypeit/scripts/arxiv_solution.py @@ -35,6 +35,8 @@ def main(args): from pypeit.wavecalib import WaveCalib from pypeit.core.wavecal import wvutils + chk_version = not args.try_old + # Set the verbosity, and create a logfile if verbosity == 2 msgs.set_logfile_and_verbosity('arxiv_solution', args.verbosity) @@ -45,7 +47,7 @@ def main(args): msgs.error("The following MasterWaveCalib file does not exist:" + msgs.newline() + args.file) # Load the wavelength calibration file - wv_calib = WaveCalib.from_file(args.file, chk_version=(not args.try_old)) + wv_calib = WaveCalib.from_file(args.file, chk_version=chk_version) # Check if a wavelength solution exists if wv_calib['wv_fits'][args.slit]['wave_soln'] is None: gd_slits = [] diff --git a/pypeit/scripts/chk_alignments.py b/pypeit/scripts/chk_alignments.py index a9a3127187..5882328a32 100644 --- a/pypeit/scripts/chk_alignments.py +++ b/pypeit/scripts/chk_alignments.py @@ -30,7 +30,8 @@ def main(args): from pypeit import alignframe # Load - alignments = alignframe.Alignments.from_file(args.file, chk_version=(not args.try_old)) + chk_version = not args.try_old + alignments = alignframe.Alignments.from_file(args.file, chk_version=chk_version) # Show alignments.show() diff --git a/pypeit/scripts/chk_edges.py b/pypeit/scripts/chk_edges.py index 2d8a8e593c..d043e76e58 100644 --- a/pypeit/scripts/chk_edges.py +++ b/pypeit/scripts/chk_edges.py @@ -35,8 +35,10 @@ def main(args): from pypeit import edgetrace, slittrace, msgs + chk_version = not args.try_old + # Load - edges = edgetrace.EdgeTraceSet.from_file(args.trace_file, chk_version=(not args.try_old)) + edges = edgetrace.EdgeTraceSet.from_file(args.trace_file, chk_version=chk_version) if args.mpl: edges.show(thin=10, include_img=True, idlabel=True) @@ -60,7 +62,7 @@ def main(args): msgs.warn(f'{slit_filename} does not exist!') # NOTE: At this point, slit_filename *must* be a Path object - slits = slittrace.SlitTraceSet.from_file(slit_filename, chk_version=(not args.try_old)) \ + slits = slittrace.SlitTraceSet.from_file(slit_filename, chk_version=chk_version) \ if slit_filename.exists() else None edges.show(thin=10, in_ginga=True, slits=slits) diff --git a/pypeit/scripts/chk_flats.py b/pypeit/scripts/chk_flats.py index e6c53d28c6..19f557eb9f 100644 --- a/pypeit/scripts/chk_flats.py +++ b/pypeit/scripts/chk_flats.py @@ -27,9 +27,11 @@ def main(args): from pypeit import flatfield + chk_version = not args.try_old + # Load - flatImages = flatfield.FlatImages.from_file(args.file, chk_version=(not args.try_old)) + flatImages = flatfield.FlatImages.from_file(args.file, chk_version=chk_version) # Show - flatImages.show(args.type, chk_version=(not args.try_old)) + flatImages.show(args.type, chk_version=chk_version) diff --git a/pypeit/scripts/chk_noise_1dspec.py b/pypeit/scripts/chk_noise_1dspec.py index bc75550c95..88577e109b 100644 --- a/pypeit/scripts/chk_noise_1dspec.py +++ b/pypeit/scripts/chk_noise_1dspec.py @@ -175,6 +175,9 @@ def get_parser(cls, width=None): @staticmethod def main(args): + + chk_version = not args.try_old + # Load em line_names, line_wav = utils.list_of_spectral_lines() @@ -200,9 +203,9 @@ def main(args): head = fits.getheader(file) # I/O spec object - specObjs = [OneSpec.from_file(file, chk_version=(not args.try_old))] \ + specObjs = [OneSpec.from_file(file, chk_version=chk_version)] \ if args.fileformat == 'coadd1d' else \ - specobjs.SpecObjs.from_fitsfile(file, chk_version=(not args.try_old)) + specobjs.SpecObjs.from_fitsfile(file, chk_version=chk_version) # loop on the spectra for spec in specObjs: diff --git a/pypeit/scripts/chk_noise_2dspec.py b/pypeit/scripts/chk_noise_2dspec.py index 1401b73d88..95e5287ae9 100644 --- a/pypeit/scripts/chk_noise_2dspec.py +++ b/pypeit/scripts/chk_noise_2dspec.py @@ -180,6 +180,8 @@ def get_parser(cls, width=None): @staticmethod def main(args): + chk_version = not args.try_old + # Parse the detector name try: det = int(args.det) @@ -207,8 +209,7 @@ def main(args): if args.list: io.fits_open(file).info() continue - spec2DObj = spec2dobj.Spec2DObj.from_file(file, detname, - chk_version=(not args.try_old)) + spec2DObj = spec2dobj.Spec2DObj.from_file(file, detname, chk_version=chk_version) # Deal with redshifts if args.z is not None: diff --git a/pypeit/scripts/chk_scattlight.py b/pypeit/scripts/chk_scattlight.py index 8171841f22..548b9fd536 100644 --- a/pypeit/scripts/chk_scattlight.py +++ b/pypeit/scripts/chk_scattlight.py @@ -40,6 +40,8 @@ def main(args): from pypeit.images.detector_container import DetectorContainer from pypeit import io + chk_version = not args.try_old + # Parse the detector name try: det = int(args.det) @@ -49,11 +51,10 @@ def main(args): detname = DetectorContainer.get_name(det) # Load scattered light calibration frame - ScattLightImage = scattlight.ScatteredLight.from_file(args.file, - chk_version=(not args.try_old)) + ScattLightImage = scattlight.ScatteredLight.from_file(args.file, chk_version=chk_version) # Load slits information - slits = slittrace.SlitTraceSet.from_file(args.slits, chk_version=(not args.try_old)) + slits = slittrace.SlitTraceSet.from_file(args.slits, chk_version=chk_version) # Load the alternate file if requested display_frame = None # The default is to display the frame used to calculate the scattered light model @@ -62,7 +63,7 @@ def main(args): try: # TODO :: the spec2d file may have already had the scattered light removed, so this is not correct. This script only works when the scattered light is turned off for the spec2d file spec2D = spec2dobj.Spec2DObj.from_file(args.spec2d, detname, - chk_version=(not args.try_old)) + chk_version=chk_version) except PypeItDataModelError: msgs.warn(f"Error loading spec2d file {args.spec2d} - attempting to load science image from fits") spec2D = None diff --git a/pypeit/scripts/chk_tilts.py b/pypeit/scripts/chk_tilts.py index 4bc9903d47..95f1d2ded2 100644 --- a/pypeit/scripts/chk_tilts.py +++ b/pypeit/scripts/chk_tilts.py @@ -26,7 +26,7 @@ def get_parser(cls, width=None): parser.add_argument('--show_traces', default=False, action='store_true', help='Show the traced tilts. This slows down the plotting (mostly in Ginga). If not set, ' 'only the fitted, masked and rejected in the fit tilts are shown.') - parser.add_argument('--try_old', default=True, action='store_true', + parser.add_argument('--try_old', default=False, action='store_true', help='Attempt to load old datamodel versions. A crash may ensue..') return parser @@ -34,10 +34,12 @@ def get_parser(cls, width=None): def main(args): from pypeit import wavetilts + chk_version = not args.try_old + # Load - tilts = wavetilts.WaveTilts.from_file(args.file, chk_version=(not args.try_old)) + tilts = wavetilts.WaveTilts.from_file(args.file, chk_version=chk_version) tilts.show(in_ginga=np.logical_not(args.mpl), show_traces=args.show_traces, - chk_version=(not args.try_old)) + chk_version=chk_version) diff --git a/pypeit/scripts/chk_wavecalib.py b/pypeit/scripts/chk_wavecalib.py index af9f20cd70..c1a510dcc0 100644 --- a/pypeit/scripts/chk_wavecalib.py +++ b/pypeit/scripts/chk_wavecalib.py @@ -29,6 +29,8 @@ def main(args): from astropy.io import fits from pypeit import wavecalib, spec2dobj, msgs + chk_version = not args.try_old + # Loop over the input files for in_file in args.input_file: @@ -45,13 +47,12 @@ def main(args): msgs.error("Bad file type input!") if file_type == 'WaveCalib': - waveCalib = wavecalib.WaveCalib.from_file(in_file, chk_version=(not args.try_old)) + waveCalib = wavecalib.WaveCalib.from_file(in_file, chk_version=chk_version) waveCalib.wave_diagnostics(print_diag=True) continue elif file_type == 'AllSpec2D': - allspec2D = spec2dobj.AllSpec2DObj.from_fits(in_file, - chk_version=(not args.try_old)) + allspec2D = spec2dobj.AllSpec2DObj.from_fits(in_file, chk_version=chk_version) for det in allspec2D.detectors: print('') print('='*50 + f'{det:^7}' + '='*51) diff --git a/pypeit/scripts/edge_inspector.py b/pypeit/scripts/edge_inspector.py index 0874d881db..c0cf4c6d12 100644 --- a/pypeit/scripts/edge_inspector.py +++ b/pypeit/scripts/edge_inspector.py @@ -29,10 +29,12 @@ def main(args): from pypeit import edgetrace from pypeit.core.gui import edge_inspector + chk_version = not args.try_old + # Set the file name to the full path trace_file = Path(args.trace_file).resolve() # Load - edges = edgetrace.EdgeTraceSet.from_file(trace_file, chk_version=(not args.try_old)) + edges = edgetrace.EdgeTraceSet.from_file(trace_file, chk_version=chk_version) # Inspector object pointer = edge_inspector.EdgeInspectorGUI(edges) # Run. Ends when window is closed diff --git a/pypeit/scripts/flux_calib.py b/pypeit/scripts/flux_calib.py index 2962c0c822..8155da6a51 100644 --- a/pypeit/scripts/flux_calib.py +++ b/pypeit/scripts/flux_calib.py @@ -74,6 +74,9 @@ def get_parser(cls, width=None): def main(args): """ Runs fluxing steps """ + + chk_version = not args.try_old + # Set the verbosity, and create a logfile if verbosity == 2 msgs.set_logfile_and_verbosity('flux_calib', args.verbosity) @@ -111,7 +114,7 @@ def main(args): # Instantiate fluxcalibrate.flux_calibrate(fluxFile.filenames, sensfiles, par=par['fluxcalib'], - chk_version=(not args.try_old)) + chk_version=chk_version) msgs.info('Flux calibration complete') return 0 diff --git a/pypeit/scripts/identify.py b/pypeit/scripts/identify.py index 41bde34050..f69cb5f9f1 100644 --- a/pypeit/scripts/identify.py +++ b/pypeit/scripts/identify.py @@ -60,11 +60,13 @@ def main(args): from pypeit import slittrace from pypeit.images.buildimage import ArcImage + chk_version = not args.try_old + # Set the verbosity, and create a logfile if verbosity == 2 msgs.set_logfile_and_verbosity('identify', args.verbosity) # Load the Arc file - msarc = ArcImage.from_file(args.arc_file, chk_version=(not args.try_old)) + msarc = ArcImage.from_file(args.arc_file, chk_version=chk_version) # Load the spectrograph spec = load_spectrograph(msarc.PYP_SPEC) @@ -80,13 +82,13 @@ def main(args): par['lamps'] = lamps # Load the slits - slits = slittrace.SlitTraceSet.from_file(args.slits_file, chk_version=(not args.try_old)) + slits = slittrace.SlitTraceSet.from_file(args.slits_file, chk_version=chk_version) # Reset the mask slits.mask = slits.mask_init # Check if a solution exists solnname = WaveCalib.construct_file_name(msarc.calib_key, calib_dir=msarc.calib_dir) - wv_calib = WaveCalib.from_file(solnname, chk_version=(not args.try_old)) \ + wv_calib = WaveCalib.from_file(solnname, chk_version=chk_version) \ if os.path.exists(solnname) and args.solution else None # Load the calibration frame (if it exists and is desired). Bad-pixel mask diff --git a/pypeit/scripts/parse_slits.py b/pypeit/scripts/parse_slits.py index b99f33e5d6..d5d58bee26 100644 --- a/pypeit/scripts/parse_slits.py +++ b/pypeit/scripts/parse_slits.py @@ -66,6 +66,8 @@ def get_parser(cls, width=None): @staticmethod def main(args): + chk_version = not args.try_old + # What kind of file are we?? hdul = fits.open(args.input_file) head0 = hdul[0].header @@ -81,14 +83,12 @@ def main(args): msgs.error("Bad file type input!") if file_type == 'Slits': - slits = slittrace.SlitTraceSet.from_file(args.input_file, - chk_version=(not args.try_old)) + slits = slittrace.SlitTraceSet.from_file(args.input_file, chk_version=chk_version) print('') print_slits(slits) elif file_type == 'AllSpec2D': - allspec2D = spec2dobj.AllSpec2DObj.from_fits(args.input_file, - chk_version=(not args.try_old)) + allspec2D = spec2dobj.AllSpec2DObj.from_fits(args.input_file, chk_version=chk_version) # Loop on Detectors for det in allspec2D.detectors: print('') diff --git a/pypeit/scripts/ql.py b/pypeit/scripts/ql.py index 1b91d9d798..2049632882 100644 --- a/pypeit/scripts/ql.py +++ b/pypeit/scripts/ql.py @@ -744,7 +744,6 @@ def get_parser(cls, width=None): help='If coadding, adjust the spatial grid sampling by this ' 'factor. For a finer grid, set value to <1.0; for coarser ' 'sampling, set value to >1.0).') - parser.add_argument('--try_old', default=False, action='store_true', help='Attempt to load old datamodel versions. A crash may ensue..') @@ -756,6 +755,8 @@ def main(args): tstart = time.perf_counter() + chk_version = not args.try_old + # Parse the raw files files = get_files(args.raw_files, args.raw_path) if len(files) == 0: @@ -965,7 +966,7 @@ def main(args): maskID=args.maskID, boxcar_radius=args.boxcar_radius, snr_thresh=args.snr_thresh, - chk_version=(not args.try_old)) + chk_version=chk_version) # Run it pypeIt = pypeit.PypeIt(sci_pypeit_file, reuse_calibs=True) diff --git a/pypeit/scripts/show_2dspec.py b/pypeit/scripts/show_2dspec.py index 8e1c98ced7..e426872f47 100644 --- a/pypeit/scripts/show_2dspec.py +++ b/pypeit/scripts/show_2dspec.py @@ -116,6 +116,8 @@ def get_parser(cls, width=None): @staticmethod def main(args): + chk_version = not args.try_old + # List only? if args.list: io.fits_open(args.file).info() @@ -139,8 +141,7 @@ def main(args): # Try to read the Spec2DObj using the current datamodel, but allowing # for the datamodel version to be different try: - spec2DObj = spec2dobj.Spec2DObj.from_file(args.file, detname, - chk_version=(not args.try_old)) + spec2DObj = spec2dobj.Spec2DObj.from_file(args.file, detname, chk_version=chk_version) except PypeItDataModelError: try: # Try to get the pypeit version used to write this file diff --git a/pypeit/scripts/show_wvcalib.py b/pypeit/scripts/show_wvcalib.py index fd6d07b9ac..712234c5e2 100644 --- a/pypeit/scripts/show_wvcalib.py +++ b/pypeit/scripts/show_wvcalib.py @@ -37,11 +37,12 @@ def main(args, unit_test=False): from matplotlib import pyplot as plt + chk_version = not args.try_old + # Load - wvcalib = wavecalib.WaveCalib.from_file(args.file, chk_version=(not args.try_old)) + wvcalib = wavecalib.WaveCalib.from_file(args.file, chk_version=chk_version) if args.slit_file is not None: - slits = slittrace.SlitTraceSet.from_file(args.slit_file, - chk_version=(not args.try_old)) + slits = slittrace.SlitTraceSet.from_file(args.slit_file, chk_version=chk_version) # Parse if args.is_order: diff --git a/pypeit/scripts/skysub_regions.py b/pypeit/scripts/skysub_regions.py index d492271a09..516d24a691 100644 --- a/pypeit/scripts/skysub_regions.py +++ b/pypeit/scripts/skysub_regions.py @@ -48,6 +48,8 @@ def main(args): from pypeit.images.detector_container import DetectorContainer from pypeit.edgetrace import EdgeTraceSet + chk_version = not args.try_old + # Parse the detector name try: det = int(args.det) @@ -57,8 +59,7 @@ def main(args): detname = DetectorContainer.get_name(det) # Load it up - spec2DObj = spec2dobj.Spec2DObj.from_file(args.file, detname, - chk_version=(not args.try_old)) + spec2DObj = spec2dobj.Spec2DObj.from_file(args.file, detname, chk_version=chk_version) frame = spec2DObj.sciimg hdr = fits.open(args.file)[0].header fname = hdr['FILENAME'] diff --git a/pypeit/slittrace.py b/pypeit/slittrace.py index b37673d0f1..0d6845abcf 100644 --- a/pypeit/slittrace.py +++ b/pypeit/slittrace.py @@ -335,7 +335,13 @@ def from_hdu(cls, hdu, chk_version=True, **kwargs): # Instantiate self = super().from_dict(d=d) - + + # Calibration frame attributes + # NOTE: If multiple HDUs are parsed, this assumes that the information + # necessary to set all the calib internals is always in *every* header. + # BEWARE! + self.calib_keys_from_header(hdu[parsed_hdus[0]].header) + # Check the bitmasks. Bits should have been written to *any* header # associated with the object hdr = hdu[parsed_hdus[0]].header if isinstance(hdu, fits.HDUList) else hdu.header diff --git a/pypeit/wavecalib.py b/pypeit/wavecalib.py index a9287bd431..765d547d15 100644 --- a/pypeit/wavecalib.py +++ b/pypeit/wavecalib.py @@ -182,28 +182,28 @@ def _parse(cls, hdu, **kwargs): else: # TODO -- Replace the following with WaveFit._parse() and pass that back!! iwavefit = wv_fitting.WaveFit.from_hdu(ihdu)# , chk_version=False) - parsed_hdus += ihdu.name + parsed_hdus += [ihdu.name] if iwavefit.version != wv_fitting.WaveFit.version: msgs.warn("Your WaveFit is out of date!!") # Grab PypeItFit (if it exists) hdname = ihdu.name.replace('WAVEFIT', 'PYPEITFIT') if hdname in [khdu.name for khdu in hdu]: iwavefit.pypeitfit = fitting.PypeItFit.from_hdu(hdu[hdname]) - parsed_hdus += hdname + parsed_hdus += [hdname] list_of_wave_fits.append(iwavefit) # Grab SPAT_ID for checking spat_ids.append(iwavefit.spat_id) elif 'WAVE2DFIT' in ihdu.name: iwave2dfit = fitting.PypeItFit.from_hdu(ihdu) list_of_wave2d_fits.append(iwave2dfit) - parsed_hdus += ihdu.name + parsed_hdus += [ihdu.name] elif 'FWHMFIT' in ihdu.name: # TODO: This is a hack. We shouldn't be writing empty HDUs, # except for the primary HDU. ifwhmfit = fitting.PypeItFit() if len(ihdu.data) == 0 \ else fitting.PypeItFit.from_hdu(ihdu) list_of_fwhm_fits.append(ifwhmfit) - parsed_hdus += ihdu.name + parsed_hdus += [ihdu.name] # Check if spat_ids != _d['spat_ids'].tolist(): #embed(header="198 of wavecalib.py") From 2a252393295b3998935997bdecb478e00812cb82 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Mon, 22 Jan 2024 16:03:40 -0800 Subject: [PATCH 208/244] PR comments --- doc/calibrations/wvcalib.rst | 16 ++++++++-------- doc/tutorials/lris_howto.rst | 14 +++++++------- doc/tutorials/mosfire_howto.rst | 20 ++++++++++---------- doc/tutorials/nires_howto.rst | 14 +++++++------- pypeit/par/pypeitpar.py | 16 ---------------- pypeit/scripts/chk_wavecalib.py | 2 +- pypeit/scripts/collate_1d.py | 16 ++-------------- pypeit/slittrace.py | 8 +++++++- pypeit/wavecalib.py | 5 ++--- 9 files changed, 44 insertions(+), 67 deletions(-) diff --git a/doc/calibrations/wvcalib.rst b/doc/calibrations/wvcalib.rst index 1f7367c706..148268c118 100644 --- a/doc/calibrations/wvcalib.rst +++ b/doc/calibrations/wvcalib.rst @@ -31,13 +31,13 @@ with the **pypeit_chk_wavecalib** script, e.g. : $ pypeit_chk_wavecalib Calibrations/WaveCalib_A_1_MSC03.fits - N. SpatID minWave Wave_cen maxWave dWave Nlin IDs_Wave_range IDs_Wave_cov(%) mesured_fwhm RMS - --- ------ ------- -------- ------- ----- ---- --------------------- --------------- ------------ ----- - 0 35 6422.5 7753.8 9053.2 0.325 48 6508.325 - 9047.930 96.5 3.5 0.046 - 1 93 6310.0 7641.4 8940.8 0.325 49 6336.179 - 8931.145 98.6 3.6 0.036 - 2 140 6440.8 7772.1 9071.5 0.325 47 6508.325 - 9047.930 96.5 3.6 0.049 - 3 184 6301.2 7632.6 8932.0 0.325 50 6306.533 - 8931.145 99.8 3.6 0.037 - 4 243 6257.1 7588.5 8887.9 0.325 49 6268.229 - 8821.832 97.1 3.6 0.034 + N. SpatID minWave Wave_cen maxWave dWave Nlin IDs_Wave_range IDs_Wave_cov(%) measured_fwhm RMS + --- ------ ------- -------- ------- ----- ---- --------------------- --------------- ------------- ----- + 0 35 6422.5 7753.8 9053.2 0.325 48 6508.325 - 9047.930 96.5 3.5 0.046 + 1 93 6310.0 7641.4 8940.8 0.325 49 6336.179 - 8931.145 98.6 3.6 0.036 + 2 140 6440.8 7772.1 9071.5 0.325 47 6508.325 - 9047.930 96.5 3.6 0.049 + 3 184 6301.2 7632.6 8932.0 0.325 50 6306.533 - 8931.145 99.8 3.6 0.037 + 4 243 6257.1 7588.5 8887.9 0.325 49 6268.229 - 8821.832 97.1 3.6 0.034 - ``SpatID`` is the spatial position of the slit/order. @@ -50,7 +50,7 @@ with the **pypeit_chk_wavecalib** script, e.g. : number, the wavelength range, and the spectral coverage of the identified and fitted arc lines. -- ``mesured_fwhm`` is the measured arc lines FWHM (in binned pixels of the input +- ``measured_fwhm`` is the measured arc lines FWHM (in binned pixels of the input arc frame), i.e, the approximate spectral resolution. Note that this not necessarily the ``fwhm`` used to identify the arc lines during the wavelength calibration, see :ref:`wvcalib-fwhm`. diff --git a/doc/tutorials/lris_howto.rst b/doc/tutorials/lris_howto.rst index 3bb07e011c..61e46535f9 100644 --- a/doc/tutorials/lris_howto.rst +++ b/doc/tutorials/lris_howto.rst @@ -287,13 +287,13 @@ and it prints on screen the following (here truncated) table: .. code-block:: bash - N. SpatID minWave Wave_cen maxWave dWave Nlin IDs_Wave_range IDs_Wave_cov(%) mesured_fwhm RMS - --- ------ ------- -------- ------- ----- ---- --------------------- --------------- ------------ ----- - 0 212 6835.4 8469.9 10121.3 1.610 43 6931.379 - 9925.919 91.1 2.7 0.027 - 1 265 6038.3 7674.1 9327.6 1.612 45 6336.179 - 9227.030 87.9 2.7 0.025 - 2 335 7169.5 8831.4 10485.5 1.611 39 7427.339 - 10472.923 91.8 2.7 0.348 - 3 384 0.0 0.0 0.0 0.000 0 0.000 - 0.000 0.0 0.0 0.000 - 4 474 0.0 0.0 0.0 0.000 0 0.000 - 0.000 0.0 0.0 0.000 + N. SpatID minWave Wave_cen maxWave dWave Nlin IDs_Wave_range IDs_Wave_cov(%) measured_fwhm RMS + --- ------ ------- -------- ------- ----- ---- --------------------- --------------- ------------- ----- + 0 212 6835.4 8469.9 10121.3 1.610 43 6931.379 - 9925.919 91.1 2.7 0.027 + 1 265 6038.3 7674.1 9327.6 1.612 45 6336.179 - 9227.030 87.9 2.7 0.025 + 2 335 7169.5 8831.4 10485.5 1.611 39 7427.339 - 10472.923 91.8 2.7 0.348 + 3 384 0.0 0.0 0.0 0.000 0 0.000 - 0.000 0.0 0.0 0.000 + 4 474 0.0 0.0 0.0 0.000 0 0.000 - 0.000 0.0 0.0 0.000 See :ref:`pypeit-chk-wavecalib` for a detailed description of all the columns. Note that the slits with ``SpatID`` 384 and 474 have all the values set to 0.0. This is because diff --git a/doc/tutorials/mosfire_howto.rst b/doc/tutorials/mosfire_howto.rst index afb605aed2..3843f7afee 100644 --- a/doc/tutorials/mosfire_howto.rst +++ b/doc/tutorials/mosfire_howto.rst @@ -348,16 +348,16 @@ and it prints on screen the following: .. code-block:: bash - N. SpatID minWave Wave_cen maxWave dWave Nlin IDs_Wave_range IDs_Wave_cov(%) mesured_fwhm RMS - --- ------ ------- -------- ------- ----- ---- --------------------- --------------- ------------ ----- - 0 306 20252.3 22473.7 24689.0 2.167 32 20275.839 - 23914.989 82.0 2.7 0.041 - 1 656 19145.6 21364.7 23574.6 2.164 49 19193.537 - 22741.961 80.1 2.6 0.048 - 2 766 19963.1 22181.5 24392.9 2.163 33 20007.951 - 23914.989 88.2 2.6 0.028 - 3 877 19232.6 21450.8 23659.5 2.163 48 19250.306 - 22741.961 78.9 2.5 0.050 - 4 1010 19213.0 21430.9 23641.0 2.163 48 19250.306 - 22741.961 78.9 2.6 0.047 - 5 1297 20173.6 22392.2 24603.7 2.164 32 20193.227 - 23914.989 84.0 2.6 0.036 - 6 1653 19244.3 21465.0 23678.2 2.166 49 19250.306 - 22741.961 78.7 2.7 0.060 - 7 1920 19251.7 21475.0 23689.8 2.169 47 19350.119 - 22741.961 76.4 2.7 0.042 + N. SpatID minWave Wave_cen maxWave dWave Nlin IDs_Wave_range IDs_Wave_cov(%) measured_fwhm RMS + --- ------ ------- -------- ------- ----- ---- --------------------- --------------- ------------- ----- + 0 306 20252.3 22473.7 24689.0 2.167 32 20275.839 - 23914.989 82.0 2.7 0.041 + 1 656 19145.6 21364.7 23574.6 2.164 49 19193.537 - 22741.961 80.1 2.6 0.048 + 2 766 19963.1 22181.5 24392.9 2.163 33 20007.951 - 23914.989 88.2 2.6 0.028 + 3 877 19232.6 21450.8 23659.5 2.163 48 19250.306 - 22741.961 78.9 2.5 0.050 + 4 1010 19213.0 21430.9 23641.0 2.163 48 19250.306 - 22741.961 78.9 2.6 0.047 + 5 1297 20173.6 22392.2 24603.7 2.164 32 20193.227 - 23914.989 84.0 2.6 0.036 + 6 1653 19244.3 21465.0 23678.2 2.166 49 19250.306 - 22741.961 78.7 2.7 0.060 + 7 1920 19251.7 21475.0 23689.8 2.169 47 19350.119 - 22741.961 76.4 2.7 0.042 See :ref:`pypeit-chk-wavecalib` for a detailed description of all the columns. diff --git a/doc/tutorials/nires_howto.rst b/doc/tutorials/nires_howto.rst index f17a10677d..6fd9923892 100644 --- a/doc/tutorials/nires_howto.rst +++ b/doc/tutorials/nires_howto.rst @@ -236,13 +236,13 @@ terminal to see the full output): .. code-block:: bash - N. SpatID minWave Wave_cen maxWave dWave Nlin IDs_Wave_range IDs_Wave_cov(%) mesured_fwhm RMS - --- ------ ------- -------- ------- ----- ---- --------------------- --------------- ------------ ----- - 0 234 8131.7 9408.9 10646.5 1.225 30 9793.676 - 10527.657 29.2 2.1 0.085 - 1 416 9496.8 10961.4 12401.6 1.419 91 9793.676 - 12351.597 88.1 2.1 0.078 - 2 574 11380.8 13133.2 14860.1 1.699 95 11439.783 - 14833.093 97.5 2.1 0.129 - 3 720 14203.1 16389.5 18547.8 2.122 106 14227.201 - 18526.181 98.9 2.2 0.106 - 4 885 18895.7 21806.9 24686.8 2.827 84 18914.824 - 24627.748 98.6 2.0 0.165 + N. SpatID minWave Wave_cen maxWave dWave Nlin IDs_Wave_range IDs_Wave_cov(%) measured_fwhm RMS + --- ------ ------- -------- ------- ----- ---- --------------------- --------------- ------------- ----- + 0 234 8131.7 9408.9 10646.5 1.225 30 9793.676 - 10527.657 29.2 2.1 0.085 + 1 416 9496.8 10961.4 12401.6 1.419 91 9793.676 - 12351.597 88.1 2.1 0.078 + 2 574 11380.8 13133.2 14860.1 1.699 95 11439.783 - 14833.093 97.5 2.1 0.129 + 3 720 14203.1 16389.5 18547.8 2.122 106 14227.201 - 18526.181 98.9 2.2 0.106 + 4 885 18895.7 21806.9 24686.8 2.827 84 18914.824 - 24627.748 98.6 2.0 0.165 See :ref:`pypeit-chk-wavecalib` for a detailed description of all the columns. diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index fc3305b43c..945b5fa0b6 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -1321,17 +1321,6 @@ def __init__(self, ex_value=None, flux_value=None, nmaskedge=None, dtypes['coaddfile'] = str descr['coaddfile'] = 'Output filename' - # NOTE: Deprecated in favor of the parameter in ReduxPar! -# defaults['chk_version'] = True -# dtypes['chk_version'] = bool -# descr['chk_version'] = 'If True, enforce strict PypeIt version checking to ensure that ' \ -# 'all reused PypeIt files were created with the current version ' \ -# 'of PypeIt. If set to False, the code will attempt to read ' \ -# 'out-of-date files and keep going. Beware (!!) that this can ' \ -# 'lead to unforeseen bugs that either cause the code to crash or ' \ -# 'produce erroneous results. I.e., you really need to know what ' \ -# 'you are doing if you set this to False!' - # Instantiate the parameter set super(Coadd1DPar, self).__init__(list(pars.keys()), values=list(pars.values()), @@ -5261,11 +5250,6 @@ def __init__(self, tolerance=None, dry_run=None, ignore_flux=None, flux=None, ma descr['refframe'] = 'Perform reference frame correction prior to coadding. ' \ 'Options are: {0}'.format(', '.join(options['refframe'])) - # NOTE: Deprecated in favor of the parameter in ReduxPar! -# defaults['chk_version'] = False -# dtypes['chk_version'] = bool -# descr['chk_version'] = "Whether to check the data model versions of spec1d files and sensfunc files." - # Instantiate the parameter set super(Collate1DPar, self).__init__(list(pars.keys()), values=list(pars.values()), diff --git a/pypeit/scripts/chk_wavecalib.py b/pypeit/scripts/chk_wavecalib.py index c1a510dcc0..9c8e767e1b 100644 --- a/pypeit/scripts/chk_wavecalib.py +++ b/pypeit/scripts/chk_wavecalib.py @@ -58,7 +58,7 @@ def main(args): print('='*50 + f'{det:^7}' + '='*51) wave_diag = allspec2D[det].wavesol for colname in ['minWave', 'Wave_cen', 'maxWave', 'IDs_Wave_cov(%)', - 'mesured_fwhm']: + 'measured_fwhm']: wave_diag[colname].format = '0.1f' for colname in ['dWave', 'RMS']: wave_diag[colname].format = '0.3f' diff --git a/pypeit/scripts/collate_1d.py b/pypeit/scripts/collate_1d.py index 6bdb60903f..5ea76dec52 100644 --- a/pypeit/scripts/collate_1d.py +++ b/pypeit/scripts/collate_1d.py @@ -124,9 +124,7 @@ def find_slits_to_exclude(spec2d_files, par): exclude_map = dict() for spec2d_file in spec2d_files: - allspec2d = AllSpec2DObj.from_fits(spec2d_file, - chk_version=par['rdx']['chk_version']) -# chk_version=par['collate1d']['chk_version']) + allspec2d = AllSpec2DObj.from_fits(spec2d_file, chk_version=par['rdx']['chk_version']) for sobj2d in [allspec2d[det] for det in allspec2d.detectors]: for (slit_id, mask, slit_mask_id) in sobj2d['slits'].slit_info: for flag in exclude_flags: @@ -249,9 +247,7 @@ def read_spec1d_files(par, spec1d_files, failure_msgs): good_spec1d_files = [] for spec1d_file in spec1d_files: try: - sobjs = SpecObjs.from_fitsfile(spec1d_file, - chk_version=par['rdx']['chk_version']) -# chk_version=par['collate1d']['chk_version']) + sobjs = SpecObjs.from_fitsfile(spec1d_file, chk_version=par['rdx']['chk_version']) specobjs_list.append(sobjs) good_spec1d_files.append(spec1d_file) except Exception as e: @@ -307,7 +303,6 @@ def flux(par, spectrograph, spec1d_files, failed_fluxing_msgs): msgs.info(f"Running flux calibrate on {spec1d_file}") FxCalib = fluxcalibrate.flux_calibrate([spec1d_file], [sens_file], par=par['fluxcalib'], chk_version=par['rdx']['chk_version']) -# chk_version=par['collate1d']['chk_version']) flux_calibrated_files.append(spec1d_file) except Exception: @@ -431,9 +426,6 @@ def coadd(par, coaddfile, source): # Set destination file for coadding par['coadd1d']['coaddfile'] = coaddfile -# # Whether to be forgiving of data model versions -# par['coadd1d']['chk_version'] = par['collate1d']['chk_version'] - # Determine if we should coadd flux calibrated data flux_key = par['coadd1d']['ex_value'] + "_FLAM" @@ -616,7 +608,6 @@ def build_parameters(args): params['collate1d']['refframe'] = args.refframe if args.chk_version is True: -# params['collate1d']['chk_version'] = True params['rdx']['chk_version'] = True return params, spectrograph, spec1d_files @@ -678,9 +669,6 @@ def get_parser(cls, width=None): 'nights and prepare a directory for the KOA.', width=width, formatter=scriptbase.SmartFormatter) -# 'F| chk_version If true, spec1ds and archival sensfuncs must match the currently\n' -# 'F| supported versions. If false (the default) version numbers are not checked.\n' - # TODO: Is the file optional? If so, shouldn't the first argument start # with '--'? parser.add_argument('input_file', type=str, diff --git a/pypeit/slittrace.py b/pypeit/slittrace.py index 0d6845abcf..7fd002c612 100644 --- a/pypeit/slittrace.py +++ b/pypeit/slittrace.py @@ -30,7 +30,13 @@ class SlitTraceBitMask(BitMask): Mask bits used during slit tracing. """ version = '1.0.1' - # TODO: Need a unique bit prefix? + + # TODO: Consider using a unique bit prefix for when the bits are written to + # the header, like so: + # + # prefix = 'SLITB' + # + # It's not necessary, though. def __init__(self): # Only ever append new bits (and don't remove old ones) diff --git a/pypeit/wavecalib.py b/pypeit/wavecalib.py index 765d547d15..e930034bd8 100644 --- a/pypeit/wavecalib.py +++ b/pypeit/wavecalib.py @@ -401,9 +401,8 @@ def wave_diagnostics(self, print_diag=False): diag['IDs_Wave_cov(%)'] = lines_cov diag['IDs_Wave_cov(%)'].format = '0.1f' # FWHM - # TODO: Why not me*a*sured_fwhm? - diag['mesured_fwhm'] = [0. if wvfit.fwhm is None else wvfit.fwhm for wvfit in self.wv_fits] - diag['mesured_fwhm'].format = '0.1f' + diag['measured_fwhm'] = [0. if wvfit.fwhm is None else wvfit.fwhm for wvfit in self.wv_fits] + diag['measured_fwhm'].format = '0.1f' # RMS diag['RMS'] = [0 if wvfit.rms is None else wvfit.rms for wvfit in self.wv_fits] diag['RMS'].format = '0.3f' From 5de20fd42cbfb41da09ac89be8174f76fde1d1e0 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Wed, 24 Jan 2024 07:55:27 -0800 Subject: [PATCH 209/244] import clean-up gone wrong --- pypeit/specobjs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pypeit/specobjs.py b/pypeit/specobjs.py index 3031b08657..15328c3354 100644 --- a/pypeit/specobjs.py +++ b/pypeit/specobjs.py @@ -24,6 +24,9 @@ from pypeit.spectrographs.util import load_spectrograph from pypeit.core import parse from pypeit.images.detector_container import DetectorContainer +# NOTE: Mosaic cannot be found in this module explicitly, but it is used in +# statements like: dmodcls = eval(hdu.header['DMODCLS']) +from pypeit.images.mosaic import Mosaic from pypeit import utils From bfcd69f9c171850a90baa721f7deea95f8c94ab7 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Wed, 24 Jan 2024 13:43:54 -0800 Subject: [PATCH 210/244] testing --- pypeit/edgetrace.py | 74 +++++++++++++++++++++++++++++++++------------ 1 file changed, 55 insertions(+), 19 deletions(-) diff --git a/pypeit/edgetrace.py b/pypeit/edgetrace.py index 684a13d5d8..757ba200cb 100644 --- a/pypeit/edgetrace.py +++ b/pypeit/edgetrace.py @@ -2403,23 +2403,23 @@ def check_synced(self, rebuild_pca=False): """ Quality check and masking of the synchronized edges. - Before executing this method, the slit edges must be - synchronized (see :func:`sync`) and ordered spatially in - left-right pairs (see :func:`spatial_sort`). The former is - checked explicitly. Any traces fully masked as bad (see - :func:`clean_traces`) are removed, along with its - synchronized partner. + Before executing this method, the slit edges must be synchronized (see + :func:`sync`) and ordered spatially in left-right pairs (see + :func:`spatial_sort`); only the former is checked explicitly. Any traces + fully masked as bad (see :func:`clean_traces`) are removed, along with + its synchronized partner. Used parameters from :attr:`par` - (:class:`~pypeit.par.pypeitpar.EdgeTracePar`) are - `minimum_slit_gap`, `minimum_slit_length`, - `minimum_slit_length_sci`, and `length_range`. + (:class:`~pypeit.par.pypeitpar.EdgeTracePar`) are ``minimum_slit_gap``, + ``minimum_slit_length``, ``minimum_slit_length_sci``, and + ``length_range``. Checks are: - - Any trace falling off the edge of the detector is - masked (see :class:`EdgeTraceBitMask`). This is the - only check performed by default (i.e., when no keyword - arguments are provided). + + - Any trace falling off the edge of the detector is masked (see + :class:`EdgeTraceBitMask`). This is the only check performed by + default (i.e., when no keyword arguments are provided). + - Traces that form slit gaps (the median difference between the right and left traces of adjacent slits) that are below an absolute tolerance are removed and @@ -2427,6 +2427,7 @@ def check_synced(self, rebuild_pca=False): the checks of the slit length below such that the merged slit is assessed in any expected slit length constraints. + - Traces that form a slit with a length (the median difference between the left and right edges) below an absolute tolerance (i.e., `right-left < atol`) are @@ -2469,9 +2470,12 @@ def check_synced(self, rebuild_pca=False): """ if self.is_empty: msgs.warn('No traces to check.') + return # Decide if the PCA should be rebuilt _rebuild_pca = rebuild_pca and self.pcatype is not None and self.can_pca() + if rebuild_pca and not _rebuild_pca: + msgs.warn('Rebuilding the PCA was requested but is not possible.') # Remove any fully masked traces and its synced counterpart; # force the removal of traces marked as SYNCERROR, even if @@ -2479,6 +2483,8 @@ def check_synced(self, rebuild_pca=False): self.clean_traces(force_flag='SYNCERROR', rebuild_pca=_rebuild_pca, sync_mode='both', assume_synced=True) + embed(header='check_synced: after clean') + # Use the fit data if available trace_cen = self.edge_cen if self.edge_fit is None else self.edge_fit @@ -2496,6 +2502,8 @@ def check_synced(self, rebuild_pca=False): msgs.error('Edge traces are not yet (or improperly) synced. Either sync() failed ' 'or has not yet been executed.') + embed(header='check_synced: after checking is_synced') + # Parse parameters and report gap_atol = None dlength_rtol = self.par['dlength_range'] @@ -2545,7 +2553,7 @@ def check_synced(self, rebuild_pca=False): msgs.info('Found {0} slit(s) with gaps below {1} arcsec ({2:.2f} pixels).'.format( np.sum(indx), self.par['minimum_slit_gap'], gap_atol)) rmtrace = np.concatenate(([False],np.repeat(indx,2),[False])) - self.remove_traces(rmtrace, rebuild_pca=rebuild_pca) + self.remove_traces(rmtrace, rebuild_pca=_rebuild_pca) # TODO: This should never happen, but keep this around # until we're sure it doesn't. if self.is_empty: @@ -2553,6 +2561,8 @@ def check_synced(self, rebuild_pca=False): # Reset the trace center data to use trace_cen = self.edge_cen if self.edge_fit is None else self.edge_fit + embed(header='check_synced: after gap_atol') + # Calculate the slit length and gap slit_length = np.squeeze(np.diff(trace_cen.reshape(self.nspec,-1,2), axis=-1)) med_slit_length = np.median(slit_length, axis=0) @@ -2607,6 +2617,17 @@ def check_synced(self, rebuild_pca=False): self.edge_msk[:,long] \ = self.bitmask.turn_on(self.edge_msk[:,long], 'ABNORMALSLIT_LONG') + # TODO: Consider removing slits that have large length changes. Like so: +# # Remove traces that have significant changes in their spatial extent +# # along the dispersion direction. +# dl_flag = self.fully_masked_traces(flag='LARGELENGTHCHANGE') +# if np.any(dl_flag): +# msgs.info(f'Removing {np.sum(dl_flag)} traces because of large spatial extent ' +# ' changes along the dispersion direction.') +# self.remove_traces(dl_flag, rebuild_pca=_rebuild_pca) + + embed(header='check_synced: after length checks') + # Get the slits that have been flagged as abnormally short. This should # be the same as the definition above, it's just redone here to ensure # `short` is defined when `length_rtol` is None. @@ -2634,6 +2655,8 @@ def check_synced(self, rebuild_pca=False): # Remove the flagged traces, resort the edges, and rebuild the pca self.remove_traces(rmtrace, rebuild_pca=_rebuild_pca) + embed(header='check_synced: after removing traces prompted by overlap check') + # If this de-synchronizes the traces, we effectively have to start # the synchronization process over again, with the adjustments for # the "short" slits that are assumed to be overlap regions. @@ -2641,6 +2664,9 @@ def check_synced(self, rebuild_pca=False): msgs.info('Checking/cleaning traces for overlap led to de-syncronization.') return False + embed(header='check_synced: after overlap') + exit() + # TODO: Check that slit edges meet a minimum slit gap? # Find all traces to remove @@ -3442,8 +3468,8 @@ def peak_refine(self, rebuild_pca=False, debug=False): troughs in :attr:`sobelsig` are detected and traced. Note that this effectively reinstantiates much of the object - attributes, including :attr:`traceid` :attr:`edge_cen` - :attr:`edge_err` :attr:`edge_msk` :attr:`edge_img` + attributes, including :attr:`traceid`, :attr:`edge_cen`, + :attr:`edge_err`, :attr:`edge_msk`, :attr:`edge_img`, :attr:`edge_fit`, and :attr:`fittype`. Used parameters from :attr:`par` @@ -3528,8 +3554,6 @@ def peak_refine(self, rebuild_pca=False, debug=False): # Iterate through each side for side in ['left', 'right']: # Get the image relevant to tracing -# _sobelsig = trace.prepare_sobel_for_trace(self.sobelsig, bpm=self.bpm, boxcar=5, -# side=side) _sobelsig = self._side_dependent_sobel(side) _pca = self.left_pca if side == 'left' else self.right_pca @@ -3575,7 +3599,15 @@ def peak_refine(self, rebuild_pca=False, debug=False): if ntrace < self.ntrace: msgs.warn('Found fewer traces using peak finding than originally available. ' 'May want to reset peak threshold.') - + + embed(header='peak refine') + exit() + + # TODO: Possibly put in a check that compares the locations of the new + # and old traces, which removes new traces that are too different (above + # some tolerance) from the old traces. Maybe this is a way to catch + # traces that result from poorly constrained polynomial fits. + # Reset the trace data self.traceid = np.zeros(ntrace, dtype=int) self.traceid[:nleft] = -1-np.arange(nleft) @@ -4040,11 +4072,15 @@ def sync(self, rebuild_pca=True, debug=False): msgs.info('Show instance includes inserted traces but before checking the sync.') self.show(flag='any') + embed(header=f'iter {i+1}: after insert') + # Check the full synchronized list and log completion of the # method if self.check_synced(rebuild_pca=rebuild_pca): break + embed(header=f'iter {i+1}: after sync') + i += 1 if i == maxiter: msgs.error('Fatal left-right trace de-synchronization error.') From 0f509807f0c6a08d1cacb9b52893c347158e7496 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 25 Jan 2024 08:40:29 -0800 Subject: [PATCH 211/244] remove peak_refine traces that are too discrepant from originals --- pypeit/core/slitdesign_matching.py | 2 +- pypeit/edgetrace.py | 55 +++++++++++++++++---------- pypeit/par/pypeitpar.py | 61 +++++++++++++++++------------- 3 files changed, 70 insertions(+), 48 deletions(-) diff --git a/pypeit/core/slitdesign_matching.py b/pypeit/core/slitdesign_matching.py index eeac5d2f20..30aeb148e7 100644 --- a/pypeit/core/slitdesign_matching.py +++ b/pypeit/core/slitdesign_matching.py @@ -394,7 +394,7 @@ def match_positions_1D(measured, nominal, tol=None): # Calculate the (m,n) separation matrix # NOTE: This is the brute force approach. For *lots* of measurements, this # can be sped up by using a KDTree to build a sparse matrix with only a - # subset of the distances calculated. + # subset of the separations calculated. sep = np.absolute(nominal[:,None] - measured[None,:]) # Perform the match diff --git a/pypeit/edgetrace.py b/pypeit/edgetrace.py index 8cf654960e..76f182e31d 100644 --- a/pypeit/edgetrace.py +++ b/pypeit/edgetrace.py @@ -2480,8 +2480,6 @@ def check_synced(self, rebuild_pca=False): self.clean_traces(force_flag='SYNCERROR', rebuild_pca=_rebuild_pca, sync_mode='both', assume_synced=True) - embed(header='check_synced: after clean') - # Use the fit data if available trace_cen = self.edge_cen if self.edge_fit is None else self.edge_fit @@ -2499,8 +2497,6 @@ def check_synced(self, rebuild_pca=False): msgs.error('Edge traces are not yet (or improperly) synced. Either sync() failed ' 'or has not yet been executed.') - embed(header='check_synced: after checking is_synced') - # Parse parameters and report gap_atol = None dlength_rtol = self.par['dlength_range'] @@ -2558,8 +2554,6 @@ def check_synced(self, rebuild_pca=False): # Reset the trace center data to use trace_cen = self.edge_cen if self.edge_fit is None else self.edge_fit - embed(header='check_synced: after gap_atol') - # Calculate the slit length and gap slit_length = np.squeeze(np.diff(trace_cen.reshape(self.nspec,-1,2), axis=-1)) med_slit_length = np.median(slit_length, axis=0) @@ -2623,8 +2617,6 @@ def check_synced(self, rebuild_pca=False): # ' changes along the dispersion direction.') # self.remove_traces(dl_flag, rebuild_pca=_rebuild_pca) - embed(header='check_synced: after length checks') - # Get the slits that have been flagged as abnormally short. This should # be the same as the definition above, it's just redone here to ensure # `short` is defined when `length_rtol` is None. @@ -2652,8 +2644,6 @@ def check_synced(self, rebuild_pca=False): # Remove the flagged traces, resort the edges, and rebuild the pca self.remove_traces(rmtrace, rebuild_pca=_rebuild_pca) - embed(header='check_synced: after removing traces prompted by overlap check') - # If this de-synchronizes the traces, we effectively have to start # the synchronization process over again, with the adjustments for # the "short" slits that are assumed to be overlap regions. @@ -2661,9 +2651,6 @@ def check_synced(self, rebuild_pca=False): msgs.info('Checking/cleaning traces for overlap led to de-syncronization.') return False - embed(header='check_synced: after overlap') - exit() - # TODO: Check that slit edges meet a minimum slit gap? # Find all traces to remove @@ -3597,9 +3584,41 @@ def peak_refine(self, rebuild_pca=False, debug=False): msgs.warn('Found fewer traces using peak finding than originally available. ' 'May want to reset peak threshold.') - embed(header='peak refine') - exit() - + if self.par['trace_rms_tol'] is not None: + # Get the PCA reference row. The PCA *must* have been defined to get + # this far (see pcatype test at the beginning of the function). + reference_row = self.left_pca.reference_row if self.par['left_right_pca'] \ + else self.pca.reference_row + # Match the new trace positions to the input ones + gpm = np.logical_not(self.bitmask.flagged(self.edge_msk[reference_row])) + # TODO: Include a tolerance here? Needs testing with more datasets. + peak_indx = slitdesign_matching.match_positions_1D( + self.edge_fit[reference_row][gpm], + fit[reference_row]) + + # Determine the RMS difference between the input and output traces. + # This allows us to compare traces that had alreaday been identified + # to their new measurements resulting from peak_trace, and remove + # them if they are too discrepant from their original form. This is + # largely meant to find and remove poorly constrained traces, where + # the polynomial fit goes wonky. + diff = fit - self.edge_fit.T[gpm][peak_indx].T + rms = np.sqrt(np.mean((diff - np.mean(diff, axis=0)[None,:])**2, axis=0)) + + # TODO: Add a report to the screen or a QA plot? + + # Select traces below the RMS tolerance or that were newly + # identified by peak_trace + indx = (rms < self.par['trace_rms_tol']) | (peak_indx == -1) + if not all(indx): + msgs.info(f'Removing {indx.size - np.sum(indx)} trace(s) due to large RMS ' + 'difference with previous trace locations.') + fit = fit[:,indx] + cen = cen[:,indx] + err = err[:,indx] + msk = msk[:,indx] + nleft -= np.sum(np.where(np.logical_not(indx))[0] < nleft) + # TODO: Possibly put in a check that compares the locations of the new # and old traces, which removes new traces that are too different (above # some tolerance) from the old traces. Maybe this is a way to catch @@ -4069,15 +4088,11 @@ def sync(self, rebuild_pca=True, debug=False): msgs.info('Show instance includes inserted traces but before checking the sync.') self.show(flag='any') - embed(header=f'iter {i+1}: after insert') - # Check the full synchronized list and log completion of the # method if self.check_synced(rebuild_pca=rebuild_pca): break - embed(header=f'iter {i+1}: after sync') - i += 1 if i == maxiter: msgs.error('Fatal left-right trace de-synchronization error.') diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index 8409519610..521ce6dae5 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -3112,20 +3112,20 @@ class EdgeTracePar(ParSet): see :ref:`parameters`. """ prefix = 'ETP' # Prefix for writing parameters to a header is a class attribute - def __init__(self, filt_iter=None, sobel_mode=None, edge_thresh=None, sobel_enhance=None, exclude_regions=None, - follow_span=None, det_min_spec_length=None, max_shift_abs=None, max_shift_adj=None, - max_spat_error=None, match_tol=None, fit_function=None, fit_order=None, - fit_maxdev=None, fit_maxiter=None, fit_niter=None, fit_min_spec_length=None, - auto_pca=None, left_right_pca=None, pca_min_edges=None, pca_n=None, - pca_var_percent=None, pca_function=None, pca_order=None, pca_sigrej=None, - pca_maxrej=None, pca_maxiter=None, smash_range=None, edge_detect_clip=None, - trace_median_frac=None, trace_thresh=None, fwhm_uniform=None, niter_uniform=None, - fwhm_gaussian=None, niter_gaussian=None, det_buffer=None, max_nudge=None, - sync_predict=None, sync_center=None, gap_offset=None, sync_to_edge=None, - bound_detector=None, minimum_slit_dlength=None, dlength_range=None, - minimum_slit_length=None, minimum_slit_length_sci=None, - length_range=None, minimum_slit_gap=None, clip=None, order_match=None, - order_offset=None, add_missed_orders=None, order_width_poly=None, + def __init__(self, filt_iter=None, sobel_mode=None, edge_thresh=None, sobel_enhance=None, + exclude_regions=None, follow_span=None, det_min_spec_length=None, + max_shift_abs=None, max_shift_adj=None, max_spat_error=None, match_tol=None, + fit_function=None, fit_order=None, fit_maxdev=None, fit_maxiter=None, + fit_niter=None, fit_min_spec_length=None, auto_pca=None, left_right_pca=None, + pca_min_edges=None, pca_n=None, pca_var_percent=None, pca_function=None, + pca_order=None, pca_sigrej=None, pca_maxrej=None, pca_maxiter=None, + smash_range=None, edge_detect_clip=None, trace_median_frac=None, trace_thresh=None, + trace_rms_tol=None, fwhm_uniform=None, niter_uniform=None, fwhm_gaussian=None, + niter_gaussian=None, det_buffer=None, max_nudge=None, sync_predict=None, + sync_center=None, gap_offset=None, sync_to_edge=None, bound_detector=None, + minimum_slit_dlength=None, dlength_range=None, minimum_slit_length=None, + minimum_slit_length_sci=None, length_range=None, minimum_slit_gap=None, clip=None, + order_match=None, order_offset=None, add_missed_orders=None, order_width_poly=None, order_gap_poly=None, order_spat_range=None, overlap=None, use_maskdesign=None, maskdesign_maxsep=None, maskdesign_step=None, maskdesign_sigrej=None, pad=None, add_slits=None, add_predict=None, rm_slits=None, maskdesign_filename=None): @@ -3325,6 +3325,13 @@ def __init__(self, filt_iter=None, sobel_mode=None, edge_thresh=None, sobel_enha 'image (see `trace_median_frac`), values in the median-filtered ' \ 'image *below* this threshold are masked in the refitting of ' \ 'the edge trace data. If None, no masking applied.' + + dtypes['trace_rms_tol'] = [int, float] + descr['trace_rms_tol'] = 'After retracing edges using peaks detected in the rectified ' \ + 'and collapsed image, the RMS difference between the original ' \ + 'and refit traces are calculated. This sets the upper limit ' \ + 'of the RMS for traces that will be removed. If None, no ' \ + 'limit is set and all new traces are kept.' defaults['fwhm_uniform'] = 3.0 dtypes['fwhm_uniform'] = [int, float] @@ -3625,19 +3632,19 @@ def from_dict(cls, cfg): # TODO Please provide docs k = np.array([*cfg.keys()]) parkeys = ['filt_iter', 'sobel_mode', 'edge_thresh', 'sobel_enhance', 'exclude_regions', - 'follow_span', 'det_min_spec_length', - 'max_shift_abs', 'max_shift_adj', 'max_spat_error', 'match_tol', 'fit_function', - 'fit_order', 'fit_maxdev', 'fit_maxiter', 'fit_niter', 'fit_min_spec_length', - 'auto_pca', 'left_right_pca', 'pca_min_edges', 'pca_n', 'pca_var_percent', - 'pca_function', 'pca_order', 'pca_sigrej', 'pca_maxrej', 'pca_maxiter', - 'smash_range', 'edge_detect_clip', 'trace_median_frac', 'trace_thresh', - 'fwhm_uniform', 'niter_uniform', 'fwhm_gaussian', 'niter_gaussian', - 'det_buffer', 'max_nudge', 'sync_predict', 'sync_center', 'gap_offset', - 'sync_to_edge', 'bound_detector', 'minimum_slit_dlength', 'dlength_range', - 'minimum_slit_length', 'minimum_slit_length_sci', 'length_range', - 'minimum_slit_gap', 'clip', 'order_match', 'order_offset', 'add_missed_orders', - 'order_width_poly', 'order_gap_poly', 'order_spat_range', 'overlap', - 'use_maskdesign', 'maskdesign_maxsep', 'maskdesign_step', 'maskdesign_sigrej', + 'follow_span', 'det_min_spec_length', 'max_shift_abs', 'max_shift_adj', + 'max_spat_error', 'match_tol', 'fit_function', 'fit_order', 'fit_maxdev', + 'fit_maxiter', 'fit_niter', 'fit_min_spec_length', 'auto_pca', 'left_right_pca', + 'pca_min_edges', 'pca_n', 'pca_var_percent', 'pca_function', 'pca_order', + 'pca_sigrej', 'pca_maxrej', 'pca_maxiter', 'smash_range', 'edge_detect_clip', + 'trace_median_frac', 'trace_thresh', 'trace_rms_tol', 'fwhm_uniform', + 'niter_uniform', 'fwhm_gaussian', 'niter_gaussian', 'det_buffer', 'max_nudge', + 'sync_predict', 'sync_center', 'gap_offset', 'sync_to_edge', 'bound_detector', + 'minimum_slit_dlength', 'dlength_range', 'minimum_slit_length', + 'minimum_slit_length_sci', 'length_range', 'minimum_slit_gap', 'clip', + 'order_match', 'order_offset', 'add_missed_orders', 'order_width_poly', + 'order_gap_poly', 'order_spat_range', 'overlap', 'use_maskdesign', + 'maskdesign_maxsep', 'maskdesign_step', 'maskdesign_sigrej', 'maskdesign_filename', 'pad', 'add_slits', 'add_predict', 'rm_slits'] badkeys = np.array([pk not in parkeys for pk in k]) From 6b3687570dff7be4df7dc53dfa053ecba7f0a3ae Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 25 Jan 2024 08:57:52 -0800 Subject: [PATCH 212/244] doc update --- doc/help/run_pypeit.rst | 2 +- doc/include/class_datamodel_onespec.rst | 3 +- doc/include/class_datamodel_telluric.rst | 4 +- doc/include/datamodel_onespec.rst | 2 +- doc/include/datamodel_sensfunc.rst | 47 +++-- doc/include/datamodel_telluric.rst | 47 +++-- doc/pypeit_par.rst | 212 +++++++++++++---------- pypeit/core/slitdesign_matching.py | 8 +- 8 files changed, 174 insertions(+), 151 deletions(-) diff --git a/doc/help/run_pypeit.rst b/doc/help/run_pypeit.rst index 6a5293f4c7..f65bce9629 100644 --- a/doc/help/run_pypeit.rst +++ b/doc/help/run_pypeit.rst @@ -4,7 +4,7 @@ usage: run_pypeit [-h] [-v VERBOSITY] [-r REDUX_PATH] [-m] [-s] [-o] [-c] pypeit_file - ## PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.14.1.dev477+g4a02a85ca + ## PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.14.1.dev696+g00535ea2e.d20240117 ## ## Available spectrographs include: ## bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, diff --git a/doc/include/class_datamodel_onespec.rst b/doc/include/class_datamodel_onespec.rst index eff409ec4d..a773421d33 100644 --- a/doc/include/class_datamodel_onespec.rst +++ b/doc/include/class_datamodel_onespec.rst @@ -1,5 +1,5 @@ -**Version**: 1.0.1 +**Version**: 1.0.2 ================= ================ ================= ========================================================================================================================================== Attribute Type Array Type Description @@ -11,6 +11,7 @@ Attribute Type Array Type Description ``ivar`` `numpy.ndarray`_ `numpy.floating`_ Inverse variance array (matches units of flux) ``mask`` `numpy.ndarray`_ `numpy.integer`_ Mask array (1=Good,0=Bad) ``obj_model`` `numpy.ndarray`_ `numpy.floating`_ Object model for tellurics +``sigma`` `numpy.ndarray`_ `numpy.floating`_ One sigma noise array, equivalent to 1/sqrt(ivar) (matches units of flux) ``spect_meta`` dict header dict ``telluric`` `numpy.ndarray`_ `numpy.floating`_ Telluric model ``wave`` `numpy.ndarray`_ `numpy.floating`_ Wavelength array (angstroms in vacuum), weighted by pixel contributions diff --git a/doc/include/class_datamodel_telluric.rst b/doc/include/class_datamodel_telluric.rst index 7143c793ac..f9f749f007 100644 --- a/doc/include/class_datamodel_telluric.rst +++ b/doc/include/class_datamodel_telluric.rst @@ -20,8 +20,10 @@ Attribute Type Array Type Description ``std_name`` str Type of standard source ``std_ra`` float RA of the standard source ``std_src`` str Name of the standard source -``telgrid`` str File containing grid of HITRAN atmosphere models +``telgrid`` str File containing PCA components or grid of HITRAN atmosphere models ``tell_norm_thresh`` float ?? +``tell_npca`` int Number of telluric PCA components used +``teltype`` str Type of telluric model, `pca` or `grid` ``tol`` float Relative tolerance for converage of the differential evolution optimization. ``ubound_norm`` float Flux normalization upper bound ``z_qso`` float Redshift of the QSO diff --git a/doc/include/datamodel_onespec.rst b/doc/include/datamodel_onespec.rst index 6eac2e340a..302e3a1fe5 100644 --- a/doc/include/datamodel_onespec.rst +++ b/doc/include/datamodel_onespec.rst @@ -1,5 +1,5 @@ -Version 1.0.1 +Version 1.0.2 ============ ============================== ========= =========================================================== HDU Name HDU Type Data Type Description diff --git a/doc/include/datamodel_sensfunc.rst b/doc/include/datamodel_sensfunc.rst index 2d9e9b5bf2..165d92e3b9 100644 --- a/doc/include/datamodel_sensfunc.rst +++ b/doc/include/datamodel_sensfunc.rst @@ -15,31 +15,28 @@ HDU Name HDU Type Data Type Description TELLURIC table (if present) -================= ========= =========================================================== -Column Data Type Description -================= ========= =========================================================== -``WAVE`` float64 Wavelength vector -``TELLURIC`` float64 Best-fitting telluric model spectrum -``OBJ_MODEL`` float64 Best-fitting object model spectrum -``TELL_THETA`` float64 Best-fitting telluric model parameters -``TELL_PRESS`` float64 Best-fitting telluric model pressure -``TELL_TEMP`` float64 Best-fitting telluric model temperature -``TELL_H2O`` float64 Best-fitting telluric model humidity -``TELL_AIRMASS`` float64 Best-fitting telluric model airmass -``TELL_RESLN`` float64 Best-fitting telluric model spectral resolution -``TELL_SHIFT`` float64 Best-fitting shift applied to the telluric model spectrum -``TELL_STRETCH`` float64 Best-fitting stretch applied to the telluric model spectrum -``OBJ_THETA`` float64 Best-fitting object model parameters -``CHI2`` float64 Chi-square of the best-fit model -``SUCCESS`` bool Flag that fit was successful -``NITER`` int64 Number of fit iterations -``ECH_ORDERS`` int64 Echelle order for this specrum (echelle data only) -``POLYORDER_VEC`` int64 Polynomial order for each slit/echelle (if applicable) -``IND_LOWER`` int64 Lowest index of a spectral pixel included in the fit -``IND_UPPER`` int64 Highest index of a spectral pixel included in the fit -``WAVE_MIN`` float64 Minimum wavelength included in the fit -``WAVE_MAX`` float64 Maximum wavelength included in the fit -================= ========= =========================================================== +================= ========= ================================================================ +Column Data Type Description +================= ========= ================================================================ +``WAVE`` float64 Wavelength vector +``TELLURIC`` float64 Best-fitting telluric model spectrum +``OBJ_MODEL`` float64 Best-fitting object model spectrum +``TELL_THETA`` float64 Best-fitting telluric model parameters +``TELL_PARAM`` float64 Best-fitting telluric atmospheric parameters or PCA coefficients +``TELL_RESLN`` float64 Best-fitting telluric model spectral resolution +``TELL_SHIFT`` float64 Best-fitting shift applied to the telluric model spectrum +``TELL_STRETCH`` float64 Best-fitting stretch applied to the telluric model spectrum +``OBJ_THETA`` float64 Best-fitting object model parameters +``CHI2`` float64 Chi-square of the best-fit model +``SUCCESS`` bool Flag that fit was successful +``NITER`` int64 Number of fit iterations +``ECH_ORDERS`` int64 Echelle order for this specrum (echelle data only) +``POLYORDER_VEC`` int64 Polynomial order for each slit/echelle (if applicable) +``IND_LOWER`` int64 Lowest index of a spectral pixel included in the fit +``IND_UPPER`` int64 Highest index of a spectral pixel included in the fit +``WAVE_MIN`` float64 Minimum wavelength included in the fit +``WAVE_MAX`` float64 Maximum wavelength included in the fit +================= ========= ================================================================ SENS table diff --git a/doc/include/datamodel_telluric.rst b/doc/include/datamodel_telluric.rst index aac5007b4e..d22cbf732c 100644 --- a/doc/include/datamodel_telluric.rst +++ b/doc/include/datamodel_telluric.rst @@ -11,28 +11,25 @@ HDU Name HDU Type Data Type Description TELLURIC table -================= ========= =========================================================== -Column Data Type Description -================= ========= =========================================================== -``WAVE`` float64 Wavelength vector -``TELLURIC`` float64 Best-fitting telluric model spectrum -``OBJ_MODEL`` float64 Best-fitting object model spectrum -``TELL_THETA`` float64 Best-fitting telluric model parameters -``TELL_PRESS`` float64 Best-fitting telluric model pressure -``TELL_TEMP`` float64 Best-fitting telluric model temperature -``TELL_H2O`` float64 Best-fitting telluric model humidity -``TELL_AIRMASS`` float64 Best-fitting telluric model airmass -``TELL_RESLN`` float64 Best-fitting telluric model spectral resolution -``TELL_SHIFT`` float64 Best-fitting shift applied to the telluric model spectrum -``TELL_STRETCH`` float64 Best-fitting stretch applied to the telluric model spectrum -``OBJ_THETA`` float64 Best-fitting object model parameters -``CHI2`` float64 Chi-square of the best-fit model -``SUCCESS`` bool Flag that fit was successful -``NITER`` int64 Number of fit iterations -``ECH_ORDERS`` int64 Echelle order for this specrum (echelle data only) -``POLYORDER_VEC`` int64 Polynomial order for each slit/echelle (if applicable) -``IND_LOWER`` int64 Lowest index of a spectral pixel included in the fit -``IND_UPPER`` int64 Highest index of a spectral pixel included in the fit -``WAVE_MIN`` float64 Minimum wavelength included in the fit -``WAVE_MAX`` float64 Maximum wavelength included in the fit -================= ========= =========================================================== +================= ========= ================================================================ +Column Data Type Description +================= ========= ================================================================ +``WAVE`` float64 Wavelength vector +``TELLURIC`` float64 Best-fitting telluric model spectrum +``OBJ_MODEL`` float64 Best-fitting object model spectrum +``TELL_THETA`` float64 Best-fitting telluric model parameters +``TELL_PARAM`` float64 Best-fitting telluric atmospheric parameters or PCA coefficients +``TELL_RESLN`` float64 Best-fitting telluric model spectral resolution +``TELL_SHIFT`` float64 Best-fitting shift applied to the telluric model spectrum +``TELL_STRETCH`` float64 Best-fitting stretch applied to the telluric model spectrum +``OBJ_THETA`` float64 Best-fitting object model parameters +``CHI2`` float64 Chi-square of the best-fit model +``SUCCESS`` bool Flag that fit was successful +``NITER`` int64 Number of fit iterations +``ECH_ORDERS`` int64 Echelle order for this specrum (echelle data only) +``POLYORDER_VEC`` int64 Polynomial order for each slit/echelle (if applicable) +``IND_LOWER`` int64 Lowest index of a spectral pixel included in the fit +``IND_UPPER`` int64 Highest index of a spectral pixel included in the fit +``WAVE_MIN`` float64 Minimum wavelength included in the fit +``WAVE_MAX`` float64 Maximum wavelength included in the fit +================= ========= ================================================================ diff --git a/doc/pypeit_par.rst b/doc/pypeit_par.rst index da16794add..7d594ef779 100644 --- a/doc/pypeit_par.rst +++ b/doc/pypeit_par.rst @@ -411,6 +411,7 @@ Key Type Options ``sync_predict`` str ``pca``, ``nearest``, ``auto`` ``pca`` Mode to use when predicting the form of the trace to insert. Use `pca` to use the PCA decomposition, `nearest` to reproduce the shape of the nearest trace, or `auto` to let PypeIt decide which mode to use between `pca` and `nearest`. In general, it will first try `pca`, and if that is not possible, it will use `nearest`. ``sync_to_edge`` bool .. True If adding a first left edge or a last right edge, ignore `center_mode` for these edges and place them at the edge of the detector (with the relevant shape). ``trace_median_frac`` int, float .. .. After detection of peaks in the rectified Sobel-filtered image and before refitting the edge traces, the rectified image is median filtered with a kernel width of `trace_median_frac*nspec` along the spectral dimension. +``trace_rms_tol`` int, float .. .. After retracing edges using peaks detected in the rectified and collapsed image, the RMS difference between the original and refit traces are calculated. This sets the upper limit of the RMS for traces that will be removed. If None, no limit is set and all new traces are kept. ``trace_thresh`` int, float .. .. After rectification and median filtering of the Sobel-filtered image (see `trace_median_frac`), values in the median-filtered image *below* this threshold are masked in the refitting of the edge trace data. If None, no masking applied. ``use_maskdesign`` bool .. False Use slit-mask designs to identify slits. =========================== ================ =========================================== ============== ====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== @@ -506,40 +507,41 @@ Coadd1DPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.Coadd1DPar` -==================== ========== ======= ========== ========================================================================================================================================================================================================================================================================================================================================================================================================================================= -Key Type Options Default Description -==================== ========== ======= ========== ========================================================================================================================================================================================================================================================================================================================================================================================================================================= -``chk_version`` bool .. True If True enforce strict PypeIt version checking to ensure that spec1d*.fits files were createdwith the current version of PypeIt -``coaddfile`` str .. .. Output filename -``dloglam`` int, float .. .. Dispersion in units of log10(wave) in case you want to specify it in the get_wave_grid (for the 'velocity' or 'log10' options), otherwise a median value is computed from the data. -``dv`` int, float .. .. Dispersion in units of km/s in case you want to specify it in the get_wave_grid (for the 'velocity' option), otherwise a median value is computed from the data. -``dwave`` int, float .. .. Dispersion in Angstroms in case you want to specify it in the get_wave_grid (for the 'linear' option), otherwise a median value is computed from the data. -``ex_value`` str .. ``OPT`` The extraction to coadd, i.e. optimal or boxcar. Must be either 'OPT' or 'BOX' -``filter`` str .. ``none`` Filter for scaling. See flux_calib.load_fitler_file() for naming. Ignore if none -``filter_mag`` float .. .. Magnitude of the source in the given filter -``filter_mask`` str, list .. .. List of wavelength regions to mask when doing the scaling (`i.e.`, occasional junk pixels). Colon and comma separateed, e.g. 5552:5559,6010:6030 -``flux_value`` bool .. True If True (default), the code will coadd the fluxed spectra (i.e. the FLAM) in the spec1d files. If False, it will coadd the counts. -``lower`` int, float .. 3.0 Lower rejection threshold used for rejecting pixels when combining spectra in units of sigma. -``mag_type`` str .. ``AB`` Magnitude type. AB is the only option currently allowed -``maxiter_reject`` int .. 5 Maximum number of iterations for stacking and rejection. The code stops iterating either when the output mask does not change betweeen successive iterations or when maxiter_reject is reached. -``maxiter_scale`` int .. 5 Maximum number of iterations performed for rescaling spectra. -``maxrej`` int .. .. Coadding performs iterative rejection by comparing each exposure to a preliminary stack of all the exposures. If this parameter is set then it will not reject more than maxrej pixels per iteration of this rejection. The default is None, which means no maximum on rejected pixels. -``nbests`` list, int .. .. Number of orders to use for estimating the per exposure weights. Default is None, which will just use one fourth of the total number of orders. This is only used for Echelle -``nmaskedge`` int .. 2 Number of edge pixels to mask. This should be removed/fixed. -``ref_percentile`` int, float .. 70.0 Percentile used for selecting the minimum SNR cut from a reference spectrum used to robustly determine the median ratio between spectra. This parameter is used by coadd1d.robust_median_ratio as part of the automatic rescaling procedure. Pixels above this percentile cut are deemed the "good" pixels and are used to compute the ratio of two spectra. This must be a number between 0 and 100. -``scale_method`` str .. ``auto`` Method used to rescale the spectra prior to coadding. The options are: 'auto' -- Determine the scaling method automatically based on the S/N ratio which works well. 'poly' -- Polynomial rescaling. 'median' -- Median rescaling 'none' -- Do not rescale. 'hand' -- Pass in hand scaling factors. This option is not well tested. -``sigrej_exp`` int, float .. .. Rejection threshold used for rejecting exposures with S/N more than sigrej_exp*sigma above the median S/N. If None (the default), no rejection is performed. Currently, only available for multi-slit observations. -``sigrej_scale`` int, float .. 3.0 Rejection threshold used for rejecting pixels when rescaling spectra with scale_spec. -``sn_clip`` int, float .. 30.0 Errors are capped during rejection so that the S/N is never greater than sn_clip. This prevents overly aggressive rejection in high S/N ratio spectrum which neverthless differ at a level greater than the formal S/N due to systematics. -``sn_min_medscale`` int, float .. 0.5 For scale method set to ``auto``, this sets the minimum SNR for which median scaling is attempted. -``sn_min_polyscale`` int, float .. 2.0 For scale method set to ``auto``, this sets the minimum SNR for which polynomial scaling is attempted. -``sn_smooth_npix`` int, float .. .. Number of pixels to median filter by when computing S/N used to decide how to scale and weight spectra. If set to None (default), the code will determine the effective number of good pixels per spectrum in the stack that is being co-added and use 10% of this neff. -``spec_samp_fact`` float .. 1.0 Make the wavelength grid sampling finer (spec_samp_fact < 1.0) or coarser (spec_samp_fact > 1.0) by this sampling factor. This basically multiples the 'native' spectral pixels by spec_samp_fact, i.e. units spec_samp_fact are pixels. -``upper`` int, float .. 3.0 Upper rejection threshold used for rejecting pixels when combining spectra in units of sigma. -``wave_grid_max`` int, float .. .. Used in case you want to specify the maximum wavelength in your wavelength grid, default=None computes from data -``wave_grid_min`` int, float .. .. Used in case you want to specify the minimum wavelength in your wavelength grid, default=None computes from data -``wave_method`` str .. ``linear`` Method used to construct wavelength grid for coadding spectra. The routine that creates the wavelength is :func:`~pypeit.core.wavecal.wvutils.get_wave_grid`. The options are: 'iref' -- Use the first wavelength array. 'velocity' -- Grid is uniform in velocity. 'log10' -- Grid is uniform in log10(wave). This is the same as velocity. 'linear' -- Grid is uniform in lambda. 'concatenate' -- Meld the input wavelength arrays -==================== ========== ======= ========== ========================================================================================================================================================================================================================================================================================================================================================================================================================================= +==================== ========== ======= ========== ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= +Key Type Options Default Description +==================== ========== ======= ========== ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= +``chk_version`` bool .. True If True enforce strict PypeIt version checking to ensure that spec1d*.fits files were createdwith the current version of PypeIt +``coaddfile`` str .. .. Output filename +``dloglam`` int, float .. .. Dispersion in units of log10(wave) in case you want to specify it in the get_wave_grid (for the 'velocity' or 'log10' options), otherwise a median value is computed from the data. +``dv`` int, float .. .. Dispersion in units of km/s in case you want to specify it in the get_wave_grid (for the 'velocity' option), otherwise a median value is computed from the data. +``dwave`` int, float .. .. Dispersion in Angstroms in case you want to specify it in the get_wave_grid (for the 'linear' option), otherwise a median value is computed from the data. +``ex_value`` str .. ``OPT`` The extraction to coadd, i.e. optimal or boxcar. Must be either 'OPT' or 'BOX' +``filter`` str .. ``none`` Filter for scaling. See flux_calib.load_fitler_file() for naming. Ignore if none +``filter_mag`` float .. .. Magnitude of the source in the given filter +``filter_mask`` str, list .. .. List of wavelength regions to mask when doing the scaling (`i.e.`, occasional junk pixels). Colon and comma separateed, e.g. 5552:5559,6010:6030 +``flux_value`` bool .. True If True (default), the code will coadd the fluxed spectra (i.e. the FLAM) in the spec1d files. If False, it will coadd the counts. +``lower`` int, float .. 3.0 Lower rejection threshold used for rejecting pixels when combining spectra in units of sigma. +``mag_type`` str .. ``AB`` Magnitude type. AB is the only option currently allowed +``maxiter_reject`` int .. 5 Maximum number of iterations for stacking and rejection. The code stops iterating either when the output mask does not change betweeen successive iterations or when maxiter_reject is reached. +``maxiter_scale`` int .. 5 Maximum number of iterations performed for rescaling spectra. +``maxrej`` int .. .. Coadding performs iterative rejection by comparing each exposure to a preliminary stack of all the exposures. If this parameter is set then it will not reject more than maxrej pixels per iteration of this rejection. The default is None, which means no maximum on rejected pixels. +``nbests`` list, int .. .. Number of orders to use for estimating the per exposure weights. Default is None, which will just use one fourth of the total number of orders. This is only used for Echelle +``nmaskedge`` int .. 2 Number of edge pixels to mask. This should be removed/fixed. +``ref_percentile`` int, float .. 70.0 Percentile used for selecting the minimum SNR cut from a reference spectrum used to robustly determine the median ratio between spectra. This parameter is used by coadd1d.robust_median_ratio as part of the automatic rescaling procedure. Pixels above this percentile cut are deemed the "good" pixels and are used to compute the ratio of two spectra. This must be a number between 0 and 100. +``scale_method`` str .. ``auto`` Method used to rescale the spectra prior to coadding. The options are: 'auto' -- Determine the scaling method automatically based on the S/N ratio which works well. 'poly' -- Polynomial rescaling. 'median' -- Median rescaling 'none' -- Do not rescale. 'hand' -- Pass in hand scaling factors. This option is not well tested. +``sigrej_exp`` int, float .. .. Rejection threshold used for rejecting exposures with S/N more than sigrej_exp*sigma above the median S/N. If None (the default), no rejection is performed. Currently, only available for multi-slit observations. +``sigrej_scale`` int, float .. 3.0 Rejection threshold used for rejecting pixels when rescaling spectra with scale_spec. +``sn_clip`` int, float .. 30.0 Errors are capped during rejection so that the S/N is never greater than sn_clip. This prevents overly aggressive rejection in high S/N ratio spectrum which neverthless differ at a level greater than the formal S/N due to systematics. +``sn_min_medscale`` int, float .. 0.5 For scale method set to ``auto``, this sets the minimum SNR for which median scaling is attempted. +``sn_min_polyscale`` int, float .. 2.0 For scale method set to ``auto``, this sets the minimum SNR for which polynomial scaling is attempted. +``sn_smooth_npix`` int, float .. .. Number of pixels to median filter by when computing S/N used to decide how to scale and weight spectra. If set to None (default), the code will determine the effective number of good pixels per spectrum in the stack that is being co-added and use 10% of this neff. +``spec_samp_fact`` float .. 1.0 Make the wavelength grid sampling finer (spec_samp_fact < 1.0) or coarser (spec_samp_fact > 1.0) by this sampling factor. This basically multiples the 'native' spectral pixels by spec_samp_fact, i.e. units spec_samp_fact are pixels. +``upper`` int, float .. 3.0 Upper rejection threshold used for rejecting pixels when combining spectra in units of sigma. +``wave_grid_max`` int, float .. .. Used in case you want to specify the maximum wavelength in your wavelength grid, default=None computes from data +``wave_grid_min`` int, float .. .. Used in case you want to specify the minimum wavelength in your wavelength grid, default=None computes from data +``wave_method`` str .. ``linear`` Method used to construct wavelength grid for coadding spectra. The routine that creates the wavelength is :func:`~pypeit.core.wavecal.wvutils.get_wave_grid`. The options are: 'iref' -- Use the first wavelength array. 'velocity' -- Grid is uniform in velocity. 'log10' -- Grid is uniform in log10(wave). This is the same as velocity. 'linear' -- Grid is uniform in lambda. 'concatenate' -- Meld the input wavelength arrays +``weight_method`` str .. ``auto`` Method used to weight the spectra for coadding. The options are: 'auto' -- Use constant weights if rms_sn < 3.0, otherwise use wavelength dependent.'constant' -- Constant weights based on rms_sn**2'uniform' -- Uniform weighting'wave_dependent' -- Wavelength dependent weights will be used irrespective of the rms_sn ratio. This option will not work well at low S/N ratio although it is useful for objects where only a small fraction of the spectral coverage has high S/N ratio (like high-z quasars).'relative' -- Apply relative weights implying one reference exposure will receive unit weight at all wavelengths and all others receive relatively wavelength dependent weights . Note, relative weighting will only work well when there is at least one spectrum with a reasonable S/N, and a continuum. This option may only be better when the object being used has a strong continuum + emission lines. This is particularly useful if you are dealing with highly variable spectra (e.g. emission lines) andrequire a precision better than ~1 per cent.'ivar' -- Use inverse variance weighting. This is not well tested and should probably be deprecated. +==================== ========== ======= ========== ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= ---- @@ -690,34 +692,34 @@ CubePar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.CubePar` -==================== ===== ===================== ============ =========================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== -Key Type Options Default Description -==================== ===== ===================== ============ =========================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== -``align`` bool .. False If set to True, the input frames will be spatially aligned by cross-correlating the whitelight images with either a reference image (see ``reference_image``) or the whitelight image that is generated using the first spec2d listed in the coadd3d file. Alternatively, the user can specify the offsets (i.e. Delta RA x cos(dec) and Delta Dec, both in arcsec) in the spec2d block of the coadd3d file. See the documentation for examples of this usage. -``astrometric`` bool .. True If true, an astrometric correction will be applied using the alignment frames. -``combine`` bool .. False If set to True, the input frames will be combined. Otherwise, a separate datacube will be generated for each input spec2d file, and will be saved as a spec3d file. -``dec_max`` float .. .. Maximum DEC to use when generating the WCS. If None, the default is maximum DEC based on the WCS of all spaxels. Units should be degrees. -``dec_min`` float .. .. Minimum DEC to use when generating the WCS. If None, the default is minimum DEC based on the WCS of all spaxels. Units should be degrees. -``grating_corr`` bool .. True This option performs a small correction for the relative blaze function of all input frames that have (even slightly) different grating angles, or if you are flux calibrating your science data with a standard star that was observed with a slightly different setup. -``method`` str ``subpixel``, ``ngp`` ``subpixel`` What method should be used to generate the datacube. There are currently two options: (1) "subpixel" (default) - this algorithm divides each pixel in the spec2d frames into subpixels, and assigns each subpixel to a voxel of the datacube. Flux is conserved, but voxels are correlated, and the error spectrum does not account for covariance between adjacent voxels. See also, spec_subpixel and spat_subpixel. (2) "ngp" (nearest grid point) - this algorithm is effectively a 3D histogram. Flux is conserved, voxels are not correlated, however this option suffers the same downsides as any histogram; the choice of bin sizes can change how the datacube appears. This algorithm takes each pixel on the spec2d frame and puts the flux of this pixel into one voxel in the datacube. Depending on the binning used, some voxels may be empty (zero flux) while a neighboring voxel might contain the flux from two spec2d pixels. Note that all spec2d pixels that contribute to the same voxel are inverse variance weighted (e.g. if two pixels have the same variance, the voxel would be assigned the average flux of the two pixels). -``output_filename`` str .. .. If combining multiple frames, this string sets the output filename of the combined datacube. If combine=False, the output filenames will be prefixed with ``spec3d_*`` -``ra_max`` float .. .. Maximum RA to use when generating the WCS. If None, the default is maximum RA based on the WCS of all spaxels. Units should be degrees. -``ra_min`` float .. .. Minimum RA to use when generating the WCS. If None, the default is minimum RA based on the WCS of all spaxels. Units should be degrees. -``reference_image`` str .. .. White light image of a previously combined datacube. The white light image will be used as a reference when calculating the offsets of the input spec2d files. Ideally, the reference image should have the same shape as the data to be combined (i.e. set the ra_min, ra_max etc. params so they are identical to the reference image). -``relative_weights`` bool .. False If set to True, the combined frames will use a relative weighting scheme. This only works well if there is a common continuum source in the field of view of all input observations, and is generally only required if high relative precision is desired. -``save_whitelight`` bool .. False Save a white light image of the combined datacube. The output filename will be given by the "output_filename" variable with a suffix "_whitelight". Note that the white light image collapses the flux along the wavelength axis, so some spaxels in the 2D white light image may have different wavelength ranges. To set the wavelength range, use the "whitelight_range" parameter. If combine=False, the individual spec3d files will have a suffix "_whitelight". -``scale_corr`` str .. .. This option performs a small correction for the relative spectral illumination scale of different spec2D files. Specify the relative path+file to the spec2D file that you would like to use for the relative scaling. If you want to perform this correction, it is best to use the spec2d file with the highest S/N sky spectrum. You should choose the same frame for both the standards and science frames. -``skysub_frame`` str .. ``image`` Set the sky subtraction to be implemented. The default behaviour is to subtract the sky using the model that is derived from each individual image (i.e. set this parameter to "image"). To turn off sky subtraction completely, set this parameter to "none" (all lowercase). Finally, if you want to use a different frame for the sky subtraction, specify the relative path+file to the spec2D file that you would like to use for the sky subtraction. The model fit to the sky of the specified frame will be used. Note, the sky and science frames do not need to have the same exposure time; the sky model will be scaled to the science frame based on the relative exposure time. -``slit_spec`` bool .. True If the data use slits in one spatial direction, set this to True. If the data uses fibres for all spaxels, set this to False. -``spat_subpixel`` int .. 5 When method=subpixel, spat_subpixel sets the subpixellation scale of each detector pixel in the spatial direction. The total number of subpixels in each pixel is given by spec_subpixel x spat_subpixel. The default option is to divide each spec2d pixel into 25 subpixels during datacube creation. See also, spec_subpixel. -``spatial_delta`` float .. .. The spatial size of each spaxel to use when generating the WCS (in arcsec). If None, the default is set by the spectrograph file. -``spec_subpixel`` int .. 5 When method=subpixel, spec_subpixel sets the subpixellation scale of each detector pixel in the spectral direction. The total number of subpixels in each pixel is given by spec_subpixel x spat_subpixel. The default option is to divide each spec2d pixel into 25 subpixels during datacube creation. See also, spat_subpixel. -``standard_cube`` str .. .. Filename of a standard star datacube. This cube will be used to correct the relative scales of the slits, and to flux calibrate the science datacube. -``wave_delta`` float .. .. The wavelength step to use when generating the WCS (in Angstroms). If None, the default is set by the wavelength solution. -``wave_max`` float .. .. Maximum wavelength to use when generating the WCS. If None, the default is maximum wavelength based on the WCS of all spaxels. Units should be Angstroms. -``wave_min`` float .. .. Minimum wavelength to use when generating the WCS. If None, the default is minimum wavelength based on the WCS of all spaxels. Units should be Angstroms. -``whitelight_range`` list .. None, None A two element list specifying the wavelength range over which to generate the white light image. The first (second) element is the minimum (maximum) wavelength to use. If either of these elements are None, PypeIt will automatically use a wavelength range that ensures all spaxels have the same wavelength coverage. Note, if you are using a reference_image to align all frames, it is preferable to use the same white light wavelength range for all white light images. For example, you may wish to use an emission line map to register two frames. -==================== ===== ===================== ============ =========================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== +==================== ===== =============================================================================== ============ ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= +Key Type Options Default Description +==================== ===== =============================================================================== ============ ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= +``align`` bool .. False If set to True, the input frames will be spatially aligned by cross-correlating the whitelight images with either a reference image (see ``reference_image``) or the whitelight image that is generated using the first spec2d listed in the coadd3d file. Alternatively, the user can specify the offsets (i.e. Delta RA x cos(dec) and Delta Dec, both in arcsec) in the spec2d block of the coadd3d file. See the documentation for examples of this usage. +``astrometric`` bool .. True If true, an astrometric correction will be applied using the alignment frames. +``combine`` bool .. False If set to True, the input frames will be combined. Otherwise, a separate datacube will be generated for each input spec2d file, and will be saved as a spec3d file. +``dec_max`` float .. .. Maximum DEC to use when generating the WCS. If None, the default is maximum DEC based on the WCS of all spaxels. Units should be degrees. +``dec_min`` float .. .. Minimum DEC to use when generating the WCS. If None, the default is minimum DEC based on the WCS of all spaxels. Units should be degrees. +``grating_corr`` bool .. True This option performs a small correction for the relative blaze function of all input frames that have (even slightly) different grating angles, or if you are flux calibrating your science data with a standard star that was observed with a slightly different setup. +``method`` str ``subpixel``, ``ngp`` ``subpixel`` What method should be used to generate the datacube. There are currently two options: (1) "subpixel" (default) - this algorithm divides each pixel in the spec2d frames into subpixels, and assigns each subpixel to a voxel of the datacube. Flux is conserved, but voxels are correlated, and the error spectrum does not account for covariance between adjacent voxels. See also, spec_subpixel and spat_subpixel. (2) "ngp" (nearest grid point) - this algorithm is effectively a 3D histogram. Flux is conserved, voxels are not correlated, however this option suffers the same downsides as any histogram; the choice of bin sizes can change how the datacube appears. This algorithm takes each pixel on the spec2d frame and puts the flux of this pixel into one voxel in the datacube. Depending on the binning used, some voxels may be empty (zero flux) while a neighboring voxel might contain the flux from two spec2d pixels. Note that all spec2d pixels that contribute to the same voxel are inverse variance weighted (e.g. if two pixels have the same variance, the voxel would be assigned the average flux of the two pixels). +``output_filename`` str .. .. If combining multiple frames, this string sets the output filename of the combined datacube. If combine=False, the output filenames will be prefixed with ``spec3d_*`` +``ra_max`` float .. .. Maximum RA to use when generating the WCS. If None, the default is maximum RA based on the WCS of all spaxels. Units should be degrees. +``ra_min`` float .. .. Minimum RA to use when generating the WCS. If None, the default is minimum RA based on the WCS of all spaxels. Units should be degrees. +``reference_image`` str .. .. White light image of a previously combined datacube. The white light image will be used as a reference when calculating the offsets of the input spec2d files. Ideally, the reference image should have the same shape as the data to be combined (i.e. set the ra_min, ra_max etc. params so they are identical to the reference image). +``save_whitelight`` bool .. False Save a white light image of the combined datacube. The output filename will be given by the "output_filename" variable with a suffix "_whitelight". Note that the white light image collapses the flux along the wavelength axis, so some spaxels in the 2D white light image may have different wavelength ranges. To set the wavelength range, use the "whitelight_range" parameter. If combine=False, the individual spec3d files will have a suffix "_whitelight". +``scale_corr`` str .. .. This option performs a small correction for the relative spectral illumination scale of different spec2D files. Specify the relative path+file to the spec2D file that you would like to use for the relative scaling. If you want to perform this correction, it is best to use the spec2d file with the highest S/N sky spectrum. You should choose the same frame for both the standards and science frames. +``skysub_frame`` str .. ``image`` Set the sky subtraction to be implemented. The default behaviour is to subtract the sky using the model that is derived from each individual image (i.e. set this parameter to "image"). To turn off sky subtraction completely, set this parameter to "none" (all lowercase). Finally, if you want to use a different frame for the sky subtraction, specify the relative path+file to the spec2D file that you would like to use for the sky subtraction. The model fit to the sky of the specified frame will be used. Note, the sky and science frames do not need to have the same exposure time; the sky model will be scaled to the science frame based on the relative exposure time. +``slit_spec`` bool .. True If the data use slits in one spatial direction, set this to True. If the data uses fibres for all spaxels, set this to False. +``spat_subpixel`` int .. 5 When method=subpixel, spat_subpixel sets the subpixellation scale of each detector pixel in the spatial direction. The total number of subpixels in each pixel is given by spec_subpixel x spat_subpixel. The default option is to divide each spec2d pixel into 25 subpixels during datacube creation. See also, spec_subpixel. +``spatial_delta`` float .. .. The spatial size of each spaxel to use when generating the WCS (in arcsec). If None, the default is set by the spectrograph file. +``spec_subpixel`` int .. 5 When method=subpixel, spec_subpixel sets the subpixellation scale of each detector pixel in the spectral direction. The total number of subpixels in each pixel is given by spec_subpixel x spat_subpixel. The default option is to divide each spec2d pixel into 25 subpixels during datacube creation. See also, spat_subpixel. +``standard_cube`` str .. .. Filename of a standard star datacube. This cube will be used to correct the relative scales of the slits, and to flux calibrate the science datacube. +``wave_delta`` float .. .. The wavelength step to use when generating the WCS (in Angstroms). If None, the default is set by the wavelength solution. +``wave_max`` float .. .. Maximum wavelength to use when generating the WCS. If None, the default is maximum wavelength based on the WCS of all spaxels. Units should be Angstroms. +``wave_min`` float .. .. Minimum wavelength to use when generating the WCS. If None, the default is minimum wavelength based on the WCS of all spaxels. Units should be Angstroms. +``weight_method`` str ``auto``, ``constant``, ``uniform``, ``wave_dependent``, ``relative``, ``ivar`` ``auto`` Method used to weight the spectra for coadding. The options are: 'auto' -- Use constant weights if rms_sn < 3.0, otherwise use wavelength dependent.'constant' -- Constant weights based on rms_sn**2'uniform' -- Uniform weighting'wave_dependent' -- Wavelength dependent weights will be used irrespective of the rms_sn ratio. This option will not work well at low S/N ratio although it is useful for objects where only a small fraction of the spectral coverage has high S/N ratio (like high-z quasars).'relative' -- Apply relative weights implying one reference exposure will receive unit weight at all wavelengths and all others receive relatively wavelength dependent weights . Note, relative weighting will only work well when there is at least one spectrum with a reasonable S/N, and a continuum. This option may only be better when the object being used has a strong continuum + emission lines. This is particularly useful if you are dealing with highly variable spectra (e.g. emission lines) andrequire a precision better than ~1 per cent.'ivar' -- Use inverse variance weighting. This is not well tested and should probably be deprecated. +``whitelight_range`` list .. None, None A two element list specifying the wavelength range over which to generate the white light image. The first (second) element is the minimum (maximum) wavelength to use. If either of these elements are None, PypeIt will automatically use a wavelength range that ensures all spaxels have the same wavelength coverage. Note, if you are using a reference_image to align all frames, it is preferable to use the same white light wavelength range for all white light images. For example, you may wish to use an emission line map to register two frames. +==================== ===== =============================================================================== ============ ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= ---- @@ -1006,7 +1008,7 @@ Key Type Options Default ``popsize`` int .. 30 A multiplier for setting the total population size for the differential evolution optimization. See scipy.optimize.differential_evolution for details. ``recombination`` int, float .. 0.7 The recombination constant for the differential evolution optimization. This should be in the range [0, 1]. See scipy.optimize.differential_evolution for details. ``redshift`` int, float .. 0.0 The redshift for the object model. This is currently only used by objmodel=qso -``resln_frac_bounds`` tuple .. (0.5, 1.5) Bounds for the resolution fit optimization which is part of the telluric model. This range is in units of the resln_guess, so the (0.5, 1.5) would bound the spectral resolution fit to be within the range bounds_resln = (0.5*resln_guess, 1.5*resln_guess) +``resln_frac_bounds`` tuple .. (0.6, 1.4) Bounds for the resolution fit optimization which is part of the telluric model. This range is in units of the resln_guess, so the (0.6, 1.4) would bound the spectral resolution fit to be within the range bounds_resln = (0.6*resln_guess, 1.4*resln_guess) ``resln_guess`` int, float .. .. A guess for the resolution of your spectrum expressed as lambda/dlambda. The resolution is fit explicitly as part of the telluric model fitting, but this guess helps determine the bounds for the optimization (see next). If not provided, the wavelength sampling of your spectrum will be used and the resolution calculated using a typical sampling of 3 spectral pixels per resolution element. ``seed`` int .. 777 An initial seed for the differential evolution optimization, which is a random process. The default is a seed = 777 which will be used to generate a unique seed for every order. A specific seed is used because otherwise the random number generator will use the time for the seed, and the results will not be reproducible. ``sn_clip`` int, float .. 30.0 This adds an error floor to the ivar, preventing too much rejection at high-S/N (`i.e.`, standard stars, bright objects) using the function utils.clip_ivar. A small erorr is added to the input ivar so that the output ivar_out will never give S/N greater than sn_clip. This prevents overly aggressive rejection in high S/N ratio spectra which neverthless differ at a level greater than the formal S/N due to the fact that our telluric models are only good to about 3%. @@ -1017,6 +1019,8 @@ Key Type Options Default ``sticky`` bool .. True Sticky parameter for the utils.djs_reject algorithm for iterative model fit rejection. If set to True then points rejected from a previous iteration are kept rejected, in other words the bad pixel mask is the OR of all previous iterations and rejected pixels accumulate. If set to False, the bad pixel mask is the mask from the previous iteration, and if the model fit changes between iterations, points can alternate from being rejected to not rejected. At present this code only performs optimizations with differential evolution and experience shows that sticky needs to be True in order for these to converge. This is because the outliers can be so large that they dominate the loss function, and one never iteratively converges to a good model fit. In other words, the deformations in the model between iterations with sticky=False are too small to approach a reasonable fit. ``telgridfile`` str .. .. File containing the telluric grid for the observatory in question. These grids are generated from HITRAN models for each observatory using nominal site parameters. They must be downloaded from the GoogleDrive and installed in your PypeIt installation via the pypeit_install_telluric script. NOTE: This parameter no longer includes the full pathname to the Telluric Grid file, but is just the filename of the grid itself. ``tell_norm_thresh`` int, float .. 0.9 Threshold of telluric absorption region +``tell_npca`` int .. 5 Number of telluric PCA components used. Can be set to any number from 1 to 10. +``teltype`` str .. ``pca`` Method used to evaluate telluric models, either pca or grid. The grid option uses a fixed grid of pre-computed HITRAN+LBLRTM atmospheric transmission models for each observatory, whereas the pca option uses principal components of a larger model grid to compute an accurate pseudo-telluric model with a much lighter telgridfile. ``tol`` float .. 0.001 Relative tolerance for converage of the differential evolution optimization. See scipy.optimize.differential_evolution for details. ``upper`` int, float .. 3.0 Upper rejection threshold in units of sigma_corr*sigma, where sigma is the formal noise of the spectrum, and sigma_corr is an empirically determined correction to the formal error. See above for description. ======================= ================== ======= ========================== ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= @@ -1425,7 +1429,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-gemini_gmos_north_e2v: @@ -1786,7 +1790,7 @@ Alterations to the default parameters are: [sensfunc] algorithm = IR [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-gemini_gnirs_echelle: @@ -1915,7 +1919,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 6 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-gemini_gnirs_ifu: @@ -2058,7 +2062,7 @@ Alterations to the default parameters are: [[UVIS]] extinct_correct = False [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-gtc_maat: @@ -2134,7 +2138,7 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = None, 180, + exprng = None, 300, [[[process]]] mask_cr = True noise_floor = 0.01 @@ -2347,7 +2351,7 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = None, 180, + exprng = None, 300, [[[process]]] mask_cr = True noise_floor = 0.01 @@ -2665,7 +2669,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R15000.fits .. _instr_par-keck_esi: @@ -2930,7 +2934,11 @@ Alterations to the default parameters are: [sensfunc] algorithm = IR [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_10500_R120000.fits + pix_shift_bounds = (-40.0, 40.0) + [telluric] + resln_frac_bounds = (0.25, 1.25) + pix_shift_bounds = (-40.0, 40.0) .. _instr_par-keck_kcrm: @@ -3249,7 +3257,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_lris_blue_orig: @@ -3351,7 +3359,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_lris_red: @@ -3464,7 +3472,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 9 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_lris_red_mark4: @@ -3577,7 +3585,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 9 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_lris_red_orig: @@ -3690,7 +3698,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 9 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_mosfire: @@ -3817,7 +3825,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 13 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_nires: @@ -3960,7 +3968,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-keck_nirspec_low: @@ -4089,7 +4097,10 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R25000.fits + pix_shift_bounds = (-8.0, 8.0) + [telluric] + pix_shift_bounds = (-8.0, 8.0) .. _instr_par-lbt_luci1: @@ -5018,7 +5029,7 @@ Alterations to the default parameters are: [sensfunc] algorithm = IR [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R15000.fits .. _instr_par-magellan_fire_long: @@ -5148,7 +5159,7 @@ Alterations to the default parameters are: find_trim_edge = 50, 50, [sensfunc] [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-magellan_mage: @@ -5665,7 +5676,7 @@ Alterations to the default parameters are: [sensfunc] polyorder = 7 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-mmt_bluechannel: @@ -5914,7 +5925,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-not_alfosc: @@ -6411,7 +6422,7 @@ Alterations to the default parameters are: [[UVIS]] polycorrect = False [[IR]] - telgridfile = TelFit_Lick_3100_11100_R10000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-p200_tspec: @@ -6557,7 +6568,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_MaunaKea_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-shane_kast_blue: @@ -6658,7 +6669,7 @@ Alterations to the default parameters are: spectrum = sky_kastb_600.fits [sensfunc] [[IR]] - telgridfile = TelFit_Lick_3100_11100_R10000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-shane_kast_red: @@ -6750,7 +6761,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_Lick_3100_11100_R10000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-shane_kast_red_ret: @@ -6844,7 +6855,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_Lick_3100_11100_R10000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-soar_goodman_blue: @@ -6947,7 +6958,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R15000.fits .. _instr_par-soar_goodman_red: @@ -7052,7 +7063,7 @@ Alterations to the default parameters are: spec_method = boxcar [sensfunc] [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R15000.fits .. _instr_par-tng_dolores: @@ -7242,7 +7253,7 @@ Alterations to the default parameters are: [sensfunc] algorithm = IR [[IR]] - telgridfile = TelFit_Paranal_VIS_9800_25000_R25000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-vlt_sinfoni: @@ -7383,7 +7394,7 @@ Alterations to the default parameters are: algorithm = IR polyorder = 7 [[IR]] - telgridfile = TelFit_Paranal_NIR_9800_25000_R25000.fits + telgridfile = TellPCA_3000_26000_R10000.fits .. _instr_par-vlt_xshooter_nir: @@ -7534,7 +7545,11 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_Paranal_NIR_9800_25000_R25000.fits + telgridfile = TellPCA_3000_26000_R25000.fits + pix_shift_bounds = (-10.0, 10.0) + [telluric] + resln_frac_bounds = (0.4, 2.0) + pix_shift_bounds = (-10.0, 10.0) .. _instr_par-vlt_xshooter_uvb: @@ -7676,7 +7691,10 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_LasCampanas_3100_26100_R20000.fits + telgridfile = TellPCA_3000_26000_R25000.fits + pix_shift_bounds = (-8.0, 8.0) + [telluric] + pix_shift_bounds = (-8.0, 8.0) .. _instr_par-vlt_xshooter_vis: @@ -7820,7 +7838,11 @@ Alterations to the default parameters are: algorithm = IR polyorder = 8 [[IR]] - telgridfile = TelFit_Paranal_VIS_4900_11100_R25000.fits + telgridfile = TellPCA_3000_26000_R25000.fits + pix_shift_bounds = (-10.0, 10.0) + [telluric] + resln_frac_bounds = (0.4, 2.0) + pix_shift_bounds = (-10.0, 10.0) .. _instr_par-wht_isis_blue: diff --git a/pypeit/core/slitdesign_matching.py b/pypeit/core/slitdesign_matching.py index 30aeb148e7..60da8e7e94 100644 --- a/pypeit/core/slitdesign_matching.py +++ b/pypeit/core/slitdesign_matching.py @@ -376,9 +376,9 @@ def match_positions_1D(measured, nominal, tol=None): position to every nominal position is used as the cost matrix. Args: - measured (`numpy.array`_): + measured (`numpy.ndarray`_): Measured positions. Shape is ``(n,)``. - nominal (`numpy.array`_): + nominal (`numpy.ndarray`_): Expected positions. Shape is ``(m,)``. tol (:obj:`float`, optional): Maximum separation between measured and nominal positions to be @@ -401,6 +401,10 @@ def match_positions_1D(measured, nominal, tol=None): n_m, m_m = optimize.linear_sum_assignment(sep) # Remove any matches that don't meet the provided tolerance. + # NOTE: It's possible this approach yields a non-optimal match. I.e., when + # multiple matches are near the tolerance, removing the largest separations + # might yield a more optimal result for the ones that remain. But this + # approach has worked so far. if tol is not None: indx = sep[n_m,m_m] < tol n_m = n_m[indx] From af3831dd7367c267048e4259e30b52ae64312c28 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 25 Jan 2024 09:10:05 -0800 Subject: [PATCH 213/244] clean up --- pypeit/core/slitdesign_matching.py | 5 ++- pypeit/edgetrace.py | 62 +++++++++++++++--------------- 2 files changed, 33 insertions(+), 34 deletions(-) diff --git a/pypeit/core/slitdesign_matching.py b/pypeit/core/slitdesign_matching.py index 60da8e7e94..eda3db0682 100644 --- a/pypeit/core/slitdesign_matching.py +++ b/pypeit/core/slitdesign_matching.py @@ -403,8 +403,9 @@ def match_positions_1D(measured, nominal, tol=None): # Remove any matches that don't meet the provided tolerance. # NOTE: It's possible this approach yields a non-optimal match. I.e., when # multiple matches are near the tolerance, removing the largest separations - # might yield a more optimal result for the ones that remain. But this - # approach has worked so far. + # *before* performing the match (above) might ultimately yield a more + # optimal result for the ones that remain. But this approach has worked so + # far. if tol is not None: indx = sep[n_m,m_m] < tol n_m = n_m[indx] diff --git a/pypeit/edgetrace.py b/pypeit/edgetrace.py index 76f182e31d..941c4d0dba 100644 --- a/pypeit/edgetrace.py +++ b/pypeit/edgetrace.py @@ -2416,7 +2416,6 @@ def check_synced(self, rebuild_pca=False): - Any trace falling off the edge of the detector is masked (see :class:`EdgeTraceBitMask`). This is the only check performed by default (i.e., when no keyword arguments are provided). - - Traces that form slit gaps (the median difference between the right and left traces of adjacent slits) that are below an absolute tolerance are removed and @@ -2424,7 +2423,6 @@ def check_synced(self, rebuild_pca=False): the checks of the slit length below such that the merged slit is assessed in any expected slit length constraints. - - Traces that form a slit with a length (the median difference between the left and right edges) below an absolute tolerance (i.e., `right-left < atol`) are @@ -3435,33 +3433,36 @@ def pca_refine(self, use_center=False, debug=False, force=False): def peak_refine(self, rebuild_pca=False, debug=False): """ - Refine the trace by isolating peaks and troughs in the - Sobel-filtered image. + Refine the trace by isolating peaks and troughs in the Sobel-filtered + image. This function *requires* that the PCA model exists; see - :func:`build_pca` or :func:`pca_refine`. It is also primarily - a wrapper for :func:`~pypeit.core.trace.peak_trace`. See the - documentation of that function for the explanation of the - algorithm. - - If the left and right traces have separate PCA - decompositions, this function makes one call to - :func:`~pypeit.core.trace.peak_trace` for each side. - Otherwise, a single call is made to - :func:`~pypeit.core.trace.peak_trace` where both the peak and - troughs in :attr:`sobelsig` are detected and traced. - - Note that this effectively reinstantiates much of the object - attributes, including :attr:`traceid`, :attr:`edge_cen`, - :attr:`edge_err`, :attr:`edge_msk`, :attr:`edge_img`, - :attr:`edge_fit`, and :attr:`fittype`. + :func:`build_pca` or :func:`pca_refine`. It is also primarily a wrapper + for :func:`~pypeit.core.trace.peak_trace`. See the documentation of that + function for the explanation of the algorithm. + + If the left and right traces have separate PCA decompositions, this + function makes one call to :func:`~pypeit.core.trace.peak_trace` for + each side. Otherwise, a single call is made to + :func:`~pypeit.core.trace.peak_trace` where both the peak and troughs in + :attr:`sobelsig` are detected and traced. + + Optionally, the code will match and compare the traces found and fit by + :func:`~pypeit.core.trace.peak_trace` to the original traces. If the + RMS difference between the matched traces is large, they can be removed + (see ``trace_rms_tol`` in :class:`~pypeit.par.pypeitpar.EdgeTracePar`). + + Note that this effectively reinstantiates much of the object attributes, + including :attr:`traceid`, :attr:`edge_cen`, :attr:`edge_err`, + :attr:`edge_msk`, :attr:`edge_img`, :attr:`edge_fit`, and + :attr:`fittype`. Used parameters from :attr:`par` - (:class:`~pypeit.par.pypeitpar.EdgeTracePar`) are - ``left_right_pca``, ``edge_thresh``, ``smash_range``, - ``edge_detect_clip``, ``trace_median_frac``, ``trace_thresh``, - ``fit_function``, ``fit_order``, ``fwhm_uniform``, ``fwhm_uniform``, - ``niter_gaussian``, ``niter_gaussian``, ``fit_maxdev``, and + (:class:`~pypeit.par.pypeitpar.EdgeTracePar`) are ``left_right_pca``, + ``edge_thresh``, ``smash_range``, ``edge_detect_clip``, + ``trace_median_frac``, ``trace_thresh``, ``trace_rms_tol``, + ``fit_function``, ``fit_order``, ``fwhm_uniform``, ``niter_uniform``, + ``fwhm_gaussian``, ``niter_gaussian``, ``fit_maxdev``, and ``fit_maxiter``. Args: @@ -3597,7 +3598,7 @@ def peak_refine(self, rebuild_pca=False, debug=False): fit[reference_row]) # Determine the RMS difference between the input and output traces. - # This allows us to compare traces that had alreaday been identified + # This allows us to compare traces that had already been identified # to their new measurements resulting from peak_trace, and remove # them if they are too discrepant from their original form. This is # largely meant to find and remove poorly constrained traces, where @@ -3608,7 +3609,9 @@ def peak_refine(self, rebuild_pca=False, debug=False): # TODO: Add a report to the screen or a QA plot? # Select traces below the RMS tolerance or that were newly - # identified by peak_trace + # identified by peak_trace. I.e., this will *not* catch newly + # identified traces found by peak_trace that are also poorly + # constrained! indx = (rms < self.par['trace_rms_tol']) | (peak_indx == -1) if not all(indx): msgs.info(f'Removing {indx.size - np.sum(indx)} trace(s) due to large RMS ' @@ -3619,11 +3622,6 @@ def peak_refine(self, rebuild_pca=False, debug=False): msk = msk[:,indx] nleft -= np.sum(np.where(np.logical_not(indx))[0] < nleft) - # TODO: Possibly put in a check that compares the locations of the new - # and old traces, which removes new traces that are too different (above - # some tolerance) from the old traces. Maybe this is a way to catch - # traces that result from poorly constrained polynomial fits. - # Reset the trace data self.traceid = np.zeros(ntrace, dtype=int) self.traceid[:nleft] = -1-np.arange(nleft) From fe9d98ebbdf3205935cd0ded71edfd6af71cd139 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 25 Jan 2024 14:01:08 -0800 Subject: [PATCH 214/244] PR comments --- doc/releases/1.14.1dev.rst | 2 ++ pypeit/edgetrace.py | 13 ++++++++++++- pypeit/par/pypeitpar.py | 8 ++++---- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/doc/releases/1.14.1dev.rst b/doc/releases/1.14.1dev.rst index cd6e40a49a..27744e2130 100644 --- a/doc/releases/1.14.1dev.rst +++ b/doc/releases/1.14.1dev.rst @@ -45,6 +45,8 @@ Functionality/Performance Improvements and Additions future. - Introduced PCA method for telluric corrections - Added a new GUI for creating and editing PypeIt input files: ``pypeit_setup_gui`` +- Added ``trace_rms_tol`` parameter for edge tracing, which helps guard against + poorly constrained traces for spectrally truncated slits/orders. Instrument-specific Updates --------------------------- diff --git a/pypeit/edgetrace.py b/pypeit/edgetrace.py index 941c4d0dba..2a6941d81e 100644 --- a/pypeit/edgetrace.py +++ b/pypeit/edgetrace.py @@ -3606,6 +3606,17 @@ def peak_refine(self, rebuild_pca=False, debug=False): diff = fit - self.edge_fit.T[gpm][peak_indx].T rms = np.sqrt(np.mean((diff - np.mean(diff, axis=0)[None,:])**2, axis=0)) + # Report + msgs.info('-'*30) + msgs.info('Matched spatial locations and RMS difference along spectral direction') + msgs.info(f' {"OLD":>8} {"NEW":>8} {"RMS":>8}') + msgs.info(' '+'-'*8+' '+'-'*8+' '+'-'*8) + for i in range(len(peak_indx)): + if peak_indx[i] < 0: + continue + msgs.info(f' {self.edge_fit[reference_row][gpm][peak_indx[i]]:8.1f}' + f' {fit[reference_row][i]:8.1f} {rms[i]:8.3f}') + # TODO: Add a report to the screen or a QA plot? # Select traces below the RMS tolerance or that were newly @@ -3613,7 +3624,7 @@ def peak_refine(self, rebuild_pca=False, debug=False): # identified traces found by peak_trace that are also poorly # constrained! indx = (rms < self.par['trace_rms_tol']) | (peak_indx == -1) - if not all(indx): + if not np.all(indx): msgs.info(f'Removing {indx.size - np.sum(indx)} trace(s) due to large RMS ' 'difference with previous trace locations.') fit = fit[:,indx] diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index 521ce6dae5..db8dd4bf3b 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -3328,10 +3328,10 @@ def __init__(self, filt_iter=None, sobel_mode=None, edge_thresh=None, sobel_enha dtypes['trace_rms_tol'] = [int, float] descr['trace_rms_tol'] = 'After retracing edges using peaks detected in the rectified ' \ - 'and collapsed image, the RMS difference between the original ' \ - 'and refit traces are calculated. This sets the upper limit ' \ - 'of the RMS for traces that will be removed. If None, no ' \ - 'limit is set and all new traces are kept.' + 'and collapsed image, the RMS difference (in pixels) between ' \ + 'the original and refit traces are calculated. This sets the ' \ + 'upper limit of the RMS for traces that will be removed. If ' \ + 'None, no limit is set and all new traces are kept.' defaults['fwhm_uniform'] = 3.0 dtypes['fwhm_uniform'] = [int, float] From 7f9c55922c5a1e95fdacd0736fedc2fe60adeff3 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 25 Jan 2024 14:02:58 -0800 Subject: [PATCH 215/244] rm comment --- pypeit/edgetrace.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pypeit/edgetrace.py b/pypeit/edgetrace.py index 2a6941d81e..f6e2ae7db1 100644 --- a/pypeit/edgetrace.py +++ b/pypeit/edgetrace.py @@ -3617,8 +3617,6 @@ def peak_refine(self, rebuild_pca=False, debug=False): msgs.info(f' {self.edge_fit[reference_row][gpm][peak_indx[i]]:8.1f}' f' {fit[reference_row][i]:8.1f} {rms[i]:8.3f}') - # TODO: Add a report to the screen or a QA plot? - # Select traces below the RMS tolerance or that were newly # identified by peak_trace. I.e., this will *not* catch newly # identified traces found by peak_trace that are also poorly From 78eb4b90843a458e796aa0e2d4af08d8e28446f0 Mon Sep 17 00:00:00 2001 From: profxj Date: Fri, 26 Jan 2024 05:51:04 -0800 Subject: [PATCH 216/244] doc update --- doc/spectrographs/soar_goodman.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/spectrographs/soar_goodman.rst b/doc/spectrographs/soar_goodman.rst index d8f6f34726..2a43f2e556 100644 --- a/doc/spectrographs/soar_goodman.rst +++ b/doc/spectrographs/soar_goodman.rst @@ -6,12 +6,13 @@ Overview ======== This file summarizes several instrument specific -items for the SOAR/Goodman spectrograph. Note that -there are two ways to download your data from the +items for the SOAR/Goodman spectrograph. + +Note that there are two ways to download your data from the archive: -(1) in .fits format; and -(2) in .fits.fz format. +(1) in .fits format -- these have been pre-processed and are not PypeIt compliant +(2) in .fits.fz format -- these are the raw frames from the telescope PypeIt will only work with the .fz format, so please ensure that you are only using data in this format. From 48e174c8d9f9076027992f55576a635dc8d20656 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Fri, 26 Jan 2024 07:38:36 -0800 Subject: [PATCH 217/244] bug --- pypeit/edgetrace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/edgetrace.py b/pypeit/edgetrace.py index f6e2ae7db1..3401413f9e 100644 --- a/pypeit/edgetrace.py +++ b/pypeit/edgetrace.py @@ -2511,7 +2511,7 @@ def check_synced(self, rebuild_pca=False): msgs.info('Binning: {0}'.format(self.traceimg.detector.binning)) msgs.info('Platescale per binned pixel: {0}'.format(platescale)) if self.par['minimum_slit_dlength'] is not None: - length_atol = self.par['minimum_slit_dlength']/platescale + dlength_atol = self.par['minimum_slit_dlength']/platescale if self.par['minimum_slit_length'] is not None: length_atol = self.par['minimum_slit_length']/platescale if self.par['minimum_slit_length_sci'] is not None: From 7be66baad621bc60b94294e053c473b35d64f77a Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Fri, 26 Jan 2024 09:22:12 -0800 Subject: [PATCH 218/244] flagging usage recheck --- pypeit/coadd2d.py | 2 -- pypeit/edgetrace.py | 32 ++++++++++++++++---------------- pypeit/metadata.py | 4 ++-- pypeit/scripts/collate_1d.py | 2 +- pypeit/scripts/parse_slits.py | 2 +- pypeit/slittrace.py | 20 ++++++++++---------- pypeit/spectrographs/slitmask.py | 8 ++++---- pypeit/wavetilts.py | 2 +- 8 files changed, 35 insertions(+), 37 deletions(-) diff --git a/pypeit/coadd2d.py b/pypeit/coadd2d.py index 73edac0f7d..871e33e09b 100644 --- a/pypeit/coadd2d.py +++ b/pypeit/coadd2d.py @@ -819,8 +819,6 @@ def reduce(self, pseudo_dict, show=False, clear_ginga=True, show_peaks=False, sh # Get bpm mask. There should not be any masked slits because we excluded those already # before the coadd, but we need to pass a bpm to FindObjects and Extract slits = pseudo_dict['slits'] - #pseudo_reduce_bpm = (slits.mask > 0) & (np.invert(slits.bitmask.flagged(slits.mask, - # flag=slits.bitmask.exclude_for_reducing))) # Initiate FindObjects object objFind = find_objects.FindObjects.get_instance(sciImage, pseudo_dict['slits'], self.spectrograph, parcopy, diff --git a/pypeit/edgetrace.py b/pypeit/edgetrace.py index 684a13d5d8..53aeb8560f 100644 --- a/pypeit/edgetrace.py +++ b/pypeit/edgetrace.py @@ -1057,13 +1057,13 @@ def initial_trace(self, bpm=None): # coordinates self.edge_img = np.zeros((self.nspec, self.ntrace), dtype=int) for i in range(self.ntrace): - row, col = np.where(np.invert(trace_id_img.mask) + row, col = np.where(np.logical_not(trace_id_img.mask) & (trace_id_img.data == self.traceid[i])) self.edge_img[row,i] = col self.edge_msk[row,i] = 0 # Turn-off the mask # Flag any insert traces - row, col = np.where(np.invert(trace_id_img.mask) & inserted_edge + row, col = np.where(np.logical_not(trace_id_img.mask) & inserted_edge & (trace_id_img.data == self.traceid[i])) if len(row) > 0: self.edge_msk[row,i] = self.bitmask.turn_on(self.edge_msk[row,i], 'ORPHANINSERT') @@ -1416,7 +1416,7 @@ def show(self, include_error=False, thin=10, in_ginga=False, include_img=True, fit = self.edge_fit err = np.ma.MaskedArray(self.edge_err, mask=np.ma.getmaskarray(cen).copy()) msk = None if flag is None \ - else np.ma.MaskedArray(self.edge_cen, mask=np.invert( + else np.ma.MaskedArray(self.edge_cen, mask=np.logical_not( self.bitmask.flagged(self.edge_msk, flag=_flag))) is_left = self.is_left is_right = self.is_right @@ -1451,7 +1451,7 @@ def show(self, include_error=False, thin=10, in_ginga=False, include_img=True, nslits = slits.nslits is_left = np.ones(2*nslits, dtype=bool) is_left[nslits:] = False - is_right = np.invert(is_left) + is_right = np.logical_not(is_left) gpm = np.ones(2*nslits, dtype=bool) traceid = np.concatenate((-np.arange(nslits), np.arange(nslits))) synced = True @@ -2082,7 +2082,7 @@ def check_traces(self, cen=None, msk=None, subset=None, min_spatial=None, max_sp if subset is not None: msgs.info('Tolerance for finding repeat traces: {0:.1f}'.format(self.par['match_tol'])) side = -1 if np.all(self.traceid[indx] < 0) else 1 - compare = (side*self.traceid > 0) & np.invert(indx) + compare = (side*self.traceid > 0) & np.logical_not(indx) if np.any(compare): # Use masked arrays to ease exclusion of masked data _col = np.ma.MaskedArray(np.round(_cen).astype(int), mask=_bpm) @@ -2093,7 +2093,7 @@ def check_traces(self, cen=None, msk=None, subset=None, min_spatial=None, max_sp # TODO: This tolerance uses the integer image # coordinates, not the floating-point centroid # coordinates.... - repeat[indx] = (mindiff.data < self.par['match_tol']) & np.invert(mindiff.mask) + repeat[indx] = (mindiff.data < self.par['match_tol']) & np.logical_not(mindiff.mask) if np.any(repeat): _msk[:,repeat] = self.bitmask.turn_on(_msk[:,repeat], 'DUPLICATE') msgs.info('Found {0} repeat trace(s).'.format(np.sum(repeat))) @@ -2103,7 +2103,7 @@ def check_traces(self, cen=None, msk=None, subset=None, min_spatial=None, max_sp if minimum_spec_length is not None: msgs.info('Minimum spectral length of any trace (pixels): {0:.2f}'.format( minimum_spec_length)) - short[indx] = np.sum(np.invert(_bpm[:,indx]), axis=0) < minimum_spec_length + short[indx] = np.sum(np.logical_not(_bpm[:,indx]), axis=0) < minimum_spec_length if np.any(short): _msk[:,short] = self.bitmask.turn_on(_msk[:,short], 'SHORTRANGE') msgs.info('Found {0} short trace(s).'.format(np.sum(short))) @@ -2115,7 +2115,7 @@ def check_traces(self, cen=None, msk=None, subset=None, min_spatial=None, max_sp hit_min = np.zeros_like(indx, dtype=bool) if min_spatial is not None: hit_min[indx] = (col[self.nspec//2,indx] <= min_spatial) \ - & np.invert(_bpm[self.nspec//2,indx]) + & np.logical_not(_bpm[self.nspec//2,indx]) if np.any(hit_min): _msk[:,hit_min] = self.bitmask.turn_on(_msk[:,hit_min], 'HITMIN') msgs.info('{0} trace(s) hit the minimum centroid value.'.format(np.sum(hit_min))) @@ -2126,7 +2126,7 @@ def check_traces(self, cen=None, msk=None, subset=None, min_spatial=None, max_sp hit_max = np.zeros_like(indx, dtype=bool) if max_spatial is not None: hit_max[indx] = (col[self.nspec//2,indx] >= max_spatial) \ - & np.invert(_bpm[self.nspec//2,indx]) + & np.logical_not(_bpm[self.nspec//2,indx]) if np.any(hit_max): _msk[:,hit_max] = self.bitmask.turn_on(_msk[:,hit_max], 'HITMAX') msgs.info('{0} trace(s) hit the maximum centroid value.'.format(np.sum(hit_max))) @@ -2143,7 +2143,7 @@ def check_traces(self, cen=None, msk=None, subset=None, min_spatial=None, max_sp # Good traces bad = indx & (repeat | short | hit_min | hit_max | off_detector) msgs.info('Identified {0} bad trace(s) in all.'.format(np.sum(bad))) - good = indx & np.invert(bad) + good = indx & np.logical_not(bad) return good, bad def merge_traces(self, merge_frac=0.5, refit=True, debug=False): @@ -2214,7 +2214,7 @@ def merge_traces(self, merge_frac=0.5, refit=True, debug=False): for side in ['left', 'right']: # Match traces on the same side and indx = np.where((self.is_left if side == 'left' else self.is_right) - & np.invert(np.all(cen_match.mask, axis=0)))[0] + & np.logical_not(np.all(cen_match.mask, axis=0)))[0] if indx.size == 0: continue @@ -2230,7 +2230,7 @@ def merge_traces(self, merge_frac=0.5, refit=True, debug=False): merge = np.append(indx[i], indx[i+1:][merge]) msgs.info('Merging traces: {0}'.format(self.traceid[merge])) merged_trace = np.ma.mean(cen_merge[:,merge], axis=1) - gpm = np.invert(np.ma.getmaskarray(merged_trace)) + gpm = np.logical_not(np.ma.getmaskarray(merged_trace)) self.edge_cen[gpm,indx[i]] = merged_trace[gpm] self.edge_msk[gpm,indx[i]] = 0 @@ -2952,7 +2952,7 @@ def spatial_sort(self, use_mean=False, use_fit=True): # Reorder the trace numbers indx = self.traceid < 0 self.traceid[indx] = -1-np.arange(np.sum(indx)) - indx = np.invert(indx) + indx = np.logical_not(indx) self.traceid[indx] = 1+np.arange(np.sum(indx)) def _reset_pca(self, rebuild): @@ -3077,7 +3077,7 @@ def fit_refine(self, weighting='uniform', debug=False, idx=None): _sobelsig = self._side_dependent_sobel(side) # Select traces on this side and that are not fully masked indx = (self.is_left if side == 'left' else self.is_right) \ - & np.invert(np.all(edge_bpm, axis=0)) + & np.logical_not(np.all(edge_bpm, axis=0)) if not np.any(indx): continue @@ -3153,7 +3153,7 @@ def can_pca(self): # NOTE: Because of the run of check_traces above, short traces # are fully flagged meaning that we can just check if the # length of the trace is larger than 0. - good = np.sum(np.invert(self.bitmask.flagged(self.edge_msk)), axis=0) > 0 + good = np.sum(np.logical_not(self.bitmask.flagged(self.edge_msk)), axis=0) > 0 # Returned value depends on whether or not the left and right # traces are done separately @@ -3707,7 +3707,7 @@ def _get_reference_locations(self, trace_cen, add_edge): # Build a masked array with the trace positions at that # spectral row, masked where new traces are supposed to go. trace_ref = np.ma.masked_all(add_edge.size) - trace_ref[np.invert(add_edge)] = trace_cen[reference_row,:] + trace_ref[np.logical_not(add_edge)] = trace_cen[reference_row,:] trace_ref = trace_ref.reshape(-1,2) # Get the length and center of each slit in pixels diff --git a/pypeit/metadata.py b/pypeit/metadata.py index b45cca9815..66130515e3 100644 --- a/pypeit/metadata.py +++ b/pypeit/metadata.py @@ -1257,7 +1257,7 @@ def find_frames(self, ftype, calib_ID=None, index=False): if ftype == 'None': return self['framebit'] == 0 # Select frames - indx = self.type_bitmask.flagged(self['framebit'], ftype) + indx = self.type_bitmask.flagged(self['framebit'], flag=ftype) if calib_ID is not None: # Select frames in the same calibration group @@ -1954,7 +1954,7 @@ def find_calib_group(self, grp): """ if 'calibbit' not in self.keys(): msgs.error('Calibration groups are not set. First run set_calibration_groups.') - return self.calib_bitmask.flagged(self['calibbit'].data, grp) + return self.calib_bitmask.flagged(self['calibbit'].data, flag=grp) def find_frame_calib_groups(self, row): """ diff --git a/pypeit/scripts/collate_1d.py b/pypeit/scripts/collate_1d.py index 6e99c789ad..a67be80f50 100644 --- a/pypeit/scripts/collate_1d.py +++ b/pypeit/scripts/collate_1d.py @@ -128,7 +128,7 @@ def find_slits_to_exclude(spec2d_files, par): for sobj2d in [allspec2d[det] for det in allspec2d.detectors]: for (slit_id, mask, slit_mask_id) in sobj2d['slits'].slit_info: for flag in exclude_flags: - if bit_mask.flagged(mask, flag): + if bit_mask.flagged(mask, flag=flag): if slit_mask_id not in exclude_map: exclude_map[slit_mask_id] = {flag} else: diff --git a/pypeit/scripts/parse_slits.py b/pypeit/scripts/parse_slits.py index e11d1785f2..c9a4b94749 100644 --- a/pypeit/scripts/parse_slits.py +++ b/pypeit/scripts/parse_slits.py @@ -41,7 +41,7 @@ def print_slits(slits): this_flags = [] if slits.mask[slit_idx] != 0: for key in bitmask.keys(): - if bitmask.flagged(slits.mask[slit_idx], key): + if bitmask.flagged(slits.mask[slit_idx], flag=key): this_flags += [key] allflags[slit_idx] = ', '.join(this_flags) diff --git a/pypeit/slittrace.py b/pypeit/slittrace.py index fd7943c3cf..4063140be4 100644 --- a/pypeit/slittrace.py +++ b/pypeit/slittrace.py @@ -521,9 +521,8 @@ def select_edges(self, initial=False, flexure=None): # Return return left.copy(), right.copy(), self.mask.copy() - def slit_img(self, pad=None, slitidx=None, initial=False, - flexure=None, - exclude_flag=None, use_spatial=True): + def slit_img(self, pad=None, slitidx=None, initial=False, flexure=None, exclude_flag=None, + use_spatial=True): r""" Construct an image identifying each pixel with its associated slit. @@ -567,8 +566,8 @@ def slit_img(self, pad=None, slitidx=None, initial=False, :attr:`right_init`) are used. To use the nominal edges regardless of the presence of the tweaked edges, set this to True. See :func:`select_edges`. - exclude_flag (:obj:`str`, optional): - Bitmask flag to ignore when masking + exclude_flag (:obj:`str`, :obj:`list`, optional): + One or more bitmask flag names to ignore when masking Warning -- This could conflict with input slitids, i.e. avoid using both use_spatial (bool, optional): If True, use self.spat_id value instead of 0-based indices @@ -600,10 +599,11 @@ def slit_img(self, pad=None, slitidx=None, initial=False, if slitidx is not None: slitidx = np.atleast_1d(slitidx).ravel() else: - bpm = self.mask.astype(bool) - if exclude_flag: - bpm &= np.invert(self.bitmask.flagged(self.mask, flag=exclude_flag)) - slitidx = np.where(np.invert(bpm))[0] + bpm = self.bitmask.flagged(self.mask, and_not=exclude_flag) +# bpm = self.mask.astype(bool) +# if exclude_flag: +# bpm &= np.invert(self.bitmask.flagged(self.mask, flag=exclude_flag)) + slitidx = np.where(np.logical_not(bpm))[0] # TODO: When specific slits are chosen, need to check that the # padding doesn't lead to slit overlap. @@ -1410,7 +1410,7 @@ def mask_flats(self, flatImages): """ # Loop on all the FLATFIELD BPM keys for flag in ['SKIPFLATCALIB', 'BADFLATCALIB']: - bad_flats = self.bitmask.flagged(flatImages.get_bpmflats(), flag) + bad_flats = self.bitmask.flagged(flatImages.get_bpmflats(), flag=flag) if np.any(bad_flats): self.mask[bad_flats] = self.bitmask.turn_on(self.mask[bad_flats], flag) diff --git a/pypeit/spectrographs/slitmask.py b/pypeit/spectrographs/slitmask.py index 47fa898b6b..cd45452741 100644 --- a/pypeit/spectrographs/slitmask.py +++ b/pypeit/spectrographs/slitmask.py @@ -254,20 +254,20 @@ def nslits(self): @property def alignment_slit(self): """Boolean array selecting the alignment slits.""" - return self.bitmask.flagged(self.mask, 'ALIGN') + return self.bitmask.flagged(self.mask, flag='ALIGN') def is_alignment(self, i): """Check if specific slit is an alignment slit.""" - return self.bitmask.flagged(self.mask[i], 'ALIGN') + return self.bitmask.flagged(self.mask[i], flag='ALIGN') @property def science_slit(self): """Boolean array selecting the slits with science targets.""" - return self.bitmask.flagged(self.mask, 'SCIENCE') + return self.bitmask.flagged(self.mask, flag='SCIENCE') def is_science(self, i): """Check if specific slit should have a science target.""" - return self.bitmask.flagged(self.mask[i], 'SCIENCE') + return self.bitmask.flagged(self.mask[i], flag='SCIENCE') class SlitRegister: diff --git a/pypeit/wavetilts.py b/pypeit/wavetilts.py index 7b4e805db9..51c41d2b6f 100644 --- a/pypeit/wavetilts.py +++ b/pypeit/wavetilts.py @@ -780,7 +780,7 @@ def run(self, doqa=True, debug=False, show=False): # Record the Mask bpmtilts = np.zeros_like(self.slits.mask, dtype=self.slits.bitmask.minimum_dtype()) for flag in ['BADTILTCALIB']: - bpm = self.slits.bitmask.flagged(self.slits.mask, flag) + bpm = self.slits.bitmask.flagged(self.slits.mask, flag=flag) if np.any(bpm): bpmtilts[bpm] = self.slits.bitmask.turn_on(bpmtilts[bpm], flag) From b4b52045d77db6b6b5319b1da5b2c18f6d5c76b1 Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 26 Jan 2024 17:52:11 +0000 Subject: [PATCH 219/244] slicer subpixel update --- doc/releases/1.14.1dev.rst | 1 + pypeit/coadd3d.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/releases/1.14.1dev.rst b/doc/releases/1.14.1dev.rst index f08e47dccf..608651f9b0 100644 --- a/doc/releases/1.14.1dev.rst +++ b/doc/releases/1.14.1dev.rst @@ -45,6 +45,7 @@ Functionality/Performance Improvements and Additions future. - Introduced PCA method for telluric corrections - Added a new GUI for creating and editing PypeIt input files: ``pypeit_setup_gui`` +- Added slicer subpixel sampling for DataCube generation Instrument-specific Updates diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index 770ee8d6f5..4b7d645b3f 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -390,7 +390,6 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs Show QA for debugging. """ - # TODO :: Before PR merge, add information to the release notes about the slicer subpixellation. # TODO :: Consider loading all calibrations into a single variable within the main CoAdd3D parent class. # Set the variables self.spec2d = spec2dfiles From bed83a383076edfb6413ac0c5374d09f0b6026da Mon Sep 17 00:00:00 2001 From: profxj Date: Sat, 27 Jan 2024 05:20:16 -0800 Subject: [PATCH 220/244] tickling tests --- doc/spectrographs/soar_goodman.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/spectrographs/soar_goodman.rst b/doc/spectrographs/soar_goodman.rst index 2a43f2e556..c7a95de3b3 100644 --- a/doc/spectrographs/soar_goodman.rst +++ b/doc/spectrographs/soar_goodman.rst @@ -17,3 +17,5 @@ archive: PypeIt will only work with the .fz format, so please ensure that you are only using data in this format. +We recommend downloading the data from here: +https://archive.lco.global/ \ No newline at end of file From dbb10c0fb4763f0388188149c01611d733408571 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Tue, 30 Jan 2024 08:15:07 -0800 Subject: [PATCH 221/244] add test --- pypeit/tests/test_bitmask.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/pypeit/tests/test_bitmask.py b/pypeit/tests/test_bitmask.py index 0f443f5de1..b8dcc8f220 100644 --- a/pypeit/tests/test_bitmask.py +++ b/pypeit/tests/test_bitmask.py @@ -7,6 +7,7 @@ from astropy.io import fits from pypeit.bitmask import BitMask +from pypeit.slittrace import SlitTraceBitMask #----------------------------------------------------------------------------- @@ -124,7 +125,7 @@ def test_flag_order(): assert not bm.correct_flag_order(flags), 'Reordering the flags is not okay' -def test_exclude_expunge(): +def test_exclude_and_not(): n = 1024 shape = (n,n) @@ -154,4 +155,17 @@ def test_exclude_expunge(): assert numpy.array_equal(image_bm.flagged(mask, and_not='SATURATED'), cosmics_indx & numpy.logical_not(saturated_indx)), 'Expunge incorrect' +def test_boxslit(): + """ + Tests old vs. new bpm after adding `and_not` functionality. + """ + bm = SlitTraceBitMask() + v = numpy.array([10,0,2]) + + desired_bpm = (v > 2) & (numpy.invert(bm.flagged(v, flag=bm.exclude_for_reducing))) + + new_bpm = bm.flagged(v, exclude='BOXSLIT', and_not=bm.exclude_for_reducing) + + assert numpy.all(new_bpm == desired_bpm) + From 96db45a6c1dc98cc410955c2c958390ba2a030a3 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Tue, 30 Jan 2024 11:13:55 -0800 Subject: [PATCH 222/244] change wavcalib reader --- pypeit/calibframe.py | 1 - pypeit/core/wavecal/wv_fitting.py | 89 ++++++++++++++++++++++++------ pypeit/tests/test_wvcalib.py | 7 +++ pypeit/wavecalib.py | 91 ++++++++++++++++++++++++++++++- 4 files changed, 170 insertions(+), 18 deletions(-) diff --git a/pypeit/calibframe.py b/pypeit/calibframe.py index 7acc033dff..a626a4482d 100644 --- a/pypeit/calibframe.py +++ b/pypeit/calibframe.py @@ -202,7 +202,6 @@ def calib_keys_from_header(self, hdr): self.calib_id = self.ingest_calib_id(hdr['CALIBID']) else: msgs.warn('Header does not have CALIBID card; cannot parse calibration IDs.') - return self @staticmethod def parse_key_dir(inp, from_filename=False): diff --git a/pypeit/core/wavecal/wv_fitting.py b/pypeit/core/wavecal/wv_fitting.py index 2481f89d50..c8d0eb242a 100644 --- a/pypeit/core/wavecal/wv_fitting.py +++ b/pypeit/core/wavecal/wv_fitting.py @@ -68,8 +68,35 @@ class WaveFit(datamodel.DataContainer): @staticmethod def hduext_prefix_from_spatid(spat_id): - """ Naming for HDU extensions""" + """ + Use the slit spatial ID number to construct the prefix for an output + HDU. + + Args: + spat_id (:obj:`int`): + Slit/order spatial ID number + + Returns: + :obj:`str`: The prefix to use for HDU extensions associated with + this data container. + """ return 'SPAT_ID-{}_'.format(spat_id) + + @staticmethod + def parse_spatid_from_hduext(ext): + """ + Parse the slit spatial ID integer from a *full* HDU extension name. + + Args: + ext (:obj:`str`): + Full extension name. + + Returns: + :obj:`int`: The parsed slit spatial ID number. Returns None if the + extension name does not start with ``'SPAT_ID-'``. + """ + return int(ext.replace('SPAT_ID-', '').split('_')[0]) \ + if ext.startswith('SPAT_ID-') else None def __init__(self, spat_id, ech_order=None, pypeitfit=None, pixel_fit=None, wave_fit=None, ion_bits=None, cen_wave=None, cen_disp=None, spec=None, wave_soln=None, @@ -128,31 +155,61 @@ def to_hdu(self, **kwargs): return super().to_hdu(force_to_bintbl=True, **kwargs) @classmethod - def from_hdu(cls, hdu, **kwargs): + def from_hdu(cls, hdu, hdu_prefix=None, spat_id=None, **kwargs): """ - Parse the data from the provided HDU. + Parse the data from one or more fits objects. - See the base-class :func:`~pypeit.datamodel.DataContainer.from_hdu` for - the argument descriptions. + .. note:: + ``spat_id`` and ``hdu_prefix`` are mutually exclusive, with + ``hdu_prefix`` taking precedence. If *both* are None, the code + attempts to read the spatial ID number from the HDU extension + name(s) to construct the appropriate prefix. In the latter + case, only the first :class:`WaveFit` object will be read if the + HDU object contains many. + Args: + hdu (`astropy.io.fits.HDUList`_, `astropy.io.fits.ImageHDU`_, `astropy.io.fits.BinTableHDU`_): + The HDU(s) with the data to use for instantiation. + hdu_prefix (:obj:`str`, optional): + Only parse HDUs with extension names matched to this prefix. If + None, ``spat_id`` will be used to construct the prefix if it is + provided, otherwise all extensions are parsed. + spat_id (:obj:`int`, optional): + Spatial ID number for the relevant slit/order for this + wavelength fit. If provided, this is used to construct the + ``hdu_prefix`` using + :func:`~pypeit.core.wavecal.wv_fitting.WaveFit.hduext_prefix_from_spatid`. kwargs: Passed directly to the base-class - :class:`~pypeit.datamodel.DataContainer.from_hdu`. Should not - include ``hdu_prefix`` because this is set directly by the - class, read from the SPAT_ID card in a relevant header. + :class:`~pypeit.datamodel.DataContainer.from_hdu`. Returns: :class:`WaveFit`: Object instantiated from data in the provided HDU. """ - if 'hdu_prefix' in kwargs: - kwargs.pop('hdu_prefix') - # Set hdu_prefix - spat_id = hdu[1].header['SPAT_ID'] if isinstance(hdu, fits.HDUList)\ - else hdu.header['SPAT_ID'] - hdu_prefix = cls.hduext_prefix_from_spatid(spat_id) - # Run the default parser to get the data - return super().from_hdu(hdu, hdu_prefix=hdu_prefix, **kwargs) + # Get the HDU prefix + if hdu_prefix is not None: + _hdu_prefix = hdu_prefix + elif spat_id is not None: + _hdu_prefix = cls.hduext_prefix_from_spatid(spat_id) + else: + # Try to get it from the provided HDU(s) + _hdu = hdu if isinstance(hdu, fits.HDUList) else [hdu] + _hdu_prefix = None + for h in _hdu: + _spat_id = cls.parse_spatid_from_hduext(h.name) + if _spat_id is None: + continue + _hdu_prefix = cls.hduext_prefix_from_spatid(_spat_id) + break + # At this point, _hdu_prefix will either be None because it couldn't + # find parse the slit ID, or it will be the correct prefix. + # TODO: Try to read the data in either case (what is currently + # done), or fault because the _bundle function *always* includes a + # prefix? + + # Construct and return the object + return super().from_hdu(hdu, hdu_prefix=_hdu_prefix, **kwargs) @property def ions(self): diff --git a/pypeit/tests/test_wvcalib.py b/pypeit/tests/test_wvcalib.py index 4e85aef2a4..d49100fb95 100644 --- a/pypeit/tests/test_wvcalib.py +++ b/pypeit/tests/test_wvcalib.py @@ -17,6 +17,13 @@ from pypeit.tests.tstutils import data_path +def test_wavefit_hduprefix(): + spat_id = 175 + prefix = wv_fitting.WaveFit.hduext_prefix_from_spatid(spat_id) + _spat_id = wv_fitting.WaveFit.parse_spatid_from_hduext(prefix) + assert spat_id == _spat_id, 'Bad parse' + + def test_wavefit(): "Fuss with the WaveFit DataContainer" out_file = Path(data_path('test_wavefit.fits')).resolve() diff --git a/pypeit/wavecalib.py b/pypeit/wavecalib.py index e930034bd8..f30d59ecc9 100644 --- a/pypeit/wavecalib.py +++ b/pypeit/wavecalib.py @@ -12,6 +12,7 @@ from linetools.utils import jsonify from astropy.table import Table +from astropy.io import fits from pypeit import msgs from pypeit.core import arc, qa @@ -158,6 +159,7 @@ def _bundle(self): # Return return _d + ''' @classmethod def _parse(cls, hdu, **kwargs): """ @@ -178,8 +180,13 @@ def _parse(cls, hdu, **kwargs): if len(ihdu.data) == 0: # TODO: This is a hack. We shouldn't be writing empty HDUs, # except for the primary HDU. - iwavefit = wv_fitting.WaveFit(ihdu.header['SPAT_ID'], ech_order=ihdu.header.get('ECH_ORDER')) + iwavefit = wv_fitting.WaveFit(ihdu.header['SPAT_ID'], + ech_order=ihdu.header.get('ECH_ORDER')) else: + + embed() + exit() + # TODO -- Replace the following with WaveFit._parse() and pass that back!! iwavefit = wv_fitting.WaveFit.from_hdu(ihdu)# , chk_version=False) parsed_hdus += [ihdu.name] @@ -215,6 +222,88 @@ def _parse(cls, hdu, **kwargs): if len(list_of_fwhm_fits) > 0: _d['fwhm_map'] = np.asarray(list_of_fwhm_fits) return _d, dm_version_passed, dm_type_passed, parsed_hdus + ''' + + @classmethod + def from_hdu(cls, hdu, chk_version=True, **kwargs): + """ + Instantiate the object from an HDU extension. + + This overrides the base-class method. Overriding this method is + preferrable to overriding the ``_parse`` method because it makes it + easier to deal with the :class:`~pypeit.datamodel.DataContainer` nesting + of this object. + + Args: + hdu (`astropy.io.fits.HDUList`_, `astropy.io.fits.ImageHDU`_, `astropy.io.fits.BinTableHDU`_): + The HDU(s) with the data to use for instantiation. + chk_version (:obj:`bool`, optional): + If True, raise an error if the datamodel version or + type check failed. If False, throw a warning only. + kwargs (:obj:`dict`, optional): + Used for consistency with base class. Ignored. + """ + # Run the default parser to get most of the data. This won't parse the + # extensions with the WAVEFIT, WAVE2DFIT, or FWHMFIT results. + d, version_passed, type_passed, parsed_hdus = super()._parse(hdu) + # Check + cls._check_parsed(version_passed, type_passed, chk_version=chk_version) + + # Get the list of all extensions + ext = [h.name for h in hdu] if isinstance(hdu, fits.HDUList) else [hdu.name] + + # Get the SPAT_IDs + if 'SPAT_IDS' in parsed_hdus: + # Use the ones parsed above + spat_ids = d['spat_ids'] + else: + # This line parses all the spat_ids from the extension names, + # filters out any None values from the list, and gets the unique set + # of integers + spat_ids = np.unique(list(filter(None.__ne__, + [wv_fitting.WaveFit.parse_spatid_from_hduext(e) for e in ext]))) + + # Parse all the WAVEFIT extensions + wave_fits = [] + for spat_id in spat_ids: + _ext = wv_fitting.WaveFit.hduext_prefix_from_spatid(spat_id)+'WAVEFIT' + if _ext not in ext: + continue + # TODO: I (KBW) don't think we should be writing empty HDUs + if len(hdu[_ext].data) == 0: + wave_fits += [wv_fitting.WaveFit(hdu[_ext].header['SPAT_ID'], + ech_order=hdu[_ext].header.get('ECH_ORDER'))] + else: + wave_fits += [wv_fitting.WaveFit.from_hdu(hdu, spat_id=spat_id, + chk_version=chk_version)] + if len(wave_fits) > 0: + d['wv_fits'] = np.asarray(wave_fits) + + # Parse all the WAVE2DFIT extensions + # TODO: It would be good to have the WAVE2DFIT extensions follow the + # same naming convention as the WAVEFIT extensions... + wave2d_fits = [fitting.PypeItFit.from_hdu(hdu[e], chk_version=chk_version) + for e in ext if 'WAVE2DFIT' in e] + if len(wave2d_fits) > 0: + d['wv_fit2d'] = np.asarray(wave2d_fits) + + # Parse all the FWHMFIT extensions + fwhm_fits = [] + for _ext in ext: + if 'FWHMFIT' not in _ext: + continue + # TODO: I (KBW) don't think we should be writing empty HDUs + fwhm_fits += [fitting.PypeItFit() if len(hdu[_ext].data) == 0 \ + else fitting.PypeItFit.from_hdu(hdu[_ext], chk_version=chk_version)] + if len(fwhm_fits) > 0: + d['fwhm_map'] = np.asarray(fwhm_fits) + + # Instantiate the object + self = cls.from_dict(d=d) + # This is a CalibFrame, so parse the relevant keys for the naming system + self.calib_keys_from_header(hdu[parsed_hdus[0]].header) + # Return the constructed object + return self @property def par(self): From ff3d2485d979e85f725380fa781654829da3aee0 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Tue, 30 Jan 2024 11:54:16 -0800 Subject: [PATCH 223/244] PR comments --- pypeit/flatfield.py | 4 +-- pypeit/pypeit.py | 16 ++++----- pypeit/scripts/show_2dspec.py | 10 ++++-- pypeit/wavecalib.py | 65 ----------------------------------- 4 files changed, 17 insertions(+), 78 deletions(-) diff --git a/pypeit/flatfield.py b/pypeit/flatfield.py index cb7c5e8a31..6c212510fb 100644 --- a/pypeit/flatfield.py +++ b/pypeit/flatfield.py @@ -16,7 +16,7 @@ from IPython import embed from pypeit import msgs -from pypeit.pypmsgs import PypeItError +from pypeit.pypmsgs import PypeItDataModelError from pypeit import utils from pypeit import bspline @@ -420,7 +420,7 @@ def show(self, frametype='all', slits=None, wcs_match=True, chk_version=True): calib_dir=self.calib_dir) try: slits = slittrace.SlitTraceSet.from_file(slits_file, chk_version=chk_version) - except (FileNotFoundError, PypeItError): + except (FileNotFoundError, PypeItDataModelError): msgs.warn('Could not load slits to include when showing flat-field images. File ' 'was either not provided directly, or it could not be read based on its ' f'expected name: {slits_file}.') diff --git a/pypeit/pypeit.py b/pypeit/pypeit.py index 06fdca97d6..cb199c91f5 100644 --- a/pypeit/pypeit.py +++ b/pypeit/pypeit.py @@ -161,7 +161,6 @@ def __init__(self, pypeit_file, verbosity=2, overwrite=True, reuse_calibs=False, self.det = None self.tstart = None self.basename = None -# self.sciI = None self.obstime = None @property @@ -283,12 +282,13 @@ def calib_all(self): for self.det in detectors: msgs.info(f'Working on detector {self.det}') # Instantiate Calibrations class + user_slits = slittrace.merge_user_slit(self.par['rdx']['slitspatnum'], + self.par['rdx']['maskIDs']) self.caliBrate = calibrations.Calibrations.get_instance( self.fitstbl, self.par['calibrations'], self.spectrograph, self.calibrations_path, qadir=self.qa_path, reuse_calibs=self.reuse_calibs, - show=self.show, - user_slits=slittrace.merge_user_slit(self.par['rdx']['slitspatnum'], - self.par['rdx']['maskIDs'])) + show=self.show, user_slits=user_slits, + chk_version=self.par['rdx']['chk_version']) # Do it # These need to be separate to accommodate COADD2D self.caliBrate.set_config(grp_frames[0], self.det, self.par['calibrations']) @@ -695,13 +695,13 @@ def calib_one(self, frames, det): msgs.info(f'Building/loading calibrations for detector {det}') # Instantiate Calibrations class + user_slits=slittrace.merge_user_slit(self.par['rdx']['slitspatnum'], + self.par['rdx']['maskIDs']) caliBrate = calibrations.Calibrations.get_instance( self.fitstbl, self.par['calibrations'], self.spectrograph, self.calibrations_path, qadir=self.qa_path, - reuse_calibs=self.reuse_calibs, show=self.show, - user_slits=slittrace.merge_user_slit( - self.par['rdx']['slitspatnum'], self.par['rdx']['maskIDs'])) - #slitspat_num=self.par['rdx']['slitspatnum']) + reuse_calibs=self.reuse_calibs, show=self.show, user_slits=user_slits, + chk_version=self.par['rdx']['chk_version']) # These need to be separate to accomodate COADD2D caliBrate.set_config(frames[0], det, self.par['calibrations']) caliBrate.run_the_steps() diff --git a/pypeit/scripts/show_2dspec.py b/pypeit/scripts/show_2dspec.py index e426872f47..d95b9446a9 100644 --- a/pypeit/scripts/show_2dspec.py +++ b/pypeit/scripts/show_2dspec.py @@ -148,12 +148,16 @@ def main(args): file_pypeit_version = fits.getval(args.file, 'VERSPYP', 0) except KeyError: file_pypeit_version = '*unknown*' + addendum = 'You can also try setting --try_old. That will ignore any version ' \ + 'differences, but still may be unable to parse your spec2d file into the ' \ + 'relevant PypeIt object if the changes to the datamodel are too extensive.' msgs.warn(f'Your installed version of PypeIt ({__version__}) cannot be used to parse ' f'{args.file}, which was reduced using version {file_pypeit_version}. You ' 'are strongly encouraged to re-reduce your data using this (or, better yet, ' - 'the most recent) version of PypeIt. Script will try to parse only the ' - 'relevant bits from the spec2d file and continue (possibly with more ' - 'limited functionality).') + 'the most recent) version of PypeIt. This script will continue by trying ' + 'to parse only the relevant bits from the spec2d file and continue ' + '(possibly with more limited functionality).' + + (f' {addendum}' if chk_version else '')) spec2DObj = None if spec2DObj is None: diff --git a/pypeit/wavecalib.py b/pypeit/wavecalib.py index f30d59ecc9..f6c4beec35 100644 --- a/pypeit/wavecalib.py +++ b/pypeit/wavecalib.py @@ -159,71 +159,6 @@ def _bundle(self): # Return return _d - ''' - @classmethod - def _parse(cls, hdu, **kwargs): - """ - See :func:`~pypeit.datamodel.DataContainer._parse` for description and - list of returned objects. All keyword arguments are ignored by this - function! - """ - # Grab everything but the bsplines - _d, dm_version_passed, dm_type_passed, parsed_hdus = super()._parse(hdu) - # Now the wave_fits - list_of_wave_fits = [] - list_of_wave2d_fits = [] - list_of_fwhm_fits = [] - spat_ids = [] - for ihdu in hdu: - if 'WAVEFIT' in ihdu.name: - # Allow for empty - if len(ihdu.data) == 0: - # TODO: This is a hack. We shouldn't be writing empty HDUs, - # except for the primary HDU. - iwavefit = wv_fitting.WaveFit(ihdu.header['SPAT_ID'], - ech_order=ihdu.header.get('ECH_ORDER')) - else: - - embed() - exit() - - # TODO -- Replace the following with WaveFit._parse() and pass that back!! - iwavefit = wv_fitting.WaveFit.from_hdu(ihdu)# , chk_version=False) - parsed_hdus += [ihdu.name] - if iwavefit.version != wv_fitting.WaveFit.version: - msgs.warn("Your WaveFit is out of date!!") - # Grab PypeItFit (if it exists) - hdname = ihdu.name.replace('WAVEFIT', 'PYPEITFIT') - if hdname in [khdu.name for khdu in hdu]: - iwavefit.pypeitfit = fitting.PypeItFit.from_hdu(hdu[hdname]) - parsed_hdus += [hdname] - list_of_wave_fits.append(iwavefit) - # Grab SPAT_ID for checking - spat_ids.append(iwavefit.spat_id) - elif 'WAVE2DFIT' in ihdu.name: - iwave2dfit = fitting.PypeItFit.from_hdu(ihdu) - list_of_wave2d_fits.append(iwave2dfit) - parsed_hdus += [ihdu.name] - elif 'FWHMFIT' in ihdu.name: - # TODO: This is a hack. We shouldn't be writing empty HDUs, - # except for the primary HDU. - ifwhmfit = fitting.PypeItFit() if len(ihdu.data) == 0 \ - else fitting.PypeItFit.from_hdu(ihdu) - list_of_fwhm_fits.append(ifwhmfit) - parsed_hdus += [ihdu.name] - # Check - if spat_ids != _d['spat_ids'].tolist(): - #embed(header="198 of wavecalib.py") - msgs.error("Bad parsing of WaveCalib") - # Finish - _d['wv_fits'] = np.asarray(list_of_wave_fits) - if len(list_of_wave2d_fits) > 0: - _d['wv_fit2d'] = np.asarray(list_of_wave2d_fits) - if len(list_of_fwhm_fits) > 0: - _d['fwhm_map'] = np.asarray(list_of_fwhm_fits) - return _d, dm_version_passed, dm_type_passed, parsed_hdus - ''' - @classmethod def from_hdu(cls, hdu, chk_version=True, **kwargs): """ From 1fab88ca4fe09c16ab71f8d27a2d587d43694593 Mon Sep 17 00:00:00 2001 From: rcooke Date: Wed, 31 Jan 2024 21:30:34 +0000 Subject: [PATCH 224/244] spec2d hotfix --- pypeit/scripts/show_2dspec.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/pypeit/scripts/show_2dspec.py b/pypeit/scripts/show_2dspec.py index 5a9c13029b..ec2726e810 100644 --- a/pypeit/scripts/show_2dspec.py +++ b/pypeit/scripts/show_2dspec.py @@ -134,6 +134,9 @@ def main(args): show_channels = [0,1,2,3] if args.channels is None \ else [int(item) for item in args.channels.split(',')] + # Need to update clear throughout in case only some channels are being displayed + _clear = args.clear + # Try to read the Spec2DObj using the current datamodel, but allowing # for the datamodel version to be different try: @@ -266,8 +269,9 @@ def main(args): # Show the bitmask? if args.showmask and bpmmask is not None: - viewer, ch_mask = display.show_image(bpmmask, chname='MASK', waveimg=waveimg, - clear=args.clear) + viewer, ch_mask = display.show_image(bpmmask.mask, chname='MASK', waveimg=waveimg, + clear=_clear) + _clear = False channel_names = [] # SCIIMG @@ -278,7 +282,8 @@ def main(args): chname_sci = args.prefix+f'sciimg-{detname}' # Clear all channels at the beginning viewer, ch_sci = display.show_image(sciimg, chname=chname_sci, waveimg=waveimg, - clear=args.clear, cuts=(cut_min, cut_max)) + clear=_clear, cuts=(cut_min, cut_max)) + _clear=False if sobjs is not None: show_trace(sobjs, detname, viewer, ch_sci) display.show_slits(viewer, ch_sci, left, right, slit_ids=slid_IDs, @@ -294,8 +299,9 @@ def main(args): cut_max = mean + 4.0 * sigma chname_skysub = args.prefix+f'skysub-{detname}' viewer, ch_skysub = display.show_image(image, chname=chname_skysub, - waveimg=waveimg, cuts=(cut_min, cut_max), + waveimg=waveimg, clear=_clear, cuts=(cut_min, cut_max), wcs_match=True) + _clear = False if not args.removetrace and sobjs is not None: show_trace(sobjs, detname, viewer, ch_skysub) display.show_slits(viewer, ch_skysub, left, right, slit_ids=slid_IDs, @@ -328,7 +334,8 @@ def main(args): chname_skyresids = args.prefix+f'sky_resid-{detname}' image = (sciimg - skymodel) * np.sqrt(ivarmodel) * model_gpm.astype(float) viewer, ch_sky_resids = display.show_image(image, chname_skyresids, waveimg=waveimg, - cuts=(-5.0, 5.0)) + clear=_clear, cuts=(-5.0, 5.0)) + _clear = False if not args.removetrace and sobjs is not None: show_trace(sobjs, detname, viewer, ch_sky_resids) display.show_slits(viewer, ch_sky_resids, left, right, slit_ids=slid_IDs, @@ -341,7 +348,8 @@ def main(args): # full model residual map image = (sciimg - skymodel - objmodel) * np.sqrt(ivarmodel) * img_gpm.astype(float) viewer, ch_resids = display.show_image(image, chname=chname_resids, waveimg=waveimg, - cuts=(-5.0, 5.0), wcs_match=True) + clear=_clear, cuts=(-5.0, 5.0), wcs_match=True) + _clear = False if not args.removetrace and sobjs is not None: show_trace(sobjs, detname, viewer, ch_resids) display.show_slits(viewer, ch_resids, left, right, slit_ids=slid_IDs, From 3c38093330e9421e29c65ad6ac60bc4efb130785 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 1 Feb 2024 07:38:02 -0800 Subject: [PATCH 225/244] doc update --- doc/pypeit_par.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/pypeit_par.rst b/doc/pypeit_par.rst index 3af698f876..1f81e12f9e 100644 --- a/doc/pypeit_par.rst +++ b/doc/pypeit_par.rst @@ -411,7 +411,7 @@ Key Type Options ``sync_predict`` str ``pca``, ``nearest``, ``auto`` ``pca`` Mode to use when predicting the form of the trace to insert. Use `pca` to use the PCA decomposition, `nearest` to reproduce the shape of the nearest trace, or `auto` to let PypeIt decide which mode to use between `pca` and `nearest`. In general, it will first try `pca`, and if that is not possible, it will use `nearest`. ``sync_to_edge`` bool .. True If adding a first left edge or a last right edge, ignore `center_mode` for these edges and place them at the edge of the detector (with the relevant shape). ``trace_median_frac`` int, float .. .. After detection of peaks in the rectified Sobel-filtered image and before refitting the edge traces, the rectified image is median filtered with a kernel width of `trace_median_frac*nspec` along the spectral dimension. -``trace_rms_tol`` int, float .. .. After retracing edges using peaks detected in the rectified and collapsed image, the RMS difference between the original and refit traces are calculated. This sets the upper limit of the RMS for traces that will be removed. If None, no limit is set and all new traces are kept. +``trace_rms_tol`` int, float .. .. After retracing edges using peaks detected in the rectified and collapsed image, the RMS difference (in pixels) between the original and refit traces are calculated. This sets the upper limit of the RMS for traces that will be removed. If None, no limit is set and all new traces are kept. ``trace_thresh`` int, float .. .. After rectification and median filtering of the Sobel-filtered image (see `trace_median_frac`), values in the median-filtered image *below* this threshold are masked in the refitting of the edge trace data. If None, no masking applied. ``use_maskdesign`` bool .. False Use slit-mask designs to identify slits. =========================== ================ =========================================== ============== ====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== @@ -510,7 +510,6 @@ Class Instantiation: :class:`~pypeit.par.pypeitpar.Coadd1DPar` ==================== ========== ======= ========== ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= Key Type Options Default Description ==================== ========== ======= ========== ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= -``chk_version`` bool .. True If True enforce strict PypeIt version checking to ensure that spec1d*.fits files were createdwith the current version of PypeIt ``coaddfile`` str .. .. Output filename ``dloglam`` int, float .. .. Dispersion in units of log10(wave) in case you want to specify it in the get_wave_grid (for the 'velocity' or 'log10' options), otherwise a median value is computed from the data. ``dv`` int, float .. .. Dispersion in units of km/s in case you want to specify it in the get_wave_grid (for the 'velocity' option), otherwise a median value is computed from the data. From adccc3bd93aca800bf4e6e2bd1168fc4a7033198 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 1 Feb 2024 11:30:19 -0800 Subject: [PATCH 226/244] bit selection and doc update --- doc/help/pypeit_show_2dspec.rst | 15 ++++-- doc/out_masks.rst | 87 ++++++++++++++++++--------------- doc/outputs.rst | 1 + doc/pypeit_par.rst | 2 +- pypeit/images/bitmaskarray.py | 7 +++ pypeit/scripts/show_2dspec.py | 19 +++++-- 6 files changed, 82 insertions(+), 49 deletions(-) diff --git a/doc/help/pypeit_show_2dspec.rst b/doc/help/pypeit_show_2dspec.rst index 77d84129bd..471f218ea2 100644 --- a/doc/help/pypeit_show_2dspec.rst +++ b/doc/help/pypeit_show_2dspec.rst @@ -2,8 +2,8 @@ $ pypeit_show_2dspec -h usage: pypeit_show_2dspec [-h] [--list] [--det DET] [--spat_id SPAT_ID] - [--maskID MASKID] [--showmask] [--removetrace] - [--embed] [--ignore_extract_mask] + [--maskID MASKID] [--showmask [SHOWMASK ...]] + [--removetrace] [--embed] [--ignore_extract_mask] [--channels CHANNELS] [--prefix PREFIX] [--no_clear] [-v VERBOSITY] file @@ -24,7 +24,16 @@ --spat_id SPAT_ID Restrict plotting to this slit (PypeIt ID notation) (default: None) --maskID MASKID Restrict plotting to this maskID (default: None) - --showmask Overplot masked pixels (default: False) + --showmask [SHOWMASK ...] + Include a channel showing the mask. If no arguments are + provided, the mask bit values are provided directly. You + can also specify one or more mask flags used to + construct an images identifying which pixels are flagged + by any of these issues. E.g., to show pixels flagged by + the instrument specific bad-pixel mask or cosmic arrays, + use --showmask BPM CR . See + https://pypeit.readthedocs.io/en/release/out_masks.html + for the list of flags. (default: None) --removetrace Do not overplot traces in the skysub, sky_resid, and resid channels (default: False) --embed Upon completion embed in ipython shell (default: False) diff --git a/doc/out_masks.rst b/doc/out_masks.rst index fefde58c76..d64e654f8b 100644 --- a/doc/out_masks.rst +++ b/doc/out_masks.rst @@ -28,65 +28,72 @@ the following bits: .. include:: include/imagebitmask_table.rst -Interpreting the bit values ---------------------------- +Viewing and interpreting the bit mask image +------------------------------------------- -You can construct the appropriate bitmask in two ways: +To access the bitmask array, we recommend using +:class:`~pypeit.spec2dobj.Spec2DObj` directly: - - (Recommended) Instantiate the bitmask directly: +.. code-block:: python - .. code-block:: python + from pathlib import Path + from pypeit.spec2dobj import Spec2DObj - from pypeit.images.imagebitmask import ImageBitMask - bm = ImageBitMask() + f = Path('spec2d_b27-J1217p3905_KASTb_20150520T045733.560.fits').resolve() + detname = 'DET01' + spec2d = Spec2DObj.from_file(f, detname) + mask = spec2d.bpmmask - - Specifically for the ``spec2d*`` files, the names of the bits - and their order is saved to the header. You can instantiate a - generic :class:`~pypeit.bitmask.BitMask` from the header; - however, it's not clear how long this behavior will remain. To - instantiate the relevant :class:`~pypeit.bitmask.BitMask` from - the header: +The bitmask array is held by the ``spec2d.bpmmask`` attribute. - .. code-block:: python +But you can read it directly from the spec2d file: - from astropy.io import fits - from pypeit.bitmask import BitMask - hdu = fits.open('spec2d_DE.20100913.22358-CFHQS1_DEIMOS_20100913T061231.334.fits') - bm = BitMask(hdu[1].header['IMGBITM'].split(',')) +.. code-block:: python + from astropy.io import fits + from pypeit.images.imagebitmask import ImageBitMaskArray -With the :class:`~pypeit.bitmask.BitMask` or -:class:`~pypeit.images.imagebitmask.ImageBitMask` instantiated, you -can interpret the meaning of the ``BPMMASK`` extensions as follows: + f = Path('spec2d_b27-J1217p3905_KASTb_20150520T045733.560.fits').resolve() + detname = 'DET01' + hdu = fits.open(f) + mask = ImageBitMaskArray.from_array(hdu[f'{detname}-BPMMASK'].data) - - Use the :func:`~pypeit.bitmask.BitMask.flagged` method to - produce a boolean array that selects pixels flagged with a - given bit: +To show all the bit values directly: - .. code-block:: python +.. code-block:: python - extract_flag = bm.flagged(hdu['DET01-BPMMASK'].data, flag='EXTRACT') + from matplotlib import pyplot as plt - - Use the same method to select multiple bits: + plt.imshow(mask, origin='lower', interpolation='nearest') + plt.show() - .. code-block:: python +However, this isn't necessarily as useful as creating boolean arrays that +identify which pixels are flagged due to one or more reasons. - process_flag = bm.flagged(hdu['DET01-BPMMASK'].data, flag=['BPM', 'CR', 'SATURATION']) +E.g., to show all pixels flagged for having cosmic rays: - - Or select bits that are flagged for *any* reason: +.. code-block:: python - .. code-block:: python + plt.imshow(mask.flagged(flag='CR').astype(int), origin='lower', interpolation='nearest') + plt.show() - # You can use the `flagged` method for this - any_flag = bm.flagged(hdu['DET01-BPMMASK'].data) - # or equivalently - _any_flag = hdu['DET01-BPMMASK'].data > 0 +or as being part of the instrument-specific bad-pixel mask *or* not part of any slit: - - To get the human-readable reason that any given value is flagged, use the - :func:`~pypeit.bitmask.BitMask.flagged_bits` method. Currently this can - only be used with a *single* bit value: +.. code-block:: python + + plt.imshow(mask.flagged(flag=['BPM', 'OFFSLITS']).astype(int), + origin='lower', interpolation='nearest') + plt.show() + +You can also use the :ref:`pypeit_show_2dspec` script to include an image that +shows the full mask or an image that selects specific flags. + +To print the human-readable reason(s) any given value is flagged: + +.. code-block:: python + + coo = (0,0) # Tuple with the 2D coordinate of the pixel + print(mask.flagged_bits(coo)) - .. code-block:: python - print(bm.flagged_bits(hdu['DET01-BPMMASK'].data[0,0])) diff --git a/doc/outputs.rst b/doc/outputs.rst index 5e0171820d..cb503788bb 100644 --- a/doc/outputs.rst +++ b/doc/outputs.rst @@ -62,6 +62,7 @@ they're produced, and their current datamodel. calibrations/calibrations out_spec2D out_spec1D + out_masks .. note:: diff --git a/doc/pypeit_par.rst b/doc/pypeit_par.rst index 7d594ef779..f18d9f942f 100644 --- a/doc/pypeit_par.rst +++ b/doc/pypeit_par.rst @@ -411,7 +411,7 @@ Key Type Options ``sync_predict`` str ``pca``, ``nearest``, ``auto`` ``pca`` Mode to use when predicting the form of the trace to insert. Use `pca` to use the PCA decomposition, `nearest` to reproduce the shape of the nearest trace, or `auto` to let PypeIt decide which mode to use between `pca` and `nearest`. In general, it will first try `pca`, and if that is not possible, it will use `nearest`. ``sync_to_edge`` bool .. True If adding a first left edge or a last right edge, ignore `center_mode` for these edges and place them at the edge of the detector (with the relevant shape). ``trace_median_frac`` int, float .. .. After detection of peaks in the rectified Sobel-filtered image and before refitting the edge traces, the rectified image is median filtered with a kernel width of `trace_median_frac*nspec` along the spectral dimension. -``trace_rms_tol`` int, float .. .. After retracing edges using peaks detected in the rectified and collapsed image, the RMS difference between the original and refit traces are calculated. This sets the upper limit of the RMS for traces that will be removed. If None, no limit is set and all new traces are kept. +``trace_rms_tol`` int, float .. .. After retracing edges using peaks detected in the rectified and collapsed image, the RMS difference (in pixels) between the original and refit traces are calculated. This sets the upper limit of the RMS for traces that will be removed. If None, no limit is set and all new traces are kept. ``trace_thresh`` int, float .. .. After rectification and median filtering of the Sobel-filtered image (see `trace_median_frac`), values in the median-filtered image *below* this threshold are masked in the refitting of the edge trace data. If None, no masking applied. ``use_maskdesign`` bool .. False Use slit-mask designs to identify slits. =========================== ================ =========================================== ============== ====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== diff --git a/pypeit/images/bitmaskarray.py b/pypeit/images/bitmaskarray.py index db9c6898b6..2d03c08330 100644 --- a/pypeit/images/bitmaskarray.py +++ b/pypeit/images/bitmaskarray.py @@ -58,6 +58,13 @@ def __init__(self, shape, asuint=False): self._set_keys() self.mask = np.zeros(shape, dtype=self.bitmask.minimum_dtype(asuint=asuint)) + @classmethod + def from_array(cls, arr): + # Instantiate using the shape of the provided array + self = cls(arr.shape, asuint=np.issubdtype(arr.dtype, np.unsignedinteger)) + self.mask[...] = arr[...] + return self + def _set_keys(self): """ Set :attr:`lower_keys`, which are needed for the bit access convenience diff --git a/pypeit/scripts/show_2dspec.py b/pypeit/scripts/show_2dspec.py index ec2726e810..5ee9b6d619 100644 --- a/pypeit/scripts/show_2dspec.py +++ b/pypeit/scripts/show_2dspec.py @@ -88,8 +88,15 @@ def get_parser(cls, width=None): help='Restrict plotting to this slit (PypeIt ID notation)') parser.add_argument('--maskID', type=int, default=None, help='Restrict plotting to this maskID') - parser.add_argument('--showmask', default=False, help='Overplot masked pixels', - action='store_true') + parser.add_argument('--showmask', nargs='*', default=None, + help='Include a channel showing the mask. If no arguments are ' + 'provided, the mask bit values are provided directly. You can ' + 'also specify one or more mask flags used to construct an ' + 'images identifying which pixels are flagged by any of these ' + 'issues. E.g., to show pixels flagged by the instrument ' + 'specific bad-pixel mask or cosmic arrays, use --showmask BPM CR ' + '. See https://pypeit.readthedocs.io/en/release/out_masks.html ' + 'for the list of flags.') parser.add_argument('--removetrace', default=False, action='store_true', help='Do not overplot traces in the skysub, sky_resid, and resid ' 'channels') @@ -246,7 +253,7 @@ def main(args): sobjs = None msgs.warn('Could not find spec1d file: {:s}'.format(spec1d_file) + msgs.newline() + ' No objects were extracted.') - + # TODO: This may be too restrictive, i.e. ignore BADFLTCALIB?? slit_gpm = slit_mask == 0 @@ -268,8 +275,10 @@ def main(args): display.connect_to_ginga(raise_err=True, allow_new=True) # Show the bitmask? - if args.showmask and bpmmask is not None: - viewer, ch_mask = display.show_image(bpmmask.mask, chname='MASK', waveimg=waveimg, + if args.showmask is not None and bpmmask is not None: + _mask = bpmmask.mask if len(args.showmask) == 0 \ + else bpmmask.flagged(flag=args.showmask) + viewer, ch_mask = display.show_image(_mask, chname='MASK', waveimg=waveimg, clear=_clear) _clear = False From d83cac308eeb8efd4a7ffafc962c3311464c728d Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 1 Feb 2024 11:32:51 -0800 Subject: [PATCH 227/244] changes --- doc/releases/1.14.1dev.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/releases/1.14.1dev.rst b/doc/releases/1.14.1dev.rst index 27744e2130..726e6eab94 100644 --- a/doc/releases/1.14.1dev.rst +++ b/doc/releases/1.14.1dev.rst @@ -81,7 +81,8 @@ Script Changes now show the echelle order number, if applicable, otherwise the slit number. - ``pypeit_chk_edges`` now load SlitTraceSet (if available) to be able to overlay the echelle order numbers. - Added a -G option to ``pypeit_setup`` and ``pypeit_obslog`` that will start the new - Setup GUI. + Setup GUI. +- Improvements and bug fixes for how the mask is displayed by ``pypeit_show_2dspec``. Datamodel Changes ----------------- From 70ea85f8157ad93a85c7ea6ff88d5a706ed9583f Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 2 Feb 2024 14:45:25 +0000 Subject: [PATCH 228/244] init script --- pypeit/scripts/print_bpm.py | 103 ++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 pypeit/scripts/print_bpm.py diff --git a/pypeit/scripts/print_bpm.py b/pypeit/scripts/print_bpm.py new file mode 100644 index 0000000000..4475e75494 --- /dev/null +++ b/pypeit/scripts/print_bpm.py @@ -0,0 +1,103 @@ +""" +This script prints a user-friendly description of the bad pixel mask +based on a spec2d file. + +.. include common links, assuming primary doc root is up one directory +.. include:: ../include/links.rst +""" +import os + +import numpy as np + +from IPython import embed + +from astropy.io import fits +from astropy.stats import sigma_clipped_stats + +from pypeit import msgs +from pypeit import specobjs +from pypeit import io +from pypeit import utils +from pypeit import __version__ +from pypeit.pypmsgs import PypeItError, PypeItDataModelError + +from pypeit.display import display +from pypeit.images.imagebitmask import ImageBitMask +from pypeit.images.detector_container import DetectorContainer +from pypeit import spec2dobj +from pypeit.scripts import scriptbase + + +class Show2DSpec(scriptbase.ScriptBase): + + @classmethod + def get_parser(cls, width=None): + parser = super().get_parser(description='Print out an informative description of a ' + 'bad pixel masked value. Usually, you should ' + 'run pypeit_show_2dspec --showmask first to ' + 'see the bad pixel mask values. Then, call this ' + 'script with the BPM value that you want to find' + 'more information about.', + width=width) + + parser.add_argument('file', type=str, default=None, help='Path to a PypeIt spec2d file') + parser.add_argument('bpm', type=int, default=None, help='Bad pixel mask value to describe in plain text') + parser.add_argument('--det', default='1', type=str, + help='Detector name or number. If a number, the name is constructed ' + 'assuming the reduction is for a single detector. If a string, ' + 'it must match the name of the detector object (e.g., DET01 for ' + 'a detector, MSC01 for a mosaic). Not essential, as the BPM is ' + 'stored in the header, independently of the detector.') + + return parser + + @staticmethod + def main(args): + + # List only? + if args.list: + io.fits_open(args.file).info() + return + + # Set the verbosity, and create a logfile if verbosity == 2 + msgs.set_logfile_and_verbosity('print_bpm', args.verbosity) + + # Parse the detector name + try: + det = int(args.det) + except: + detname = args.det + else: + detname = DetectorContainer.get_name(det) + + # Try to read the Spec2DObj using the current datamodel, but allowing + # for the datamodel version to be different + try: + spec2DObj = spec2dobj.Spec2DObj.from_file(args.file, detname, chk_version=False) + except PypeItDataModelError: + try: + # Try to get the pypeit version used to write this file + file_pypeit_version = fits.getval(args.file, 'VERSPYP', 0) + except KeyError: + file_pypeit_version = '*unknown*' + msgs.warn(f'Your installed version of PypeIt ({__version__}) cannot be used to parse ' + f'{args.file}, which was reduced using version {file_pypeit_version}. You ' + 'are strongly encouraged to re-reduce your data using this (or, better yet, ' + 'the most recent) version of PypeIt. Script will try to parse only the ' + 'relevant bits from the spec2d file and continue (possibly with more ' + 'limited functionality).') + spec2DObj = None + + if spec2DObj is None: + # Try to get the relevant elements directly from the fits file + with io.fits_open(args.file) as hdu: + names = [h.name for h in hdu] + _ext = f'{detname}-BPMMASK' + bpmmask = hdu[_ext].data if _ext in names else None + else: + # Use the parsed SpecObjs object + bpmmask = spec2DObj.bpmmask + + # Finally, print out a message to point users to the online documentation + msgs.info("Please see the following website for more information:" + msgs.newline() + + "https://pypeit.readthedocs.io/en/release/out_masks.html") From 3c8e9c82b3cd6ceccd69a17f5a8f23cd8883fbc6 Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 2 Feb 2024 14:47:28 +0000 Subject: [PATCH 229/244] init script --- pypeit/scripts/__init__.py | 1 + pypeit/scripts/print_bpm.py | 2 +- setup.cfg | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pypeit/scripts/__init__.py b/pypeit/scripts/__init__.py index e08966dc63..a16a0f9573 100644 --- a/pypeit/scripts/__init__.py +++ b/pypeit/scripts/__init__.py @@ -31,6 +31,7 @@ from pypeit.scripts import multislit_flexure from pypeit.scripts import obslog from pypeit.scripts import parse_slits +from pypeit.scripts import print_bpm from pypeit.scripts import qa_html from pypeit.scripts import ql #from pypeit.scripts import ql_multislit diff --git a/pypeit/scripts/print_bpm.py b/pypeit/scripts/print_bpm.py index 4475e75494..83198ba386 100644 --- a/pypeit/scripts/print_bpm.py +++ b/pypeit/scripts/print_bpm.py @@ -28,7 +28,7 @@ from pypeit.scripts import scriptbase -class Show2DSpec(scriptbase.ScriptBase): +class PrintBPM(scriptbase.ScriptBase): @classmethod def get_parser(cls, width=None): diff --git a/setup.cfg b/setup.cfg index 969670b068..5e57b0d31d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -118,6 +118,7 @@ console_scripts = pypeit_obslog = pypeit.scripts.obslog:ObsLog.entry_point #pypeit_parse_calib_id = pypeit.scripts.parse_calib_id:ParseCalibID.entry_point pypeit_parse_slits = pypeit.scripts.parse_slits:ParseSlits.entry_point + pypeit_print_bpm = pypeit.scripts.print_bpm:PrintBPM.entry_point pypeit_qa_html = pypeit.scripts.qa_html:QAHtml.entry_point pypeit_ql = pypeit.scripts.ql:QL.entry_point #pypeit_ql_jfh_multislit = pypeit.scripts.ql_multislit:QL_Multislit.entry_point From fe91d836cf5c08553ad66d5202c3ad9ffe60f71c Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 2 Feb 2024 14:48:03 +0000 Subject: [PATCH 230/244] progress --- pypeit/scripts/print_bpm.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/pypeit/scripts/print_bpm.py b/pypeit/scripts/print_bpm.py index 83198ba386..782713d95b 100644 --- a/pypeit/scripts/print_bpm.py +++ b/pypeit/scripts/print_bpm.py @@ -5,24 +5,13 @@ .. include common links, assuming primary doc root is up one directory .. include:: ../include/links.rst """ -import os - -import numpy as np - -from IPython import embed - from astropy.io import fits -from astropy.stats import sigma_clipped_stats from pypeit import msgs -from pypeit import specobjs from pypeit import io -from pypeit import utils from pypeit import __version__ -from pypeit.pypmsgs import PypeItError, PypeItDataModelError +from pypeit.pypmsgs import PypeItDataModelError -from pypeit.display import display -from pypeit.images.imagebitmask import ImageBitMask from pypeit.images.detector_container import DetectorContainer from pypeit import spec2dobj from pypeit.scripts import scriptbase From 919d24c78eb9db432cfd1088ab6f54a2f97a09a4 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Fri, 2 Feb 2024 09:37:40 -0800 Subject: [PATCH 231/244] control flow changes to pypeit_show_2dspec --- pypeit/scripts/show_2dspec.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/pypeit/scripts/show_2dspec.py b/pypeit/scripts/show_2dspec.py index d95b9446a9..6aaa9633ae 100644 --- a/pypeit/scripts/show_2dspec.py +++ b/pypeit/scripts/show_2dspec.py @@ -148,16 +148,25 @@ def main(args): file_pypeit_version = fits.getval(args.file, 'VERSPYP', 0) except KeyError: file_pypeit_version = '*unknown*' - addendum = 'You can also try setting --try_old. That will ignore any version ' \ - 'differences, but still may be unable to parse your spec2d file into the ' \ - 'relevant PypeIt object if the changes to the datamodel are too extensive.' - msgs.warn(f'Your installed version of PypeIt ({__version__}) cannot be used to parse ' + if chk_version: + msgs_func = msgs.error + addendum = 'To allow the script to attempt to read the data anyway, use the ' \ + '--try_old command-line option. This will first try to simply ' \ + 'ignore the version number. If the datamodels are incompatible ' \ + '(e.g., the new datamodel contains components not in a previous ' \ + 'version), this may not be enough and the script will continue by ' \ + 'trying to parse only the components necessary for use by this ' \ + 'script. In either case, BEWARE that the displayed data may be in ' \ + 'error!' + else: + msgs_func = msgs.warn + addendum = 'The datamodels are sufficiently different that the script will now ' \ + 'try to parse only the components necessary for use by this ' \ + 'script. BEWARE that the displayed data may be in error!' + msgs_func(f'Your installed version of PypeIt ({__version__}) cannot be used to parse ' f'{args.file}, which was reduced using version {file_pypeit_version}. You ' 'are strongly encouraged to re-reduce your data using this (or, better yet, ' - 'the most recent) version of PypeIt. This script will continue by trying ' - 'to parse only the relevant bits from the spec2d file and continue ' - '(possibly with more limited functionality).' - + (f' {addendum}' if chk_version else '')) + 'the most recent) version of PypeIt. ' + addendum) spec2DObj = None if spec2DObj is None: From ea7d7a68a3cf07da12cae776d736f4611953c2a0 Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 2 Feb 2024 17:42:43 +0000 Subject: [PATCH 232/244] printed BPM --- pypeit/scripts/print_bpm.py | 70 ++++++++----------------------------- 1 file changed, 14 insertions(+), 56 deletions(-) diff --git a/pypeit/scripts/print_bpm.py b/pypeit/scripts/print_bpm.py index 782713d95b..97a87a1706 100644 --- a/pypeit/scripts/print_bpm.py +++ b/pypeit/scripts/print_bpm.py @@ -5,15 +5,10 @@ .. include common links, assuming primary doc root is up one directory .. include:: ../include/links.rst """ -from astropy.io import fits +from IPython import embed from pypeit import msgs -from pypeit import io -from pypeit import __version__ -from pypeit.pypmsgs import PypeItDataModelError - -from pypeit.images.detector_container import DetectorContainer -from pypeit import spec2dobj +from pypeit.images.imagebitmask import ImageBitMask from pypeit.scripts import scriptbase @@ -29,63 +24,26 @@ def get_parser(cls, width=None): 'more information about.', width=width) - parser.add_argument('file', type=str, default=None, help='Path to a PypeIt spec2d file') - parser.add_argument('bpm', type=int, default=None, help='Bad pixel mask value to describe in plain text') - parser.add_argument('--det', default='1', type=str, - help='Detector name or number. If a number, the name is constructed ' - 'assuming the reduction is for a single detector. If a string, ' - 'it must match the name of the detector object (e.g., DET01 for ' - 'a detector, MSC01 for a mosaic). Not essential, as the BPM is ' - 'stored in the header, independently of the detector.') - + parser.add_argument('bit', type=int, default=None, help='Bad pixel mask value to describe in plain text') return parser @staticmethod def main(args): - # List only? - if args.list: - io.fits_open(args.file).info() - return - - # Set the verbosity, and create a logfile if verbosity == 2 - msgs.set_logfile_and_verbosity('print_bpm', args.verbosity) + # Convert the integer bitmask value to a list of binary numbers + binvals = [int(x) for x in bin(args.bit)[2:]][::-1] - # Parse the detector name - try: - det = int(args.det) - except: - detname = args.det - else: - detname = DetectorContainer.get_name(det) + # Generate an Image BitMask object + bpm = ImageBitMask() - # Try to read the Spec2DObj using the current datamodel, but allowing - # for the datamodel version to be different - try: - spec2DObj = spec2dobj.Spec2DObj.from_file(args.file, detname, chk_version=False) - except PypeItDataModelError: - try: - # Try to get the pypeit version used to write this file - file_pypeit_version = fits.getval(args.file, 'VERSPYP', 0) - except KeyError: - file_pypeit_version = '*unknown*' - msgs.warn(f'Your installed version of PypeIt ({__version__}) cannot be used to parse ' - f'{args.file}, which was reduced using version {file_pypeit_version}. You ' - 'are strongly encouraged to re-reduce your data using this (or, better yet, ' - 'the most recent) version of PypeIt. Script will try to parse only the ' - 'relevant bits from the spec2d file and continue (possibly with more ' - 'limited functionality).') - spec2DObj = None + # Print the description of the bad pixel mask value + outstr = "This bad pixel mask value corresponds to the following:" + msgs.newline() + msgs.newline() + for i in range(len(binvals)): + if binvals[i] == 1: + outstr += "* " + bpm.descr[i] + msgs.newline() - if spec2DObj is None: - # Try to get the relevant elements directly from the fits file - with io.fits_open(args.file) as hdu: - names = [h.name for h in hdu] - _ext = f'{detname}-BPMMASK' - bpmmask = hdu[_ext].data if _ext in names else None - else: - # Use the parsed SpecObjs object - bpmmask = spec2DObj.bpmmask + # Print the list of bad pixel mask value descriptions + msgs.info(outstr) # Finally, print out a message to point users to the online documentation msgs.info("Please see the following website for more information:" + msgs.newline() + From 64fcd65c9fe7b67e84b8b1020352c787f5b3841e Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 2 Feb 2024 17:43:19 +0000 Subject: [PATCH 233/244] bugfix --- pypeit/spec2dobj.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/spec2dobj.py b/pypeit/spec2dobj.py index fd6906288d..7e8d1d3578 100644 --- a/pypeit/spec2dobj.py +++ b/pypeit/spec2dobj.py @@ -160,7 +160,7 @@ def from_hdu(cls, hdu, detname, chk_version=True): if len(ext) == 0: # No relevant extensions! - msgs.error(f'{detname} not available in any extension of {file}') + msgs.error(f'{detname} not available in any extension of the input HDUList.') mask_ext = f'{detname}-BPMMASK' has_mask = mask_ext in ext From 2dfe7280893b7fd6ae888f00137689ff1c9c7f40 Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 2 Feb 2024 18:08:22 +0000 Subject: [PATCH 234/244] implement spec2d --- pypeit/scripts/print_bpm.py | 71 +++++++++++++++++++++++++++++++++---- 1 file changed, 65 insertions(+), 6 deletions(-) diff --git a/pypeit/scripts/print_bpm.py b/pypeit/scripts/print_bpm.py index 97a87a1706..2d9b2f0937 100644 --- a/pypeit/scripts/print_bpm.py +++ b/pypeit/scripts/print_bpm.py @@ -7,8 +7,13 @@ """ from IPython import embed -from pypeit import msgs +from astropy.io import fits + +from pypeit import __version__ +from pypeit import msgs, spec2dobj +from pypeit.images.detector_container import DetectorContainer from pypeit.images.imagebitmask import ImageBitMask +from pypeit.pypmsgs import PypeItDataModelError from pypeit.scripts import scriptbase @@ -25,6 +30,17 @@ def get_parser(cls, width=None): width=width) parser.add_argument('bit', type=int, default=None, help='Bad pixel mask value to describe in plain text') + parser.add_argument('--file', type=str, default=None, help='PypeIt spec2d file to use for the description' + '(optional). If provided, the bitmask contained ' + 'in the spec2d file will be used to describe the ' + 'bad pixel mask value. If not provided, the default ' + 'pypeit bad pixel mask will be used.') + parser.add_argument('--det', default='1', type=str, + help='Detector name or number. If a number, the name is constructed ' + 'assuming the reduction is for a single detector. If a string, ' + 'it must match the name of the detector object (e.g., DET01 for ' + 'a detector, MSC01 for a mosaic). This is not required, and the ' + 'value is acceptable. Default is 1.') return parser @staticmethod @@ -33,16 +49,59 @@ def main(args): # Convert the integer bitmask value to a list of binary numbers binvals = [int(x) for x in bin(args.bit)[2:]][::-1] - # Generate an Image BitMask object - bpm = ImageBitMask() + if args.file is None: + msgs.info("Using the default PypeIt bad pixel mask.") + # Generate an Image BitMask object + bpm = ImageBitMask() + descr = bpm.descr + else: + # Read the spec2d file + msgs.info("Using the bad pixel mask from the following spec2d file:" + msgs.newline() + f"{args.file}.") + spec2d_file = args.file + + # Parse the detector name + try: + det = int(args.det) + except: + detname = args.det + else: + detname = DetectorContainer.get_name(det) + + # Try to read the Spec2DObj using the current datamodel, but allowing + # for the datamodel version to be different + try: + spec2DObj = spec2dobj.Spec2DObj.from_file(args.file, detname, chk_version=False) + except PypeItDataModelError: + try: + # Try to get the pypeit version used to write this file + file_pypeit_version = fits.getval(args.file, 'VERSPYP', 0) + except KeyError: + file_pypeit_version = '*unknown*' + msgs.warn(f'Your installed version of PypeIt ({__version__}) cannot be used to parse ' + f'{args.file}, which was reduced using version {file_pypeit_version}. You ' + 'are strongly encouraged to re-reduce your data using this (or, better yet, ' + 'the most recent) version of PypeIt. Script will try to parse only the ' + 'relevant bits from the spec2d file and continue (possibly with more ' + 'limited functionality).') + # Generate an Image BitMask object + msgs.info("Using the default PypeIt bad pixel mask.") + bpm = ImageBitMask() + descr = bpm.descr + else: + bpm = spec2DObj.bpmmask + descr = bpm.bitmask.descr # Print the description of the bad pixel mask value - outstr = "This bad pixel mask value corresponds to the following:" + msgs.newline() + msgs.newline() + outstr = f"The bad pixel mask value ({args.bit}) corresponds to the following:" \ + + msgs.newline() + msgs.newline() + bitkeys = list(bpm.bits.keys()) + # Pad the bit keys with spaces so that they all have the same length + bitlen = len(max(bitkeys, key=len)) for i in range(len(binvals)): if binvals[i] == 1: - outstr += "* " + bpm.descr[i] + msgs.newline() + outstr += f"* {bitkeys[i].ljust(bitlen)} : {descr[i]}" + msgs.newline() - # Print the list of bad pixel mask value descriptions + # Print the message to the user msgs.info(outstr) # Finally, print out a message to point users to the online documentation From 948bfc22966f3cd303458bc5225bb5ba4a6275fb Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Fri, 2 Feb 2024 13:12:45 -0800 Subject: [PATCH 235/244] update docs --- doc/api/pypeit.scripts.print_bpm.rst | 8 ++++++++ doc/api/pypeit.scripts.rst | 1 + doc/help/pypeit_print_bpm.rst | 25 +++++++++++++++++++++++++ doc/help/pypeit_show_2dspec.rst | 2 +- pypeit/scripts/show_2dspec.py | 2 +- 5 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 doc/api/pypeit.scripts.print_bpm.rst create mode 100644 doc/help/pypeit_print_bpm.rst diff --git a/doc/api/pypeit.scripts.print_bpm.rst b/doc/api/pypeit.scripts.print_bpm.rst new file mode 100644 index 0000000000..d042ad948d --- /dev/null +++ b/doc/api/pypeit.scripts.print_bpm.rst @@ -0,0 +1,8 @@ +pypeit.scripts.print\_bpm module +================================ + +.. automodule:: pypeit.scripts.print_bpm + :members: + :private-members: + :undoc-members: + :show-inheritance: diff --git a/doc/api/pypeit.scripts.rst b/doc/api/pypeit.scripts.rst index b94e122b86..4f768ea755 100644 --- a/doc/api/pypeit.scripts.rst +++ b/doc/api/pypeit.scripts.rst @@ -35,6 +35,7 @@ Submodules pypeit.scripts.multislit_flexure pypeit.scripts.obslog pypeit.scripts.parse_slits + pypeit.scripts.print_bpm pypeit.scripts.qa_html pypeit.scripts.ql pypeit.scripts.run_pypeit diff --git a/doc/help/pypeit_print_bpm.rst b/doc/help/pypeit_print_bpm.rst new file mode 100644 index 0000000000..73e2508173 --- /dev/null +++ b/doc/help/pypeit_print_bpm.rst @@ -0,0 +1,25 @@ +.. code-block:: console + + $ pypeit_print_bpm -h + usage: pypeit_print_bpm [-h] [--file FILE] [--det DET] bit + + Print out an informative description of a bad pixel masked value. Usually, you + should run pypeit_show_2dspec --showmask first to see the bad pixel mask values. + Then, call this script with the BPM value that you want to findmore information + about. + + positional arguments: + bit Bad pixel mask value to describe in plain text + + options: + -h, --help show this help message and exit + --file FILE PypeIt spec2d file to use for the description(optional). If + provided, the bitmask contained in the spec2d file will be used + to describe the bad pixel mask value. If not provided, the + default pypeit bad pixel mask will be used. (default: None) + --det DET Detector name or number. If a number, the name is constructed + assuming the reduction is for a single detector. If a string, it + must match the name of the detector object (e.g., DET01 for a + detector, MSC01 for a mosaic). This is not required, and the + value is acceptable. Default is 1. (default: 1) + \ No newline at end of file diff --git a/doc/help/pypeit_show_2dspec.rst b/doc/help/pypeit_show_2dspec.rst index 471f218ea2..080e081181 100644 --- a/doc/help/pypeit_show_2dspec.rst +++ b/doc/help/pypeit_show_2dspec.rst @@ -28,7 +28,7 @@ Include a channel showing the mask. If no arguments are provided, the mask bit values are provided directly. You can also specify one or more mask flags used to - construct an images identifying which pixels are flagged + construct an image identifying which pixels are flagged by any of these issues. E.g., to show pixels flagged by the instrument specific bad-pixel mask or cosmic arrays, use --showmask BPM CR . See diff --git a/pypeit/scripts/show_2dspec.py b/pypeit/scripts/show_2dspec.py index 5ee9b6d619..41e5769884 100644 --- a/pypeit/scripts/show_2dspec.py +++ b/pypeit/scripts/show_2dspec.py @@ -92,7 +92,7 @@ def get_parser(cls, width=None): help='Include a channel showing the mask. If no arguments are ' 'provided, the mask bit values are provided directly. You can ' 'also specify one or more mask flags used to construct an ' - 'images identifying which pixels are flagged by any of these ' + 'image identifying which pixels are flagged by any of these ' 'issues. E.g., to show pixels flagged by the instrument ' 'specific bad-pixel mask or cosmic arrays, use --showmask BPM CR ' '. See https://pypeit.readthedocs.io/en/release/out_masks.html ' From d0b2f5d3cb90332e2755f9d5e68caeeb09053c04 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Fri, 2 Feb 2024 13:23:04 -0800 Subject: [PATCH 236/244] doc update --- doc/help/run_pypeit.rst | 2 +- doc/out_masks.rst | 30 ++++++++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/doc/help/run_pypeit.rst b/doc/help/run_pypeit.rst index f65bce9629..43055026a6 100644 --- a/doc/help/run_pypeit.rst +++ b/doc/help/run_pypeit.rst @@ -4,7 +4,7 @@ usage: run_pypeit [-h] [-v VERBOSITY] [-r REDUX_PATH] [-m] [-s] [-o] [-c] pypeit_file - ## PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.14.1.dev696+g00535ea2e.d20240117 + ## PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.14.1.dev729+g948bfc229.d20240202 ## ## Available spectrographs include: ## bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, diff --git a/doc/out_masks.rst b/doc/out_masks.rst index d64e654f8b..1cb6e41b7d 100644 --- a/doc/out_masks.rst +++ b/doc/out_masks.rst @@ -28,8 +28,8 @@ the following bits: .. include:: include/imagebitmask_table.rst -Viewing and interpreting the bit mask image -------------------------------------------- +Viewing and interpreting the bit mask image (python) +---------------------------------------------------- To access the bitmask array, we recommend using :class:`~pypeit.spec2dobj.Spec2DObj` directly: @@ -95,5 +95,31 @@ To print the human-readable reason(s) any given value is flagged: coo = (0,0) # Tuple with the 2D coordinate of the pixel print(mask.flagged_bits(coo)) +.. _pypeit_print_bpm: + +pypeit_print_bpm +---------------- + +This simple executable allows you to effectively do the above via a command-line +script. The script usage can be displayed by calling the script with the +``-h`` option: + +.. include:: help/pypeit_print_bpm.rst + +A typical call and its output looks like this: + +.. code-block:: bash + + % pypeit_print_bpm 23 + [INFO] :: Using the default PypeIt bad pixel mask. + [INFO] :: The bad pixel mask value (23) corresponds to the following: + + * BPM : Component of the instrument-specific bad pixel mask + * CR : Cosmic ray detected + * SATURATION : Saturated pixel + * OFFSLITS : Pixel does not belong to any slit + + [INFO] :: Please see the following website for more information: + https://pypeit.readthedocs.io/en/release/out_masks.html From f564537d5fe1a826b820e3678e72dd534197d604 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Fri, 2 Feb 2024 13:24:24 -0800 Subject: [PATCH 237/244] changes --- doc/releases/1.14.1dev.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/releases/1.14.1dev.rst b/doc/releases/1.14.1dev.rst index 726e6eab94..456df329ea 100644 --- a/doc/releases/1.14.1dev.rst +++ b/doc/releases/1.14.1dev.rst @@ -82,7 +82,9 @@ Script Changes - ``pypeit_chk_edges`` now load SlitTraceSet (if available) to be able to overlay the echelle order numbers. - Added a -G option to ``pypeit_setup`` and ``pypeit_obslog`` that will start the new Setup GUI. -- Improvements and bug fixes for how the mask is displayed by ``pypeit_show_2dspec``. +- Improvements and bug fixes for how the mask is displayed by + ``pypeit_show_2dspec``, and added ``pypeit_print_bpm`` to allow for a quick + report on the flags associated with a given bit value. Datamodel Changes ----------------- From 12162908637f9eb8f6f92d8a0a036d9672db56a1 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Fri, 2 Feb 2024 16:32:00 -0800 Subject: [PATCH 238/244] doc update --- doc/pypeit_par.rst | 72 +++++++++++++++++++++++----------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/doc/pypeit_par.rst b/doc/pypeit_par.rst index efe017ab3a..ce1532dd1d 100644 --- a/doc/pypeit_par.rst +++ b/doc/pypeit_par.rst @@ -323,7 +323,7 @@ Key Type Options ``slit_illum_finecorr`` bool .. True If True, a fine correction to the spatial illumination profile will be performed. The fine correction is a low order 2D polynomial fit to account for a gradual change to the spatial illumination profile as a function of wavelength. ``slit_illum_pad`` int, float .. 5.0 The number of pixels to pad the slit edges when constructing the slit-illumination profile. Single value applied to both edges. ``slit_illum_ref_idx`` int .. 0 The index of a reference slit (0-indexed) used for estimating the relative spectral sensitivity (or the relative blaze). This parameter is only used if ``slit_illum_relative = True``. -``slit_illum_relative`` bool .. False Generate an image of the relative spectral illumination for a multi-slit setup. If you set ``use_specillum = True`` for any of the frames that use the flatfield model, this *must* be set to True. Currently, this is only used for SlicerIFU reductions. +``slit_illum_relative`` bool .. False Generate an image of the relative spectral illumination for a multi-slit setup. If you set ``use_specillum = True`` for any of the frames that use the flatfield model, this *must* be set to True. Currently, this is only used for SlicerIFU reductions. ``slit_illum_smooth_npix`` int .. 10 The number of pixels used to determine smoothly varying relative weights is given by ``nspec/slit_illum_smooth_npix``, where nspec is the number of spectral pixels. ``slit_trim`` int, float, tuple .. 3.0 The number of pixels to trim each side of the slit when selecting pixels to use for fitting the spectral response function. Single values are used for both slit edges; a two-tuple can be used to trim the left and right sides differently. ``spat_samp`` int, float .. 5.0 Spatial sampling for slit illumination function. This is the width of the median filter in pixels used to determine the slit illumination function, and thus sets the minimum scale on which the illumination function will have features. @@ -508,7 +508,7 @@ Coadd1DPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.Coadd1DPar` ==================== ========== ======= ========== ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= -Key Type Options Default Description +Key Type Options Default Description ==================== ========== ======= ========== ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= ``coaddfile`` str .. .. Output filename ``dloglam`` int, float .. .. Dispersion in units of log10(wave) in case you want to specify it in the get_wave_grid (for the 'velocity' or 'log10' options), otherwise a median value is computed from the data. @@ -692,33 +692,33 @@ CubePar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.CubePar` ==================== ===== =============================================================================== ============ ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= -Key Type Options Default Description +Key Type Options Default Description ==================== ===== =============================================================================== ============ ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= -``align`` bool .. False If set to True, the input frames will be spatially aligned by cross-correlating the whitelight images with either a reference image (see ``reference_image``) or the whitelight image that is generated using the first spec2d listed in the coadd3d file. Alternatively, the user can specify the offsets (i.e. Delta RA x cos(dec) and Delta Dec, both in arcsec) in the spec2d block of the coadd3d file. See the documentation for examples of this usage. -``astrometric`` bool .. True If true, an astrometric correction will be applied using the alignment frames. -``combine`` bool .. False If set to True, the input frames will be combined. Otherwise, a separate datacube will be generated for each input spec2d file, and will be saved as a spec3d file. -``dec_max`` float .. .. Maximum DEC to use when generating the WCS. If None, the default is maximum DEC based on the WCS of all spaxels. Units should be degrees. -``dec_min`` float .. .. Minimum DEC to use when generating the WCS. If None, the default is minimum DEC based on the WCS of all spaxels. Units should be degrees. -``grating_corr`` bool .. True This option performs a small correction for the relative blaze function of all input frames that have (even slightly) different grating angles, or if you are flux calibrating your science data with a standard star that was observed with a slightly different setup. -``method`` str ``subpixel``, ``ngp`` ``subpixel`` What method should be used to generate the datacube. There are currently two options: (1) "subpixel" (default) - this algorithm divides each pixel in the spec2d frames into subpixels, and assigns each subpixel to a voxel of the datacube. Flux is conserved, but voxels are correlated, and the error spectrum does not account for covariance between adjacent voxels. See also, spec_subpixel and spat_subpixel. (2) "ngp" (nearest grid point) - this algorithm is effectively a 3D histogram. Flux is conserved, voxels are not correlated, however this option suffers the same downsides as any histogram; the choice of bin sizes can change how the datacube appears. This algorithm takes each pixel on the spec2d frame and puts the flux of this pixel into one voxel in the datacube. Depending on the binning used, some voxels may be empty (zero flux) while a neighboring voxel might contain the flux from two spec2d pixels. Note that all spec2d pixels that contribute to the same voxel are inverse variance weighted (e.g. if two pixels have the same variance, the voxel would be assigned the average flux of the two pixels). -``output_filename`` str .. .. If combining multiple frames, this string sets the output filename of the combined datacube. If combine=False, the output filenames will be prefixed with ``spec3d_*`` -``ra_max`` float .. .. Maximum RA to use when generating the WCS. If None, the default is maximum RA based on the WCS of all spaxels. Units should be degrees. -``ra_min`` float .. .. Minimum RA to use when generating the WCS. If None, the default is minimum RA based on the WCS of all spaxels. Units should be degrees. -``reference_image`` str .. .. White light image of a previously combined datacube. The white light image will be used as a reference when calculating the offsets of the input spec2d files. Ideally, the reference image should have the same shape as the data to be combined (i.e. set the ra_min, ra_max etc. params so they are identical to the reference image). -``save_whitelight`` bool .. False Save a white light image of the combined datacube. The output filename will be given by the "output_filename" variable with a suffix "_whitelight". Note that the white light image collapses the flux along the wavelength axis, so some spaxels in the 2D white light image may have different wavelength ranges. To set the wavelength range, use the "whitelight_range" parameter. If combine=False, the individual spec3d files will have a suffix "_whitelight". -``scale_corr`` str .. .. This option performs a small correction for the relative spectral illumination scale of different spec2D files. Specify the relative path+file to the spec2D file that you would like to use for the relative scaling. If you want to perform this correction, it is best to use the spec2d file with the highest S/N sky spectrum. You should choose the same frame for both the standards and science frames. -``skysub_frame`` str .. ``image`` Set the sky subtraction to be implemented. The default behaviour is to subtract the sky using the model that is derived from each individual image (i.e. set this parameter to "image"). To turn off sky subtraction completely, set this parameter to "none" (all lowercase). Finally, if you want to use a different frame for the sky subtraction, specify the relative path+file to the spec2D file that you would like to use for the sky subtraction. The model fit to the sky of the specified frame will be used. Note, the sky and science frames do not need to have the same exposure time; the sky model will be scaled to the science frame based on the relative exposure time. -``slice_subpixel`` int .. 5 When method=subpixel, slice_subpixel sets the subpixellation scale of each IFU slice. The default option is to divide each slice into 5 sub-slices during datacube creation. See also, spec_subpixel and spat_subpixel. -``slit_spec`` bool .. True If the data use slits in one spatial direction, set this to True. If the data uses fibres for all spaxels, set this to False. -``spat_subpixel`` int .. 5 When method=subpixel, spat_subpixel sets the subpixellation scale of each detector pixel in the spatial direction. The total number of subpixels in each pixel is given by spec_subpixel x spat_subpixel. The default option is to divide each spec2d pixel into 25 subpixels during datacube creation. See also, spec_subpixel and slice_subpixel. -``spatial_delta`` float .. .. The spatial size of each spaxel to use when generating the WCS (in arcsec). If None, the default is set by the spectrograph file. -``spec_subpixel`` int .. 5 When method=subpixel, spec_subpixel sets the subpixellation scale of each detector pixel in the spectral direction. The total number of subpixels in each pixel is given by spec_subpixel x spat_subpixel. The default option is to divide each spec2d pixel into 25 subpixels during datacube creation. See also, spat_subpixel and slice_subpixel. -``standard_cube`` str .. .. Filename of a standard star datacube. This cube will be used to correct the relative scales of the slits, and to flux calibrate the science datacube. -``wave_delta`` float .. .. The wavelength step to use when generating the WCS (in Angstroms). If None, the default is set by the wavelength solution. -``wave_max`` float .. .. Maximum wavelength to use when generating the WCS. If None, the default is maximum wavelength based on the WCS of all spaxels. Units should be Angstroms. -``wave_min`` float .. .. Minimum wavelength to use when generating the WCS. If None, the default is minimum wavelength based on the WCS of all spaxels. Units should be Angstroms. +``align`` bool .. False If set to True, the input frames will be spatially aligned by cross-correlating the whitelight images with either a reference image (see ``reference_image``) or the whitelight image that is generated using the first spec2d listed in the coadd3d file. Alternatively, the user can specify the offsets (i.e. Delta RA x cos(dec) and Delta Dec, both in arcsec) in the spec2d block of the coadd3d file. See the documentation for examples of this usage. +``astrometric`` bool .. True If true, an astrometric correction will be applied using the alignment frames. +``combine`` bool .. False If set to True, the input frames will be combined. Otherwise, a separate datacube will be generated for each input spec2d file, and will be saved as a spec3d file. +``dec_max`` float .. .. Maximum DEC to use when generating the WCS. If None, the default is maximum DEC based on the WCS of all spaxels. Units should be degrees. +``dec_min`` float .. .. Minimum DEC to use when generating the WCS. If None, the default is minimum DEC based on the WCS of all spaxels. Units should be degrees. +``grating_corr`` bool .. True This option performs a small correction for the relative blaze function of all input frames that have (even slightly) different grating angles, or if you are flux calibrating your science data with a standard star that was observed with a slightly different setup. +``method`` str ``subpixel``, ``ngp`` ``subpixel`` What method should be used to generate the datacube. There are currently two options: (1) "subpixel" (default) - this algorithm divides each pixel in the spec2d frames into subpixels, and assigns each subpixel to a voxel of the datacube. Flux is conserved, but voxels are correlated, and the error spectrum does not account for covariance between adjacent voxels. See also, spec_subpixel and spat_subpixel. (2) "ngp" (nearest grid point) - this algorithm is effectively a 3D histogram. Flux is conserved, voxels are not correlated, however this option suffers the same downsides as any histogram; the choice of bin sizes can change how the datacube appears. This algorithm takes each pixel on the spec2d frame and puts the flux of this pixel into one voxel in the datacube. Depending on the binning used, some voxels may be empty (zero flux) while a neighboring voxel might contain the flux from two spec2d pixels. Note that all spec2d pixels that contribute to the same voxel are inverse variance weighted (e.g. if two pixels have the same variance, the voxel would be assigned the average flux of the two pixels). +``output_filename`` str .. .. If combining multiple frames, this string sets the output filename of the combined datacube. If combine=False, the output filenames will be prefixed with ``spec3d_*`` +``ra_max`` float .. .. Maximum RA to use when generating the WCS. If None, the default is maximum RA based on the WCS of all spaxels. Units should be degrees. +``ra_min`` float .. .. Minimum RA to use when generating the WCS. If None, the default is minimum RA based on the WCS of all spaxels. Units should be degrees. +``reference_image`` str .. .. White light image of a previously combined datacube. The white light image will be used as a reference when calculating the offsets of the input spec2d files. Ideally, the reference image should have the same shape as the data to be combined (i.e. set the ra_min, ra_max etc. params so they are identical to the reference image). +``save_whitelight`` bool .. False Save a white light image of the combined datacube. The output filename will be given by the "output_filename" variable with a suffix "_whitelight". Note that the white light image collapses the flux along the wavelength axis, so some spaxels in the 2D white light image may have different wavelength ranges. To set the wavelength range, use the "whitelight_range" parameter. If combine=False, the individual spec3d files will have a suffix "_whitelight". +``scale_corr`` str .. .. This option performs a small correction for the relative spectral illumination scale of different spec2D files. Specify the relative path+file to the spec2D file that you would like to use for the relative scaling. If you want to perform this correction, it is best to use the spec2d file with the highest S/N sky spectrum. You should choose the same frame for both the standards and science frames. +``skysub_frame`` str .. ``image`` Set the sky subtraction to be implemented. The default behaviour is to subtract the sky using the model that is derived from each individual image (i.e. set this parameter to "image"). To turn off sky subtraction completely, set this parameter to "none" (all lowercase). Finally, if you want to use a different frame for the sky subtraction, specify the relative path+file to the spec2D file that you would like to use for the sky subtraction. The model fit to the sky of the specified frame will be used. Note, the sky and science frames do not need to have the same exposure time; the sky model will be scaled to the science frame based on the relative exposure time. +``slice_subpixel`` int .. 5 When method=subpixel, slice_subpixel sets the subpixellation scale of each IFU slice. The default option is to divide each slice into 5 sub-slices during datacube creation. See also, spec_subpixel and spat_subpixel. +``slit_spec`` bool .. True If the data use slits in one spatial direction, set this to True. If the data uses fibres for all spaxels, set this to False. +``spat_subpixel`` int .. 5 When method=subpixel, spat_subpixel sets the subpixellation scale of each detector pixel in the spatial direction. The total number of subpixels in each pixel is given by spec_subpixel x spat_subpixel. The default option is to divide each spec2d pixel into 25 subpixels during datacube creation. See also, spec_subpixel and slice_subpixel. +``spatial_delta`` float .. .. The spatial size of each spaxel to use when generating the WCS (in arcsec). If None, the default is set by the spectrograph file. +``spec_subpixel`` int .. 5 When method=subpixel, spec_subpixel sets the subpixellation scale of each detector pixel in the spectral direction. The total number of subpixels in each pixel is given by spec_subpixel x spat_subpixel. The default option is to divide each spec2d pixel into 25 subpixels during datacube creation. See also, spat_subpixel and slice_subpixel. +``standard_cube`` str .. .. Filename of a standard star datacube. This cube will be used to correct the relative scales of the slits, and to flux calibrate the science datacube. +``wave_delta`` float .. .. The wavelength step to use when generating the WCS (in Angstroms). If None, the default is set by the wavelength solution. +``wave_max`` float .. .. Maximum wavelength to use when generating the WCS. If None, the default is maximum wavelength based on the WCS of all spaxels. Units should be Angstroms. +``wave_min`` float .. .. Minimum wavelength to use when generating the WCS. If None, the default is minimum wavelength based on the WCS of all spaxels. Units should be Angstroms. ``weight_method`` str ``auto``, ``constant``, ``uniform``, ``wave_dependent``, ``relative``, ``ivar`` ``auto`` Method used to weight the spectra for coadding. The options are: 'auto' -- Use constant weights if rms_sn < 3.0, otherwise use wavelength dependent.'constant' -- Constant weights based on rms_sn**2'uniform' -- Uniform weighting'wave_dependent' -- Wavelength dependent weights will be used irrespective of the rms_sn ratio. This option will not work well at low S/N ratio although it is useful for objects where only a small fraction of the spectral coverage has high S/N ratio (like high-z quasars).'relative' -- Apply relative weights implying one reference exposure will receive unit weight at all wavelengths and all others receive relatively wavelength dependent weights . Note, relative weighting will only work well when there is at least one spectrum with a reasonable S/N, and a continuum. This option may only be better when the object being used has a strong continuum + emission lines. This is particularly useful if you are dealing with highly variable spectra (e.g. emission lines) andrequire a precision better than ~1 per cent.'ivar' -- Use inverse variance weighting. This is not well tested and should probably be deprecated. -``whitelight_range`` list .. None, None A two element list specifying the wavelength range over which to generate the white light image. The first (second) element is the minimum (maximum) wavelength to use. If either of these elements are None, PypeIt will automatically use a wavelength range that ensures all spaxels have the same wavelength coverage. Note, if you are using a reference_image to align all frames, it is preferable to use the same white light wavelength range for all white light images. For example, you may wish to use an emission line map to register two frames. +``whitelight_range`` list .. None, None A two element list specifying the wavelength range over which to generate the white light image. The first (second) element is the minimum (maximum) wavelength to use. If either of these elements are None, PypeIt will automatically use a wavelength range that ensures all spaxels have the same wavelength coverage. Note, if you are using a reference_image to align all frames, it is preferable to use the same white light wavelength range for all white light images. For example, you may wish to use an emission line map to register two frames. ==================== ===== =============================================================================== ============ ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= @@ -904,12 +904,12 @@ ScatteredLightPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.ScatteredLightPar` =================== ========= ================================= ========= ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================ -Key Type Options Default Description +Key Type Options Default Description =================== ========= ================================= ========= ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================ -``finecorr_mask`` int, list .. .. A list containing the inter-slit regions that the user wishes to mask during the fine correction to the scattered light. Each integer corresponds to an inter-slit region. For example, "0" corresponds to all pixels left of the leftmost slit, while a value of "1" corresponds to all pixels between the first and second slit (counting from the left). It should be either a single integer value, or a list of integer values. The default (None) means that no inter-slit regions will be masked. -``finecorr_method`` str ``median``, ``poly`` .. If None, a fine correction to the scattered light will not be performed. Otherwise, the allowed methods include: median, poly. 'median' will subtract a constant value from an entire CCD row, based on a median of the pixels that are not on slits (see also, 'finecorr_pad'). 'poly' will fit a polynomial to the scattered light in each row, based on the pixels that are not on slits (see also, 'finecorr_pad'). -``finecorr_order`` int .. 2 Polynomial order to use for the fine correction to the scattered light subtraction. It should be a low value. -``finecorr_pad`` int .. 4 Number of unbinned pixels to extend the slit edges by when masking the slits for the fine correction to the scattered light. +``finecorr_mask`` int, list .. .. A list containing the inter-slit regions that the user wishes to mask during the fine correction to the scattered light. Each integer corresponds to an inter-slit region. For example, "0" corresponds to all pixels left of the leftmost slit, while a value of "1" corresponds to all pixels between the first and second slit (counting from the left). It should be either a single integer value, or a list of integer values. The default (None) means that no inter-slit regions will be masked. +``finecorr_method`` str ``median``, ``poly`` .. If None, a fine correction to the scattered light will not be performed. Otherwise, the allowed methods include: median, poly. 'median' will subtract a constant value from an entire CCD row, based on a median of the pixels that are not on slits (see also, 'finecorr_pad'). 'poly' will fit a polynomial to the scattered light in each row, based on the pixels that are not on slits (see also, 'finecorr_pad'). +``finecorr_order`` int .. 2 Polynomial order to use for the fine correction to the scattered light subtraction. It should be a low value. +``finecorr_pad`` int .. 4 Number of unbinned pixels to extend the slit edges by when masking the slits for the fine correction to the scattered light. ``method`` str ``model``, ``frame``, ``archive`` ``model`` Method used to fit the overscan. Options are: model, frame, archive. 'model' will the scattered light model parameters derived from a user-specified frame during their reduction (note, you will need to make sure that you set appropriate scattlight frames in your .pypeit file for this option). 'frame' will use each individual frame to determine the scattered light that affects this frame. 'archive' will use an archival model parameter solution for the scattered light (note that this option is not currently available for all spectrographs). =================== ========= ================================= ========= ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================ @@ -1008,7 +1008,7 @@ Key Type Options Default ``popsize`` int .. 30 A multiplier for setting the total population size for the differential evolution optimization. See scipy.optimize.differential_evolution for details. ``recombination`` int, float .. 0.7 The recombination constant for the differential evolution optimization. This should be in the range [0, 1]. See scipy.optimize.differential_evolution for details. ``redshift`` int, float .. 0.0 The redshift for the object model. This is currently only used by objmodel=qso -``resln_frac_bounds`` tuple .. (0.6, 1.4) Bounds for the resolution fit optimization which is part of the telluric model. This range is in units of the resln_guess, so the (0.6, 1.4) would bound the spectral resolution fit to be within the range bounds_resln = (0.6*resln_guess, 1.4*resln_guess) +``resln_frac_bounds`` tuple .. (0.6, 1.4) Bounds for the resolution fit optimization which is part of the telluric model. This range is in units of the resln_guess, so the (0.6, 1.4) would bound the spectral resolution fit to be within the range bounds_resln = (0.6*resln_guess, 1.4*resln_guess) ``resln_guess`` int, float .. .. A guess for the resolution of your spectrum expressed as lambda/dlambda. The resolution is fit explicitly as part of the telluric model fitting, but this guess helps determine the bounds for the optimization (see next). If not provided, the wavelength sampling of your spectrum will be used and the resolution calculated using a typical sampling of 3 spectral pixels per resolution element. ``seed`` int .. 777 An initial seed for the differential evolution optimization, which is a random process. The default is a seed = 777 which will be used to generate a unique seed for every order. A specific seed is used because otherwise the random number generator will use the time for the seed, and the results will not be reproducible. ``sn_clip`` int, float .. 30.0 This adds an error floor to the ivar, preventing too much rejection at high-S/N (`i.e.`, standard stars, bright objects) using the function utils.clip_ivar. A small erorr is added to the input ivar so that the output ivar_out will never give S/N greater than sn_clip. This prevents overly aggressive rejection in high S/N ratio spectra which neverthless differ at a level greater than the formal S/N due to the fact that our telluric models are only good to about 3%. @@ -1019,9 +1019,9 @@ Key Type Options Default ``sticky`` bool .. True Sticky parameter for the utils.djs_reject algorithm for iterative model fit rejection. If set to True then points rejected from a previous iteration are kept rejected, in other words the bad pixel mask is the OR of all previous iterations and rejected pixels accumulate. If set to False, the bad pixel mask is the mask from the previous iteration, and if the model fit changes between iterations, points can alternate from being rejected to not rejected. At present this code only performs optimizations with differential evolution and experience shows that sticky needs to be True in order for these to converge. This is because the outliers can be so large that they dominate the loss function, and one never iteratively converges to a good model fit. In other words, the deformations in the model between iterations with sticky=False are too small to approach a reasonable fit. ``telgridfile`` str .. .. File containing the telluric grid for the observatory in question. These grids are generated from HITRAN models for each observatory using nominal site parameters. They must be downloaded from the GoogleDrive and installed in your PypeIt installation via the pypeit_install_telluric script. NOTE: This parameter no longer includes the full pathname to the Telluric Grid file, but is just the filename of the grid itself. ``tell_norm_thresh`` int, float .. 0.9 Threshold of telluric absorption region -``tell_npca`` int .. 5 Number of telluric PCA components used. Can be set to any number from 1 to 10. -``teltype`` str .. ``pca`` Method used to evaluate telluric models, either pca or grid. The grid option uses a fixed grid of pre-computed HITRAN+LBLRTM atmospheric transmission models for each observatory, whereas the pca option uses principal components of a larger model grid to compute an accurate pseudo-telluric model with a much lighter telgridfile. -``tol`` float .. 0.001 Relative tolerance for converage of the differential evolution optimization. See scipy.optimize.differential_evolution for details. +``tell_npca`` int .. 5 Number of telluric PCA components used. Can be set to any number from 1 to 10. +``teltype`` str .. ``pca`` Method used to evaluate telluric models, either pca or grid. The grid option uses a fixed grid of pre-computed HITRAN+LBLRTM atmospheric transmission models for each observatory, whereas the pca option uses principal components of a larger model grid to compute an accurate pseudo-telluric model with a much lighter telgridfile. +``tol`` float .. 0.001 Relative tolerance for converage of the differential evolution optimization. See scipy.optimize.differential_evolution for details. ``upper`` int, float .. 3.0 Upper rejection threshold in units of sigma_corr*sigma, where sigma is the formal noise of the spectrum, and sigma_corr is an empirically determined correction to the formal error. See above for description. ======================= ================== ======= ========================== ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= From 5a87aafe4180e061370676dcb7e0d015fc568974 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Mon, 5 Feb 2024 10:34:34 -0800 Subject: [PATCH 239/244] mv deprecated --- {pypeit/deprecated => deprecated}/.bumpversion.cfg | 0 {pypeit/deprecated => deprecated}/.travis.yml | 0 {pypeit/deprecated => deprecated}/ararclines.py | 0 {pypeit/deprecated => deprecated}/arc_old.py | 0 {pypeit/deprecated => deprecated}/arcid_plot.py | 0 {pypeit/deprecated => deprecated}/arproc.py | 0 {pypeit/deprecated => deprecated}/arsciexp.py | 0 {pypeit/deprecated => deprecated}/arspecobj.py | 0 {pypeit/deprecated => deprecated}/artrace.py | 0 {pypeit/deprecated => deprecated}/biasframe.py | 0 {pypeit/deprecated => deprecated}/bin/pypeit_arcid_plot | 0 {pypeit/deprecated => deprecated}/bin/pypeit_chk_2dslits | 0 {pypeit/deprecated => deprecated}/bin/pypeit_chk_alignments | 0 {pypeit/deprecated => deprecated}/bin/pypeit_chk_edges | 0 {pypeit/deprecated => deprecated}/bin/pypeit_chk_flats | 0 {pypeit/deprecated => deprecated}/bin/pypeit_chk_for_calibs | 0 {pypeit/deprecated => deprecated}/bin/pypeit_chk_tilts | 0 {pypeit/deprecated => deprecated}/bin/pypeit_chk_wavecalib | 0 {pypeit/deprecated => deprecated}/bin/pypeit_coadd_1dspec | 0 {pypeit/deprecated => deprecated}/bin/pypeit_coadd_2dspec | 0 {pypeit/deprecated => deprecated}/bin/pypeit_coadd_datacube | 0 {pypeit/deprecated => deprecated}/bin/pypeit_compare_sky | 0 {pypeit/deprecated => deprecated}/bin/pypeit_find_objects | 0 {pypeit/deprecated => deprecated}/bin/pypeit_flux_calib | 0 {pypeit/deprecated => deprecated}/bin/pypeit_flux_setup | 0 {pypeit/deprecated => deprecated}/bin/pypeit_identify | 0 {pypeit/deprecated => deprecated}/bin/pypeit_lowrdx_pixflat | 0 {pypeit/deprecated => deprecated}/bin/pypeit_lowrdx_skyspec | 0 {pypeit/deprecated => deprecated}/bin/pypeit_obslog | 0 {pypeit/deprecated => deprecated}/bin/pypeit_qa_html | 0 {pypeit/deprecated => deprecated}/bin/pypeit_ql_keck_mosfire | 0 {pypeit/deprecated => deprecated}/bin/pypeit_ql_keck_nires | 0 {pypeit/deprecated => deprecated}/bin/pypeit_ql_mos | 0 {pypeit/deprecated => deprecated}/bin/pypeit_sensfunc | 0 {pypeit/deprecated => deprecated}/bin/pypeit_setup | 0 {pypeit/deprecated => deprecated}/bin/pypeit_show_1dspec | 0 {pypeit/deprecated => deprecated}/bin/pypeit_show_2dspec | 0 {pypeit/deprecated => deprecated}/bin/pypeit_show_arxiv | 0 {pypeit/deprecated => deprecated}/bin/pypeit_show_wvcalib | 0 {pypeit/deprecated => deprecated}/bin/pypeit_skysub_regions | 0 {pypeit/deprecated => deprecated}/bin/pypeit_tellfit | 0 {pypeit/deprecated => deprecated}/bin/pypeit_trace_edges | 0 {pypeit/deprecated => deprecated}/bin/pypeit_view_fits | 0 {pypeit/deprecated => deprecated}/bin/run_pypeit | 0 {pypeit/deprecated => deprecated}/bpmimage.py | 0 {pypeit/deprecated => deprecated}/check_requirements.py | 0 {pypeit/deprecated => deprecated}/chk_tilts.py | 0 {pypeit/deprecated => deprecated}/coadd.py | 0 {pypeit/deprecated => deprecated}/coadd2d.py | 0 {pypeit/deprecated => deprecated}/coadd_1dspec_old.py | 0 {pypeit/deprecated => deprecated}/combine.py | 0 {pypeit/deprecated => deprecated}/datacube.py | 0 {pypeit/deprecated => deprecated}/debugger.py | 0 {pypeit/deprecated => deprecated}/defs.py | 0 {pypeit/deprecated => deprecated}/ech_coadd.py | 0 {pypeit/deprecated => deprecated}/extract.py | 0 {pypeit/deprecated => deprecated}/filter.py | 0 {pypeit/deprecated => deprecated}/find_objects.py | 0 {pypeit/deprecated => deprecated}/flat.py | 0 {pypeit/deprecated => deprecated}/flux.py | 0 {pypeit/deprecated => deprecated}/flux_old.py | 0 {pypeit/deprecated => deprecated}/fluxcalibrate.py | 0 {pypeit/deprecated => deprecated}/fluxspec.py | 0 {pypeit/deprecated => deprecated}/load.py | 0 {pypeit/deprecated => deprecated}/lowrdx_pixflat.py | 0 {pypeit/deprecated => deprecated}/masters.py | 0 {pypeit/deprecated => deprecated}/old_test_objfind.py | 0 {pypeit/deprecated => deprecated}/orig_save.py | 0 {pypeit/deprecated => deprecated}/orig_specobjs.py | 0 {pypeit/deprecated => deprecated}/par.py | 0 {pypeit/deprecated => deprecated}/parse.py | 0 {pypeit/deprecated => deprecated}/parse_calib_id.py | 0 {pypeit/deprecated => deprecated}/pca.py | 0 {pypeit/deprecated => deprecated}/pixels.py | 0 {pypeit/deprecated => deprecated}/processimages.py | 0 {pypeit/deprecated => deprecated}/procimg.py | 0 {pypeit/deprecated => deprecated}/pydl.py | 0 {pypeit/deprecated => deprecated}/pypsetup.py | 0 {pypeit/deprecated => deprecated}/qa.py | 0 {pypeit/deprecated => deprecated}/ql_keck_deimos.py | 0 {pypeit/deprecated => deprecated}/ql_keck_lris.py | 0 {pypeit/deprecated => deprecated}/ql_keck_mosfire.py | 0 {pypeit/deprecated => deprecated}/ql_keck_nires.py | 0 {pypeit/deprecated => deprecated}/ql_multislit.py | 0 {pypeit/deprecated => deprecated}/ql_vlt_fors2.py | 0 {pypeit/deprecated => deprecated}/rawimage.py | 0 {pypeit/deprecated => deprecated}/reduce.py | 0 {pypeit/deprecated => deprecated}/requirements.txt | 0 {pypeit/deprecated => deprecated}/requirements_doc.txt | 0 {pypeit/deprecated => deprecated}/save.py | 0 {pypeit/deprecated => deprecated}/sciimgstack.py | 0 {pypeit/deprecated => deprecated}/script_utils.py | 0 .../settings/archive/settings.2017-02-06.armed | 0 .../settings/archive/settings.2017-02-06.armlsd | 0 .../settings/archive/settings.2017-02-06.baseargflag | 0 .../settings/archive/settings.2017-02-06.basespect | 0 .../settings/archive/settings.2017-02-06.isis_blue | 0 .../settings/archive/settings.2017-02-06.kast_blue | 0 .../settings/archive/settings.2017-02-06.kast_red | 0 .../settings/archive/settings.2017-02-06.lris_blue | 0 .../settings/archive/settings.2017-02-06.lris_red | 0 .../settings/archive/settings.2017-02-21.basespect | 0 .../settings/archive/settings.2017-02-21.isis_blue | 0 .../settings/archive/settings.2017-02-21.kast_blue | 0 .../settings/archive/settings.2017-02-21.kast_red | 0 .../settings/archive/settings.2017-02-21.levy | 0 .../settings/archive/settings.2017-02-21.lris_blue | 0 .../settings/archive/settings.2017-02-21.lris_red | 0 .../settings/archive/settings.2017-03-24.baseargflag | 0 .../settings/archive/settings.2017-03-24.kast_blue | 0 .../settings/archive/settings.2017-03-24.kast_red | 0 .../settings/archive/settings.2017-03-24.kast_red_ret | 0 .../settings/archive/settings.2017-03-24.lris_blue | 0 .../settings/archive/settings.2017-03-24.lris_red | 0 .../settings/archive/settings.2017-04-12.armed | 0 .../settings/archive/settings.2017-04-12.armlsd | 0 .../settings/archive/settings.2017-04-12.baseargflag | 0 .../settings/archive/settings.2017-04-24.lris_red | 0 .../settings/archive/settings.2017-05-19.armed | 0 .../settings/archive/settings.2017-05-19.baseargflag | 0 .../settings/archive/settings.2017-05-19.levy | 0 .../settings/archive/settings.2017-06-05.basespect | 0 .../settings/archive/settings.2017-06-05.isis_blue | 0 .../settings/archive/settings.2017-06-05.kast_blue | 0 .../settings/archive/settings.2017-06-05.kast_red | 0 .../settings/archive/settings.2017-06-05.kast_red_ret | 0 .../settings/archive/settings.2017-06-05.keck_hires | 0 .../settings/archive/settings.2017-06-05.levy | 0 .../settings/archive/settings.2017-07-08.isis_blue | 0 .../settings/archive/settings.2017-07-16.armlsd | 0 .../settings/archive/settings.2017-07-16.baseargflag | 0 .../settings/archive/settings.2017-08-25.baseargflag | 0 .../settings/archive/settings.2017-08-29.dolores | 0 .../settings/archive/settings.2017-11-17.apf_levy | 0 .../settings/archive/settings.2017-11-17.baseargflag | 0 .../settings/archive/settings.2017-11-17.keck_hires | 0 .../settings/archive/settings.2017-11-17.keck_lris_blue | 0 .../settings/archive/settings.2017-11-17.keck_lris_red | 0 .../settings/archive/settings.2017-11-17.shane_kast_blue | 0 .../settings/archive/settings.2017-11-17.shane_kast_red | 0 .../settings/archive/settings.2017-11-17.shane_kast_red_ret | 0 .../settings/archive/settings.2017-11-17.tng_dolores | 0 .../settings/archive/settings.2017-11-17.wht_isis_blue | 0 .../settings/archive/settings.2018-04-05.armlsd | 0 .../settings/archive/settings.2018-04-05.baseargflag | 0 .../settings/archive/settings.2018-04-05.keck_deimos | 0 .../settings/archive/settings.2018-04-30.keck_deimos | 0 .../settings/archive/settings.2018-05-10.armed | 0 .../settings/archive/settings.2018-05-10.armlsd | 0 .../settings/archive/settings.2018-05-10.keck_deimos | 0 .../settings/archive/settings.2018-05-10.keck_lris_blue | 0 .../settings/archive/settings.2018-05-10.keck_lris_red | 0 .../settings/archive/settings.2018-05-22.keck_deimos | 0 .../settings/archive/settings.2018-05-22.keck_lris_red | 0 .../settings/archive/settings.2018-05-22.shane_kast_red_ret | 0 .../settings/archive/settings.2018-05-31.keck_nirspec | 0 .../settings/archive/settings.2018-06-07.arms | 0 .../settings/archive/settings.2018-06-07.baseargflag | 0 .../settings/archive/settings.2018-06-07.keck_deimos | 0 .../settings/archive/settings.2018-06-07.keck_lris_blue | 0 .../settings/archive/settings.2018-06-07.keck_lris_red | 0 .../settings/archive/settings.2018-06-07.keck_nirspec | 0 .../settings/archive/settings.2018-06-07.shane_kast_blue | 0 .../settings/archive/settings.2018-06-07.shane_kast_red | 0 .../settings/archive/settings.2018-06-07.shane_kast_red_ret | 0 .../settings/archive/settings.2018-06-07.tng_dolores | 0 .../settings/archive/settings.2018-06-07.wht_isis_blue | 0 .../settings/archive/settings.2018-06-19.baseargflag | 0 .../settings/archive/settings.2018-06-26.baseargflag | 0 .../settings/archive/settings.2018-07-03.keck_nires | 0 {pypeit/deprecated => deprecated}/settings/settings.apf_levy | 0 {pypeit/deprecated => deprecated}/settings/settings.armed | 0 {pypeit/deprecated => deprecated}/settings/settings.arms | 0 {pypeit/deprecated => deprecated}/settings/settings.baseargflag | 0 {pypeit/deprecated => deprecated}/settings/settings.basespect | 0 {pypeit/deprecated => deprecated}/settings/settings.keck_deimos | 0 {pypeit/deprecated => deprecated}/settings/settings.keck_hires | 0 .../deprecated => deprecated}/settings/settings.keck_lris_blue | 0 {pypeit/deprecated => deprecated}/settings/settings.keck_lris_red | 0 {pypeit/deprecated => deprecated}/settings/settings.keck_nires | 0 {pypeit/deprecated => deprecated}/settings/settings.keck_nirspec | 0 {pypeit/deprecated => deprecated}/settings/settings.py | 0 .../deprecated => deprecated}/settings/settings.shane_kast_blue | 0 .../deprecated => deprecated}/settings/settings.shane_kast_red | 0 .../settings/settings.shane_kast_red_ret | 0 {pypeit/deprecated => deprecated}/settings/settings.tng_dolores | 0 {pypeit/deprecated => deprecated}/settings/settings.wht_isis_blue | 0 {pypeit/deprecated => deprecated}/setup.py | 0 {pypeit/deprecated => deprecated}/trace_slits.py | 0 {pypeit/deprecated => deprecated}/traceslits.py | 0 {pypeit/deprecated => deprecated}/tracewave_old.py | 0 {pypeit/deprecated => deprecated}/utils.py | 0 {pypeit/deprecated => deprecated}/waveimage.py | 0 {pypeit/deprecated => deprecated}/waveio_old.py | 0 {pypeit/deprecated => deprecated}/wvutils.py | 0 195 files changed, 0 insertions(+), 0 deletions(-) rename {pypeit/deprecated => deprecated}/.bumpversion.cfg (100%) rename {pypeit/deprecated => deprecated}/.travis.yml (100%) rename {pypeit/deprecated => deprecated}/ararclines.py (100%) rename {pypeit/deprecated => deprecated}/arc_old.py (100%) rename {pypeit/deprecated => deprecated}/arcid_plot.py (100%) rename {pypeit/deprecated => deprecated}/arproc.py (100%) rename {pypeit/deprecated => deprecated}/arsciexp.py (100%) rename {pypeit/deprecated => deprecated}/arspecobj.py (100%) rename {pypeit/deprecated => deprecated}/artrace.py (100%) rename {pypeit/deprecated => deprecated}/biasframe.py (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_arcid_plot (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_chk_2dslits (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_chk_alignments (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_chk_edges (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_chk_flats (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_chk_for_calibs (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_chk_tilts (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_chk_wavecalib (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_coadd_1dspec (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_coadd_2dspec (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_coadd_datacube (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_compare_sky (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_find_objects (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_flux_calib (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_flux_setup (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_identify (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_lowrdx_pixflat (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_lowrdx_skyspec (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_obslog (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_qa_html (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_ql_keck_mosfire (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_ql_keck_nires (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_ql_mos (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_sensfunc (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_setup (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_show_1dspec (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_show_2dspec (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_show_arxiv (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_show_wvcalib (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_skysub_regions (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_tellfit (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_trace_edges (100%) rename {pypeit/deprecated => deprecated}/bin/pypeit_view_fits (100%) rename {pypeit/deprecated => deprecated}/bin/run_pypeit (100%) rename {pypeit/deprecated => deprecated}/bpmimage.py (100%) rename {pypeit/deprecated => deprecated}/check_requirements.py (100%) rename {pypeit/deprecated => deprecated}/chk_tilts.py (100%) rename {pypeit/deprecated => deprecated}/coadd.py (100%) rename {pypeit/deprecated => deprecated}/coadd2d.py (100%) rename {pypeit/deprecated => deprecated}/coadd_1dspec_old.py (100%) rename {pypeit/deprecated => deprecated}/combine.py (100%) rename {pypeit/deprecated => deprecated}/datacube.py (100%) rename {pypeit/deprecated => deprecated}/debugger.py (100%) rename {pypeit/deprecated => deprecated}/defs.py (100%) rename {pypeit/deprecated => deprecated}/ech_coadd.py (100%) rename {pypeit/deprecated => deprecated}/extract.py (100%) rename {pypeit/deprecated => deprecated}/filter.py (100%) rename {pypeit/deprecated => deprecated}/find_objects.py (100%) rename {pypeit/deprecated => deprecated}/flat.py (100%) rename {pypeit/deprecated => deprecated}/flux.py (100%) rename {pypeit/deprecated => deprecated}/flux_old.py (100%) rename {pypeit/deprecated => deprecated}/fluxcalibrate.py (100%) rename {pypeit/deprecated => deprecated}/fluxspec.py (100%) rename {pypeit/deprecated => deprecated}/load.py (100%) rename {pypeit/deprecated => deprecated}/lowrdx_pixflat.py (100%) rename {pypeit/deprecated => deprecated}/masters.py (100%) rename {pypeit/deprecated => deprecated}/old_test_objfind.py (100%) rename {pypeit/deprecated => deprecated}/orig_save.py (100%) rename {pypeit/deprecated => deprecated}/orig_specobjs.py (100%) rename {pypeit/deprecated => deprecated}/par.py (100%) rename {pypeit/deprecated => deprecated}/parse.py (100%) rename {pypeit/deprecated => deprecated}/parse_calib_id.py (100%) rename {pypeit/deprecated => deprecated}/pca.py (100%) rename {pypeit/deprecated => deprecated}/pixels.py (100%) rename {pypeit/deprecated => deprecated}/processimages.py (100%) rename {pypeit/deprecated => deprecated}/procimg.py (100%) rename {pypeit/deprecated => deprecated}/pydl.py (100%) rename {pypeit/deprecated => deprecated}/pypsetup.py (100%) rename {pypeit/deprecated => deprecated}/qa.py (100%) rename {pypeit/deprecated => deprecated}/ql_keck_deimos.py (100%) rename {pypeit/deprecated => deprecated}/ql_keck_lris.py (100%) rename {pypeit/deprecated => deprecated}/ql_keck_mosfire.py (100%) rename {pypeit/deprecated => deprecated}/ql_keck_nires.py (100%) rename {pypeit/deprecated => deprecated}/ql_multislit.py (100%) rename {pypeit/deprecated => deprecated}/ql_vlt_fors2.py (100%) rename {pypeit/deprecated => deprecated}/rawimage.py (100%) rename {pypeit/deprecated => deprecated}/reduce.py (100%) rename {pypeit/deprecated => deprecated}/requirements.txt (100%) rename {pypeit/deprecated => deprecated}/requirements_doc.txt (100%) rename {pypeit/deprecated => deprecated}/save.py (100%) rename {pypeit/deprecated => deprecated}/sciimgstack.py (100%) rename {pypeit/deprecated => deprecated}/script_utils.py (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-02-06.armed (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-02-06.armlsd (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-02-06.baseargflag (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-02-06.basespect (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-02-06.isis_blue (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-02-06.kast_blue (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-02-06.kast_red (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-02-06.lris_blue (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-02-06.lris_red (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-02-21.basespect (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-02-21.isis_blue (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-02-21.kast_blue (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-02-21.kast_red (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-02-21.levy (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-02-21.lris_blue (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-02-21.lris_red (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-03-24.baseargflag (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-03-24.kast_blue (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-03-24.kast_red (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-03-24.kast_red_ret (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-03-24.lris_blue (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-03-24.lris_red (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-04-12.armed (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-04-12.armlsd (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-04-12.baseargflag (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-04-24.lris_red (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-05-19.armed (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-05-19.baseargflag (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-05-19.levy (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-06-05.basespect (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-06-05.isis_blue (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-06-05.kast_blue (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-06-05.kast_red (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-06-05.kast_red_ret (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-06-05.keck_hires (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-06-05.levy (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-07-08.isis_blue (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-07-16.armlsd (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-07-16.baseargflag (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-08-25.baseargflag (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-08-29.dolores (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-11-17.apf_levy (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-11-17.baseargflag (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-11-17.keck_hires (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-11-17.keck_lris_blue (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-11-17.keck_lris_red (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-11-17.shane_kast_blue (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-11-17.shane_kast_red (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-11-17.shane_kast_red_ret (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-11-17.tng_dolores (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2017-11-17.wht_isis_blue (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-04-05.armlsd (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-04-05.baseargflag (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-04-05.keck_deimos (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-04-30.keck_deimos (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-05-10.armed (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-05-10.armlsd (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-05-10.keck_deimos (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-05-10.keck_lris_blue (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-05-10.keck_lris_red (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-05-22.keck_deimos (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-05-22.keck_lris_red (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-05-22.shane_kast_red_ret (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-05-31.keck_nirspec (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-06-07.arms (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-06-07.baseargflag (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-06-07.keck_deimos (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-06-07.keck_lris_blue (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-06-07.keck_lris_red (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-06-07.keck_nirspec (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-06-07.shane_kast_blue (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-06-07.shane_kast_red (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-06-07.shane_kast_red_ret (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-06-07.tng_dolores (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-06-07.wht_isis_blue (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-06-19.baseargflag (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-06-26.baseargflag (100%) rename {pypeit/deprecated => deprecated}/settings/archive/settings.2018-07-03.keck_nires (100%) rename {pypeit/deprecated => deprecated}/settings/settings.apf_levy (100%) rename {pypeit/deprecated => deprecated}/settings/settings.armed (100%) rename {pypeit/deprecated => deprecated}/settings/settings.arms (100%) rename {pypeit/deprecated => deprecated}/settings/settings.baseargflag (100%) rename {pypeit/deprecated => deprecated}/settings/settings.basespect (100%) rename {pypeit/deprecated => deprecated}/settings/settings.keck_deimos (100%) rename {pypeit/deprecated => deprecated}/settings/settings.keck_hires (100%) rename {pypeit/deprecated => deprecated}/settings/settings.keck_lris_blue (100%) rename {pypeit/deprecated => deprecated}/settings/settings.keck_lris_red (100%) rename {pypeit/deprecated => deprecated}/settings/settings.keck_nires (100%) rename {pypeit/deprecated => deprecated}/settings/settings.keck_nirspec (100%) rename {pypeit/deprecated => deprecated}/settings/settings.py (100%) rename {pypeit/deprecated => deprecated}/settings/settings.shane_kast_blue (100%) rename {pypeit/deprecated => deprecated}/settings/settings.shane_kast_red (100%) rename {pypeit/deprecated => deprecated}/settings/settings.shane_kast_red_ret (100%) rename {pypeit/deprecated => deprecated}/settings/settings.tng_dolores (100%) rename {pypeit/deprecated => deprecated}/settings/settings.wht_isis_blue (100%) rename {pypeit/deprecated => deprecated}/setup.py (100%) rename {pypeit/deprecated => deprecated}/trace_slits.py (100%) rename {pypeit/deprecated => deprecated}/traceslits.py (100%) rename {pypeit/deprecated => deprecated}/tracewave_old.py (100%) rename {pypeit/deprecated => deprecated}/utils.py (100%) rename {pypeit/deprecated => deprecated}/waveimage.py (100%) rename {pypeit/deprecated => deprecated}/waveio_old.py (100%) rename {pypeit/deprecated => deprecated}/wvutils.py (100%) diff --git a/pypeit/deprecated/.bumpversion.cfg b/deprecated/.bumpversion.cfg similarity index 100% rename from pypeit/deprecated/.bumpversion.cfg rename to deprecated/.bumpversion.cfg diff --git a/pypeit/deprecated/.travis.yml b/deprecated/.travis.yml similarity index 100% rename from pypeit/deprecated/.travis.yml rename to deprecated/.travis.yml diff --git a/pypeit/deprecated/ararclines.py b/deprecated/ararclines.py similarity index 100% rename from pypeit/deprecated/ararclines.py rename to deprecated/ararclines.py diff --git a/pypeit/deprecated/arc_old.py b/deprecated/arc_old.py similarity index 100% rename from pypeit/deprecated/arc_old.py rename to deprecated/arc_old.py diff --git a/pypeit/deprecated/arcid_plot.py b/deprecated/arcid_plot.py similarity index 100% rename from pypeit/deprecated/arcid_plot.py rename to deprecated/arcid_plot.py diff --git a/pypeit/deprecated/arproc.py b/deprecated/arproc.py similarity index 100% rename from pypeit/deprecated/arproc.py rename to deprecated/arproc.py diff --git a/pypeit/deprecated/arsciexp.py b/deprecated/arsciexp.py similarity index 100% rename from pypeit/deprecated/arsciexp.py rename to deprecated/arsciexp.py diff --git a/pypeit/deprecated/arspecobj.py b/deprecated/arspecobj.py similarity index 100% rename from pypeit/deprecated/arspecobj.py rename to deprecated/arspecobj.py diff --git a/pypeit/deprecated/artrace.py b/deprecated/artrace.py similarity index 100% rename from pypeit/deprecated/artrace.py rename to deprecated/artrace.py diff --git a/pypeit/deprecated/biasframe.py b/deprecated/biasframe.py similarity index 100% rename from pypeit/deprecated/biasframe.py rename to deprecated/biasframe.py diff --git a/pypeit/deprecated/bin/pypeit_arcid_plot b/deprecated/bin/pypeit_arcid_plot similarity index 100% rename from pypeit/deprecated/bin/pypeit_arcid_plot rename to deprecated/bin/pypeit_arcid_plot diff --git a/pypeit/deprecated/bin/pypeit_chk_2dslits b/deprecated/bin/pypeit_chk_2dslits similarity index 100% rename from pypeit/deprecated/bin/pypeit_chk_2dslits rename to deprecated/bin/pypeit_chk_2dslits diff --git a/pypeit/deprecated/bin/pypeit_chk_alignments b/deprecated/bin/pypeit_chk_alignments similarity index 100% rename from pypeit/deprecated/bin/pypeit_chk_alignments rename to deprecated/bin/pypeit_chk_alignments diff --git a/pypeit/deprecated/bin/pypeit_chk_edges b/deprecated/bin/pypeit_chk_edges similarity index 100% rename from pypeit/deprecated/bin/pypeit_chk_edges rename to deprecated/bin/pypeit_chk_edges diff --git a/pypeit/deprecated/bin/pypeit_chk_flats b/deprecated/bin/pypeit_chk_flats similarity index 100% rename from pypeit/deprecated/bin/pypeit_chk_flats rename to deprecated/bin/pypeit_chk_flats diff --git a/pypeit/deprecated/bin/pypeit_chk_for_calibs b/deprecated/bin/pypeit_chk_for_calibs similarity index 100% rename from pypeit/deprecated/bin/pypeit_chk_for_calibs rename to deprecated/bin/pypeit_chk_for_calibs diff --git a/pypeit/deprecated/bin/pypeit_chk_tilts b/deprecated/bin/pypeit_chk_tilts similarity index 100% rename from pypeit/deprecated/bin/pypeit_chk_tilts rename to deprecated/bin/pypeit_chk_tilts diff --git a/pypeit/deprecated/bin/pypeit_chk_wavecalib b/deprecated/bin/pypeit_chk_wavecalib similarity index 100% rename from pypeit/deprecated/bin/pypeit_chk_wavecalib rename to deprecated/bin/pypeit_chk_wavecalib diff --git a/pypeit/deprecated/bin/pypeit_coadd_1dspec b/deprecated/bin/pypeit_coadd_1dspec similarity index 100% rename from pypeit/deprecated/bin/pypeit_coadd_1dspec rename to deprecated/bin/pypeit_coadd_1dspec diff --git a/pypeit/deprecated/bin/pypeit_coadd_2dspec b/deprecated/bin/pypeit_coadd_2dspec similarity index 100% rename from pypeit/deprecated/bin/pypeit_coadd_2dspec rename to deprecated/bin/pypeit_coadd_2dspec diff --git a/pypeit/deprecated/bin/pypeit_coadd_datacube b/deprecated/bin/pypeit_coadd_datacube similarity index 100% rename from pypeit/deprecated/bin/pypeit_coadd_datacube rename to deprecated/bin/pypeit_coadd_datacube diff --git a/pypeit/deprecated/bin/pypeit_compare_sky b/deprecated/bin/pypeit_compare_sky similarity index 100% rename from pypeit/deprecated/bin/pypeit_compare_sky rename to deprecated/bin/pypeit_compare_sky diff --git a/pypeit/deprecated/bin/pypeit_find_objects b/deprecated/bin/pypeit_find_objects similarity index 100% rename from pypeit/deprecated/bin/pypeit_find_objects rename to deprecated/bin/pypeit_find_objects diff --git a/pypeit/deprecated/bin/pypeit_flux_calib b/deprecated/bin/pypeit_flux_calib similarity index 100% rename from pypeit/deprecated/bin/pypeit_flux_calib rename to deprecated/bin/pypeit_flux_calib diff --git a/pypeit/deprecated/bin/pypeit_flux_setup b/deprecated/bin/pypeit_flux_setup similarity index 100% rename from pypeit/deprecated/bin/pypeit_flux_setup rename to deprecated/bin/pypeit_flux_setup diff --git a/pypeit/deprecated/bin/pypeit_identify b/deprecated/bin/pypeit_identify similarity index 100% rename from pypeit/deprecated/bin/pypeit_identify rename to deprecated/bin/pypeit_identify diff --git a/pypeit/deprecated/bin/pypeit_lowrdx_pixflat b/deprecated/bin/pypeit_lowrdx_pixflat similarity index 100% rename from pypeit/deprecated/bin/pypeit_lowrdx_pixflat rename to deprecated/bin/pypeit_lowrdx_pixflat diff --git a/pypeit/deprecated/bin/pypeit_lowrdx_skyspec b/deprecated/bin/pypeit_lowrdx_skyspec similarity index 100% rename from pypeit/deprecated/bin/pypeit_lowrdx_skyspec rename to deprecated/bin/pypeit_lowrdx_skyspec diff --git a/pypeit/deprecated/bin/pypeit_obslog b/deprecated/bin/pypeit_obslog similarity index 100% rename from pypeit/deprecated/bin/pypeit_obslog rename to deprecated/bin/pypeit_obslog diff --git a/pypeit/deprecated/bin/pypeit_qa_html b/deprecated/bin/pypeit_qa_html similarity index 100% rename from pypeit/deprecated/bin/pypeit_qa_html rename to deprecated/bin/pypeit_qa_html diff --git a/pypeit/deprecated/bin/pypeit_ql_keck_mosfire b/deprecated/bin/pypeit_ql_keck_mosfire similarity index 100% rename from pypeit/deprecated/bin/pypeit_ql_keck_mosfire rename to deprecated/bin/pypeit_ql_keck_mosfire diff --git a/pypeit/deprecated/bin/pypeit_ql_keck_nires b/deprecated/bin/pypeit_ql_keck_nires similarity index 100% rename from pypeit/deprecated/bin/pypeit_ql_keck_nires rename to deprecated/bin/pypeit_ql_keck_nires diff --git a/pypeit/deprecated/bin/pypeit_ql_mos b/deprecated/bin/pypeit_ql_mos similarity index 100% rename from pypeit/deprecated/bin/pypeit_ql_mos rename to deprecated/bin/pypeit_ql_mos diff --git a/pypeit/deprecated/bin/pypeit_sensfunc b/deprecated/bin/pypeit_sensfunc similarity index 100% rename from pypeit/deprecated/bin/pypeit_sensfunc rename to deprecated/bin/pypeit_sensfunc diff --git a/pypeit/deprecated/bin/pypeit_setup b/deprecated/bin/pypeit_setup similarity index 100% rename from pypeit/deprecated/bin/pypeit_setup rename to deprecated/bin/pypeit_setup diff --git a/pypeit/deprecated/bin/pypeit_show_1dspec b/deprecated/bin/pypeit_show_1dspec similarity index 100% rename from pypeit/deprecated/bin/pypeit_show_1dspec rename to deprecated/bin/pypeit_show_1dspec diff --git a/pypeit/deprecated/bin/pypeit_show_2dspec b/deprecated/bin/pypeit_show_2dspec similarity index 100% rename from pypeit/deprecated/bin/pypeit_show_2dspec rename to deprecated/bin/pypeit_show_2dspec diff --git a/pypeit/deprecated/bin/pypeit_show_arxiv b/deprecated/bin/pypeit_show_arxiv similarity index 100% rename from pypeit/deprecated/bin/pypeit_show_arxiv rename to deprecated/bin/pypeit_show_arxiv diff --git a/pypeit/deprecated/bin/pypeit_show_wvcalib b/deprecated/bin/pypeit_show_wvcalib similarity index 100% rename from pypeit/deprecated/bin/pypeit_show_wvcalib rename to deprecated/bin/pypeit_show_wvcalib diff --git a/pypeit/deprecated/bin/pypeit_skysub_regions b/deprecated/bin/pypeit_skysub_regions similarity index 100% rename from pypeit/deprecated/bin/pypeit_skysub_regions rename to deprecated/bin/pypeit_skysub_regions diff --git a/pypeit/deprecated/bin/pypeit_tellfit b/deprecated/bin/pypeit_tellfit similarity index 100% rename from pypeit/deprecated/bin/pypeit_tellfit rename to deprecated/bin/pypeit_tellfit diff --git a/pypeit/deprecated/bin/pypeit_trace_edges b/deprecated/bin/pypeit_trace_edges similarity index 100% rename from pypeit/deprecated/bin/pypeit_trace_edges rename to deprecated/bin/pypeit_trace_edges diff --git a/pypeit/deprecated/bin/pypeit_view_fits b/deprecated/bin/pypeit_view_fits similarity index 100% rename from pypeit/deprecated/bin/pypeit_view_fits rename to deprecated/bin/pypeit_view_fits diff --git a/pypeit/deprecated/bin/run_pypeit b/deprecated/bin/run_pypeit similarity index 100% rename from pypeit/deprecated/bin/run_pypeit rename to deprecated/bin/run_pypeit diff --git a/pypeit/deprecated/bpmimage.py b/deprecated/bpmimage.py similarity index 100% rename from pypeit/deprecated/bpmimage.py rename to deprecated/bpmimage.py diff --git a/pypeit/deprecated/check_requirements.py b/deprecated/check_requirements.py similarity index 100% rename from pypeit/deprecated/check_requirements.py rename to deprecated/check_requirements.py diff --git a/pypeit/deprecated/chk_tilts.py b/deprecated/chk_tilts.py similarity index 100% rename from pypeit/deprecated/chk_tilts.py rename to deprecated/chk_tilts.py diff --git a/pypeit/deprecated/coadd.py b/deprecated/coadd.py similarity index 100% rename from pypeit/deprecated/coadd.py rename to deprecated/coadd.py diff --git a/pypeit/deprecated/coadd2d.py b/deprecated/coadd2d.py similarity index 100% rename from pypeit/deprecated/coadd2d.py rename to deprecated/coadd2d.py diff --git a/pypeit/deprecated/coadd_1dspec_old.py b/deprecated/coadd_1dspec_old.py similarity index 100% rename from pypeit/deprecated/coadd_1dspec_old.py rename to deprecated/coadd_1dspec_old.py diff --git a/pypeit/deprecated/combine.py b/deprecated/combine.py similarity index 100% rename from pypeit/deprecated/combine.py rename to deprecated/combine.py diff --git a/pypeit/deprecated/datacube.py b/deprecated/datacube.py similarity index 100% rename from pypeit/deprecated/datacube.py rename to deprecated/datacube.py diff --git a/pypeit/deprecated/debugger.py b/deprecated/debugger.py similarity index 100% rename from pypeit/deprecated/debugger.py rename to deprecated/debugger.py diff --git a/pypeit/deprecated/defs.py b/deprecated/defs.py similarity index 100% rename from pypeit/deprecated/defs.py rename to deprecated/defs.py diff --git a/pypeit/deprecated/ech_coadd.py b/deprecated/ech_coadd.py similarity index 100% rename from pypeit/deprecated/ech_coadd.py rename to deprecated/ech_coadd.py diff --git a/pypeit/deprecated/extract.py b/deprecated/extract.py similarity index 100% rename from pypeit/deprecated/extract.py rename to deprecated/extract.py diff --git a/pypeit/deprecated/filter.py b/deprecated/filter.py similarity index 100% rename from pypeit/deprecated/filter.py rename to deprecated/filter.py diff --git a/pypeit/deprecated/find_objects.py b/deprecated/find_objects.py similarity index 100% rename from pypeit/deprecated/find_objects.py rename to deprecated/find_objects.py diff --git a/pypeit/deprecated/flat.py b/deprecated/flat.py similarity index 100% rename from pypeit/deprecated/flat.py rename to deprecated/flat.py diff --git a/pypeit/deprecated/flux.py b/deprecated/flux.py similarity index 100% rename from pypeit/deprecated/flux.py rename to deprecated/flux.py diff --git a/pypeit/deprecated/flux_old.py b/deprecated/flux_old.py similarity index 100% rename from pypeit/deprecated/flux_old.py rename to deprecated/flux_old.py diff --git a/pypeit/deprecated/fluxcalibrate.py b/deprecated/fluxcalibrate.py similarity index 100% rename from pypeit/deprecated/fluxcalibrate.py rename to deprecated/fluxcalibrate.py diff --git a/pypeit/deprecated/fluxspec.py b/deprecated/fluxspec.py similarity index 100% rename from pypeit/deprecated/fluxspec.py rename to deprecated/fluxspec.py diff --git a/pypeit/deprecated/load.py b/deprecated/load.py similarity index 100% rename from pypeit/deprecated/load.py rename to deprecated/load.py diff --git a/pypeit/deprecated/lowrdx_pixflat.py b/deprecated/lowrdx_pixflat.py similarity index 100% rename from pypeit/deprecated/lowrdx_pixflat.py rename to deprecated/lowrdx_pixflat.py diff --git a/pypeit/deprecated/masters.py b/deprecated/masters.py similarity index 100% rename from pypeit/deprecated/masters.py rename to deprecated/masters.py diff --git a/pypeit/deprecated/old_test_objfind.py b/deprecated/old_test_objfind.py similarity index 100% rename from pypeit/deprecated/old_test_objfind.py rename to deprecated/old_test_objfind.py diff --git a/pypeit/deprecated/orig_save.py b/deprecated/orig_save.py similarity index 100% rename from pypeit/deprecated/orig_save.py rename to deprecated/orig_save.py diff --git a/pypeit/deprecated/orig_specobjs.py b/deprecated/orig_specobjs.py similarity index 100% rename from pypeit/deprecated/orig_specobjs.py rename to deprecated/orig_specobjs.py diff --git a/pypeit/deprecated/par.py b/deprecated/par.py similarity index 100% rename from pypeit/deprecated/par.py rename to deprecated/par.py diff --git a/pypeit/deprecated/parse.py b/deprecated/parse.py similarity index 100% rename from pypeit/deprecated/parse.py rename to deprecated/parse.py diff --git a/pypeit/deprecated/parse_calib_id.py b/deprecated/parse_calib_id.py similarity index 100% rename from pypeit/deprecated/parse_calib_id.py rename to deprecated/parse_calib_id.py diff --git a/pypeit/deprecated/pca.py b/deprecated/pca.py similarity index 100% rename from pypeit/deprecated/pca.py rename to deprecated/pca.py diff --git a/pypeit/deprecated/pixels.py b/deprecated/pixels.py similarity index 100% rename from pypeit/deprecated/pixels.py rename to deprecated/pixels.py diff --git a/pypeit/deprecated/processimages.py b/deprecated/processimages.py similarity index 100% rename from pypeit/deprecated/processimages.py rename to deprecated/processimages.py diff --git a/pypeit/deprecated/procimg.py b/deprecated/procimg.py similarity index 100% rename from pypeit/deprecated/procimg.py rename to deprecated/procimg.py diff --git a/pypeit/deprecated/pydl.py b/deprecated/pydl.py similarity index 100% rename from pypeit/deprecated/pydl.py rename to deprecated/pydl.py diff --git a/pypeit/deprecated/pypsetup.py b/deprecated/pypsetup.py similarity index 100% rename from pypeit/deprecated/pypsetup.py rename to deprecated/pypsetup.py diff --git a/pypeit/deprecated/qa.py b/deprecated/qa.py similarity index 100% rename from pypeit/deprecated/qa.py rename to deprecated/qa.py diff --git a/pypeit/deprecated/ql_keck_deimos.py b/deprecated/ql_keck_deimos.py similarity index 100% rename from pypeit/deprecated/ql_keck_deimos.py rename to deprecated/ql_keck_deimos.py diff --git a/pypeit/deprecated/ql_keck_lris.py b/deprecated/ql_keck_lris.py similarity index 100% rename from pypeit/deprecated/ql_keck_lris.py rename to deprecated/ql_keck_lris.py diff --git a/pypeit/deprecated/ql_keck_mosfire.py b/deprecated/ql_keck_mosfire.py similarity index 100% rename from pypeit/deprecated/ql_keck_mosfire.py rename to deprecated/ql_keck_mosfire.py diff --git a/pypeit/deprecated/ql_keck_nires.py b/deprecated/ql_keck_nires.py similarity index 100% rename from pypeit/deprecated/ql_keck_nires.py rename to deprecated/ql_keck_nires.py diff --git a/pypeit/deprecated/ql_multislit.py b/deprecated/ql_multislit.py similarity index 100% rename from pypeit/deprecated/ql_multislit.py rename to deprecated/ql_multislit.py diff --git a/pypeit/deprecated/ql_vlt_fors2.py b/deprecated/ql_vlt_fors2.py similarity index 100% rename from pypeit/deprecated/ql_vlt_fors2.py rename to deprecated/ql_vlt_fors2.py diff --git a/pypeit/deprecated/rawimage.py b/deprecated/rawimage.py similarity index 100% rename from pypeit/deprecated/rawimage.py rename to deprecated/rawimage.py diff --git a/pypeit/deprecated/reduce.py b/deprecated/reduce.py similarity index 100% rename from pypeit/deprecated/reduce.py rename to deprecated/reduce.py diff --git a/pypeit/deprecated/requirements.txt b/deprecated/requirements.txt similarity index 100% rename from pypeit/deprecated/requirements.txt rename to deprecated/requirements.txt diff --git a/pypeit/deprecated/requirements_doc.txt b/deprecated/requirements_doc.txt similarity index 100% rename from pypeit/deprecated/requirements_doc.txt rename to deprecated/requirements_doc.txt diff --git a/pypeit/deprecated/save.py b/deprecated/save.py similarity index 100% rename from pypeit/deprecated/save.py rename to deprecated/save.py diff --git a/pypeit/deprecated/sciimgstack.py b/deprecated/sciimgstack.py similarity index 100% rename from pypeit/deprecated/sciimgstack.py rename to deprecated/sciimgstack.py diff --git a/pypeit/deprecated/script_utils.py b/deprecated/script_utils.py similarity index 100% rename from pypeit/deprecated/script_utils.py rename to deprecated/script_utils.py diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-06.armed b/deprecated/settings/archive/settings.2017-02-06.armed similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-06.armed rename to deprecated/settings/archive/settings.2017-02-06.armed diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-06.armlsd b/deprecated/settings/archive/settings.2017-02-06.armlsd similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-06.armlsd rename to deprecated/settings/archive/settings.2017-02-06.armlsd diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-06.baseargflag b/deprecated/settings/archive/settings.2017-02-06.baseargflag similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-06.baseargflag rename to deprecated/settings/archive/settings.2017-02-06.baseargflag diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-06.basespect b/deprecated/settings/archive/settings.2017-02-06.basespect similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-06.basespect rename to deprecated/settings/archive/settings.2017-02-06.basespect diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-06.isis_blue b/deprecated/settings/archive/settings.2017-02-06.isis_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-06.isis_blue rename to deprecated/settings/archive/settings.2017-02-06.isis_blue diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-06.kast_blue b/deprecated/settings/archive/settings.2017-02-06.kast_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-06.kast_blue rename to deprecated/settings/archive/settings.2017-02-06.kast_blue diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-06.kast_red b/deprecated/settings/archive/settings.2017-02-06.kast_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-06.kast_red rename to deprecated/settings/archive/settings.2017-02-06.kast_red diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-06.lris_blue b/deprecated/settings/archive/settings.2017-02-06.lris_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-06.lris_blue rename to deprecated/settings/archive/settings.2017-02-06.lris_blue diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-06.lris_red b/deprecated/settings/archive/settings.2017-02-06.lris_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-06.lris_red rename to deprecated/settings/archive/settings.2017-02-06.lris_red diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-21.basespect b/deprecated/settings/archive/settings.2017-02-21.basespect similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-21.basespect rename to deprecated/settings/archive/settings.2017-02-21.basespect diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-21.isis_blue b/deprecated/settings/archive/settings.2017-02-21.isis_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-21.isis_blue rename to deprecated/settings/archive/settings.2017-02-21.isis_blue diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-21.kast_blue b/deprecated/settings/archive/settings.2017-02-21.kast_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-21.kast_blue rename to deprecated/settings/archive/settings.2017-02-21.kast_blue diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-21.kast_red b/deprecated/settings/archive/settings.2017-02-21.kast_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-21.kast_red rename to deprecated/settings/archive/settings.2017-02-21.kast_red diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-21.levy b/deprecated/settings/archive/settings.2017-02-21.levy similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-21.levy rename to deprecated/settings/archive/settings.2017-02-21.levy diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-21.lris_blue b/deprecated/settings/archive/settings.2017-02-21.lris_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-21.lris_blue rename to deprecated/settings/archive/settings.2017-02-21.lris_blue diff --git a/pypeit/deprecated/settings/archive/settings.2017-02-21.lris_red b/deprecated/settings/archive/settings.2017-02-21.lris_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-02-21.lris_red rename to deprecated/settings/archive/settings.2017-02-21.lris_red diff --git a/pypeit/deprecated/settings/archive/settings.2017-03-24.baseargflag b/deprecated/settings/archive/settings.2017-03-24.baseargflag similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-03-24.baseargflag rename to deprecated/settings/archive/settings.2017-03-24.baseargflag diff --git a/pypeit/deprecated/settings/archive/settings.2017-03-24.kast_blue b/deprecated/settings/archive/settings.2017-03-24.kast_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-03-24.kast_blue rename to deprecated/settings/archive/settings.2017-03-24.kast_blue diff --git a/pypeit/deprecated/settings/archive/settings.2017-03-24.kast_red b/deprecated/settings/archive/settings.2017-03-24.kast_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-03-24.kast_red rename to deprecated/settings/archive/settings.2017-03-24.kast_red diff --git a/pypeit/deprecated/settings/archive/settings.2017-03-24.kast_red_ret b/deprecated/settings/archive/settings.2017-03-24.kast_red_ret similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-03-24.kast_red_ret rename to deprecated/settings/archive/settings.2017-03-24.kast_red_ret diff --git a/pypeit/deprecated/settings/archive/settings.2017-03-24.lris_blue b/deprecated/settings/archive/settings.2017-03-24.lris_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-03-24.lris_blue rename to deprecated/settings/archive/settings.2017-03-24.lris_blue diff --git a/pypeit/deprecated/settings/archive/settings.2017-03-24.lris_red b/deprecated/settings/archive/settings.2017-03-24.lris_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-03-24.lris_red rename to deprecated/settings/archive/settings.2017-03-24.lris_red diff --git a/pypeit/deprecated/settings/archive/settings.2017-04-12.armed b/deprecated/settings/archive/settings.2017-04-12.armed similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-04-12.armed rename to deprecated/settings/archive/settings.2017-04-12.armed diff --git a/pypeit/deprecated/settings/archive/settings.2017-04-12.armlsd b/deprecated/settings/archive/settings.2017-04-12.armlsd similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-04-12.armlsd rename to deprecated/settings/archive/settings.2017-04-12.armlsd diff --git a/pypeit/deprecated/settings/archive/settings.2017-04-12.baseargflag b/deprecated/settings/archive/settings.2017-04-12.baseargflag similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-04-12.baseargflag rename to deprecated/settings/archive/settings.2017-04-12.baseargflag diff --git a/pypeit/deprecated/settings/archive/settings.2017-04-24.lris_red b/deprecated/settings/archive/settings.2017-04-24.lris_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-04-24.lris_red rename to deprecated/settings/archive/settings.2017-04-24.lris_red diff --git a/pypeit/deprecated/settings/archive/settings.2017-05-19.armed b/deprecated/settings/archive/settings.2017-05-19.armed similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-05-19.armed rename to deprecated/settings/archive/settings.2017-05-19.armed diff --git a/pypeit/deprecated/settings/archive/settings.2017-05-19.baseargflag b/deprecated/settings/archive/settings.2017-05-19.baseargflag similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-05-19.baseargflag rename to deprecated/settings/archive/settings.2017-05-19.baseargflag diff --git a/pypeit/deprecated/settings/archive/settings.2017-05-19.levy b/deprecated/settings/archive/settings.2017-05-19.levy similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-05-19.levy rename to deprecated/settings/archive/settings.2017-05-19.levy diff --git a/pypeit/deprecated/settings/archive/settings.2017-06-05.basespect b/deprecated/settings/archive/settings.2017-06-05.basespect similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-06-05.basespect rename to deprecated/settings/archive/settings.2017-06-05.basespect diff --git a/pypeit/deprecated/settings/archive/settings.2017-06-05.isis_blue b/deprecated/settings/archive/settings.2017-06-05.isis_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-06-05.isis_blue rename to deprecated/settings/archive/settings.2017-06-05.isis_blue diff --git a/pypeit/deprecated/settings/archive/settings.2017-06-05.kast_blue b/deprecated/settings/archive/settings.2017-06-05.kast_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-06-05.kast_blue rename to deprecated/settings/archive/settings.2017-06-05.kast_blue diff --git a/pypeit/deprecated/settings/archive/settings.2017-06-05.kast_red b/deprecated/settings/archive/settings.2017-06-05.kast_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-06-05.kast_red rename to deprecated/settings/archive/settings.2017-06-05.kast_red diff --git a/pypeit/deprecated/settings/archive/settings.2017-06-05.kast_red_ret b/deprecated/settings/archive/settings.2017-06-05.kast_red_ret similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-06-05.kast_red_ret rename to deprecated/settings/archive/settings.2017-06-05.kast_red_ret diff --git a/pypeit/deprecated/settings/archive/settings.2017-06-05.keck_hires b/deprecated/settings/archive/settings.2017-06-05.keck_hires similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-06-05.keck_hires rename to deprecated/settings/archive/settings.2017-06-05.keck_hires diff --git a/pypeit/deprecated/settings/archive/settings.2017-06-05.levy b/deprecated/settings/archive/settings.2017-06-05.levy similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-06-05.levy rename to deprecated/settings/archive/settings.2017-06-05.levy diff --git a/pypeit/deprecated/settings/archive/settings.2017-07-08.isis_blue b/deprecated/settings/archive/settings.2017-07-08.isis_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-07-08.isis_blue rename to deprecated/settings/archive/settings.2017-07-08.isis_blue diff --git a/pypeit/deprecated/settings/archive/settings.2017-07-16.armlsd b/deprecated/settings/archive/settings.2017-07-16.armlsd similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-07-16.armlsd rename to deprecated/settings/archive/settings.2017-07-16.armlsd diff --git a/pypeit/deprecated/settings/archive/settings.2017-07-16.baseargflag b/deprecated/settings/archive/settings.2017-07-16.baseargflag similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-07-16.baseargflag rename to deprecated/settings/archive/settings.2017-07-16.baseargflag diff --git a/pypeit/deprecated/settings/archive/settings.2017-08-25.baseargflag b/deprecated/settings/archive/settings.2017-08-25.baseargflag similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-08-25.baseargflag rename to deprecated/settings/archive/settings.2017-08-25.baseargflag diff --git a/pypeit/deprecated/settings/archive/settings.2017-08-29.dolores b/deprecated/settings/archive/settings.2017-08-29.dolores similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-08-29.dolores rename to deprecated/settings/archive/settings.2017-08-29.dolores diff --git a/pypeit/deprecated/settings/archive/settings.2017-11-17.apf_levy b/deprecated/settings/archive/settings.2017-11-17.apf_levy similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-11-17.apf_levy rename to deprecated/settings/archive/settings.2017-11-17.apf_levy diff --git a/pypeit/deprecated/settings/archive/settings.2017-11-17.baseargflag b/deprecated/settings/archive/settings.2017-11-17.baseargflag similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-11-17.baseargflag rename to deprecated/settings/archive/settings.2017-11-17.baseargflag diff --git a/pypeit/deprecated/settings/archive/settings.2017-11-17.keck_hires b/deprecated/settings/archive/settings.2017-11-17.keck_hires similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-11-17.keck_hires rename to deprecated/settings/archive/settings.2017-11-17.keck_hires diff --git a/pypeit/deprecated/settings/archive/settings.2017-11-17.keck_lris_blue b/deprecated/settings/archive/settings.2017-11-17.keck_lris_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-11-17.keck_lris_blue rename to deprecated/settings/archive/settings.2017-11-17.keck_lris_blue diff --git a/pypeit/deprecated/settings/archive/settings.2017-11-17.keck_lris_red b/deprecated/settings/archive/settings.2017-11-17.keck_lris_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-11-17.keck_lris_red rename to deprecated/settings/archive/settings.2017-11-17.keck_lris_red diff --git a/pypeit/deprecated/settings/archive/settings.2017-11-17.shane_kast_blue b/deprecated/settings/archive/settings.2017-11-17.shane_kast_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-11-17.shane_kast_blue rename to deprecated/settings/archive/settings.2017-11-17.shane_kast_blue diff --git a/pypeit/deprecated/settings/archive/settings.2017-11-17.shane_kast_red b/deprecated/settings/archive/settings.2017-11-17.shane_kast_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-11-17.shane_kast_red rename to deprecated/settings/archive/settings.2017-11-17.shane_kast_red diff --git a/pypeit/deprecated/settings/archive/settings.2017-11-17.shane_kast_red_ret b/deprecated/settings/archive/settings.2017-11-17.shane_kast_red_ret similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-11-17.shane_kast_red_ret rename to deprecated/settings/archive/settings.2017-11-17.shane_kast_red_ret diff --git a/pypeit/deprecated/settings/archive/settings.2017-11-17.tng_dolores b/deprecated/settings/archive/settings.2017-11-17.tng_dolores similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-11-17.tng_dolores rename to deprecated/settings/archive/settings.2017-11-17.tng_dolores diff --git a/pypeit/deprecated/settings/archive/settings.2017-11-17.wht_isis_blue b/deprecated/settings/archive/settings.2017-11-17.wht_isis_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2017-11-17.wht_isis_blue rename to deprecated/settings/archive/settings.2017-11-17.wht_isis_blue diff --git a/pypeit/deprecated/settings/archive/settings.2018-04-05.armlsd b/deprecated/settings/archive/settings.2018-04-05.armlsd similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-04-05.armlsd rename to deprecated/settings/archive/settings.2018-04-05.armlsd diff --git a/pypeit/deprecated/settings/archive/settings.2018-04-05.baseargflag b/deprecated/settings/archive/settings.2018-04-05.baseargflag similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-04-05.baseargflag rename to deprecated/settings/archive/settings.2018-04-05.baseargflag diff --git a/pypeit/deprecated/settings/archive/settings.2018-04-05.keck_deimos b/deprecated/settings/archive/settings.2018-04-05.keck_deimos similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-04-05.keck_deimos rename to deprecated/settings/archive/settings.2018-04-05.keck_deimos diff --git a/pypeit/deprecated/settings/archive/settings.2018-04-30.keck_deimos b/deprecated/settings/archive/settings.2018-04-30.keck_deimos similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-04-30.keck_deimos rename to deprecated/settings/archive/settings.2018-04-30.keck_deimos diff --git a/pypeit/deprecated/settings/archive/settings.2018-05-10.armed b/deprecated/settings/archive/settings.2018-05-10.armed similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-05-10.armed rename to deprecated/settings/archive/settings.2018-05-10.armed diff --git a/pypeit/deprecated/settings/archive/settings.2018-05-10.armlsd b/deprecated/settings/archive/settings.2018-05-10.armlsd similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-05-10.armlsd rename to deprecated/settings/archive/settings.2018-05-10.armlsd diff --git a/pypeit/deprecated/settings/archive/settings.2018-05-10.keck_deimos b/deprecated/settings/archive/settings.2018-05-10.keck_deimos similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-05-10.keck_deimos rename to deprecated/settings/archive/settings.2018-05-10.keck_deimos diff --git a/pypeit/deprecated/settings/archive/settings.2018-05-10.keck_lris_blue b/deprecated/settings/archive/settings.2018-05-10.keck_lris_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-05-10.keck_lris_blue rename to deprecated/settings/archive/settings.2018-05-10.keck_lris_blue diff --git a/pypeit/deprecated/settings/archive/settings.2018-05-10.keck_lris_red b/deprecated/settings/archive/settings.2018-05-10.keck_lris_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-05-10.keck_lris_red rename to deprecated/settings/archive/settings.2018-05-10.keck_lris_red diff --git a/pypeit/deprecated/settings/archive/settings.2018-05-22.keck_deimos b/deprecated/settings/archive/settings.2018-05-22.keck_deimos similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-05-22.keck_deimos rename to deprecated/settings/archive/settings.2018-05-22.keck_deimos diff --git a/pypeit/deprecated/settings/archive/settings.2018-05-22.keck_lris_red b/deprecated/settings/archive/settings.2018-05-22.keck_lris_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-05-22.keck_lris_red rename to deprecated/settings/archive/settings.2018-05-22.keck_lris_red diff --git a/pypeit/deprecated/settings/archive/settings.2018-05-22.shane_kast_red_ret b/deprecated/settings/archive/settings.2018-05-22.shane_kast_red_ret similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-05-22.shane_kast_red_ret rename to deprecated/settings/archive/settings.2018-05-22.shane_kast_red_ret diff --git a/pypeit/deprecated/settings/archive/settings.2018-05-31.keck_nirspec b/deprecated/settings/archive/settings.2018-05-31.keck_nirspec similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-05-31.keck_nirspec rename to deprecated/settings/archive/settings.2018-05-31.keck_nirspec diff --git a/pypeit/deprecated/settings/archive/settings.2018-06-07.arms b/deprecated/settings/archive/settings.2018-06-07.arms similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-06-07.arms rename to deprecated/settings/archive/settings.2018-06-07.arms diff --git a/pypeit/deprecated/settings/archive/settings.2018-06-07.baseargflag b/deprecated/settings/archive/settings.2018-06-07.baseargflag similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-06-07.baseargflag rename to deprecated/settings/archive/settings.2018-06-07.baseargflag diff --git a/pypeit/deprecated/settings/archive/settings.2018-06-07.keck_deimos b/deprecated/settings/archive/settings.2018-06-07.keck_deimos similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-06-07.keck_deimos rename to deprecated/settings/archive/settings.2018-06-07.keck_deimos diff --git a/pypeit/deprecated/settings/archive/settings.2018-06-07.keck_lris_blue b/deprecated/settings/archive/settings.2018-06-07.keck_lris_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-06-07.keck_lris_blue rename to deprecated/settings/archive/settings.2018-06-07.keck_lris_blue diff --git a/pypeit/deprecated/settings/archive/settings.2018-06-07.keck_lris_red b/deprecated/settings/archive/settings.2018-06-07.keck_lris_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-06-07.keck_lris_red rename to deprecated/settings/archive/settings.2018-06-07.keck_lris_red diff --git a/pypeit/deprecated/settings/archive/settings.2018-06-07.keck_nirspec b/deprecated/settings/archive/settings.2018-06-07.keck_nirspec similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-06-07.keck_nirspec rename to deprecated/settings/archive/settings.2018-06-07.keck_nirspec diff --git a/pypeit/deprecated/settings/archive/settings.2018-06-07.shane_kast_blue b/deprecated/settings/archive/settings.2018-06-07.shane_kast_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-06-07.shane_kast_blue rename to deprecated/settings/archive/settings.2018-06-07.shane_kast_blue diff --git a/pypeit/deprecated/settings/archive/settings.2018-06-07.shane_kast_red b/deprecated/settings/archive/settings.2018-06-07.shane_kast_red similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-06-07.shane_kast_red rename to deprecated/settings/archive/settings.2018-06-07.shane_kast_red diff --git a/pypeit/deprecated/settings/archive/settings.2018-06-07.shane_kast_red_ret b/deprecated/settings/archive/settings.2018-06-07.shane_kast_red_ret similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-06-07.shane_kast_red_ret rename to deprecated/settings/archive/settings.2018-06-07.shane_kast_red_ret diff --git a/pypeit/deprecated/settings/archive/settings.2018-06-07.tng_dolores b/deprecated/settings/archive/settings.2018-06-07.tng_dolores similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-06-07.tng_dolores rename to deprecated/settings/archive/settings.2018-06-07.tng_dolores diff --git a/pypeit/deprecated/settings/archive/settings.2018-06-07.wht_isis_blue b/deprecated/settings/archive/settings.2018-06-07.wht_isis_blue similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-06-07.wht_isis_blue rename to deprecated/settings/archive/settings.2018-06-07.wht_isis_blue diff --git a/pypeit/deprecated/settings/archive/settings.2018-06-19.baseargflag b/deprecated/settings/archive/settings.2018-06-19.baseargflag similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-06-19.baseargflag rename to deprecated/settings/archive/settings.2018-06-19.baseargflag diff --git a/pypeit/deprecated/settings/archive/settings.2018-06-26.baseargflag b/deprecated/settings/archive/settings.2018-06-26.baseargflag similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-06-26.baseargflag rename to deprecated/settings/archive/settings.2018-06-26.baseargflag diff --git a/pypeit/deprecated/settings/archive/settings.2018-07-03.keck_nires b/deprecated/settings/archive/settings.2018-07-03.keck_nires similarity index 100% rename from pypeit/deprecated/settings/archive/settings.2018-07-03.keck_nires rename to deprecated/settings/archive/settings.2018-07-03.keck_nires diff --git a/pypeit/deprecated/settings/settings.apf_levy b/deprecated/settings/settings.apf_levy similarity index 100% rename from pypeit/deprecated/settings/settings.apf_levy rename to deprecated/settings/settings.apf_levy diff --git a/pypeit/deprecated/settings/settings.armed b/deprecated/settings/settings.armed similarity index 100% rename from pypeit/deprecated/settings/settings.armed rename to deprecated/settings/settings.armed diff --git a/pypeit/deprecated/settings/settings.arms b/deprecated/settings/settings.arms similarity index 100% rename from pypeit/deprecated/settings/settings.arms rename to deprecated/settings/settings.arms diff --git a/pypeit/deprecated/settings/settings.baseargflag b/deprecated/settings/settings.baseargflag similarity index 100% rename from pypeit/deprecated/settings/settings.baseargflag rename to deprecated/settings/settings.baseargflag diff --git a/pypeit/deprecated/settings/settings.basespect b/deprecated/settings/settings.basespect similarity index 100% rename from pypeit/deprecated/settings/settings.basespect rename to deprecated/settings/settings.basespect diff --git a/pypeit/deprecated/settings/settings.keck_deimos b/deprecated/settings/settings.keck_deimos similarity index 100% rename from pypeit/deprecated/settings/settings.keck_deimos rename to deprecated/settings/settings.keck_deimos diff --git a/pypeit/deprecated/settings/settings.keck_hires b/deprecated/settings/settings.keck_hires similarity index 100% rename from pypeit/deprecated/settings/settings.keck_hires rename to deprecated/settings/settings.keck_hires diff --git a/pypeit/deprecated/settings/settings.keck_lris_blue b/deprecated/settings/settings.keck_lris_blue similarity index 100% rename from pypeit/deprecated/settings/settings.keck_lris_blue rename to deprecated/settings/settings.keck_lris_blue diff --git a/pypeit/deprecated/settings/settings.keck_lris_red b/deprecated/settings/settings.keck_lris_red similarity index 100% rename from pypeit/deprecated/settings/settings.keck_lris_red rename to deprecated/settings/settings.keck_lris_red diff --git a/pypeit/deprecated/settings/settings.keck_nires b/deprecated/settings/settings.keck_nires similarity index 100% rename from pypeit/deprecated/settings/settings.keck_nires rename to deprecated/settings/settings.keck_nires diff --git a/pypeit/deprecated/settings/settings.keck_nirspec b/deprecated/settings/settings.keck_nirspec similarity index 100% rename from pypeit/deprecated/settings/settings.keck_nirspec rename to deprecated/settings/settings.keck_nirspec diff --git a/pypeit/deprecated/settings/settings.py b/deprecated/settings/settings.py similarity index 100% rename from pypeit/deprecated/settings/settings.py rename to deprecated/settings/settings.py diff --git a/pypeit/deprecated/settings/settings.shane_kast_blue b/deprecated/settings/settings.shane_kast_blue similarity index 100% rename from pypeit/deprecated/settings/settings.shane_kast_blue rename to deprecated/settings/settings.shane_kast_blue diff --git a/pypeit/deprecated/settings/settings.shane_kast_red b/deprecated/settings/settings.shane_kast_red similarity index 100% rename from pypeit/deprecated/settings/settings.shane_kast_red rename to deprecated/settings/settings.shane_kast_red diff --git a/pypeit/deprecated/settings/settings.shane_kast_red_ret b/deprecated/settings/settings.shane_kast_red_ret similarity index 100% rename from pypeit/deprecated/settings/settings.shane_kast_red_ret rename to deprecated/settings/settings.shane_kast_red_ret diff --git a/pypeit/deprecated/settings/settings.tng_dolores b/deprecated/settings/settings.tng_dolores similarity index 100% rename from pypeit/deprecated/settings/settings.tng_dolores rename to deprecated/settings/settings.tng_dolores diff --git a/pypeit/deprecated/settings/settings.wht_isis_blue b/deprecated/settings/settings.wht_isis_blue similarity index 100% rename from pypeit/deprecated/settings/settings.wht_isis_blue rename to deprecated/settings/settings.wht_isis_blue diff --git a/pypeit/deprecated/setup.py b/deprecated/setup.py similarity index 100% rename from pypeit/deprecated/setup.py rename to deprecated/setup.py diff --git a/pypeit/deprecated/trace_slits.py b/deprecated/trace_slits.py similarity index 100% rename from pypeit/deprecated/trace_slits.py rename to deprecated/trace_slits.py diff --git a/pypeit/deprecated/traceslits.py b/deprecated/traceslits.py similarity index 100% rename from pypeit/deprecated/traceslits.py rename to deprecated/traceslits.py diff --git a/pypeit/deprecated/tracewave_old.py b/deprecated/tracewave_old.py similarity index 100% rename from pypeit/deprecated/tracewave_old.py rename to deprecated/tracewave_old.py diff --git a/pypeit/deprecated/utils.py b/deprecated/utils.py similarity index 100% rename from pypeit/deprecated/utils.py rename to deprecated/utils.py diff --git a/pypeit/deprecated/waveimage.py b/deprecated/waveimage.py similarity index 100% rename from pypeit/deprecated/waveimage.py rename to deprecated/waveimage.py diff --git a/pypeit/deprecated/waveio_old.py b/deprecated/waveio_old.py similarity index 100% rename from pypeit/deprecated/waveio_old.py rename to deprecated/waveio_old.py diff --git a/pypeit/deprecated/wvutils.py b/deprecated/wvutils.py similarity index 100% rename from pypeit/deprecated/wvutils.py rename to deprecated/wvutils.py From 34f45c2898a9da122524ec78fb1792b152eec5e6 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Mon, 5 Feb 2024 10:44:15 -0800 Subject: [PATCH 240/244] rm deprecated from doc makefile; fix telluric test file (bad merge) --- doc/Makefile | 2 +- pypeit/tests/tstutils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/Makefile b/doc/Makefile index c71afd4a43..61786be58d 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -56,7 +56,7 @@ clean: rm -rf $(LOCALFILES) apirst: - SPHINX_APIDOC_OPTIONS=$(SPHINXAPIOPT) $(SPHINXAPI) --separate -o ./api ../pypeit ../pypeit/tests/* ../pypeit/deprecated/* ../pypeit/version.py ../pypeit/compiler_version* + SPHINX_APIDOC_OPTIONS=$(SPHINXAPIOPT) $(SPHINXAPI) --separate -o ./api ../pypeit ../pypeit/tests/* ../pypeit/version.py ../pypeit/compiler_version* wget -O ./include/dev_suite_readme.rst https://raw.githubusercontent.com/pypeit/PypeIt-development-suite/main/README.rst python ./scripts/build_datacontainer_datamodels.py python ./scripts/build_dependency_rst.py diff --git a/pypeit/tests/tstutils.py b/pypeit/tests/tstutils.py index b648348d73..68d1c71028 100644 --- a/pypeit/tests/tstutils.py +++ b/pypeit/tests/tstutils.py @@ -36,7 +36,7 @@ # Tests require the Telluric file (Mauna Kea) par = Spectrograph.default_pypeit_par() -tell_test_grid = data.get_telgrid_filepath('TelFit_MaunaKea_3100_26100_R20000.fits') +tell_test_grid = data.get_telgrid_filepath('TellPCA_3000_26000_R25000.fits') telluric_required = pytest.mark.skipif(not tell_test_grid.is_file(), reason='no Mauna Kea telluric file') From 9bf41aeef586f071a3a76077c4866f64b50403b1 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Mon, 5 Feb 2024 12:39:55 -0800 Subject: [PATCH 241/244] dev doc updated --- doc/dev/development.rst | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/doc/dev/development.rst b/doc/dev/development.rst index 0f62f73c29..935051d54a 100644 --- a/doc/dev/development.rst +++ b/doc/dev/development.rst @@ -422,6 +422,12 @@ are as follows: * The docstrings for any changes to existing methods that were altered must have been modified so that they are up-to-date and accurate. + * The documentation must be successfully recompiled, either using the + ``update_docs`` scripts or but running ``make clean ; make html`` in the + ``doc/`` directory. (We plan for this to be added to the dev-suite testing; + in the meantime, PR authors simply need to affirm that the documentation + builds successfully.) + * Spurious commented code used for debugging or testing is fine, but please let us know if you want it to be kept by adding a relevant comment, something like ``# TODO: Keep this around for now``, at the @@ -491,7 +497,11 @@ tagging process is as follows: # Push the new tag git push --tags - * The tag is released for `pip`_ installation. + Similarly, a matching tag is executed for the dev-suite code (these tags only + exist for versions 1.15 and later). + + * The tag of the ``pypeit`` code-base (not the dev-suite) is released for + `pip`_ installation. .. code-block:: bash From af4f05641c41787cef460df473c56eaf395d64d3 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Mon, 5 Feb 2024 13:52:52 -0800 Subject: [PATCH 242/244] review changes --- doc/releases/1.15.0.rst | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/doc/releases/1.15.0.rst b/doc/releases/1.15.0.rst index e2140f4fc0..4e4d90ce88 100644 --- a/doc/releases/1.15.0.rst +++ b/doc/releases/1.15.0.rst @@ -34,7 +34,6 @@ Functionality/Performance Improvements and Additions - Add a sensible error message to the pypeit Spectrum1D loaders in the event a user inadvertently tries to use Spectrum1D instead of SpectrumList for a ``spec1d`` file. -- Add support for the R4K detector for MDM OSMOS - Enabled interpolation and extrapolation of missing echelle orders. This is currently only used for Keck/HIRES, but the code is general. - Allow the specification of wavelength limits on the flexure cross-correlation @@ -45,15 +44,14 @@ Functionality/Performance Improvements and Additions code, but the name ``'polynomial'`` is deprecated and will be removed in the future. - Introduced PCA method for telluric corrections -- Added a new GUI for creating and editing PypeIt input files: ``pypeit_setup_gui`` - Added slicer subpixel sampling for DataCube generation - - Added ``trace_rms_tol`` parameter for edge tracing, which helps guard against poorly constrained traces for spectrally truncated slits/orders. Instrument-specific Updates --------------------------- +- Add support for the R4K detector for MDM OSMOS - Updated archival sensitivity functions for DEIMOS 1200G, 800G, and 600ZD gratings. - Keck/KCWI and Keck/KCRM: Turned on polynomial correction for sky subtraction. - We now support the reduction of VLT/FORS2 data taken in MOS mode. @@ -83,6 +81,7 @@ Script Changes - Column ``SpatID`` in the output of ``pypeit_chk_wavecalib`` changed to ``SpatOrderID`` and now show the echelle order number, if applicable, otherwise the slit number. - ``pypeit_chk_edges`` now load SlitTraceSet (if available) to be able to overlay the echelle order numbers. +- Added a new GUI for creating and editing PypeIt input files: ``pypeit_setup_gui`` - Added a -G option to ``pypeit_setup`` and ``pypeit_obslog`` that will start the new Setup GUI. - Improvements and bug fixes for how the mask is displayed by @@ -93,9 +92,9 @@ Datamodel Changes ----------------- - A wavelength array is now stored for DataCube() -- Wavecalib and Wavefit datacontainers now store information about echelle order +- WaveCalib and WaveFit datacontainers now store information about echelle order number, if applicable. -- Change to how Slit datamodel stores and checks bit flag values. +- Change to how SlitTraceSet datamodel stores and checks bit flag values. Under-the-hood Improvements --------------------------- From f176578a39200d07e4d690469921a46592afb505 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Mon, 5 Feb 2024 13:53:30 -0800 Subject: [PATCH 243/244] review changes --- doc/releases/1.15.0.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/releases/1.15.0.rst b/doc/releases/1.15.0.rst index 4e4d90ce88..83d0e236a8 100644 --- a/doc/releases/1.15.0.rst +++ b/doc/releases/1.15.0.rst @@ -1,6 +1,6 @@ -Version 1.14.1dev -================= +Version 1.15.0 +============== Installation Changes -------------------- From 6bd6548ba53f3e71cfdaf963cc0b2bcafae67b7c Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Wed, 7 Feb 2024 07:50:00 -0800 Subject: [PATCH 244/244] chmod and fixed MANIFEST --- MANIFEST.in | 2 +- {pypeit => deprecated}/aroutput.yml | 0 pypeit/archive.py | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename {pypeit => deprecated}/aroutput.yml (100%) mode change 100755 => 100644 pypeit/archive.py diff --git a/MANIFEST.in b/MANIFEST.in index 019c6268bc..f7b4bd542a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -15,7 +15,7 @@ recursive-include scripts * prune build prune docs/_build prune docs/api -prune pypeit/deprecated +prune deprecated global-exclude *.pyc *.o *.so *.DS_Store diff --git a/pypeit/aroutput.yml b/deprecated/aroutput.yml similarity index 100% rename from pypeit/aroutput.yml rename to deprecated/aroutput.yml diff --git a/pypeit/archive.py b/pypeit/archive.py old mode 100755 new mode 100644