Fixes to pep style stuff

Jenkinsfile

@@ -136,6 +136,44 @@ pipeline {
 				}
 			}
 		}
+				stage('Test-Arc-Frame-Creation') {
+			agent {
+				label 'helm'
+			}
+			environment {
+				// store stage start time in the environment so it has stage scope
+				START_TIME = sh(script: 'date +%s', returnStdout: true).trim()
+			}
+			when {
+				anyOf {
+					branch 'PR-*'
+					expression { return params.forceEndToEnd }
+				}
+			}
+			steps {
+				script {
+                    withKubeConfig([credentialsId: "build-kube-config"]) {
+						sh("kubectl exec ${podName} -c banzai-floyds-e2e-listener -- " +
+						        "pytest -s --durations=0 --junitxml=/home/archive/pytest-arc-frames.xml " +
+						        "-m arc_frames /lco/banzai-floyds/")
+					}
+				}
+			}
+			post {
+				always {
+					script {
+					    withKubeConfig([credentialsId: "build-kube-config"]) {
+					    	env.LOGS_SINCE = sh(script: 'expr `date +%s` - ${START_TIME}', returnStdout: true).trim()
+    					    sh("kubectl logs ${podName} --since=${LOGS_SINCE}s --all-containers")
+						    sh("kubectl cp -c banzai-floyds-e2e-listener ${podName}:/home/archive/pytest-arc-frames.xml " +
+						            "pytest-arc-frames.xml")
+						    junit "pytest-arc-frames.xml"
+						}
+					}
+				}
+			}
+		}
+
 		stage('Test-Science-Frame-Creation') {
 			agent {
 				label 'helm'

banzai_floyds/extract.py

@@ -106,7 +106,9 @@ def fit_profile_width(data, profile_fits, poly_order=3, background_poly_order=2,
         # Short circuit if the trace is not significantly brighter than the background in this bin
         if peak_snr < 2.0 * median_snr:
             continue
-
+
+        # TODO: Only fit the profile width where it is much larger than the background value,
+        # otherwise use a heuristic width
         # Pass a match filter (with correct s/n scaling) with a gaussian with a default width
         initial_coeffs = np.zeros(background_poly_order + 1)
         initial_coeffs[0] = np.median(data_to_fit['data']) / data_to_fit['data'][peak]
@@ -140,6 +142,8 @@ def fit_background(data, profile_centers, profile_widths, x_poly_order=2, y_poly
         # Pass a match filter (with correct s/n scaling) with a gaussian with a default width
         initial_coeffs = np.zeros((x_poly_order + 1) + y_poly_order)
         initial_coeffs[0] = np.median(data_to_fit['data']) / data_to_fit['data'][peak]
+        # TODO: Fit the background with a totally fixed profile, and no need to iterate
+        # since our filter is linear 
         best_fit_coeffs = optimize_match_filter(initial_coeffs, data_to_fit['data'],
                                                 data_to_fit['uncertainty'],
                                                 background_fixed_profile,

banzai_floyds/flux.py

@@ -72,6 +72,13 @@ def apply_master_calibration(self, image, master_calibration_image):
     def calibration_type(self):
         return 'STANDARD'
 
+    def on_missing_master_calibration(self, image):
+        flux_standard = get_standard(image.ra, image.dec, self.runtime_context.db_address)
+        if flux_standard is not None:
+            return super().on_missing_master_calibration(image)
+        else: 
+            return image
+
 
 class FluxCalibrator(Stage):
     def do_stage(self, image):

banzai_floyds/frames.py

@@ -9,7 +9,6 @@
 import os
 from astropy.io import fits
 from banzai_floyds.utils.flux_utils import rescale_by_airmass
-from banzai.dbs import get_session, Site
 
 
 class FLOYDSObservationFrame(LCOObservationFrame):
@@ -28,10 +27,11 @@ def __init__(self, hdu_list: list, file_path: str, frame_id: int = None, hdu_ord
 
     def get_1d_and_2d_spectra_products(self, runtime_context):
         filename_1d = self.get_output_filename(runtime_context).replace('.fits', '-1d.fits')
+        self.meta.pop('EXTNAME')
         frame_1d = LCOObservationFrame([HeaderOnly(self.meta.copy()), self['EXTRACTED']],
                                        os.path.join(self.get_output_directory(runtime_context), filename_1d))
         fits_1d = frame_1d.to_fits(runtime_context)
-        fits_1d['SPECTRUM1D'].name = 'SPECTRUM'
+        fits_1d['EXTRACTED'].name = 'SPECTRUM'
         # TODO: Save telluric and sensitivity corrections that were applied
 
         filename_2d = filename_1d.replace('-1d.fits', '-2d.fits')

banzai_floyds/fringe.py

@@ -82,10 +82,10 @@ def make_master_calibration_frame(self, images):
         grouping = self.runtime_context.CALIBRATION_SET_CRITERIA.get(images[0].obstype, [])
         master_frame_class = import_utils.import_attribute(self.runtime_context.CALIBRATION_FRAME_CLASS)
         hdu_order = self.runtime_context.MASTER_CALIBRATION_EXTENSION_ORDER.get(self.calibration_type)
-
         super_frame = master_frame_class.init_master_frame(images, master_calibration_filename,
                                                            grouping_criteria=grouping, hdu_order=hdu_order)
         super_frame.primary_hdu.data[:, :] = super_fringe[:, :]
+        super_frame.primary_hdu.name = 'FRINGE'
         return super_frame
 
 
@@ -112,7 +112,7 @@ def on_missing_master_calibration(self, image):
 
     @property
     def calibration_type(self):
-        return 'FRINGE'
+        return 'LAMPFLAT'
 
     def apply_master_calibration(self, image, master_calibration_image):
         image.fringe = master_calibration_image.fringe

banzai_floyds/settings.py

@@ -8,8 +8,12 @@
     'banzai_floyds.orders.OrderLoader',
     'banzai_floyds.orders.OrderTweaker',
     'banzai_floyds.wavelengths.WavelengthSolutionLoader',
+    'banzai_floyds.fringe.FringeLoader',
     'banzai_floyds.fringe.FringeCorrector',
-    'banzai_floyds.extract.Extractor'
+    'banzai_floyds.extract.Extractor',
+    'banzai_floyds.flux.StandardLoader'
+    'banzai_floyds.flux.FluxSensitivity',
+    'banzai_floyds.flux.FluxCalibrator'
 ]
 
 FRAME_SELECTION_CRITERIA = [('type', 'contains', 'FLOYDS')]
@@ -18,7 +22,7 @@
 
 LAST_STAGE = {
     'SPECTRUM': None,
-    'LAMPFLAT': 'banzai_floyds.wavelengths.WavelengthSolutionLoader',
+    'LAMPFLAT': 'banzai_floyds.fringe.FringeLoader',
     'ARC': 'banzai_floyds.orders.OrderTweaker',
     'SKYFLAT': 'banzai.uncertainty.PoissonInitializer'
 }

banzai_floyds/tests/test_e2e.py

@@ -192,6 +192,32 @@ def test_if_fringe_frames_were_created(self):
         assert len(calibrations_in_db) == 2
 
 
+@pytest.mark.e2e
+@pytest.mark.standards
+class TestStandardFileCreation:
+    @pytest.fixture(autouse=True)
+    def process_standards(self):
+        logger.info('Reducing individual frames')
+
+        exchange = Exchange(os.getenv('FITS_EXCHANGE', 'fits_files'), type='fanout')
+        test_data = ascii.read(DATA_FILELIST)
+        with Connection(os.getenv('FITS_BROKER')) as conn:
+            producer = conn.Producer(exchange=exchange)
+            for row in test_data:
+                archive_record = requests.get(f'{os.getenv("API_ROOT")}frames/{row["frameid"]}').json()
+                archive_record['frameid'] = archive_record['id']
+                producer.publish(archive_record)
+            producer.release()
+
+        celery_join()
+        logger.info('Finished reducing individual frames')
+
+    def test_if_standards_were_created(self):
+        test_data = ascii.read(DATA_FILELIST)
+        for expected_file in expected_filenames(test_data):
+            assert os.path.exists(expected_file)
+
+
 @pytest.mark.e2e
 @pytest.mark.science_frames
 class TestScienceFileCreation:

banzai_floyds/wavelengths.py

@@ -248,7 +248,7 @@ class CalibrateWavelengths(Stage):
     LINES = arc_lines_table()
     # All in angstroms, measured by Curtis McCully
     # FWHM is , 5 pixels
-    INITIAL_LINE_WIDTHS = {1: 15.6, 2: 8.6}
+    INITIAL_LINE_WIDTHS = {1: 10, 2: 6}
     INITIAL_DISPERSIONS = {1: 3.51, 2: 1.72}
     # Tilts in degrees measured counterclockwise (right-handed coordinates)
     INITIAL_LINE_TILTS = {1: 8., 2: 8.}