Initial version to deploy.

banzai_floyds/data/standards/README.md

@@ -0,0 +1,40 @@
+Relevant links:
+https://www.eso.org/sci/observing/tools/standards/spectra/stanlis.html
+https://ftp.eso.org/pub/usg/standards/ctiostan/
+https://ftp.eso.org/pub/stecf/standards/okestan/
+
+We prioritize the HST standards, followed by the X-Shooter standards, followed by the traditional CTIO observations.
+
+To generate the standard files in the pipeline format, download them from the ESO ftp servers and run the following:
+
+```
+from astropy.io import fits, ascii
+from astropy.table import Table
+
+
+standards = [{'name': 'gd108', 'input_file': 'fgd108.dat', 'ra': 115.086642, 'dec': -17.413667},
+             {'name': 'eg274', 'input_file': 'fEG274.dat', 'ra': 245.890989, 'dec': -39.229487},
+             {'name': 'feige110', 'input_file': 'ffeige110.dat', 'ra': 349.99332558, 'dec': -5.16560011},
+             {'name': 'feige34', 'input_file': 'ffeige34.dat', 'ra': 159.903066, 'dec': 43.102559},
+             {'name': 'bdp284211', 'input_file': 'fbd28d4211.dat', 'ra': 327.795923, 'dec': 28.863988}]
+
+for standard in standards:
+    eso_data = ascii.read(standard['input_file'])
+    data = Table({'wavelength': eso_data['col1'], 'flux': eso_data['col2']})
+    hdu_list = fits.HDUList([fits.PrimaryHDU(header=fits.Header({'RA': standard['ra'], 'DEC': standard['dec']})), 
+                             fits.BinTableHDU(data)])
+    hdu_list.writeto(f'{standard["name"]}.fits', overwrite=True)
+```
+Note that we use the same units as ESO: $\frac{ergs}{s cm^2 \AA}$.
+
+I have currently omitted L745-46A as a standard because it was not available through ESO and appears to not have been calibrated
+since 1984 (Baldwin & Stone). We should bootstrap this fluxed file based on standards observed on the same night with FLOYDS. LTT 3218 is a decent candidate to cross calibrate with. EG21 was reproduced in Hamuy 1994 but at very low resolution. The fluxes from Hamuy also do not look telluric corrected to me. We should probably use GD50 to bootstrap the EG21. Feige 67 will need to be bootstrapped from the HST CALSPEC spectrum of GD153. HZ44 will also need to be derived from its observations in CALSPEC + a model for minor extrapolation. Similarly for G191-B2B.
+
+Calibration Source:
+| Object | Source                                                    |
+| ------ | --------------------------------------------------------- |
+| GD108  | ftp://ftp.eso.org/pub/stecf/standards/hststan/fgd108.dat  |
+| EG274  | ftp://ftp.eso.org/pub/stecf/standards/Xshooter/fEG274.dat |
+| FEIGE110 | ftp://ftp.eso.org/pub/stecf/standards/hststan/ffeige110.dat |
+| FEIGE34 | ftp://ftp.eso.org/pub/stecf/standards/hststan/ffeige110.dat |
+| BD+28-4211 | ftp://ftp.eso.org/pub/stecf/standards/hststan/fbd28d4211.dat |

banzai_floyds/dbs.py

@@ -46,7 +46,6 @@ class FluxStandard(Base):
     id = Column(Integer, primary_key=True, autoincrement=True)
     filename = Column(String(100), unique=True)
     filepath = Column(String(150))
-    frameid = Column(Integer, nullable=True)
     ra = Column(Float)
     dec = Column(Float)
 

banzai_floyds/flux.py

@@ -39,7 +39,6 @@ def do_stage(self, image):
                 telluric_model = telluric_utils.estimate_telluric(data_to_fit['wavelength'],
                                                                   image.airmass, image.elevation,
                                                                   telluric_model=image.telluric)
-
             else:
                 telluric_model = np.ones_like(data_to_fit['wavelength'])
             # Divide the data by the flux standard in the blue

banzai_floyds/main.py

@@ -1,13 +1,13 @@
 from banzai_floyds import settings
 from banzai.main import parse_args, start_listener
 import argparse
-from banzai.main import add_settings_to_context
-import requests
-from banzai.utils import import_utils
 from banzai import logs
-from banzai.data import DataProduct
 import banzai_floyds.dbs
 import logging
+import celery
+import celery.bin.beat
+from celery.schedules import crontab
+from banzai.celery import app, schedule_calibration_stacking
 
 
 logger = logging.getLogger('banzai')
@@ -46,31 +46,24 @@ def create_db():
     banzai_floyds.dbs.create_db(args.db_address)
 
 
-def floyds_add_spectrophotometric_standard():
-    parser = argparse.ArgumentParser(description="Add bad pixel mask from a given archive api")
+def populate_photometric_standards():
+    parser = argparse.ArgumentParser("Ingest the location of the known flux standard tables.\n\n"
+                                     "This only needs to be run once on initialization of the database.")
     parser.add_argument('--db-address', dest='db_address',
-                        default='mysql://cmccully:password@localhost/test',
+                        default='sqlite3:///test.db',
                         help='Database address: Should be in SQLAlchemy form')
     args = parser.parse_args()
-    add_settings_to_context(args, settings)
-    # Query the archive for all bpm files
-    url = f'{settings.ARCHIVE_FRAME_URL}/?OBSTYPE=BPM&limit=1000'
-    archive_auth_header = settings.ARCHIVE_AUTH_HEADER
-    response = requests.get(url, headers=archive_auth_header)
-    response.raise_for_status()
-    results = response.json()['results']
-
-    # Load each one, saving the calibration info for each
-    frame_factory = import_utils.import_attribute(settings.FRAME_FACTORY)()
-    for frame in results:
-        frame['frameid'] = frame['id']
-        try:
-            bpm_image = frame_factory.open(frame, args)
-            if bpm_image is not None:
-                bpm_image.is_master = True
-                dbs.save_calibration_info(bpm_image.to_db_record(DataProduct(None, filename=bpm_image.filename,
-                                                                             filepath=None)),
-                                          args.db_address)
-        except Exception:
-            logger.error(f"BPM not added to database: {logs.format_exception()}",
-                         extra_tags={'filename': frame.get('filename')})
+    banzai_floyds.dbs.ingest_standards(args.db_address)
+
+
+def start_flat_stacking_scheduler():
+    logger.info('Started Flat Stacking Scheduler')
+    runtime_context = parse_args(settings)
+    for site, hour in zip(['coj', 'ogg'], [0, 4]):
+        app.add_periodic_task(crontab(minute=0, hour=hour),
+                              schedule_calibration_stacking.s(site=site, runtime_context=vars(runtime_context)),
+                              queue=runtime_context.CELERY_TASK_QUEUE_NAME)
+
+    beat = celery.bin.beat.beat(app=app)
+    logger.info('Starting celery beat')
+    beat.run(schedule='/tmp/celerybeat-schedule', pidfile='/tmp/celerybeat.pid', working_directory='/tmp')

banzai_floyds/settings.py

@@ -45,3 +45,5 @@
 CALIBRATION_FRAME_CLASS = 'banzai_floyds.frames.FLOYDSCalibrationFrame'
 
 CALIBRATION_IMAGE_TYPES = ['BIAS', 'DARK', 'SKYFLAT', 'BPM', 'LAMPFLAT', 'ARC', 'STANDARD']
+
+CALIBRATION_LOOKBACK = {'LAMPFLAT': 2.5}

helm-chart/banzai-floyds/templates/listener.yaml

@@ -51,6 +51,42 @@ spec:
             - name: listener-tmp
               mountPath: /tmp
               readOnly: false
+        - name: 'banzai-floyds-calibration-scheduler'
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          command:
+            - "banzai_floyds_start_stacker"
+            - "--post-to-archive"
+            - "--post-to-opensearch"
+            - "--opensearch-url=$(OPENSEARCH_URL)"
+            - "--fpack"
+            - "--rlevel=91"
+            - "--db-address=$(DB_ADDRESS)"
+            - "--broker-url=$(FITS_BROKER)"
+            - "--log-level=info"
+          env:
+            {{- include "banzai.Env" . | nindent 12 }}
+          resources:
+            requests:
+              cpu: "0.1"
+              memory: "512M"
+            limits:
+              cpu: "1"
+              memory: "1Gi"
+          volumeMounts:
+            - name: scheduler-tmp
+              mountPath: /tmp
+              readOnly: false
+
+      volumes:
+        - name: listener-tmp
+          emptyDir:
+            sizeLimit: 1Gi
+        - name: scheduler-tmp
+          emptyDir:
+            sizeLimit: 1Gi
       volumes:
         - name: listener-tmp
           emptyDir:

setup.cfg

@@ -33,6 +33,8 @@ docs =
 console_scripts =
     banzai_floyds_run_realtime_pipeline=banzai_floyds.main:floyds_run_realtime_pipeline
     banzai_floyds_create_db=banzai_floyds.main:create_db
+    banzai_floyds_start_stacker=banzai_floyds.main:start_flat_stacking_scheduler
+    banzai_floyds_populate_standards=banzai_floyds.main:populate_photometric_standards
 
 [options.package_data]
 banzai_floyds =